diff --git a/config/Service.pkl b/config/Service.pkl index 6ad587ab962aae5286a468a0dca91f5ffe43a75f..dea7bcc624297f43a3cebb8bd0b0defb701eebd5 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -29,6 +29,8 @@ class Http { /// Internal Prometheus endpoint collected by the home server dashboard. metricsPath: String? checkHeaders: Mapping = new {} + /// API paths authenticated by a sibling task, bypassing the browser login proxy. + apiPaths: Mapping = new {} /// Request path to a file or directory in this service's folder. overrideFiles: Mapping = new {} /// HTML inserted before for the listed page paths. diff --git a/service/substation/monitor.js b/service/substation/monitor.js new file mode 100644 index 0000000000000000000000000000000000000000..608067df2c779f32b6ca59bc47fe2c377cb1b2ee --- /dev/null +++ b/service/substation/monitor.js @@ -0,0 +1,47 @@ +export async function probe(url, clientKey, managementKey) { + const checks = []; + for (const [path, key, expected] of [ + ["/v1/models", null, 401], + ["/v1/models", clientKey, 200], + ["/v0/management/version", managementKey, 200], + ]) { + const started = performance.now(); + let status = null; + let requestId = null; + let ok = false; + try { + const response = await fetch(new URL(path, url), { + headers: key ? { Authorization: `Bearer ${key}` } : {}, + redirect: "manual", + signal: AbortSignal.timeout(10_000), + }); + status = response.status; + const id = response.headers.get("x-request-id"); + if (/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(id ?? "")) requestId = id; + const body = await response.json(); + ok = status === expected && requestId !== null && (expected === 401 + ? typeof body.error === "string" + : path === "/v1/models" + ? body.object === "list" && Array.isArray(body.data) + : typeof body.version === "string"); + } catch {} + checks.push({ path, authenticated: key !== null, status, requestId, ok, + elapsedMs: Math.round(performance.now() - started) }); + } + return checks; +} + +if (import.meta.main) { + const { SUBSTATION_URL: url, SUBSTATION_CLIENT_KEY: clientKey, SUBSTATION_MANAGEMENT_KEY: managementKey } = process.env; + if (!url || !clientKey || !managementKey) throw new Error("Substation monitor configuration is incomplete"); + for (;;) { + const checks = await probe(url, clientKey, managementKey); + const ok = checks.every(check => check.ok); + console.log(JSON.stringify({ event: "substation-probe", level: ok ? "info" : "error", checks })); + if (process.argv.includes("--once")) { + process.exitCode = ok ? 0 : 1; + break; + } + await Bun.sleep(60_000); + } +} diff --git a/service/substation/service.pkl b/service/substation/service.pkl index a00f7091b16a7e695408ee827e7f2464e68ba13d..69a40815ab587509d487a0ca90cebc9c770ce20f 100644 --- a/service/substation/service.pkl +++ b/service/substation/service.pkl @@ -88,10 +88,14 @@ containers { containerPort = 4180 loopback = true subdomain = "substation.ai" + apiPaths { + ["/v1/*"] = "hub" + ["/v0/management/*"] = "hub" + } } envTemplate = """ OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }} - OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ + OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ """ env { ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)" @@ -105,7 +109,22 @@ containers { ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false" ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false" ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s" - ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/" + } + } + ["monitor"] = (runtime) { + entrypoint = "/usr/local/bin/bun" + args { "/monitor.js" } + cpu = 25 + memory = 64 + volumes = new { + ["/monitor.js"] { config = "monitor.js" } + ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true } + } + extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" } + env = new { + ["SUBSTATION_URL"] = "https://\(module.containers["app"].http.hostname)" + ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}" + ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}" } } } diff --git a/tools/router-api-test.py b/tools/router-api-test.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2a6c2bfec9e33a814f2987671e85a682f0b3bf --- /dev/null +++ b/tools/router-api-test.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +import argparse +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import threading +import time +from unittest.mock import patch + +import router +import snowglobe + + +class Hub(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + started = threading.Event() + + def do_GET(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" + if self.headers.get("Authorization") != "Bearer " + key: + status, body = 401, b'{"error":"unauthorized"}' + elif self.path.startswith("/v1/slow"): + type(self).started.set() + time.sleep(12) + status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n" + elif self.path.startswith("/v1/cancel"): + type(self).started.set() + time.sleep(1) + status, body = 200, b"finished" + elif self.path.startswith("/v1/broken"): + self.close_connection = True + return + else: + status, body = 200, b'{"object":"list"}' + try: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json") + self.send_header("Set-Cookie", "response-cookie-private-fixture") + self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) + self.send_header("X-Request-ID", "upstream-private-fixture") + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + do_POST = do_GET + + def log_message(self, *_): + pass + + +class Login(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(302) + self.send_header("Location", "/snow.oauth2/start") + self.end_headers() + + def log_message(self, *_): + pass + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--caddy", required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login: + root = Path(temporary) + routes = root / "routes" + routes.mkdir() + proof = root / "proof" + proof.write_text("a" * 64) + service = "substation-preview-1234abcd" + host = service + ".snowglobe.test" + paths = {"/v1/*": "hub", "/v0/management/*": "hub"} + data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False, + "uid": os.getuid(), "secrets": {}, "requiredSecrets": [], + "containers": {"app": {"rootGroup": False, "volumes": {}, "http": { + "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}}, + "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}} + with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0): + snowglobe.provision(data) + manifest = json.loads((routes / (service + ".json")).read_text()) + assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths} + instances = { + service: {"ServiceName": service, "JobID": service, "AllocID": service, + "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]}, + service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service, + "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []}, + "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria", + "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]}, + } + + def nomad(path, _): + if path == "/v1/services": + return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}] + if path.startswith("/v1/service/"): + item = instances.get(path.rsplit("/", 1)[1]) + return [item] if item else [] + return {"http": {"Status": "success"}} + + environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port), + "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary, + "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")} + environment.pop("SNOWGLOBE_INTERNAL_PORT", None) + with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)): + rendered = router.render("fixture") + assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered + instances[service + "-hub"]["JobID"] = "substation" + try: + router.render("fixture") + except ValueError: + pass + else: + raise AssertionError("preview routed to production") + instances[service + "-hub"]["JobID"] = service + missing = instances.pop(service + "-hub") + assert "respond 503" in router.render("fixture") + instances[service + "-hub"] = missing + ports = [] + for _ in range(2): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + ports.append(listener.getsockname()[1]) + port, admin = ports + site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] + site = site.replace(" tls internal\n", "") + config = root / "Caddyfile" + content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n" + config.write_text(content) + for backend in (hub, login): + threading.Thread(target=backend.serve_forever, daemon=True).start() + log_path = root / "caddy.log" + with log_path.open("w") as log: + process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) + try: + def request(path, key=None, method="GET", body=None, **headers): + if key: + headers["Authorization"] = "Bearer " + key + connection = HTTPConnection("127.0.0.1", port, timeout=20) + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + result = response.status, dict(response.getheaders()), response.read() + connection.close() + return result + + deadline = time.monotonic() + 10 + while True: + try: + assert request("/")[0] == 302 + break + except OSError: + assert process.poll() is None and time.monotonic() < deadline, log_path.read_text() + time.sleep(.05) + for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401), + ("/v0/management/version", "client-fixture", 401), + ("/v1/models", "client-fixture", 200), + ("/v0/management/version", "management-fixture", 200)]: + result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"}) + assert result == status, (path, result) + assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] + assert headers["X-Request-Id"] != "spoof-private-fixture" + assert request("/api/summary", "client-fixture")[0] == 302 + assert request("/v1", "client-fixture")[0] == 302 + Hub.started.clear() + outcome = [] + worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture"))) + worker.start() + assert Hub.started.wait(5) + config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0")) + subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment) + worker.join(timeout=20) + assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome + assert request("/v1/broken", "client-fixture")[0] == 502 + Hub.started.clear() + client = socket.create_connection(("127.0.0.1", port)) + client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) + assert Hub.started.wait(5) + client.close() + deadline = time.monotonic() + 5 + while True: + entries = [] + for line in log_path.read_text().splitlines(): + try: + entry = json.loads(line) + except ValueError: + continue + if entry.get("service") == service: + entries.append(entry) + if any(entry.get("status") == 499 for entry in entries): + break + assert time.monotonic() < deadline, entries + time.sleep(.05) + assert all("request" not in entry and "resp_headers" not in entry for entry in entries) + assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) + assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) + assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) + assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + assert "private-fixture" not in log_path.read_text() + assert "client-fixture" not in log_path.read_text() + assert "management-fixture" not in log_path.read_text() + finally: + process.terminate() + process.wait(timeout=10) + for backend in (hub, login): + backend.shutdown() + print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") + + +if __name__ == "__main__": + main() diff --git a/tools/router.py b/tools/router.py index 113adcfb37a50a2b7b3277ac696a21a991be5306..ecdf535ab8471e80b5079be7f8e60d64017f61fc 100644 --- a/tools/router.py +++ b/tools/router.py @@ -359,6 +359,32 @@ def render(token): if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path): raise ValueError(f"invalid metrics path: {host}") lines += [f" handle {metrics_path} {{", " respond 404", " }"] + api_paths = assets.get("apiPaths", {}).get(host, {}) + if api_paths: + if any(not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) or not re.fullmatch(r"[a-z][a-z0-9-]*", target) + or not (target == service or target.startswith(service + "-")) for path, target in api_paths.items()): + raise ValueError(f"invalid API routes: {host}") + lines += [f" log {service} {{", " output stdout", " format filter {", + " request delete", " resp_headers delete", " user_id delete", " }", " }", + " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", + f" log_append service {service}", " log_append request_id {http.request.uuid}", + " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", + " log_append upstream {http.reverse_proxy.upstream.hostport}", + " log_append upstream_status {http.reverse_proxy.status_code}", + " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", + " log_append proxy_ms {http.reverse_proxy.duration_ms}"] + for index, target in enumerate(sorted(set(api_paths.values()))): + upstreams = set() + for instance in nomad("/v1/service/" + target, token): + address, port = instance["Address"], instance["Port"] + if instance.get("JobID") != service or not re.fullmatch(r"127\.0\.0\.1|::1", address) or type(port) is not int or not 1 <= port <= 65535: + raise ValueError(f"API upstream must belong to {service} on loopback: {target}") + upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") + paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) + lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", + " request_header X-Request-ID {http.request.uuid}"] + lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] + lines.append(" }") upstreams = " ".join(sorted(route["upstreams"])) if role: auth = " ".join(sorted(auth_upstreams)) @@ -438,10 +464,10 @@ def render(token): raise ValueError("invalid login domain") lines += [f"{auth_host} {{", *([" tls internal"] if auth_host.endswith(".test") else []), *webauthn, " handle {", f" redir https://{dashboard_host}/ 308", " }", "}"] - traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None) + traces = internal_services.get("victoria-traces") if traces: lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1", - *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"] + *proxy(" ".join(sorted(traces)), " "), "}"] dashboard_port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) if not 1 <= dashboard_port <= 65535: raise ValueError("invalid dashboard route") diff --git a/tools/snowglobe.py b/tools/snowglobe.py index 18801a587db22acf0de3e79d59c0b8f47832a658..ac91593f3b5979d1ad3e525da5e108cb90ae2fa6 100644 --- a/tools/snowglobe.py +++ b/tools/snowglobe.py @@ -558,6 +558,7 @@ def provision(data): identity = {} head_html = {} metrics_paths = {} + api_paths = {} for task in data["containers"].values(): if task.get("http"): http = task["http"] @@ -567,6 +568,15 @@ def provision(data): head_html[http["hostname"]] = http["headHtml"] if http.get("metricsPath") and http["hostname"]: metrics_paths[http["hostname"]] = http["metricsPath"] + if http.get("apiPaths"): + targets = {} + for path, task_name in http["apiPaths"].items(): + target = data["containers"].get(task_name) + if (not http["hostname"] or not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) + or not target or not (target.get("http") or target.get("tcp"))): + raise ValueError(f"invalid API route for {data['id']}: {path}") + targets[path] = data["id"] if task_name == "app" else data["id"] + "-" + task_name + api_paths[http["hostname"]] = targets for request, rel in http["overrideFiles"].items(): origin = assets[rel] target = directories if origin.is_dir() else static @@ -625,7 +635,7 @@ def provision(data): proof[http["hostname"]] = "X-Snowglobe-Idp-" + values[name] write_private(route_dir / f"{data['id']}.json", json.dumps({ "files": static, "dirs": directories, "identity": identity, - "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, + "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, "apiPaths": api_paths, })) diff --git a/tools/substation-monitor-test.js b/tools/substation-monitor-test.js new file mode 100644 index 0000000000000000000000000000000000000000..1a8a53cb2a63e83fbcdc85f5ce5839b954497923 --- /dev/null +++ b/tools/substation-monitor-test.js @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { probe } from "../service/substation/monitor.js"; + +const requestId = "2c42c339-cd80-4f22-a4d7-61a411bba5e8"; +const replies = [ + [401, { error: "unauthorized" }], + [200, { object: "list", data: [] }], + [200, { version: "fixture" }], +]; +let calls = []; +let scenario = "healthy"; +globalThis.fetch = async (url, options) => { + const index = calls.length; + calls.push({ url: String(url), options }); + if (scenario === "network") throw new Error("client-private-fixture"); + const [status, body] = replies[index]; + return new Response(JSON.stringify(scenario === "body" ? { private: "client-private-fixture" } : body), { + status: scenario === "redirect" ? 302 : status, + headers: { "X-Request-ID": scenario === "id" ? "client-private-fixture" : requestId }, + }); +}; +for (scenario of ["healthy", "redirect", "body", "id", "network"]) { + calls = []; + const checks = await probe("https://substation.invalid", "client-private-fixture", "management-private-fixture"); + assert.equal(checks.length, 3); + assert.equal(checks.every(check => check.ok), scenario === "healthy"); + assert.ok(calls.every(call => call.options.redirect === "manual" && call.options.signal instanceof AbortSignal)); + assert.equal(calls[0].options.headers.Authorization, undefined); + assert.equal(calls[1].options.headers.Authorization, "Bearer client-private-fixture"); + assert.equal(calls[2].options.headers.Authorization, "Bearer management-private-fixture"); + assert.ok(!JSON.stringify(checks).includes("private-fixture")); + if (scenario === "healthy") assert.ok(checks.every(check => check.requestId === requestId)); + if (scenario === "id" || scenario === "network") assert.ok(checks.every(check => check.requestId === null)); +} +console.log("Substation monitor: authentication, redirect rejection, response validation, request IDs and private failures passed"); diff --git a/tools/substation.md b/tools/substation.md index f6bbfe5054adc1d695e1734ab1df1aff4c48f862..9d66d714533c47c46e37cd14301b46049ceca3fe 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -3,7 +3,8 @@ `https://substation.ai.paperclover.net` serves the Substation dashboard and its native APIs. Snow OAuth protects the dashboard and requires `infra-admin`. Substation's trusted-proxy mode accepts the identity header from the local -OAuth proxy. Hub requests authenticate with Substation's own keys. +OAuth proxy. Caddy sends `/v1/*` and `/v0/management/*` directly to the hub, +which checks its own client and management keys. ## Set up accounts @@ -94,3 +95,23 @@ The in-app updater is unavailable inside the immutable image. CPA's live dataset, preview, keys, and routing are retired after the replacement passes production checks. Restricted off-server recovery copies retain the old store; restoring them requires a separate explicit request. + +## Request diagnostics + +API responses include a Caddy-generated `X-Request-ID`. API access records in +Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, +duration, upstream address, upstream status, and time to upstream headers. +Headers, query strings, cookies, and request/response bodies are excluded from +these access records. The log shipper retains them in Victoria Logs. + +The `monitor` task checks anonymous rejection, client-key model access, and +management-key version access through HTTPS every minute. It requires the edge +request ID and rejects redirects, so an OAuth login page cannot pass as a healthy +API. It sends no generation requests and logs only check outcomes and IDs. + +Inspect `substation` task `monitor` in the service logs, and search Caddy's +journal for the response's request ID. A 499 with no upstream status marks a +cancellation before upstream headers; an upstream 499 is recorded separately. +A 200 followed by a truncated stream needs the edge byte count and the hub's +stream outcome. A successful probe does not establish that every generation +stream completed.