From 665ddec25df72359c856c0603d356d5ce7b2831f Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 12:36:39 -0800 Subject: [PATCH] Bypass OAuth for Substation APIs and monitor request failures Route declared API paths to sibling tasks over loopback. Substation keeps client and management key checks in the hub; OAuth protects its dashboard. Generate edge request IDs and record metadata without headers, queries or bodies, including upstream status and header latency for cancellation diagnosis. Probe anonymous rejection and both key roles through HTTPS every minute. Restore the trace ingestion route by selecting the registered task name. Verified actual staging hub, OAuth and monitor containers behind isolated Caddy; key separation and preview isolation; slow headers across reload; client cancellation 499 and upstream failure 502; private logs; existing WebSocket, cookie, Shale and Zig routes; ten deployment export tests. Assisted-by: gpt-6.1-sol --- config/Service.pkl | 2 + service/substation/monitor.js | 47 +++++++ service/substation/service.pkl | 23 +++- tools/router-api-test.py | 217 +++++++++++++++++++++++++++++++ tools/router.py | 30 ++++- tools/snowglobe.py | 12 +- tools/substation-monitor-test.js | 35 +++++ tools/substation.md | 23 +++- 8 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 service/substation/monitor.js create mode 100644 tools/router-api-test.py create mode 100644 tools/substation-monitor-test.js diff --git a/config/Service.pkl b/config/Service.pkl index 6ad587ab962aae5286a468a0dca91f5ffe43a75f..dea7bcc624297f43a3cebb8bd0b0defb701eebd5 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -29,6 +29,8 @@ class Http { /// Internal Prometheus endpoint collected by the home server dashboard. metricsPath: String? checkHeaders: Mapping = new {} + /// API paths authenticated by a sibling task, bypassing the browser login proxy. + apiPaths: Mapping = new {} /// Request path to a file or directory in this service's folder. overrideFiles: Mapping = new {} /// HTML inserted before for the listed page paths. diff --git a/service/substation/monitor.js b/service/substation/monitor.js new file mode 100644 index 0000000000000000000000000000000000000000..608067df2c779f32b6ca59bc47fe2c377cb1b2ee --- /dev/null +++ b/service/substation/monitor.js @@ -0,0 +1,47 @@ +export async function probe(url, clientKey, managementKey) { + const checks = []; + for (const [path, key, expected] of [ + ["/v1/models", null, 401], + ["/v1/models", clientKey, 200], + ["/v0/management/version", managementKey, 200], + ]) { + const started = performance.now(); + let status = null; + let requestId = null; + let ok = false; + try { + const response = await fetch(new URL(path, url), { + headers: key ? { Authorization: `Bearer ${key}` } : {}, + redirect: "manual", + signal: AbortSignal.timeout(10_000), + }); + status = response.status; + const id = response.headers.get("x-request-id"); + if (/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(id ?? "")) requestId = id; + const body = await response.json(); + ok = status === expected && requestId !== null && (expected === 401 + ? typeof body.error === "string" + : path === "/v1/models" + ? body.object === "list" && Array.isArray(body.data) + : typeof body.version === "string"); + } catch {} + checks.push({ path, authenticated: key !== null, status, requestId, ok, + elapsedMs: Math.round(performance.now() - started) }); + } + return checks; +} + +if (import.meta.main) { + const { SUBSTATION_URL: url, SUBSTATION_CLIENT_KEY: clientKey, SUBSTATION_MANAGEMENT_KEY: managementKey } = process.env; + if (!url || !clientKey || !managementKey) throw new Error("Substation monitor configuration is incomplete"); + for (;;) { + const checks = await probe(url, clientKey, managementKey); + const ok = checks.every(check => check.ok); + console.log(JSON.stringify({ event: "substation-probe", level: ok ? "info" : "error", checks })); + if (process.argv.includes("--once")) { + process.exitCode = ok ? 0 : 1; + break; + } + await Bun.sleep(60_000); + } +} diff --git a/service/substation/service.pkl b/service/substation/service.pkl index a00f7091b16a7e695408ee827e7f2464e68ba13d..69a40815ab587509d487a0ca90cebc9c770ce20f 100644 --- a/service/substation/service.pkl +++ b/service/substation/service.pkl @@ -88,10 +88,14 @@ containers { containerPort = 4180 loopback = true subdomain = "substation.ai" + apiPaths { + ["/v1/*"] = "hub" + ["/v0/management/*"] = "hub" + } } envTemplate = """ OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }} - OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ + OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ """ env { ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)" @@ -105,7 +109,22 @@ containers { ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false" ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false" ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s" - ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/" + } + } + ["monitor"] = (runtime) { + entrypoint = "/usr/local/bin/bun" + args { "/monitor.js" } + cpu = 25 + memory = 64 + volumes = new { + ["/monitor.js"] { config = "monitor.js" } + ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true } + } + extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" } + env = new { + ["SUBSTATION_URL"] = "https://\(module.containers["app"].http.hostname)" + ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}" + ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}" } } } diff --git a/tools/router-api-test.py b/tools/router-api-test.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2a6c2bfec9e33a814f2987671e85a682f0b3bf --- /dev/null +++ b/tools/router-api-test.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +import argparse +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import threading +import time +from unittest.mock import patch + +import router +import snowglobe + + +class Hub(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + started = threading.Event() + + def do_GET(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" + if self.headers.get("Authorization") != "Bearer " + key: + status, body = 401, b'{"error":"unauthorized"}' + elif self.path.startswith("/v1/slow"): + type(self).started.set() + time.sleep(12) + status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n" + elif self.path.startswith("/v1/cancel"): + type(self).started.set() + time.sleep(1) + status, body = 200, b"finished" + elif self.path.startswith("/v1/broken"): + self.close_connection = True + return + else: + status, body = 200, b'{"object":"list"}' + try: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json") + self.send_header("Set-Cookie", "response-cookie-private-fixture") + self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) + self.send_header("X-Request-ID", "upstream-private-fixture") + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + do_POST = do_GET + + def log_message(self, *_): + pass + + +class Login(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(302) + self.send_header("Location", "/snow.oauth2/start") + self.end_headers() + + def log_message(self, *_): + pass + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--caddy", required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login: + root = Path(temporary) + routes = root / "routes" + routes.mkdir() + proof = root / "proof" + proof.write_text("a" * 64) + service = "substation-preview-1234abcd" + host = service + ".snowglobe.test" + paths = {"/v1/*": "hub", "/v0/management/*": "hub"} + data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False, + "uid": os.getuid(), "secrets": {}, "requiredSecrets": [], + "containers": {"app": {"rootGroup": False, "volumes": {}, "http": { + "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}}, + "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}} + with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0): + snowglobe.provision(data) + manifest = json.loads((routes / (service + ".json")).read_text()) + assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths} + instances = { + service: {"ServiceName": service, "JobID": service, "AllocID": service, + "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]}, + service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service, + "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []}, + "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria", + "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]}, + } + + def nomad(path, _): + if path == "/v1/services": + return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}] + if path.startswith("/v1/service/"): + item = instances.get(path.rsplit("/", 1)[1]) + return [item] if item else [] + return {"http": {"Status": "success"}} + + environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port), + "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary, + "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")} + environment.pop("SNOWGLOBE_INTERNAL_PORT", None) + with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)): + rendered = router.render("fixture") + assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered + instances[service + "-hub"]["JobID"] = "substation" + try: + router.render("fixture") + except ValueError: + pass + else: + raise AssertionError("preview routed to production") + instances[service + "-hub"]["JobID"] = service + missing = instances.pop(service + "-hub") + assert "respond 503" in router.render("fixture") + instances[service + "-hub"] = missing + ports = [] + for _ in range(2): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + ports.append(listener.getsockname()[1]) + port, admin = ports + site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] + site = site.replace(" tls internal\n", "") + config = root / "Caddyfile" + content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n" + config.write_text(content) + for backend in (hub, login): + threading.Thread(target=backend.serve_forever, daemon=True).start() + log_path = root / "caddy.log" + with log_path.open("w") as log: + process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) + try: + def request(path, key=None, method="GET", body=None, **headers): + if key: + headers["Authorization"] = "Bearer " + key + connection = HTTPConnection("127.0.0.1", port, timeout=20) + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + result = response.status, dict(response.getheaders()), response.read() + connection.close() + return result + + deadline = time.monotonic() + 10 + while True: + try: + assert request("/")[0] == 302 + break + except OSError: + assert process.poll() is None and time.monotonic() < deadline, log_path.read_text() + time.sleep(.05) + for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401), + ("/v0/management/version", "client-fixture", 401), + ("/v1/models", "client-fixture", 200), + ("/v0/management/version", "management-fixture", 200)]: + result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"}) + assert result == status, (path, result) + assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] + assert headers["X-Request-Id"] != "spoof-private-fixture" + assert request("/api/summary", "client-fixture")[0] == 302 + assert request("/v1", "client-fixture")[0] == 302 + Hub.started.clear() + outcome = [] + worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture"))) + worker.start() + assert Hub.started.wait(5) + config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0")) + subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment) + worker.join(timeout=20) + assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome + assert request("/v1/broken", "client-fixture")[0] == 502 + Hub.started.clear() + client = socket.create_connection(("127.0.0.1", port)) + client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) + assert Hub.started.wait(5) + client.close() + deadline = time.monotonic() + 5 + while True: + entries = [] + for line in log_path.read_text().splitlines(): + try: + entry = json.loads(line) + except ValueError: + continue + if entry.get("service") == service: + entries.append(entry) + if any(entry.get("status") == 499 for entry in entries): + break + assert time.monotonic() < deadline, entries + time.sleep(.05) + assert all("request" not in entry and "resp_headers" not in entry for entry in entries) + assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) + assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) + assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) + assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + assert "private-fixture" not in log_path.read_text() + assert "client-fixture" not in log_path.read_text() + assert "management-fixture" not in log_path.read_text() + finally: + process.terminate() + process.wait(timeout=10) + for backend in (hub, login): + backend.shutdown() + print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") + + +if __name__ == "__main__": + main() diff --git a/tools/router.py b/tools/router.py index 113adcfb37a50a2b7b3277ac696a21a991be5306..ecdf535ab8471e80b5079be7f8e60d64017f61fc 100644 --- a/tools/router.py +++ b/tools/router.py @@ -359,6 +359,32 @@ def render(token): if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path): raise ValueError(f"invalid metrics path: {host}") lines += [f" handle {metrics_path} {{", " respond 404", " }"] + api_paths = assets.get("apiPaths", {}).get(host, {}) + if api_paths: + if any(not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) or not re.fullmatch(r"[a-z][a-z0-9-]*", target) + or not (target == service or target.startswith(service + "-")) for path, target in api_paths.items()): + raise ValueError(f"invalid API routes: {host}") + lines += [f" log {service} {{", " output stdout", " format filter {", + " request delete", " resp_headers delete", " user_id delete", " }", " }", + " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", + f" log_append service {service}", " log_append request_id {http.request.uuid}", + " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", + " log_append upstream {http.reverse_proxy.upstream.hostport}", + " log_append upstream_status {http.reverse_proxy.status_code}", + " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", + " log_append proxy_ms {http.reverse_proxy.duration_ms}"] + for index, target in enumerate(sorted(set(api_paths.values()))): + upstreams = set() + for instance in nomad("/v1/service/" + target, token): + address, port = instance["Address"], instance["Port"] + if instance.get("JobID") != service or not re.fullmatch(r"127\.0\.0\.1|::1", address) or type(port) is not int or not 1 <= port <= 65535: + raise ValueError(f"API upstream must belong to {service} on loopback: {target}") + upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") + paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) + lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", + " request_header X-Request-ID {http.request.uuid}"] + lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] + lines.append(" }") upstreams = " ".join(sorted(route["upstreams"])) if role: auth = " ".join(sorted(auth_upstreams)) @@ -438,10 +464,10 @@ def render(token): raise ValueError("invalid login domain") lines += [f"{auth_host} {{", *([" tls internal"] if auth_host.endswith(".test") else []), *webauthn, " handle {", f" redir https://{dashboard_host}/ 308", " }", "}"] - traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None) + traces = internal_services.get("victoria-traces") if traces: lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1", - *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"] + *proxy(" ".join(sorted(traces)), " "), "}"] dashboard_port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) if not 1 <= dashboard_port <= 65535: raise ValueError("invalid dashboard route") diff --git a/tools/snowglobe.py b/tools/snowglobe.py index 18801a587db22acf0de3e79d59c0b8f47832a658..ac91593f3b5979d1ad3e525da5e108cb90ae2fa6 100644 --- a/tools/snowglobe.py +++ b/tools/snowglobe.py @@ -558,6 +558,7 @@ def provision(data): identity = {} head_html = {} metrics_paths = {} + api_paths = {} for task in data["containers"].values(): if task.get("http"): http = task["http"] @@ -567,6 +568,15 @@ def provision(data): head_html[http["hostname"]] = http["headHtml"] if http.get("metricsPath") and http["hostname"]: metrics_paths[http["hostname"]] = http["metricsPath"] + if http.get("apiPaths"): + targets = {} + for path, task_name in http["apiPaths"].items(): + target = data["containers"].get(task_name) + if (not http["hostname"] or not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) + or not target or not (target.get("http") or target.get("tcp"))): + raise ValueError(f"invalid API route for {data['id']}: {path}") + targets[path] = data["id"] if task_name == "app" else data["id"] + "-" + task_name + api_paths[http["hostname"]] = targets for request, rel in http["overrideFiles"].items(): origin = assets[rel] target = directories if origin.is_dir() else static @@ -625,7 +635,7 @@ def provision(data): proof[http["hostname"]] = "X-Snowglobe-Idp-" + values[name] write_private(route_dir / f"{data['id']}.json", json.dumps({ "files": static, "dirs": directories, "identity": identity, - "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, + "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, "apiPaths": api_paths, })) diff --git a/tools/substation-monitor-test.js b/tools/substation-monitor-test.js new file mode 100644 index 0000000000000000000000000000000000000000..1a8a53cb2a63e83fbcdc85f5ce5839b954497923 --- /dev/null +++ b/tools/substation-monitor-test.js @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { probe } from "../service/substation/monitor.js"; + +const requestId = "2c42c339-cd80-4f22-a4d7-61a411bba5e8"; +const replies = [ + [401, { error: "unauthorized" }], + [200, { object: "list", data: [] }], + [200, { version: "fixture" }], +]; +let calls = []; +let scenario = "healthy"; +globalThis.fetch = async (url, options) => { + const index = calls.length; + calls.push({ url: String(url), options }); + if (scenario === "network") throw new Error("client-private-fixture"); + const [status, body] = replies[index]; + return new Response(JSON.stringify(scenario === "body" ? { private: "client-private-fixture" } : body), { + status: scenario === "redirect" ? 302 : status, + headers: { "X-Request-ID": scenario === "id" ? "client-private-fixture" : requestId }, + }); +}; +for (scenario of ["healthy", "redirect", "body", "id", "network"]) { + calls = []; + const checks = await probe("https://substation.invalid", "client-private-fixture", "management-private-fixture"); + assert.equal(checks.length, 3); + assert.equal(checks.every(check => check.ok), scenario === "healthy"); + assert.ok(calls.every(call => call.options.redirect === "manual" && call.options.signal instanceof AbortSignal)); + assert.equal(calls[0].options.headers.Authorization, undefined); + assert.equal(calls[1].options.headers.Authorization, "Bearer client-private-fixture"); + assert.equal(calls[2].options.headers.Authorization, "Bearer management-private-fixture"); + assert.ok(!JSON.stringify(checks).includes("private-fixture")); + if (scenario === "healthy") assert.ok(checks.every(check => check.requestId === requestId)); + if (scenario === "id" || scenario === "network") assert.ok(checks.every(check => check.requestId === null)); +} +console.log("Substation monitor: authentication, redirect rejection, response validation, request IDs and private failures passed"); diff --git a/tools/substation.md b/tools/substation.md index f6bbfe5054adc1d695e1734ab1df1aff4c48f862..9d66d714533c47c46e37cd14301b46049ceca3fe 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -3,7 +3,8 @@ `https://substation.ai.paperclover.net` serves the Substation dashboard and its native APIs. Snow OAuth protects the dashboard and requires `infra-admin`. Substation's trusted-proxy mode accepts the identity header from the local -OAuth proxy. Hub requests authenticate with Substation's own keys. +OAuth proxy. Caddy sends `/v1/*` and `/v0/management/*` directly to the hub, +which checks its own client and management keys. ## Set up accounts @@ -94,3 +95,23 @@ The in-app updater is unavailable inside the immutable image. CPA's live dataset, preview, keys, and routing are retired after the replacement passes production checks. Restricted off-server recovery copies retain the old store; restoring them requires a separate explicit request. + +## Request diagnostics + +API responses include a Caddy-generated `X-Request-ID`. API access records in +Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, +duration, upstream address, upstream status, and time to upstream headers. +Headers, query strings, cookies, and request/response bodies are excluded from +these access records. The log shipper retains them in Victoria Logs. + +The `monitor` task checks anonymous rejection, client-key model access, and +management-key version access through HTTPS every minute. It requires the edge +request ID and rejects redirects, so an OAuth login page cannot pass as a healthy +API. It sends no generation requests and logs only check outcomes and IDs. + +Inspect `substation` task `monitor` in the service logs, and search Caddy's +journal for the response's request ID. A 499 with no upstream status marks a +cancellation before upstream headers; an upstream 499 is recorded separately. +A 200 followed by a truncated stream needs the edge byte count and the hub's +stream outcome. A successful probe does not establish that every generation +stream completed. -- 2.54.1