diff --git a/tools/router-api-test.py b/tools/router-api-test.py index 6a4c24dfcdfc42b8e10cb92c416fbacab892683e..79a1910c438ef023e57b06d77e5d2a2c973bcf63 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import argparse -from http.client import HTTPConnection +from http.client import HTTPSConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import re import socket +import ssl import subprocess import tempfile import threading @@ -28,6 +29,8 @@ class Hub(BaseHTTPRequestHandler): key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" if self.headers.get("Authorization") != "Bearer " + key: status, body = 401, b'{"error":"unauthorized"}' + elif self.path.startswith("/v1/transport-canceled"): + status, body = 499, b"" elif self.path.startswith("/v1/slow"): type(self).started.set() time.sleep(12) @@ -48,6 +51,8 @@ class Hub(BaseHTTPRequestHandler): self.send_header("Set-Cookie", "response-cookie-private-fixture") self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) self.send_header("X-Request-ID", "upstream-private-fixture") + if self.path == "/v1/transport-canceled-no-retry": + self.send_header("X-Should-Retry", "false") self.end_headers() self.wfile.write(body) except (BrokenPipeError, ConnectionResetError): @@ -135,18 +140,19 @@ def main(): site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] site = site.replace(" tls internal\n", "") config = root / "Caddyfile" - content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n" + content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n" config.write_text(content) for backend in (hub, login): threading.Thread(target=backend.serve_forever, daemon=True).start() log_path = root / "caddy.log" + tls = ssl._create_unverified_context() with log_path.open("w") as log: process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) try: def request(path, key=None, method="GET", body=None, **headers): if key: headers["Authorization"] = "Bearer " + key - connection = HTTPConnection("127.0.0.1", port, timeout=20) + connection = HTTPSConnection("localhost", port, timeout=20, context=tls) connection.request(method, path, body=body, headers=headers) response = connection.getresponse() result = response.status, dict(response.getheaders()), response.read() @@ -169,6 +175,22 @@ def main(): assert result == status, (path, result) assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] assert headers["X-Request-Id"] != "spoof-private-fixture" + assert "X-Should-Retry" not in headers + status, headers, _ = request("/v1/transport-canceled", "client-fixture") + assert status == 499 and headers["X-Should-Retry"] == "true" + status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture") + assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers) + client = HTTPSConnection("localhost", port, timeout=5, context=tls) + client.putrequest("POST", "/v1/upload-idle") + client.putheader("Authorization", "Bearer client-fixture") + client.putheader("Content-Length", "1000000") + client.endheaders(b"x" * 40000) + response = client.getresponse() + assert response.status in (499, 504), response.status + if response.status == 499: + assert response.headers["X-Should-Retry"] == "true" + response.read() + client.close() assert request("/api/summary", "client-fixture")[0] == 302 assert request("/v1", "client-fixture")[0] == 302 Hub.started.clear() @@ -182,13 +204,13 @@ def main(): assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome assert request("/v1/broken", "client-fixture")[0] == 502 Hub.started.clear() - client = socket.create_connection(("127.0.0.1", port)) - client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) + client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") + client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) assert Hub.started.wait(5) client.close() Hub.started.clear() - client = socket.create_connection(("127.0.0.1", port)) - client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) + client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") + client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) assert Hub.started.wait(5) client.close() deadline = time.monotonic() + 5 @@ -201,7 +223,7 @@ def main(): continue if entry.get("service") == service: entries.append(entry) - if any(entry.get("status") == 499 for entry in entries): + if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries): break assert time.monotonic() < deadline, entries time.sleep(.05) @@ -210,7 +232,7 @@ def main(): assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) - assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel") upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") diff --git a/tools/router.py b/tools/router.py index 5e28dd9564aaf5c442fd376e3e7596d389b91264..e2bc9a3832947234c4ec6c3485c48bcb0d2dc109 100644 --- a/tools/router.py +++ b/tools/router.py @@ -384,6 +384,9 @@ def render(token): upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", + # Caddy can return 499 to a connected client when its request-body idle timer expires. + " header X-Should-Retry true {", " match {", " status 499", + " header !X-Should-Retry", " }", " }", " request_header X-Request-ID {http.request.uuid}"] lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] lines.append(" }") diff --git a/tools/substation.md b/tools/substation.md index c985e9e3e094844669aad8b2b34cc8471f480418..b1e4f01b2b8aaca8c5d219dea54c372ced39837b 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -114,6 +114,8 @@ Inspect `substation` task `monitor` in the service logs, and search Caddy's journal for the response's request ID. A 499 with no upstream status can mean client cancellation or Caddy's request-body idle timeout; an upstream 499 is recorded separately. +API 499 responses default to `X-Should-Retry: true` so Claude can recover from +request-body timeouts; an upstream's explicit retry decision takes precedence. Compare `bytes_read` with `content_length` to identify incomplete fixed-length uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's