diff --git a/nixos/caddy-body-timeout.patch b/nixos/caddy-body-timeout.patch new file mode 100644 index 0000000000000000000000000000000000000000..d20db88727f41eeb32bb388a4c2fbfb33f622da1 --- /dev/null +++ b/nixos/caddy-body-timeout.patch @@ -0,0 +1,35 @@ +--- a/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/idletimeout.go ++++ b/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/idletimeout.go +@@ -69,8 +69,8 @@ + // IdleTimeoutReader wraps a request body with IdleDeadline, resetting + // the read deadline before every Read call instead of bounding the + // whole body transfer with a single hard deadline. +-// The deadline is cleared after an error (most likely io.EOF) is encountered when reading the body +-// to prevent context canceled error for h1 requests. ++// EOF clears the deadline to permit HTTP/1 keep-alive. ++// Failed reads keep it armed so net/http cannot block draining the body in Close. + // see: https://github.com/caddyserver/caddy/issues/8103 + type IdleTimeoutReader struct { + io.ReadCloser +@@ -108,7 +108,7 @@ + r.Deadline.transferred += int64(n) + if err != nil { + r.terminalErr = err +- if !r.finished { ++ if err == io.EOF && !r.finished { + r.clearDeadlineLocked() + } + } +--- a/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go ++++ b/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go +@@ -735,6 +735,10 @@ + // request, which is not our failure; don't retry or ding the upstream. + // Record a 499 (client closed request) so the access log reflects the + // disconnect instead of a misleading 0 status (see #7396). ++ // Cancellation can leave body bytes queued on a still-open client socket. ++ if origReq.ProtoMajor == 1 { ++ w.Header().Set("Connection", "close") ++ } + w.WriteHeader(499) + return true, nil + } diff --git a/nixos/configuration.nix b/nixos/configuration.nix index bccf9b25f415a4e75d26e77da9049251486d1396..fcdbf2c7aa0127aedd95d99e7d4492812531ad80 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -83,10 +83,12 @@ in services.openssh.enable = true; services.caddy = { enable = true; - package = pkgs.caddy.withPlugins { + package = (pkgs.caddy.withPlugins { plugins = [ "github.com/caddyserver/replace-response@v0.0.0-20250618171559-80962887e4c6" ]; hash = "sha256-iWtaVD4LrOgCFZc90crAlHYbHHxz8zCFnvD2ZLRkO0Q="; - }; + }).overrideAttrs (old: { + patches = (old.patches or []) ++ [ ./caddy-body-timeout.patch ]; + }); extraConfig = "import /var/lib/caddy/routes.caddy"; virtualHosts = { ":8082".extraConfig = "reverse_proxy 127.0.0.1:4646"; diff --git a/tools/net.paperclover.tcp-rack-off.plist b/tools/net.paperclover.tcp-rack-off.plist new file mode 100644 index 0000000000000000000000000000000000000000..99bf76e7618520021f29b1b371db75f5aa7f9ea6 --- /dev/null +++ b/tools/net.paperclover.tcp-rack-off.plist @@ -0,0 +1,16 @@ + + + + + Label + net.paperclover.tcp-rack-off + ProgramArguments + + /usr/sbin/sysctl + -w + net.inet.tcp.rack=0 + + RunAtLoad + + + diff --git a/tools/router-api-test.py b/tools/router-api-test.py index 79a1910c438ef023e57b06d77e5d2a2c973bcf63..9dfa3ba81c79cd3a8bf255fa1778cfc8adbf815c 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import argparse -from http.client import HTTPSConnection +from http.client import HTTPResponse, HTTPSConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os @@ -31,6 +31,8 @@ class Hub(BaseHTTPRequestHandler): status, body = 401, b'{"error":"unauthorized"}' elif self.path.startswith("/v1/transport-canceled"): status, body = 499, b"" + elif self.path.startswith("/v1/upstream-timeout"): + status, body = 504, b"" elif self.path.startswith("/v1/slow"): type(self).started.set() time.sleep(12) @@ -51,7 +53,7 @@ class Hub(BaseHTTPRequestHandler): self.send_header("Set-Cookie", "response-cookie-private-fixture") self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) self.send_header("X-Request-ID", "upstream-private-fixture") - if self.path == "/v1/transport-canceled-no-retry": + if self.path in {"/v1/transport-canceled-no-retry", "/v1/upstream-timeout"}: self.send_header("X-Should-Retry", "false") self.end_headers() self.wfile.write(body) @@ -180,16 +182,34 @@ def main(): assert status == 499 and headers["X-Should-Retry"] == "true" status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture") assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers) - client = HTTPSConnection("localhost", port, timeout=5, context=tls) - client.putrequest("POST", "/v1/upload-idle") - client.putheader("Authorization", "Bearer client-fixture") - client.putheader("Content-Length", "1000000") - client.endheaders(b"x" * 40000) - response = client.getresponse() - assert response.status in (499, 504), response.status - if response.status == 499: + assert "Connection" not in headers + status, headers, _ = request("/v1/upstream-timeout", "client-fixture") + assert status == 504 and headers["X-Should-Retry"] == "false" and "Connection" not in headers + for size in (100000, 2000000): + wire = tls.wrap_socket(socket.create_connection(("127.0.0.1", port), timeout=5), server_hostname="localhost") + wire.sendall(("POST /v1/upload-idle HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer client-fixture\r\n" + f"Content-Length: {size}\r\n\r\n").encode() + b"x" * 40000) + response = HTTPResponse(wire) + response.begin() + assert response.status in (499, 504), response.status + response.read() + assert response.headers["Connection"] == "close", dict(response.getheaders()) assert response.headers["X-Should-Retry"] == "true" - response.read() + assert re.fullmatch(r"[0-9a-f-]{36}", response.headers["X-Request-ID"]) + try: + wire.sendall(b"x" * (size - 40000)) + assert wire.recv(4096) == b"", "late request body became another response" + except (BrokenPipeError, ConnectionResetError, ssl.SSLError): + pass + finally: + response.close() + wire.close() + client = HTTPSConnection("localhost", port, timeout=5, context=tls) + for _ in range(2): + client.request("GET", "/v1/models", headers={"Authorization": "Bearer client-fixture"}) + response = client.getresponse() + assert response.status == 200 and response.headers.get("Connection") != "close" + response.read() client.close() assert request("/api/summary", "client-fixture")[0] == 302 assert request("/v1", "client-fixture")[0] == 302 diff --git a/tools/router.py b/tools/router.py index e2bc9a3832947234c4ec6c3485c48bcb0d2dc109..1aab5b7fd45cbb065515ef9a73abb7bb412bc0fc 100644 --- a/tools/router.py +++ b/tools/router.py @@ -375,6 +375,11 @@ def render(token): " log_append upstream_status {http.reverse_proxy.status_code}", " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", " log_append proxy_ms {http.reverse_proxy.duration_ms}"] + lines += [" handle_errors 499 504 {", " @api_error path " + " ".join(sorted(api_paths)), + " handle @api_error {", " @http1 not protocol http/2+", + # Late body bytes must not become the next HTTP/1 request after a timeout. + " header @http1 Connection close", " header X-Request-ID {http.request.uuid}", + " header X-Should-Retry true", " }", ' respond "" {err.status_code}', " }"] for index, target in enumerate(sorted(set(api_paths.values()))): upstreams = set() for instance in nomad("/v1/service/" + target, token): diff --git a/tools/substation.md b/tools/substation.md index 3894ff356393326752ea42b6678f5ad061ec22cd..8b278ca6923b39c39cc415419119cdd3eb09b892 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -116,6 +116,14 @@ client cancellation or Caddy's request-body idle timeout; an upstream 499 is recorded separately. API 499 responses default to `X-Should-Retry: true` so Claude can recover from request-body timeouts; an upstream's explicit retry decision takes precedence. +Caddy-generated API 499/504 errors close HTTP/1 connections and include the +request ID and retry hint. A timed-out upload can still have queued body bytes; +reusing that connection lets those bytes become an invalid next request and +can produce an unlogged 431 before routing. Upstream responses keep their +retry decisions and connection behavior. +The packaged [Caddy patch](../nixos/caddy-body-timeout.patch) preserves failed +read deadlines while the upstream transport closes the body. Otherwise, +net/http can block draining a small unfinished upload before error handling. Compare `bytes_read` with `content_length` to identify incomplete fixed-length uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's @@ -142,6 +150,11 @@ Worktop's `net.paperclover.substation-watch` LaunchAgent records matching Substation sockets under `~/.local/state/substation-watch`. Minute files retain 12 hours of process IDs, CPU time/state, TCP queues, byte counters, retransmissions and windows. +Paired captures showed Worktop leave a SACK-reported hole unrepaired for 75–109 +seconds. `net.paperclover.tcp-rack-off` disables RACK before login while retaining +SACK recovery; it is a system-wide transport trade, not a Substation setting. +The root-owned LaunchDaemon is installed from the exported +`tools/net.paperclover.tcp-rack-off.plist` and should exit zero after each boot. The installed script is an exported revision of `tools/substation-client-watch.py`; the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. Match UTC timestamps to the server incident before comparing TCP progress.