From 9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 16:07:30 -0800 Subject: [PATCH] Let Claude retry stalled Substation uploads A captured Worktop upload stopped at 1.70 MB of 4.19 MB with an open server receive window, an unfilled TCP sequence hole, repeated SACKs and 1.78 MB queued at the sender. Caddy can report its body idle timeout as 499; Claude then treats this transient transport failure as a terminal client error. Default API 499 responses to X-Should-Retry: true while preserving an upstream retry decision. Keep the original status and diagnostics. Validated the generated routes against the deployed Caddy 2.11.7 over TLS: idle upload timeout, retry hints and explicit refusal, key separation, browser login, preview isolation, stream completion through reload and log privacy. Worktop Claude 2.1.322 failed a bare-499 fixture in one attempt, then retried and completed OK in two attempts with the hint. Staging healthy. Assisted-by: gpt-6.1-sol --- tools/router-api-test.py | 40 +++++++++++++++++++++++++++++++--------- tools/router.py | 3 +++ tools/substation.md | 2 ++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tools/router-api-test.py b/tools/router-api-test.py index 6a4c24dfcdfc42b8e10cb92c416fbacab892683e..79a1910c438ef023e57b06d77e5d2a2c973bcf63 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import argparse -from http.client import HTTPConnection +from http.client import HTTPSConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import re import socket +import ssl import subprocess import tempfile import threading @@ -28,6 +29,8 @@ class Hub(BaseHTTPRequestHandler): key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" if self.headers.get("Authorization") != "Bearer " + key: status, body = 401, b'{"error":"unauthorized"}' + elif self.path.startswith("/v1/transport-canceled"): + status, body = 499, b"" elif self.path.startswith("/v1/slow"): type(self).started.set() time.sleep(12) @@ -48,6 +51,8 @@ class Hub(BaseHTTPRequestHandler): self.send_header("Set-Cookie", "response-cookie-private-fixture") self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) self.send_header("X-Request-ID", "upstream-private-fixture") + if self.path == "/v1/transport-canceled-no-retry": + self.send_header("X-Should-Retry", "false") self.end_headers() self.wfile.write(body) except (BrokenPipeError, ConnectionResetError): @@ -135,18 +140,19 @@ def main(): site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] site = site.replace(" tls internal\n", "") config = root / "Caddyfile" - content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n" + content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n" config.write_text(content) for backend in (hub, login): threading.Thread(target=backend.serve_forever, daemon=True).start() log_path = root / "caddy.log" + tls = ssl._create_unverified_context() with log_path.open("w") as log: process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) try: def request(path, key=None, method="GET", body=None, **headers): if key: headers["Authorization"] = "Bearer " + key - connection = HTTPConnection("127.0.0.1", port, timeout=20) + connection = HTTPSConnection("localhost", port, timeout=20, context=tls) connection.request(method, path, body=body, headers=headers) response = connection.getresponse() result = response.status, dict(response.getheaders()), response.read() @@ -169,6 +175,22 @@ def main(): assert result == status, (path, result) assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] assert headers["X-Request-Id"] != "spoof-private-fixture" + assert "X-Should-Retry" not in headers + status, headers, _ = request("/v1/transport-canceled", "client-fixture") + assert status == 499 and headers["X-Should-Retry"] == "true" + status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture") + assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers) + client = HTTPSConnection("localhost", port, timeout=5, context=tls) + client.putrequest("POST", "/v1/upload-idle") + client.putheader("Authorization", "Bearer client-fixture") + client.putheader("Content-Length", "1000000") + client.endheaders(b"x" * 40000) + response = client.getresponse() + assert response.status in (499, 504), response.status + if response.status == 499: + assert response.headers["X-Should-Retry"] == "true" + response.read() + client.close() assert request("/api/summary", "client-fixture")[0] == 302 assert request("/v1", "client-fixture")[0] == 302 Hub.started.clear() @@ -182,13 +204,13 @@ def main(): assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome assert request("/v1/broken", "client-fixture")[0] == 502 Hub.started.clear() - client = socket.create_connection(("127.0.0.1", port)) - client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) + client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") + client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) assert Hub.started.wait(5) client.close() Hub.started.clear() - client = socket.create_connection(("127.0.0.1", port)) - client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) + client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") + client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) assert Hub.started.wait(5) client.close() deadline = time.monotonic() + 5 @@ -201,7 +223,7 @@ def main(): continue if entry.get("service") == service: entries.append(entry) - if any(entry.get("status") == 499 for entry in entries): + if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries): break assert time.monotonic() < deadline, entries time.sleep(.05) @@ -210,7 +232,7 @@ def main(): assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) - assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel") upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") diff --git a/tools/router.py b/tools/router.py index 5e28dd9564aaf5c442fd376e3e7596d389b91264..e2bc9a3832947234c4ec6c3485c48bcb0d2dc109 100644 --- a/tools/router.py +++ b/tools/router.py @@ -384,6 +384,9 @@ def render(token): upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", + # Caddy can return 499 to a connected client when its request-body idle timer expires. + " header X-Should-Retry true {", " match {", " status 499", + " header !X-Should-Retry", " }", " }", " request_header X-Request-ID {http.request.uuid}"] lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] lines.append(" }") diff --git a/tools/substation.md b/tools/substation.md index c985e9e3e094844669aad8b2b34cc8471f480418..b1e4f01b2b8aaca8c5d219dea54c372ced39837b 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -114,6 +114,8 @@ Inspect `substation` task `monitor` in the service logs, and search Caddy's journal for the response's request ID. A 499 with no upstream status can mean client cancellation or Caddy's request-body idle timeout; an upstream 499 is recorded separately. +API 499 responses default to `X-Should-Retry: true` so Claude can recover from +request-body timeouts; an upstream's explicit retry decision takes precedence. Compare `bytes_read` with `content_length` to identify incomplete fixed-length uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's -- 2.54.1