authoravatarclover caruso<git@paperclover.net>2026-11-11 16:07:30-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640
tree3b9faa611d878270804a80696fe449a765ce6bca
parentc65330b68ae0ff9f783e00a339d61006cf9c8ff0
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Let Claude retry stalled Substation uploads

A captured Worktop upload stopped at 1.70 MB of 4.19 MB with an open server receive window, an unfilled TCP sequence hole, repeated SACKs and 1.78 MB queued at the sender. Caddy can report its body idle timeout as 499; Claude then treats this transient transport failure as a terminal client error. Default API 499 responses to X-Should-Retry: true while preserving an upstream retry decision. Keep the original status and diagnostics. Validated the generated routes against the deployed Caddy 2.11.7 over TLS: idle upload timeout, retry hints and explicit refusal, key separation, browser login, preview isolation, stream completion through reload and log privacy. Worktop Claude 2.1.322 failed a bare-499 fixture in one attempt, then retried and completed OK in two attempts with the hint. Staging healthy. Assisted-by: gpt-6.1-sol

3 files changed, 36 insertions(+), 9 deletions(-)

tools/router-api-test.py+31-9
...@@ -1,12 +1,13 @@...@@ -1,12 +1,13 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import argparse2import argparse
3from http.client import HTTPConnection3from http.client import HTTPSConnection
4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json5import json
6import os6import os
7from pathlib import Path7from pathlib import Path
8import re8import re
9import socket9import socket
10import ssl
10import subprocess11import subprocess
11import tempfile12import tempfile
12import threading13import threading
...@@ -28,6 +29,8 @@ class Hub(BaseHTTPRequestHandler):...@@ -28,6 +29,8 @@ class Hub(BaseHTTPRequestHandler):
28 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"29 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
29 if self.headers.get("Authorization") != "Bearer " + key:30 if self.headers.get("Authorization") != "Bearer " + key:
30 status, body = 401, b'{"error":"unauthorized"}'31 status, body = 401, b'{"error":"unauthorized"}'
32 elif self.path.startswith("/v1/transport-canceled"):
33 status, body = 499, b""
31 elif self.path.startswith("/v1/slow"):34 elif self.path.startswith("/v1/slow"):
32 type(self).started.set()35 type(self).started.set()
33 time.sleep(12)36 time.sleep(12)
...@@ -48,6 +51,8 @@ class Hub(BaseHTTPRequestHandler):...@@ -48,6 +51,8 @@ class Hub(BaseHTTPRequestHandler):
48 self.send_header("Set-Cookie", "response-cookie-private-fixture")51 self.send_header("Set-Cookie", "response-cookie-private-fixture")
49 self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", ""))52 self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", ""))
50 self.send_header("X-Request-ID", "upstream-private-fixture")53 self.send_header("X-Request-ID", "upstream-private-fixture")
54 if self.path == "/v1/transport-canceled-no-retry":
55 self.send_header("X-Should-Retry", "false")
51 self.end_headers()56 self.end_headers()
52 self.wfile.write(body)57 self.wfile.write(body)
53 except (BrokenPipeError, ConnectionResetError):58 except (BrokenPipeError, ConnectionResetError):
...@@ -135,18 +140,19 @@ def main():...@@ -135,18 +140,19 @@ def main():
135 site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0]140 site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0]
136 site = site.replace(" tls internal\n", "")141 site = site.replace(" tls internal\n", "")
137 config = root / "Caddyfile"142 config = root / "Caddyfile"
138 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n"143 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n"
139 config.write_text(content)144 config.write_text(content)
140 for backend in (hub, login):145 for backend in (hub, login):
141 threading.Thread(target=backend.serve_forever, daemon=True).start()146 threading.Thread(target=backend.serve_forever, daemon=True).start()
142 log_path = root / "caddy.log"147 log_path = root / "caddy.log"
148 tls = ssl._create_unverified_context()
143 with log_path.open("w") as log:149 with log_path.open("w") as log:
144 process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment)150 process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment)
145 try:151 try:
146 def request(path, key=None, method="GET", body=None, **headers):152 def request(path, key=None, method="GET", body=None, **headers):
147 if key:153 if key:
148 headers["Authorization"] = "Bearer " + key154 headers["Authorization"] = "Bearer " + key
149 connection = HTTPConnection("127.0.0.1", port, timeout=20)155 connection = HTTPSConnection("localhost", port, timeout=20, context=tls)
150 connection.request(method, path, body=body, headers=headers)156 connection.request(method, path, body=body, headers=headers)
151 response = connection.getresponse()157 response = connection.getresponse()
152 result = response.status, dict(response.getheaders()), response.read()158 result = response.status, dict(response.getheaders()), response.read()
...@@ -169,6 +175,22 @@ def main():...@@ -169,6 +175,22 @@ def main():
169 assert result == status, (path, result)175 assert result == status, (path, result)
170 assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"]176 assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"]
171 assert headers["X-Request-Id"] != "spoof-private-fixture"177 assert headers["X-Request-Id"] != "spoof-private-fixture"
178 assert "X-Should-Retry" not in headers
179 status, headers, _ = request("/v1/transport-canceled", "client-fixture")
180 assert status == 499 and headers["X-Should-Retry"] == "true"
181 status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture")
182 assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers)
183 client = HTTPSConnection("localhost", port, timeout=5, context=tls)
184 client.putrequest("POST", "/v1/upload-idle")
185 client.putheader("Authorization", "Bearer client-fixture")
186 client.putheader("Content-Length", "1000000")
187 client.endheaders(b"x" * 40000)
188 response = client.getresponse()
189 assert response.status in (499, 504), response.status
190 if response.status == 499:
191 assert response.headers["X-Should-Retry"] == "true"
192 response.read()
193 client.close()
172 assert request("/api/summary", "client-fixture")[0] == 302194 assert request("/api/summary", "client-fixture")[0] == 302
173 assert request("/v1", "client-fixture")[0] == 302195 assert request("/v1", "client-fixture")[0] == 302
174 Hub.started.clear()196 Hub.started.clear()
...@@ -182,13 +204,13 @@ def main():...@@ -182,13 +204,13 @@ def main():
182 assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome204 assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome
183 assert request("/v1/broken", "client-fixture")[0] == 502205 assert request("/v1/broken", "client-fixture")[0] == 502
184 Hub.started.clear()206 Hub.started.clear()
185 client = socket.create_connection(("127.0.0.1", port))207 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
186 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)208 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
187 assert Hub.started.wait(5)209 assert Hub.started.wait(5)
188 client.close()210 client.close()
189 Hub.started.clear()211 Hub.started.clear()
190 client = socket.create_connection(("127.0.0.1", port))212 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
191 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())213 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
192 assert Hub.started.wait(5)214 assert Hub.started.wait(5)
193 client.close()215 client.close()
194 deadline = time.monotonic() + 5216 deadline = time.monotonic() + 5
...@@ -201,7 +223,7 @@ def main():...@@ -201,7 +223,7 @@ def main():
201 continue223 continue
202 if entry.get("service") == service:224 if entry.get("service") == service:
203 entries.append(entry)225 entries.append(entry)
204 if any(entry.get("status") == 499 for entry in entries):226 if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries):
205 break227 break
206 assert time.monotonic() < deadline, entries228 assert time.monotonic() < deadline, entries
207 time.sleep(.05)229 time.sleep(.05)
...@@ -210,7 +232,7 @@ def main():...@@ -210,7 +232,7 @@ def main():
210 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)232 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)
211 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)233 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
212 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)234 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
213 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)235 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel")
214 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")236 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")
215 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000237 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000
216 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")238 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")
tools/router.py+3
...@@ -384,6 +384,9 @@ def render(token):...@@ -384,6 +384,9 @@ def render(token):
384 upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}")384 upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}")
385 paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target))385 paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target))
386 lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}",386 lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}",
387 # Caddy can return 499 to a connected client when its request-body idle timer expires.
388 " header X-Should-Retry true {", " match {", " status 499",
389 " header !X-Should-Retry", " }", " }",
387 " request_header X-Request-ID {http.request.uuid}"]390 " request_header X-Request-ID {http.request.uuid}"]
388 lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"]391 lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"]
389 lines.append(" }")392 lines.append(" }")
tools/substation.md+2
...@@ -114,6 +114,8 @@ Inspect `substation` task `monitor` in the service logs, and search Caddy's...@@ -114,6 +114,8 @@ Inspect `substation` task `monitor` in the service logs, and search Caddy's
114journal for the response's request ID. A 499 with no upstream status can mean114journal for the response's request ID. A 499 with no upstream status can mean
115client cancellation or Caddy's request-body idle timeout; an upstream 499 is115client cancellation or Caddy's request-body idle timeout; an upstream 499 is
116recorded separately.116recorded separately.
117API 499 responses default to `X-Should-Retry: true` so Claude can recover from
118request-body timeouts; an upstream's explicit retry decision takes precedence.
117Compare `bytes_read` with `content_length` to identify incomplete fixed-length119Compare `bytes_read` with `content_length` to identify incomplete fixed-length
118uploads. A missing length can mean a chunked request; it does not prove completion.120uploads. A missing length can mean a chunked request; it does not prove completion.
119A 200 followed by a truncated stream needs the edge byte count and the hub's121A 200 followed by a truncated stream needs the edge byte count and the hub's