authoravatarclover caruso<git@paperclover.net>2026-11-11 15:00:14-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log53fd9c969ec080da95877749caebd841358fe8c7
tree0d7dba6fc9295e7badb50705abc049b76280f8b7
parent665ddec25df72359c856c0603d356d5ce7b2831f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Record declared API upload size for cancellation diagnosis

The recurring Substation error is a Caddy cancellation after 60.045 seconds, with no upstream response headers. Two other failures hit a client-upload read timeout at 60 seconds. Full uploads with 65-second upstream header delays pass against the deployed Caddy binary, so response latency alone does not reproduce the failure. Record declared Content-Length beside bytes received to distinguish an incomplete upload on recurrence. Preserve header and body privacy. Validated complete and interrupted upload accounting, key separation, preview isolation, cancellations, slow headers across reload, and private logs with the deployed Caddy binary. Substation staging remains healthy. Assisted-by: gpt-6.1-sol

3 files changed, 19 insertions(+), 4 deletions(-)

tools/router-api-test.py+12-1
...@@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler):...@@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler):
21 started = threading.Event()21 started = threading.Event()
2222
23 def do_GET(self):23 def do_GET(self):
24 if self.path == "/v1/upload-canceled":
25 type(self).started.set()
24 self.rfile.read(int(self.headers.get("Content-Length", 0)))26 self.rfile.read(int(self.headers.get("Content-Length", 0)))
25 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"27 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
26 if self.headers.get("Authorization") != "Bearer " + key:28 if self.headers.get("Authorization") != "Bearer " + key:
...@@ -180,6 +182,11 @@ def main():...@@ -180,6 +182,11 @@ def main():
180 assert request("/v1/broken", "client-fixture")[0] == 502182 assert request("/v1/broken", "client-fixture")[0] == 502
181 Hub.started.clear()183 Hub.started.clear()
182 client = socket.create_connection(("127.0.0.1", port))184 client = socket.create_connection(("127.0.0.1", port))
185 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
186 assert Hub.started.wait(5)
187 client.close()
188 Hub.started.clear()
189 client = socket.create_connection(("127.0.0.1", port))
183 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())190 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
184 assert Hub.started.wait(5)191 assert Hub.started.wait(5)
185 client.close()192 client.close()
...@@ -202,6 +209,10 @@ def main():...@@ -202,6 +209,10 @@ def main():
202 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)209 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
203 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)210 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
204 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)211 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
212 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")
213 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000
214 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")
215 assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture")
205 assert "private-fixture" not in log_path.read_text()216 assert "private-fixture" not in log_path.read_text()
206 assert "client-fixture" not in log_path.read_text()217 assert "client-fixture" not in log_path.read_text()
207 assert "management-fixture" not in log_path.read_text()218 assert "management-fixture" not in log_path.read_text()
...@@ -210,7 +221,7 @@ def main():...@@ -210,7 +221,7 @@ def main():
210 process.wait(timeout=10)221 process.wait(timeout=10)
211 for backend in (hub, login):222 for backend in (hub, login):
212 backend.shutdown()223 backend.shutdown()
213 print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed")224 print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed")
214225
215226
216if __name__ == "__main__":227if __name__ == "__main__":
tools/router.py+1
...@@ -369,6 +369,7 @@ def render(token):...@@ -369,6 +369,7 @@ def render(token):
369 " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api",369 " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api",
370 f" log_append service {service}", " log_append request_id {http.request.uuid}",370 f" log_append service {service}", " log_append request_id {http.request.uuid}",
371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",
372 " log_append content_length {http.request.header.Content-Length}",
372 " log_append upstream {http.reverse_proxy.upstream.hostport}",373 " log_append upstream {http.reverse_proxy.upstream.hostport}",
373 " log_append upstream_status {http.reverse_proxy.status_code}",374 " log_append upstream_status {http.reverse_proxy.status_code}",
374 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",375 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",
tools/substation.md+6-3
...@@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request....@@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request.
100100
101API responses include a Caddy-generated `X-Request-ID`. API access records in101API responses include a Caddy-generated `X-Request-ID`. API access records in
102Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes,102Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes,
103duration, upstream address, upstream status, and time to upstream headers.103duration, declared content length, upstream address, upstream status, and time
104Headers, query strings, cookies, and request/response bodies are excluded from104to upstream headers. Other headers, query strings, cookies, and request/response
105these access records. The log shipper retains them in Victoria Logs.105bodies are excluded from these access records. The log shipper retains them in
106Victoria Logs.
106107
107The `monitor` task checks anonymous rejection, client-key model access, and108The `monitor` task checks anonymous rejection, client-key model access, and
108management-key version access through HTTPS every minute. It requires the edge109management-key version access through HTTPS every minute. It requires the edge
...@@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs....@@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs.
112Inspect `substation` task `monitor` in the service logs, and search Caddy's113Inspect `substation` task `monitor` in the service logs, and search Caddy's
113journal for the response's request ID. A 499 with no upstream status marks a114journal for the response's request ID. A 499 with no upstream status marks a
114cancellation before upstream headers; an upstream 499 is recorded separately.115cancellation before upstream headers; an upstream 499 is recorded separately.
116Compare `bytes_read` with `content_length` to identify incomplete fixed-length
117uploads. A missing length can mean a chunked request; it does not prove completion.
115A 200 followed by a truncated stream needs the edge byte count and the hub's118A 200 followed by a truncated stream needs the edge byte count and the hub's
116stream outcome. A successful probe does not establish that every generation119stream outcome. A successful probe does not establish that every generation
117stream completed.120stream completed.