authoravatarclover caruso<git@paperclover.net>2026-11-11 15:00:14-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log53fd9c969ec080da95877749caebd841358fe8c7
tree0d7dba6fc9295e7badb50705abc049b76280f8b7
parent665ddec25df72359c856c0603d356d5ce7b2831f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Record declared API upload size for cancellation diagnosis

The recurring Substation error is a Caddy cancellation after 60.045 seconds, with no upstream response headers. Two other failures hit a client-upload read timeout at 60 seconds. Full uploads with 65-second upstream header delays pass against the deployed Caddy binary, so response latency alone does not reproduce the failure. Record declared Content-Length beside bytes received to distinguish an incomplete upload on recurrence. Preserve header and body privacy. Validated complete and interrupted upload accounting, key separation, preview isolation, cancellations, slow headers across reload, and private logs with the deployed Caddy binary. Substation staging remains healthy. Assisted-by: gpt-6.1-sol

3 files changed, 19 insertions(+), 4 deletions(-)

tools/router-api-test.py+12-1
......@@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler):
2121 started = threading.Event()
2222
2323 def do_GET(self):
24 if self.path == "/v1/upload-canceled":
25 type(self).started.set()
2426 self.rfile.read(int(self.headers.get("Content-Length", 0)))
2527 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
2628 if self.headers.get("Authorization") != "Bearer " + key:
......@@ -180,6 +182,11 @@ def main():
180182 assert request("/v1/broken", "client-fixture")[0] == 502
181183 Hub.started.clear()
182184 client = socket.create_connection(("127.0.0.1", port))
185 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
186 assert Hub.started.wait(5)
187 client.close()
188 Hub.started.clear()
189 client = socket.create_connection(("127.0.0.1", port))
183190 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
184191 assert Hub.started.wait(5)
185192 client.close()
......@@ -202,6 +209,10 @@ def main():
202209 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
203210 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
204211 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
212 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")
213 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000
214 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")
215 assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture")
205216 assert "private-fixture" not in log_path.read_text()
206217 assert "client-fixture" not in log_path.read_text()
207218 assert "management-fixture" not in log_path.read_text()
......@@ -210,7 +221,7 @@ def main():
210221 process.wait(timeout=10)
211222 for backend in (hub, login):
212223 backend.shutdown()
213 print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed")
224 print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed")
214225
215226
216227if __name__ == "__main__":
tools/router.py+1
......@@ -369,6 +369,7 @@ def render(token):
369369 " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api",
370370 f" log_append service {service}", " log_append request_id {http.request.uuid}",
371371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",
372 " log_append content_length {http.request.header.Content-Length}",
372373 " log_append upstream {http.reverse_proxy.upstream.hostport}",
373374 " log_append upstream_status {http.reverse_proxy.status_code}",
374375 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",
tools/substation.md+6-3
......@@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request.
100100
101101API responses include a Caddy-generated `X-Request-ID`. API access records in
102102Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes,
103duration, upstream address, upstream status, and time to upstream headers.
104Headers, query strings, cookies, and request/response bodies are excluded from
105these access records. The log shipper retains them in Victoria Logs.
103duration, declared content length, upstream address, upstream status, and time
104to upstream headers. Other headers, query strings, cookies, and request/response
105bodies are excluded from these access records. The log shipper retains them in
106Victoria Logs.
106107
107108The `monitor` task checks anonymous rejection, client-key model access, and
108109management-key version access through HTTPS every minute. It requires the edge
......@@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs.
112113Inspect `substation` task `monitor` in the service logs, and search Caddy's
113114journal for the response's request ID. A 499 with no upstream status marks a
114115cancellation before upstream headers; an upstream 499 is recorded separately.
116Compare `bytes_read` with `content_length` to identify incomplete fixed-length
117uploads. A missing length can mean a chunked request; it does not prove completion.
115118A 200 followed by a truncated stream needs the edge byte count and the hub's
116119stream outcome. A successful probe does not establish that every generation
117120stream completed.