authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 00:19:46-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 15:36:02-07:00
logd9b295aa93ef00565b91d9caa3ec4875b94596a4
treeb83e1578da850692b072ab5fa6e213c93628fed3
parentfed95315bf3a060a272a75be193773cde19900f9
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Proxy Shale pages through the authenticated internal gateway

Keep upstream streaming independent of the dashboard request timeout. Assisted-by: gpt-6

3 files changed, 39 insertions(+), 16 deletions(-)

dashboard/src/main.rs+18
......@@ -91,6 +91,7 @@ struct App {
9191 relay: relay::Broker,
9292 shale: shale::Backend,
9393 http: reqwest::Client,
94 shale_http: reqwest::Client,
9495 internal: Option<(url::Url, reqwest::Client)>,
9596 cache: cache::Cache,
9697 nomad_slots: Semaphore,
......@@ -382,6 +383,22 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
382383 },
383384 )
384385 .transpose()?;
386 let mut shale_http = reqwest::Client::builder()
387 .connect_timeout(Duration::from_secs(5))
388 .read_timeout(Duration::from_secs(30))
389 .redirect(reqwest::redirect::Policy::none())
390 .retry(reqwest::retry::never());
391 for certificate in &certificates {
392 shale_http = shale_http.add_root_certificate(certificate.clone());
393 }
394 if internal.is_some() {
395 let mut token = axum::http::HeaderValue::from_str(proof.as_ref().unwrap())?;
396 token.set_sensitive(true);
397 let mut headers = HeaderMap::new();
398 headers.insert("Studio-Proxy-Token", token);
399 shale_http = shale_http.default_headers(headers);
400 }
401 let shale_http = shale_http.build()?;
385402 let (live, _) = watch::channel(Bytes::new());
386403 let index = std::env::var("STUDIO_INDEX_POOL").ok().map(|pool| {
387404 Arc::new(index::Index::new(
......@@ -459,6 +476,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
459476 )
460477 .map_err(|error| std::io::Error::other(error.message))?,
461478 http: client().build()?,
479 shale_http,
462480 internal,
463481 cache: cache::Cache::default(),
464482 nomad_slots: Semaphore::new(4),
dashboard/src/shale_page.rs+10-11
......@@ -182,6 +182,13 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
182182 )
183183 .into_response());
184184 }
185 let target = match &app.internal {
186 Some((base, _)) => format!("{base}services/{service}{uri}"),
187 None => format!("http://{upstream}{uri}"),
188 };
189 if app.internal.is_some() {
190 parts.headers.remove("host");
191 }
185192 strip_hop_headers(&mut parts.headers);
186193 for name in [
187194 "studio-shale-upstream",
......@@ -199,17 +206,9 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
199206 parts
200207 .headers
201208 .insert("accept-encoding", "identity".parse().unwrap());
202 static HTTP: std::sync::LazyLock<reqwest::Client> = std::sync::LazyLock::new(|| {
203 reqwest::Client::builder()
204 .connect_timeout(Duration::from_secs(5))
205 .read_timeout(Duration::from_secs(30))
206 .redirect(reqwest::redirect::Policy::none())
207 .retry(reqwest::retry::never())
208 .build()
209 .unwrap()
210 });
211 let mut upstream_response = HTTP
212 .request(parts.method.clone(), format!("http://{upstream}{uri}"))
209 let mut upstream_response = app
210 .shale_http
211 .request(parts.method.clone(), target)
213212 .headers(parts.headers)
214213 .body(reqwest::Body::wrap_stream(body.into_data_stream()))
215214 .send()
tools/dashboard-shale-page-test.py+11-5
......@@ -52,7 +52,7 @@ def main():
5252 protocol_version = 'HTTP/1.1'
5353
5454 def do_GET(self):
55 observations.append((self.command, self.path, dict(self.headers)))
55 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
5656 if self.path == '/binary':
5757 return self.respond(blob, 'application/octet-stream')
5858 if self.path == '/large':
......@@ -69,7 +69,7 @@ def main():
6969
7070 def do_POST(self):
7171 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
72 observations.append((self.command, self.path, dict(self.headers), body))
72 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
7373 self.respond(body, 'application/octet-stream', 201)
7474
7575 def respond(self, body, content_type, status=200, headers=()):
......@@ -95,13 +95,15 @@ def main():
9595 stack.callback(fixture.server_close)
9696 stack.callback(fixture.shutdown)
9797 threading.Thread(target=fixture.serve_forever, daemon=True).start()
98 dashboard_port, gateway_port = port(), port()
98 dashboard_port, gateway_port, internal_port = port(), port(), port()
9999 proof = 'a' * 64
100100 token = root / 'proxy.token'
101101 token.write_text(proof)
102 certificate, gateway_key = root / 'gateway.pem', root / 'gateway.key'
103 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
102104 data = root / 'data'
103 environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')}
104 for key in ['STUDIO_INTERNAL_URL', 'STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']:
105 environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_INTERNAL_URL': f'https://localhost:{internal_port}', 'STUDIO_CA_BUNDLE': str(certificate), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')}
106 for key in ['STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']:
105107 environment.pop(key, None)
106108 log = stack.enter_context((root / 'dashboard.log').open('w'))
107109 dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log)
......@@ -149,6 +151,10 @@ def main():
149151
150152 preview_port = port()
151153 config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.studio.test', gateway_port) + '\n' + local_site('shale-preview-12345678.studio.test', preview_port)
154 config += f'\nhttps://localhost:{internal_port} {{\n tls {certificate} {gateway_key}\n @trusted header Studio-Proxy-Token {proof}\n handle @trusted {{\n request_header -Studio-Proxy-Token\n'
155 for service in ['shale', 'shale-preview-12345678']:
156 config += f' handle_path /services/{service}/* {{\n reverse_proxy 127.0.0.1:{fixture.server_port}\n }}\n'
157 config += ' }\n handle {\n respond 403\n }\n}\n'
152158 config_path = root / 'Caddyfile'
153159 config_path.write_text(config)
154160 subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)