From d9b295aa93ef00565b91d9caa3ec4875b94596a4 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 5 Oct 2026 00:19:46 -0700 Subject: [PATCH] Proxy Shale pages through the authenticated internal gateway Keep upstream streaming independent of the dashboard request timeout. Assisted-by: gpt-6 --- dashboard/src/main.rs | 18 ++++++++++++++++++ dashboard/src/shale_page.rs | 21 ++++++++++----------- tools/dashboard-shale-page-test.py | 16 +++++++++++----- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index 1082d8038f884e17f583f96c66a4f1cca91522cd..ec3e3bcc338c6f04fc897f12c56bfa4577169f89 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -91,6 +91,7 @@ struct App { relay: relay::Broker, shale: shale::Backend, http: reqwest::Client, + shale_http: reqwest::Client, internal: Option<(url::Url, reqwest::Client)>, cache: cache::Cache, nomad_slots: Semaphore, @@ -382,6 +383,22 @@ async fn main() -> std::result::Result<(), Box> { }, ) .transpose()?; + let mut shale_http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()); + for certificate in &certificates { + shale_http = shale_http.add_root_certificate(certificate.clone()); + } + if internal.is_some() { + let mut token = axum::http::HeaderValue::from_str(proof.as_ref().unwrap())?; + token.set_sensitive(true); + let mut headers = HeaderMap::new(); + headers.insert("Studio-Proxy-Token", token); + shale_http = shale_http.default_headers(headers); + } + let shale_http = shale_http.build()?; let (live, _) = watch::channel(Bytes::new()); let index = std::env::var("STUDIO_INDEX_POOL").ok().map(|pool| { Arc::new(index::Index::new( @@ -459,6 +476,7 @@ async fn main() -> std::result::Result<(), Box> { ) .map_err(|error| std::io::Error::other(error.message))?, http: client().build()?, + shale_http, internal, cache: cache::Cache::default(), nomad_slots: Semaphore::new(4), diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index 17f1383fadb10474063c30a65af5b436c4f1632e..1d079ee7a7ee1e9f64408180867effb246f9328d 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -182,6 +182,13 @@ pub async fn proxy(app: Arc, request: Request) -> Result { ) .into_response()); } + let target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; + if app.internal.is_some() { + parts.headers.remove("host"); + } strip_hop_headers(&mut parts.headers); for name in [ "studio-shale-upstream", @@ -199,17 +206,9 @@ pub async fn proxy(app: Arc, request: Request) -> Result { parts .headers .insert("accept-encoding", "identity".parse().unwrap()); - static HTTP: std::sync::LazyLock = std::sync::LazyLock::new(|| { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(5)) - .read_timeout(Duration::from_secs(30)) - .redirect(reqwest::redirect::Policy::none()) - .retry(reqwest::retry::never()) - .build() - .unwrap() - }); - let mut upstream_response = HTTP - .request(parts.method.clone(), format!("http://{upstream}{uri}")) + let mut upstream_response = app + .shale_http + .request(parts.method.clone(), target) .headers(parts.headers) .body(reqwest::Body::wrap_stream(body.into_data_stream())) .send() diff --git a/tools/dashboard-shale-page-test.py b/tools/dashboard-shale-page-test.py index 4a71e59c209a27d4cc0b0c26c5698fdc21dabc19..ff5af3a3672a9e1f6b9dcb7e4ba1bedc6a0a0ec9 100644 --- a/tools/dashboard-shale-page-test.py +++ b/tools/dashboard-shale-page-test.py @@ -52,7 +52,7 @@ def main(): protocol_version = 'HTTP/1.1' def do_GET(self): - observations.append((self.command, self.path, dict(self.headers))) + observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()})) if self.path == '/binary': return self.respond(blob, 'application/octet-stream') if self.path == '/large': @@ -69,7 +69,7 @@ def main(): def do_POST(self): body = self.rfile.read(int(self.headers.get('Content-Length', 0))) - observations.append((self.command, self.path, dict(self.headers), body)) + observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body)) self.respond(body, 'application/octet-stream', 201) def respond(self, body, content_type, status=200, headers=()): @@ -95,13 +95,15 @@ def main(): stack.callback(fixture.server_close) stack.callback(fixture.shutdown) threading.Thread(target=fixture.serve_forever, daemon=True).start() - dashboard_port, gateway_port = port(), port() + dashboard_port, gateway_port, internal_port = port(), port(), port() proof = 'a' * 64 token = root / 'proxy.token' token.write_text(proof) + certificate, gateway_key = root / 'gateway.pem', root / 'gateway.key' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True) data = root / 'data' - environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')} - for key in ['STUDIO_INTERNAL_URL', 'STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']: + environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_INTERNAL_URL': f'https://localhost:{internal_port}', 'STUDIO_CA_BUNDLE': str(certificate), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')} + for key in ['STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']: environment.pop(key, None) log = stack.enter_context((root / 'dashboard.log').open('w')) dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log) @@ -149,6 +151,10 @@ def main(): preview_port = port() config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.studio.test', gateway_port) + '\n' + local_site('shale-preview-12345678.studio.test', preview_port) + config += f'\nhttps://localhost:{internal_port} {{\n tls {certificate} {gateway_key}\n @trusted header Studio-Proxy-Token {proof}\n handle @trusted {{\n request_header -Studio-Proxy-Token\n' + for service in ['shale', 'shale-preview-12345678']: + config += f' handle_path /services/{service}/* {{\n reverse_proxy 127.0.0.1:{fixture.server_port}\n }}\n' + config += ' }\n handle {\n respond 403\n }\n}\n' config_path = root / 'Caddyfile' config_path.write_text(config) subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env) -- 2.54.0