diff --git a/dashboard/Cargo.lock b/dashboard/Cargo.lock index 9300a90690c8b109e60ae8c9d0b1b015bd2c619e..9500fd93e90dd7b9b479cec57fd5bb74fa9d0ff4 100644 --- a/dashboard/Cargo.lock +++ b/dashboard/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.6" @@ -324,19 +330,42 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros 0.6.1", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + [[package]] name = "cssparser" version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" dependencies = [ - "cssparser-macros", + "cssparser-macros 0.7.1", "dtoa-short", "itoa", "phf", "smallvec", ] +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "cssparser-macros" version = "0.7.1" @@ -503,6 +532,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -694,7 +729,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -702,6 +737,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -729,6 +769,7 @@ dependencies = [ "chrono", "futures", "globset", + "lol_html", "openssl", "rand 0.9.5", "regex", @@ -1080,6 +1121,25 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lol_html" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5adbb62638edf7e6bc88835cd3ea388bdd53382af42045da0e414ea78aa0c91a" +dependencies = [ + "bitflags", + "cfg-if", + "cssparser 0.36.0", + "encoding_rs", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "memchr", + "mime", + "precomputed-hash", + "selectors 0.37.0", + "thiserror 2.0.21", +] + [[package]] name = "lru-slab" version = "0.1.3" @@ -1649,12 +1709,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", ] @@ -1836,14 +1898,33 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2" dependencies = [ - "cssparser", + "cssparser 0.37.0", "ego-tree", "html5ever", "precomputed-hash", - "selectors", + "selectors 0.38.0", "tendril", ] +[[package]] +name = "selectors" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" +dependencies = [ + "bitflags", + "cssparser 0.36.0", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "selectors" version = "0.38.0" @@ -1851,7 +1932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" dependencies = [ "bitflags", - "cssparser", + "cssparser 0.37.0", "derive_more", "log", "new_debug_unreachable", @@ -2582,6 +2663,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.106" diff --git a/dashboard/Cargo.toml b/dashboard/Cargo.toml index 0d43b62eb3a91716c8e07d1ccc9fb9409677ae62..14be074d6b72e34aa060aba83fb4fb170f3e2713 100644 --- a/dashboard/Cargo.toml +++ b/dashboard/Cargo.toml @@ -13,8 +13,9 @@ futures = "0.3" globset = "0.4" openssl = "0.10" rand = "0.9" +lol_html = "3" regex = "1" -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "multipart"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "multipart", "stream"] } rmcp = { version = "3.5.0", default-features = false, features = ["server", "transport-streamable-http-server"] } rusqlite = { version = "0.37", features = ["bundled"] } serde = { version = "1", features = ["derive"] } diff --git a/dashboard/src/astheno.svg b/dashboard/src/astheno.svg new file mode 100644 index 0000000000000000000000000000000000000000..2c5fe85aad44c9bce6bd5e45a6ae9c3f9d3aa1c8 --- /dev/null +++ b/dashboard/src/astheno.svg @@ -0,0 +1,50 @@ + + + + + + diff --git a/dashboard/src/auth.rs b/dashboard/src/auth.rs index 87b0a25bbf73d835b578d973e142687db4cb3e8b..f4d98867023d8c3e8579812771f4a466988c15c3 100644 --- a/dashboard/src/auth.rs +++ b/dashboard/src/auth.rs @@ -560,6 +560,9 @@ impl Store { } pub async fn route(State(app): State>, request: Request) -> Result { + if request.uri().path() == "/auth/shale/page" { + return shale_page::proxy(app, request).await; + } if request.uri().path().starts_with("/auth/guest/") { return Ok(guest::route(State(app), request).await); } diff --git a/dashboard/src/guest.rs b/dashboard/src/guest.rs index 60be1d70349053da8f8cdd86b0ba99140a64fbdf..3e4c592074438ecea29c3a416bf6b0f916ac1325 100644 --- a/dashboard/src/guest.rs +++ b/dashboard/src/guest.rs @@ -299,7 +299,7 @@ async fn exchange( code: &str, callback: &str, flow: &Value, -) -> Result<(String, String)> { +) -> Result<(String, String, Option)> { let form = [ ("client_id", client), ("client_secret", secret), @@ -356,7 +356,7 @@ async fn exchange( "GitHub couldn't verify your account. Return to Shale and try again.", ) })?; - return Ok((id.to_string(), login.to_owned())); + return Ok((id.to_string(), login.to_owned(), None)); } let mut form = form.to_vec(); form.push(("state", "none")); @@ -430,10 +430,27 @@ async fn exchange( .chars() .take(128) .collect(); - Ok((string(&profile["sub"]).to_owned(), name)) + let picture = profile["picture"] + .as_str() + .filter(|value| value.len() <= 2048) + .and_then(|value| url::Url::parse(value).ok()) + .filter(|url| { + url.scheme() == "https" + && url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + }) + .map(String::from); + Ok((string(&profile["sub"]).to_owned(), name, picture)) } -fn account(auth: &auth::Store, provider: &str, subject: &str, name: &str) -> Result { +fn account( + auth: &auth::Store, + provider: &str, + subject: &str, + name: &str, + picture: Option<&str>, +) -> Result { let mut db = auth.db.lock().unwrap(); let tx = db.transaction()?; let existing: Option = tx @@ -451,6 +468,11 @@ fn account(auth: &auth::Store, provider: &str, subject: &str, name: &str) -> Res "This guest account is disabled. Contact Clover.", )); } + tx.execute( + "UPDATE users SET profile=json_patch(profile,?) WHERE id=?", + sql![json!({"firstName":name,"attributes":{"picture":picture.map(|picture| vec![picture])}}).to_string(), id], + )?; + tx.commit()?; return Ok(id); } let id = uuid::Uuid::new_v4().to_string(); @@ -459,7 +481,10 @@ fn account(auth: &auth::Store, provider: &str, subject: &str, name: &str) -> Res } else { mcp::hash(subject)[..24].to_owned() }; - let profile = json!({"kind":"guest","guestProvider":provider,"username":format!("guest-{provider}-{suffix}"),"enabled":true,"email":null,"emailVerified":false,"firstName":name,"lastName":null,"requiredActions":[],"attributes":{},"createdTimestamp":(now()*1000.0) as i64}); + let mut profile = json!({"kind":"guest","guestProvider":provider,"username":format!("guest-{provider}-{suffix}"),"enabled":true,"email":null,"emailVerified":false,"firstName":name,"lastName":null,"requiredActions":[],"attributes":{},"createdTimestamp":(now()*1000.0) as i64}); + if let Some(picture) = picture { + profile["attributes"]["picture"] = json!([picture]); + } tx.execute( "INSERT INTO users(id,profile) VALUES (?,?)", sql![id, profile.to_string()], @@ -598,9 +623,9 @@ async fn handle(app: &App, request: Request) -> Result { headers }) .build()?; - let (subject, name) = + let (subject, name, picture) = exchange(&http, provider, &client, &secret, code, &callback, &flow).await?; - let id = account(auth, provider, &subject, &name)?; + let id = account(auth, provider, &subject, &name, picture.as_deref())?; auth.create_session(&id, "dashboard", headers, None) } .await; @@ -781,16 +806,34 @@ mod tests { db.execute("INSERT INTO roles VALUES ('admin','infra-admin')", []) .unwrap(); } - let first = account(&auth, "github", "123", "clover").unwrap(); - let repeat = account(&auth, "github", "123", "renamed").unwrap(); - let other = account(&auth, "astheno", "123", "clover").unwrap(); + let first = account(&auth, "github", "123", "clover", None).unwrap(); + let repeat = account(&auth, "github", "123", "renamed", None).unwrap(); + let other = account( + &auth, + "astheno", + "123", + "clover", + Some("https://identity.astheno.software/avatar/123"), + ) + .unwrap(); assert_eq!(first, repeat); assert_ne!(first, "owner"); assert_ne!(first, other); + assert_eq!( + auth::user(&auth.db.lock().unwrap(), &other).unwrap()["attributes"]["picture"][0], + "https://identity.astheno.software/avatar/123" + ); + account(&auth, "astheno", "123", "clover", None).unwrap(); + assert!( + auth::user(&auth.db.lock().unwrap(), &other).unwrap()["attributes"] + .get("picture") + .is_none() + ); { let db = auth.db.lock().unwrap(); let profile = auth::user(&db, &first).unwrap(); assert!(is_guest(&profile)); + assert_eq!(profile["firstName"], "renamed"); assert!(profile["email"].is_null()); assert_eq!(profile["groups"], json!([])); assert!( @@ -811,7 +854,7 @@ mod tests { ) .unwrap(); } - assert!(account(&auth, "github", "123", "clover").is_err()); + assert!(account(&auth, "github", "123", "clover", None).is_err()); assert!( auth.create_session(&other, "file", &HeaderMap::new(), None) .is_err() diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index dcb5c295334b76e58bdd6e78fc5bfb87e8ef76b4..1082d8038f884e17f583f96c66a4f1cca91522cd 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -12,6 +12,7 @@ mod observability; mod oidc; mod relay; mod shale; +mod shale_page; mod storage; mod telemetry; mod users; @@ -527,6 +528,9 @@ async fn main() -> std::result::Result<(), Box> { } request.headers_mut().remove("Studio-Proxy-Token"); } + if request.uri().path() == "/auth/shale/page" { + return next.run(request).await; + } let path = request.uri().path().to_owned(); let asset = path.starts_with("/assets/") || path.starts_with("/fonts/") diff --git a/dashboard/src/shale.rs b/dashboard/src/shale.rs index 3226ae2bc9368653b5879fc7700d81c436a8ad53..39d5dfa82bddc47abd893ea4161cac0e9c9dff65 100644 --- a/dashboard/src/shale.rs +++ b/dashboard/src/shale.rs @@ -326,7 +326,7 @@ fn issue(html: &str, repository: &str, id: Option) -> Result { ) })?; let author = comment - .select(&Selector::parse(".n-card__header a[href^='/~']").unwrap()) + .select(&Selector::parse(".n-card__header a[href^='/~'], .n-card__header a[href^='https://github.com/'], .n-card__header a[href^='https://identity.astheno.software/user/']").unwrap()) .next() .map(text); let time = comment @@ -935,6 +935,14 @@ mod tests { assert!(issue(&page.replace("r1616-ga87d2f5.zig.0.16.0", "r1758-new"), "owned", Some(3)).is_err()); assert!(issue(&page.replace("Issue #3", "Issue #0"), "owned", None).is_err()); } + #[test] + fn issue_comment_authors_include_external_guest_profiles() { + let page = "

#3Title

Done
"; + let parsed = issue(page, "owned", Some(3)).unwrap(); + assert_eq!(parsed["comments"][0]["author"], "paperclover"); + assert_eq!(parsed["comments"][1]["author"], "Astheno user"); + } + #[test] fn account_identity_uses_html_text_and_rejects_login_or_changed_markup() { assert_eq!( diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs new file mode 100644 index 0000000000000000000000000000000000000000..17f1383fadb10474063c30a65af5b436c4f1632e --- /dev/null +++ b/dashboard/src/shale_page.rs @@ -0,0 +1,336 @@ +use crate::*; +use axum::body::Body; +use lol_html::{RewriteStrSettings, element, html_content::ContentType, text}; + +struct Profile { + name: String, + url: String, + icon: &'static str, + picture: Option, +} + +fn profiles(auth: &auth::Store) -> Result> { + let db = auth.db.lock().unwrap(); + let mut query = db.prepare("SELECT json_extract(profile,'$.username'),coalesce(json_extract(profile,'$.firstName'),''),provider,subject,json_extract(profile,'$.attributes.picture[0]') FROM users JOIN external_identities ON user_id=users.id WHERE json_extract(profile,'$.kind')='guest'")?; + let rows = query.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + )) + })?; + let mut profiles = HashMap::new(); + for row in rows { + let (username, name, provider, subject, picture) = row?; + let (mut url, id, icon) = match provider.as_str() { + "github" => ( + url::Url::parse("https://github.com/")?, + name.as_str(), + include_str!("../web/sso/github.svg"), + ), + "astheno" => ( + url::Url::parse("https://identity.astheno.software/user/")?, + subject.as_str(), + include_str!("astheno.svg"), + ), + _ => continue, + }; + if name.is_empty() || id.is_empty() || matches!(id, "." | "..") { + continue; + } + url.path_segments_mut().unwrap().pop_if_empty().push(id); + profiles.insert( + username, + Profile { + name, + url: url.into(), + icon, + picture: if provider == "github" { + Some(format!( + "https://avatars.githubusercontent.com/u/{subject}?s=64" + )) + } else { + picture + }, + }, + ); + } + Ok(profiles) +} + +fn account_path(path: &str) -> Option<&str> { + let name = path.strip_prefix("/~")?; + let name = name.strip_suffix('/').unwrap_or(name); + (name.starts_with("guest-") && !name.contains('/')).then_some(name) +} + +fn rewrite(html: &str, origin: &url::Url, profiles: &HashMap) -> Result { + use std::{cell::Cell, rc::Rc}; + let active = Rc::new(Cell::new(None::)); + let images = active.clone(); + let labels = active.clone(); + Ok(lol_html::rewrite_str(html, RewriteStrSettings::new() + .append_element_content_handler(element!("a[href]", |element| { + let profile = element.get_attribute("href") + .and_then(|href| origin.join(&href).ok()) + .filter(|target| target.origin() == origin.origin()) + .and_then(|target| account_path(target.path()).and_then(|name| profiles.get(name))); + active.set(profile.map(|profile| profile.picture.is_some())); + let Some(profile) = profile else { return Ok(()); }; + let closing = active.clone(); + element.on_end_tag(lol_html::end_tag!(move |_| { closing.set(None); Ok(()) }))?; + element.set_attribute("href", &profile.url)?; + element.set_attribute("target", "_blank")?; + element.set_attribute("rel", "noreferrer")?; + if let Some(picture) = &profile.picture { + let picture = picture.replace('&', "&").replace('"', """).replace('<', "<"); + element.prepend(&format!("\"\""), ContentType::Html); + } + let icon = profile.icon.trim().replace(" = headers + .get_all("connection") + .iter() + .filter_map(|v| v.to_str().ok()) + .flat_map(|v| v.split(',')) + .map(|v| v.trim().to_owned()) + .collect(); + for name in named { + headers.remove(name); + } + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ] { + headers.remove(name); + } +} + +pub async fn proxy(app: Arc, request: Request) -> Result { + let (mut parts, body) = request.into_parts(); + let host = parts + .headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + let domain = env("STUDIO_DOMAIN", "studio.test"); + let service = host.strip_suffix(&format!(".{domain}")).unwrap_or_default(); + if service != "shale" + && !(service + .strip_prefix("shale-preview-") + .is_some_and(|id| id.len() == 8 && id.bytes().all(|b| b.is_ascii_hexdigit()))) + { + return Err(Error::new(403, "Open Shale to continue.")); + } + let upstream: std::net::SocketAddr = parts + .headers + .get("studio-shale-upstream") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| Error::new(403, "Open Shale to continue."))? + .parse()?; + let uri: axum::http::Uri = parts + .headers + .get("studio-shale-uri") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| Error::new(403, "Open Shale to continue."))? + .parse()?; + if uri.scheme().is_some() || uri.authority().is_some() || !uri.path().starts_with('/') { + return Err(Error::new(403, "Open Shale to continue.")); + } + let origin = url::Url::parse(&format!("https://{host}{uri}"))?; + let profiles = profiles(&app.auth)?; + if matches!(parts.method, Method::GET | Method::HEAD) + && let Some(profile) = account_path(uri.path()).and_then(|name| profiles.get(name)) + { + return Ok(( + StatusCode::FOUND, + [ + ("location", profile.url.as_str()), + ("cache-control", "no-store"), + ("referrer-policy", "no-referrer"), + ], + ) + .into_response()); + } + strip_hop_headers(&mut parts.headers); + for name in [ + "studio-shale-upstream", + "studio-shale-uri", + "studio-proxy-token", + "user-name", + "user-groups", + "user-id", + "if-none-match", + "if-modified-since", + "accept-encoding", + ] { + parts.headers.remove(name); + } + parts + .headers + .insert("accept-encoding", "identity".parse().unwrap()); + static HTTP: std::sync::LazyLock = std::sync::LazyLock::new(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .build() + .unwrap() + }); + let mut upstream_response = HTTP + .request(parts.method.clone(), format!("http://{upstream}{uri}")) + .headers(parts.headers) + .body(reqwest::Body::wrap_stream(body.into_data_stream())) + .send() + .await?; + let status = upstream_response.status(); + let mut headers = upstream_response.headers().clone(); + strip_hop_headers(&mut headers); + let html = parts.method != Method::HEAD + && headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| { + v.split(';') + .next() + .unwrap_or_default() + .trim() + .eq_ignore_ascii_case("text/html") + }) + && headers + .get("content-encoding") + .is_none_or(|v| v == "identity"); + let mut prefix = Vec::new(); + if html { + while let Some(chunk) = upstream_response.chunk().await? { + prefix.extend_from_slice(&chunk); + if prefix.len() > 8 * 1024 * 1024 { + break; + } + } + if prefix.len() <= 8 * 1024 * 1024 + && let Ok(text) = std::str::from_utf8(&prefix) + { + let rewritten = rewrite(text, &origin, &profiles)?; + for name in [ + "content-length", + "etag", + "last-modified", + "content-md5", + "digest", + "content-digest", + "accept-ranges", + ] { + headers.remove(name); + } + headers.insert("cache-control", "no-store".parse().unwrap()); + return Ok((status, headers, rewritten).into_response()); + } + } + let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) + .chain(upstream_response.bytes_stream()); + Ok((status, headers, Body::from_stream(stream)).into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use scraper::{Html, Selector}; + + #[test] + fn rewrites_only_known_local_account_links_and_escapes_provider_names() { + let origin = url::Url::parse("https://shale.paperclover.net/").unwrap(); + let profiles = HashMap::from([ + ( + "guest-github-123".into(), + Profile { + name: "<&\"clover".into(), + url: "https://github.com/clover".into(), + icon: include_str!("../web/sso/github.svg"), + picture: Some("https://avatars.githubusercontent.com/u/123?s=64".into()), + }, + ), + ( + "guest-astheno-abc".into(), + Profile { + name: "Astheno user".into(), + url: "https://identity.astheno.software/user/123".into(), + icon: include_str!("astheno.svg"), + picture: None, + }, + ), + ]); + let html = r#"~guest-github-123avatarguestunknownexternalrepo
~guest-github-123
"#; + let rewritten = rewrite(html, &origin, &profiles).unwrap(); + let page = Html::parse_document(&rewritten); + let links: Vec<_> = page.select(&Selector::parse("a").unwrap()).collect(); + for link in &links[..2] { + assert_eq!(link.attr("target"), Some("_blank")); + assert_eq!(link.attr("rel"), Some("noreferrer")); + assert_eq!( + link.select(&Selector::parse("svg[aria-hidden=true]").unwrap()) + .count(), + 1 + ); + } + assert_eq!( + links[1] + .select(&Selector::parse("img").unwrap()) + .next() + .unwrap() + .attr("src"), + Some("astheno-avatar") + ); + assert_eq!(links[0].select(&Selector::parse("img").unwrap()).count(), 1); + assert_eq!( + links[0] + .select(&Selector::parse("img").unwrap()) + .next() + .unwrap() + .attr("src"), + Some("https://avatars.githubusercontent.com/u/123?s=64") + ); + assert_eq!(links[0].text().collect::().trim(), "<&\"clover"); + assert_eq!(links[0].attr("class"), Some("usa-nav-link")); + assert_eq!( + links[1].attr("href"), + Some("https://identity.astheno.software/user/123") + ); + for link in &links[2..] { + assert_eq!(link.attr("target"), None); + } + assert!(rewritten.contains("
~guest-github-123
")); + assert_eq!(account_path("/~guest-github-123//"), None); + } +} diff --git a/nixos/dashboard.nix b/nixos/dashboard.nix index 8c221112cf3da7aae250cce7fd66b7145070adc1..04b65180719b8b781064a07afa2469d5dece3fc8 100644 --- a/nixos/dashboard.nix +++ b/nixos/dashboard.nix @@ -38,7 +38,7 @@ let src = lib.fileset.toSource { root = ../dashboard; fileset = lib.fileset.unions [ - ../dashboard/src ../dashboard/tests ../dashboard/Cargo.toml ../dashboard/Cargo.lock + ../dashboard/src ../dashboard/web/sso/github.svg ../dashboard/tests ../dashboard/Cargo.toml ../dashboard/Cargo.lock ../dashboard/agent/install.sh ../dashboard/agent/install.ps1 ]; }; diff --git a/tools/dashboard-shale-page-test.py b/tools/dashboard-shale-page-test.py new file mode 100644 index 0000000000000000000000000000000000000000..4a71e59c209a27d4cc0b0c26c5698fdc21dabc19 --- /dev/null +++ b/tools/dashboard-shale-page-test.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +import argparse +from contextlib import ExitStack +from html.parser import HTMLParser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import sqlite3 +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.request + +import router + + +class Links(HTMLParser): + def __init__(self, html): + super().__init__() + self.links = [] + self.feed(html) + + def handle_starttag(self, tag, attrs): + if tag == 'a': + self.links.append(dict(attrs)) + + +def port(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--caddy', required=True, type=Path) + parser.add_argument('--binary', type=Path, default=Path(__file__).resolve().parent.parent / 'dashboard/target/debug/home-dashboard') + parser.add_argument('--page', type=Path) + parser.add_argument('--browser-ready-file', type=Path) + args = parser.parse_args() + repo = Path(__file__).resolve().parent.parent + observations = [] + issue = args.page.read_bytes() if args.page else b'~guest-github-24465214' + sample = b'guestunknownrepo' + blob = bytes(range(256)) * 65536 + + class Fixture(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def do_GET(self): + observations.append((self.command, self.path, dict(self.headers))) + if self.path == '/binary': + return self.respond(blob, 'application/octet-stream') + if self.path == '/large': + return self.respond(b' ' * (9 * 1024 * 1024), 'text/html') + if self.path == '/redirect': + return self.respond(b'', 'text/plain', 302, [('Location', '/snowbound/issues/26'), ('Set-Cookie', 'SessionID=new; Path=/; HttpOnly'), ('Set-Cookie', 'other=kept; Path=/')]) + if self.path.startswith('/-/') and args.page: + with urllib.request.urlopen('https://shale.paperclover.net' + self.path, timeout=15) as response: + return self.respond(response.read(), response.headers['Content-Type']) + return self.respond(issue + sample, 'text/html; charset=utf-8', headers=[('ETag', '"stale"'), ('Last-Modified', 'Mon, 05 Oct 2026 00:00:00 GMT')]) + + def do_HEAD(self): + self.respond(b'', 'text/html') + + def do_POST(self): + body = self.rfile.read(int(self.headers.get('Content-Length', 0))) + observations.append((self.command, self.path, dict(self.headers), body)) + self.respond(body, 'application/octet-stream', 201) + + def respond(self, body, content_type, status=200, headers=()): + self.send_response(status) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(body))) + for key, value in headers: + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + opener = urllib.request.build_opener(NoRedirect) + with tempfile.TemporaryDirectory(prefix='shale-page-') as temporary, ExitStack() as stack: + root = Path(temporary).resolve() + fixture = ThreadingHTTPServer(('127.0.0.1', 0), Fixture) + stack.callback(fixture.server_close) + stack.callback(fixture.shutdown) + threading.Thread(target=fixture.serve_forever, daemon=True).start() + dashboard_port, gateway_port = port(), port() + proof = 'a' * 64 + token = root / 'proxy.token' + token.write_text(proof) + data = root / 'data' + environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')} + for key in ['STUDIO_INTERNAL_URL', 'STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']: + environment.pop(key, None) + log = stack.enter_context((root / 'dashboard.log').open('w')) + dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log) + stack.callback(lambda: dashboard.wait(timeout=10)) + stack.callback(dashboard.terminate) + deadline = time.monotonic() + 15 + while True: + assert dashboard.poll() is None, (root / 'dashboard.log').read_text() + try: + with socket.create_connection(('127.0.0.1', dashboard_port), timeout=.1): + break + except OSError: + assert time.monotonic() < deadline + time.sleep(.05) + with sqlite3.connect(data / 'accounts.sqlite') as db: + for provider, subject, suffix, name in [('github', '24465214', '24465214', 'paperclover'), ('astheno', '00653DG7HZ7MCGTW2BPG7RMGQB', 'abc', 'Astheno user'), ('github', '777', '777', None)]: + username = f'guest-{provider}-{suffix}' + profile = {'kind': 'guest', 'enabled': True, 'username': username, 'guestProvider': provider, 'firstName': name} + db.execute('INSERT INTO users(id,profile) VALUES (?,?)', [username, json.dumps(profile)]) + db.execute('INSERT INTO external_identities VALUES (?,?,?)', [provider, subject, username]) + os.environ.update(STUDIO_DOMAIN='studio.test', STUDIO_DASHBOARD_PORT=str(dashboard_port), STUDIO_PROXY_TOKEN_FILE=str(token)) + router.ROUTE_DIR = str(root / 'routes') + Path(router.ROUTE_DIR).mkdir() + (Path(router.ROUTE_DIR) / 'shale.json').write_text(json.dumps({'headHtml': {'shale.studio.test': {'/snowbound/*': ''}}})) + + def nomad(path, token): + if path == '/v1/services': + return [{'Namespace': 'default', 'Services': [{'ServiceName': 'shale'}, {'ServiceName': 'shale-preview-12345678'}]}] + if path.startswith('/v1/service/'): + service = path.rsplit('/', 1)[1] + host = 'shale.studio.test' if service == 'shale' else service + '.studio.test' + return [{'Address': '127.0.0.1', 'Port': fixture.server_port, 'ServiceName': service, 'AllocID': service, 'JobID': service, 'Tags': ['caddy-host=' + host]}] + return {'ready': {'Status': 'success'}} + + router.nomad = nomad + rendered = router.render('fixture') + full_config = root / 'routes.caddy' + full_config.write_text('{\n admin off\n auto_https off\n}\n' + rendered) + caddy_env = {**os.environ, 'XDG_DATA_HOME': str(root / 'caddy-data'), 'XDG_CONFIG_HOME': str(root / 'caddy-config')} + subprocess.run([str(args.caddy), 'validate', '--config', str(full_config), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env) + def local_site(host, listener): + start = rendered.index(host + ' {') + end = rendered.index('\n}', start) + 2 + return rendered[start:end].replace(host + ' {', f'http://127.0.0.1:{listener} {{', 1).replace(' tls internal\n', '').replace(' route {\n', f' route {{\n request_header Host {host}\n', 1) + + preview_port = port() + config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.studio.test', gateway_port) + '\n' + local_site('shale-preview-12345678.studio.test', preview_port) + config_path = root / 'Caddyfile' + config_path.write_text(config) + subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env) + caddy_log = stack.enter_context((root / 'caddy.log').open('w')) + caddy = subprocess.Popen([str(args.caddy), 'run', '--config', str(config_path), '--adapter', 'caddyfile'], stdout=caddy_log, stderr=caddy_log, env=caddy_env) + stack.callback(lambda: caddy.wait(timeout=10)) + stack.callback(caddy.terminate) + origin = f'http://127.0.0.1:{gateway_port}' + + def request(path, *, headers=None, body=None, direct=False): + supplied = {'Cookie': 'SessionID=preserved', 'Origin': 'https://shale.studio.test', **(headers or {})} + url = (f'http://127.0.0.1:{dashboard_port}' if direct else origin) + path + try: + response = opener.open(urllib.request.Request(url, headers=supplied, data=body), timeout=15) + except urllib.error.HTTPError as error: + response = error + with response: + return response.status, response.headers, response.read() + + deadline = time.monotonic() + 15 + while True: + try: + status, headers, body = request('/snowbound/issues/26?query=kept') + break + except urllib.error.URLError: + assert time.monotonic() < deadline and caddy.poll() is None + time.sleep(.05) + assert status == 200, (status, body, (root / 'dashboard.log').read_text(), (root / 'caddy.log').read_text()) + links = Links(body.decode()).links + for url in ['https://github.com/paperclover', 'https://identity.astheno.software/user/00653DG7HZ7MCGTW2BPG7RMGQB']: + link = next(link for link in links if link['href'] == url) + assert link['target'] == '_blank' and link['rel'] == 'noreferrer', link + assert any(link['href'] == '/~guest-github-unknown' for link in links) + assert any(link['href'] == '/~guest-github-24465214/repo' for link in links) + assert b'https://avatars.githubusercontent.com/u/24465214?s=64' in body + assert b'rewrite-test' in body + assert headers['Cache-Control'] == 'no-store' and headers.get('ETag') is None and headers.get('Last-Modified') is None + assert observations[-1][1] == '/snowbound/issues/26?query=kept', observations[-1] + assert observations[-1][2]['cookie'] == 'SessionID=preserved', observations[-1] + assert not any(key.lower().startswith('studio-') for key in observations[-1][2]), observations[-1] + for path, target in [('/~guest-github-24465214', 'https://github.com/paperclover'), ('/~guest-astheno-abc/', 'https://identity.astheno.software/user/00653DG7HZ7MCGTW2BPG7RMGQB')]: + status, headers, body = request(path) + assert status == 302 and headers['Location'] == target and headers['Referrer-Policy'] == 'no-referrer', (path, status, dict(headers), observations[-1][:2]) + status, headers, body = request('/redirect') + assert status == 302 and headers['Location'] == '/snowbound/issues/26' and len(headers.get_all('Set-Cookie')) == 2 + with opener.open(f'http://127.0.0.1:{preview_port}/snowbound/issues/26') as response: + assert response.status == 200 + assert any(link['href'] == 'https://github.com/paperclover' for link in Links(response.read().decode()).links) + assert b'/~guest-github-24465214' in request('/repo/info/refs')[2] + assert request('/binary')[2] == blob + assert len(request('/large')[2]) == 9 * 1024 * 1024 + status, headers, body = request('/submit?keep=yes', body=b'field=unchanged') + assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes' + assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403 + assert request('/auth/shale/page', direct=True)[0] == 403 + assert request('/auth/shale/page', direct=True, headers={'Studio-Proxy-Token': proof, 'Studio-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Studio-Shale-Uri': '/', 'Host': 'snowglobe.studio.test'})[0] == 403 + print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True) + if args.browser_ready_file: + args.browser_ready_file.write_text(json.dumps({'origin': origin})) + stop = args.browser_ready_file.with_suffix('.stop') + deadline = time.monotonic() + 600 + while not stop.exists() and time.monotonic() < deadline: + time.sleep(.2) + args.browser_ready_file.unlink(missing_ok=True) + stop.unlink(missing_ok=True) + + +if __name__ == '__main__': + main() diff --git a/tools/router.py b/tools/router.py index 5d90516318213ac028cd30cf958f5dab75a7984a..46dc62c7713a2a7460810853a952cfc0713e9922 100644 --- a/tools/router.py +++ b/tools/router.py @@ -297,14 +297,27 @@ def render(token): f" request_header -{scrub[0]}", " }", " }", *proxy(upstreams, " "), " }", "}"] else: + page_upstream = upstreams + shale_page = service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service) + page_headers = [] + if shale_page: + page_upstream = f"127.0.0.1:{int(os.environ['STUDIO_DASHBOARD_PORT'])}" + page_headers = [f" request_header Studio-Proxy-Token {dashboard_proof}", + f" request_header Studio-Shale-Upstream {json.dumps(sorted(route['upstreams'])[0])}", + " request_header Studio-Shale-Uri {uri}", + " rewrite * /auth/shale/page"] for index, (request, markup) in enumerate(head_html.items()): name = f"@studio_head_{index}" lines += [f" {name} path {request}", f" handle {name} {{", " route {", f" replace {json.dumps(markup + '')} {{", " match {", " header Content-Type text/html*", " }", " }", - *proxy(upstreams, " ", uncompressed=True), " }", " }"] + *[" " + line for line in page_headers], + *proxy(page_upstream, " ", uncompressed=True), " }", " }"] + page_handler = [*page_headers, *proxy(page_upstream, " ")] + if shale_page: + page_handler = [" route {", *[" " + line for line in page_handler], " }"] lines += [" handle {", *(f" request_header -{name}" for name in scrub), - *proxy(upstreams, " "), " }", "}"] + *page_handler, " }", "}"] if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): lines.insert(len(lines) - 1, " }") else: