diff --git a/config/mac-studio-sandbox.json b/config/mac-studio-sandbox.json index 5c369e99f3ceeed7cc38bac3cddcebe155dd0a8d..8da3cac8e807f04115cdf2af537d91e2146ddca9 100644 --- a/config/mac-studio-sandbox.json +++ b/config/mac-studio-sandbox.json @@ -1,5 +1,6 @@ { "dataset": "globe/clover/mac-studio-sandbox", "mountPoint": "/srv/mac-studio-vm/sandbox", - "clientAddress": "10.0.0.222" + "hostAddress": "10.0.0.189", + "guestAddress": "10.0.0.124" } diff --git a/config/shale.json b/config/shale.json index a1b954e2576117f2f7a83803615bcb6ebf6f1bc4..8669e03b5264e833420068132ba418e91f9287cc 100644 --- a/config/shale.json +++ b/config/shale.json @@ -1 +1 @@ -{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}} +{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}} diff --git a/dashboard/src/auth.rs b/dashboard/src/auth.rs index e7f69f029fe32b654812ada76260418ff82c7a86..2bbf7265f0b79c9d0739bb20d252fede14e14ed9 100644 --- a/dashboard/src/auth.rs +++ b/dashboard/src/auth.rs @@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session"; const FLOW_COOKIE: &str = "__Host-snow-flow"; const CSRF_COOKIE: &str = "__Host-snow-csrf"; const SESSION_TTL: i64 = 30 * 86400; -const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"]; +const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"]; pub struct Store { pub db: Mutex, @@ -201,14 +201,13 @@ impl Store { oidc::initialise(&db)?; guest::initialise(&db)?; for name in GROUPS { - let added = db.execute( + db.execute( "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)", sql![uuid::Uuid::new_v4().to_string(), name], )?; - if *name == "knife-native" && added != 0 { - db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?; - } } + db.execute_batch("DELETE FROM memberships WHERE role_id IN (SELECT id FROM roles WHERE name='knife-native'); + DELETE FROM roles WHERE name='knife-native';")?; Ok(Self { db: Mutex::new(db), origin, diff --git a/dashboard/src/guest.rs b/dashboard/src/guest.rs index f94f043c4b18bf144f4582bd439a86a013bc8049..ad7ac392cbe407882b56f265ae94f65ed58c609f 100644 --- a/dashboard/src/guest.rs +++ b/dashboard/src/guest.rs @@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> { db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL); CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id)); DROP TRIGGER IF EXISTS guest_no_groups; - CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; - CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; + CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END; + DROP TRIGGER IF EXISTS guest_no_group_updates; CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END; CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?; Ok(()) @@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) if known.len() > 1 { return Err(Error::new( 409, - "Several accounts have signed in with that name. Open the right one under guests and join it to knife-native.", + "Several accounts have signed in with that name. Find them under guests.", )); } let username = match known.first() { @@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) id } }; - tx.execute( - "INSERT OR IGNORE INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", - [&id], - )?; tx.commit()?; Ok(json!({"id":id,"created":existing.is_none()})) } @@ -1185,7 +1181,7 @@ mod tests { } #[test] - fn identities_stay_separate_and_guests_can_only_join_native_access() { + fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() { let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4())); let auth = auth::Store::new( &path, @@ -1234,9 +1230,6 @@ mod tests { db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first]) .is_err() ); - db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap(); - assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native"); - assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err()); assert!(auth::set_password(&db, &first, "a-password-hash").is_err()); assert!( db.execute( @@ -1261,10 +1254,6 @@ mod tests { std::fs::remove_dir_all(path).unwrap(); } - fn members(auth: &auth::Store, id: &str) -> Value { - auth::user(&auth.db.lock().unwrap(), id).unwrap()["groups"].clone() - } - #[test] fn added_guests_are_the_accounts_their_first_sign_in_finds() { let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4())); @@ -1279,10 +1268,8 @@ mod tests { let added = grant(&auth, provider, Some(subject), "octo").unwrap(); let id = string(&added["id"]); assert_eq!(added["created"], true); - assert_eq!(members(&auth, id)[0]["name"], "knife-native"); assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id); assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false); - assert_eq!(members(&auth, id).as_array().unwrap().len(), 1); } let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned(); @@ -1294,19 +1281,15 @@ mod tests { assert_eq!(identities(&reserved), 0); let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap(); assert_ne!(stranger, reserved); - assert_eq!(members(&auth, &stranger), json!([])); claim(&auth, "sub-1", "alice").unwrap(); assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved); - assert_eq!(members(&auth, &reserved)[0]["name"], "knife-native"); claim(&auth, "sub-3", "alice").unwrap(); let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap(); assert_ne!(impostor, reserved); - assert_eq!(members(&auth, &impostor), json!([])); let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap(); let added = grant(&auth, "astheno", None, "carol").unwrap(); assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false))); - assert_eq!(members(&auth, &earlier)[0]["name"], "knife-native"); account(&auth, "astheno", "sub-5", "dave", None).unwrap(); account(&auth, "astheno", "sub-6", "dave", None).unwrap(); assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409); diff --git a/dashboard/src/knife-restricted.html b/dashboard/src/knife-restricted.html deleted file mode 100644 index 3538a53cb93f2c30012a02f65a98f8ec521a0e51..0000000000000000000000000000000000000000 --- a/dashboard/src/knife-restricted.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - -The Knife's Native UI is restricted - -
-

The Knife's Native UI is restricted

-

This part of the code is sensitive. Please contact clover to request private access.

-Login -
- diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index e34b96ecd2d18058431511bc9c826019bdef9bf5..787f5411398505c46cfcf546596e604d7ae2d8d6 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) { } } -fn knife_uri(path: &str) -> bool { - let mut decoded = path.to_owned(); - for _ in 0..5 { - if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) { - return true; - } - let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned(); - if next == decoded { break; } - decoded = next; - } - false -} - -fn session_account(html: &str) -> Option { - let document = scraper::Html::parse_document(html); - let selector = scraper::Selector::parse("body > header.usa-header--basic nav a.usa-nav-link[href^='/~']").unwrap(); - let mut accounts = document.select(&selector); - let name = accounts.next()?.attr("href")?.strip_prefix("/~")?; - let name = percent_encoding::percent_decode_str(name).decode_utf8().ok()?; - (accounts.next().is_none() && !name.is_empty() && !name.contains(['/', '\\', '?', '#'])) - .then(|| name.into_owned()) -} - -fn restricted(authenticate: bool) -> Response { - (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "private, no-store"), - ("vary", "Cookie, Authorization"), - ("referrer-policy", "same-origin"), - ("x-content-type-options", "nosniff"), - ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"), - ("www-authenticate", "Basic realm=\"The Knife\""), - ], include_str!("knife-restricted.html")).into_response() -} - -fn redact_knife(html: &str, policy: &Value) -> Result { - let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect(); - let document = scraper::Html::parse_document(html); - let diff = policy["safeDiff"].as_str(); - if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 { - return Err(Error::new(403, "The Knife's Native UI is restricted")); - } - let links = scraper::Selector::parse("a[href]").unwrap(); - let rows: Vec = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| { - row.select(&links).any(|link| link.attr("href").is_some_and(|href| { - href.split(['/', '?', '#', '.']).any(|part| commits.contains(part)) - })) - }).collect(); - let mut row_index = 0; - Ok(lol_html::rewrite_str(html, RewriteStrSettings::new() - .append_element_content_handler(element!("#m-code", |element| { - if let Some(diff) = diff { - let diff = diff.replace('&', "&").replace('<', "<").replace('>', ">"); - element.set_inner_content(&format!("
{diff}
"), ContentType::Html); - } - Ok(()) - })) - .append_element_content_handler(element!("tr", |element| { - if *rows.get(row_index).unwrap_or(&true) { element.remove(); } - row_index += 1; - Ok(()) - })) - )?) -} - -/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch. -async fn clone(request: Value, body: Body) -> Result { - use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; - let (_, stream) = host::open(request).await?; - let (reader, mut writer) = stream.into_split(); - tokio::spawn(async move { - let mut body = body.into_data_stream(); - while let Some(Ok(chunk)) = body.next().await { - if writer.write_all(&chunk).await.is_err() { return; } - } - let _ = writer.shutdown().await; - }); - let mut reader = tokio::io::BufReader::new(reader); - let mut response = Response::builder(); - loop { - let mut line = String::new(); - if reader.read_line(&mut line).await? == 0 { - return Err(Error::new(502, "The Knife clone ended early.")); - } - let line = line.trim_end(); - if line.is_empty() { break; } - let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?; - let value = value.trim(); - if name.eq_ignore_ascii_case("status") { - response = response.status(value.split_whitespace().next().unwrap_or_default().parse::()?); - } else { - response = response.header(name, value); - } - } - let stream = futures::stream::unfold(reader, |mut reader| async move { - let mut buffer = vec![0; 64 * 1024]; - match reader.read(&mut buffer).await { - Ok(0) => None, - Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) } - Err(error) => Some((Err(error), reader)), - } - }); - Ok(response.body(Body::from_stream(stream))?) -} - pub async fn proxy(app: Arc, request: Request) -> Result { let (mut parts, body) = request.into_parts(); let host = parts @@ -484,52 +379,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Err(Error::new(403, "Open Shale to continue.")); } let origin = url::Url::parse(&format!("https://{host}{uri}"))?; - let knife = knife_uri(uri.path()); - let mut target = match &app.internal { - Some((base, _)) => format!("{base}services/{service}{uri}"), - None => format!("http://{upstream}{uri}"), - }; - let mut knife_policy = Value::Null; - if knife { - let mut account = String::new(); - if let Some(cookie) = parts.headers.get("cookie") { - // Opening settings rotates the session's CSRF token, invalidating pending forms. - let index = match &app.internal { - Some((base, _)) => format!("{base}services/{service}/"), - None => format!("http://{upstream}/"), - }; - let mut identity = app.shale_http.get(index).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3)); - if app.internal.is_none() { identity = identity.header("host", host); } - for name in ["x-forwarded-for", "x-real-ip", "user-agent"] { - if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); } - } - if let Ok(response) = identity.send().await - && response.status() == StatusCode::OK - && let Ok(body) = response.text().await - { - account = session_account(&body).unwrap_or_default(); - } - } - knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await { - Ok(policy) => policy, - Err(_) => return Ok(restricted(false)), - }; - if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } - if knife_policy["clone"] == true { - let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default(); - let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")}); - return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true))); - } - if let Some(uri) = knife_policy["uri"].as_str() { - target = match &app.internal { - Some((base, _)) => format!("{base}services/{service}{uri}"), - None => format!("http://{upstream}{uri}"), - }; - } - if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() { - return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response()); - } - } let profiles = profiles(&app.auth)?; if matches!(parts.method, Method::GET | Method::HEAD) && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref()) @@ -544,6 +393,10 @@ pub async fn proxy(app: Arc, request: Request) -> Result { ) .into_response()); } + let target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; if app.internal.is_some() { parts.headers.remove("host"); } @@ -578,10 +431,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { let mut headers = upstream_response.headers().clone(); strip_hop_headers(&mut headers); headers.remove("status"); - if knife { - headers.insert("cache-control", "private, no-store".parse().unwrap()); - headers.insert("vary", "Cookie, Authorization".parse().unwrap()); - } let html = parts.method != Method::HEAD && headers .get("content-type") @@ -637,14 +486,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { profiles.insert(name.clone(), profile); } } - let redacted; - let text = if knife && knife_policy["allowed"] != true { - redacted = match redact_knife(text, &knife_policy) { - Ok(html) => html, - Err(_) => return Ok(restricted(false)), - }; - redacted.as_str() - } else { text }; let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?; for name in [ "content-length", @@ -661,9 +502,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Ok((status, headers, rewritten).into_response()); } } - if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) { - return Ok(restricted(false)); - } let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) .chain(upstream_response.bytes_stream()); Ok((status, headers, Body::from_stream(stream)).into_response()) @@ -675,33 +513,6 @@ mod tests { use rusqlite::params as sql; use scraper::{Html, Selector}; - #[test] - fn session_account_uses_only_unambiguous_site_navigation() { - let header = |name: &str| format!("
"); - assert_eq!(session_account(&header("clover")), Some("clover".into())); - assert_eq!(session_account(&header("guest%2Dname")), Some("guest-name".into())); - assert_eq!(session_account(&format!("{}
", header("guest"))), Some("guest".into())); - assert_eq!(session_account(&format!("{}{}", header("guest"), header("clover"))), None); - for name in ["", "%FF", "clover%2Frepo", "clover%5Crepo", "clover?fake", "clover#fake"] { - assert_eq!(session_account(&header(name)), None, "{name}"); - } - assert_eq!(session_account("fake"), None); - } - - #[test] - fn native_content_is_removed_from_both_diff_styles_and_history_rows() { - let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"}); - let html = "
PRIVATE MESSAGE
PUBLIC ROW
PRIVATE PATHS
knife/private.rs
PRIVATE SOURCE
src/public.rs
PUBLIC SOURCE
nested/Cargo.lock
PRIVATE LOCK
"; - let output = redact_knife(html, &policy).unwrap(); - assert!(!output.contains("PRIVATE"), "{output}"); - assert!(output.contains("PUBLIC ROW"), "{output}"); - assert!(output.contains("PUBLIC SOURCE"), "{output}"); - assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1); - assert!(redact_knife("
unknown diff markup
", &policy).is_err()); - for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); } - assert!(!knife_uri("/snowbound/commit/native")); - } - #[test] fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() { let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4())); diff --git a/dashboard/src/users.rs b/dashboard/src/users.rs index f2852a0403300ac3d830b168a29f67c18446d0f1..d7030f9c39218e89eae069fd658b21e8d9900641 100644 --- a/dashboard/src/users.rs +++ b/dashboard/src/users.rs @@ -158,13 +158,10 @@ pub async fn route( for id in &users { uuid(id)?; let user = auth::user(&transaction, id)?; - let guest_groups = if guest::is_guest(&user) { - add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true)) - } else { false }; - if guest_groups || user["kind"] == "ai" { + if guest::is_guest(&user) || user["kind"] == "ai" { return Err(Error::new( 400, - "Guests can only join knife-native.", + "Guests can use Shale only. Select regular accounts to edit groups.", )); } for group in &add { @@ -304,7 +301,7 @@ pub async fn route( let transaction = db.transaction()?; let mut user = auth::user(&transaction, id)?; if guest::is_guest(&user) - && matches!(parts, [_, "password"]) + && matches!(parts, [_, "password"] | [_, "groups", _]) && method != Method::GET { return Err(Error::new( @@ -354,9 +351,6 @@ pub async fn route( .optional()?; let name = name .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?; - if guest::is_guest(&user) && name != "knife-native" { - return Err(Error::new(400, "Guests can only join knife-native.")); - } if own && method == Method::DELETE && name == "infra-admin" { return Err(Error::new( 400, diff --git a/dashboard/web/pages/Users.tsx b/dashboard/web/pages/Users.tsx index 38fdb13d4b84676197b5925c7dce921a5a8a733c..ce953cba70e8b4e040c091974145bfc932db9670 100644 --- a/dashboard/web/pages/Users.tsx +++ b/dashboard/web/pages/Users.tsx @@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) { /** "30 members · opens Jellyfin, Navidrome" */ function groupTip(group: string, d: Data) { - if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`; const more = adds(group, d.services); const members = d.users.filter((user) => names(user).includes(group)).length; const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : []; @@ -230,7 +229,7 @@ function List(props: { data: Resource; ready: Accessor; guestMenu.hidePopover(); showTextDialog({ title: `Add ${guest.name} guest`, - description: "They can sign in to Shale and open The Knife’s Native UI.", + description: "They can sign in to Shale.", label: guest.field, placeholder: guest.placeholder, body: provider === "astheno" ? () =>
Astheno can’t look names up. The first person to sign in with this username gets the access.
: undefined, @@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {
{(group) => ( - diff --git a/flake.nix b/flake.nix index f273ba765485351d76a06557b45e2b6725b294ad..9382098b14adbba6dbc6320d8483f2ba6137bd55 100644 --- a/flake.nix +++ b/flake.nix @@ -5,7 +5,6 @@ outputs = { self, nixpkgs, ... }: { packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { }; - packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { }; packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { }; packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { }; packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 04491685e48de19fc0b03d6ea855664b57d89eab..4f8faf94dafaaeed466710f7920c48c7f79adba9 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,12 +6,11 @@ let proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; }; computerAhk = pkgs.callPackage ./computer-ahk.nix { }; computerWindows = pkgs.callPackage ./computer-windows.nix { }; nativePkl = pkgs.callPackage ./pkl.nix { }; - hostPython = pkgs.callPackage ./host-python.nix { }; screen = pkgs.callPackage ./vm-screen.nix { }; secureOvmf = pkgs.OVMF.override { secureBoot = true; @@ -264,7 +263,7 @@ in SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables; }; serviceConfig = { - ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py"; + ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py"; Type = "notify"; Group = "snowglobe-dashboard"; StateDirectory = "snowglobe/host"; diff --git a/nixos/host-python.nix b/nixos/host-python.nix deleted file mode 100644 index a9d47aee38f4142cd6d623f45c50c98f52b355c6..0000000000000000000000000000000000000000 --- a/nixos/host-python.nix +++ /dev/null @@ -1,6 +0,0 @@ -{ python3, git }: -python3.buildEnv.override { - extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ]; - # shale-access serves anonymous Knife clones through git http-backend. - makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ]; -} diff --git a/nixos/mac-studio-sandbox.nix b/nixos/mac-studio-sandbox.nix index ddc6fea68a22b3470235c845eac24271011c92a4..d0cead8860873630e5228ac2cadd61994784abb7 100644 --- a/nixos/mac-studio-sandbox.nix +++ b/nixos/mac-studio-sandbox.nix @@ -50,11 +50,11 @@ in enable = true; hostName = "10.0.0.1"; exports = '' - ${mountPoint} ${workspace.clientAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) + ${mountPoint} ${workspace.guestAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) ''; }; environment.etc."exports.d/snowglobe-work.exports".text = '' - ${mountPoint}/.work ${workspace.clientAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) + ${mountPoint}/.work ${workspace.guestAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) ''; systemd.services.mac-work-share-check = { requires = [ "zfs-import-work.service" ]; diff --git a/nixos/mac-vm.nix b/nixos/mac-vm.nix index b584af2d9578735c0eae16ccb1ab1916b08f1192..0c9dc64bae1c3cbbd50b86c09495b9dcbd98834b 100644 --- a/nixos/mac-vm.nix +++ b/nixos/mac-vm.nix @@ -5,7 +5,7 @@ let in { systemd.services.snowglobe-host = { environment.SNOWGLOBE_MAC_VM = builtins.toJSON { - address = workspace.clientAddress; + address = workspace.hostAddress; port = 22; user = "clo"; identity = "/var/lib/snowglobe/mac-vm/ssh_key"; @@ -15,6 +15,6 @@ in { { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; } ]; }; - serviceConfig.IPAddressAllow = [ "${workspace.clientAddress}/32" ]; + serviceConfig.IPAddressAllow = [ "${workspace.hostAddress}/32" ]; }; } diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 8aa274d8c8b6082e5e883b6dad5ebf7249e99063..80e28cfbe1e52d617665946c4ef5e37839f28aa4 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -29,7 +29,7 @@ in networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept '') config.networking.interfaces.enp4s0.ipv4.addresses + '' - iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).clientAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept + iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).guestAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept ''; diff --git a/service/shale/prepare.py b/service/shale/prepare.py index 6afa2038cfcb5a63d0da74b8f853085f9e0e60bb..2036ddeb14f8872353425f9b100b5c08b188cf78 100644 --- a/service/shale/prepare.py +++ b/service/shale/prepare.py @@ -9,13 +9,8 @@ import sys data = json.load(sys.stdin) root = Path(data["hostRoot"]) -for config in (root / "repositories_owned").glob("*/config"): - text = config.read_text() - if not re.search(r"(?im)^\s*receivepack\s*=", text): - # Shale authorizes pushes before CGI; its own repository initializer enables this. - with config.open("a") as file: - file.write("\n[http]\n\treceivepack = true\n") path = root / "data/astheno.shale.db" +webhooks = {} if path.exists(): domain = os.environ["SNOWGLOBE_DOMAIN"] old, new = "auth." + domain + "/realms/master", "snowglobe." + domain @@ -25,6 +20,47 @@ if path.exists(): raise ValueError("duplicate Shale identity; resolve it before moving the issuer") count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount db.commit() + webhooks = dict(db.execute( + "SELECT uuid, name FROM repositories WHERE name IN ('snowcord', 'snowcord-server')" + )) assert db.execute("PRAGMA quick_check").fetchone() == ("ok",) db.close() print("Moved", count, "existing Shale identity bindings to Snowglobe") + +executable = root / "data/shale-webhook" +source = Path(__file__).with_name("webhook.sh").read_bytes() +if not executable.exists() or executable.read_bytes() != source: + pending = executable.with_suffix(".pending") + pending.write_bytes(source) + if os.geteuid() == 0: + os.chown(pending, data["uid"], data["uid"]) + pending.replace(executable) +executable.chmod(0o750) +if os.geteuid() == 0: + os.chown(executable, data["uid"], data["uid"]) + +target = "/data/shale-webhook" +for config in (root / "repositories_owned").glob("*/config"): + original = text = config.read_text() + if not re.search(r"(?im)^\s*receivepack\s*=", text): + # Shale authorizes pushes before CGI; its own repository initializer enables this. + text += "\n[http]\n\treceivepack = true\n" + text = re.sub(r'(?m)\n+\[shale "snowglobe"\]\n(?:[ \t].*\n?)*', "\n", text) + name = webhooks.get(config.parent.name) + if name: + text += f'\n[shale "snowglobe"]\n\trepository = {name}\n' + if text != original: + config.write_text(text) + + hook = config.parent / "hooks/post-receive" + if name: + hook.parent.mkdir(exist_ok=True) + if os.path.lexists(hook): + if not hook.is_symlink() or os.readlink(hook) != target: + raise ValueError(f"refusing to replace existing hook: {hook}") + if not os.path.lexists(hook): + hook.symlink_to(target) + if os.geteuid() == 0: + os.lchown(hook, data["uid"], data["uid"]) + elif hook.is_symlink() and os.readlink(hook) == target: + hook.unlink() diff --git a/service/shale/service.pkl b/service/shale/service.pkl index e140ae188cfd3e2cf3aa4eb595121905797d607e..d74f6537601195ddc6b84f366aa3fb9b2ba0ee51 100644 --- a/service/shale/service.pkl +++ b/service/shale/service.pkl @@ -61,6 +61,8 @@ container { ["MIRRORS_DISABLE_CLONE"] = "true" ["MIRRORS_DISABLE_PULL_POLL"] = "true" } + ["SHALE_WEBHOOK_ENABLED"] = if (site.preview) "false" else "true" + ["SHALE_WEBHOOK_URL"] = "${secret.own.webhook_url}" ["SESSION_SECRET"] = "${secret.own.session_secret}" ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}" } @@ -69,3 +71,4 @@ container { secrets { ["session_secret"] {} } +requiredSecrets { "webhook_url" } diff --git a/service/shale/webhook.sh b/service/shale/webhook.sh new file mode 100755 index 0000000000000000000000000000000000000000..a971cd0330283a55f867d1d1f9bc6ab041ab9e11 --- /dev/null +++ b/service/shale/webhook.sh @@ -0,0 +1,144 @@ +#!/bin/sh +set -u + +[ "${SHALE_WEBHOOK_ENABLED:-false}" = true ] || exit 0 +[ -n "${SHALE_WEBHOOK_URL:-}" ] || exit 0 +case "$SHALE_WEBHOOK_URL" in + */github|*/github/) webhook_url=${SHALE_WEBHOOK_URL%/}/ ;; + *) webhook_url=${SHALE_WEBHOOK_URL%/}/github/ ;; +esac + +repository=$(git config --get shale.snowglobe.repository) || exit 0 +origin="https://${DOMAIN}/${repository}" +avatar="https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png" + +json_string() { + awk 'BEGIN { ORS = "" } + { + if (NR > 1) printf "\\n" + for (i = 1; i <= length($0); i++) { + c = substr($0, i, 1) + if (c == "\\") printf "\\\\" + else if (c == "\"") printf "\\\"" + else if (c == "\t") printf "\\t" + else if (c == "\r") printf "\\r" + else if (c == sprintf("%c", 8)) printf "\\b" + else if (c == sprintf("%c", 12)) printf "\\f" + else printf "%s", c + } + }' +} + +quote() { + printf '"' + printf '%s' "$1" | json_string + printf '"' +} + +is_zero() { + case "$1" in + *[!0]*) return 1 ;; + *) return 0 ;; + esac +} + +write_ref_payload() { + { + printf '{"ref":'; quote "$short_ref" + printf ',"ref_type":'; quote "$ref_type" + printf ',"repository":{"name":'; quote "$repository" + printf ',"full_name":'; quote "$repository" + printf ',"html_url":'; quote "$origin/" + printf '},"sender":{"login":'; quote "$actor" + printf ',"avatar_url":'; quote "$avatar" + printf ',"html_url":'; quote "$origin/" + printf '}}\n' + } >"$payload" +} + +send() { + event=$1 + payload=$2 + delivery=$(printf '%s\n' "$repository:$event:$(date +%s):$$:$index" | git hash-object --stdin) + curl --fail --silent --show-error --max-time 10 \ + -H "Content-Type: application/json" \ + -H "User-Agent: Shale-Hookshot/1.0" \ + -H "X-GitHub-Event: $event" \ + -H "X-GitHub-Delivery: $delivery" \ + --data-binary "@$payload" "$webhook_url" >/dev/null +} + +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT HUP INT TERM +index=0 +while read -r before after ref; do + index=$((index + 1)) + payload="$temporary/$index.json" + short_ref=${ref#refs/heads/} + if [ "$short_ref" = "$ref" ]; then + short_ref=${ref#refs/tags/} + ref_type=tag + else + ref_type=branch + fi + + if is_zero "$after"; then + actor=${REMOTE_USER:-$(git show -s --format=%an "$before")} + write_ref_payload + send delete "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 + continue + fi + + actor=${REMOTE_USER:-$(git show -s --format=%an "$after")} + if [ "$ref_type" = tag ]; then + if is_zero "$before"; then + write_ref_payload + send create "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 + fi + continue + fi + if is_zero "$before"; then + created=true + forced=false + commits=$(git rev-list --reverse "$after") + else + created=false + if git merge-base --is-ancestor "$before" "$after"; then forced=false; else forced=true; fi + commits=$(git rev-list --reverse "$before..$after") + fi + compare="$origin/commit/$after" + { + printf '{"ref":'; quote "$ref" + printf ',"before":'; quote "$before" + printf ',"after":'; quote "$after" + printf ',"created":%s,"deleted":false,"forced":%s' "$created" "$forced" + printf ',"compare":'; quote "$compare" + printf ',"repository":{"name":'; quote "$repository" + printf ',"full_name":'; quote "$repository" + printf ',"html_url":'; quote "$origin/" + printf '},"sender":{"login":'; quote "$actor" + printf ',"avatar_url":'; quote "$avatar" + printf ',"html_url":'; quote "$origin/" + printf '},"commits":[' + separator= + for commit in $commits; do + message=$(git show -s --format=%B "$commit") + author=$(git show -s --format=%an "$commit") + timestamp=$(git show -s --format=%aI "$commit") + printf '%s{"id":' "$separator"; quote "$commit" + printf ',"message":'; quote "$message" + printf ',"timestamp":'; quote "$timestamp" + printf ',"url":'; quote "$origin/commit/$commit" + printf ',"author":{"name":'; quote "$author" + printf ',"username":'; quote "$author" + printf '}}' + separator=, + done + printf '],"head_commit":{"id":'; quote "$after" + printf ',"message":'; quote "$(git show -s --format=%B "$after")" + printf ',"timestamp":'; quote "$(git show -s --format=%aI "$after")" + printf ',"url":'; quote "$origin/commit/$after" + printf '}}\n' + } >"$payload" + send push "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 +done diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index 69e167d2990d685ec8fdfaf960d76caf7fccaf27..1953fede3c5be5e8123698a9c13aaba73124200b 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -16,7 +16,6 @@ import signal import socket import struct import subprocess -import sys import threading import time import vms @@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4 WORK_SLOTS = threading.BoundedSemaphore(WORKERS) STREAM_SLOTS = threading.BoundedSemaphore(4) GUEST_SLOTS = threading.BoundedSemaphore(4) -CLONE_SLOTS = threading.BoundedSemaphore(2) VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" @@ -97,8 +95,6 @@ class Host: "index.diff": {"dataset", "from", "to"}, "index.discard": {"dataset", "snapshot"}, "shale.metadata": {"repository", "accounts"}, - "shale.access": {"service", "uri", "account", "authorization"}, - "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, @@ -113,10 +109,6 @@ class Host: raise Rejected("Use only the fields required by this host operation.") if operation == "copyparty.manage": return copyparty.handle(request) - if operation == "shale.access": - return importlib.import_module("shale-access").access(request) - if operation == "shale.clone": - return importlib.import_module("shale-access").clone(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": @@ -473,20 +465,6 @@ def stream_upload(connection, request): Path(temporary).unlink(missing_ok=True) -def stream_clone(connection, plan): - owner = os.stat(plan["objects"]) - # Service identities have numeric owners without passwd entries, which systemd's User= rejects. - command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", - *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", - "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), - *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), - "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", - sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] - send_response(connection, {"value": None}) - connection.setblocking(True) - subprocess.run(command, stdin=connection, stdout=connection) - - def serve_connection(connection, host, allowed_uid, allowed_operations=None): with connection, contextlib.ExitStack() as resources: connection.settimeout(65) @@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): return finally: streams.release() - if operation == "shale.clone": - if not CLONE_SLOTS.acquire(blocking=False): - raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") - try: - stream_clone(connection, host.handle(request)) - return - finally: - CLONE_SLOTS.release() response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} except Rejected as error: response = {"error": str(error), "status": 400} diff --git a/tools/dashboard-shale-page-test.py b/tools/dashboard-shale-page-test.py index ce668c6e98274955371460fe3c128e396cfa6702..cb699ed5864c899021e15a62806a453a7094efe2 100644 --- a/tools/dashboard-shale-page-test.py +++ b/tools/dashboard-shale-page-test.py @@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path -import re import socket -import socketserver import sqlite3 import subprocess import tempfile @@ -49,21 +47,12 @@ def main(): issue = args.page.read_bytes() if args.page else b'~guest-github-24465214' sample = b'guestunknownrepo' blob = bytes(range(256)) * 65536 - csrf = 'initial' class Fixture(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' def do_GET(self): - nonlocal csrf observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()})) - if self.path == '/': - account = b'~fixture-member' if self.headers.get('Cookie') == 'SessionID=preserved' else b'' - return self.respond(b'
', 'text/html') - if self.path == '/-/settings' or self.path.endswith('/issues/new'): - csrf += 'x' - account = 'fixture-member' if self.path == '/-/settings' else '' - return self.respond(('' + account + '
').encode(), 'text/html') if self.path == '/binary': return self.respond(blob, 'application/octet-stream') if self.path == '/json': @@ -86,8 +75,6 @@ def main(): def do_POST(self): body = self.rfile.read(int(self.headers.get('Content-Length', 0))) observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body)) - if self.path.endswith('/issues/new') and urllib.parse.parse_qs(body.decode()).get('csrf_token') != [csrf]: - return self.respond(b'E: unexpected csrf token', 'text/plain', 403) self.respond(body, 'application/octet-stream', 201) def respond(self, body, content_type, status=200, headers=()): @@ -121,21 +108,7 @@ def main(): subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True) data = root / 'data' data.mkdir() - class Host(socketserver.StreamRequestHandler): - def handle(self): - request = json.loads(self.rfile.readline()) - value = {} - if request['operation'] == 'shale.access': - allowed = request['account'] == 'fixture-member' - value = {'allowed': allowed, 'deny': not allowed} - response = json.dumps({'value': value}).encode() - self.wfile.write(len(response).to_bytes(4, 'big') + response) - - host = socketserver.UnixStreamServer(str(root / 'host.sock'), Host) - stack.callback(host.server_close) - stack.callback(host.shutdown) - threading.Thread(target=host.serve_forever, daemon=True).start() - environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_HOST_SOCKET': str(root / 'host.sock'), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')} + environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')} for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']: environment.pop(key, None) log = stack.enter_context((root / 'dashboard.log').open('w')) @@ -253,22 +226,10 @@ def main(): assert len(request('/large')[2]) == 9 * 1024 * 1024 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged') assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes' - for path in ['/the-knife/issues/new', '/another-repo/issues/new']: - status, _, form = request(path) - assert status == 200 - csrf_field = re.search(rb'name="csrf_token" value="([^"]+)"', form)[1] - assert request('/the-knife/')[0] == 200 - submitted = b'csrf_token=' + csrf_field + b'&title=preserved' - status, _, body = request(path, body=submitted) - assert status == 201 and body == submitted, (path, status, body) - for invalid in [b'title=missing', b'csrf_token=invalid']: - assert request(path, body=invalid)[0] == 403 - assert not any(item[1] == '/-/settings' for item in observations) - assert request('/the-knife/issues/new', headers={'Cookie': ''})[0] == 403 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403 assert request('/auth/shale/page', direct=True)[0] == 403 assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403 - print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'csrf_preservation_and_rejection': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True) + print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True) if args.browser_ready_file: args.browser_ready_file.write_text(json.dumps({'origin': origin})) stop = args.browser_ready_file.with_suffix('.stop') diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index 28af941551a0e62b5aff913ff5be529b991584c1..fb143552a80b8836a202a2085a645996d6ba42eb 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -21,7 +21,6 @@ import snowglobe OPERATIONS = frozenset({ - "shale.access", "shale.clone", "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", @@ -56,16 +55,6 @@ class PreviewHost(boundary.Host): actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") - if operation in {"shale.access", "shale.clone"}: - service = request.get("service", "") - if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): - raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") - linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) - if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: - raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") - if operation == "shale.clone": - return import_module("shale-access").clone(request) - return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") if operation in {"files.snapshot", "files.snapshots", "files.discard"}: datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: @@ -234,14 +223,13 @@ def stage(computer=None): for entry in mac["vms"]: entry["owner"] = owner[0] computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac)) - host_python = snowglobe.command(*nix, "build", "--out-link", str(root / "host-python"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip() subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True) snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32", "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], *computer_environment, - host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity) + sys.executable, str(Path(__file__).resolve()), "--serve", identity) for _ in range(100): if Path(runtime, "host.sock").exists(): break @@ -263,7 +251,7 @@ def stage(computer=None): "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock", "--setenv=XDG_RUNTIME_DIR=" + screen_runtime, "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache", - host_python + "/bin/python3", "-c", launcher) + sys.executable, "-c", launcher) for _ in range(100): if Path(screen_runtime, "screen.sock").exists(): break diff --git a/tools/mac-agent-forward-test.py b/tools/mac-agent-forward-test.py index 6fb89ea874678bc50026ef70b0351e60d21be644..9b4fdd281f7da545c508ae021b6819cc5de49646 100644 --- a/tools/mac-agent-forward-test.py +++ b/tools/mac-agent-forward-test.py @@ -35,8 +35,7 @@ sys.stdin.buffer.read() server.shutdown() server.server_close() ''' - ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip() - server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)], + server = subprocess.Popen([*vm.ssh_args(), vm.GUEST, "python3 -u -c " + shlex.quote(code)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) tunnel = None try: diff --git a/tools/mac-agent-vm.md b/tools/mac-agent-vm.md index 38cf7e87c111fea2bafb7c03bb0548bb8d5fa1ce..c4a1d3a81dfbccaa06e1d522c37526d26e95063c 100644 --- a/tools/mac-agent-vm.md +++ b/tools/mac-agent-vm.md @@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console python3 tools/mac-agent-vm.py stop ``` +Everything except `exec` reaches the guest over pinned SSH at its Bonjour name. +`exec` and the dashboard's computer operations go through Tart's guest agent so +they run in the logged-in GUI session, and that path can crash the VM runner +(`/Library/Logs/DiagnosticReports/tart-*.ips`, `-[VZVirtioSocketDevice +connectToPort:]`). Use `ssh` for anything that does not need the GUI session. + +The `net.paperclover.mac-vm-revive` LaunchAgent runs `revive` every 30 seconds +through `ssh localhost` (launchd jobs cannot read the VM volume). It starts the +VM when its runner is gone unless `stop` or `suspend` left `private/stopped`; +`start` clears that marker. Restarts and the newest Tart crash report are logged +to `~/Library/Logs/mac-vm-revive.log`. A hung guest under a live runner is not +restarted. + To reach a server bound to guest loopback from this Mac: ```sh @@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running VM holds a process-scoped idle-sleep assertion, so locking the host does not cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. +## Network and iPhones + +The guest is bridged onto the LAN through the host's default-route interface +so iPhones can find it over Bonjour and connect in; Virtualization.framework +has no USB passthrough for them. Its DHCP address and the host's are reserved +on the router and recorded in +[mac-studio-sandbox.json](../config/mac-studio-sandbox.json): Snow Globe +exports NFS to `guestAddress` and the dashboard reaches the host at +`hostAddress`. Every `start` loads a pf anchor (`com.apple/net.paperclover`) +in the guest that refuses new inbound TCP below port 49152 from anyone but +the host, plus Kerberos and NetBIOS over UDP. A guest that reboots itself +stays open to the LAN until the next `start`. iPhones on iOS 27 pair in Device +Hub with **+ → Pair Nearby Device…** and Settings → Privacy & Security → +Developer Mode on the phone. Older iOS needs a cable, which the guest cannot +have. + The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve physical cores. CPU count and RAM changes require a cold boot. No automatic lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget @@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker is present. The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address -`10.0.0.1`. Traffic uses the VM's NAT gateway `192.168.64.1`; the host's hosts -file is not inherited. The previous guest file is `/etc/hosts.before-code-nas`. +`10.0.0.1`. The host's hosts file is not inherited. The previous guest file is +`/etc/hosts.before-code-nas`. SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned in `private/known_hosts`, and disables password authentication. No host SSH @@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles the guest account authentication from `private/password` server-side. Screen -Sharing listens on all interfaces inside the private NAT guest; the host relay's +Sharing listens on all guest interfaces behind the pf anchor; the host relay's TCP listeners default to loopback and check the exact HTTP Host and WebSocket Origin. Tart's experimental host VNC server is disabled. The viewer trusts every client that can reach its listener; Origin checking protects against foreign diff --git a/tools/mac-agent-vm.py b/tools/mac-agent-vm.py index 327c3f81946b8f4f27712acada7fe525393057a9..64cd9602f48b1782acc2f3115b77d2720aafb5d3 100644 --- a/tools/mac-agent-vm.py +++ b/tools/mac-agent-vm.py @@ -8,6 +8,7 @@ import logging import multiprocessing import os from pathlib import Path +import re import select import secrets import shlex @@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod TART = ROOT / "runtime/tart.app/Contents/MacOS/tart" NAME = "gui-agent" PRIVATE = ROOT / "private" +# Never reach the guest with `tart exec` or `tart ip --resolver=agent` unless it needs the GUI session: +# their vsock connect intermittently traps inside Virtualization.framework and kills the whole VM. +GUEST = "admin@AgentVM.local" def tart(*args, **kwargs): @@ -35,12 +39,27 @@ def tart(*args, **kwargs): def ssh_args(): return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"), - "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no", + "-o", "AddressFamily=inet", "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no", "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME, "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"', "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"] +def guest(*command, **kwargs): + kwargs.setdefault("check", True) + return subprocess.run([*ssh_args(), GUEST, shlex.join(command)], **kwargs) + + +def guest_ready(**kwargs): + guest("/bin/sh", "-c", '[ "$(/usr/bin/stat -f %Su /dev/console)" = admin ]', **kwargs) + + +def shut_down(): + # ssh can drop as the guest powers off, so the stopped state is the proof. + guest("sudo", "-n", "shutdown", "-h", "now", check=False, timeout=15) + wait_for_state("stopped") + + def exact(stream, size): data = bytearray() while len(data) < size: @@ -51,10 +70,14 @@ def exact(stream, size): return bytes(data) +def vm(): + rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) + return next(row for row in rows if row["Name"] == NAME and row["Source"] == "local") + + def wait_for_state(state): for _ in range(90): - rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) - if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["State"] == state: + if vm()["State"] == state: return time.sleep(1) raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.") @@ -62,13 +85,12 @@ def wait_for_state(state): @contextlib.contextmanager def guest_screen(): - ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory: path = Path(directory) / "console.sock" tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", "-o", "StreamLocalBindMask=0177", "-L", - f"{path}:127.0.0.1:5900", "admin@" + ip], + f"{path}:127.0.0.1:5900", GUEST], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: @@ -208,6 +230,7 @@ def main(): sub.add_parser("stop") sub.add_parser("suspend") sub.add_parser("reboot") + sub.add_parser("revive", help="Start the VM unless it is running or was stopped on purpose") for action in ["exec", "ssh"]: command = sub.add_parser(action) command.add_argument("command", nargs=argparse.REMAINDER) @@ -222,12 +245,17 @@ def main(): local_port = args.local_port if args.local_port is not None else args.port if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535: parser.error("Ports must be between 1 and 65535.") - ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.", file=sys.stderr, flush=True) os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", - "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip]) + "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", GUEST]) + if args.action == "revive": + if (PRIVATE / "stopped").exists() or vm()["Running"]: + return + crashes = sorted(Path("/Library/Logs/DiagnosticReports").glob("tart-*.ips")) + print(f"{time.ctime()}: VM is down; newest tart crash report: {crashes[-1] if crashes else 'none'}", flush=True) + args.action = "start" if args.action in {"start", "stop", "suspend", "reboot"}: lock = (PRIVATE / "lifecycle.lock").open("a") os.chmod(PRIVATE / "lifecycle.lock", 0o600) @@ -240,34 +268,47 @@ def main(): lock.write(args.action) lock.flush() if args.action == "reboot": - tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now", timeout=15) - wait_for_state("stopped") + shut_down() args.action = "start" if args.action == "start": - rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) - if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]: + (PRIVATE / "stopped").unlink(missing_ok=True) + interface = re.search(r"interface: (\S+)", subprocess.run(["/sbin/route", "-n", "get", "default"], + capture_output=True, text=True, check=True).stdout)[1] + if vm()["Running"]: print("GUI-agent VM is already running.") - return - # Closing an agent session kills every process it ever spawned, so sshd must own the VM. - run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", - str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", - "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"]) - (PRIVATE / "run.log").write_text("") - os.chmod(PRIVATE / "run.log", 0o600) - subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", - f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 {shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 deadline: + raise ValueError("VM startup is still pending; inspect status before retrying.") + if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode: + raise ValueError("VM startup failed; inspect the private run log locally.") + try: + guest_ready(capture_output=True, timeout=5) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + time.sleep(1) + else: + break + print("GUI-agent VM is ready in its logged-in guest session.") + # The bridged guest is on the LAN: only this Mac may reach its services; iPhones pair on ephemeral ports. + host = subprocess.run(["/usr/sbin/ipconfig", "getifaddr", interface], + capture_output=True, text=True, check=True).stdout.strip() + guest("sudo", "-n", "/bin/sh", "-c", + "/sbin/pfctl -q -a com.apple/net.paperclover -f - && /sbin/pfctl -q -E", + input=f"table {{ {host} }}\n" + "block in quick on ! lo0 proto tcp from ! to any port < 49152 flags S/SA\n" + "block in quick on ! lo0 proto udp to any port { 88 137 138 }\n", + capture_output=True, text=True) elif args.action == "status": tart("get", NAME) elif args.action == "exec": @@ -275,18 +316,18 @@ def main(): parser.error("exec requires a guest command") tart("exec", "-i", NAME, *args.command) elif args.action == "ssh": - ip = tart("ip", NAME, capture_output=True, text=True).stdout.strip() command = ssh_args() if not args.command: command.append("-t") - os.execvp("ssh", [*command, "admin@" + ip, *args.command]) + os.execvp("ssh", [*command, GUEST, *args.command]) elif args.action == "stop": - tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now") - wait_for_state("stopped") + shut_down() + (PRIVATE / "stopped").touch() print("GUI-agent VM shut down.") elif args.action == "suspend": tart("suspend", NAME) wait_for_state("suspended") + (PRIVATE / "stopped").touch() print("GUI-agent VM suspended.") elif args.action == "console": python = ROOT / "runtime/venv/bin/python" diff --git a/tools/mac-vm-bridge.py b/tools/mac-vm-bridge.py index e3a5bdec36b3291d25e36c7eaf896f3ce4ec3015..252228a651fff01d3a7a1a4dd303a4f293776555 100644 --- a/tools/mac-vm-bridge.py +++ b/tools/mac-vm-bridge.py @@ -32,7 +32,7 @@ def status(): agent = None if current["Running"]: try: - vm.tart("exec", vm.NAME, "/usr/bin/true", capture_output=True, timeout=2) + vm.guest_ready(capture_output=True, timeout=5) agent = "connected" except subprocess.SubprocessError: agent = "disconnected" diff --git a/tools/release.py b/tools/release.py index 4961e86946c52af89f5436ac2880bce749859e60..bb2ed731f6733af41f69f85967a114f0cab78fd2 100644 --- a/tools/release.py +++ b/tools/release.py @@ -60,7 +60,7 @@ def host_digest(root): digest = hashlib.sha256() paths = sorted(path for path, relative in files(root) if relative.parts[0] in {"nixos", "dashboard", "config", "service"} - or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) + or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) for path in paths: digest.update(str(path.relative_to(root)).encode()) _hash_contents(digest, path) diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py deleted file mode 100644 index d32ca65892524367159722b3e8a2e8ce7b571b32..0000000000000000000000000000000000000000 --- a/tools/shale-access-test.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python3 -import base64 -from contextlib import closing -import importlib -import json -import os -from pathlib import Path -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import sqlite3 -import subprocess -import sys -import tempfile -import threading -import unittest -from unittest.mock import patch - -import pygit2 -from argon2 import PasswordHasher - -shale = importlib.import_module('shale-access') - - -class CGI(BaseHTTPRequestHandler): - """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" - def __init__(self, test, *args): - self.test = test - super().__init__(*args) - - def log_message(self, *args): - pass - - def do_GET(self): - self.relay() - - def do_POST(self): - self.relay() - - def relay(self): - try: - plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) - except shale.Error: - self.send_response(401) - self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') - self.end_headers() - return - body = self.rfile.read(int(self.headers.get('content-length', 0))) - output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, - env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout - head, _, payload = output.partition(b'\r\n\r\n') - headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) - self.send_response(int(headers.pop('Status', '200').split()[0])) - for name, value in headers.items(): - self.send_header(name, value) - self.end_headers() - self.wfile.write(payload) - - -class Access(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'}) - self.env.start() - self.addCleanup(self.env.stop) - (self.root / 'data').mkdir() - self.uuid = '0' * 26 - self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) - self.addCleanup(self.repo.free) - self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) - self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) - self.repo.create_reference('refs/heads/native', self.native) - self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native]) - self.repo.create_reference('refs/heads/main', self.public) - self.repo.set_head('refs/heads/main') - (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) - with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: - db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') - for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): - db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) - if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) - with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: - db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') - db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) - for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): - db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name)) - db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret'))) - - def commit(self, files, parents=None): - index = pygit2.Index() - for path, content in files.items(): - index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB)) - tree = index.write_tree(self.repo) - author = pygit2.Signature('Fixture', 'fixture@example.invalid') - return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or []) - - def access(self, path, account='', authorization=''): - return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) - - def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self): - released = [] - class TrackedRepository(pygit2.Repository): - def free(self): - released.append(True) - super().free() - with patch.object(shale.pygit2, 'Repository', TrackedRepository): - self.assertFalse(self.access('/the-knife/')['deny']) - self.assertTrue(self.access('/the-knife/objects/x')['deny']) - with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')): - with self.assertRaises(ValueError): - self.access('/the-knife/') - self.assertEqual(len(released), 3) - - def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): - public = self.access('/the-knife/tree/main/') - self.assertFalse(public['deny']) - self.assertIn(str(self.public), public['uri']) - for rev in ('native', str(self.native), str(self.native)[:12]): - for route in ('tree', 'plain', 'commit'): - self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev)) - self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny']) - self.repo.references.delete('refs/heads/native') - self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny']) - - def test_every_revision_blocks_files_even_when_absent(self): - for rev in ('main', str(self.base)): - for route in ('tree', 'plain'): - for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'): - self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path)) - self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny']) - - def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self): - result = self.access('/the-knife/commit/' + str(self.public)) - self.assertFalse(result['deny']) - self.assertNotIn('SECRET', result['safeDiff']) - self.assertIn('PUBLIC', result['safeDiff']) - for suffix in ('diff', 'patch'): - result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix) - self.assertFalse(result['deny']) - self.assertNotIn('SECRET', result['safeDiff']) - self.assertIn('PUBLIC', result['safeDiff']) - for suffix in ('stat', 'diffraw'): - self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny']) - - def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self): - path = '/the-knife/plain/native/knife/private.rs' - for name in ('snow', 'guest'): - self.assertTrue(self.access(path, name)['allowed']) - for name in ('stranger', 'disabled', 'ai', ''): - self.assertTrue(self.access(path, name)['deny']) - with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: - db.execute("DELETE FROM memberships WHERE user_id='snow'") - self.assertTrue(self.access(path, 'snow')['deny']) - - def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): - path = '/the-knife/info/refs?service=git-receive-pack' - self.assertTrue(self.access(path, 'snow')['deny']) - self.assertTrue(self.access(path)['authenticate']) - token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() - self.assertTrue(self.access(path, authorization=token)['allowed']) - self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) - with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: - db.execute('DELETE FROM personal_tokens') - self.assertTrue(self.access(path, authorization=token)['deny']) - - def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): - for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): - self.assertTrue(self.access(path)['clone'], path) - token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() - self.assertTrue(self.access(path, authorization=token)['allowed'], path) - for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): - self.assertNotIn('clone', self.access(path), path) - - def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): - with self.assertRaises(shale.Error): - self.clone('/the-knife.git/info/refs?service=git-upload-pack') - self.repo.lookup_reference('refs/heads/main').set_target(self.base) - self.repo.create_reference('refs/heads/merge', self.public) - server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) - self.addCleanup(server.server_close) - threading.Thread(target=server.serve_forever, daemon=True).start() - self.addCleanup(server.shutdown) - url = f'http://127.0.0.1:{server.server_port}/the-knife' - environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} - for version in ('0', '1', '2'): - checkout = self.root / ('clone-v' + version) - subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) - refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout - self.assertEqual(refs, f'refs/heads/main {self.base}\n') - for revision in (str(self.native), str(self.public)): - fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) - self.assertNotEqual(fetched.returncode, 0, (version, revision)) - self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) - pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) - self.assertNotEqual(pushed.returncode, 0) - - def clone(self, uri, method='GET', content_type='', content_encoding=''): - return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) - - def test_native_push_during_classification_cannot_change_the_checked_revision(self): - newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) - repo = self.repo - class ConcurrentPush: - def __getattr__(self, name): return getattr(repo, name) - def __getitem__(self, key): return repo[key] - def walk(self, *args): - repo.lookup_reference('refs/heads/native').set_target(newer) - repo.create_reference('refs/heads/deadbeef', newer, force=True) - return repo.walk(*args) - for revision in ('native', 'deadbeef'): - repo.lookup_reference('refs/heads/native').set_target(self.native) - if 'refs/heads/deadbeef' in repo.references: - repo.references.delete('refs/heads/deadbeef') - with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()): - self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) - - def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): - with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: - db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") - self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) - token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() - self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny']) - - def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self): - for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'): - self.assertTrue(self.access(path)['deny'], path) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/shale-access.py b/tools/shale-access.py deleted file mode 100644 index c1d88779af251ce2bc2192789abfe257b9a5b803..0000000000000000000000000000000000000000 --- a/tools/shale-access.py +++ /dev/null @@ -1,250 +0,0 @@ -import base64 -from contextlib import closing -import importlib -import json -import os -from pathlib import Path -import posixpath -import re -import sqlite3 -import subprocess -import sys -import tempfile -from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit - -import pygit2 -from argon2 import PasswordHasher -from argon2.exceptions import VerificationError, InvalidHashError - -Error = importlib.import_module('dashboard-run').Error -# The root host helper reads repositories owned by the Shale service UID. -pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) - - -def restricted_path(path): - parts = path.split('/') - return parts[0] == 'knife' or 'Cargo.lock' in parts - - -def allowed_account(db, user_id): - row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake " - "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m " - "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')", - (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone() - if not row: - return False - profile = json.loads(row[0]) - return (profile.get('enabled') is True and not profile.get('requiredActions') - and profile.get('kind') != 'ai') - - -def instance(service): - if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): - raise Error(400, 'Choose a Shale instance.') - if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): - return Path(fixture_root) - site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) - return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) - - -def target(request_uri): - uri = urlsplit(request_uri) - path = uri.path - for _ in range(4): - decoded = unquote(path, errors='strict') - if decoded == path: - break - path = decoded - path = posixpath.normpath('/' + path.lstrip('/')) - return uri, path, path.strip('/').split('/') - - -def upload_pack(uri, path, parts): - """Returns the CGI path of an anonymous fetch request, the only Git route served without a token.""" - if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path: - return None - if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query: - return '/the-knife.git/' + '/'.join(parts[1:]) - return None - - -def repository(root, db): - row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() - return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None - - -def protected_commits(repo, native): - policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] - tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] - if native: - tips.append(native) - return {str(commit.id) for tip in tips for commit in repo.walk(tip)} - - -def access(request, accounts=None): - root = instance(request['service']) - uri, path, parts = target(request['uri']) - if parts[0].lower() not in ('the-knife', 'the-knife.git'): - return {'allowed': False, 'deny': True} - git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') - if not request['authorization'] and upload_pack(uri, path, parts): - return {'allowed': False, 'deny': False, 'clone': True} - with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: - accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) - db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) - authorization = request['authorization'] - allowed = False - if authorization: - try: - scheme, value = authorization.split(' ', 1) - if scheme.lower() == 'basic': - name, secret = base64.b64decode(value, validate=True).decode().split(':', 1) - users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall() - if len(users) == 1 and allowed_account(db, users[0][0]): - for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]): - try: - if PasswordHasher().verify(hashed, secret): - allowed = True - break - except (VerificationError, InvalidHashError): - pass - except (ValueError, UnicodeError): - pass - elif request['account'] and not git: - users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall() - allowed = len(users) == 1 and allowed_account(db, users[0][0]) - if allowed: - return {'allowed': True, 'deny': False} - location = repository(root, db) - if not location: - return {'allowed': False, 'deny': True} - repo = pygit2.Repository(str(location)) - try: - head = repo.lookup_reference('HEAD').target - references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} - protected = protected_commits(repo, references.get('refs/heads/native')) - result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} - if parts[0] != 'the-knife' or '%' in path or '\\' in path: - return result - route = parts[1] if len(parts) > 1 else '' - query = dict(parse_qsl(uri.query, keep_blank_values=True)) - if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): - return result - if route in ('', 'issues'): - if route == '' and query: - return result - result['htmlOnly'] = route != 'issues' - result['deny'] = False - return result - if route in ('heads', 'tags') and len(parts) == 2 and not query: - result['deny'] = False - return result - if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): - return result - if route not in ('log', 'tree', 'plain', 'commit'): - return result - revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' - suffix = '' - if route == 'commit': - revision, dot, suffix = revision.partition('.') - if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): - return result - if not revision or revision == '-': - revision = 'HEAD' - # Only named refs and object IDs have the same meaning in Shale and libgit2. - if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): - return result - try: - if revision == 'HEAD': - oid = references.get(head) if isinstance(head, str) else head - else: - candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} - if len(candidates) > 1: - return result - if candidates: - oid = candidates.pop() - elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): - oid = repo[revision].id - else: - return result - if oid is None: - return result - commit = repo[oid].peel(pygit2.Commit) - except (KeyError, ValueError, pygit2.GitError): - return result - if str(commit.id) in protected: - return result - if route in ('tree', 'plain'): - result['htmlOnly'] = route != 'plain' - file = '/'.join(parts[3:]) - if restricted_path(file): - return result - if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): - return result - if route == 'commit': - diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] - sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) - if sensitive: - if suffix in ('stat', 'diffraw'): - return result - result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" - if suffix: - result['rawDiff'] = True - result['deny'] = False - return result - if route == 'log': - result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) - else: - parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] - result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') - if query: - result['uri'] += '?' + urlencode(query) - result['deny'] = False - return result - finally: - repo.free() - - -def clone(request): - root = instance(request['service']) - uri, path, parts = target(request['uri']) - location = upload_pack(uri, path, parts) - if not location: - raise Error(404, 'Clone The Knife over HTTPS.') - with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: - knife = repository(root, db) - if not knife: - raise Error(404, 'The Knife is unavailable.') - repo = pygit2.Repository(str(knife)) - try: - main = repo.lookup_reference('refs/heads/main').resolve().target - native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None - protected = protected_commits(repo, native) - if any(str(commit.id) in protected for commit in repo.walk(main)): - raise Error(403, "The Knife's main branch includes native history.") - finally: - repo.free() - return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': { - 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query, - 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'], - }} - - -def serve(objects, main): - """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects.""" - repository = Path(tempfile.mkdtemp()) / 'the-knife.git' - (repository / 'objects/info').mkdir(parents=True) - (repository / 'refs/heads').mkdir(parents=True) - (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n') - (repository / 'HEAD').write_text('ref: refs/heads/main\n') - (repository / 'refs/heads/main').write_text(main + '\n') - (repository / 'objects/info/alternates').write_text(objects + '\n') - # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main. - environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)} - os.execvpe('git', ['git', 'http-backend'], environment | { - 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', - }) - - -if __name__ == '__main__': - serve(*sys.argv[1:]) diff --git a/tools/shale-webhook-test.py b/tools/shale-webhook-test.py new file mode 100644 index 0000000000000000000000000000000000000000..44f115bc86af6ea1460896ca645bde05ce5ac7f7 --- /dev/null +++ b/tools/shale-webhook-test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +import http.server +import json +import os +from pathlib import Path +import sqlite3 +import subprocess +import tempfile +import threading +import unittest + +REPO = Path(__file__).resolve().parent.parent +HOOK = REPO / "service/shale/webhook.sh" +PREPARE = REPO / "service/shale/prepare.py" + + +class Receiver(http.server.BaseHTTPRequestHandler): + events = [] + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + self.events.append((self.headers["X-GitHub-Event"], self.path, json.loads(body))) + self.send_response(204) + self.end_headers() + + def log_message(self, *_): + pass + + +class ShaleWebhookTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Receiver) + Receiver.events = [] + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def tearDown(self): + self.server.shutdown() + self.thread.join() + self.server.server_close() + self.temp.cleanup() + + def git(self, *args, cwd=None, env=None): + return subprocess.run( + ["git", *args], cwd=cwd, env=env, check=True, text=True, + capture_output=True, + ).stdout.strip() + + def test_push_and_delete_match_snowcord_github_contract(self): + bare = self.root / "origin.git" + work = self.root / "work" + self.git("init", "--bare", "--initial-branch=main", bare) + self.git("config", "--file", bare / "config", "shale.snowglobe.repository", "snowcord") + (bare / "hooks/post-receive").symlink_to(HOOK) + self.git("init", "--initial-branch=main", work) + self.git("config", "user.name", "Clover", cwd=work) + self.git("config", "user.email", "clover@example.test", cwd=work) + (work / "file.txt").write_text("hello\n") + self.git("add", "file.txt", cwd=work) + self.git("commit", "-m", 'first "line"\n\nsecond \\ line', cwd=work) + self.git("remote", "add", "origin", str(bare), cwd=work) + env = { + **os.environ, + "DOMAIN": "shale.test", + "SHALE_WEBHOOK_ENABLED": "true", + "SHALE_WEBHOOK_URL": f"http://127.0.0.1:{self.server.server_port}/hook", + } + self.git("push", "origin", "main", cwd=work, env=env) + + event, path, payload = Receiver.events.pop(0) + self.assertEqual(event, "push") + self.assertEqual(path, "/hook/github/") + self.assertEqual(payload["ref"], "refs/heads/main") + self.assertEqual(payload["repository"]["name"], "snowcord") + self.assertEqual(payload["sender"]["login"], "Clover") + self.assertEqual(payload["commits"][0]["message"], 'first "line"\n\nsecond \\ line') + self.assertEqual(payload["head_commit"]["id"], payload["after"]) + + self.git("push", "origin", "main:delete-me", cwd=work, env=env) + Receiver.events.pop(0) + self.git("push", "origin", ":delete-me", cwd=work, env=env) + event, _, payload = Receiver.events.pop(0) + self.assertEqual((event, payload["ref_type"], payload["ref"]), ("delete", "branch", "delete-me")) + + def test_prepare_installs_only_named_repository_hooks(self): + owned = self.root / "repositories_owned" + database = self.root / "data/astheno.shale.db" + database.parent.mkdir() + with sqlite3.connect(database) as db: + db.executescript(""" + CREATE TABLE users (provider TEXT, snowflake TEXT); + CREATE TABLE repositories (uuid TEXT, name TEXT); + INSERT INTO repositories VALUES ('snowcord-id', 'snowcord'); + INSERT INTO repositories VALUES ('other-id', 'other'); + """) + for name in ("snowcord-id", "other-id"): + self.git("init", "--bare", owned / name) + env = {**os.environ, "SNOWGLOBE_DOMAIN": "test"} + subprocess.run( + ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}), + env=env, check=True, text=True, capture_output=True, + ) + hook = owned / "snowcord-id/hooks/post-receive" + self.assertEqual(os.readlink(hook), "/data/shale-webhook") + self.assertEqual((self.root / "data/shale-webhook").stat().st_mode & 0o777, 0o750) + self.assertEqual( + self.git("config", "--file", owned / "snowcord-id/config", "--get", "shale.snowglobe.repository"), + "snowcord", + ) + self.assertFalse((owned / "other-id/hooks/post-receive").exists()) + + with sqlite3.connect(database) as db: + db.execute("UPDATE repositories SET name='renamed' WHERE uuid='snowcord-id'") + subprocess.run( + ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}), + env=env, check=True, text=True, capture_output=True, + ) + self.assertFalse(os.path.lexists(hook)) + + +if __name__ == "__main__": + unittest.main()