| author | |
| committer | |
| log | 9acd963a146bafc2595b79916b2f1df24cc4eefe |
| tree | 261a78d8790d924564c2ad56abfb918d468c5956 |
| parent | 17dc455c3819b2d2563acaf514669bb33a593f0f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
The Knife is private in Shale now, so the dashboard no longer gates
/the-knife: shale-access.py, anonymous main-branch clones, native-history
redaction, the restricted page, and nativeRestriction are gone, along with
host-python.nix, which existed only for them. The knife-native group is
deleted from existing databases; guests join no groups again, and adding a
guest only creates the account.
Assisted-by: claude-opus-5.528 files changed, 456 insertions(+), 890 deletions(-)
config/mac-studio-sandbox.json+2-1| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | { | 1 | { |
| 2 | "dataset": "globe/clover/mac-studio-sandbox", | 2 | "dataset": "globe/clover/mac-studio-sandbox", |
| 3 | "mountPoint": "/srv/mac-studio-vm/sandbox", | 3 | "mountPoint": "/srv/mac-studio-vm/sandbox", |
| 4 | "clientAddress": "10.0.0.222" | 4 | "hostAddress": "10.0.0.189", |
| 5 | "guestAddress": "10.0.0.124" | ||
| 5 | } | 6 | } |
config/shale.json+1-1| ... | @@ -1 +1 @@ | ... | @@ -1 +1 @@ |
| 1 | {"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}} | 1 | {"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}} |
dashboard/src/auth.rs+4-5| ... | @@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session"; | ... | @@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session"; |
| 12 | const FLOW_COOKIE: &str = "__Host-snow-flow"; | 12 | const FLOW_COOKIE: &str = "__Host-snow-flow"; |
| 13 | const CSRF_COOKIE: &str = "__Host-snow-csrf"; | 13 | const CSRF_COOKIE: &str = "__Host-snow-csrf"; |
| 14 | const SESSION_TTL: i64 = 30 * 86400; | 14 | const SESSION_TTL: i64 = 30 * 86400; |
| 15 | const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"]; | 15 | const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"]; |
| 16 | 16 | ||
| 17 | pub struct Store { | 17 | pub struct Store { |
| 18 | pub db: Mutex<Connection>, | 18 | pub db: Mutex<Connection>, |
| ... | @@ -201,14 +201,13 @@ impl Store { | ... | @@ -201,14 +201,13 @@ impl Store { |
| 201 | oidc::initialise(&db)?; | 201 | oidc::initialise(&db)?; |
| 202 | guest::initialise(&db)?; | 202 | guest::initialise(&db)?; |
| 203 | for name in GROUPS { | 203 | for name in GROUPS { |
| 204 | let added = db.execute( | 204 | db.execute( |
| 205 | "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)", | 205 | "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)", |
| 206 | sql![uuid::Uuid::new_v4().to_string(), name], | 206 | sql![uuid::Uuid::new_v4().to_string(), name], |
| 207 | )?; | 207 | )?; |
| 208 | if *name == "knife-native" && added != 0 { | ||
| 209 | db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?; | ||
| 210 | } | ||
| 211 | } | 208 | } |
| 209 | db.execute_batch("DELETE FROM memberships WHERE role_id IN (SELECT id FROM roles WHERE name='knife-native'); | ||
| 210 | DELETE FROM roles WHERE name='knife-native';")?; | ||
| 212 | Ok(Self { | 211 | Ok(Self { |
| 213 | db: Mutex::new(db), | 212 | db: Mutex::new(db), |
| 214 | origin, | 213 | origin, |
dashboard/src/guest.rs+4-21| ... | @@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> { | ... | @@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> { |
| 23 | db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL); | 23 | db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL); |
| 24 | CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id)); | 24 | CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id)); |
| 25 | DROP TRIGGER IF EXISTS guest_no_groups; | 25 | DROP TRIGGER IF EXISTS guest_no_groups; |
| 26 | CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; | 26 | CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END; |
| 27 | CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; | 27 | DROP TRIGGER IF EXISTS guest_no_group_updates; |
| 28 | CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END; | 28 | CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END; |
| 29 | CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?; | 29 | CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?; |
| 30 | Ok(()) | 30 | Ok(()) |
| ... | @@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) | ... | @@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) |
| 650 | if known.len() > 1 { | 650 | if known.len() > 1 { |
| 651 | return Err(Error::new( | 651 | return Err(Error::new( |
| 652 | 409, | 652 | 409, |
| 653 | "Several accounts have signed in with that name. Open the right one under guests and join it to knife-native.", | 653 | "Several accounts have signed in with that name. Find them under guests.", |
| 654 | )); | 654 | )); |
| 655 | } | 655 | } |
| 656 | let username = match known.first() { | 656 | let username = match known.first() { |
| ... | @@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) | ... | @@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str) |
| 673 | id | 673 | id |
| 674 | } | 674 | } |
| 675 | }; | 675 | }; |
| 676 | tx.execute( | ||
| 677 | "INSERT OR IGNORE INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", | ||
| 678 | [&id], | ||
| 679 | )?; | ||
| 680 | tx.commit()?; | 676 | tx.commit()?; |
| 681 | Ok(json!({"id":id,"created":existing.is_none()})) | 677 | Ok(json!({"id":id,"created":existing.is_none()})) |
| 682 | } | 678 | } |
| ... | @@ -1185,7 +1181,7 @@ mod tests { | ... | @@ -1185,7 +1181,7 @@ mod tests { |
| 1185 | } | 1181 | } |
| 1186 | 1182 | ||
| 1187 | #[test] | 1183 | #[test] |
| 1188 | fn identities_stay_separate_and_guests_can_only_join_native_access() { | 1184 | fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() { |
| 1189 | let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4())); | 1185 | let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4())); |
| 1190 | let auth = auth::Store::new( | 1186 | let auth = auth::Store::new( |
| 1191 | &path, | 1187 | &path, |
| ... | @@ -1234,9 +1230,6 @@ mod tests { | ... | @@ -1234,9 +1230,6 @@ mod tests { |
| 1234 | db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first]) | 1230 | db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first]) |
| 1235 | .is_err() | 1231 | .is_err() |
| 1236 | ); | 1232 | ); |
| 1237 | db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap(); | ||
| 1238 | assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native"); | ||
| 1239 | assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err()); | ||
| 1240 | assert!(auth::set_password(&db, &first, "a-password-hash").is_err()); | 1233 | assert!(auth::set_password(&db, &first, "a-password-hash").is_err()); |
| 1241 | assert!( | 1234 | assert!( |
| 1242 | db.execute( | 1235 | db.execute( |
| ... | @@ -1261,10 +1254,6 @@ mod tests { | ... | @@ -1261,10 +1254,6 @@ mod tests { |
| 1261 | std::fs::remove_dir_all(path).unwrap(); | 1254 | std::fs::remove_dir_all(path).unwrap(); |
| 1262 | } | 1255 | } |
| 1263 | 1256 | ||
| 1264 | fn members(auth: &auth::Store, id: &str) -> Value { | ||
| 1265 | auth::user(&auth.db.lock().unwrap(), id).unwrap()["groups"].clone() | ||
| 1266 | } | ||
| 1267 | |||
| 1268 | #[test] | 1257 | #[test] |
| 1269 | fn added_guests_are_the_accounts_their_first_sign_in_finds() { | 1258 | fn added_guests_are_the_accounts_their_first_sign_in_finds() { |
| 1270 | let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4())); | 1259 | let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4())); |
| ... | @@ -1279,10 +1268,8 @@ mod tests { | ... | @@ -1279,10 +1268,8 @@ mod tests { |
| 1279 | let added = grant(&auth, provider, Some(subject), "octo").unwrap(); | 1268 | let added = grant(&auth, provider, Some(subject), "octo").unwrap(); |
| 1280 | let id = string(&added["id"]); | 1269 | let id = string(&added["id"]); |
| 1281 | assert_eq!(added["created"], true); | 1270 | assert_eq!(added["created"], true); |
| 1282 | assert_eq!(members(&auth, id)[0]["name"], "knife-native"); | ||
| 1283 | assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id); | 1271 | assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id); |
| 1284 | assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false); | 1272 | assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false); |
| 1285 | assert_eq!(members(&auth, id).as_array().unwrap().len(), 1); | ||
| 1286 | } | 1273 | } |
| 1287 | 1274 | ||
| 1288 | let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned(); | 1275 | let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned(); |
| ... | @@ -1294,19 +1281,15 @@ mod tests { | ... | @@ -1294,19 +1281,15 @@ mod tests { |
| 1294 | assert_eq!(identities(&reserved), 0); | 1281 | assert_eq!(identities(&reserved), 0); |
| 1295 | let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap(); | 1282 | let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap(); |
| 1296 | assert_ne!(stranger, reserved); | 1283 | assert_ne!(stranger, reserved); |
| 1297 | assert_eq!(members(&auth, &stranger), json!([])); | ||
| 1298 | claim(&auth, "sub-1", "alice").unwrap(); | 1284 | claim(&auth, "sub-1", "alice").unwrap(); |
| 1299 | assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved); | 1285 | assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved); |
| 1300 | assert_eq!(members(&auth, &reserved)[0]["name"], "knife-native"); | ||
| 1301 | claim(&auth, "sub-3", "alice").unwrap(); | 1286 | claim(&auth, "sub-3", "alice").unwrap(); |
| 1302 | let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap(); | 1287 | let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap(); |
| 1303 | assert_ne!(impostor, reserved); | 1288 | assert_ne!(impostor, reserved); |
| 1304 | assert_eq!(members(&auth, &impostor), json!([])); | ||
| 1305 | 1289 | ||
| 1306 | let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap(); | 1290 | let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap(); |
| 1307 | let added = grant(&auth, "astheno", None, "carol").unwrap(); | 1291 | let added = grant(&auth, "astheno", None, "carol").unwrap(); |
| 1308 | assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false))); | 1292 | assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false))); |
| 1309 | assert_eq!(members(&auth, &earlier)[0]["name"], "knife-native"); | ||
| 1310 | account(&auth, "astheno", "sub-5", "dave", None).unwrap(); | 1293 | account(&auth, "astheno", "sub-5", "dave", None).unwrap(); |
| 1311 | account(&auth, "astheno", "sub-6", "dave", None).unwrap(); | 1294 | account(&auth, "astheno", "sub-6", "dave", None).unwrap(); |
| 1312 | assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409); | 1295 | assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409); |
dashboard/src/knife-restricted.html deleted-28| ... | @@ -1,28 +0,0 @@ | ||
| 1 | <!doctype html> | ||
| 2 | <html lang="en"> | ||
| 3 | <meta charset="utf-8"> | ||
| 4 | <meta name="viewport" content="width=device-width, initial-scale=1"> | ||
| 5 | <title>The Knife's Native UI is restricted</title> | ||
| 6 | <style> | ||
| 7 | :root { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; background: #faf8fc; color: #29232e; } | ||
| 8 | body { margin: 0; min-height: 100svh; display: grid; place-items: center; } | ||
| 9 | main { box-sizing: border-box; width: min(100%, 42rem); padding: 2rem; text-align: center; } | ||
| 10 | h1 { margin: 0 0 1rem; font-size: clamp(1.5rem, 4vw, 2rem); line-height: 1.25; } | ||
| 11 | p { margin: 0 auto 1.75rem; max-width: 34rem; color: #62596b; } | ||
| 12 | a { display: inline-block; padding: .625rem 1.375rem; border: 1px solid transparent; border-radius: .375rem; background: #6b3e9e; color: white; text-decoration: none; font-weight: 600; } | ||
| 13 | a:hover { background: #552a84; } | ||
| 14 | a:focus-visible { outline: 3px solid #8857be; outline-offset: 4px; } | ||
| 15 | @media (prefers-color-scheme: dark) { | ||
| 16 | :root { background: #1c1821; color: #f4eff8; } | ||
| 17 | p { color: #c2b6cd; } | ||
| 18 | a { background: #b495db; color: #21162e; } | ||
| 19 | a:hover { background: #c8ade9; } | ||
| 20 | a:focus-visible { outline-color: #d7b8ff; } | ||
| 21 | } | ||
| 22 | </style> | ||
| 23 | <main> | ||
| 24 | <h1>The Knife's Native UI is restricted</h1> | ||
| 25 | <p>This part of the code is sensitive. Please contact clover to request private access.</p> | ||
| 26 | <a href="/-/login">Login</a> | ||
| 27 | </main> | ||
| 28 | </html> | ||
dashboard/src/shale_page.rs+4-193| ... | @@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) { | ... | @@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) { |
| 347 | } | 347 | } |
| 348 | } | 348 | } |
| 349 | 349 | ||
| 350 | fn knife_uri(path: &str) -> bool { | ||
| 351 | let mut decoded = path.to_owned(); | ||
| 352 | for _ in 0..5 { | ||
| 353 | if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) { | ||
| 354 | return true; | ||
| 355 | } | ||
| 356 | let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned(); | ||
| 357 | if next == decoded { break; } | ||
| 358 | decoded = next; | ||
| 359 | } | ||
| 360 | false | ||
| 361 | } | ||
| 362 | |||
| 363 | fn session_account(html: &str) -> Option<String> { | ||
| 364 | let document = scraper::Html::parse_document(html); | ||
| 365 | let selector = scraper::Selector::parse("body > header.usa-header--basic nav a.usa-nav-link[href^='/~']").unwrap(); | ||
| 366 | let mut accounts = document.select(&selector); | ||
| 367 | let name = accounts.next()?.attr("href")?.strip_prefix("/~")?; | ||
| 368 | let name = percent_encoding::percent_decode_str(name).decode_utf8().ok()?; | ||
| 369 | (accounts.next().is_none() && !name.is_empty() && !name.contains(['/', '\\', '?', '#'])) | ||
| 370 | .then(|| name.into_owned()) | ||
| 371 | } | ||
| 372 | |||
| 373 | fn restricted(authenticate: bool) -> Response { | ||
| 374 | (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [ | ||
| 375 | ("content-type", "text/html; charset=utf-8"), | ||
| 376 | ("cache-control", "private, no-store"), | ||
| 377 | ("vary", "Cookie, Authorization"), | ||
| 378 | ("referrer-policy", "same-origin"), | ||
| 379 | ("x-content-type-options", "nosniff"), | ||
| 380 | ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"), | ||
| 381 | ("www-authenticate", "Basic realm=\"The Knife\""), | ||
| 382 | ], include_str!("knife-restricted.html")).into_response() | ||
| 383 | } | ||
| 384 | |||
| 385 | fn redact_knife(html: &str, policy: &Value) -> Result<String> { | ||
| 386 | let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect(); | ||
| 387 | let document = scraper::Html::parse_document(html); | ||
| 388 | let diff = policy["safeDiff"].as_str(); | ||
| 389 | if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 { | ||
| 390 | return Err(Error::new(403, "The Knife's Native UI is restricted")); | ||
| 391 | } | ||
| 392 | let links = scraper::Selector::parse("a[href]").unwrap(); | ||
| 393 | let rows: Vec<bool> = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| { | ||
| 394 | row.select(&links).any(|link| link.attr("href").is_some_and(|href| { | ||
| 395 | href.split(['/', '?', '#', '.']).any(|part| commits.contains(part)) | ||
| 396 | })) | ||
| 397 | }).collect(); | ||
| 398 | let mut row_index = 0; | ||
| 399 | Ok(lol_html::rewrite_str(html, RewriteStrSettings::new() | ||
| 400 | .append_element_content_handler(element!("#m-code", |element| { | ||
| 401 | if let Some(diff) = diff { | ||
| 402 | let diff = diff.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;"); | ||
| 403 | element.set_inner_content(&format!("<pre><code>{diff}</code></pre>"), ContentType::Html); | ||
| 404 | } | ||
| 405 | Ok(()) | ||
| 406 | })) | ||
| 407 | .append_element_content_handler(element!("tr", |element| { | ||
| 408 | if *rows.get(row_index).unwrap_or(&true) { element.remove(); } | ||
| 409 | row_index += 1; | ||
| 410 | Ok(()) | ||
| 411 | })) | ||
| 412 | )?) | ||
| 413 | } | ||
| 414 | |||
| 415 | /// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch. | ||
| 416 | async fn clone(request: Value, body: Body) -> Result<Response> { | ||
| 417 | use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; | ||
| 418 | let (_, stream) = host::open(request).await?; | ||
| 419 | let (reader, mut writer) = stream.into_split(); | ||
| 420 | tokio::spawn(async move { | ||
| 421 | let mut body = body.into_data_stream(); | ||
| 422 | while let Some(Ok(chunk)) = body.next().await { | ||
| 423 | if writer.write_all(&chunk).await.is_err() { return; } | ||
| 424 | } | ||
| 425 | let _ = writer.shutdown().await; | ||
| 426 | }); | ||
| 427 | let mut reader = tokio::io::BufReader::new(reader); | ||
| 428 | let mut response = Response::builder(); | ||
| 429 | loop { | ||
| 430 | let mut line = String::new(); | ||
| 431 | if reader.read_line(&mut line).await? == 0 { | ||
| 432 | return Err(Error::new(502, "The Knife clone ended early.")); | ||
| 433 | } | ||
| 434 | let line = line.trim_end(); | ||
| 435 | if line.is_empty() { break; } | ||
| 436 | let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?; | ||
| 437 | let value = value.trim(); | ||
| 438 | if name.eq_ignore_ascii_case("status") { | ||
| 439 | response = response.status(value.split_whitespace().next().unwrap_or_default().parse::<u16>()?); | ||
| 440 | } else { | ||
| 441 | response = response.header(name, value); | ||
| 442 | } | ||
| 443 | } | ||
| 444 | let stream = futures::stream::unfold(reader, |mut reader| async move { | ||
| 445 | let mut buffer = vec![0; 64 * 1024]; | ||
| 446 | match reader.read(&mut buffer).await { | ||
| 447 | Ok(0) => None, | ||
| 448 | Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) } | ||
| 449 | Err(error) => Some((Err(error), reader)), | ||
| 450 | } | ||
| 451 | }); | ||
| 452 | Ok(response.body(Body::from_stream(stream))?) | ||
| 453 | } | ||
| 454 | |||
| 455 | pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | 350 | pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 456 | let (mut parts, body) = request.into_parts(); | 351 | let (mut parts, body) = request.into_parts(); |
| 457 | let host = parts | 352 | let host = parts |
| ... | @@ -484,52 +379,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -484,52 +379,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 484 | return Err(Error::new(403, "Open Shale to continue.")); | 379 | return Err(Error::new(403, "Open Shale to continue.")); |
| 485 | } | 380 | } |
| 486 | let origin = url::Url::parse(&format!("https://{host}{uri}"))?; | 381 | let origin = url::Url::parse(&format!("https://{host}{uri}"))?; |
| 487 | let knife = knife_uri(uri.path()); | ||
| 488 | let mut target = match &app.internal { | ||
| 489 | Some((base, _)) => format!("{base}services/{service}{uri}"), | ||
| 490 | None => format!("http://{upstream}{uri}"), | ||
| 491 | }; | ||
| 492 | let mut knife_policy = Value::Null; | ||
| 493 | if knife { | ||
| 494 | let mut account = String::new(); | ||
| 495 | if let Some(cookie) = parts.headers.get("cookie") { | ||
| 496 | // Opening settings rotates the session's CSRF token, invalidating pending forms. | ||
| 497 | let index = match &app.internal { | ||
| 498 | Some((base, _)) => format!("{base}services/{service}/"), | ||
| 499 | None => format!("http://{upstream}/"), | ||
| 500 | }; | ||
| 501 | let mut identity = app.shale_http.get(index).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3)); | ||
| 502 | if app.internal.is_none() { identity = identity.header("host", host); } | ||
| 503 | for name in ["x-forwarded-for", "x-real-ip", "user-agent"] { | ||
| 504 | if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); } | ||
| 505 | } | ||
| 506 | if let Ok(response) = identity.send().await | ||
| 507 | && response.status() == StatusCode::OK | ||
| 508 | && let Ok(body) = response.text().await | ||
| 509 | { | ||
| 510 | account = session_account(&body).unwrap_or_default(); | ||
| 511 | } | ||
| 512 | } | ||
| 513 | knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await { | ||
| 514 | Ok(policy) => policy, | ||
| 515 | Err(_) => return Ok(restricted(false)), | ||
| 516 | }; | ||
| 517 | if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } | ||
| 518 | if knife_policy["clone"] == true { | ||
| 519 | let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default(); | ||
| 520 | let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")}); | ||
| 521 | return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true))); | ||
| 522 | } | ||
| 523 | if let Some(uri) = knife_policy["uri"].as_str() { | ||
| 524 | target = match &app.internal { | ||
| 525 | Some((base, _)) => format!("{base}services/{service}{uri}"), | ||
| 526 | None => format!("http://{upstream}{uri}"), | ||
| 527 | }; | ||
| 528 | } | ||
| 529 | if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() { | ||
| 530 | return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response()); | ||
| 531 | } | ||
| 532 | } | ||
| 533 | let profiles = profiles(&app.auth)?; | 382 | let profiles = profiles(&app.auth)?; |
| 534 | if matches!(parts.method, Method::GET | Method::HEAD) | 383 | if matches!(parts.method, Method::GET | Method::HEAD) |
| 535 | && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref()) | 384 | && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref()) |
| ... | @@ -544,6 +393,10 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -544,6 +393,10 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 544 | ) | 393 | ) |
| 545 | .into_response()); | 394 | .into_response()); |
| 546 | } | 395 | } |
| 396 | let target = match &app.internal { | ||
| 397 | Some((base, _)) => format!("{base}services/{service}{uri}"), | ||
| 398 | None => format!("http://{upstream}{uri}"), | ||
| 399 | }; | ||
| 547 | if app.internal.is_some() { | 400 | if app.internal.is_some() { |
| 548 | parts.headers.remove("host"); | 401 | parts.headers.remove("host"); |
| 549 | } | 402 | } |
| ... | @@ -578,10 +431,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -578,10 +431,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 578 | let mut headers = upstream_response.headers().clone(); | 431 | let mut headers = upstream_response.headers().clone(); |
| 579 | strip_hop_headers(&mut headers); | 432 | strip_hop_headers(&mut headers); |
| 580 | headers.remove("status"); | 433 | headers.remove("status"); |
| 581 | if knife { | ||
| 582 | headers.insert("cache-control", "private, no-store".parse().unwrap()); | ||
| 583 | headers.insert("vary", "Cookie, Authorization".parse().unwrap()); | ||
| 584 | } | ||
| 585 | let html = parts.method != Method::HEAD | 434 | let html = parts.method != Method::HEAD |
| 586 | && headers | 435 | && headers |
| 587 | .get("content-type") | 436 | .get("content-type") |
| ... | @@ -637,14 +486,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -637,14 +486,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 637 | profiles.insert(name.clone(), profile); | 486 | profiles.insert(name.clone(), profile); |
| 638 | } | 487 | } |
| 639 | } | 488 | } |
| 640 | let redacted; | ||
| 641 | let text = if knife && knife_policy["allowed"] != true { | ||
| 642 | redacted = match redact_knife(text, &knife_policy) { | ||
| 643 | Ok(html) => html, | ||
| 644 | Err(_) => return Ok(restricted(false)), | ||
| 645 | }; | ||
| 646 | redacted.as_str() | ||
| 647 | } else { text }; | ||
| 648 | let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?; | 489 | let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?; |
| 649 | for name in [ | 490 | for name in [ |
| 650 | "content-length", | 491 | "content-length", |
| ... | @@ -661,9 +502,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -661,9 +502,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 661 | return Ok((status, headers, rewritten).into_response()); | 502 | return Ok((status, headers, rewritten).into_response()); |
| 662 | } | 503 | } |
| 663 | } | 504 | } |
| 664 | if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) { | ||
| 665 | return Ok(restricted(false)); | ||
| 666 | } | ||
| 667 | let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) | 505 | let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) |
| 668 | .chain(upstream_response.bytes_stream()); | 506 | .chain(upstream_response.bytes_stream()); |
| 669 | Ok((status, headers, Body::from_stream(stream)).into_response()) | 507 | Ok((status, headers, Body::from_stream(stream)).into_response()) |
| ... | @@ -675,33 +513,6 @@ mod tests { | ... | @@ -675,33 +513,6 @@ mod tests { |
| 675 | use rusqlite::params as sql; | 513 | use rusqlite::params as sql; |
| 676 | use scraper::{Html, Selector}; | 514 | use scraper::{Html, Selector}; |
| 677 | 515 | ||
| 678 | #[test] | ||
| 679 | fn session_account_uses_only_unambiguous_site_navigation() { | ||
| 680 | let header = |name: &str| format!("<header class='usa-header usa-header--basic'><nav><a class='usa-nav-link' href='/~{name}'>account</a></nav></header>"); | ||
| 681 | assert_eq!(session_account(&header("clover")), Some("clover".into())); | ||
| 682 | assert_eq!(session_account(&header("guest%2Dname")), Some("guest-name".into())); | ||
| 683 | assert_eq!(session_account(&format!("{}<main><header class='usa-header--basic'><nav><a class='usa-nav-link' href='/~clover'>fake</a></nav></header></main>", header("guest"))), Some("guest".into())); | ||
| 684 | assert_eq!(session_account(&format!("{}{}", header("guest"), header("clover"))), None); | ||
| 685 | for name in ["", "%FF", "clover%2Frepo", "clover%5Crepo", "clover?fake", "clover#fake"] { | ||
| 686 | assert_eq!(session_account(&header(name)), None, "{name}"); | ||
| 687 | } | ||
| 688 | assert_eq!(session_account("<a class='usa-nav-link' href='/~clover'>fake</a>"), None); | ||
| 689 | } | ||
| 690 | |||
| 691 | #[test] | ||
| 692 | fn native_content_is_removed_from_both_diff_styles_and_history_rows() { | ||
| 693 | let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"}); | ||
| 694 | let html = "<html><body id=page-commit><table><tr><td><a href='/the-knife/commit/deadbeef'>PRIVATE MESSAGE</a></td></tr><tr><td>PUBLIC ROW</td></tr></table><div id=m-code><div id=m-changedfiles>PRIVATE PATHS</div><details class=m-usa-accordion><summary><code>knife/private.rs</code></summary><table><tr><td>PRIVATE SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>src/public.rs</code></summary><table><tr><td>PUBLIC SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>nested/Cargo.lock</code></summary><table><tr><td>PRIVATE LOCK</td></tr></table></details></div></body></html>"; | ||
| 695 | let output = redact_knife(html, &policy).unwrap(); | ||
| 696 | assert!(!output.contains("PRIVATE"), "{output}"); | ||
| 697 | assert!(output.contains("PUBLIC ROW"), "{output}"); | ||
| 698 | assert!(output.contains("PUBLIC SOURCE"), "{output}"); | ||
| 699 | assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1); | ||
| 700 | assert!(redact_knife("<div>unknown diff markup</div>", &policy).is_err()); | ||
| 701 | for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); } | ||
| 702 | assert!(!knife_uri("/snowbound/commit/native")); | ||
| 703 | } | ||
| 704 | |||
| 705 | #[test] | 516 | #[test] |
| 706 | fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() { | 517 | fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() { |
| 707 | let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4())); | 518 | let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4())); |
dashboard/src/users.rs+3-9| ... | @@ -158,13 +158,10 @@ pub async fn route( | ... | @@ -158,13 +158,10 @@ pub async fn route( |
| 158 | for id in &users { | 158 | for id in &users { |
| 159 | uuid(id)?; | 159 | uuid(id)?; |
| 160 | let user = auth::user(&transaction, id)?; | 160 | let user = auth::user(&transaction, id)?; |
| 161 | let guest_groups = if guest::is_guest(&user) { | 161 | if guest::is_guest(&user) || user["kind"] == "ai" { |
| 162 | add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true)) | ||
| 163 | } else { false }; | ||
| 164 | if guest_groups || user["kind"] == "ai" { | ||
| 165 | return Err(Error::new( | 162 | return Err(Error::new( |
| 166 | 400, | 163 | 400, |
| 167 | "Guests can only join knife-native.", | 164 | "Guests can use Shale only. Select regular accounts to edit groups.", |
| 168 | )); | 165 | )); |
| 169 | } | 166 | } |
| 170 | for group in &add { | 167 | for group in &add { |
| ... | @@ -304,7 +301,7 @@ pub async fn route( | ... | @@ -304,7 +301,7 @@ pub async fn route( |
| 304 | let transaction = db.transaction()?; | 301 | let transaction = db.transaction()?; |
| 305 | let mut user = auth::user(&transaction, id)?; | 302 | let mut user = auth::user(&transaction, id)?; |
| 306 | if guest::is_guest(&user) | 303 | if guest::is_guest(&user) |
| 307 | && matches!(parts, [_, "password"]) | 304 | && matches!(parts, [_, "password"] | [_, "groups", _]) |
| 308 | && method != Method::GET | 305 | && method != Method::GET |
| 309 | { | 306 | { |
| 310 | return Err(Error::new( | 307 | return Err(Error::new( |
| ... | @@ -354,9 +351,6 @@ pub async fn route( | ... | @@ -354,9 +351,6 @@ pub async fn route( |
| 354 | .optional()?; | 351 | .optional()?; |
| 355 | let name = name | 352 | let name = name |
| 356 | .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?; | 353 | .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?; |
| 357 | if guest::is_guest(&user) && name != "knife-native" { | ||
| 358 | return Err(Error::new(400, "Guests can only join knife-native.")); | ||
| 359 | } | ||
| 360 | if own && method == Method::DELETE && name == "infra-admin" { | 354 | if own && method == Method::DELETE && name == "infra-admin" { |
| 361 | return Err(Error::new( | 355 | return Err(Error::new( |
| 362 | 400, | 356 | 400, |
dashboard/web/pages/Users.tsx+2-3| ... | @@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) { | ... | @@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) { |
| 77 | 77 | ||
| 78 | /** "30 members · opens Jellyfin, Navidrome" */ | 78 | /** "30 members · opens Jellyfin, Navidrome" */ |
| 79 | function groupTip(group: string, d: Data) { | 79 | function groupTip(group: string, d: Data) { |
| 80 | if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`; | ||
| 81 | const more = adds(group, d.services); | 80 | const more = adds(group, d.services); |
| 82 | const members = d.users.filter((user) => names(user).includes(group)).length; | 81 | const members = d.users.filter((user) => names(user).includes(group)).length; |
| 83 | const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : []; | 82 | const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : []; |
| ... | @@ -230,7 +229,7 @@ function List(props: { data: Resource<Data>; ready: Accessor<Data | undefined>; | ... | @@ -230,7 +229,7 @@ function List(props: { data: Resource<Data>; ready: Accessor<Data | undefined>; |
| 230 | guestMenu.hidePopover(); | 229 | guestMenu.hidePopover(); |
| 231 | showTextDialog({ | 230 | showTextDialog({ |
| 232 | title: `Add ${guest.name} guest`, | 231 | title: `Add ${guest.name} guest`, |
| 233 | description: "They can sign in to Shale and open The Knife’s Native UI.", | 232 | description: "They can sign in to Shale.", |
| 234 | label: guest.field, | 233 | label: guest.field, |
| 235 | placeholder: guest.placeholder, | 234 | placeholder: guest.placeholder, |
| 236 | body: provider === "astheno" ? () => <div class="field"><span class="hint">Astheno can’t look names up. The first person to sign in with this username gets the access.</span></div> : undefined, | 235 | body: provider === "astheno" ? () => <div class="field"><span class="hint">Astheno can’t look names up. The first person to sign in with this username gets the access.</span></div> : undefined, |
| ... | @@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) { | ... | @@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) { |
| 638 | <div class="toggles" role="group" aria-label="Groups"> | 637 | <div class="toggles" role="group" aria-label="Groups"> |
| 639 | <For each={props.data.groups}> | 638 | <For each={props.data.groups}> |
| 640 | {(group) => ( | 639 | {(group) => ( |
| 641 | <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" && group.name !== "knife-native" || busy() === group.id} | 640 | <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" || busy() === group.id} |
| 642 | data-tip={groupTip(group.name, props.data)} onClick={() => toggleGroup(group)}> | 641 | data-tip={groupTip(group.name, props.data)} onClick={() => toggleGroup(group)}> |
| 643 | {group.name} | 642 | {group.name} |
| 644 | </button> | 643 | </button> |
flake.nix-1| ... | @@ -5,7 +5,6 @@ | ... | @@ -5,7 +5,6 @@ |
| 5 | 5 | ||
| 6 | outputs = { self, nixpkgs, ... }: { | 6 | outputs = { self, nixpkgs, ... }: { |
| 7 | packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { }; | 7 | packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { }; |
| 8 | packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { }; | ||
| 9 | packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { }; | 8 | packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { }; |
| 10 | packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { }; | 9 | packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { }; |
| 11 | packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image; | 10 | packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image; |
nixos/configuration.nix+2-3| ... | @@ -6,12 +6,11 @@ let | ... | @@ -6,12 +6,11 @@ let |
| 6 | proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; | 6 | proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; |
| 7 | hostTools = lib.fileset.toSource { | 7 | hostTools = lib.fileset.toSource { |
| 8 | root = ../.; | 8 | root = ../.; |
| 9 | fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; | 9 | fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; |
| 10 | }; | 10 | }; |
| 11 | computerAhk = pkgs.callPackage ./computer-ahk.nix { }; | 11 | computerAhk = pkgs.callPackage ./computer-ahk.nix { }; |
| 12 | computerWindows = pkgs.callPackage ./computer-windows.nix { }; | 12 | computerWindows = pkgs.callPackage ./computer-windows.nix { }; |
| 13 | nativePkl = pkgs.callPackage ./pkl.nix { }; | 13 | nativePkl = pkgs.callPackage ./pkl.nix { }; |
| 14 | hostPython = pkgs.callPackage ./host-python.nix { }; | ||
| 15 | screen = pkgs.callPackage ./vm-screen.nix { }; | 14 | screen = pkgs.callPackage ./vm-screen.nix { }; |
| 16 | secureOvmf = pkgs.OVMF.override { | 15 | secureOvmf = pkgs.OVMF.override { |
| 17 | secureBoot = true; | 16 | secureBoot = true; |
| ... | @@ -264,7 +263,7 @@ in | ... | @@ -264,7 +263,7 @@ in |
| 264 | SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables; | 263 | SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables; |
| 265 | }; | 264 | }; |
| 266 | serviceConfig = { | 265 | serviceConfig = { |
| 267 | ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py"; | 266 | ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py"; |
| 268 | Type = "notify"; | 267 | Type = "notify"; |
| 269 | Group = "snowglobe-dashboard"; | 268 | Group = "snowglobe-dashboard"; |
| 270 | StateDirectory = "snowglobe/host"; | 269 | StateDirectory = "snowglobe/host"; |
nixos/host-python.nix deleted-6| ... | @@ -1,6 +0,0 @@ | ||
| 1 | { python3, git }: | ||
| 2 | python3.buildEnv.override { | ||
| 3 | extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ]; | ||
| 4 | # shale-access serves anonymous Knife clones through git http-backend. | ||
| 5 | makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ]; | ||
| 6 | } | ||
nixos/mac-studio-sandbox.nix+2-2| ... | @@ -50,11 +50,11 @@ in | ... | @@ -50,11 +50,11 @@ in |
| 50 | enable = true; | 50 | enable = true; |
| 51 | hostName = "10.0.0.1"; | 51 | hostName = "10.0.0.1"; |
| 52 | exports = '' | 52 | exports = '' |
| 53 | ${mountPoint} ${workspace.clientAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) | 53 | ${mountPoint} ${workspace.guestAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) |
| 54 | ''; | 54 | ''; |
| 55 | }; | 55 | }; |
| 56 | environment.etc."exports.d/snowglobe-work.exports".text = '' | 56 | environment.etc."exports.d/snowglobe-work.exports".text = '' |
| 57 | ${mountPoint}/.work ${workspace.clientAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) | 57 | ${mountPoint}/.work ${workspace.guestAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) |
| 58 | ''; | 58 | ''; |
| 59 | systemd.services.mac-work-share-check = { | 59 | systemd.services.mac-work-share-check = { |
| 60 | requires = [ "zfs-import-work.service" ]; | 60 | requires = [ "zfs-import-work.service" ]; |
nixos/mac-vm.nix+2-2| ... | @@ -5,7 +5,7 @@ let | ... | @@ -5,7 +5,7 @@ let |
| 5 | in { | 5 | in { |
| 6 | systemd.services.snowglobe-host = { | 6 | systemd.services.snowglobe-host = { |
| 7 | environment.SNOWGLOBE_MAC_VM = builtins.toJSON { | 7 | environment.SNOWGLOBE_MAC_VM = builtins.toJSON { |
| 8 | address = workspace.clientAddress; | 8 | address = workspace.hostAddress; |
| 9 | port = 22; | 9 | port = 22; |
| 10 | user = "clo"; | 10 | user = "clo"; |
| 11 | identity = "/var/lib/snowglobe/mac-vm/ssh_key"; | 11 | identity = "/var/lib/snowglobe/mac-vm/ssh_key"; |
| ... | @@ -15,6 +15,6 @@ in { | ... | @@ -15,6 +15,6 @@ in { |
| 15 | { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; } | 15 | { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; } |
| 16 | ]; | 16 | ]; |
| 17 | }; | 17 | }; |
| 18 | serviceConfig.IPAddressAllow = [ "${workspace.clientAddress}/32" ]; | 18 | serviceConfig.IPAddressAllow = [ "${workspace.hostAddress}/32" ]; |
| 19 | }; | 19 | }; |
| 20 | } | 20 | } |
nixos/snowglobe.nix+1-1| ... | @@ -29,7 +29,7 @@ in | ... | @@ -29,7 +29,7 @@ in |
| 29 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' | 29 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' |
| 30 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept | 30 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept |
| 31 | '') config.networking.interfaces.enp4s0.ipv4.addresses + '' | 31 | '') config.networking.interfaces.enp4s0.ipv4.addresses + '' |
| 32 | iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).clientAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept | 32 | iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).guestAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept |
| 33 | ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept | 33 | ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept |
| 34 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept | 34 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept |
| 35 | ''; | 35 | ''; |
service/shale/prepare.py+42-6| ... | @@ -9,13 +9,8 @@ import sys | ... | @@ -9,13 +9,8 @@ import sys |
| 9 | 9 | ||
| 10 | data = json.load(sys.stdin) | 10 | data = json.load(sys.stdin) |
| 11 | root = Path(data["hostRoot"]) | 11 | root = Path(data["hostRoot"]) |
| 12 | for config in (root / "repositories_owned").glob("*/config"): | ||
| 13 | text = config.read_text() | ||
| 14 | if not re.search(r"(?im)^\s*receivepack\s*=", text): | ||
| 15 | # Shale authorizes pushes before CGI; its own repository initializer enables this. | ||
| 16 | with config.open("a") as file: | ||
| 17 | file.write("\n[http]\n\treceivepack = true\n") | ||
| 18 | path = root / "data/astheno.shale.db" | 12 | path = root / "data/astheno.shale.db" |
| 13 | webhooks = {} | ||
| 19 | if path.exists(): | 14 | if path.exists(): |
| 20 | domain = os.environ["SNOWGLOBE_DOMAIN"] | 15 | domain = os.environ["SNOWGLOBE_DOMAIN"] |
| 21 | old, new = "auth." + domain + "/realms/master", "snowglobe." + domain | 16 | old, new = "auth." + domain + "/realms/master", "snowglobe." + domain |
| ... | @@ -25,6 +20,47 @@ if path.exists(): | ... | @@ -25,6 +20,47 @@ if path.exists(): |
| 25 | raise ValueError("duplicate Shale identity; resolve it before moving the issuer") | 20 | raise ValueError("duplicate Shale identity; resolve it before moving the issuer") |
| 26 | count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount | 21 | count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount |
| 27 | db.commit() | 22 | db.commit() |
| 23 | webhooks = dict(db.execute( | ||
| 24 | "SELECT uuid, name FROM repositories WHERE name IN ('snowcord', 'snowcord-server')" | ||
| 25 | )) | ||
| 28 | assert db.execute("PRAGMA quick_check").fetchone() == ("ok",) | 26 | assert db.execute("PRAGMA quick_check").fetchone() == ("ok",) |
| 29 | db.close() | 27 | db.close() |
| 30 | print("Moved", count, "existing Shale identity bindings to Snowglobe") | 28 | print("Moved", count, "existing Shale identity bindings to Snowglobe") |
| 29 | |||
| 30 | executable = root / "data/shale-webhook" | ||
| 31 | source = Path(__file__).with_name("webhook.sh").read_bytes() | ||
| 32 | if not executable.exists() or executable.read_bytes() != source: | ||
| 33 | pending = executable.with_suffix(".pending") | ||
| 34 | pending.write_bytes(source) | ||
| 35 | if os.geteuid() == 0: | ||
| 36 | os.chown(pending, data["uid"], data["uid"]) | ||
| 37 | pending.replace(executable) | ||
| 38 | executable.chmod(0o750) | ||
| 39 | if os.geteuid() == 0: | ||
| 40 | os.chown(executable, data["uid"], data["uid"]) | ||
| 41 | |||
| 42 | target = "/data/shale-webhook" | ||
| 43 | for config in (root / "repositories_owned").glob("*/config"): | ||
| 44 | original = text = config.read_text() | ||
| 45 | if not re.search(r"(?im)^\s*receivepack\s*=", text): | ||
| 46 | # Shale authorizes pushes before CGI; its own repository initializer enables this. | ||
| 47 | text += "\n[http]\n\treceivepack = true\n" | ||
| 48 | text = re.sub(r'(?m)\n+\[shale "snowglobe"\]\n(?:[ \t].*\n?)*', "\n", text) | ||
| 49 | name = webhooks.get(config.parent.name) | ||
| 50 | if name: | ||
| 51 | text += f'\n[shale "snowglobe"]\n\trepository = {name}\n' | ||
| 52 | if text != original: | ||
| 53 | config.write_text(text) | ||
| 54 | |||
| 55 | hook = config.parent / "hooks/post-receive" | ||
| 56 | if name: | ||
| 57 | hook.parent.mkdir(exist_ok=True) | ||
| 58 | if os.path.lexists(hook): | ||
| 59 | if not hook.is_symlink() or os.readlink(hook) != target: | ||
| 60 | raise ValueError(f"refusing to replace existing hook: {hook}") | ||
| 61 | if not os.path.lexists(hook): | ||
| 62 | hook.symlink_to(target) | ||
| 63 | if os.geteuid() == 0: | ||
| 64 | os.lchown(hook, data["uid"], data["uid"]) | ||
| 65 | elif hook.is_symlink() and os.readlink(hook) == target: | ||
| 66 | hook.unlink() |
service/shale/service.pkl+3| ... | @@ -61,6 +61,8 @@ container { | ... | @@ -61,6 +61,8 @@ container { |
| 61 | ["MIRRORS_DISABLE_CLONE"] = "true" | 61 | ["MIRRORS_DISABLE_CLONE"] = "true" |
| 62 | ["MIRRORS_DISABLE_PULL_POLL"] = "true" | 62 | ["MIRRORS_DISABLE_PULL_POLL"] = "true" |
| 63 | } | 63 | } |
| 64 | ["SHALE_WEBHOOK_ENABLED"] = if (site.preview) "false" else "true" | ||
| 65 | ["SHALE_WEBHOOK_URL"] = "${secret.own.webhook_url}" | ||
| 64 | ["SESSION_SECRET"] = "${secret.own.session_secret}" | 66 | ["SESSION_SECRET"] = "${secret.own.session_secret}" |
| 65 | ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}" | 67 | ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}" |
| 66 | } | 68 | } |
| ... | @@ -69,3 +71,4 @@ container { | ... | @@ -69,3 +71,4 @@ container { |
| 69 | secrets { | 71 | secrets { |
| 70 | ["session_secret"] {} | 72 | ["session_secret"] {} |
| 71 | } | 73 | } |
| 74 | requiredSecrets { "webhook_url" } |
service/shale/webhook.sh created+144| ... | @@ -0,0 +1,144 @@ | ||
| 1 | #!/bin/sh | ||
| 2 | set -u | ||
| 3 | |||
| 4 | [ "${SHALE_WEBHOOK_ENABLED:-false}" = true ] || exit 0 | ||
| 5 | [ -n "${SHALE_WEBHOOK_URL:-}" ] || exit 0 | ||
| 6 | case "$SHALE_WEBHOOK_URL" in | ||
| 7 | */github|*/github/) webhook_url=${SHALE_WEBHOOK_URL%/}/ ;; | ||
| 8 | *) webhook_url=${SHALE_WEBHOOK_URL%/}/github/ ;; | ||
| 9 | esac | ||
| 10 | |||
| 11 | repository=$(git config --get shale.snowglobe.repository) || exit 0 | ||
| 12 | origin="https://${DOMAIN}/${repository}" | ||
| 13 | avatar="https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png" | ||
| 14 | |||
| 15 | json_string() { | ||
| 16 | awk 'BEGIN { ORS = "" } | ||
| 17 | { | ||
| 18 | if (NR > 1) printf "\\n" | ||
| 19 | for (i = 1; i <= length($0); i++) { | ||
| 20 | c = substr($0, i, 1) | ||
| 21 | if (c == "\\") printf "\\\\" | ||
| 22 | else if (c == "\"") printf "\\\"" | ||
| 23 | else if (c == "\t") printf "\\t" | ||
| 24 | else if (c == "\r") printf "\\r" | ||
| 25 | else if (c == sprintf("%c", 8)) printf "\\b" | ||
| 26 | else if (c == sprintf("%c", 12)) printf "\\f" | ||
| 27 | else printf "%s", c | ||
| 28 | } | ||
| 29 | }' | ||
| 30 | } | ||
| 31 | |||
| 32 | quote() { | ||
| 33 | printf '"' | ||
| 34 | printf '%s' "$1" | json_string | ||
| 35 | printf '"' | ||
| 36 | } | ||
| 37 | |||
| 38 | is_zero() { | ||
| 39 | case "$1" in | ||
| 40 | *[!0]*) return 1 ;; | ||
| 41 | *) return 0 ;; | ||
| 42 | esac | ||
| 43 | } | ||
| 44 | |||
| 45 | write_ref_payload() { | ||
| 46 | { | ||
| 47 | printf '{"ref":'; quote "$short_ref" | ||
| 48 | printf ',"ref_type":'; quote "$ref_type" | ||
| 49 | printf ',"repository":{"name":'; quote "$repository" | ||
| 50 | printf ',"full_name":'; quote "$repository" | ||
| 51 | printf ',"html_url":'; quote "$origin/" | ||
| 52 | printf '},"sender":{"login":'; quote "$actor" | ||
| 53 | printf ',"avatar_url":'; quote "$avatar" | ||
| 54 | printf ',"html_url":'; quote "$origin/" | ||
| 55 | printf '}}\n' | ||
| 56 | } >"$payload" | ||
| 57 | } | ||
| 58 | |||
| 59 | send() { | ||
| 60 | event=$1 | ||
| 61 | payload=$2 | ||
| 62 | delivery=$(printf '%s\n' "$repository:$event:$(date +%s):$$:$index" | git hash-object --stdin) | ||
| 63 | curl --fail --silent --show-error --max-time 10 \ | ||
| 64 | -H "Content-Type: application/json" \ | ||
| 65 | -H "User-Agent: Shale-Hookshot/1.0" \ | ||
| 66 | -H "X-GitHub-Event: $event" \ | ||
| 67 | -H "X-GitHub-Delivery: $delivery" \ | ||
| 68 | --data-binary "@$payload" "$webhook_url" >/dev/null | ||
| 69 | } | ||
| 70 | |||
| 71 | temporary=$(mktemp -d) | ||
| 72 | trap 'rm -rf "$temporary"' EXIT HUP INT TERM | ||
| 73 | index=0 | ||
| 74 | while read -r before after ref; do | ||
| 75 | index=$((index + 1)) | ||
| 76 | payload="$temporary/$index.json" | ||
| 77 | short_ref=${ref#refs/heads/} | ||
| 78 | if [ "$short_ref" = "$ref" ]; then | ||
| 79 | short_ref=${ref#refs/tags/} | ||
| 80 | ref_type=tag | ||
| 81 | else | ||
| 82 | ref_type=branch | ||
| 83 | fi | ||
| 84 | |||
| 85 | if is_zero "$after"; then | ||
| 86 | actor=${REMOTE_USER:-$(git show -s --format=%an "$before")} | ||
| 87 | write_ref_payload | ||
| 88 | send delete "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 | ||
| 89 | continue | ||
| 90 | fi | ||
| 91 | |||
| 92 | actor=${REMOTE_USER:-$(git show -s --format=%an "$after")} | ||
| 93 | if [ "$ref_type" = tag ]; then | ||
| 94 | if is_zero "$before"; then | ||
| 95 | write_ref_payload | ||
| 96 | send create "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 | ||
| 97 | fi | ||
| 98 | continue | ||
| 99 | fi | ||
| 100 | if is_zero "$before"; then | ||
| 101 | created=true | ||
| 102 | forced=false | ||
| 103 | commits=$(git rev-list --reverse "$after") | ||
| 104 | else | ||
| 105 | created=false | ||
| 106 | if git merge-base --is-ancestor "$before" "$after"; then forced=false; else forced=true; fi | ||
| 107 | commits=$(git rev-list --reverse "$before..$after") | ||
| 108 | fi | ||
| 109 | compare="$origin/commit/$after" | ||
| 110 | { | ||
| 111 | printf '{"ref":'; quote "$ref" | ||
| 112 | printf ',"before":'; quote "$before" | ||
| 113 | printf ',"after":'; quote "$after" | ||
| 114 | printf ',"created":%s,"deleted":false,"forced":%s' "$created" "$forced" | ||
| 115 | printf ',"compare":'; quote "$compare" | ||
| 116 | printf ',"repository":{"name":'; quote "$repository" | ||
| 117 | printf ',"full_name":'; quote "$repository" | ||
| 118 | printf ',"html_url":'; quote "$origin/" | ||
| 119 | printf '},"sender":{"login":'; quote "$actor" | ||
| 120 | printf ',"avatar_url":'; quote "$avatar" | ||
| 121 | printf ',"html_url":'; quote "$origin/" | ||
| 122 | printf '},"commits":[' | ||
| 123 | separator= | ||
| 124 | for commit in $commits; do | ||
| 125 | message=$(git show -s --format=%B "$commit") | ||
| 126 | author=$(git show -s --format=%an "$commit") | ||
| 127 | timestamp=$(git show -s --format=%aI "$commit") | ||
| 128 | printf '%s{"id":' "$separator"; quote "$commit" | ||
| 129 | printf ',"message":'; quote "$message" | ||
| 130 | printf ',"timestamp":'; quote "$timestamp" | ||
| 131 | printf ',"url":'; quote "$origin/commit/$commit" | ||
| 132 | printf ',"author":{"name":'; quote "$author" | ||
| 133 | printf ',"username":'; quote "$author" | ||
| 134 | printf '}}' | ||
| 135 | separator=, | ||
| 136 | done | ||
| 137 | printf '],"head_commit":{"id":'; quote "$after" | ||
| 138 | printf ',"message":'; quote "$(git show -s --format=%B "$after")" | ||
| 139 | printf ',"timestamp":'; quote "$(git show -s --format=%aI "$after")" | ||
| 140 | printf ',"url":'; quote "$origin/commit/$after" | ||
| 141 | printf '}}\n' | ||
| 142 | } >"$payload" | ||
| 143 | send push "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2 | ||
| 144 | done | ||
tools/dashboard-host.py-30| ... | @@ -16,7 +16,6 @@ import signal | ... | @@ -16,7 +16,6 @@ import signal |
| 16 | import socket | 16 | import socket |
| 17 | import struct | 17 | import struct |
| 18 | import subprocess | 18 | import subprocess |
| 19 | import sys | ||
| 20 | import threading | 19 | import threading |
| 21 | import time | 20 | import time |
| 22 | import vms | 21 | import vms |
| ... | @@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4 | ... | @@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4 |
| 45 | WORK_SLOTS = threading.BoundedSemaphore(WORKERS) | 44 | WORK_SLOTS = threading.BoundedSemaphore(WORKERS) |
| 46 | STREAM_SLOTS = threading.BoundedSemaphore(4) | 45 | STREAM_SLOTS = threading.BoundedSemaphore(4) |
| 47 | GUEST_SLOTS = threading.BoundedSemaphore(4) | 46 | GUEST_SLOTS = threading.BoundedSemaphore(4) |
| 48 | CLONE_SLOTS = threading.BoundedSemaphore(2) | ||
| 49 | VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) | 47 | VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) |
| 50 | INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" | 48 | INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" |
| 51 | 49 | ||
| ... | @@ -97,8 +95,6 @@ class Host: | ... | @@ -97,8 +95,6 @@ class Host: |
| 97 | "index.diff": {"dataset", "from", "to"}, | 95 | "index.diff": {"dataset", "from", "to"}, |
| 98 | "index.discard": {"dataset", "snapshot"}, | 96 | "index.discard": {"dataset", "snapshot"}, |
| 99 | "shale.metadata": {"repository", "accounts"}, | 97 | "shale.metadata": {"repository", "accounts"}, |
| 100 | "shale.access": {"service", "uri", "account", "authorization"}, | ||
| 101 | "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, | ||
| 102 | "shale.write": {"grant", "repository", "action", "arguments"}, | 98 | "shale.write": {"grant", "repository", "action", "arguments"}, |
| 103 | "host.sample": set(), | 99 | "host.sample": set(), |
| 104 | "host.usage": {"refresh"}, | 100 | "host.usage": {"refresh"}, |
| ... | @@ -113,10 +109,6 @@ class Host: | ... | @@ -113,10 +109,6 @@ class Host: |
| 113 | raise Rejected("Use only the fields required by this host operation.") | 109 | raise Rejected("Use only the fields required by this host operation.") |
| 114 | if operation == "copyparty.manage": | 110 | if operation == "copyparty.manage": |
| 115 | return copyparty.handle(request) | 111 | return copyparty.handle(request) |
| 116 | if operation == "shale.access": | ||
| 117 | return importlib.import_module("shale-access").access(request) | ||
| 118 | if operation == "shale.clone": | ||
| 119 | return importlib.import_module("shale-access").clone(request) | ||
| 120 | if operation == "shale.metadata": | 112 | if operation == "shale.metadata": |
| 121 | return shale_labels.metadata(request) | 113 | return shale_labels.metadata(request) |
| 122 | if operation == "shale.write": | 114 | if operation == "shale.write": |
| ... | @@ -473,20 +465,6 @@ def stream_upload(connection, request): | ... | @@ -473,20 +465,6 @@ def stream_upload(connection, request): |
| 473 | Path(temporary).unlink(missing_ok=True) | 465 | Path(temporary).unlink(missing_ok=True) |
| 474 | 466 | ||
| 475 | 467 | ||
| 476 | def stream_clone(connection, plan): | ||
| 477 | owner = os.stat(plan["objects"]) | ||
| 478 | # Service identities have numeric owners without passwd entries, which systemd's User= rejects. | ||
| 479 | command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", | ||
| 480 | *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", | ||
| 481 | "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), | ||
| 482 | *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), | ||
| 483 | "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", | ||
| 484 | sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] | ||
| 485 | send_response(connection, {"value": None}) | ||
| 486 | connection.setblocking(True) | ||
| 487 | subprocess.run(command, stdin=connection, stdout=connection) | ||
| 488 | |||
| 489 | |||
| 490 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): | 468 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 491 | with connection, contextlib.ExitStack() as resources: | 469 | with connection, contextlib.ExitStack() as resources: |
| 492 | connection.settimeout(65) | 470 | connection.settimeout(65) |
| ... | @@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): | ... | @@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 553 | return | 531 | return |
| 554 | finally: | 532 | finally: |
| 555 | streams.release() | 533 | streams.release() |
| 556 | if operation == "shale.clone": | ||
| 557 | if not CLONE_SLOTS.acquire(blocking=False): | ||
| 558 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") | ||
| 559 | try: | ||
| 560 | stream_clone(connection, host.handle(request)) | ||
| 561 | return | ||
| 562 | finally: | ||
| 563 | CLONE_SLOTS.release() | ||
| 564 | response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} | 534 | response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} |
| 565 | except Rejected as error: | 535 | except Rejected as error: |
| 566 | response = {"error": str(error), "status": 400} | 536 | response = {"error": str(error), "status": 400} |
tools/dashboard-shale-page-test.py+2-41| ... | @@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ... | @@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 6 | import json | 6 | import json |
| 7 | import os | 7 | import os |
| 8 | from pathlib import Path | 8 | from pathlib import Path |
| 9 | import re | ||
| 10 | import socket | 9 | import socket |
| 11 | import socketserver | ||
| 12 | import sqlite3 | 10 | import sqlite3 |
| 13 | import subprocess | 11 | import subprocess |
| 14 | import tempfile | 12 | import tempfile |
| ... | @@ -49,21 +47,12 @@ def main(): | ... | @@ -49,21 +47,12 @@ def main(): |
| 49 | issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>' | 47 | issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>' |
| 50 | sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>' | 48 | sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>' |
| 51 | blob = bytes(range(256)) * 65536 | 49 | blob = bytes(range(256)) * 65536 |
| 52 | csrf = 'initial' | ||
| 53 | 50 | ||
| 54 | class Fixture(BaseHTTPRequestHandler): | 51 | class Fixture(BaseHTTPRequestHandler): |
| 55 | protocol_version = 'HTTP/1.1' | 52 | protocol_version = 'HTTP/1.1' |
| 56 | 53 | ||
| 57 | def do_GET(self): | 54 | def do_GET(self): |
| 58 | nonlocal csrf | ||
| 59 | observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()})) | 55 | observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()})) |
| 60 | if self.path == '/': | ||
| 61 | account = b'<a class="usa-nav-link" href="/~fixture-member">~fixture-member</a>' if self.headers.get('Cookie') == 'SessionID=preserved' else b'' | ||
| 62 | return self.respond(b'<html><body><header class="usa-header--basic"><nav>' + account + b'</nav></header></body></html>', 'text/html') | ||
| 63 | if self.path == '/-/settings' or self.path.endswith('/issues/new'): | ||
| 64 | csrf += 'x' | ||
| 65 | account = '<body id="page-user-settings"><kbd>fixture-member</kbd>' if self.path == '/-/settings' else '<body>' | ||
| 66 | return self.respond(('<html>' + account + '<form method="post"><input type="hidden" name="csrf_token" value="' + csrf + '"></form></body></html>').encode(), 'text/html') | ||
| 67 | if self.path == '/binary': | 56 | if self.path == '/binary': |
| 68 | return self.respond(blob, 'application/octet-stream') | 57 | return self.respond(blob, 'application/octet-stream') |
| 69 | if self.path == '/json': | 58 | if self.path == '/json': |
| ... | @@ -86,8 +75,6 @@ def main(): | ... | @@ -86,8 +75,6 @@ def main(): |
| 86 | def do_POST(self): | 75 | def do_POST(self): |
| 87 | body = self.rfile.read(int(self.headers.get('Content-Length', 0))) | 76 | body = self.rfile.read(int(self.headers.get('Content-Length', 0))) |
| 88 | observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body)) | 77 | observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body)) |
| 89 | if self.path.endswith('/issues/new') and urllib.parse.parse_qs(body.decode()).get('csrf_token') != [csrf]: | ||
| 90 | return self.respond(b'E: unexpected csrf token', 'text/plain', 403) | ||
| 91 | self.respond(body, 'application/octet-stream', 201) | 78 | self.respond(body, 'application/octet-stream', 201) |
| 92 | 79 | ||
| 93 | def respond(self, body, content_type, status=200, headers=()): | 80 | def respond(self, body, content_type, status=200, headers=()): |
| ... | @@ -121,21 +108,7 @@ def main(): | ... | @@ -121,21 +108,7 @@ def main(): |
| 121 | subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True) | 108 | subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True) |
| 122 | data = root / 'data' | 109 | data = root / 'data' |
| 123 | data.mkdir() | 110 | data.mkdir() |
| 124 | class Host(socketserver.StreamRequestHandler): | 111 | environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')} |
| 125 | def handle(self): | ||
| 126 | request = json.loads(self.rfile.readline()) | ||
| 127 | value = {} | ||
| 128 | if request['operation'] == 'shale.access': | ||
| 129 | allowed = request['account'] == 'fixture-member' | ||
| 130 | value = {'allowed': allowed, 'deny': not allowed} | ||
| 131 | response = json.dumps({'value': value}).encode() | ||
| 132 | self.wfile.write(len(response).to_bytes(4, 'big') + response) | ||
| 133 | |||
| 134 | host = socketserver.UnixStreamServer(str(root / 'host.sock'), Host) | ||
| 135 | stack.callback(host.server_close) | ||
| 136 | stack.callback(host.shutdown) | ||
| 137 | threading.Thread(target=host.serve_forever, daemon=True).start() | ||
| 138 | environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_HOST_SOCKET': str(root / 'host.sock'), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')} | ||
| 139 | for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']: | 112 | for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']: |
| 140 | environment.pop(key, None) | 113 | environment.pop(key, None) |
| 141 | log = stack.enter_context((root / 'dashboard.log').open('w')) | 114 | log = stack.enter_context((root / 'dashboard.log').open('w')) |
| ... | @@ -253,22 +226,10 @@ def main(): | ... | @@ -253,22 +226,10 @@ def main(): |
| 253 | assert len(request('/large')[2]) == 9 * 1024 * 1024 | 226 | assert len(request('/large')[2]) == 9 * 1024 * 1024 |
| 254 | status, headers, body = request('/submit?keep=yes', body=b'field=unchanged') | 227 | status, headers, body = request('/submit?keep=yes', body=b'field=unchanged') |
| 255 | assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes' | 228 | assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes' |
| 256 | for path in ['/the-knife/issues/new', '/another-repo/issues/new']: | ||
| 257 | status, _, form = request(path) | ||
| 258 | assert status == 200 | ||
| 259 | csrf_field = re.search(rb'name="csrf_token" value="([^"]+)"', form)[1] | ||
| 260 | assert request('/the-knife/')[0] == 200 | ||
| 261 | submitted = b'csrf_token=' + csrf_field + b'&title=preserved' | ||
| 262 | status, _, body = request(path, body=submitted) | ||
| 263 | assert status == 201 and body == submitted, (path, status, body) | ||
| 264 | for invalid in [b'title=missing', b'csrf_token=invalid']: | ||
| 265 | assert request(path, body=invalid)[0] == 403 | ||
| 266 | assert not any(item[1] == '/-/settings' for item in observations) | ||
| 267 | assert request('/the-knife/issues/new', headers={'Cookie': ''})[0] == 403 | ||
| 268 | assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403 | 229 | assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403 |
| 269 | assert request('/auth/shale/page', direct=True)[0] == 403 | 230 | assert request('/auth/shale/page', direct=True)[0] == 403 |
| 270 | assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403 | 231 | assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403 |
| 271 | print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'csrf_preservation_and_rejection': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True) | 232 | print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True) |
| 272 | if args.browser_ready_file: | 233 | if args.browser_ready_file: |
| 273 | args.browser_ready_file.write_text(json.dumps({'origin': origin})) | 234 | args.browser_ready_file.write_text(json.dumps({'origin': origin})) |
| 274 | stop = args.browser_ready_file.with_suffix('.stop') | 235 | stop = args.browser_ready_file.with_suffix('.stop') |
tools/dashboard-stage.py+2-14| ... | @@ -21,7 +21,6 @@ import snowglobe | ... | @@ -21,7 +21,6 @@ import snowglobe |
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | OPERATIONS = frozenset({ | 23 | OPERATIONS = frozenset({ |
| 24 | "shale.access", "shale.clone", | ||
| 25 | "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", | 24 | "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", |
| 26 | "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", | 25 | "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", |
| 27 | "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", | 26 | "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", |
| ... | @@ -56,16 +55,6 @@ class PreviewHost(boundary.Host): | ... | @@ -56,16 +55,6 @@ class PreviewHost(boundary.Host): |
| 56 | actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} | 55 | actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} |
| 57 | if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: | 56 | if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: |
| 58 | raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") | 57 | raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") |
| 59 | if operation in {"shale.access", "shale.clone"}: | ||
| 60 | service = request.get("service", "") | ||
| 61 | if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): | ||
| 62 | raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") | ||
| 63 | linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) | ||
| 64 | if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: | ||
| 65 | raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") | ||
| 66 | if operation == "shale.clone": | ||
| 67 | return import_module("shale-access").clone(request) | ||
| 68 | return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") | ||
| 69 | if operation in {"files.snapshot", "files.snapshots", "files.discard"}: | 58 | if operation in {"files.snapshot", "files.snapshots", "files.discard"}: |
| 70 | datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] | 59 | datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] |
| 71 | if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: | 60 | if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: |
| ... | @@ -234,14 +223,13 @@ def stage(computer=None): | ... | @@ -234,14 +223,13 @@ def stage(computer=None): |
| 234 | for entry in mac["vms"]: | 223 | for entry in mac["vms"]: |
| 235 | entry["owner"] = owner[0] | 224 | entry["owner"] = owner[0] |
| 236 | computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac)) | 225 | computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac)) |
| 237 | host_python = snowglobe.command(*nix, "build", "--out-link", str(root / "host-python"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip() | ||
| 238 | subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) | 226 | subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) |
| 239 | subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True) | 227 | subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True) |
| 240 | snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, | 228 | snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, |
| 241 | "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32", | 229 | "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32", |
| 242 | "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], | 230 | "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], |
| 243 | *computer_environment, | 231 | *computer_environment, |
| 244 | host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity) | 232 | sys.executable, str(Path(__file__).resolve()), "--serve", identity) |
| 245 | for _ in range(100): | 233 | for _ in range(100): |
| 246 | if Path(runtime, "host.sock").exists(): | 234 | if Path(runtime, "host.sock").exists(): |
| 247 | break | 235 | break |
| ... | @@ -263,7 +251,7 @@ def stage(computer=None): | ... | @@ -263,7 +251,7 @@ def stage(computer=None): |
| 263 | "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock", | 251 | "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock", |
| 264 | "--setenv=XDG_RUNTIME_DIR=" + screen_runtime, | 252 | "--setenv=XDG_RUNTIME_DIR=" + screen_runtime, |
| 265 | "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache", | 253 | "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache", |
| 266 | host_python + "/bin/python3", "-c", launcher) | 254 | sys.executable, "-c", launcher) |
| 267 | for _ in range(100): | 255 | for _ in range(100): |
| 268 | if Path(screen_runtime, "screen.sock").exists(): | 256 | if Path(screen_runtime, "screen.sock").exists(): |
| 269 | break | 257 | break |
tools/mac-agent-forward-test.py+1-2| ... | @@ -35,8 +35,7 @@ sys.stdin.buffer.read() | ... | @@ -35,8 +35,7 @@ sys.stdin.buffer.read() |
| 35 | server.shutdown() | 35 | server.shutdown() |
| 36 | server.server_close() | 36 | server.server_close() |
| 37 | ''' | 37 | ''' |
| 38 | ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip() | 38 | server = subprocess.Popen([*vm.ssh_args(), vm.GUEST, "python3 -u -c " + shlex.quote(code)], |
| 39 | server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)], | ||
| 40 | stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) | 39 | stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) |
| 41 | tunnel = None | 40 | tunnel = None |
| 42 | try: | 41 | try: |
tools/mac-agent-vm.md+32-3| ... | @@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console | ... | @@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console |
| 27 | python3 tools/mac-agent-vm.py stop | 27 | python3 tools/mac-agent-vm.py stop |
| 28 | ``` | 28 | ``` |
| 29 | 29 | ||
| 30 | Everything except `exec` reaches the guest over pinned SSH at its Bonjour name. | ||
| 31 | `exec` and the dashboard's computer operations go through Tart's guest agent so | ||
| 32 | they run in the logged-in GUI session, and that path can crash the VM runner | ||
| 33 | (`/Library/Logs/DiagnosticReports/tart-*.ips`, `-[VZVirtioSocketDevice | ||
| 34 | connectToPort:]`). Use `ssh` for anything that does not need the GUI session. | ||
| 35 | |||
| 36 | The `net.paperclover.mac-vm-revive` LaunchAgent runs `revive` every 30 seconds | ||
| 37 | through `ssh localhost` (launchd jobs cannot read the VM volume). It starts the | ||
| 38 | VM when its runner is gone unless `stop` or `suspend` left `private/stopped`; | ||
| 39 | `start` clears that marker. Restarts and the newest Tart crash report are logged | ||
| 40 | to `~/Library/Logs/mac-vm-revive.log`. A hung guest under a live runner is not | ||
| 41 | restarted. | ||
| 42 | |||
| 30 | To reach a server bound to guest loopback from this Mac: | 43 | To reach a server bound to guest loopback from this Mac: |
| 31 | 44 | ||
| 32 | ```sh | 45 | ```sh |
| ... | @@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running | ... | @@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running |
| 50 | VM holds a process-scoped idle-sleep assertion, so locking the host does not | 63 | VM holds a process-scoped idle-sleep assertion, so locking the host does not |
| 51 | cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. | 64 | cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. |
| 52 | 65 | ||
| 66 | ## Network and iPhones | ||
| 67 | |||
| 68 | The guest is bridged onto the LAN through the host's default-route interface | ||
| 69 | so iPhones can find it over Bonjour and connect in; Virtualization.framework | ||
| 70 | has no USB passthrough for them. Its DHCP address and the host's are reserved | ||
| 71 | on the router and recorded in | ||
| 72 | [mac-studio-sandbox.json](../config/mac-studio-sandbox.json): Snow Globe | ||
| 73 | exports NFS to `guestAddress` and the dashboard reaches the host at | ||
| 74 | `hostAddress`. Every `start` loads a pf anchor (`com.apple/net.paperclover`) | ||
| 75 | in the guest that refuses new inbound TCP below port 49152 from anyone but | ||
| 76 | the host, plus Kerberos and NetBIOS over UDP. A guest that reboots itself | ||
| 77 | stays open to the LAN until the next `start`. iPhones on iOS 27 pair in Device | ||
| 78 | Hub with **+ → Pair Nearby Device…** and Settings → Privacy & Security → | ||
| 79 | Developer Mode on the phone. Older iOS needs a cable, which the guest cannot | ||
| 80 | have. | ||
| 81 | |||
| 53 | The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve | 82 | The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve |
| 54 | physical cores. CPU count and RAM changes require a cold boot. No automatic | 83 | physical cores. CPU count and RAM changes require a cold boot. No automatic |
| 55 | lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget | 84 | lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget |
| ... | @@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker | ... | @@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker |
| 180 | is present. | 209 | is present. |
| 181 | 210 | ||
| 182 | The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address | 211 | The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address |
| 183 | `10.0.0.1`. Traffic uses the VM's NAT gateway `192.168.64.1`; the host's hosts | 212 | `10.0.0.1`. The host's hosts file is not inherited. The previous guest file is |
| 184 | file is not inherited. The previous guest file is `/etc/hosts.before-code-nas`. | 213 | `/etc/hosts.before-code-nas`. |
| 185 | 214 | ||
| 186 | SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned | 215 | SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned |
| 187 | in `private/known_hosts`, and disables password authentication. No host SSH | 216 | in `private/known_hosts`, and disables password authentication. No host SSH |
| ... | @@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut | ... | @@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut |
| 222 | It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding | 251 | It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding |
| 223 | guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles | 252 | guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles |
| 224 | the guest account authentication from `private/password` server-side. Screen | 253 | the guest account authentication from `private/password` server-side. Screen |
| 225 | Sharing listens on all interfaces inside the private NAT guest; the host relay's | 254 | Sharing listens on all guest interfaces behind the pf anchor; the host relay's |
| 226 | TCP listeners default to loopback and check the exact HTTP Host and WebSocket | 255 | TCP listeners default to loopback and check the exact HTTP Host and WebSocket |
| 227 | Origin. Tart's experimental host VNC server is disabled. The viewer trusts | 256 | Origin. Tart's experimental host VNC server is disabled. The viewer trusts |
| 228 | every client that can reach its listener; Origin checking protects against foreign | 257 | every client that can reach its listener; Origin checking protects against foreign |
tools/mac-agent-vm.py+77-36| ... | @@ -8,6 +8,7 @@ import logging | ... | @@ -8,6 +8,7 @@ import logging |
| 8 | import multiprocessing | 8 | import multiprocessing |
| 9 | import os | 9 | import os |
| 10 | from pathlib import Path | 10 | from pathlib import Path |
| 11 | import re | ||
| 11 | import select | 12 | import select |
| 12 | import secrets | 13 | import secrets |
| 13 | import shlex | 14 | import shlex |
| ... | @@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod | ... | @@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod |
| 26 | TART = ROOT / "runtime/tart.app/Contents/MacOS/tart" | 27 | TART = ROOT / "runtime/tart.app/Contents/MacOS/tart" |
| 27 | NAME = "gui-agent" | 28 | NAME = "gui-agent" |
| 28 | PRIVATE = ROOT / "private" | 29 | PRIVATE = ROOT / "private" |
| 30 | # Never reach the guest with `tart exec` or `tart ip --resolver=agent` unless it needs the GUI session: | ||
| 31 | # their vsock connect intermittently traps inside Virtualization.framework and kills the whole VM. | ||
| 32 | GUEST = "admin@AgentVM.local" | ||
| 29 | 33 | ||
| 30 | 34 | ||
| 31 | def tart(*args, **kwargs): | 35 | def tart(*args, **kwargs): |
| ... | @@ -35,12 +39,27 @@ def tart(*args, **kwargs): | ... | @@ -35,12 +39,27 @@ def tart(*args, **kwargs): |
| 35 | 39 | ||
| 36 | def ssh_args(): | 40 | def ssh_args(): |
| 37 | return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"), | 41 | return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"), |
| 38 | "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no", | 42 | "-o", "AddressFamily=inet", "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no", |
| 39 | "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME, | 43 | "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME, |
| 40 | "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"', | 44 | "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"', |
| 41 | "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"] | 45 | "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"] |
| 42 | 46 | ||
| 43 | 47 | ||
| 48 | def guest(*command, **kwargs): | ||
| 49 | kwargs.setdefault("check", True) | ||
| 50 | return subprocess.run([*ssh_args(), GUEST, shlex.join(command)], **kwargs) | ||
| 51 | |||
| 52 | |||
| 53 | def guest_ready(**kwargs): | ||
| 54 | guest("/bin/sh", "-c", '[ "$(/usr/bin/stat -f %Su /dev/console)" = admin ]', **kwargs) | ||
| 55 | |||
| 56 | |||
| 57 | def shut_down(): | ||
| 58 | # ssh can drop as the guest powers off, so the stopped state is the proof. | ||
| 59 | guest("sudo", "-n", "shutdown", "-h", "now", check=False, timeout=15) | ||
| 60 | wait_for_state("stopped") | ||
| 61 | |||
| 62 | |||
| 44 | def exact(stream, size): | 63 | def exact(stream, size): |
| 45 | data = bytearray() | 64 | data = bytearray() |
| 46 | while len(data) < size: | 65 | while len(data) < size: |
| ... | @@ -51,10 +70,14 @@ def exact(stream, size): | ... | @@ -51,10 +70,14 @@ def exact(stream, size): |
| 51 | return bytes(data) | 70 | return bytes(data) |
| 52 | 71 | ||
| 53 | 72 | ||
| 73 | def vm(): | ||
| 74 | rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) | ||
| 75 | return next(row for row in rows if row["Name"] == NAME and row["Source"] == "local") | ||
| 76 | |||
| 77 | |||
| 54 | def wait_for_state(state): | 78 | def wait_for_state(state): |
| 55 | for _ in range(90): | 79 | for _ in range(90): |
| 56 | rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) | 80 | if vm()["State"] == state: |
| 57 | if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["State"] == state: | ||
| 58 | return | 81 | return |
| 59 | time.sleep(1) | 82 | time.sleep(1) |
| 60 | raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.") | 83 | raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.") |
| ... | @@ -62,13 +85,12 @@ def wait_for_state(state): | ... | @@ -62,13 +85,12 @@ def wait_for_state(state): |
| 62 | 85 | ||
| 63 | @contextlib.contextmanager | 86 | @contextlib.contextmanager |
| 64 | def guest_screen(): | 87 | def guest_screen(): |
| 65 | ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() | ||
| 66 | with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory: | 88 | with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory: |
| 67 | path = Path(directory) / "console.sock" | 89 | path = Path(directory) / "console.sock" |
| 68 | tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes", | 90 | tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes", |
| 69 | "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", | 91 | "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", |
| 70 | "-o", "StreamLocalBindMask=0177", "-L", | 92 | "-o", "StreamLocalBindMask=0177", "-L", |
| 71 | f"{path}:127.0.0.1:5900", "admin@" + ip], | 93 | f"{path}:127.0.0.1:5900", GUEST], |
| 72 | stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, | 94 | stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, |
| 73 | stderr=subprocess.DEVNULL) | 95 | stderr=subprocess.DEVNULL) |
| 74 | try: | 96 | try: |
| ... | @@ -208,6 +230,7 @@ def main(): | ... | @@ -208,6 +230,7 @@ def main(): |
| 208 | sub.add_parser("stop") | 230 | sub.add_parser("stop") |
| 209 | sub.add_parser("suspend") | 231 | sub.add_parser("suspend") |
| 210 | sub.add_parser("reboot") | 232 | sub.add_parser("reboot") |
| 233 | sub.add_parser("revive", help="Start the VM unless it is running or was stopped on purpose") | ||
| 211 | for action in ["exec", "ssh"]: | 234 | for action in ["exec", "ssh"]: |
| 212 | command = sub.add_parser(action) | 235 | command = sub.add_parser(action) |
| 213 | command.add_argument("command", nargs=argparse.REMAINDER) | 236 | command.add_argument("command", nargs=argparse.REMAINDER) |
| ... | @@ -222,12 +245,17 @@ def main(): | ... | @@ -222,12 +245,17 @@ def main(): |
| 222 | local_port = args.local_port if args.local_port is not None else args.port | 245 | local_port = args.local_port if args.local_port is not None else args.port |
| 223 | if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535: | 246 | if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535: |
| 224 | parser.error("Ports must be between 1 and 65535.") | 247 | parser.error("Ports must be between 1 and 65535.") |
| 225 | ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() | ||
| 226 | print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.", | 248 | print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.", |
| 227 | file=sys.stderr, flush=True) | 249 | file=sys.stderr, flush=True) |
| 228 | os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes", | 250 | os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes", |
| 229 | "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", | 251 | "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", |
| 230 | "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip]) | 252 | "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", GUEST]) |
| 253 | if args.action == "revive": | ||
| 254 | if (PRIVATE / "stopped").exists() or vm()["Running"]: | ||
| 255 | return | ||
| 256 | crashes = sorted(Path("/Library/Logs/DiagnosticReports").glob("tart-*.ips")) | ||
| 257 | print(f"{time.ctime()}: VM is down; newest tart crash report: {crashes[-1] if crashes else 'none'}", flush=True) | ||
| 258 | args.action = "start" | ||
| 231 | if args.action in {"start", "stop", "suspend", "reboot"}: | 259 | if args.action in {"start", "stop", "suspend", "reboot"}: |
| 232 | lock = (PRIVATE / "lifecycle.lock").open("a") | 260 | lock = (PRIVATE / "lifecycle.lock").open("a") |
| 233 | os.chmod(PRIVATE / "lifecycle.lock", 0o600) | 261 | os.chmod(PRIVATE / "lifecycle.lock", 0o600) |
| ... | @@ -240,34 +268,47 @@ def main(): | ... | @@ -240,34 +268,47 @@ def main(): |
| 240 | lock.write(args.action) | 268 | lock.write(args.action) |
| 241 | lock.flush() | 269 | lock.flush() |
| 242 | if args.action == "reboot": | 270 | if args.action == "reboot": |
| 243 | tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now", timeout=15) | 271 | shut_down() |
| 244 | wait_for_state("stopped") | ||
| 245 | args.action = "start" | 272 | args.action = "start" |
| 246 | if args.action == "start": | 273 | if args.action == "start": |
| 247 | rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout) | 274 | (PRIVATE / "stopped").unlink(missing_ok=True) |
| 248 | if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]: | 275 | interface = re.search(r"interface: (\S+)", subprocess.run(["/sbin/route", "-n", "get", "default"], |
| 276 | capture_output=True, text=True, check=True).stdout)[1] | ||
| 277 | if vm()["Running"]: | ||
| 249 | print("GUI-agent VM is already running.") | 278 | print("GUI-agent VM is already running.") |
| 250 | return | 279 | else: |
| 251 | # Closing an agent session kills every process it ever spawned, so sshd must own the VM. | 280 | # Closing an agent session kills every process it ever spawned, so sshd must own the VM. |
| 252 | run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", | 281 | # Bridged so iPhones can find and pair with the guest's Xcode over Bonjour. |
| 253 | str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", | 282 | run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", |
| 254 | "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"]) | 283 | str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", |
| 255 | (PRIVATE / "run.log").write_text("") | 284 | "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full", |
| 256 | os.chmod(PRIVATE / "run.log", 0o600) | 285 | f"--net-bridged={interface}"]) |
| 257 | subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", | 286 | (PRIVATE / "run.log").write_text("") |
| 258 | f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True) | 287 | os.chmod(PRIVATE / "run.log", 0o600) |
| 259 | deadline = time.monotonic() + 90 | 288 | subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", |
| 260 | while time.monotonic() < deadline: | 289 | f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True) |
| 261 | if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode: | 290 | deadline = time.monotonic() + 90 |
| 262 | raise ValueError("VM startup failed; inspect the private run log locally.") | 291 | while True: |
| 263 | try: | 292 | if time.monotonic() > deadline: |
| 264 | tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5) | 293 | raise ValueError("VM startup is still pending; inspect status before retrying.") |
| 265 | except (subprocess.CalledProcessError, subprocess.TimeoutExpired): | 294 | if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode: |
| 266 | time.sleep(1) | 295 | raise ValueError("VM startup failed; inspect the private run log locally.") |
| 267 | else: | 296 | try: |
| 268 | print("GUI-agent VM is ready in its logged-in guest session.") | 297 | guest_ready(capture_output=True, timeout=5) |
| 269 | return | 298 | except (subprocess.CalledProcessError, subprocess.TimeoutExpired): |
| 270 | raise ValueError("VM startup is still pending; inspect status before retrying.") | 299 | time.sleep(1) |
| 300 | else: | ||
| 301 | break | ||
| 302 | print("GUI-agent VM is ready in its logged-in guest session.") | ||
| 303 | # The bridged guest is on the LAN: only this Mac may reach its services; iPhones pair on ephemeral ports. | ||
| 304 | host = subprocess.run(["/usr/sbin/ipconfig", "getifaddr", interface], | ||
| 305 | capture_output=True, text=True, check=True).stdout.strip() | ||
| 306 | guest("sudo", "-n", "/bin/sh", "-c", | ||
| 307 | "/sbin/pfctl -q -a com.apple/net.paperclover -f - && /sbin/pfctl -q -E", | ||
| 308 | input=f"table <mac-host> {{ {host} }}\n" | ||
| 309 | "block in quick on ! lo0 proto tcp from ! <mac-host> to any port < 49152 flags S/SA\n" | ||
| 310 | "block in quick on ! lo0 proto udp to any port { 88 137 138 }\n", | ||
| 311 | capture_output=True, text=True) | ||
| 271 | elif args.action == "status": | 312 | elif args.action == "status": |
| 272 | tart("get", NAME) | 313 | tart("get", NAME) |
| 273 | elif args.action == "exec": | 314 | elif args.action == "exec": |
| ... | @@ -275,18 +316,18 @@ def main(): | ... | @@ -275,18 +316,18 @@ def main(): |
| 275 | parser.error("exec requires a guest command") | 316 | parser.error("exec requires a guest command") |
| 276 | tart("exec", "-i", NAME, *args.command) | 317 | tart("exec", "-i", NAME, *args.command) |
| 277 | elif args.action == "ssh": | 318 | elif args.action == "ssh": |
| 278 | ip = tart("ip", NAME, capture_output=True, text=True).stdout.strip() | ||
| 279 | command = ssh_args() | 319 | command = ssh_args() |
| 280 | if not args.command: | 320 | if not args.command: |
| 281 | command.append("-t") | 321 | command.append("-t") |
| 282 | os.execvp("ssh", [*command, "admin@" + ip, *args.command]) | 322 | os.execvp("ssh", [*command, GUEST, *args.command]) |
| 283 | elif args.action == "stop": | 323 | elif args.action == "stop": |
| 284 | tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now") | 324 | shut_down() |
| 285 | wait_for_state("stopped") | 325 | (PRIVATE / "stopped").touch() |
| 286 | print("GUI-agent VM shut down.") | 326 | print("GUI-agent VM shut down.") |
| 287 | elif args.action == "suspend": | 327 | elif args.action == "suspend": |
| 288 | tart("suspend", NAME) | 328 | tart("suspend", NAME) |
| 289 | wait_for_state("suspended") | 329 | wait_for_state("suspended") |
| 330 | (PRIVATE / "stopped").touch() | ||
| 290 | print("GUI-agent VM suspended.") | 331 | print("GUI-agent VM suspended.") |
| 291 | elif args.action == "console": | 332 | elif args.action == "console": |
| 292 | python = ROOT / "runtime/venv/bin/python" | 333 | python = ROOT / "runtime/venv/bin/python" |
tools/mac-vm-bridge.py+1-1| ... | @@ -32,7 +32,7 @@ def status(): | ... | @@ -32,7 +32,7 @@ def status(): |
| 32 | agent = None | 32 | agent = None |
| 33 | if current["Running"]: | 33 | if current["Running"]: |
| 34 | try: | 34 | try: |
| 35 | vm.tart("exec", vm.NAME, "/usr/bin/true", capture_output=True, timeout=2) | 35 | vm.guest_ready(capture_output=True, timeout=5) |
| 36 | agent = "connected" | 36 | agent = "connected" |
| 37 | except subprocess.SubprocessError: | 37 | except subprocess.SubprocessError: |
| 38 | agent = "disconnected" | 38 | agent = "disconnected" |
tools/release.py+1-1| ... | @@ -60,7 +60,7 @@ def host_digest(root): | ... | @@ -60,7 +60,7 @@ def host_digest(root): |
| 60 | digest = hashlib.sha256() | 60 | digest = hashlib.sha256() |
| 61 | paths = sorted(path for path, relative in files(root) | 61 | paths = sorted(path for path, relative in files(root) |
| 62 | if relative.parts[0] in {"nixos", "dashboard", "config", "service"} | 62 | if relative.parts[0] in {"nixos", "dashboard", "config", "service"} |
| 63 | or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) | 63 | or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) |
| 64 | for path in paths: | 64 | for path in paths: |
| 65 | digest.update(str(path.relative_to(root)).encode()) | 65 | digest.update(str(path.relative_to(root)).encode()) |
| 66 | _hash_contents(digest, path) | 66 | _hash_contents(digest, path) |
tools/shale-access-test.py deleted-230| ... | @@ -1,230 +0,0 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import base64 | ||
| 3 | from contextlib import closing | ||
| 4 | import importlib | ||
| 5 | import json | ||
| 6 | import os | ||
| 7 | from pathlib import Path | ||
| 8 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ||
| 9 | import sqlite3 | ||
| 10 | import subprocess | ||
| 11 | import sys | ||
| 12 | import tempfile | ||
| 13 | import threading | ||
| 14 | import unittest | ||
| 15 | from unittest.mock import patch | ||
| 16 | |||
| 17 | import pygit2 | ||
| 18 | from argon2 import PasswordHasher | ||
| 19 | |||
| 20 | shale = importlib.import_module('shale-access') | ||
| 21 | |||
| 22 | |||
| 23 | class CGI(BaseHTTPRequestHandler): | ||
| 24 | """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" | ||
| 25 | def __init__(self, test, *args): | ||
| 26 | self.test = test | ||
| 27 | super().__init__(*args) | ||
| 28 | |||
| 29 | def log_message(self, *args): | ||
| 30 | pass | ||
| 31 | |||
| 32 | def do_GET(self): | ||
| 33 | self.relay() | ||
| 34 | |||
| 35 | def do_POST(self): | ||
| 36 | self.relay() | ||
| 37 | |||
| 38 | def relay(self): | ||
| 39 | try: | ||
| 40 | plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) | ||
| 41 | except shale.Error: | ||
| 42 | self.send_response(401) | ||
| 43 | self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') | ||
| 44 | self.end_headers() | ||
| 45 | return | ||
| 46 | body = self.rfile.read(int(self.headers.get('content-length', 0))) | ||
| 47 | output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, | ||
| 48 | env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout | ||
| 49 | head, _, payload = output.partition(b'\r\n\r\n') | ||
| 50 | headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) | ||
| 51 | self.send_response(int(headers.pop('Status', '200').split()[0])) | ||
| 52 | for name, value in headers.items(): | ||
| 53 | self.send_header(name, value) | ||
| 54 | self.end_headers() | ||
| 55 | self.wfile.write(payload) | ||
| 56 | |||
| 57 | |||
| 58 | class Access(unittest.TestCase): | ||
| 59 | def setUp(self): | ||
| 60 | self.temp = tempfile.TemporaryDirectory() | ||
| 61 | self.addCleanup(self.temp.cleanup) | ||
| 62 | self.root = Path(self.temp.name) | ||
| 63 | self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'}) | ||
| 64 | self.env.start() | ||
| 65 | self.addCleanup(self.env.stop) | ||
| 66 | (self.root / 'data').mkdir() | ||
| 67 | self.uuid = '0' * 26 | ||
| 68 | self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) | ||
| 69 | self.addCleanup(self.repo.free) | ||
| 70 | self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) | ||
| 71 | self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) | ||
| 72 | self.repo.create_reference('refs/heads/native', self.native) | ||
| 73 | self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native]) | ||
| 74 | self.repo.create_reference('refs/heads/main', self.public) | ||
| 75 | self.repo.set_head('refs/heads/main') | ||
| 76 | (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) | ||
| 77 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: | ||
| 78 | db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') | ||
| 79 | for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): | ||
| 80 | db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) | ||
| 81 | if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) | ||
| 82 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: | ||
| 83 | db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') | ||
| 84 | db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) | ||
| 85 | for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): | ||
| 86 | db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name)) | ||
| 87 | db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret'))) | ||
| 88 | |||
| 89 | def commit(self, files, parents=None): | ||
| 90 | index = pygit2.Index() | ||
| 91 | for path, content in files.items(): | ||
| 92 | index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB)) | ||
| 93 | tree = index.write_tree(self.repo) | ||
| 94 | author = pygit2.Signature('Fixture', 'fixture@example.invalid') | ||
| 95 | return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or []) | ||
| 96 | |||
| 97 | def access(self, path, account='', authorization=''): | ||
| 98 | return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) | ||
| 99 | |||
| 100 | def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self): | ||
| 101 | released = [] | ||
| 102 | class TrackedRepository(pygit2.Repository): | ||
| 103 | def free(self): | ||
| 104 | released.append(True) | ||
| 105 | super().free() | ||
| 106 | with patch.object(shale.pygit2, 'Repository', TrackedRepository): | ||
| 107 | self.assertFalse(self.access('/the-knife/')['deny']) | ||
| 108 | self.assertTrue(self.access('/the-knife/objects/x')['deny']) | ||
| 109 | with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')): | ||
| 110 | with self.assertRaises(ValueError): | ||
| 111 | self.access('/the-knife/') | ||
| 112 | self.assertEqual(len(released), 3) | ||
| 113 | |||
| 114 | def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): | ||
| 115 | public = self.access('/the-knife/tree/main/') | ||
| 116 | self.assertFalse(public['deny']) | ||
| 117 | self.assertIn(str(self.public), public['uri']) | ||
| 118 | for rev in ('native', str(self.native), str(self.native)[:12]): | ||
| 119 | for route in ('tree', 'plain', 'commit'): | ||
| 120 | self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev)) | ||
| 121 | self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny']) | ||
| 122 | self.repo.references.delete('refs/heads/native') | ||
| 123 | self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny']) | ||
| 124 | |||
| 125 | def test_every_revision_blocks_files_even_when_absent(self): | ||
| 126 | for rev in ('main', str(self.base)): | ||
| 127 | for route in ('tree', 'plain'): | ||
| 128 | for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'): | ||
| 129 | self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path)) | ||
| 130 | self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny']) | ||
| 131 | |||
| 132 | def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self): | ||
| 133 | result = self.access('/the-knife/commit/' + str(self.public)) | ||
| 134 | self.assertFalse(result['deny']) | ||
| 135 | self.assertNotIn('SECRET', result['safeDiff']) | ||
| 136 | self.assertIn('PUBLIC', result['safeDiff']) | ||
| 137 | for suffix in ('diff', 'patch'): | ||
| 138 | result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix) | ||
| 139 | self.assertFalse(result['deny']) | ||
| 140 | self.assertNotIn('SECRET', result['safeDiff']) | ||
| 141 | self.assertIn('PUBLIC', result['safeDiff']) | ||
| 142 | for suffix in ('stat', 'diffraw'): | ||
| 143 | self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny']) | ||
| 144 | |||
| 145 | def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self): | ||
| 146 | path = '/the-knife/plain/native/knife/private.rs' | ||
| 147 | for name in ('snow', 'guest'): | ||
| 148 | self.assertTrue(self.access(path, name)['allowed']) | ||
| 149 | for name in ('stranger', 'disabled', 'ai', ''): | ||
| 150 | self.assertTrue(self.access(path, name)['deny']) | ||
| 151 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: | ||
| 152 | db.execute("DELETE FROM memberships WHERE user_id='snow'") | ||
| 153 | self.assertTrue(self.access(path, 'snow')['deny']) | ||
| 154 | |||
| 155 | def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): | ||
| 156 | path = '/the-knife/info/refs?service=git-receive-pack' | ||
| 157 | self.assertTrue(self.access(path, 'snow')['deny']) | ||
| 158 | self.assertTrue(self.access(path)['authenticate']) | ||
| 159 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | ||
| 160 | self.assertTrue(self.access(path, authorization=token)['allowed']) | ||
| 161 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) | ||
| 162 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: | ||
| 163 | db.execute('DELETE FROM personal_tokens') | ||
| 164 | self.assertTrue(self.access(path, authorization=token)['deny']) | ||
| 165 | |||
| 166 | def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): | ||
| 167 | for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): | ||
| 168 | self.assertTrue(self.access(path)['clone'], path) | ||
| 169 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | ||
| 170 | self.assertTrue(self.access(path, authorization=token)['allowed'], path) | ||
| 171 | for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): | ||
| 172 | self.assertNotIn('clone', self.access(path), path) | ||
| 173 | |||
| 174 | def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): | ||
| 175 | with self.assertRaises(shale.Error): | ||
| 176 | self.clone('/the-knife.git/info/refs?service=git-upload-pack') | ||
| 177 | self.repo.lookup_reference('refs/heads/main').set_target(self.base) | ||
| 178 | self.repo.create_reference('refs/heads/merge', self.public) | ||
| 179 | server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) | ||
| 180 | self.addCleanup(server.server_close) | ||
| 181 | threading.Thread(target=server.serve_forever, daemon=True).start() | ||
| 182 | self.addCleanup(server.shutdown) | ||
| 183 | url = f'http://127.0.0.1:{server.server_port}/the-knife' | ||
| 184 | environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} | ||
| 185 | for version in ('0', '1', '2'): | ||
| 186 | checkout = self.root / ('clone-v' + version) | ||
| 187 | subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) | ||
| 188 | refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout | ||
| 189 | self.assertEqual(refs, f'refs/heads/main {self.base}\n') | ||
| 190 | for revision in (str(self.native), str(self.public)): | ||
| 191 | fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) | ||
| 192 | self.assertNotEqual(fetched.returncode, 0, (version, revision)) | ||
| 193 | self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) | ||
| 194 | pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) | ||
| 195 | self.assertNotEqual(pushed.returncode, 0) | ||
| 196 | |||
| 197 | def clone(self, uri, method='GET', content_type='', content_encoding=''): | ||
| 198 | return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) | ||
| 199 | |||
| 200 | def test_native_push_during_classification_cannot_change_the_checked_revision(self): | ||
| 201 | newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) | ||
| 202 | repo = self.repo | ||
| 203 | class ConcurrentPush: | ||
| 204 | def __getattr__(self, name): return getattr(repo, name) | ||
| 205 | def __getitem__(self, key): return repo[key] | ||
| 206 | def walk(self, *args): | ||
| 207 | repo.lookup_reference('refs/heads/native').set_target(newer) | ||
| 208 | repo.create_reference('refs/heads/deadbeef', newer, force=True) | ||
| 209 | return repo.walk(*args) | ||
| 210 | for revision in ('native', 'deadbeef'): | ||
| 211 | repo.lookup_reference('refs/heads/native').set_target(self.native) | ||
| 212 | if 'refs/heads/deadbeef' in repo.references: | ||
| 213 | repo.references.delete('refs/heads/deadbeef') | ||
| 214 | with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()): | ||
| 215 | self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) | ||
| 216 | |||
| 217 | def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): | ||
| 218 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: | ||
| 219 | db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") | ||
| 220 | self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) | ||
| 221 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | ||
| 222 | self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny']) | ||
| 223 | |||
| 224 | def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self): | ||
| 225 | for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'): | ||
| 226 | self.assertTrue(self.access(path)['deny'], path) | ||
| 227 | |||
| 228 | |||
| 229 | if __name__ == '__main__': | ||
| 230 | unittest.main() | ||
tools/shale-access.py deleted-250| ... | @@ -1,250 +0,0 @@ | ||
| 1 | import base64 | ||
| 2 | from contextlib import closing | ||
| 3 | import importlib | ||
| 4 | import json | ||
| 5 | import os | ||
| 6 | from pathlib import Path | ||
| 7 | import posixpath | ||
| 8 | import re | ||
| 9 | import sqlite3 | ||
| 10 | import subprocess | ||
| 11 | import sys | ||
| 12 | import tempfile | ||
| 13 | from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit | ||
| 14 | |||
| 15 | import pygit2 | ||
| 16 | from argon2 import PasswordHasher | ||
| 17 | from argon2.exceptions import VerificationError, InvalidHashError | ||
| 18 | |||
| 19 | Error = importlib.import_module('dashboard-run').Error | ||
| 20 | # The root host helper reads repositories owned by the Shale service UID. | ||
| 21 | pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) | ||
| 22 | |||
| 23 | |||
| 24 | def restricted_path(path): | ||
| 25 | parts = path.split('/') | ||
| 26 | return parts[0] == 'knife' or 'Cargo.lock' in parts | ||
| 27 | |||
| 28 | |||
| 29 | def allowed_account(db, user_id): | ||
| 30 | row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake " | ||
| 31 | "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m " | ||
| 32 | "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')", | ||
| 33 | (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone() | ||
| 34 | if not row: | ||
| 35 | return False | ||
| 36 | profile = json.loads(row[0]) | ||
| 37 | return (profile.get('enabled') is True and not profile.get('requiredActions') | ||
| 38 | and profile.get('kind') != 'ai') | ||
| 39 | |||
| 40 | |||
| 41 | def instance(service): | ||
| 42 | if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): | ||
| 43 | raise Error(400, 'Choose a Shale instance.') | ||
| 44 | if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): | ||
| 45 | return Path(fixture_root) | ||
| 46 | site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) | ||
| 47 | return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) | ||
| 48 | |||
| 49 | |||
| 50 | def target(request_uri): | ||
| 51 | uri = urlsplit(request_uri) | ||
| 52 | path = uri.path | ||
| 53 | for _ in range(4): | ||
| 54 | decoded = unquote(path, errors='strict') | ||
| 55 | if decoded == path: | ||
| 56 | break | ||
| 57 | path = decoded | ||
| 58 | path = posixpath.normpath('/' + path.lstrip('/')) | ||
| 59 | return uri, path, path.strip('/').split('/') | ||
| 60 | |||
| 61 | |||
| 62 | def upload_pack(uri, path, parts): | ||
| 63 | """Returns the CGI path of an anonymous fetch request, the only Git route served without a token.""" | ||
| 64 | if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path: | ||
| 65 | return None | ||
| 66 | if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query: | ||
| 67 | return '/the-knife.git/' + '/'.join(parts[1:]) | ||
| 68 | return None | ||
| 69 | |||
| 70 | |||
| 71 | def repository(root, db): | ||
| 72 | row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() | ||
| 73 | return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None | ||
| 74 | |||
| 75 | |||
| 76 | def protected_commits(repo, native): | ||
| 77 | policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] | ||
| 78 | tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] | ||
| 79 | if native: | ||
| 80 | tips.append(native) | ||
| 81 | return {str(commit.id) for tip in tips for commit in repo.walk(tip)} | ||
| 82 | |||
| 83 | |||
| 84 | def access(request, accounts=None): | ||
| 85 | root = instance(request['service']) | ||
| 86 | uri, path, parts = target(request['uri']) | ||
| 87 | if parts[0].lower() not in ('the-knife', 'the-knife.git'): | ||
| 88 | return {'allowed': False, 'deny': True} | ||
| 89 | git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') | ||
| 90 | if not request['authorization'] and upload_pack(uri, path, parts): | ||
| 91 | return {'allowed': False, 'deny': False, 'clone': True} | ||
| 92 | with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: | ||
| 93 | accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) | ||
| 94 | db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) | ||
| 95 | authorization = request['authorization'] | ||
| 96 | allowed = False | ||
| 97 | if authorization: | ||
| 98 | try: | ||
| 99 | scheme, value = authorization.split(' ', 1) | ||
| 100 | if scheme.lower() == 'basic': | ||
| 101 | name, secret = base64.b64decode(value, validate=True).decode().split(':', 1) | ||
| 102 | users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall() | ||
| 103 | if len(users) == 1 and allowed_account(db, users[0][0]): | ||
| 104 | for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]): | ||
| 105 | try: | ||
| 106 | if PasswordHasher().verify(hashed, secret): | ||
| 107 | allowed = True | ||
| 108 | break | ||
| 109 | except (VerificationError, InvalidHashError): | ||
| 110 | pass | ||
| 111 | except (ValueError, UnicodeError): | ||
| 112 | pass | ||
| 113 | elif request['account'] and not git: | ||
| 114 | users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall() | ||
| 115 | allowed = len(users) == 1 and allowed_account(db, users[0][0]) | ||
| 116 | if allowed: | ||
| 117 | return {'allowed': True, 'deny': False} | ||
| 118 | location = repository(root, db) | ||
| 119 | if not location: | ||
| 120 | return {'allowed': False, 'deny': True} | ||
| 121 | repo = pygit2.Repository(str(location)) | ||
| 122 | try: | ||
| 123 | head = repo.lookup_reference('HEAD').target | ||
| 124 | references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} | ||
| 125 | protected = protected_commits(repo, references.get('refs/heads/native')) | ||
| 126 | result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} | ||
| 127 | if parts[0] != 'the-knife' or '%' in path or '\\' in path: | ||
| 128 | return result | ||
| 129 | route = parts[1] if len(parts) > 1 else '' | ||
| 130 | query = dict(parse_qsl(uri.query, keep_blank_values=True)) | ||
| 131 | if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): | ||
| 132 | return result | ||
| 133 | if route in ('', 'issues'): | ||
| 134 | if route == '' and query: | ||
| 135 | return result | ||
| 136 | result['htmlOnly'] = route != 'issues' | ||
| 137 | result['deny'] = False | ||
| 138 | return result | ||
| 139 | if route in ('heads', 'tags') and len(parts) == 2 and not query: | ||
| 140 | result['deny'] = False | ||
| 141 | return result | ||
| 142 | if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): | ||
| 143 | return result | ||
| 144 | if route not in ('log', 'tree', 'plain', 'commit'): | ||
| 145 | return result | ||
| 146 | revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' | ||
| 147 | suffix = '' | ||
| 148 | if route == 'commit': | ||
| 149 | revision, dot, suffix = revision.partition('.') | ||
| 150 | if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): | ||
| 151 | return result | ||
| 152 | if not revision or revision == '-': | ||
| 153 | revision = 'HEAD' | ||
| 154 | # Only named refs and object IDs have the same meaning in Shale and libgit2. | ||
| 155 | if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): | ||
| 156 | return result | ||
| 157 | try: | ||
| 158 | if revision == 'HEAD': | ||
| 159 | oid = references.get(head) if isinstance(head, str) else head | ||
| 160 | else: | ||
| 161 | candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} | ||
| 162 | if len(candidates) > 1: | ||
| 163 | return result | ||
| 164 | if candidates: | ||
| 165 | oid = candidates.pop() | ||
| 166 | elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): | ||
| 167 | oid = repo[revision].id | ||
| 168 | else: | ||
| 169 | return result | ||
| 170 | if oid is None: | ||
| 171 | return result | ||
| 172 | commit = repo[oid].peel(pygit2.Commit) | ||
| 173 | except (KeyError, ValueError, pygit2.GitError): | ||
| 174 | return result | ||
| 175 | if str(commit.id) in protected: | ||
| 176 | return result | ||
| 177 | if route in ('tree', 'plain'): | ||
| 178 | result['htmlOnly'] = route != 'plain' | ||
| 179 | file = '/'.join(parts[3:]) | ||
| 180 | if restricted_path(file): | ||
| 181 | return result | ||
| 182 | if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): | ||
| 183 | return result | ||
| 184 | if route == 'commit': | ||
| 185 | diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] | ||
| 186 | sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) | ||
| 187 | if sensitive: | ||
| 188 | if suffix in ('stat', 'diffraw'): | ||
| 189 | return result | ||
| 190 | result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" | ||
| 191 | if suffix: | ||
| 192 | result['rawDiff'] = True | ||
| 193 | result['deny'] = False | ||
| 194 | return result | ||
| 195 | if route == 'log': | ||
| 196 | result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) | ||
| 197 | else: | ||
| 198 | parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] | ||
| 199 | result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') | ||
| 200 | if query: | ||
| 201 | result['uri'] += '?' + urlencode(query) | ||
| 202 | result['deny'] = False | ||
| 203 | return result | ||
| 204 | finally: | ||
| 205 | repo.free() | ||
| 206 | |||
| 207 | |||
| 208 | def clone(request): | ||
| 209 | root = instance(request['service']) | ||
| 210 | uri, path, parts = target(request['uri']) | ||
| 211 | location = upload_pack(uri, path, parts) | ||
| 212 | if not location: | ||
| 213 | raise Error(404, 'Clone The Knife over HTTPS.') | ||
| 214 | with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: | ||
| 215 | knife = repository(root, db) | ||
| 216 | if not knife: | ||
| 217 | raise Error(404, 'The Knife is unavailable.') | ||
| 218 | repo = pygit2.Repository(str(knife)) | ||
| 219 | try: | ||
| 220 | main = repo.lookup_reference('refs/heads/main').resolve().target | ||
| 221 | native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None | ||
| 222 | protected = protected_commits(repo, native) | ||
| 223 | if any(str(commit.id) in protected for commit in repo.walk(main)): | ||
| 224 | raise Error(403, "The Knife's main branch includes native history.") | ||
| 225 | finally: | ||
| 226 | repo.free() | ||
| 227 | return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': { | ||
| 228 | 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query, | ||
| 229 | 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'], | ||
| 230 | }} | ||
| 231 | |||
| 232 | |||
| 233 | def serve(objects, main): | ||
| 234 | """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects.""" | ||
| 235 | repository = Path(tempfile.mkdtemp()) / 'the-knife.git' | ||
| 236 | (repository / 'objects/info').mkdir(parents=True) | ||
| 237 | (repository / 'refs/heads').mkdir(parents=True) | ||
| 238 | (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n') | ||
| 239 | (repository / 'HEAD').write_text('ref: refs/heads/main\n') | ||
| 240 | (repository / 'refs/heads/main').write_text(main + '\n') | ||
| 241 | (repository / 'objects/info/alternates').write_text(objects + '\n') | ||
| 242 | # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main. | ||
| 243 | environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)} | ||
| 244 | os.execvpe('git', ['git', 'http-backend'], environment | { | ||
| 245 | 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', | ||
| 246 | }) | ||
| 247 | |||
| 248 | |||
| 249 | if __name__ == '__main__': | ||
| 250 | serve(*sys.argv[1:]) | ||
tools/shale-webhook-test.py created+124| ... | @@ -0,0 +1,124 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import http.server | ||
| 3 | import json | ||
| 4 | import os | ||
| 5 | from pathlib import Path | ||
| 6 | import sqlite3 | ||
| 7 | import subprocess | ||
| 8 | import tempfile | ||
| 9 | import threading | ||
| 10 | import unittest | ||
| 11 | |||
| 12 | REPO = Path(__file__).resolve().parent.parent | ||
| 13 | HOOK = REPO / "service/shale/webhook.sh" | ||
| 14 | PREPARE = REPO / "service/shale/prepare.py" | ||
| 15 | |||
| 16 | |||
| 17 | class Receiver(http.server.BaseHTTPRequestHandler): | ||
| 18 | events = [] | ||
| 19 | |||
| 20 | def do_POST(self): | ||
| 21 | body = self.rfile.read(int(self.headers["Content-Length"])) | ||
| 22 | self.events.append((self.headers["X-GitHub-Event"], self.path, json.loads(body))) | ||
| 23 | self.send_response(204) | ||
| 24 | self.end_headers() | ||
| 25 | |||
| 26 | def log_message(self, *_): | ||
| 27 | pass | ||
| 28 | |||
| 29 | |||
| 30 | class ShaleWebhookTests(unittest.TestCase): | ||
| 31 | def setUp(self): | ||
| 32 | self.temp = tempfile.TemporaryDirectory() | ||
| 33 | self.root = Path(self.temp.name) | ||
| 34 | self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Receiver) | ||
| 35 | Receiver.events = [] | ||
| 36 | self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) | ||
| 37 | self.thread.start() | ||
| 38 | |||
| 39 | def tearDown(self): | ||
| 40 | self.server.shutdown() | ||
| 41 | self.thread.join() | ||
| 42 | self.server.server_close() | ||
| 43 | self.temp.cleanup() | ||
| 44 | |||
| 45 | def git(self, *args, cwd=None, env=None): | ||
| 46 | return subprocess.run( | ||
| 47 | ["git", *args], cwd=cwd, env=env, check=True, text=True, | ||
| 48 | capture_output=True, | ||
| 49 | ).stdout.strip() | ||
| 50 | |||
| 51 | def test_push_and_delete_match_snowcord_github_contract(self): | ||
| 52 | bare = self.root / "origin.git" | ||
| 53 | work = self.root / "work" | ||
| 54 | self.git("init", "--bare", "--initial-branch=main", bare) | ||
| 55 | self.git("config", "--file", bare / "config", "shale.snowglobe.repository", "snowcord") | ||
| 56 | (bare / "hooks/post-receive").symlink_to(HOOK) | ||
| 57 | self.git("init", "--initial-branch=main", work) | ||
| 58 | self.git("config", "user.name", "Clover", cwd=work) | ||
| 59 | self.git("config", "user.email", "clover@example.test", cwd=work) | ||
| 60 | (work / "file.txt").write_text("hello\n") | ||
| 61 | self.git("add", "file.txt", cwd=work) | ||
| 62 | self.git("commit", "-m", 'first "line"\n\nsecond \\ line', cwd=work) | ||
| 63 | self.git("remote", "add", "origin", str(bare), cwd=work) | ||
| 64 | env = { | ||
| 65 | **os.environ, | ||
| 66 | "DOMAIN": "shale.test", | ||
| 67 | "SHALE_WEBHOOK_ENABLED": "true", | ||
| 68 | "SHALE_WEBHOOK_URL": f"http://127.0.0.1:{self.server.server_port}/hook", | ||
| 69 | } | ||
| 70 | self.git("push", "origin", "main", cwd=work, env=env) | ||
| 71 | |||
| 72 | event, path, payload = Receiver.events.pop(0) | ||
| 73 | self.assertEqual(event, "push") | ||
| 74 | self.assertEqual(path, "/hook/github/") | ||
| 75 | self.assertEqual(payload["ref"], "refs/heads/main") | ||
| 76 | self.assertEqual(payload["repository"]["name"], "snowcord") | ||
| 77 | self.assertEqual(payload["sender"]["login"], "Clover") | ||
| 78 | self.assertEqual(payload["commits"][0]["message"], 'first "line"\n\nsecond \\ line') | ||
| 79 | self.assertEqual(payload["head_commit"]["id"], payload["after"]) | ||
| 80 | |||
| 81 | self.git("push", "origin", "main:delete-me", cwd=work, env=env) | ||
| 82 | Receiver.events.pop(0) | ||
| 83 | self.git("push", "origin", ":delete-me", cwd=work, env=env) | ||
| 84 | event, _, payload = Receiver.events.pop(0) | ||
| 85 | self.assertEqual((event, payload["ref_type"], payload["ref"]), ("delete", "branch", "delete-me")) | ||
| 86 | |||
| 87 | def test_prepare_installs_only_named_repository_hooks(self): | ||
| 88 | owned = self.root / "repositories_owned" | ||
| 89 | database = self.root / "data/astheno.shale.db" | ||
| 90 | database.parent.mkdir() | ||
| 91 | with sqlite3.connect(database) as db: | ||
| 92 | db.executescript(""" | ||
| 93 | CREATE TABLE users (provider TEXT, snowflake TEXT); | ||
| 94 | CREATE TABLE repositories (uuid TEXT, name TEXT); | ||
| 95 | INSERT INTO repositories VALUES ('snowcord-id', 'snowcord'); | ||
| 96 | INSERT INTO repositories VALUES ('other-id', 'other'); | ||
| 97 | """) | ||
| 98 | for name in ("snowcord-id", "other-id"): | ||
| 99 | self.git("init", "--bare", owned / name) | ||
| 100 | env = {**os.environ, "SNOWGLOBE_DOMAIN": "test"} | ||
| 101 | subprocess.run( | ||
| 102 | ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}), | ||
| 103 | env=env, check=True, text=True, capture_output=True, | ||
| 104 | ) | ||
| 105 | hook = owned / "snowcord-id/hooks/post-receive" | ||
| 106 | self.assertEqual(os.readlink(hook), "/data/shale-webhook") | ||
| 107 | self.assertEqual((self.root / "data/shale-webhook").stat().st_mode & 0o777, 0o750) | ||
| 108 | self.assertEqual( | ||
| 109 | self.git("config", "--file", owned / "snowcord-id/config", "--get", "shale.snowglobe.repository"), | ||
| 110 | "snowcord", | ||
| 111 | ) | ||
| 112 | self.assertFalse((owned / "other-id/hooks/post-receive").exists()) | ||
| 113 | |||
| 114 | with sqlite3.connect(database) as db: | ||
| 115 | db.execute("UPDATE repositories SET name='renamed' WHERE uuid='snowcord-id'") | ||
| 116 | subprocess.run( | ||
| 117 | ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}), | ||
| 118 | env=env, check=True, text=True, capture_output=True, | ||
| 119 | ) | ||
| 120 | self.assertFalse(os.path.lexists(hook)) | ||
| 121 | |||
| 122 | |||
| 123 | if __name__ == "__main__": | ||
| 124 | unittest.main() | ||