authoravatarclover caruso<git@paperclover.net>2026-11-19 13:21:20-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:03-08:00
log9acd963a146bafc2595b79916b2f1df24cc4eefe
tree261a78d8790d924564c2ad56abfb918d468c5956
parent17dc455c3819b2d2563acaf514669bb33a593f0f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Remove The Knife's Shale access gate and the knife-native group

The Knife is private in Shale now, so the dashboard no longer gates /the-knife: shale-access.py, anonymous main-branch clones, native-history redaction, the restricted page, and nativeRestriction are gone, along with host-python.nix, which existed only for them. The knife-native group is deleted from existing databases; guests join no groups again, and adding a guest only creates the account. Assisted-by: claude-opus-5.5

28 files changed, 456 insertions(+), 890 deletions(-)

config/mac-studio-sandbox.json+2-1
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1{1{
2 "dataset": "globe/clover/mac-studio-sandbox",2 "dataset": "globe/clover/mac-studio-sandbox",
3 "mountPoint": "/srv/mac-studio-vm/sandbox",3 "mountPoint": "/srv/mac-studio-vm/sandbox",
4 "clientAddress": "10.0.0.222"4 "hostAddress": "10.0.0.189",
5 "guestAddress": "10.0.0.124"
5}6}
config/shale.json+1-1
...@@ -1 +1 @@...@@ -1 +1 @@
1{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}}1{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}}
dashboard/src/auth.rs+4-5
...@@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session";...@@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session";
12const FLOW_COOKIE: &str = "__Host-snow-flow";12const FLOW_COOKIE: &str = "__Host-snow-flow";
13const CSRF_COOKIE: &str = "__Host-snow-csrf";13const CSRF_COOKIE: &str = "__Host-snow-csrf";
14const SESSION_TTL: i64 = 30 * 86400;14const SESSION_TTL: i64 = 30 * 86400;
15const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"];15const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"];
1616
17pub struct Store {17pub struct Store {
18 pub db: Mutex<Connection>,18 pub db: Mutex<Connection>,
...@@ -201,14 +201,13 @@ impl Store {...@@ -201,14 +201,13 @@ impl Store {
201 oidc::initialise(&db)?;201 oidc::initialise(&db)?;
202 guest::initialise(&db)?;202 guest::initialise(&db)?;
203 for name in GROUPS {203 for name in GROUPS {
204 let added = db.execute(204 db.execute(
205 "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)",205 "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)",
206 sql![uuid::Uuid::new_v4().to_string(), name],206 sql![uuid::Uuid::new_v4().to_string(), name],
207 )?;207 )?;
208 if *name == "knife-native" && added != 0 {
209 db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?;
210 }
211 }208 }
209 db.execute_batch("DELETE FROM memberships WHERE role_id IN (SELECT id FROM roles WHERE name='knife-native');
210 DELETE FROM roles WHERE name='knife-native';")?;
212 Ok(Self {211 Ok(Self {
213 db: Mutex::new(db),212 db: Mutex::new(db),
214 origin,213 origin,
dashboard/src/guest.rs+4-21
...@@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> {...@@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> {
23 db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL);23 db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL);
24 CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id));24 CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id));
25 DROP TRIGGER IF EXISTS guest_no_groups;25 DROP TRIGGER IF EXISTS guest_no_groups;
26 CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END;26 CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END;
27 CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END;27 DROP TRIGGER IF EXISTS guest_no_group_updates;
28 CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END;28 CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END;
29 CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?;29 CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?;
30 Ok(())30 Ok(())
...@@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)...@@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)
650 if known.len() > 1 {650 if known.len() > 1 {
651 return Err(Error::new(651 return Err(Error::new(
652 409,652 409,
653 "Several accounts have signed in with that name. Open the right one under guests and join it to knife-native.",653 "Several accounts have signed in with that name. Find them under guests.",
654 ));654 ));
655 }655 }
656 let username = match known.first() {656 let username = match known.first() {
...@@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)...@@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)
673 id673 id
674 }674 }
675 };675 };
676 tx.execute(
677 "INSERT OR IGNORE INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'",
678 [&id],
679 )?;
680 tx.commit()?;676 tx.commit()?;
681 Ok(json!({"id":id,"created":existing.is_none()}))677 Ok(json!({"id":id,"created":existing.is_none()}))
682}678}
...@@ -1185,7 +1181,7 @@ mod tests {...@@ -1185,7 +1181,7 @@ mod tests {
1185 }1181 }
11861182
1187 #[test]1183 #[test]
1188 fn identities_stay_separate_and_guests_can_only_join_native_access() {1184 fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() {
1189 let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4()));1185 let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4()));
1190 let auth = auth::Store::new(1186 let auth = auth::Store::new(
1191 &path,1187 &path,
...@@ -1234,9 +1230,6 @@ mod tests {...@@ -1234,9 +1230,6 @@ mod tests {
1234 db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first])1230 db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first])
1235 .is_err()1231 .is_err()
1236 );1232 );
1237 db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap();
1238 assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native");
1239 assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err());
1240 assert!(auth::set_password(&db, &first, "a-password-hash").is_err());1233 assert!(auth::set_password(&db, &first, "a-password-hash").is_err());
1241 assert!(1234 assert!(
1242 db.execute(1235 db.execute(
...@@ -1261,10 +1254,6 @@ mod tests {...@@ -1261,10 +1254,6 @@ mod tests {
1261 std::fs::remove_dir_all(path).unwrap();1254 std::fs::remove_dir_all(path).unwrap();
1262 }1255 }
12631256
1264 fn members(auth: &auth::Store, id: &str) -> Value {
1265 auth::user(&auth.db.lock().unwrap(), id).unwrap()["groups"].clone()
1266 }
1267
1268 #[test]1257 #[test]
1269 fn added_guests_are_the_accounts_their_first_sign_in_finds() {1258 fn added_guests_are_the_accounts_their_first_sign_in_finds() {
1270 let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4()));1259 let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4()));
...@@ -1279,10 +1268,8 @@ mod tests {...@@ -1279,10 +1268,8 @@ mod tests {
1279 let added = grant(&auth, provider, Some(subject), "octo").unwrap();1268 let added = grant(&auth, provider, Some(subject), "octo").unwrap();
1280 let id = string(&added["id"]);1269 let id = string(&added["id"]);
1281 assert_eq!(added["created"], true);1270 assert_eq!(added["created"], true);
1282 assert_eq!(members(&auth, id)[0]["name"], "knife-native");
1283 assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id);1271 assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id);
1284 assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false);1272 assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false);
1285 assert_eq!(members(&auth, id).as_array().unwrap().len(), 1);
1286 }1273 }
12871274
1288 let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned();1275 let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned();
...@@ -1294,19 +1281,15 @@ mod tests {...@@ -1294,19 +1281,15 @@ mod tests {
1294 assert_eq!(identities(&reserved), 0);1281 assert_eq!(identities(&reserved), 0);
1295 let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap();1282 let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap();
1296 assert_ne!(stranger, reserved);1283 assert_ne!(stranger, reserved);
1297 assert_eq!(members(&auth, &stranger), json!([]));
1298 claim(&auth, "sub-1", "alice").unwrap();1284 claim(&auth, "sub-1", "alice").unwrap();
1299 assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved);1285 assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved);
1300 assert_eq!(members(&auth, &reserved)[0]["name"], "knife-native");
1301 claim(&auth, "sub-3", "alice").unwrap();1286 claim(&auth, "sub-3", "alice").unwrap();
1302 let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap();1287 let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap();
1303 assert_ne!(impostor, reserved);1288 assert_ne!(impostor, reserved);
1304 assert_eq!(members(&auth, &impostor), json!([]));
13051289
1306 let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap();1290 let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap();
1307 let added = grant(&auth, "astheno", None, "carol").unwrap();1291 let added = grant(&auth, "astheno", None, "carol").unwrap();
1308 assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false)));1292 assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false)));
1309 assert_eq!(members(&auth, &earlier)[0]["name"], "knife-native");
1310 account(&auth, "astheno", "sub-5", "dave", None).unwrap();1293 account(&auth, "astheno", "sub-5", "dave", None).unwrap();
1311 account(&auth, "astheno", "sub-6", "dave", None).unwrap();1294 account(&auth, "astheno", "sub-6", "dave", None).unwrap();
1312 assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409);1295 assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409);
dashboard/src/knife-restricted.html deleted-28
...@@ -1,28 +0,0 @@
1<!doctype html>
2<html lang="en">
3<meta charset="utf-8">
4<meta name="viewport" content="width=device-width, initial-scale=1">
5<title>The Knife's Native UI is restricted</title>
6<style>
7:root { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; background: #faf8fc; color: #29232e; }
8body { margin: 0; min-height: 100svh; display: grid; place-items: center; }
9main { box-sizing: border-box; width: min(100%, 42rem); padding: 2rem; text-align: center; }
10h1 { margin: 0 0 1rem; font-size: clamp(1.5rem, 4vw, 2rem); line-height: 1.25; }
11p { margin: 0 auto 1.75rem; max-width: 34rem; color: #62596b; }
12a { display: inline-block; padding: .625rem 1.375rem; border: 1px solid transparent; border-radius: .375rem; background: #6b3e9e; color: white; text-decoration: none; font-weight: 600; }
13a:hover { background: #552a84; }
14a:focus-visible { outline: 3px solid #8857be; outline-offset: 4px; }
15@media (prefers-color-scheme: dark) {
16:root { background: #1c1821; color: #f4eff8; }
17p { color: #c2b6cd; }
18a { background: #b495db; color: #21162e; }
19a:hover { background: #c8ade9; }
20a:focus-visible { outline-color: #d7b8ff; }
21}
22</style>
23<main>
24<h1>The Knife's Native UI is restricted</h1>
25<p>This part of the code is sensitive. Please contact clover to request private access.</p>
26<a href="/-/login">Login</a>
27</main>
28</html>
dashboard/src/shale_page.rs+4-193
...@@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) {...@@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) {
347 }347 }
348}348}
349349
350fn knife_uri(path: &str) -> bool {
351 let mut decoded = path.to_owned();
352 for _ in 0..5 {
353 if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) {
354 return true;
355 }
356 let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned();
357 if next == decoded { break; }
358 decoded = next;
359 }
360 false
361}
362
363fn session_account(html: &str) -> Option<String> {
364 let document = scraper::Html::parse_document(html);
365 let selector = scraper::Selector::parse("body > header.usa-header--basic nav a.usa-nav-link[href^='/~']").unwrap();
366 let mut accounts = document.select(&selector);
367 let name = accounts.next()?.attr("href")?.strip_prefix("/~")?;
368 let name = percent_encoding::percent_decode_str(name).decode_utf8().ok()?;
369 (accounts.next().is_none() && !name.is_empty() && !name.contains(['/', '\\', '?', '#']))
370 .then(|| name.into_owned())
371}
372
373fn restricted(authenticate: bool) -> Response {
374 (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [
375 ("content-type", "text/html; charset=utf-8"),
376 ("cache-control", "private, no-store"),
377 ("vary", "Cookie, Authorization"),
378 ("referrer-policy", "same-origin"),
379 ("x-content-type-options", "nosniff"),
380 ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"),
381 ("www-authenticate", "Basic realm=\"The Knife\""),
382 ], include_str!("knife-restricted.html")).into_response()
383}
384
385fn redact_knife(html: &str, policy: &Value) -> Result<String> {
386 let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect();
387 let document = scraper::Html::parse_document(html);
388 let diff = policy["safeDiff"].as_str();
389 if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 {
390 return Err(Error::new(403, "The Knife's Native UI is restricted"));
391 }
392 let links = scraper::Selector::parse("a[href]").unwrap();
393 let rows: Vec<bool> = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| {
394 row.select(&links).any(|link| link.attr("href").is_some_and(|href| {
395 href.split(['/', '?', '#', '.']).any(|part| commits.contains(part))
396 }))
397 }).collect();
398 let mut row_index = 0;
399 Ok(lol_html::rewrite_str(html, RewriteStrSettings::new()
400 .append_element_content_handler(element!("#m-code", |element| {
401 if let Some(diff) = diff {
402 let diff = diff.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;");
403 element.set_inner_content(&format!("<pre><code>{diff}</code></pre>"), ContentType::Html);
404 }
405 Ok(())
406 }))
407 .append_element_content_handler(element!("tr", |element| {
408 if *rows.get(row_index).unwrap_or(&true) { element.remove(); }
409 row_index += 1;
410 Ok(())
411 }))
412 )?)
413}
414
415/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch.
416async fn clone(request: Value, body: Body) -> Result<Response> {
417 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
418 let (_, stream) = host::open(request).await?;
419 let (reader, mut writer) = stream.into_split();
420 tokio::spawn(async move {
421 let mut body = body.into_data_stream();
422 while let Some(Ok(chunk)) = body.next().await {
423 if writer.write_all(&chunk).await.is_err() { return; }
424 }
425 let _ = writer.shutdown().await;
426 });
427 let mut reader = tokio::io::BufReader::new(reader);
428 let mut response = Response::builder();
429 loop {
430 let mut line = String::new();
431 if reader.read_line(&mut line).await? == 0 {
432 return Err(Error::new(502, "The Knife clone ended early."));
433 }
434 let line = line.trim_end();
435 if line.is_empty() { break; }
436 let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?;
437 let value = value.trim();
438 if name.eq_ignore_ascii_case("status") {
439 response = response.status(value.split_whitespace().next().unwrap_or_default().parse::<u16>()?);
440 } else {
441 response = response.header(name, value);
442 }
443 }
444 let stream = futures::stream::unfold(reader, |mut reader| async move {
445 let mut buffer = vec![0; 64 * 1024];
446 match reader.read(&mut buffer).await {
447 Ok(0) => None,
448 Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) }
449 Err(error) => Some((Err(error), reader)),
450 }
451 });
452 Ok(response.body(Body::from_stream(stream))?)
453}
454
455pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {350pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
456 let (mut parts, body) = request.into_parts();351 let (mut parts, body) = request.into_parts();
457 let host = parts352 let host = parts
...@@ -484,52 +379,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {...@@ -484,52 +379,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
484 return Err(Error::new(403, "Open Shale to continue."));379 return Err(Error::new(403, "Open Shale to continue."));
485 }380 }
486 let origin = url::Url::parse(&format!("https://{host}{uri}"))?;381 let origin = url::Url::parse(&format!("https://{host}{uri}"))?;
487 let knife = knife_uri(uri.path());
488 let mut target = match &app.internal {
489 Some((base, _)) => format!("{base}services/{service}{uri}"),
490 None => format!("http://{upstream}{uri}"),
491 };
492 let mut knife_policy = Value::Null;
493 if knife {
494 let mut account = String::new();
495 if let Some(cookie) = parts.headers.get("cookie") {
496 // Opening settings rotates the session's CSRF token, invalidating pending forms.
497 let index = match &app.internal {
498 Some((base, _)) => format!("{base}services/{service}/"),
499 None => format!("http://{upstream}/"),
500 };
501 let mut identity = app.shale_http.get(index).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3));
502 if app.internal.is_none() { identity = identity.header("host", host); }
503 for name in ["x-forwarded-for", "x-real-ip", "user-agent"] {
504 if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); }
505 }
506 if let Ok(response) = identity.send().await
507 && response.status() == StatusCode::OK
508 && let Ok(body) = response.text().await
509 {
510 account = session_account(&body).unwrap_or_default();
511 }
512 }
513 knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await {
514 Ok(policy) => policy,
515 Err(_) => return Ok(restricted(false)),
516 };
517 if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); }
518 if knife_policy["clone"] == true {
519 let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default();
520 let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")});
521 return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true)));
522 }
523 if let Some(uri) = knife_policy["uri"].as_str() {
524 target = match &app.internal {
525 Some((base, _)) => format!("{base}services/{service}{uri}"),
526 None => format!("http://{upstream}{uri}"),
527 };
528 }
529 if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() {
530 return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response());
531 }
532 }
533 let profiles = profiles(&app.auth)?;382 let profiles = profiles(&app.auth)?;
534 if matches!(parts.method, Method::GET | Method::HEAD)383 if matches!(parts.method, Method::GET | Method::HEAD)
535 && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref())384 && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref())
...@@ -544,6 +393,10 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {...@@ -544,6 +393,10 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
544 )393 )
545 .into_response());394 .into_response());
546 }395 }
396 let target = match &app.internal {
397 Some((base, _)) => format!("{base}services/{service}{uri}"),
398 None => format!("http://{upstream}{uri}"),
399 };
547 if app.internal.is_some() {400 if app.internal.is_some() {
548 parts.headers.remove("host");401 parts.headers.remove("host");
549 }402 }
...@@ -578,10 +431,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {...@@ -578,10 +431,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
578 let mut headers = upstream_response.headers().clone();431 let mut headers = upstream_response.headers().clone();
579 strip_hop_headers(&mut headers);432 strip_hop_headers(&mut headers);
580 headers.remove("status");433 headers.remove("status");
581 if knife {
582 headers.insert("cache-control", "private, no-store".parse().unwrap());
583 headers.insert("vary", "Cookie, Authorization".parse().unwrap());
584 }
585 let html = parts.method != Method::HEAD434 let html = parts.method != Method::HEAD
586 && headers435 && headers
587 .get("content-type")436 .get("content-type")
...@@ -637,14 +486,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {...@@ -637,14 +486,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
637 profiles.insert(name.clone(), profile);486 profiles.insert(name.clone(), profile);
638 }487 }
639 }488 }
640 let redacted;
641 let text = if knife && knife_policy["allowed"] != true {
642 redacted = match redact_knife(text, &knife_policy) {
643 Ok(html) => html,
644 Err(_) => return Ok(restricted(false)),
645 };
646 redacted.as_str()
647 } else { text };
648 let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?;489 let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?;
649 for name in [490 for name in [
650 "content-length",491 "content-length",
...@@ -661,9 +502,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {...@@ -661,9 +502,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
661 return Ok((status, headers, rewritten).into_response());502 return Ok((status, headers, rewritten).into_response());
662 }503 }
663 }504 }
664 if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) {
665 return Ok(restricted(false));
666 }
667 let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) })505 let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) })
668 .chain(upstream_response.bytes_stream());506 .chain(upstream_response.bytes_stream());
669 Ok((status, headers, Body::from_stream(stream)).into_response())507 Ok((status, headers, Body::from_stream(stream)).into_response())
...@@ -675,33 +513,6 @@ mod tests {...@@ -675,33 +513,6 @@ mod tests {
675 use rusqlite::params as sql;513 use rusqlite::params as sql;
676 use scraper::{Html, Selector};514 use scraper::{Html, Selector};
677515
678 #[test]
679 fn session_account_uses_only_unambiguous_site_navigation() {
680 let header = |name: &str| format!("<header class='usa-header usa-header--basic'><nav><a class='usa-nav-link' href='/~{name}'>account</a></nav></header>");
681 assert_eq!(session_account(&header("clover")), Some("clover".into()));
682 assert_eq!(session_account(&header("guest%2Dname")), Some("guest-name".into()));
683 assert_eq!(session_account(&format!("{}<main><header class='usa-header--basic'><nav><a class='usa-nav-link' href='/~clover'>fake</a></nav></header></main>", header("guest"))), Some("guest".into()));
684 assert_eq!(session_account(&format!("{}{}", header("guest"), header("clover"))), None);
685 for name in ["", "%FF", "clover%2Frepo", "clover%5Crepo", "clover?fake", "clover#fake"] {
686 assert_eq!(session_account(&header(name)), None, "{name}");
687 }
688 assert_eq!(session_account("<a class='usa-nav-link' href='/~clover'>fake</a>"), None);
689 }
690
691 #[test]
692 fn native_content_is_removed_from_both_diff_styles_and_history_rows() {
693 let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"});
694 let html = "<html><body id=page-commit><table><tr><td><a href='/the-knife/commit/deadbeef'>PRIVATE MESSAGE</a></td></tr><tr><td>PUBLIC ROW</td></tr></table><div id=m-code><div id=m-changedfiles>PRIVATE PATHS</div><details class=m-usa-accordion><summary><code>knife/private.rs</code></summary><table><tr><td>PRIVATE SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>src/public.rs</code></summary><table><tr><td>PUBLIC SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>nested/Cargo.lock</code></summary><table><tr><td>PRIVATE LOCK</td></tr></table></details></div></body></html>";
695 let output = redact_knife(html, &policy).unwrap();
696 assert!(!output.contains("PRIVATE"), "{output}");
697 assert!(output.contains("PUBLIC ROW"), "{output}");
698 assert!(output.contains("PUBLIC SOURCE"), "{output}");
699 assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1);
700 assert!(redact_knife("<div>unknown diff markup</div>", &policy).is_err());
701 for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); }
702 assert!(!knife_uri("/snowbound/commit/native"));
703 }
704
705 #[test]516 #[test]
706 fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() {517 fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() {
707 let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4()));518 let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4()));
dashboard/src/users.rs+3-9
...@@ -158,13 +158,10 @@ pub async fn route(...@@ -158,13 +158,10 @@ pub async fn route(
158 for id in &users {158 for id in &users {
159 uuid(id)?;159 uuid(id)?;
160 let user = auth::user(&transaction, id)?;160 let user = auth::user(&transaction, id)?;
161 let guest_groups = if guest::is_guest(&user) {161 if guest::is_guest(&user) || user["kind"] == "ai" {
162 add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true))
163 } else { false };
164 if guest_groups || user["kind"] == "ai" {
165 return Err(Error::new(162 return Err(Error::new(
166 400,163 400,
167 "Guests can only join knife-native.",164 "Guests can use Shale only. Select regular accounts to edit groups.",
168 ));165 ));
169 }166 }
170 for group in &add {167 for group in &add {
...@@ -304,7 +301,7 @@ pub async fn route(...@@ -304,7 +301,7 @@ pub async fn route(
304 let transaction = db.transaction()?;301 let transaction = db.transaction()?;
305 let mut user = auth::user(&transaction, id)?;302 let mut user = auth::user(&transaction, id)?;
306 if guest::is_guest(&user)303 if guest::is_guest(&user)
307 && matches!(parts, [_, "password"])304 && matches!(parts, [_, "password"] | [_, "groups", _])
308 && method != Method::GET305 && method != Method::GET
309 {306 {
310 return Err(Error::new(307 return Err(Error::new(
...@@ -354,9 +351,6 @@ pub async fn route(...@@ -354,9 +351,6 @@ pub async fn route(
354 .optional()?;351 .optional()?;
355 let name = name352 let name = name
356 .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?;353 .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?;
357 if guest::is_guest(&user) && name != "knife-native" {
358 return Err(Error::new(400, "Guests can only join knife-native."));
359 }
360 if own && method == Method::DELETE && name == "infra-admin" {354 if own && method == Method::DELETE && name == "infra-admin" {
361 return Err(Error::new(355 return Err(Error::new(
362 400,356 400,
dashboard/web/pages/Users.tsx+2-3
...@@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) {...@@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) {
7777
78/** "30 members · opens Jellyfin, Navidrome" */78/** "30 members · opens Jellyfin, Navidrome" */
79function groupTip(group: string, d: Data) {79function groupTip(group: string, d: Data) {
80 if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`;
81 const more = adds(group, d.services);80 const more = adds(group, d.services);
82 const members = d.users.filter((user) => names(user).includes(group)).length;81 const members = d.users.filter((user) => names(user).includes(group)).length;
83 const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : [];82 const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : [];
...@@ -230,7 +229,7 @@ function List(props: { data: Resource<Data>; ready: Accessor<Data | undefined>;...@@ -230,7 +229,7 @@ function List(props: { data: Resource<Data>; ready: Accessor<Data | undefined>;
230 guestMenu.hidePopover();229 guestMenu.hidePopover();
231 showTextDialog({230 showTextDialog({
232 title: `Add ${guest.name} guest`,231 title: `Add ${guest.name} guest`,
233 description: "They can sign in to Shale and open The Knife’s Native UI.",232 description: "They can sign in to Shale.",
234 label: guest.field,233 label: guest.field,
235 placeholder: guest.placeholder,234 placeholder: guest.placeholder,
236 body: provider === "astheno" ? () => <div class="field"><span class="hint">Astheno can’t look names up. The first person to sign in with this username gets the access.</span></div> : undefined,235 body: provider === "astheno" ? () => <div class="field"><span class="hint">Astheno can’t look names up. The first person to sign in with this username gets the access.</span></div> : undefined,
...@@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {...@@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {
638 <div class="toggles" role="group" aria-label="Groups">637 <div class="toggles" role="group" aria-label="Groups">
639 <For each={props.data.groups}>638 <For each={props.data.groups}>
640 {(group) => (639 {(group) => (
641 <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" && group.name !== "knife-native" || busy() === group.id}640 <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" || busy() === group.id}
642 data-tip={groupTip(group.name, props.data)} onClick={() => toggleGroup(group)}>641 data-tip={groupTip(group.name, props.data)} onClick={() => toggleGroup(group)}>
643 {group.name}642 {group.name}
644 </button>643 </button>
flake.nix-1
...@@ -5,7 +5,6 @@...@@ -5,7 +5,6 @@
55
6 outputs = { self, nixpkgs, ... }: {6 outputs = { self, nixpkgs, ... }: {
7 packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { };7 packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { };
8 packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { };
9 packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { };8 packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { };
10 packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { };9 packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { };
11 packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image;10 packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image;
nixos/configuration.nix+2-3
...@@ -6,12 +6,11 @@ let...@@ -6,12 +6,11 @@ let
6 proxyToken = "/var/lib/snowglobe/dashboard-proxy.token";6 proxyToken = "/var/lib/snowglobe/dashboard-proxy.token";
7 hostTools = lib.fileset.toSource {7 hostTools = lib.fileset.toSource {
8 root = ../.;8 root = ../.;
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];
10 };10 };
11 computerAhk = pkgs.callPackage ./computer-ahk.nix { };11 computerAhk = pkgs.callPackage ./computer-ahk.nix { };
12 computerWindows = pkgs.callPackage ./computer-windows.nix { };12 computerWindows = pkgs.callPackage ./computer-windows.nix { };
13 nativePkl = pkgs.callPackage ./pkl.nix { };13 nativePkl = pkgs.callPackage ./pkl.nix { };
14 hostPython = pkgs.callPackage ./host-python.nix { };
15 screen = pkgs.callPackage ./vm-screen.nix { };14 screen = pkgs.callPackage ./vm-screen.nix { };
16 secureOvmf = pkgs.OVMF.override {15 secureOvmf = pkgs.OVMF.override {
17 secureBoot = true;16 secureBoot = true;
...@@ -264,7 +263,7 @@ in...@@ -264,7 +263,7 @@ in
264 SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables;263 SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables;
265 };264 };
266 serviceConfig = {265 serviceConfig = {
267 ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py";266 ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py";
268 Type = "notify";267 Type = "notify";
269 Group = "snowglobe-dashboard";268 Group = "snowglobe-dashboard";
270 StateDirectory = "snowglobe/host";269 StateDirectory = "snowglobe/host";
nixos/host-python.nix deleted-6
...@@ -1,6 +0,0 @@
1{ python3, git }:
2python3.buildEnv.override {
3 extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ];
4 # shale-access serves anonymous Knife clones through git http-backend.
5 makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ];
6}
nixos/mac-studio-sandbox.nix+2-2
...@@ -50,11 +50,11 @@ in...@@ -50,11 +50,11 @@ in
50 enable = true;50 enable = true;
51 hostName = "10.0.0.1";51 hostName = "10.0.0.1";
52 exports = ''52 exports = ''
53 ${mountPoint} ${workspace.clientAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)53 ${mountPoint} ${workspace.guestAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
54 '';54 '';
55 };55 };
56 environment.etc."exports.d/snowglobe-work.exports".text = ''56 environment.etc."exports.d/snowglobe-work.exports".text = ''
57 ${mountPoint}/.work ${workspace.clientAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)57 ${mountPoint}/.work ${workspace.guestAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
58 '';58 '';
59 systemd.services.mac-work-share-check = {59 systemd.services.mac-work-share-check = {
60 requires = [ "zfs-import-work.service" ];60 requires = [ "zfs-import-work.service" ];
nixos/mac-vm.nix+2-2
...@@ -5,7 +5,7 @@ let...@@ -5,7 +5,7 @@ let
5in {5in {
6 systemd.services.snowglobe-host = {6 systemd.services.snowglobe-host = {
7 environment.SNOWGLOBE_MAC_VM = builtins.toJSON {7 environment.SNOWGLOBE_MAC_VM = builtins.toJSON {
8 address = workspace.clientAddress;8 address = workspace.hostAddress;
9 port = 22;9 port = 22;
10 user = "clo";10 user = "clo";
11 identity = "/var/lib/snowglobe/mac-vm/ssh_key";11 identity = "/var/lib/snowglobe/mac-vm/ssh_key";
...@@ -15,6 +15,6 @@ in {...@@ -15,6 +15,6 @@ in {
15 { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; }15 { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; }
16 ];16 ];
17 };17 };
18 serviceConfig.IPAddressAllow = [ "${workspace.clientAddress}/32" ];18 serviceConfig.IPAddressAllow = [ "${workspace.hostAddress}/32" ];
19 };19 };
20}20}
nixos/snowglobe.nix+1-1
...@@ -29,7 +29,7 @@ in...@@ -29,7 +29,7 @@ in
29 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''29 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
30 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept30 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
31 '') config.networking.interfaces.enp4s0.ipv4.addresses + ''31 '') config.networking.interfaces.enp4s0.ipv4.addresses + ''
32 iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).clientAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept32 iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).guestAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept
33 ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept33 ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept
34 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept34 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
35 '';35 '';
service/shale/prepare.py+42-6
...@@ -9,13 +9,8 @@ import sys...@@ -9,13 +9,8 @@ import sys
99
10data = json.load(sys.stdin)10data = json.load(sys.stdin)
11root = Path(data["hostRoot"])11root = Path(data["hostRoot"])
12for config in (root / "repositories_owned").glob("*/config"):
13 text = config.read_text()
14 if not re.search(r"(?im)^\s*receivepack\s*=", text):
15 # Shale authorizes pushes before CGI; its own repository initializer enables this.
16 with config.open("a") as file:
17 file.write("\n[http]\n\treceivepack = true\n")
18path = root / "data/astheno.shale.db"12path = root / "data/astheno.shale.db"
13webhooks = {}
19if path.exists():14if path.exists():
20 domain = os.environ["SNOWGLOBE_DOMAIN"]15 domain = os.environ["SNOWGLOBE_DOMAIN"]
21 old, new = "auth." + domain + "/realms/master", "snowglobe." + domain16 old, new = "auth." + domain + "/realms/master", "snowglobe." + domain
...@@ -25,6 +20,47 @@ if path.exists():...@@ -25,6 +20,47 @@ if path.exists():
25 raise ValueError("duplicate Shale identity; resolve it before moving the issuer")20 raise ValueError("duplicate Shale identity; resolve it before moving the issuer")
26 count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount21 count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount
27 db.commit()22 db.commit()
23 webhooks = dict(db.execute(
24 "SELECT uuid, name FROM repositories WHERE name IN ('snowcord', 'snowcord-server')"
25 ))
28 assert db.execute("PRAGMA quick_check").fetchone() == ("ok",)26 assert db.execute("PRAGMA quick_check").fetchone() == ("ok",)
29 db.close()27 db.close()
30 print("Moved", count, "existing Shale identity bindings to Snowglobe")28 print("Moved", count, "existing Shale identity bindings to Snowglobe")
29
30executable = root / "data/shale-webhook"
31source = Path(__file__).with_name("webhook.sh").read_bytes()
32if not executable.exists() or executable.read_bytes() != source:
33 pending = executable.with_suffix(".pending")
34 pending.write_bytes(source)
35 if os.geteuid() == 0:
36 os.chown(pending, data["uid"], data["uid"])
37 pending.replace(executable)
38executable.chmod(0o750)
39if os.geteuid() == 0:
40 os.chown(executable, data["uid"], data["uid"])
41
42target = "/data/shale-webhook"
43for config in (root / "repositories_owned").glob("*/config"):
44 original = text = config.read_text()
45 if not re.search(r"(?im)^\s*receivepack\s*=", text):
46 # Shale authorizes pushes before CGI; its own repository initializer enables this.
47 text += "\n[http]\n\treceivepack = true\n"
48 text = re.sub(r'(?m)\n+\[shale "snowglobe"\]\n(?:[ \t].*\n?)*', "\n", text)
49 name = webhooks.get(config.parent.name)
50 if name:
51 text += f'\n[shale "snowglobe"]\n\trepository = {name}\n'
52 if text != original:
53 config.write_text(text)
54
55 hook = config.parent / "hooks/post-receive"
56 if name:
57 hook.parent.mkdir(exist_ok=True)
58 if os.path.lexists(hook):
59 if not hook.is_symlink() or os.readlink(hook) != target:
60 raise ValueError(f"refusing to replace existing hook: {hook}")
61 if not os.path.lexists(hook):
62 hook.symlink_to(target)
63 if os.geteuid() == 0:
64 os.lchown(hook, data["uid"], data["uid"])
65 elif hook.is_symlink() and os.readlink(hook) == target:
66 hook.unlink()
service/shale/service.pkl+3
...@@ -61,6 +61,8 @@ container {...@@ -61,6 +61,8 @@ container {
61 ["MIRRORS_DISABLE_CLONE"] = "true"61 ["MIRRORS_DISABLE_CLONE"] = "true"
62 ["MIRRORS_DISABLE_PULL_POLL"] = "true"62 ["MIRRORS_DISABLE_PULL_POLL"] = "true"
63 }63 }
64 ["SHALE_WEBHOOK_ENABLED"] = if (site.preview) "false" else "true"
65 ["SHALE_WEBHOOK_URL"] = "${secret.own.webhook_url}"
64 ["SESSION_SECRET"] = "${secret.own.session_secret}"66 ["SESSION_SECRET"] = "${secret.own.session_secret}"
65 ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}"67 ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
66 }68 }
...@@ -69,3 +71,4 @@ container {...@@ -69,3 +71,4 @@ container {
69secrets {71secrets {
70 ["session_secret"] {}72 ["session_secret"] {}
71}73}
74requiredSecrets { "webhook_url" }
service/shale/webhook.sh created+144
...@@ -0,0 +1,144 @@
1#!/bin/sh
2set -u
3
4[ "${SHALE_WEBHOOK_ENABLED:-false}" = true ] || exit 0
5[ -n "${SHALE_WEBHOOK_URL:-}" ] || exit 0
6case "$SHALE_WEBHOOK_URL" in
7 */github|*/github/) webhook_url=${SHALE_WEBHOOK_URL%/}/ ;;
8 *) webhook_url=${SHALE_WEBHOOK_URL%/}/github/ ;;
9esac
10
11repository=$(git config --get shale.snowglobe.repository) || exit 0
12origin="https://${DOMAIN}/${repository}"
13avatar="https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png"
14
15json_string() {
16 awk 'BEGIN { ORS = "" }
17 {
18 if (NR > 1) printf "\\n"
19 for (i = 1; i <= length($0); i++) {
20 c = substr($0, i, 1)
21 if (c == "\\") printf "\\\\"
22 else if (c == "\"") printf "\\\""
23 else if (c == "\t") printf "\\t"
24 else if (c == "\r") printf "\\r"
25 else if (c == sprintf("%c", 8)) printf "\\b"
26 else if (c == sprintf("%c", 12)) printf "\\f"
27 else printf "%s", c
28 }
29 }'
30}
31
32quote() {
33 printf '"'
34 printf '%s' "$1" | json_string
35 printf '"'
36}
37
38is_zero() {
39 case "$1" in
40 *[!0]*) return 1 ;;
41 *) return 0 ;;
42 esac
43}
44
45write_ref_payload() {
46 {
47 printf '{"ref":'; quote "$short_ref"
48 printf ',"ref_type":'; quote "$ref_type"
49 printf ',"repository":{"name":'; quote "$repository"
50 printf ',"full_name":'; quote "$repository"
51 printf ',"html_url":'; quote "$origin/"
52 printf '},"sender":{"login":'; quote "$actor"
53 printf ',"avatar_url":'; quote "$avatar"
54 printf ',"html_url":'; quote "$origin/"
55 printf '}}\n'
56 } >"$payload"
57}
58
59send() {
60 event=$1
61 payload=$2
62 delivery=$(printf '%s\n' "$repository:$event:$(date +%s):$$:$index" | git hash-object --stdin)
63 curl --fail --silent --show-error --max-time 10 \
64 -H "Content-Type: application/json" \
65 -H "User-Agent: Shale-Hookshot/1.0" \
66 -H "X-GitHub-Event: $event" \
67 -H "X-GitHub-Delivery: $delivery" \
68 --data-binary "@$payload" "$webhook_url" >/dev/null
69}
70
71temporary=$(mktemp -d)
72trap 'rm -rf "$temporary"' EXIT HUP INT TERM
73index=0
74while read -r before after ref; do
75 index=$((index + 1))
76 payload="$temporary/$index.json"
77 short_ref=${ref#refs/heads/}
78 if [ "$short_ref" = "$ref" ]; then
79 short_ref=${ref#refs/tags/}
80 ref_type=tag
81 else
82 ref_type=branch
83 fi
84
85 if is_zero "$after"; then
86 actor=${REMOTE_USER:-$(git show -s --format=%an "$before")}
87 write_ref_payload
88 send delete "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
89 continue
90 fi
91
92 actor=${REMOTE_USER:-$(git show -s --format=%an "$after")}
93 if [ "$ref_type" = tag ]; then
94 if is_zero "$before"; then
95 write_ref_payload
96 send create "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
97 fi
98 continue
99 fi
100 if is_zero "$before"; then
101 created=true
102 forced=false
103 commits=$(git rev-list --reverse "$after")
104 else
105 created=false
106 if git merge-base --is-ancestor "$before" "$after"; then forced=false; else forced=true; fi
107 commits=$(git rev-list --reverse "$before..$after")
108 fi
109 compare="$origin/commit/$after"
110 {
111 printf '{"ref":'; quote "$ref"
112 printf ',"before":'; quote "$before"
113 printf ',"after":'; quote "$after"
114 printf ',"created":%s,"deleted":false,"forced":%s' "$created" "$forced"
115 printf ',"compare":'; quote "$compare"
116 printf ',"repository":{"name":'; quote "$repository"
117 printf ',"full_name":'; quote "$repository"
118 printf ',"html_url":'; quote "$origin/"
119 printf '},"sender":{"login":'; quote "$actor"
120 printf ',"avatar_url":'; quote "$avatar"
121 printf ',"html_url":'; quote "$origin/"
122 printf '},"commits":['
123 separator=
124 for commit in $commits; do
125 message=$(git show -s --format=%B "$commit")
126 author=$(git show -s --format=%an "$commit")
127 timestamp=$(git show -s --format=%aI "$commit")
128 printf '%s{"id":' "$separator"; quote "$commit"
129 printf ',"message":'; quote "$message"
130 printf ',"timestamp":'; quote "$timestamp"
131 printf ',"url":'; quote "$origin/commit/$commit"
132 printf ',"author":{"name":'; quote "$author"
133 printf ',"username":'; quote "$author"
134 printf '}}'
135 separator=,
136 done
137 printf '],"head_commit":{"id":'; quote "$after"
138 printf ',"message":'; quote "$(git show -s --format=%B "$after")"
139 printf ',"timestamp":'; quote "$(git show -s --format=%aI "$after")"
140 printf ',"url":'; quote "$origin/commit/$after"
141 printf '}}\n'
142 } >"$payload"
143 send push "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
144done
tools/dashboard-host.py-30
...@@ -16,7 +16,6 @@ import signal...@@ -16,7 +16,6 @@ import signal
16import socket16import socket
17import struct17import struct
18import subprocess18import subprocess
19import sys
20import threading19import threading
21import time20import time
22import vms21import vms
...@@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4...@@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4
45WORK_SLOTS = threading.BoundedSemaphore(WORKERS)44WORK_SLOTS = threading.BoundedSemaphore(WORKERS)
46STREAM_SLOTS = threading.BoundedSemaphore(4)45STREAM_SLOTS = threading.BoundedSemaphore(4)
47GUEST_SLOTS = threading.BoundedSemaphore(4)46GUEST_SLOTS = threading.BoundedSemaphore(4)
48CLONE_SLOTS = threading.BoundedSemaphore(2)
49VM_PREPARATION_SLOT = threading.BoundedSemaphore(1)47VM_PREPARATION_SLOT = threading.BoundedSemaphore(1)
50INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?"48INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?"
5149
...@@ -97,8 +95,6 @@ class Host:...@@ -97,8 +95,6 @@ class Host:
97 "index.diff": {"dataset", "from", "to"},95 "index.diff": {"dataset", "from", "to"},
98 "index.discard": {"dataset", "snapshot"},96 "index.discard": {"dataset", "snapshot"},
99 "shale.metadata": {"repository", "accounts"},97 "shale.metadata": {"repository", "accounts"},
100 "shale.access": {"service", "uri", "account", "authorization"},
101 "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"},
102 "shale.write": {"grant", "repository", "action", "arguments"},98 "shale.write": {"grant", "repository", "action", "arguments"},
103 "host.sample": set(),99 "host.sample": set(),
104 "host.usage": {"refresh"},100 "host.usage": {"refresh"},
...@@ -113,10 +109,6 @@ class Host:...@@ -113,10 +109,6 @@ class Host:
113 raise Rejected("Use only the fields required by this host operation.")109 raise Rejected("Use only the fields required by this host operation.")
114 if operation == "copyparty.manage":110 if operation == "copyparty.manage":
115 return copyparty.handle(request)111 return copyparty.handle(request)
116 if operation == "shale.access":
117 return importlib.import_module("shale-access").access(request)
118 if operation == "shale.clone":
119 return importlib.import_module("shale-access").clone(request)
120 if operation == "shale.metadata":112 if operation == "shale.metadata":
121 return shale_labels.metadata(request)113 return shale_labels.metadata(request)
122 if operation == "shale.write":114 if operation == "shale.write":
...@@ -473,20 +465,6 @@ def stream_upload(connection, request):...@@ -473,20 +465,6 @@ def stream_upload(connection, request):
473 Path(temporary).unlink(missing_ok=True)465 Path(temporary).unlink(missing_ok=True)
474466
475467
476def stream_clone(connection, plan):
477 owner = os.stat(plan["objects"])
478 # Service identities have numeric owners without passwd entries, which systemd's User= rejects.
479 command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec",
480 *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes",
481 "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")),
482 *(f"--setenv={name}={value}" for name, value in plan["environment"].items()),
483 "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--",
484 sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]]
485 send_response(connection, {"value": None})
486 connection.setblocking(True)
487 subprocess.run(command, stdin=connection, stdout=connection)
488
489
490def serve_connection(connection, host, allowed_uid, allowed_operations=None):468def serve_connection(connection, host, allowed_uid, allowed_operations=None):
491 with connection, contextlib.ExitStack() as resources:469 with connection, contextlib.ExitStack() as resources:
492 connection.settimeout(65)470 connection.settimeout(65)
...@@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):...@@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
553 return531 return
554 finally:532 finally:
555 streams.release()533 streams.release()
556 if operation == "shale.clone":
557 if not CLONE_SLOTS.acquire(blocking=False):
558 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
559 try:
560 stream_clone(connection, host.handle(request))
561 return
562 finally:
563 CLONE_SLOTS.release()
564 response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)}534 response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)}
565 except Rejected as error:535 except Rejected as error:
566 response = {"error": str(error), "status": 400}536 response = {"error": str(error), "status": 400}
tools/dashboard-shale-page-test.py+2-41
...@@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer...@@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json6import json
7import os7import os
8from pathlib import Path8from pathlib import Path
9import re
10import socket9import socket
11import socketserver
12import sqlite310import sqlite3
13import subprocess11import subprocess
14import tempfile12import tempfile
...@@ -49,21 +47,12 @@ def main():...@@ -49,21 +47,12 @@ def main():
49 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'47 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'
50 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'48 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'
51 blob = bytes(range(256)) * 6553649 blob = bytes(range(256)) * 65536
52 csrf = 'initial'
5350
54 class Fixture(BaseHTTPRequestHandler):51 class Fixture(BaseHTTPRequestHandler):
55 protocol_version = 'HTTP/1.1'52 protocol_version = 'HTTP/1.1'
5653
57 def do_GET(self):54 def do_GET(self):
58 nonlocal csrf
59 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))55 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
60 if self.path == '/':
61 account = b'<a class="usa-nav-link" href="/~fixture-member">~fixture-member</a>' if self.headers.get('Cookie') == 'SessionID=preserved' else b''
62 return self.respond(b'<html><body><header class="usa-header--basic"><nav>' + account + b'</nav></header></body></html>', 'text/html')
63 if self.path == '/-/settings' or self.path.endswith('/issues/new'):
64 csrf += 'x'
65 account = '<body id="page-user-settings"><kbd>fixture-member</kbd>' if self.path == '/-/settings' else '<body>'
66 return self.respond(('<html>' + account + '<form method="post"><input type="hidden" name="csrf_token" value="' + csrf + '"></form></body></html>').encode(), 'text/html')
67 if self.path == '/binary':56 if self.path == '/binary':
68 return self.respond(blob, 'application/octet-stream')57 return self.respond(blob, 'application/octet-stream')
69 if self.path == '/json':58 if self.path == '/json':
...@@ -86,8 +75,6 @@ def main():...@@ -86,8 +75,6 @@ def main():
86 def do_POST(self):75 def do_POST(self):
87 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))76 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
88 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))77 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
89 if self.path.endswith('/issues/new') and urllib.parse.parse_qs(body.decode()).get('csrf_token') != [csrf]:
90 return self.respond(b'E: unexpected csrf token', 'text/plain', 403)
91 self.respond(body, 'application/octet-stream', 201)78 self.respond(body, 'application/octet-stream', 201)
9279
93 def respond(self, body, content_type, status=200, headers=()):80 def respond(self, body, content_type, status=200, headers=()):
...@@ -121,21 +108,7 @@ def main():...@@ -121,21 +108,7 @@ def main():
121 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)108 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
122 data = root / 'data'109 data = root / 'data'
123 data.mkdir()110 data.mkdir()
124 class Host(socketserver.StreamRequestHandler):111 environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')}
125 def handle(self):
126 request = json.loads(self.rfile.readline())
127 value = {}
128 if request['operation'] == 'shale.access':
129 allowed = request['account'] == 'fixture-member'
130 value = {'allowed': allowed, 'deny': not allowed}
131 response = json.dumps({'value': value}).encode()
132 self.wfile.write(len(response).to_bytes(4, 'big') + response)
133
134 host = socketserver.UnixStreamServer(str(root / 'host.sock'), Host)
135 stack.callback(host.server_close)
136 stack.callback(host.shutdown)
137 threading.Thread(target=host.serve_forever, daemon=True).start()
138 environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_HOST_SOCKET': str(root / 'host.sock'), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')}
139 for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']:112 for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']:
140 environment.pop(key, None)113 environment.pop(key, None)
141 log = stack.enter_context((root / 'dashboard.log').open('w'))114 log = stack.enter_context((root / 'dashboard.log').open('w'))
...@@ -253,22 +226,10 @@ def main():...@@ -253,22 +226,10 @@ def main():
253 assert len(request('/large')[2]) == 9 * 1024 * 1024226 assert len(request('/large')[2]) == 9 * 1024 * 1024
254 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')227 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')
255 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'228 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'
256 for path in ['/the-knife/issues/new', '/another-repo/issues/new']:
257 status, _, form = request(path)
258 assert status == 200
259 csrf_field = re.search(rb'name="csrf_token" value="([^"]+)"', form)[1]
260 assert request('/the-knife/')[0] == 200
261 submitted = b'csrf_token=' + csrf_field + b'&title=preserved'
262 status, _, body = request(path, body=submitted)
263 assert status == 201 and body == submitted, (path, status, body)
264 for invalid in [b'title=missing', b'csrf_token=invalid']:
265 assert request(path, body=invalid)[0] == 403
266 assert not any(item[1] == '/-/settings' for item in observations)
267 assert request('/the-knife/issues/new', headers={'Cookie': ''})[0] == 403
268 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403229 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403
269 assert request('/auth/shale/page', direct=True)[0] == 403230 assert request('/auth/shale/page', direct=True)[0] == 403
270 assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403231 assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403
271 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'csrf_preservation_and_rejection': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)232 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
272 if args.browser_ready_file:233 if args.browser_ready_file:
273 args.browser_ready_file.write_text(json.dumps({'origin': origin}))234 args.browser_ready_file.write_text(json.dumps({'origin': origin}))
274 stop = args.browser_ready_file.with_suffix('.stop')235 stop = args.browser_ready_file.with_suffix('.stop')
tools/dashboard-stage.py+2-14
...@@ -21,7 +21,6 @@ import snowglobe...@@ -21,7 +21,6 @@ import snowglobe
2121
2222
23OPERATIONS = frozenset({23OPERATIONS = frozenset({
24 "shale.access", "shale.clone",
25 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",24 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",
26 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",25 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",
27 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",26 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",
...@@ -56,16 +55,6 @@ class PreviewHost(boundary.Host):...@@ -56,16 +55,6 @@ class PreviewHost(boundary.Host):
56 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}55 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}
57 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:56 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:
58 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")57 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")
59 if operation in {"shale.access", "shale.clone"}:
60 service = request.get("service", "")
61 if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
62 raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.")
63 linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text())
64 if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name:
65 raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.")
66 if operation == "shale.clone":
67 return import_module("shale-access").clone(request)
68 return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data")
69 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:58 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:
70 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]59 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]
71 if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]:60 if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]:
...@@ -234,14 +223,13 @@ def stage(computer=None):...@@ -234,14 +223,13 @@ def stage(computer=None):
234 for entry in mac["vms"]:223 for entry in mac["vms"]:
235 entry["owner"] = owner[0]224 entry["owner"] = owner[0]
236 computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac))225 computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac))
237 host_python = snowglobe.command(*nix, "build", "--out-link", str(root / "host-python"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip()
238 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)226 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
239 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)227 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
240 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity,228 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity,
241 "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32",229 "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32",
242 "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"],230 "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"],
243 *computer_environment,231 *computer_environment,
244 host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity)232 sys.executable, str(Path(__file__).resolve()), "--serve", identity)
245 for _ in range(100):233 for _ in range(100):
246 if Path(runtime, "host.sock").exists():234 if Path(runtime, "host.sock").exists():
247 break235 break
...@@ -263,7 +251,7 @@ def stage(computer=None):...@@ -263,7 +251,7 @@ def stage(computer=None):
263 "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock",251 "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock",
264 "--setenv=XDG_RUNTIME_DIR=" + screen_runtime,252 "--setenv=XDG_RUNTIME_DIR=" + screen_runtime,
265 "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache",253 "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache",
266 host_python + "/bin/python3", "-c", launcher)254 sys.executable, "-c", launcher)
267 for _ in range(100):255 for _ in range(100):
268 if Path(screen_runtime, "screen.sock").exists():256 if Path(screen_runtime, "screen.sock").exists():
269 break257 break
tools/mac-agent-forward-test.py+1-2
...@@ -35,8 +35,7 @@ sys.stdin.buffer.read()...@@ -35,8 +35,7 @@ sys.stdin.buffer.read()
35server.shutdown()35server.shutdown()
36server.server_close()36server.server_close()
37'''37'''
38 ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip()38 server = subprocess.Popen([*vm.ssh_args(), vm.GUEST, "python3 -u -c " + shlex.quote(code)],
39 server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)],
40 stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)39 stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
41 tunnel = None40 tunnel = None
42 try:41 try:
tools/mac-agent-vm.md+32-3
...@@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console...@@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console
27python3 tools/mac-agent-vm.py stop27python3 tools/mac-agent-vm.py stop
28```28```
2929
30Everything except `exec` reaches the guest over pinned SSH at its Bonjour name.
31`exec` and the dashboard's computer operations go through Tart's guest agent so
32they run in the logged-in GUI session, and that path can crash the VM runner
33(`/Library/Logs/DiagnosticReports/tart-*.ips`, `-[VZVirtioSocketDevice
34connectToPort:]`). Use `ssh` for anything that does not need the GUI session.
35
36The `net.paperclover.mac-vm-revive` LaunchAgent runs `revive` every 30 seconds
37through `ssh localhost` (launchd jobs cannot read the VM volume). It starts the
38VM when its runner is gone unless `stop` or `suspend` left `private/stopped`;
39`start` clears that marker. Restarts and the newest Tart crash report are logged
40to `~/Library/Logs/mac-vm-revive.log`. A hung guest under a live runner is not
41restarted.
42
30To reach a server bound to guest loopback from this Mac:43To reach a server bound to guest loopback from this Mac:
3144
32```sh45```sh
...@@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running...@@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running
50VM holds a process-scoped idle-sleep assertion, so locking the host does not63VM holds a process-scoped idle-sleep assertion, so locking the host does not
51cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.64cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.
5265
66## Network and iPhones
67
68The guest is bridged onto the LAN through the host's default-route interface
69so iPhones can find it over Bonjour and connect in; Virtualization.framework
70has no USB passthrough for them. Its DHCP address and the host's are reserved
71on the router and recorded in
72[mac-studio-sandbox.json](../config/mac-studio-sandbox.json): Snow Globe
73exports NFS to `guestAddress` and the dashboard reaches the host at
74`hostAddress`. Every `start` loads a pf anchor (`com.apple/net.paperclover`)
75in the guest that refuses new inbound TCP below port 49152 from anyone but
76the host, plus Kerberos and NetBIOS over UDP. A guest that reboots itself
77stays open to the LAN until the next `start`. iPhones on iOS 27 pair in Device
78Hub with **+ → Pair Nearby Device…** and Settings → Privacy & Security →
79Developer Mode on the phone. Older iOS needs a cable, which the guest cannot
80have.
81
53The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve82The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve
54physical cores. CPU count and RAM changes require a cold boot. No automatic83physical cores. CPU count and RAM changes require a cold boot. No automatic
55lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget84lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget
...@@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker...@@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker
180is present.209is present.
181210
182The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address211The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address
183`10.0.0.1`. Traffic uses the VM's NAT gateway `192.168.64.1`; the host's hosts212`10.0.0.1`. The host's hosts file is not inherited. The previous guest file is
184file is not inherited. The previous guest file is `/etc/hosts.before-code-nas`.213`/etc/hosts.before-code-nas`.
185214
186SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned215SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned
187in `private/known_hosts`, and disables password authentication. No host SSH216in `private/known_hosts`, and disables password authentication. No host SSH
...@@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut...@@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut
222It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding251It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding
223guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles252guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles
224the guest account authentication from `private/password` server-side. Screen253the guest account authentication from `private/password` server-side. Screen
225Sharing listens on all interfaces inside the private NAT guest; the host relay's254Sharing listens on all guest interfaces behind the pf anchor; the host relay's
226TCP listeners default to loopback and check the exact HTTP Host and WebSocket255TCP listeners default to loopback and check the exact HTTP Host and WebSocket
227Origin. Tart's experimental host VNC server is disabled. The viewer trusts256Origin. Tart's experimental host VNC server is disabled. The viewer trusts
228every client that can reach its listener; Origin checking protects against foreign257every client that can reach its listener; Origin checking protects against foreign
tools/mac-agent-vm.py+77-36
...@@ -8,6 +8,7 @@ import logging...@@ -8,6 +8,7 @@ import logging
8import multiprocessing8import multiprocessing
9import os9import os
10from pathlib import Path10from pathlib import Path
11import re
11import select12import select
12import secrets13import secrets
13import shlex14import shlex
...@@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod...@@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod
26TART = ROOT / "runtime/tart.app/Contents/MacOS/tart"27TART = ROOT / "runtime/tart.app/Contents/MacOS/tart"
27NAME = "gui-agent"28NAME = "gui-agent"
28PRIVATE = ROOT / "private"29PRIVATE = ROOT / "private"
30# Never reach the guest with `tart exec` or `tart ip --resolver=agent` unless it needs the GUI session:
31# their vsock connect intermittently traps inside Virtualization.framework and kills the whole VM.
32GUEST = "admin@AgentVM.local"
2933
3034
31def tart(*args, **kwargs):35def tart(*args, **kwargs):
...@@ -35,12 +39,27 @@ def tart(*args, **kwargs):...@@ -35,12 +39,27 @@ def tart(*args, **kwargs):
3539
36def ssh_args():40def ssh_args():
37 return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"),41 return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"),
38 "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no",42 "-o", "AddressFamily=inet", "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no",
39 "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME,43 "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME,
40 "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"',44 "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"',
41 "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"]45 "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"]
4246
4347
48def guest(*command, **kwargs):
49 kwargs.setdefault("check", True)
50 return subprocess.run([*ssh_args(), GUEST, shlex.join(command)], **kwargs)
51
52
53def guest_ready(**kwargs):
54 guest("/bin/sh", "-c", '[ "$(/usr/bin/stat -f %Su /dev/console)" = admin ]', **kwargs)
55
56
57def shut_down():
58 # ssh can drop as the guest powers off, so the stopped state is the proof.
59 guest("sudo", "-n", "shutdown", "-h", "now", check=False, timeout=15)
60 wait_for_state("stopped")
61
62
44def exact(stream, size):63def exact(stream, size):
45 data = bytearray()64 data = bytearray()
46 while len(data) < size:65 while len(data) < size:
...@@ -51,10 +70,14 @@ def exact(stream, size):...@@ -51,10 +70,14 @@ def exact(stream, size):
51 return bytes(data)70 return bytes(data)
5271
5372
73def vm():
74 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)
75 return next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")
76
77
54def wait_for_state(state):78def wait_for_state(state):
55 for _ in range(90):79 for _ in range(90):
56 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)80 if vm()["State"] == state:
57 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["State"] == state:
58 return81 return
59 time.sleep(1)82 time.sleep(1)
60 raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.")83 raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.")
...@@ -62,13 +85,12 @@ def wait_for_state(state):...@@ -62,13 +85,12 @@ def wait_for_state(state):
6285
63@contextlib.contextmanager86@contextlib.contextmanager
64def guest_screen():87def guest_screen():
65 ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip()
66 with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory:88 with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory:
67 path = Path(directory) / "console.sock"89 path = Path(directory) / "console.sock"
68 tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes",90 tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes",
69 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",91 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",
70 "-o", "StreamLocalBindMask=0177", "-L",92 "-o", "StreamLocalBindMask=0177", "-L",
71 f"{path}:127.0.0.1:5900", "admin@" + ip],93 f"{path}:127.0.0.1:5900", GUEST],
72 stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,94 stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
73 stderr=subprocess.DEVNULL)95 stderr=subprocess.DEVNULL)
74 try:96 try:
...@@ -208,6 +230,7 @@ def main():...@@ -208,6 +230,7 @@ def main():
208 sub.add_parser("stop")230 sub.add_parser("stop")
209 sub.add_parser("suspend")231 sub.add_parser("suspend")
210 sub.add_parser("reboot")232 sub.add_parser("reboot")
233 sub.add_parser("revive", help="Start the VM unless it is running or was stopped on purpose")
211 for action in ["exec", "ssh"]:234 for action in ["exec", "ssh"]:
212 command = sub.add_parser(action)235 command = sub.add_parser(action)
213 command.add_argument("command", nargs=argparse.REMAINDER)236 command.add_argument("command", nargs=argparse.REMAINDER)
...@@ -222,12 +245,17 @@ def main():...@@ -222,12 +245,17 @@ def main():
222 local_port = args.local_port if args.local_port is not None else args.port245 local_port = args.local_port if args.local_port is not None else args.port
223 if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535:246 if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535:
224 parser.error("Ports must be between 1 and 65535.")247 parser.error("Ports must be between 1 and 65535.")
225 ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip()
226 print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.",248 print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.",
227 file=sys.stderr, flush=True)249 file=sys.stderr, flush=True)
228 os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes",250 os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes",
229 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",251 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",
230 "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip])252 "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", GUEST])
253 if args.action == "revive":
254 if (PRIVATE / "stopped").exists() or vm()["Running"]:
255 return
256 crashes = sorted(Path("/Library/Logs/DiagnosticReports").glob("tart-*.ips"))
257 print(f"{time.ctime()}: VM is down; newest tart crash report: {crashes[-1] if crashes else 'none'}", flush=True)
258 args.action = "start"
231 if args.action in {"start", "stop", "suspend", "reboot"}:259 if args.action in {"start", "stop", "suspend", "reboot"}:
232 lock = (PRIVATE / "lifecycle.lock").open("a")260 lock = (PRIVATE / "lifecycle.lock").open("a")
233 os.chmod(PRIVATE / "lifecycle.lock", 0o600)261 os.chmod(PRIVATE / "lifecycle.lock", 0o600)
...@@ -240,34 +268,47 @@ def main():...@@ -240,34 +268,47 @@ def main():
240 lock.write(args.action)268 lock.write(args.action)
241 lock.flush()269 lock.flush()
242 if args.action == "reboot":270 if args.action == "reboot":
243 tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now", timeout=15)271 shut_down()
244 wait_for_state("stopped")
245 args.action = "start"272 args.action = "start"
246 if args.action == "start":273 if args.action == "start":
247 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)274 (PRIVATE / "stopped").unlink(missing_ok=True)
248 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]:275 interface = re.search(r"interface: (\S+)", subprocess.run(["/sbin/route", "-n", "get", "default"],
276 capture_output=True, text=True, check=True).stdout)[1]
277 if vm()["Running"]:
249 print("GUI-agent VM is already running.")278 print("GUI-agent VM is already running.")
250 return279 else:
251 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.280 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.
252 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",281 # Bridged so iPhones can find and pair with the guest's Xcode over Bonjour.
253 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",282 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",
254 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"])283 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",
255 (PRIVATE / "run.log").write_text("")284 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full",
256 os.chmod(PRIVATE / "run.log", 0o600)285 f"--net-bridged={interface}"])
257 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",286 (PRIVATE / "run.log").write_text("")
258 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)287 os.chmod(PRIVATE / "run.log", 0o600)
259 deadline = time.monotonic() + 90288 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",
260 while time.monotonic() < deadline:289 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)
261 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:290 deadline = time.monotonic() + 90
262 raise ValueError("VM startup failed; inspect the private run log locally.")291 while True:
263 try:292 if time.monotonic() > deadline:
264 tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5)293 raise ValueError("VM startup is still pending; inspect status before retrying.")
265 except (subprocess.CalledProcessError, subprocess.TimeoutExpired):294 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:
266 time.sleep(1)295 raise ValueError("VM startup failed; inspect the private run log locally.")
267 else:296 try:
268 print("GUI-agent VM is ready in its logged-in guest session.")297 guest_ready(capture_output=True, timeout=5)
269 return298 except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
270 raise ValueError("VM startup is still pending; inspect status before retrying.")299 time.sleep(1)
300 else:
301 break
302 print("GUI-agent VM is ready in its logged-in guest session.")
303 # The bridged guest is on the LAN: only this Mac may reach its services; iPhones pair on ephemeral ports.
304 host = subprocess.run(["/usr/sbin/ipconfig", "getifaddr", interface],
305 capture_output=True, text=True, check=True).stdout.strip()
306 guest("sudo", "-n", "/bin/sh", "-c",
307 "/sbin/pfctl -q -a com.apple/net.paperclover -f - && /sbin/pfctl -q -E",
308 input=f"table <mac-host> {{ {host} }}\n"
309 "block in quick on ! lo0 proto tcp from ! <mac-host> to any port < 49152 flags S/SA\n"
310 "block in quick on ! lo0 proto udp to any port { 88 137 138 }\n",
311 capture_output=True, text=True)
271 elif args.action == "status":312 elif args.action == "status":
272 tart("get", NAME)313 tart("get", NAME)
273 elif args.action == "exec":314 elif args.action == "exec":
...@@ -275,18 +316,18 @@ def main():...@@ -275,18 +316,18 @@ def main():
275 parser.error("exec requires a guest command")316 parser.error("exec requires a guest command")
276 tart("exec", "-i", NAME, *args.command)317 tart("exec", "-i", NAME, *args.command)
277 elif args.action == "ssh":318 elif args.action == "ssh":
278 ip = tart("ip", NAME, capture_output=True, text=True).stdout.strip()
279 command = ssh_args()319 command = ssh_args()
280 if not args.command:320 if not args.command:
281 command.append("-t")321 command.append("-t")
282 os.execvp("ssh", [*command, "admin@" + ip, *args.command])322 os.execvp("ssh", [*command, GUEST, *args.command])
283 elif args.action == "stop":323 elif args.action == "stop":
284 tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now")324 shut_down()
285 wait_for_state("stopped")325 (PRIVATE / "stopped").touch()
286 print("GUI-agent VM shut down.")326 print("GUI-agent VM shut down.")
287 elif args.action == "suspend":327 elif args.action == "suspend":
288 tart("suspend", NAME)328 tart("suspend", NAME)
289 wait_for_state("suspended")329 wait_for_state("suspended")
330 (PRIVATE / "stopped").touch()
290 print("GUI-agent VM suspended.")331 print("GUI-agent VM suspended.")
291 elif args.action == "console":332 elif args.action == "console":
292 python = ROOT / "runtime/venv/bin/python"333 python = ROOT / "runtime/venv/bin/python"
tools/mac-vm-bridge.py+1-1
...@@ -32,7 +32,7 @@ def status():...@@ -32,7 +32,7 @@ def status():
32 agent = None32 agent = None
33 if current["Running"]:33 if current["Running"]:
34 try:34 try:
35 vm.tart("exec", vm.NAME, "/usr/bin/true", capture_output=True, timeout=2)35 vm.guest_ready(capture_output=True, timeout=5)
36 agent = "connected"36 agent = "connected"
37 except subprocess.SubprocessError:37 except subprocess.SubprocessError:
38 agent = "disconnected"38 agent = "disconnected"
tools/release.py+1-1
...@@ -60,7 +60,7 @@ def host_digest(root):...@@ -60,7 +60,7 @@ def host_digest(root):
60 digest = hashlib.sha256()60 digest = hashlib.sha256()
61 paths = sorted(path for path, relative in files(root)61 paths = sorted(path for path, relative in files(root)
62 if relative.parts[0] in {"nixos", "dashboard", "config", "service"}62 if relative.parts[0] in {"nixos", "dashboard", "config", "service"}
63 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"})63 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"})
64 for path in paths:64 for path in paths:
65 digest.update(str(path.relative_to(root)).encode())65 digest.update(str(path.relative_to(root)).encode())
66 _hash_contents(digest, path)66 _hash_contents(digest, path)
tools/shale-access-test.py deleted-230
...@@ -1,230 +0,0 @@
1#!/usr/bin/env python3
2import base64
3from contextlib import closing
4import importlib
5import json
6import os
7from pathlib import Path
8from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9import sqlite3
10import subprocess
11import sys
12import tempfile
13import threading
14import unittest
15from unittest.mock import patch
16
17import pygit2
18from argon2 import PasswordHasher
19
20shale = importlib.import_module('shale-access')
21
22
23class CGI(BaseHTTPRequestHandler):
24 """Plays the Shale gateway: every request passes through clone() before reaching git http-backend."""
25 def __init__(self, test, *args):
26 self.test = test
27 super().__init__(*args)
28
29 def log_message(self, *args):
30 pass
31
32 def do_GET(self):
33 self.relay()
34
35 def do_POST(self):
36 self.relay()
37
38 def relay(self):
39 try:
40 plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', ''))
41 except shale.Error:
42 self.send_response(401)
43 self.send_header('WWW-Authenticate', 'Basic realm="The Knife"')
44 self.end_headers()
45 return
46 body = self.rfile.read(int(self.headers.get('content-length', 0)))
47 output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True,
48 env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout
49 head, _, payload = output.partition(b'\r\n\r\n')
50 headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n'))
51 self.send_response(int(headers.pop('Status', '200').split()[0]))
52 for name, value in headers.items():
53 self.send_header(name, value)
54 self.end_headers()
55 self.wfile.write(payload)
56
57
58class Access(unittest.TestCase):
59 def setUp(self):
60 self.temp = tempfile.TemporaryDirectory()
61 self.addCleanup(self.temp.cleanup)
62 self.root = Path(self.temp.name)
63 self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'})
64 self.env.start()
65 self.addCleanup(self.env.stop)
66 (self.root / 'data').mkdir()
67 self.uuid = '0' * 26
68 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)
69 self.addCleanup(self.repo.free)
70 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})
71 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])
72 self.repo.create_reference('refs/heads/native', self.native)
73 self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native])
74 self.repo.create_reference('refs/heads/main', self.public)
75 self.repo.set_head('refs/heads/main')
76 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))
77 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
78 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')
79 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):
80 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))
81 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))
82 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
83 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')
84 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))
85 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):
86 db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name))
87 db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret')))
88
89 def commit(self, files, parents=None):
90 index = pygit2.Index()
91 for path, content in files.items():
92 index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB))
93 tree = index.write_tree(self.repo)
94 author = pygit2.Signature('Fixture', 'fixture@example.invalid')
95 return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or [])
96
97 def access(self, path, account='', authorization=''):
98 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})
99
100 def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self):
101 released = []
102 class TrackedRepository(pygit2.Repository):
103 def free(self):
104 released.append(True)
105 super().free()
106 with patch.object(shale.pygit2, 'Repository', TrackedRepository):
107 self.assertFalse(self.access('/the-knife/')['deny'])
108 self.assertTrue(self.access('/the-knife/objects/x')['deny'])
109 with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')):
110 with self.assertRaises(ValueError):
111 self.access('/the-knife/')
112 self.assertEqual(len(released), 3)
113
114 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):
115 public = self.access('/the-knife/tree/main/')
116 self.assertFalse(public['deny'])
117 self.assertIn(str(self.public), public['uri'])
118 for rev in ('native', str(self.native), str(self.native)[:12]):
119 for route in ('tree', 'plain', 'commit'):
120 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev))
121 self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny'])
122 self.repo.references.delete('refs/heads/native')
123 self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny'])
124
125 def test_every_revision_blocks_files_even_when_absent(self):
126 for rev in ('main', str(self.base)):
127 for route in ('tree', 'plain'):
128 for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'):
129 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path))
130 self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny'])
131
132 def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self):
133 result = self.access('/the-knife/commit/' + str(self.public))
134 self.assertFalse(result['deny'])
135 self.assertNotIn('SECRET', result['safeDiff'])
136 self.assertIn('PUBLIC', result['safeDiff'])
137 for suffix in ('diff', 'patch'):
138 result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)
139 self.assertFalse(result['deny'])
140 self.assertNotIn('SECRET', result['safeDiff'])
141 self.assertIn('PUBLIC', result['safeDiff'])
142 for suffix in ('stat', 'diffraw'):
143 self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny'])
144
145 def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self):
146 path = '/the-knife/plain/native/knife/private.rs'
147 for name in ('snow', 'guest'):
148 self.assertTrue(self.access(path, name)['allowed'])
149 for name in ('stranger', 'disabled', 'ai', ''):
150 self.assertTrue(self.access(path, name)['deny'])
151 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
152 db.execute("DELETE FROM memberships WHERE user_id='snow'")
153 self.assertTrue(self.access(path, 'snow')['deny'])
154
155 def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self):
156 path = '/the-knife/info/refs?service=git-receive-pack'
157 self.assertTrue(self.access(path, 'snow')['deny'])
158 self.assertTrue(self.access(path)['authenticate'])
159 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
160 self.assertTrue(self.access(path, authorization=token)['allowed'])
161 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
162 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
163 db.execute('DELETE FROM personal_tokens')
164 self.assertTrue(self.access(path, authorization=token)['deny'])
165
166 def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self):
167 for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'):
168 self.assertTrue(self.access(path)['clone'], path)
169 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
170 self.assertTrue(self.access(path, authorization=token)['allowed'], path)
171 for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'):
172 self.assertNotIn('clone', self.access(path), path)
173
174 def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self):
175 with self.assertRaises(shale.Error):
176 self.clone('/the-knife.git/info/refs?service=git-upload-pack')
177 self.repo.lookup_reference('refs/heads/main').set_target(self.base)
178 self.repo.create_reference('refs/heads/merge', self.public)
179 server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args))
180 self.addCleanup(server.server_close)
181 threading.Thread(target=server.serve_forever, daemon=True).start()
182 self.addCleanup(server.shutdown)
183 url = f'http://127.0.0.1:{server.server_port}/the-knife'
184 environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'}
185 for version in ('0', '1', '2'):
186 checkout = self.root / ('clone-v' + version)
187 subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment)
188 refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout
189 self.assertEqual(refs, f'refs/heads/main {self.base}\n')
190 for revision in (str(self.native), str(self.public)):
191 fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment)
192 self.assertNotEqual(fetched.returncode, 0, (version, revision))
193 self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0)
194 pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment)
195 self.assertNotEqual(pushed.returncode, 0)
196
197 def clone(self, uri, method='GET', content_type='', content_encoding=''):
198 return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding})
199
200 def test_native_push_during_classification_cannot_change_the_checked_revision(self):
201 newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native])
202 repo = self.repo
203 class ConcurrentPush:
204 def __getattr__(self, name): return getattr(repo, name)
205 def __getitem__(self, key): return repo[key]
206 def walk(self, *args):
207 repo.lookup_reference('refs/heads/native').set_target(newer)
208 repo.create_reference('refs/heads/deadbeef', newer, force=True)
209 return repo.walk(*args)
210 for revision in ('native', 'deadbeef'):
211 repo.lookup_reference('refs/heads/native').set_target(self.native)
212 if 'refs/heads/deadbeef' in repo.references:
213 repo.references.delete('refs/heads/deadbeef')
214 with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()):
215 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])
216
217 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):
218 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
219 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")
220 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])
221 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
222 self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny'])
223
224 def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self):
225 for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'):
226 self.assertTrue(self.access(path)['deny'], path)
227
228
229if __name__ == '__main__':
230 unittest.main()
tools/shale-access.py deleted-250
...@@ -1,250 +0,0 @@
1import base64
2from contextlib import closing
3import importlib
4import json
5import os
6from pathlib import Path
7import posixpath
8import re
9import sqlite3
10import subprocess
11import sys
12import tempfile
13from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
14
15import pygit2
16from argon2 import PasswordHasher
17from argon2.exceptions import VerificationError, InvalidHashError
18
19Error = importlib.import_module('dashboard-run').Error
20# The root host helper reads repositories owned by the Shale service UID.
21pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0)
22
23
24def restricted_path(path):
25 parts = path.split('/')
26 return parts[0] == 'knife' or 'Cargo.lock' in parts
27
28
29def allowed_account(db, user_id):
30 row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake "
31 "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m "
32 "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')",
33 (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone()
34 if not row:
35 return False
36 profile = json.loads(row[0])
37 return (profile.get('enabled') is True and not profile.get('requiredActions')
38 and profile.get('kind') != 'ai')
39
40
41def instance(service):
42 if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service):
43 raise Error(400, 'Choose a Shale instance.')
44 if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'):
45 return Path(fixture_root)
46 site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5))
47 return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service)
48
49
50def target(request_uri):
51 uri = urlsplit(request_uri)
52 path = uri.path
53 for _ in range(4):
54 decoded = unquote(path, errors='strict')
55 if decoded == path:
56 break
57 path = decoded
58 path = posixpath.normpath('/' + path.lstrip('/'))
59 return uri, path, path.strip('/').split('/')
60
61
62def upload_pack(uri, path, parts):
63 """Returns the CGI path of an anonymous fetch request, the only Git route served without a token."""
64 if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path:
65 return None
66 if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query:
67 return '/the-knife.git/' + '/'.join(parts[1:])
68 return None
69
70
71def repository(root, db):
72 row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone()
73 return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None
74
75
76def protected_commits(repo, native):
77 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
78 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
79 if native:
80 tips.append(native)
81 return {str(commit.id) for tip in tips for commit in repo.walk(tip)}
82
83
84def access(request, accounts=None):
85 root = instance(request['service'])
86 uri, path, parts = target(request['uri'])
87 if parts[0].lower() not in ('the-knife', 'the-knife.git'):
88 return {'allowed': False, 'deny': True}
89 git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD')
90 if not request['authorization'] and upload_pack(uri, path, parts):
91 return {'allowed': False, 'deny': False, 'clone': True}
92 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
93 accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard'))
94 db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',))
95 authorization = request['authorization']
96 allowed = False
97 if authorization:
98 try:
99 scheme, value = authorization.split(' ', 1)
100 if scheme.lower() == 'basic':
101 name, secret = base64.b64decode(value, validate=True).decode().split(':', 1)
102 users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall()
103 if len(users) == 1 and allowed_account(db, users[0][0]):
104 for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]):
105 try:
106 if PasswordHasher().verify(hashed, secret):
107 allowed = True
108 break
109 except (VerificationError, InvalidHashError):
110 pass
111 except (ValueError, UnicodeError):
112 pass
113 elif request['account'] and not git:
114 users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall()
115 allowed = len(users) == 1 and allowed_account(db, users[0][0])
116 if allowed:
117 return {'allowed': True, 'deny': False}
118 location = repository(root, db)
119 if not location:
120 return {'allowed': False, 'deny': True}
121 repo = pygit2.Repository(str(location))
122 try:
123 head = repo.lookup_reference('HEAD').target
124 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
125 protected = protected_commits(repo, references.get('refs/heads/native'))
126 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
127 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
128 return result
129 route = parts[1] if len(parts) > 1 else ''
130 query = dict(parse_qsl(uri.query, keep_blank_values=True))
131 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
132 return result
133 if route in ('', 'issues'):
134 if route == '' and query:
135 return result
136 result['htmlOnly'] = route != 'issues'
137 result['deny'] = False
138 return result
139 if route in ('heads', 'tags') and len(parts) == 2 and not query:
140 result['deny'] = False
141 return result
142 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
143 return result
144 if route not in ('log', 'tree', 'plain', 'commit'):
145 return result
146 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
147 suffix = ''
148 if route == 'commit':
149 revision, dot, suffix = revision.partition('.')
150 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
151 return result
152 if not revision or revision == '-':
153 revision = 'HEAD'
154 # Only named refs and object IDs have the same meaning in Shale and libgit2.
155 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
156 return result
157 try:
158 if revision == 'HEAD':
159 oid = references.get(head) if isinstance(head, str) else head
160 else:
161 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
162 if len(candidates) > 1:
163 return result
164 if candidates:
165 oid = candidates.pop()
166 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):
167 oid = repo[revision].id
168 else:
169 return result
170 if oid is None:
171 return result
172 commit = repo[oid].peel(pygit2.Commit)
173 except (KeyError, ValueError, pygit2.GitError):
174 return result
175 if str(commit.id) in protected:
176 return result
177 if route in ('tree', 'plain'):
178 result['htmlOnly'] = route != 'plain'
179 file = '/'.join(parts[3:])
180 if restricted_path(file):
181 return result
182 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):
183 return result
184 if route == 'commit':
185 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]
186 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
187 if sensitive:
188 if suffix in ('stat', 'diffraw'):
189 return result
190 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"
191 if suffix:
192 result['rawDiff'] = True
193 result['deny'] = False
194 return result
195 if route == 'log':
196 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})
197 else:
198 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]
199 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')
200 if query:
201 result['uri'] += '?' + urlencode(query)
202 result['deny'] = False
203 return result
204 finally:
205 repo.free()
206
207
208def clone(request):
209 root = instance(request['service'])
210 uri, path, parts = target(request['uri'])
211 location = upload_pack(uri, path, parts)
212 if not location:
213 raise Error(404, 'Clone The Knife over HTTPS.')
214 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
215 knife = repository(root, db)
216 if not knife:
217 raise Error(404, 'The Knife is unavailable.')
218 repo = pygit2.Repository(str(knife))
219 try:
220 main = repo.lookup_reference('refs/heads/main').resolve().target
221 native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None
222 protected = protected_commits(repo, native)
223 if any(str(commit.id) in protected for commit in repo.walk(main)):
224 raise Error(403, "The Knife's main branch includes native history.")
225 finally:
226 repo.free()
227 return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': {
228 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query,
229 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'],
230 }}
231
232
233def serve(objects, main):
234 """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects."""
235 repository = Path(tempfile.mkdtemp()) / 'the-knife.git'
236 (repository / 'objects/info').mkdir(parents=True)
237 (repository / 'refs/heads').mkdir(parents=True)
238 (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n')
239 (repository / 'HEAD').write_text('ref: refs/heads/main\n')
240 (repository / 'refs/heads/main').write_text(main + '\n')
241 (repository / 'objects/info/alternates').write_text(objects + '\n')
242 # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main.
243 environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)}
244 os.execvpe('git', ['git', 'http-backend'], environment | {
245 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null',
246 })
247
248
249if __name__ == '__main__':
250 serve(*sys.argv[1:])
tools/shale-webhook-test.py created+124
...@@ -0,0 +1,124 @@
1#!/usr/bin/env python3
2import http.server
3import json
4import os
5from pathlib import Path
6import sqlite3
7import subprocess
8import tempfile
9import threading
10import unittest
11
12REPO = Path(__file__).resolve().parent.parent
13HOOK = REPO / "service/shale/webhook.sh"
14PREPARE = REPO / "service/shale/prepare.py"
15
16
17class Receiver(http.server.BaseHTTPRequestHandler):
18 events = []
19
20 def do_POST(self):
21 body = self.rfile.read(int(self.headers["Content-Length"]))
22 self.events.append((self.headers["X-GitHub-Event"], self.path, json.loads(body)))
23 self.send_response(204)
24 self.end_headers()
25
26 def log_message(self, *_):
27 pass
28
29
30class ShaleWebhookTests(unittest.TestCase):
31 def setUp(self):
32 self.temp = tempfile.TemporaryDirectory()
33 self.root = Path(self.temp.name)
34 self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Receiver)
35 Receiver.events = []
36 self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
37 self.thread.start()
38
39 def tearDown(self):
40 self.server.shutdown()
41 self.thread.join()
42 self.server.server_close()
43 self.temp.cleanup()
44
45 def git(self, *args, cwd=None, env=None):
46 return subprocess.run(
47 ["git", *args], cwd=cwd, env=env, check=True, text=True,
48 capture_output=True,
49 ).stdout.strip()
50
51 def test_push_and_delete_match_snowcord_github_contract(self):
52 bare = self.root / "origin.git"
53 work = self.root / "work"
54 self.git("init", "--bare", "--initial-branch=main", bare)
55 self.git("config", "--file", bare / "config", "shale.snowglobe.repository", "snowcord")
56 (bare / "hooks/post-receive").symlink_to(HOOK)
57 self.git("init", "--initial-branch=main", work)
58 self.git("config", "user.name", "Clover", cwd=work)
59 self.git("config", "user.email", "clover@example.test", cwd=work)
60 (work / "file.txt").write_text("hello\n")
61 self.git("add", "file.txt", cwd=work)
62 self.git("commit", "-m", 'first "line"\n\nsecond \\ line', cwd=work)
63 self.git("remote", "add", "origin", str(bare), cwd=work)
64 env = {
65 **os.environ,
66 "DOMAIN": "shale.test",
67 "SHALE_WEBHOOK_ENABLED": "true",
68 "SHALE_WEBHOOK_URL": f"http://127.0.0.1:{self.server.server_port}/hook",
69 }
70 self.git("push", "origin", "main", cwd=work, env=env)
71
72 event, path, payload = Receiver.events.pop(0)
73 self.assertEqual(event, "push")
74 self.assertEqual(path, "/hook/github/")
75 self.assertEqual(payload["ref"], "refs/heads/main")
76 self.assertEqual(payload["repository"]["name"], "snowcord")
77 self.assertEqual(payload["sender"]["login"], "Clover")
78 self.assertEqual(payload["commits"][0]["message"], 'first "line"\n\nsecond \\ line')
79 self.assertEqual(payload["head_commit"]["id"], payload["after"])
80
81 self.git("push", "origin", "main:delete-me", cwd=work, env=env)
82 Receiver.events.pop(0)
83 self.git("push", "origin", ":delete-me", cwd=work, env=env)
84 event, _, payload = Receiver.events.pop(0)
85 self.assertEqual((event, payload["ref_type"], payload["ref"]), ("delete", "branch", "delete-me"))
86
87 def test_prepare_installs_only_named_repository_hooks(self):
88 owned = self.root / "repositories_owned"
89 database = self.root / "data/astheno.shale.db"
90 database.parent.mkdir()
91 with sqlite3.connect(database) as db:
92 db.executescript("""
93 CREATE TABLE users (provider TEXT, snowflake TEXT);
94 CREATE TABLE repositories (uuid TEXT, name TEXT);
95 INSERT INTO repositories VALUES ('snowcord-id', 'snowcord');
96 INSERT INTO repositories VALUES ('other-id', 'other');
97 """)
98 for name in ("snowcord-id", "other-id"):
99 self.git("init", "--bare", owned / name)
100 env = {**os.environ, "SNOWGLOBE_DOMAIN": "test"}
101 subprocess.run(
102 ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}),
103 env=env, check=True, text=True, capture_output=True,
104 )
105 hook = owned / "snowcord-id/hooks/post-receive"
106 self.assertEqual(os.readlink(hook), "/data/shale-webhook")
107 self.assertEqual((self.root / "data/shale-webhook").stat().st_mode & 0o777, 0o750)
108 self.assertEqual(
109 self.git("config", "--file", owned / "snowcord-id/config", "--get", "shale.snowglobe.repository"),
110 "snowcord",
111 )
112 self.assertFalse((owned / "other-id/hooks/post-receive").exists())
113
114 with sqlite3.connect(database) as db:
115 db.execute("UPDATE repositories SET name='renamed' WHERE uuid='snowcord-id'")
116 subprocess.run(
117 ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}),
118 env=env, check=True, text=True, capture_output=True,
119 )
120 self.assertFalse(os.path.lexists(hook))
121
122
123if __name__ == "__main__":
124 unittest.main()