authoravatarclover caruso<git@paperclover.net>2026-11-19 13:21:20-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:03-08:00
log9acd963a146bafc2595b79916b2f1df24cc4eefe
tree261a78d8790d924564c2ad56abfb918d468c5956
parent17dc455c3819b2d2563acaf514669bb33a593f0f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Remove The Knife's Shale access gate and the knife-native group

The Knife is private in Shale now, so the dashboard no longer gates /the-knife: shale-access.py, anonymous main-branch clones, native-history redaction, the restricted page, and nativeRestriction are gone, along with host-python.nix, which existed only for them. The knife-native group is deleted from existing databases; guests join no groups again, and adding a guest only creates the account. Assisted-by: claude-opus-5.5

28 files changed, 456 insertions(+), 890 deletions(-)

config/mac-studio-sandbox.json+2-1
......@@ -1,5 +1,6 @@
11{
22 "dataset": "globe/clover/mac-studio-sandbox",
33 "mountPoint": "/srv/mac-studio-vm/sandbox",
4 "clientAddress": "10.0.0.222"
4 "hostAddress": "10.0.0.189",
5 "guestAddress": "10.0.0.124"
56}
config/shale.json+1-1
......@@ -1 +1 @@
1{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}}
1{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}}
dashboard/src/auth.rs+4-5
......@@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session";
1212const FLOW_COOKIE: &str = "__Host-snow-flow";
1313const CSRF_COOKIE: &str = "__Host-snow-csrf";
1414const SESSION_TTL: i64 = 30 * 86400;
15const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"];
15const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"];
1616
1717pub struct Store {
1818 pub db: Mutex<Connection>,
......@@ -201,14 +201,13 @@ impl Store {
201201 oidc::initialise(&db)?;
202202 guest::initialise(&db)?;
203203 for name in GROUPS {
204 let added = db.execute(
204 db.execute(
205205 "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)",
206206 sql![uuid::Uuid::new_v4().to_string(), name],
207207 )?;
208 if *name == "knife-native" && added != 0 {
209 db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?;
210 }
211208 }
209 db.execute_batch("DELETE FROM memberships WHERE role_id IN (SELECT id FROM roles WHERE name='knife-native');
210 DELETE FROM roles WHERE name='knife-native';")?;
212211 Ok(Self {
213212 db: Mutex::new(db),
214213 origin,
dashboard/src/guest.rs+4-21
......@@ -23,8 +23,8 @@ pub fn initialise(db: &Connection) -> Result<()> {
2323 db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL);
2424 CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id));
2525 DROP TRIGGER IF EXISTS guest_no_groups;
26 CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END;
27 CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END;
26 CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END;
27 DROP TRIGGER IF EXISTS guest_no_group_updates;
2828 CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END;
2929 CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?;
3030 Ok(())
......@@ -650,7 +650,7 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)
650650 if known.len() > 1 {
651651 return Err(Error::new(
652652 409,
653 "Several accounts have signed in with that name. Open the right one under guests and join it to knife-native.",
653 "Several accounts have signed in with that name. Find them under guests.",
654654 ));
655655 }
656656 let username = match known.first() {
......@@ -673,10 +673,6 @@ fn grant(auth: &auth::Store, provider: &str, subject: Option<&str>, name: &str)
673673 id
674674 }
675675 };
676 tx.execute(
677 "INSERT OR IGNORE INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'",
678 [&id],
679 )?;
680676 tx.commit()?;
681677 Ok(json!({"id":id,"created":existing.is_none()}))
682678}
......@@ -1185,7 +1181,7 @@ mod tests {
11851181 }
11861182
11871183 #[test]
1188 fn identities_stay_separate_and_guests_can_only_join_native_access() {
1184 fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() {
11891185 let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4()));
11901186 let auth = auth::Store::new(
11911187 &path,
......@@ -1234,9 +1230,6 @@ mod tests {
12341230 db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first])
12351231 .is_err()
12361232 );
1237 db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap();
1238 assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native");
1239 assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err());
12401233 assert!(auth::set_password(&db, &first, "a-password-hash").is_err());
12411234 assert!(
12421235 db.execute(
......@@ -1261,10 +1254,6 @@ mod tests {
12611254 std::fs::remove_dir_all(path).unwrap();
12621255 }
12631256
1264 fn members(auth: &auth::Store, id: &str) -> Value {
1265 auth::user(&auth.db.lock().unwrap(), id).unwrap()["groups"].clone()
1266 }
1267
12681257 #[test]
12691258 fn added_guests_are_the_accounts_their_first_sign_in_finds() {
12701259 let path = std::env::temp_dir().join(format!("guest-add-test-{}", uuid::Uuid::new_v4()));
......@@ -1279,10 +1268,8 @@ mod tests {
12791268 let added = grant(&auth, provider, Some(subject), "octo").unwrap();
12801269 let id = string(&added["id"]);
12811270 assert_eq!(added["created"], true);
1282 assert_eq!(members(&auth, id)[0]["name"], "knife-native");
12831271 assert_eq!(account(&auth, provider, subject, "renamed", None).unwrap(), id);
12841272 assert_eq!(grant(&auth, provider, Some(subject), "octo").unwrap()["created"], false);
1285 assert_eq!(members(&auth, id).as_array().unwrap().len(), 1);
12861273 }
12871274
12881275 let reserved = string(&grant(&auth, "astheno", None, "alice").unwrap()["id"]).to_owned();
......@@ -1294,19 +1281,15 @@ mod tests {
12941281 assert_eq!(identities(&reserved), 0);
12951282 let stranger = account(&auth, "astheno", "sub-2", "mallory", None).unwrap();
12961283 assert_ne!(stranger, reserved);
1297 assert_eq!(members(&auth, &stranger), json!([]));
12981284 claim(&auth, "sub-1", "alice").unwrap();
12991285 assert_eq!(account(&auth, "astheno", "sub-1", "alice", None).unwrap(), reserved);
1300 assert_eq!(members(&auth, &reserved)[0]["name"], "knife-native");
13011286 claim(&auth, "sub-3", "alice").unwrap();
13021287 let impostor = account(&auth, "astheno", "sub-3", "alice", None).unwrap();
13031288 assert_ne!(impostor, reserved);
1304 assert_eq!(members(&auth, &impostor), json!([]));
13051289
13061290 let earlier = account(&auth, "astheno", "sub-9", "carol", None).unwrap();
13071291 let added = grant(&auth, "astheno", None, "carol").unwrap();
13081292 assert_eq!((string(&added["id"]), &added["created"]), (earlier.as_str(), &json!(false)));
1309 assert_eq!(members(&auth, &earlier)[0]["name"], "knife-native");
13101293 account(&auth, "astheno", "sub-5", "dave", None).unwrap();
13111294 account(&auth, "astheno", "sub-6", "dave", None).unwrap();
13121295 assert_eq!(grant(&auth, "astheno", None, "dave").unwrap_err().status, 409);
dashboard/src/knife-restricted.html deleted-28
......@@ -1,28 +0,0 @@
1<!doctype html>
2<html lang="en">
3<meta charset="utf-8">
4<meta name="viewport" content="width=device-width, initial-scale=1">
5<title>The Knife's Native UI is restricted</title>
6<style>
7:root { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; background: #faf8fc; color: #29232e; }
8body { margin: 0; min-height: 100svh; display: grid; place-items: center; }
9main { box-sizing: border-box; width: min(100%, 42rem); padding: 2rem; text-align: center; }
10h1 { margin: 0 0 1rem; font-size: clamp(1.5rem, 4vw, 2rem); line-height: 1.25; }
11p { margin: 0 auto 1.75rem; max-width: 34rem; color: #62596b; }
12a { display: inline-block; padding: .625rem 1.375rem; border: 1px solid transparent; border-radius: .375rem; background: #6b3e9e; color: white; text-decoration: none; font-weight: 600; }
13a:hover { background: #552a84; }
14a:focus-visible { outline: 3px solid #8857be; outline-offset: 4px; }
15@media (prefers-color-scheme: dark) {
16:root { background: #1c1821; color: #f4eff8; }
17p { color: #c2b6cd; }
18a { background: #b495db; color: #21162e; }
19a:hover { background: #c8ade9; }
20a:focus-visible { outline-color: #d7b8ff; }
21}
22</style>
23<main>
24<h1>The Knife's Native UI is restricted</h1>
25<p>This part of the code is sensitive. Please contact clover to request private access.</p>
26<a href="/-/login">Login</a>
27</main>
28</html>
dashboard/src/shale_page.rs+4-193
......@@ -347,111 +347,6 @@ fn strip_hop_headers(headers: &mut HeaderMap) {
347347 }
348348}
349349
350fn knife_uri(path: &str) -> bool {
351 let mut decoded = path.to_owned();
352 for _ in 0..5 {
353 if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) {
354 return true;
355 }
356 let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned();
357 if next == decoded { break; }
358 decoded = next;
359 }
360 false
361}
362
363fn session_account(html: &str) -> Option<String> {
364 let document = scraper::Html::parse_document(html);
365 let selector = scraper::Selector::parse("body > header.usa-header--basic nav a.usa-nav-link[href^='/~']").unwrap();
366 let mut accounts = document.select(&selector);
367 let name = accounts.next()?.attr("href")?.strip_prefix("/~")?;
368 let name = percent_encoding::percent_decode_str(name).decode_utf8().ok()?;
369 (accounts.next().is_none() && !name.is_empty() && !name.contains(['/', '\\', '?', '#']))
370 .then(|| name.into_owned())
371}
372
373fn restricted(authenticate: bool) -> Response {
374 (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [
375 ("content-type", "text/html; charset=utf-8"),
376 ("cache-control", "private, no-store"),
377 ("vary", "Cookie, Authorization"),
378 ("referrer-policy", "same-origin"),
379 ("x-content-type-options", "nosniff"),
380 ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"),
381 ("www-authenticate", "Basic realm=\"The Knife\""),
382 ], include_str!("knife-restricted.html")).into_response()
383}
384
385fn redact_knife(html: &str, policy: &Value) -> Result<String> {
386 let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect();
387 let document = scraper::Html::parse_document(html);
388 let diff = policy["safeDiff"].as_str();
389 if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 {
390 return Err(Error::new(403, "The Knife's Native UI is restricted"));
391 }
392 let links = scraper::Selector::parse("a[href]").unwrap();
393 let rows: Vec<bool> = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| {
394 row.select(&links).any(|link| link.attr("href").is_some_and(|href| {
395 href.split(['/', '?', '#', '.']).any(|part| commits.contains(part))
396 }))
397 }).collect();
398 let mut row_index = 0;
399 Ok(lol_html::rewrite_str(html, RewriteStrSettings::new()
400 .append_element_content_handler(element!("#m-code", |element| {
401 if let Some(diff) = diff {
402 let diff = diff.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;");
403 element.set_inner_content(&format!("<pre><code>{diff}</code></pre>"), ContentType::Html);
404 }
405 Ok(())
406 }))
407 .append_element_content_handler(element!("tr", |element| {
408 if *rows.get(row_index).unwrap_or(&true) { element.remove(); }
409 row_index += 1;
410 Ok(())
411 }))
412 )?)
413}
414
415/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch.
416async fn clone(request: Value, body: Body) -> Result<Response> {
417 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
418 let (_, stream) = host::open(request).await?;
419 let (reader, mut writer) = stream.into_split();
420 tokio::spawn(async move {
421 let mut body = body.into_data_stream();
422 while let Some(Ok(chunk)) = body.next().await {
423 if writer.write_all(&chunk).await.is_err() { return; }
424 }
425 let _ = writer.shutdown().await;
426 });
427 let mut reader = tokio::io::BufReader::new(reader);
428 let mut response = Response::builder();
429 loop {
430 let mut line = String::new();
431 if reader.read_line(&mut line).await? == 0 {
432 return Err(Error::new(502, "The Knife clone ended early."));
433 }
434 let line = line.trim_end();
435 if line.is_empty() { break; }
436 let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?;
437 let value = value.trim();
438 if name.eq_ignore_ascii_case("status") {
439 response = response.status(value.split_whitespace().next().unwrap_or_default().parse::<u16>()?);
440 } else {
441 response = response.header(name, value);
442 }
443 }
444 let stream = futures::stream::unfold(reader, |mut reader| async move {
445 let mut buffer = vec![0; 64 * 1024];
446 match reader.read(&mut buffer).await {
447 Ok(0) => None,
448 Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) }
449 Err(error) => Some((Err(error), reader)),
450 }
451 });
452 Ok(response.body(Body::from_stream(stream))?)
453}
454
455350pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
456351 let (mut parts, body) = request.into_parts();
457352 let host = parts
......@@ -484,52 +379,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
484379 return Err(Error::new(403, "Open Shale to continue."));
485380 }
486381 let origin = url::Url::parse(&format!("https://{host}{uri}"))?;
487 let knife = knife_uri(uri.path());
488 let mut target = match &app.internal {
489 Some((base, _)) => format!("{base}services/{service}{uri}"),
490 None => format!("http://{upstream}{uri}"),
491 };
492 let mut knife_policy = Value::Null;
493 if knife {
494 let mut account = String::new();
495 if let Some(cookie) = parts.headers.get("cookie") {
496 // Opening settings rotates the session's CSRF token, invalidating pending forms.
497 let index = match &app.internal {
498 Some((base, _)) => format!("{base}services/{service}/"),
499 None => format!("http://{upstream}/"),
500 };
501 let mut identity = app.shale_http.get(index).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3));
502 if app.internal.is_none() { identity = identity.header("host", host); }
503 for name in ["x-forwarded-for", "x-real-ip", "user-agent"] {
504 if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); }
505 }
506 if let Ok(response) = identity.send().await
507 && response.status() == StatusCode::OK
508 && let Ok(body) = response.text().await
509 {
510 account = session_account(&body).unwrap_or_default();
511 }
512 }
513 knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await {
514 Ok(policy) => policy,
515 Err(_) => return Ok(restricted(false)),
516 };
517 if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); }
518 if knife_policy["clone"] == true {
519 let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default();
520 let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")});
521 return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true)));
522 }
523 if let Some(uri) = knife_policy["uri"].as_str() {
524 target = match &app.internal {
525 Some((base, _)) => format!("{base}services/{service}{uri}"),
526 None => format!("http://{upstream}{uri}"),
527 };
528 }
529 if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() {
530 return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response());
531 }
532 }
533382 let profiles = profiles(&app.auth)?;
534383 if matches!(parts.method, Method::GET | Method::HEAD)
535384 && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref())
......@@ -544,6 +393,10 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
544393 )
545394 .into_response());
546395 }
396 let target = match &app.internal {
397 Some((base, _)) => format!("{base}services/{service}{uri}"),
398 None => format!("http://{upstream}{uri}"),
399 };
547400 if app.internal.is_some() {
548401 parts.headers.remove("host");
549402 }
......@@ -578,10 +431,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
578431 let mut headers = upstream_response.headers().clone();
579432 strip_hop_headers(&mut headers);
580433 headers.remove("status");
581 if knife {
582 headers.insert("cache-control", "private, no-store".parse().unwrap());
583 headers.insert("vary", "Cookie, Authorization".parse().unwrap());
584 }
585434 let html = parts.method != Method::HEAD
586435 && headers
587436 .get("content-type")
......@@ -637,14 +486,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
637486 profiles.insert(name.clone(), profile);
638487 }
639488 }
640 let redacted;
641 let text = if knife && knife_policy["allowed"] != true {
642 redacted = match redact_knife(text, &knife_policy) {
643 Ok(html) => html,
644 Err(_) => return Ok(restricted(false)),
645 };
646 redacted.as_str()
647 } else { text };
648489 let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?;
649490 for name in [
650491 "content-length",
......@@ -661,9 +502,6 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
661502 return Ok((status, headers, rewritten).into_response());
662503 }
663504 }
664 if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) {
665 return Ok(restricted(false));
666 }
667505 let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) })
668506 .chain(upstream_response.bytes_stream());
669507 Ok((status, headers, Body::from_stream(stream)).into_response())
......@@ -675,33 +513,6 @@ mod tests {
675513 use rusqlite::params as sql;
676514 use scraper::{Html, Selector};
677515
678 #[test]
679 fn session_account_uses_only_unambiguous_site_navigation() {
680 let header = |name: &str| format!("<header class='usa-header usa-header--basic'><nav><a class='usa-nav-link' href='/~{name}'>account</a></nav></header>");
681 assert_eq!(session_account(&header("clover")), Some("clover".into()));
682 assert_eq!(session_account(&header("guest%2Dname")), Some("guest-name".into()));
683 assert_eq!(session_account(&format!("{}<main><header class='usa-header--basic'><nav><a class='usa-nav-link' href='/~clover'>fake</a></nav></header></main>", header("guest"))), Some("guest".into()));
684 assert_eq!(session_account(&format!("{}{}", header("guest"), header("clover"))), None);
685 for name in ["", "%FF", "clover%2Frepo", "clover%5Crepo", "clover?fake", "clover#fake"] {
686 assert_eq!(session_account(&header(name)), None, "{name}");
687 }
688 assert_eq!(session_account("<a class='usa-nav-link' href='/~clover'>fake</a>"), None);
689 }
690
691 #[test]
692 fn native_content_is_removed_from_both_diff_styles_and_history_rows() {
693 let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"});
694 let html = "<html><body id=page-commit><table><tr><td><a href='/the-knife/commit/deadbeef'>PRIVATE MESSAGE</a></td></tr><tr><td>PUBLIC ROW</td></tr></table><div id=m-code><div id=m-changedfiles>PRIVATE PATHS</div><details class=m-usa-accordion><summary><code>knife/private.rs</code></summary><table><tr><td>PRIVATE SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>src/public.rs</code></summary><table><tr><td>PUBLIC SOURCE</td></tr></table></details><details class=m-usa-accordion><summary><code>nested/Cargo.lock</code></summary><table><tr><td>PRIVATE LOCK</td></tr></table></details></div></body></html>";
695 let output = redact_knife(html, &policy).unwrap();
696 assert!(!output.contains("PRIVATE"), "{output}");
697 assert!(output.contains("PUBLIC ROW"), "{output}");
698 assert!(output.contains("PUBLIC SOURCE"), "{output}");
699 assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1);
700 assert!(redact_knife("<div>unknown diff markup</div>", &policy).is_err());
701 for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); }
702 assert!(!knife_uri("/snowbound/commit/native"));
703 }
704
705516 #[test]
706517 fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() {
707518 let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4()));
dashboard/src/users.rs+3-9
......@@ -158,13 +158,10 @@ pub async fn route(
158158 for id in &users {
159159 uuid(id)?;
160160 let user = auth::user(&transaction, id)?;
161 let guest_groups = if guest::is_guest(&user) {
162 add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true))
163 } else { false };
164 if guest_groups || user["kind"] == "ai" {
161 if guest::is_guest(&user) || user["kind"] == "ai" {
165162 return Err(Error::new(
166163 400,
167 "Guests can only join knife-native.",
164 "Guests can use Shale only. Select regular accounts to edit groups.",
168165 ));
169166 }
170167 for group in &add {
......@@ -304,7 +301,7 @@ pub async fn route(
304301 let transaction = db.transaction()?;
305302 let mut user = auth::user(&transaction, id)?;
306303 if guest::is_guest(&user)
307 && matches!(parts, [_, "password"])
304 && matches!(parts, [_, "password"] | [_, "groups", _])
308305 && method != Method::GET
309306 {
310307 return Err(Error::new(
......@@ -354,9 +351,6 @@ pub async fn route(
354351 .optional()?;
355352 let name = name
356353 .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?;
357 if guest::is_guest(&user) && name != "knife-native" {
358 return Err(Error::new(400, "Guests can only join knife-native."));
359 }
360354 if own && method == Method::DELETE && name == "infra-admin" {
361355 return Err(Error::new(
362356 400,
dashboard/web/pages/Users.tsx+2-3
......@@ -77,7 +77,6 @@ function adds(group: string, services: ServiceSummary[]) {
7777
7878/** "30 members · opens Jellyfin, Navidrome" */
7979function groupTip(group: string, d: Data) {
80 if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`;
8180 const more = adds(group, d.services);
8281 const members = d.users.filter((user) => names(user).includes(group)).length;
8382 const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : [];
......@@ -230,7 +229,7 @@ function List(props: { data: Resource<Data>; ready: Accessor<Data | undefined>;
230229 guestMenu.hidePopover();
231230 showTextDialog({
232231 title: `Add ${guest.name} guest`,
233 description: "They can sign in to Shale and open The Knife’s Native UI.",
232 description: "They can sign in to Shale.",
234233 label: guest.field,
235234 placeholder: guest.placeholder,
236235 body: provider === "astheno" ? () => <div class="field"><span class="hint">Astheno can’t look names up. The first person to sign in with this username gets the access.</span></div> : undefined,
......@@ -638,7 +637,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {
638637 <div class="toggles" role="group" aria-label="Groups">
639638 <For each={props.data.groups}>
640639 {(group) => (
641 <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" && group.name !== "knife-native" || busy() === group.id}
640 <button class="chip toggle" aria-pressed={member(group)} disabled={props.user.kind === "guest" || busy() === group.id}
642641 data-tip={groupTip(group.name, props.data)} onClick={() => toggleGroup(group)}>
643642 {group.name}
644643 </button>
flake.nix-1
......@@ -5,7 +5,6 @@
55
66 outputs = { self, nixpkgs, ... }: {
77 packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { };
8 packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { };
98 packages.x86_64-linux.vm-screen = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/vm-screen.nix { };
109 packages.x86_64-linux.computer-ahk = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/computer-ahk.nix { };
1110 packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image;
nixos/configuration.nix+2-3
......@@ -6,12 +6,11 @@ let
66 proxyToken = "/var/lib/snowglobe/dashboard-proxy.token";
77 hostTools = lib.fileset.toSource {
88 root = ../.;
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];
1010 };
1111 computerAhk = pkgs.callPackage ./computer-ahk.nix { };
1212 computerWindows = pkgs.callPackage ./computer-windows.nix { };
1313 nativePkl = pkgs.callPackage ./pkl.nix { };
14 hostPython = pkgs.callPackage ./host-python.nix { };
1514 screen = pkgs.callPackage ./vm-screen.nix { };
1615 secureOvmf = pkgs.OVMF.override {
1716 secureBoot = true;
......@@ -264,7 +263,7 @@ in
264263 SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables;
265264 };
266265 serviceConfig = {
267 ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py";
266 ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py";
268267 Type = "notify";
269268 Group = "snowglobe-dashboard";
270269 StateDirectory = "snowglobe/host";
nixos/host-python.nix deleted-6
......@@ -1,6 +0,0 @@
1{ python3, git }:
2python3.buildEnv.override {
3 extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ];
4 # shale-access serves anonymous Knife clones through git http-backend.
5 makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ];
6}
nixos/mac-studio-sandbox.nix+2-2
......@@ -50,11 +50,11 @@ in
5050 enable = true;
5151 hostName = "10.0.0.1";
5252 exports = ''
53 ${mountPoint} ${workspace.clientAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
53 ${mountPoint} ${workspace.guestAddress}(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
5454 '';
5555 };
5656 environment.etc."exports.d/snowglobe-work.exports".text = ''
57 ${mountPoint}/.work ${workspace.clientAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
57 ${mountPoint}/.work ${workspace.guestAddress}(rw,sync,fsid=42,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure)
5858 '';
5959 systemd.services.mac-work-share-check = {
6060 requires = [ "zfs-import-work.service" ];
nixos/mac-vm.nix+2-2
......@@ -5,7 +5,7 @@ let
55in {
66 systemd.services.snowglobe-host = {
77 environment.SNOWGLOBE_MAC_VM = builtins.toJSON {
8 address = workspace.clientAddress;
8 address = workspace.hostAddress;
99 port = 22;
1010 user = "clo";
1111 identity = "/var/lib/snowglobe/mac-vm/ssh_key";
......@@ -15,6 +15,6 @@ in {
1515 { name = "m3-phoenix-box"; inherit owner; console = false; suspend = false; }
1616 ];
1717 };
18 serviceConfig.IPAddressAllow = [ "${workspace.clientAddress}/32" ];
18 serviceConfig.IPAddressAllow = [ "${workspace.hostAddress}/32" ];
1919 };
2020}
nixos/snowglobe.nix+1-1
......@@ -29,7 +29,7 @@ in
2929 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
3030 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
3131 '') config.networking.interfaces.enp4s0.ipv4.addresses + ''
32 iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).clientAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept
32 iptables -A nixos-fw -i enp4s0 -s ${(builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json)).guestAddress}/32 -p tcp --dport 2049 -j nixos-fw-accept
3333 ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept
3434 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
3535 '';
service/shale/prepare.py+42-6
......@@ -9,13 +9,8 @@ import sys
99
1010data = json.load(sys.stdin)
1111root = Path(data["hostRoot"])
12for config in (root / "repositories_owned").glob("*/config"):
13 text = config.read_text()
14 if not re.search(r"(?im)^\s*receivepack\s*=", text):
15 # Shale authorizes pushes before CGI; its own repository initializer enables this.
16 with config.open("a") as file:
17 file.write("\n[http]\n\treceivepack = true\n")
1812path = root / "data/astheno.shale.db"
13webhooks = {}
1914if path.exists():
2015 domain = os.environ["SNOWGLOBE_DOMAIN"]
2116 old, new = "auth." + domain + "/realms/master", "snowglobe." + domain
......@@ -25,6 +20,47 @@ if path.exists():
2520 raise ValueError("duplicate Shale identity; resolve it before moving the issuer")
2621 count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount
2722 db.commit()
23 webhooks = dict(db.execute(
24 "SELECT uuid, name FROM repositories WHERE name IN ('snowcord', 'snowcord-server')"
25 ))
2826 assert db.execute("PRAGMA quick_check").fetchone() == ("ok",)
2927 db.close()
3028 print("Moved", count, "existing Shale identity bindings to Snowglobe")
29
30executable = root / "data/shale-webhook"
31source = Path(__file__).with_name("webhook.sh").read_bytes()
32if not executable.exists() or executable.read_bytes() != source:
33 pending = executable.with_suffix(".pending")
34 pending.write_bytes(source)
35 if os.geteuid() == 0:
36 os.chown(pending, data["uid"], data["uid"])
37 pending.replace(executable)
38executable.chmod(0o750)
39if os.geteuid() == 0:
40 os.chown(executable, data["uid"], data["uid"])
41
42target = "/data/shale-webhook"
43for config in (root / "repositories_owned").glob("*/config"):
44 original = text = config.read_text()
45 if not re.search(r"(?im)^\s*receivepack\s*=", text):
46 # Shale authorizes pushes before CGI; its own repository initializer enables this.
47 text += "\n[http]\n\treceivepack = true\n"
48 text = re.sub(r'(?m)\n+\[shale "snowglobe"\]\n(?:[ \t].*\n?)*', "\n", text)
49 name = webhooks.get(config.parent.name)
50 if name:
51 text += f'\n[shale "snowglobe"]\n\trepository = {name}\n'
52 if text != original:
53 config.write_text(text)
54
55 hook = config.parent / "hooks/post-receive"
56 if name:
57 hook.parent.mkdir(exist_ok=True)
58 if os.path.lexists(hook):
59 if not hook.is_symlink() or os.readlink(hook) != target:
60 raise ValueError(f"refusing to replace existing hook: {hook}")
61 if not os.path.lexists(hook):
62 hook.symlink_to(target)
63 if os.geteuid() == 0:
64 os.lchown(hook, data["uid"], data["uid"])
65 elif hook.is_symlink() and os.readlink(hook) == target:
66 hook.unlink()
service/shale/service.pkl+3
......@@ -61,6 +61,8 @@ container {
6161 ["MIRRORS_DISABLE_CLONE"] = "true"
6262 ["MIRRORS_DISABLE_PULL_POLL"] = "true"
6363 }
64 ["SHALE_WEBHOOK_ENABLED"] = if (site.preview) "false" else "true"
65 ["SHALE_WEBHOOK_URL"] = "${secret.own.webhook_url}"
6466 ["SESSION_SECRET"] = "${secret.own.session_secret}"
6567 ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
6668 }
......@@ -69,3 +71,4 @@ container {
6971secrets {
7072 ["session_secret"] {}
7173}
74requiredSecrets { "webhook_url" }
service/shale/webhook.sh created+144
......@@ -0,0 +1,144 @@
1#!/bin/sh
2set -u
3
4[ "${SHALE_WEBHOOK_ENABLED:-false}" = true ] || exit 0
5[ -n "${SHALE_WEBHOOK_URL:-}" ] || exit 0
6case "$SHALE_WEBHOOK_URL" in
7 */github|*/github/) webhook_url=${SHALE_WEBHOOK_URL%/}/ ;;
8 *) webhook_url=${SHALE_WEBHOOK_URL%/}/github/ ;;
9esac
10
11repository=$(git config --get shale.snowglobe.repository) || exit 0
12origin="https://${DOMAIN}/${repository}"
13avatar="https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png"
14
15json_string() {
16 awk 'BEGIN { ORS = "" }
17 {
18 if (NR > 1) printf "\\n"
19 for (i = 1; i <= length($0); i++) {
20 c = substr($0, i, 1)
21 if (c == "\\") printf "\\\\"
22 else if (c == "\"") printf "\\\""
23 else if (c == "\t") printf "\\t"
24 else if (c == "\r") printf "\\r"
25 else if (c == sprintf("%c", 8)) printf "\\b"
26 else if (c == sprintf("%c", 12)) printf "\\f"
27 else printf "%s", c
28 }
29 }'
30}
31
32quote() {
33 printf '"'
34 printf '%s' "$1" | json_string
35 printf '"'
36}
37
38is_zero() {
39 case "$1" in
40 *[!0]*) return 1 ;;
41 *) return 0 ;;
42 esac
43}
44
45write_ref_payload() {
46 {
47 printf '{"ref":'; quote "$short_ref"
48 printf ',"ref_type":'; quote "$ref_type"
49 printf ',"repository":{"name":'; quote "$repository"
50 printf ',"full_name":'; quote "$repository"
51 printf ',"html_url":'; quote "$origin/"
52 printf '},"sender":{"login":'; quote "$actor"
53 printf ',"avatar_url":'; quote "$avatar"
54 printf ',"html_url":'; quote "$origin/"
55 printf '}}\n'
56 } >"$payload"
57}
58
59send() {
60 event=$1
61 payload=$2
62 delivery=$(printf '%s\n' "$repository:$event:$(date +%s):$$:$index" | git hash-object --stdin)
63 curl --fail --silent --show-error --max-time 10 \
64 -H "Content-Type: application/json" \
65 -H "User-Agent: Shale-Hookshot/1.0" \
66 -H "X-GitHub-Event: $event" \
67 -H "X-GitHub-Delivery: $delivery" \
68 --data-binary "@$payload" "$webhook_url" >/dev/null
69}
70
71temporary=$(mktemp -d)
72trap 'rm -rf "$temporary"' EXIT HUP INT TERM
73index=0
74while read -r before after ref; do
75 index=$((index + 1))
76 payload="$temporary/$index.json"
77 short_ref=${ref#refs/heads/}
78 if [ "$short_ref" = "$ref" ]; then
79 short_ref=${ref#refs/tags/}
80 ref_type=tag
81 else
82 ref_type=branch
83 fi
84
85 if is_zero "$after"; then
86 actor=${REMOTE_USER:-$(git show -s --format=%an "$before")}
87 write_ref_payload
88 send delete "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
89 continue
90 fi
91
92 actor=${REMOTE_USER:-$(git show -s --format=%an "$after")}
93 if [ "$ref_type" = tag ]; then
94 if is_zero "$before"; then
95 write_ref_payload
96 send create "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
97 fi
98 continue
99 fi
100 if is_zero "$before"; then
101 created=true
102 forced=false
103 commits=$(git rev-list --reverse "$after")
104 else
105 created=false
106 if git merge-base --is-ancestor "$before" "$after"; then forced=false; else forced=true; fi
107 commits=$(git rev-list --reverse "$before..$after")
108 fi
109 compare="$origin/commit/$after"
110 {
111 printf '{"ref":'; quote "$ref"
112 printf ',"before":'; quote "$before"
113 printf ',"after":'; quote "$after"
114 printf ',"created":%s,"deleted":false,"forced":%s' "$created" "$forced"
115 printf ',"compare":'; quote "$compare"
116 printf ',"repository":{"name":'; quote "$repository"
117 printf ',"full_name":'; quote "$repository"
118 printf ',"html_url":'; quote "$origin/"
119 printf '},"sender":{"login":'; quote "$actor"
120 printf ',"avatar_url":'; quote "$avatar"
121 printf ',"html_url":'; quote "$origin/"
122 printf '},"commits":['
123 separator=
124 for commit in $commits; do
125 message=$(git show -s --format=%B "$commit")
126 author=$(git show -s --format=%an "$commit")
127 timestamp=$(git show -s --format=%aI "$commit")
128 printf '%s{"id":' "$separator"; quote "$commit"
129 printf ',"message":'; quote "$message"
130 printf ',"timestamp":'; quote "$timestamp"
131 printf ',"url":'; quote "$origin/commit/$commit"
132 printf ',"author":{"name":'; quote "$author"
133 printf ',"username":'; quote "$author"
134 printf '}}'
135 separator=,
136 done
137 printf '],"head_commit":{"id":'; quote "$after"
138 printf ',"message":'; quote "$(git show -s --format=%B "$after")"
139 printf ',"timestamp":'; quote "$(git show -s --format=%aI "$after")"
140 printf ',"url":'; quote "$origin/commit/$after"
141 printf '}}\n'
142 } >"$payload"
143 send push "$payload" || printf 'Shale webhook delivery failed for %s\n' "$repository" >&2
144done
tools/dashboard-host.py-30
......@@ -16,7 +16,6 @@ import signal
1616import socket
1717import struct
1818import subprocess
19import sys
2019import threading
2120import time
2221import vms
......@@ -45,7 +44,6 @@ CONNECTIONS = WORKERS + 4
4544WORK_SLOTS = threading.BoundedSemaphore(WORKERS)
4645STREAM_SLOTS = threading.BoundedSemaphore(4)
4746GUEST_SLOTS = threading.BoundedSemaphore(4)
48CLONE_SLOTS = threading.BoundedSemaphore(2)
4947VM_PREPARATION_SLOT = threading.BoundedSemaphore(1)
5048INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?"
5149
......@@ -97,8 +95,6 @@ class Host:
9795 "index.diff": {"dataset", "from", "to"},
9896 "index.discard": {"dataset", "snapshot"},
9997 "shale.metadata": {"repository", "accounts"},
100 "shale.access": {"service", "uri", "account", "authorization"},
101 "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"},
10298 "shale.write": {"grant", "repository", "action", "arguments"},
10399 "host.sample": set(),
104100 "host.usage": {"refresh"},
......@@ -113,10 +109,6 @@ class Host:
113109 raise Rejected("Use only the fields required by this host operation.")
114110 if operation == "copyparty.manage":
115111 return copyparty.handle(request)
116 if operation == "shale.access":
117 return importlib.import_module("shale-access").access(request)
118 if operation == "shale.clone":
119 return importlib.import_module("shale-access").clone(request)
120112 if operation == "shale.metadata":
121113 return shale_labels.metadata(request)
122114 if operation == "shale.write":
......@@ -473,20 +465,6 @@ def stream_upload(connection, request):
473465 Path(temporary).unlink(missing_ok=True)
474466
475467
476def stream_clone(connection, plan):
477 owner = os.stat(plan["objects"])
478 # Service identities have numeric owners without passwd entries, which systemd's User= rejects.
479 command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec",
480 *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes",
481 "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")),
482 *(f"--setenv={name}={value}" for name, value in plan["environment"].items()),
483 "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--",
484 sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]]
485 send_response(connection, {"value": None})
486 connection.setblocking(True)
487 subprocess.run(command, stdin=connection, stdout=connection)
488
489
490468def serve_connection(connection, host, allowed_uid, allowed_operations=None):
491469 with connection, contextlib.ExitStack() as resources:
492470 connection.settimeout(65)
......@@ -553,14 +531,6 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
553531 return
554532 finally:
555533 streams.release()
556 if operation == "shale.clone":
557 if not CLONE_SLOTS.acquire(blocking=False):
558 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
559 try:
560 stream_clone(connection, host.handle(request))
561 return
562 finally:
563 CLONE_SLOTS.release()
564534 response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)}
565535 except Rejected as error:
566536 response = {"error": str(error), "status": 400}
tools/dashboard-shale-page-test.py+2-41
......@@ -6,9 +6,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
66import json
77import os
88from pathlib import Path
9import re
109import socket
11import socketserver
1210import sqlite3
1311import subprocess
1412import tempfile
......@@ -49,21 +47,12 @@ def main():
4947 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'
5048 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'
5149 blob = bytes(range(256)) * 65536
52 csrf = 'initial'
5350
5451 class Fixture(BaseHTTPRequestHandler):
5552 protocol_version = 'HTTP/1.1'
5653
5754 def do_GET(self):
58 nonlocal csrf
5955 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
60 if self.path == '/':
61 account = b'<a class="usa-nav-link" href="/~fixture-member">~fixture-member</a>' if self.headers.get('Cookie') == 'SessionID=preserved' else b''
62 return self.respond(b'<html><body><header class="usa-header--basic"><nav>' + account + b'</nav></header></body></html>', 'text/html')
63 if self.path == '/-/settings' or self.path.endswith('/issues/new'):
64 csrf += 'x'
65 account = '<body id="page-user-settings"><kbd>fixture-member</kbd>' if self.path == '/-/settings' else '<body>'
66 return self.respond(('<html>' + account + '<form method="post"><input type="hidden" name="csrf_token" value="' + csrf + '"></form></body></html>').encode(), 'text/html')
6756 if self.path == '/binary':
6857 return self.respond(blob, 'application/octet-stream')
6958 if self.path == '/json':
......@@ -86,8 +75,6 @@ def main():
8675 def do_POST(self):
8776 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
8877 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
89 if self.path.endswith('/issues/new') and urllib.parse.parse_qs(body.decode()).get('csrf_token') != [csrf]:
90 return self.respond(b'E: unexpected csrf token', 'text/plain', 403)
9178 self.respond(body, 'application/octet-stream', 201)
9279
9380 def respond(self, body, content_type, status=200, headers=()):
......@@ -121,21 +108,7 @@ def main():
121108 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
122109 data = root / 'data'
123110 data.mkdir()
124 class Host(socketserver.StreamRequestHandler):
125 def handle(self):
126 request = json.loads(self.rfile.readline())
127 value = {}
128 if request['operation'] == 'shale.access':
129 allowed = request['account'] == 'fixture-member'
130 value = {'allowed': allowed, 'deny': not allowed}
131 response = json.dumps({'value': value}).encode()
132 self.wfile.write(len(response).to_bytes(4, 'big') + response)
133
134 host = socketserver.UnixStreamServer(str(root / 'host.sock'), Host)
135 stack.callback(host.server_close)
136 stack.callback(host.shutdown)
137 threading.Thread(target=host.serve_forever, daemon=True).start()
138 environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_HOST_SOCKET': str(root / 'host.sock'), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')}
111 environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')}
139112 for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']:
140113 environment.pop(key, None)
141114 log = stack.enter_context((root / 'dashboard.log').open('w'))
......@@ -253,22 +226,10 @@ def main():
253226 assert len(request('/large')[2]) == 9 * 1024 * 1024
254227 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')
255228 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'
256 for path in ['/the-knife/issues/new', '/another-repo/issues/new']:
257 status, _, form = request(path)
258 assert status == 200
259 csrf_field = re.search(rb'name="csrf_token" value="([^"]+)"', form)[1]
260 assert request('/the-knife/')[0] == 200
261 submitted = b'csrf_token=' + csrf_field + b'&title=preserved'
262 status, _, body = request(path, body=submitted)
263 assert status == 201 and body == submitted, (path, status, body)
264 for invalid in [b'title=missing', b'csrf_token=invalid']:
265 assert request(path, body=invalid)[0] == 403
266 assert not any(item[1] == '/-/settings' for item in observations)
267 assert request('/the-knife/issues/new', headers={'Cookie': ''})[0] == 403
268229 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403
269230 assert request('/auth/shale/page', direct=True)[0] == 403
270231 assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403
271 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'csrf_preservation_and_rejection': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
232 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
272233 if args.browser_ready_file:
273234 args.browser_ready_file.write_text(json.dumps({'origin': origin}))
274235 stop = args.browser_ready_file.with_suffix('.stop')
tools/dashboard-stage.py+2-14
......@@ -21,7 +21,6 @@ import snowglobe
2121
2222
2323OPERATIONS = frozenset({
24 "shale.access", "shale.clone",
2524 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",
2625 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",
2726 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",
......@@ -56,16 +55,6 @@ class PreviewHost(boundary.Host):
5655 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}
5756 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:
5857 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")
59 if operation in {"shale.access", "shale.clone"}:
60 service = request.get("service", "")
61 if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
62 raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.")
63 linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text())
64 if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name:
65 raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.")
66 if operation == "shale.clone":
67 return import_module("shale-access").clone(request)
68 return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data")
6958 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:
7059 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]
7160 if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]:
......@@ -234,14 +223,13 @@ def stage(computer=None):
234223 for entry in mac["vms"]:
235224 entry["owner"] = owner[0]
236225 computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac))
237 host_python = snowglobe.command(*nix, "build", "--out-link", str(root / "host-python"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip()
238226 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
239227 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
240228 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity,
241229 "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32",
242230 "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"],
243231 *computer_environment,
244 host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity)
232 sys.executable, str(Path(__file__).resolve()), "--serve", identity)
245233 for _ in range(100):
246234 if Path(runtime, "host.sock").exists():
247235 break
......@@ -263,7 +251,7 @@ def stage(computer=None):
263251 "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock",
264252 "--setenv=XDG_RUNTIME_DIR=" + screen_runtime,
265253 "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache",
266 host_python + "/bin/python3", "-c", launcher)
254 sys.executable, "-c", launcher)
267255 for _ in range(100):
268256 if Path(screen_runtime, "screen.sock").exists():
269257 break
tools/mac-agent-forward-test.py+1-2
......@@ -35,8 +35,7 @@ sys.stdin.buffer.read()
3535server.shutdown()
3636server.server_close()
3737'''
38 ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip()
39 server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)],
38 server = subprocess.Popen([*vm.ssh_args(), vm.GUEST, "python3 -u -c " + shlex.quote(code)],
4039 stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
4140 tunnel = None
4241 try:
tools/mac-agent-vm.md+32-3
......@@ -27,6 +27,19 @@ python3 tools/mac-agent-vm.py console
2727python3 tools/mac-agent-vm.py stop
2828```
2929
30Everything except `exec` reaches the guest over pinned SSH at its Bonjour name.
31`exec` and the dashboard's computer operations go through Tart's guest agent so
32they run in the logged-in GUI session, and that path can crash the VM runner
33(`/Library/Logs/DiagnosticReports/tart-*.ips`, `-[VZVirtioSocketDevice
34connectToPort:]`). Use `ssh` for anything that does not need the GUI session.
35
36The `net.paperclover.mac-vm-revive` LaunchAgent runs `revive` every 30 seconds
37through `ssh localhost` (launchd jobs cannot read the VM volume). It starts the
38VM when its runner is gone unless `stop` or `suspend` left `private/stopped`;
39`start` clears that marker. Restarts and the newest Tart crash report are logged
40to `~/Library/Logs/mac-vm-revive.log`. A hung guest under a live runner is not
41restarted.
42
3043To reach a server bound to guest loopback from this Mac:
3144
3245```sh
......@@ -50,6 +63,22 @@ snapshot; it writes memory to disk and is not an instant CPU pause. A running
5063VM holds a process-scoped idle-sleep assertion, so locking the host does not
5164cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.
5265
66## Network and iPhones
67
68The guest is bridged onto the LAN through the host's default-route interface
69so iPhones can find it over Bonjour and connect in; Virtualization.framework
70has no USB passthrough for them. Its DHCP address and the host's are reserved
71on the router and recorded in
72[mac-studio-sandbox.json](../config/mac-studio-sandbox.json): Snow Globe
73exports NFS to `guestAddress` and the dashboard reaches the host at
74`hostAddress`. Every `start` loads a pf anchor (`com.apple/net.paperclover`)
75in the guest that refuses new inbound TCP below port 49152 from anyone but
76the host, plus Kerberos and NetBIOS over UDP. A guest that reboots itself
77stays open to the LAN until the next `start`. iPhones on iOS 27 pair in Device
78Hub with **+ → Pair Nearby Device…** and Settings → Privacy & Security →
79Developer Mode on the phone. Older iOS needs a cable, which the guest cannot
80have.
81
5382The guest has 28 vCPUs, 48 GiB RAM, and a 1440×900 display. vCPUs do not reserve
5483physical cores. CPU count and RAM changes require a cold boot. No automatic
5584lock/unlock controller or CPU affinity is installed. The guest's 48 GiB budget
......@@ -180,8 +209,8 @@ retries the mount every 30 seconds and unmounts it again unless the SSD marker
180209is present.
181210
182211The guest's `/etc/hosts` maps `file.paperclover.net` to the NAS LAN address
183`10.0.0.1`. Traffic uses the VM's NAT gateway `192.168.64.1`; the host's hosts
184file is not inherited. The previous guest file is `/etc/hosts.before-code-nas`.
212`10.0.0.1`. The host's hosts file is not inherited. The previous guest file is
213`/etc/hosts.before-code-nas`.
185214
186215SSH uses a new private key in `private/ssh_key`, unique guest host keys pinned
187216in `private/known_hosts`, and disables password authentication. No host SSH
......@@ -222,7 +251,7 @@ The optional local recovery console is `http://127.0.0.1:17891/vnc_lite.html?aut
222251It reaches the guest's built-in Screen Sharing through pinned SSH, forwarding
223252guest `127.0.0.1:5900` into a temporary private Unix socket. The relay handles
224253the guest account authentication from `private/password` server-side. Screen
225Sharing listens on all interfaces inside the private NAT guest; the host relay's
254Sharing listens on all guest interfaces behind the pf anchor; the host relay's
226255TCP listeners default to loopback and check the exact HTTP Host and WebSocket
227256Origin. Tart's experimental host VNC server is disabled. The viewer trusts
228257every client that can reach its listener; Origin checking protects against foreign
tools/mac-agent-vm.py+77-36
......@@ -8,6 +8,7 @@ import logging
88import multiprocessing
99import os
1010from pathlib import Path
11import re
1112import select
1213import secrets
1314import shlex
......@@ -26,6 +27,9 @@ ROOT = Path(os.environ.get("MAC_AGENT_VM_HOME", "/Volumes/Documents/VM/macOS Cod
2627TART = ROOT / "runtime/tart.app/Contents/MacOS/tart"
2728NAME = "gui-agent"
2829PRIVATE = ROOT / "private"
30# Never reach the guest with `tart exec` or `tart ip --resolver=agent` unless it needs the GUI session:
31# their vsock connect intermittently traps inside Virtualization.framework and kills the whole VM.
32GUEST = "admin@AgentVM.local"
2933
3034
3135def tart(*args, **kwargs):
......@@ -35,12 +39,27 @@ def tart(*args, **kwargs):
3539
3640def ssh_args():
3741 return ["ssh", "-F", "/dev/null", "-i", str(PRIVATE / "ssh_key"),
38 "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no",
42 "-o", "AddressFamily=inet", "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ForwardAgent=no",
3943 "-o", "StrictHostKeyChecking=yes", "-o", "HostKeyAlias=" + NAME,
4044 "-o", f'UserKnownHostsFile="{PRIVATE / "known_hosts"}"',
4145 "-o", "GlobalKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10"]
4246
4347
48def guest(*command, **kwargs):
49 kwargs.setdefault("check", True)
50 return subprocess.run([*ssh_args(), GUEST, shlex.join(command)], **kwargs)
51
52
53def guest_ready(**kwargs):
54 guest("/bin/sh", "-c", '[ "$(/usr/bin/stat -f %Su /dev/console)" = admin ]', **kwargs)
55
56
57def shut_down():
58 # ssh can drop as the guest powers off, so the stopped state is the proof.
59 guest("sudo", "-n", "shutdown", "-h", "now", check=False, timeout=15)
60 wait_for_state("stopped")
61
62
4463def exact(stream, size):
4564 data = bytearray()
4665 while len(data) < size:
......@@ -51,10 +70,14 @@ def exact(stream, size):
5170 return bytes(data)
5271
5372
73def vm():
74 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)
75 return next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")
76
77
5478def wait_for_state(state):
5579 for _ in range(90):
56 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)
57 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["State"] == state:
80 if vm()["State"] == state:
5881 return
5982 time.sleep(1)
6083 raise ValueError(f"The guest has not reached {state}; inspect status before another lifecycle action.")
......@@ -62,13 +85,12 @@ def wait_for_state(state):
6285
6386@contextlib.contextmanager
6487def guest_screen():
65 ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip()
6688 with tempfile.TemporaryDirectory(prefix="vnc-", dir=PRIVATE) as directory:
6789 path = Path(directory) / "console.sock"
6890 tunnel = subprocess.Popen([*ssh_args(), "-N", "-o", "ExitOnForwardFailure=yes",
6991 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",
7092 "-o", "StreamLocalBindMask=0177", "-L",
71 f"{path}:127.0.0.1:5900", "admin@" + ip],
93 f"{path}:127.0.0.1:5900", GUEST],
7294 stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
7395 stderr=subprocess.DEVNULL)
7496 try:
......@@ -208,6 +230,7 @@ def main():
208230 sub.add_parser("stop")
209231 sub.add_parser("suspend")
210232 sub.add_parser("reboot")
233 sub.add_parser("revive", help="Start the VM unless it is running or was stopped on purpose")
211234 for action in ["exec", "ssh"]:
212235 command = sub.add_parser(action)
213236 command.add_argument("command", nargs=argparse.REMAINDER)
......@@ -222,12 +245,17 @@ def main():
222245 local_port = args.local_port if args.local_port is not None else args.port
223246 if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535:
224247 parser.error("Ports must be between 1 and 65535.")
225 ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip()
226248 print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.",
227249 file=sys.stderr, flush=True)
228250 os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes",
229251 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",
230 "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip])
252 "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", GUEST])
253 if args.action == "revive":
254 if (PRIVATE / "stopped").exists() or vm()["Running"]:
255 return
256 crashes = sorted(Path("/Library/Logs/DiagnosticReports").glob("tart-*.ips"))
257 print(f"{time.ctime()}: VM is down; newest tart crash report: {crashes[-1] if crashes else 'none'}", flush=True)
258 args.action = "start"
231259 if args.action in {"start", "stop", "suspend", "reboot"}:
232260 lock = (PRIVATE / "lifecycle.lock").open("a")
233261 os.chmod(PRIVATE / "lifecycle.lock", 0o600)
......@@ -240,34 +268,47 @@ def main():
240268 lock.write(args.action)
241269 lock.flush()
242270 if args.action == "reboot":
243 tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now", timeout=15)
244 wait_for_state("stopped")
271 shut_down()
245272 args.action = "start"
246273 if args.action == "start":
247 rows = json.loads(tart("list", "--format", "json", capture_output=True, text=True).stdout)
248 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]:
274 (PRIVATE / "stopped").unlink(missing_ok=True)
275 interface = re.search(r"interface: (\S+)", subprocess.run(["/sbin/route", "-n", "get", "default"],
276 capture_output=True, text=True, check=True).stdout)[1]
277 if vm()["Running"]:
249278 print("GUI-agent VM is already running.")
250 return
251 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.
252 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",
253 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",
254 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"])
255 (PRIVATE / "run.log").write_text("")
256 os.chmod(PRIVATE / "run.log", 0o600)
257 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",
258 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)
259 deadline = time.monotonic() + 90
260 while time.monotonic() < deadline:
261 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:
262 raise ValueError("VM startup failed; inspect the private run log locally.")
263 try:
264 tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5)
265 except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
266 time.sleep(1)
267 else:
268 print("GUI-agent VM is ready in its logged-in guest session.")
269 return
270 raise ValueError("VM startup is still pending; inspect status before retrying.")
279 else:
280 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.
281 # Bridged so iPhones can find and pair with the guest's Xcode over Bonjour.
282 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",
283 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",
284 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full",
285 f"--net-bridged={interface}"])
286 (PRIVATE / "run.log").write_text("")
287 os.chmod(PRIVATE / "run.log", 0o600)
288 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",
289 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)
290 deadline = time.monotonic() + 90
291 while True:
292 if time.monotonic() > deadline:
293 raise ValueError("VM startup is still pending; inspect status before retrying.")
294 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:
295 raise ValueError("VM startup failed; inspect the private run log locally.")
296 try:
297 guest_ready(capture_output=True, timeout=5)
298 except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
299 time.sleep(1)
300 else:
301 break
302 print("GUI-agent VM is ready in its logged-in guest session.")
303 # The bridged guest is on the LAN: only this Mac may reach its services; iPhones pair on ephemeral ports.
304 host = subprocess.run(["/usr/sbin/ipconfig", "getifaddr", interface],
305 capture_output=True, text=True, check=True).stdout.strip()
306 guest("sudo", "-n", "/bin/sh", "-c",
307 "/sbin/pfctl -q -a com.apple/net.paperclover -f - && /sbin/pfctl -q -E",
308 input=f"table <mac-host> {{ {host} }}\n"
309 "block in quick on ! lo0 proto tcp from ! <mac-host> to any port < 49152 flags S/SA\n"
310 "block in quick on ! lo0 proto udp to any port { 88 137 138 }\n",
311 capture_output=True, text=True)
271312 elif args.action == "status":
272313 tart("get", NAME)
273314 elif args.action == "exec":
......@@ -275,18 +316,18 @@ def main():
275316 parser.error("exec requires a guest command")
276317 tart("exec", "-i", NAME, *args.command)
277318 elif args.action == "ssh":
278 ip = tart("ip", NAME, capture_output=True, text=True).stdout.strip()
279319 command = ssh_args()
280320 if not args.command:
281321 command.append("-t")
282 os.execvp("ssh", [*command, "admin@" + ip, *args.command])
322 os.execvp("ssh", [*command, GUEST, *args.command])
283323 elif args.action == "stop":
284 tart("exec", NAME, "sudo", "-n", "shutdown", "-h", "now")
285 wait_for_state("stopped")
324 shut_down()
325 (PRIVATE / "stopped").touch()
286326 print("GUI-agent VM shut down.")
287327 elif args.action == "suspend":
288328 tart("suspend", NAME)
289329 wait_for_state("suspended")
330 (PRIVATE / "stopped").touch()
290331 print("GUI-agent VM suspended.")
291332 elif args.action == "console":
292333 python = ROOT / "runtime/venv/bin/python"
tools/mac-vm-bridge.py+1-1
......@@ -32,7 +32,7 @@ def status():
3232 agent = None
3333 if current["Running"]:
3434 try:
35 vm.tart("exec", vm.NAME, "/usr/bin/true", capture_output=True, timeout=2)
35 vm.guest_ready(capture_output=True, timeout=5)
3636 agent = "connected"
3737 except subprocess.SubprocessError:
3838 agent = "disconnected"
tools/release.py+1-1
......@@ -60,7 +60,7 @@ def host_digest(root):
6060 digest = hashlib.sha256()
6161 paths = sorted(path for path, relative in files(root)
6262 if relative.parts[0] in {"nixos", "dashboard", "config", "service"}
63 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"})
63 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"})
6464 for path in paths:
6565 digest.update(str(path.relative_to(root)).encode())
6666 _hash_contents(digest, path)
tools/shale-access-test.py deleted-230
......@@ -1,230 +0,0 @@
1#!/usr/bin/env python3
2import base64
3from contextlib import closing
4import importlib
5import json
6import os
7from pathlib import Path
8from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9import sqlite3
10import subprocess
11import sys
12import tempfile
13import threading
14import unittest
15from unittest.mock import patch
16
17import pygit2
18from argon2 import PasswordHasher
19
20shale = importlib.import_module('shale-access')
21
22
23class CGI(BaseHTTPRequestHandler):
24 """Plays the Shale gateway: every request passes through clone() before reaching git http-backend."""
25 def __init__(self, test, *args):
26 self.test = test
27 super().__init__(*args)
28
29 def log_message(self, *args):
30 pass
31
32 def do_GET(self):
33 self.relay()
34
35 def do_POST(self):
36 self.relay()
37
38 def relay(self):
39 try:
40 plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', ''))
41 except shale.Error:
42 self.send_response(401)
43 self.send_header('WWW-Authenticate', 'Basic realm="The Knife"')
44 self.end_headers()
45 return
46 body = self.rfile.read(int(self.headers.get('content-length', 0)))
47 output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True,
48 env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout
49 head, _, payload = output.partition(b'\r\n\r\n')
50 headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n'))
51 self.send_response(int(headers.pop('Status', '200').split()[0]))
52 for name, value in headers.items():
53 self.send_header(name, value)
54 self.end_headers()
55 self.wfile.write(payload)
56
57
58class Access(unittest.TestCase):
59 def setUp(self):
60 self.temp = tempfile.TemporaryDirectory()
61 self.addCleanup(self.temp.cleanup)
62 self.root = Path(self.temp.name)
63 self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'})
64 self.env.start()
65 self.addCleanup(self.env.stop)
66 (self.root / 'data').mkdir()
67 self.uuid = '0' * 26
68 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)
69 self.addCleanup(self.repo.free)
70 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})
71 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])
72 self.repo.create_reference('refs/heads/native', self.native)
73 self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native])
74 self.repo.create_reference('refs/heads/main', self.public)
75 self.repo.set_head('refs/heads/main')
76 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))
77 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
78 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')
79 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):
80 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))
81 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))
82 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
83 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')
84 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))
85 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):
86 db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name))
87 db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret')))
88
89 def commit(self, files, parents=None):
90 index = pygit2.Index()
91 for path, content in files.items():
92 index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB))
93 tree = index.write_tree(self.repo)
94 author = pygit2.Signature('Fixture', 'fixture@example.invalid')
95 return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or [])
96
97 def access(self, path, account='', authorization=''):
98 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})
99
100 def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self):
101 released = []
102 class TrackedRepository(pygit2.Repository):
103 def free(self):
104 released.append(True)
105 super().free()
106 with patch.object(shale.pygit2, 'Repository', TrackedRepository):
107 self.assertFalse(self.access('/the-knife/')['deny'])
108 self.assertTrue(self.access('/the-knife/objects/x')['deny'])
109 with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')):
110 with self.assertRaises(ValueError):
111 self.access('/the-knife/')
112 self.assertEqual(len(released), 3)
113
114 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):
115 public = self.access('/the-knife/tree/main/')
116 self.assertFalse(public['deny'])
117 self.assertIn(str(self.public), public['uri'])
118 for rev in ('native', str(self.native), str(self.native)[:12]):
119 for route in ('tree', 'plain', 'commit'):
120 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev))
121 self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny'])
122 self.repo.references.delete('refs/heads/native')
123 self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny'])
124
125 def test_every_revision_blocks_files_even_when_absent(self):
126 for rev in ('main', str(self.base)):
127 for route in ('tree', 'plain'):
128 for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'):
129 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path))
130 self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny'])
131
132 def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self):
133 result = self.access('/the-knife/commit/' + str(self.public))
134 self.assertFalse(result['deny'])
135 self.assertNotIn('SECRET', result['safeDiff'])
136 self.assertIn('PUBLIC', result['safeDiff'])
137 for suffix in ('diff', 'patch'):
138 result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)
139 self.assertFalse(result['deny'])
140 self.assertNotIn('SECRET', result['safeDiff'])
141 self.assertIn('PUBLIC', result['safeDiff'])
142 for suffix in ('stat', 'diffraw'):
143 self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny'])
144
145 def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self):
146 path = '/the-knife/plain/native/knife/private.rs'
147 for name in ('snow', 'guest'):
148 self.assertTrue(self.access(path, name)['allowed'])
149 for name in ('stranger', 'disabled', 'ai', ''):
150 self.assertTrue(self.access(path, name)['deny'])
151 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
152 db.execute("DELETE FROM memberships WHERE user_id='snow'")
153 self.assertTrue(self.access(path, 'snow')['deny'])
154
155 def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self):
156 path = '/the-knife/info/refs?service=git-receive-pack'
157 self.assertTrue(self.access(path, 'snow')['deny'])
158 self.assertTrue(self.access(path)['authenticate'])
159 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
160 self.assertTrue(self.access(path, authorization=token)['allowed'])
161 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
162 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
163 db.execute('DELETE FROM personal_tokens')
164 self.assertTrue(self.access(path, authorization=token)['deny'])
165
166 def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self):
167 for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'):
168 self.assertTrue(self.access(path)['clone'], path)
169 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
170 self.assertTrue(self.access(path, authorization=token)['allowed'], path)
171 for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'):
172 self.assertNotIn('clone', self.access(path), path)
173
174 def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self):
175 with self.assertRaises(shale.Error):
176 self.clone('/the-knife.git/info/refs?service=git-upload-pack')
177 self.repo.lookup_reference('refs/heads/main').set_target(self.base)
178 self.repo.create_reference('refs/heads/merge', self.public)
179 server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args))
180 self.addCleanup(server.server_close)
181 threading.Thread(target=server.serve_forever, daemon=True).start()
182 self.addCleanup(server.shutdown)
183 url = f'http://127.0.0.1:{server.server_port}/the-knife'
184 environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'}
185 for version in ('0', '1', '2'):
186 checkout = self.root / ('clone-v' + version)
187 subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment)
188 refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout
189 self.assertEqual(refs, f'refs/heads/main {self.base}\n')
190 for revision in (str(self.native), str(self.public)):
191 fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment)
192 self.assertNotEqual(fetched.returncode, 0, (version, revision))
193 self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0)
194 pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment)
195 self.assertNotEqual(pushed.returncode, 0)
196
197 def clone(self, uri, method='GET', content_type='', content_encoding=''):
198 return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding})
199
200 def test_native_push_during_classification_cannot_change_the_checked_revision(self):
201 newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native])
202 repo = self.repo
203 class ConcurrentPush:
204 def __getattr__(self, name): return getattr(repo, name)
205 def __getitem__(self, key): return repo[key]
206 def walk(self, *args):
207 repo.lookup_reference('refs/heads/native').set_target(newer)
208 repo.create_reference('refs/heads/deadbeef', newer, force=True)
209 return repo.walk(*args)
210 for revision in ('native', 'deadbeef'):
211 repo.lookup_reference('refs/heads/native').set_target(self.native)
212 if 'refs/heads/deadbeef' in repo.references:
213 repo.references.delete('refs/heads/deadbeef')
214 with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()):
215 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])
216
217 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):
218 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
219 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")
220 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])
221 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
222 self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny'])
223
224 def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self):
225 for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'):
226 self.assertTrue(self.access(path)['deny'], path)
227
228
229if __name__ == '__main__':
230 unittest.main()
tools/shale-access.py deleted-250
......@@ -1,250 +0,0 @@
1import base64
2from contextlib import closing
3import importlib
4import json
5import os
6from pathlib import Path
7import posixpath
8import re
9import sqlite3
10import subprocess
11import sys
12import tempfile
13from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
14
15import pygit2
16from argon2 import PasswordHasher
17from argon2.exceptions import VerificationError, InvalidHashError
18
19Error = importlib.import_module('dashboard-run').Error
20# The root host helper reads repositories owned by the Shale service UID.
21pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0)
22
23
24def restricted_path(path):
25 parts = path.split('/')
26 return parts[0] == 'knife' or 'Cargo.lock' in parts
27
28
29def allowed_account(db, user_id):
30 row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake "
31 "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m "
32 "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')",
33 (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone()
34 if not row:
35 return False
36 profile = json.loads(row[0])
37 return (profile.get('enabled') is True and not profile.get('requiredActions')
38 and profile.get('kind') != 'ai')
39
40
41def instance(service):
42 if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service):
43 raise Error(400, 'Choose a Shale instance.')
44 if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'):
45 return Path(fixture_root)
46 site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5))
47 return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service)
48
49
50def target(request_uri):
51 uri = urlsplit(request_uri)
52 path = uri.path
53 for _ in range(4):
54 decoded = unquote(path, errors='strict')
55 if decoded == path:
56 break
57 path = decoded
58 path = posixpath.normpath('/' + path.lstrip('/'))
59 return uri, path, path.strip('/').split('/')
60
61
62def upload_pack(uri, path, parts):
63 """Returns the CGI path of an anonymous fetch request, the only Git route served without a token."""
64 if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path:
65 return None
66 if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query:
67 return '/the-knife.git/' + '/'.join(parts[1:])
68 return None
69
70
71def repository(root, db):
72 row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone()
73 return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None
74
75
76def protected_commits(repo, native):
77 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
78 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
79 if native:
80 tips.append(native)
81 return {str(commit.id) for tip in tips for commit in repo.walk(tip)}
82
83
84def access(request, accounts=None):
85 root = instance(request['service'])
86 uri, path, parts = target(request['uri'])
87 if parts[0].lower() not in ('the-knife', 'the-knife.git'):
88 return {'allowed': False, 'deny': True}
89 git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD')
90 if not request['authorization'] and upload_pack(uri, path, parts):
91 return {'allowed': False, 'deny': False, 'clone': True}
92 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
93 accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard'))
94 db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',))
95 authorization = request['authorization']
96 allowed = False
97 if authorization:
98 try:
99 scheme, value = authorization.split(' ', 1)
100 if scheme.lower() == 'basic':
101 name, secret = base64.b64decode(value, validate=True).decode().split(':', 1)
102 users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall()
103 if len(users) == 1 and allowed_account(db, users[0][0]):
104 for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]):
105 try:
106 if PasswordHasher().verify(hashed, secret):
107 allowed = True
108 break
109 except (VerificationError, InvalidHashError):
110 pass
111 except (ValueError, UnicodeError):
112 pass
113 elif request['account'] and not git:
114 users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall()
115 allowed = len(users) == 1 and allowed_account(db, users[0][0])
116 if allowed:
117 return {'allowed': True, 'deny': False}
118 location = repository(root, db)
119 if not location:
120 return {'allowed': False, 'deny': True}
121 repo = pygit2.Repository(str(location))
122 try:
123 head = repo.lookup_reference('HEAD').target
124 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
125 protected = protected_commits(repo, references.get('refs/heads/native'))
126 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
127 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
128 return result
129 route = parts[1] if len(parts) > 1 else ''
130 query = dict(parse_qsl(uri.query, keep_blank_values=True))
131 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
132 return result
133 if route in ('', 'issues'):
134 if route == '' and query:
135 return result
136 result['htmlOnly'] = route != 'issues'
137 result['deny'] = False
138 return result
139 if route in ('heads', 'tags') and len(parts) == 2 and not query:
140 result['deny'] = False
141 return result
142 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
143 return result
144 if route not in ('log', 'tree', 'plain', 'commit'):
145 return result
146 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
147 suffix = ''
148 if route == 'commit':
149 revision, dot, suffix = revision.partition('.')
150 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
151 return result
152 if not revision or revision == '-':
153 revision = 'HEAD'
154 # Only named refs and object IDs have the same meaning in Shale and libgit2.
155 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
156 return result
157 try:
158 if revision == 'HEAD':
159 oid = references.get(head) if isinstance(head, str) else head
160 else:
161 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
162 if len(candidates) > 1:
163 return result
164 if candidates:
165 oid = candidates.pop()
166 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):
167 oid = repo[revision].id
168 else:
169 return result
170 if oid is None:
171 return result
172 commit = repo[oid].peel(pygit2.Commit)
173 except (KeyError, ValueError, pygit2.GitError):
174 return result
175 if str(commit.id) in protected:
176 return result
177 if route in ('tree', 'plain'):
178 result['htmlOnly'] = route != 'plain'
179 file = '/'.join(parts[3:])
180 if restricted_path(file):
181 return result
182 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):
183 return result
184 if route == 'commit':
185 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]
186 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
187 if sensitive:
188 if suffix in ('stat', 'diffraw'):
189 return result
190 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"
191 if suffix:
192 result['rawDiff'] = True
193 result['deny'] = False
194 return result
195 if route == 'log':
196 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})
197 else:
198 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]
199 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')
200 if query:
201 result['uri'] += '?' + urlencode(query)
202 result['deny'] = False
203 return result
204 finally:
205 repo.free()
206
207
208def clone(request):
209 root = instance(request['service'])
210 uri, path, parts = target(request['uri'])
211 location = upload_pack(uri, path, parts)
212 if not location:
213 raise Error(404, 'Clone The Knife over HTTPS.')
214 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
215 knife = repository(root, db)
216 if not knife:
217 raise Error(404, 'The Knife is unavailable.')
218 repo = pygit2.Repository(str(knife))
219 try:
220 main = repo.lookup_reference('refs/heads/main').resolve().target
221 native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None
222 protected = protected_commits(repo, native)
223 if any(str(commit.id) in protected for commit in repo.walk(main)):
224 raise Error(403, "The Knife's main branch includes native history.")
225 finally:
226 repo.free()
227 return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': {
228 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query,
229 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'],
230 }}
231
232
233def serve(objects, main):
234 """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects."""
235 repository = Path(tempfile.mkdtemp()) / 'the-knife.git'
236 (repository / 'objects/info').mkdir(parents=True)
237 (repository / 'refs/heads').mkdir(parents=True)
238 (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n')
239 (repository / 'HEAD').write_text('ref: refs/heads/main\n')
240 (repository / 'refs/heads/main').write_text(main + '\n')
241 (repository / 'objects/info/alternates').write_text(objects + '\n')
242 # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main.
243 environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)}
244 os.execvpe('git', ['git', 'http-backend'], environment | {
245 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null',
246 })
247
248
249if __name__ == '__main__':
250 serve(*sys.argv[1:])
tools/shale-webhook-test.py created+124
......@@ -0,0 +1,124 @@
1#!/usr/bin/env python3
2import http.server
3import json
4import os
5from pathlib import Path
6import sqlite3
7import subprocess
8import tempfile
9import threading
10import unittest
11
12REPO = Path(__file__).resolve().parent.parent
13HOOK = REPO / "service/shale/webhook.sh"
14PREPARE = REPO / "service/shale/prepare.py"
15
16
17class Receiver(http.server.BaseHTTPRequestHandler):
18 events = []
19
20 def do_POST(self):
21 body = self.rfile.read(int(self.headers["Content-Length"]))
22 self.events.append((self.headers["X-GitHub-Event"], self.path, json.loads(body)))
23 self.send_response(204)
24 self.end_headers()
25
26 def log_message(self, *_):
27 pass
28
29
30class ShaleWebhookTests(unittest.TestCase):
31 def setUp(self):
32 self.temp = tempfile.TemporaryDirectory()
33 self.root = Path(self.temp.name)
34 self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Receiver)
35 Receiver.events = []
36 self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
37 self.thread.start()
38
39 def tearDown(self):
40 self.server.shutdown()
41 self.thread.join()
42 self.server.server_close()
43 self.temp.cleanup()
44
45 def git(self, *args, cwd=None, env=None):
46 return subprocess.run(
47 ["git", *args], cwd=cwd, env=env, check=True, text=True,
48 capture_output=True,
49 ).stdout.strip()
50
51 def test_push_and_delete_match_snowcord_github_contract(self):
52 bare = self.root / "origin.git"
53 work = self.root / "work"
54 self.git("init", "--bare", "--initial-branch=main", bare)
55 self.git("config", "--file", bare / "config", "shale.snowglobe.repository", "snowcord")
56 (bare / "hooks/post-receive").symlink_to(HOOK)
57 self.git("init", "--initial-branch=main", work)
58 self.git("config", "user.name", "Clover", cwd=work)
59 self.git("config", "user.email", "clover@example.test", cwd=work)
60 (work / "file.txt").write_text("hello\n")
61 self.git("add", "file.txt", cwd=work)
62 self.git("commit", "-m", 'first "line"\n\nsecond \\ line', cwd=work)
63 self.git("remote", "add", "origin", str(bare), cwd=work)
64 env = {
65 **os.environ,
66 "DOMAIN": "shale.test",
67 "SHALE_WEBHOOK_ENABLED": "true",
68 "SHALE_WEBHOOK_URL": f"http://127.0.0.1:{self.server.server_port}/hook",
69 }
70 self.git("push", "origin", "main", cwd=work, env=env)
71
72 event, path, payload = Receiver.events.pop(0)
73 self.assertEqual(event, "push")
74 self.assertEqual(path, "/hook/github/")
75 self.assertEqual(payload["ref"], "refs/heads/main")
76 self.assertEqual(payload["repository"]["name"], "snowcord")
77 self.assertEqual(payload["sender"]["login"], "Clover")
78 self.assertEqual(payload["commits"][0]["message"], 'first "line"\n\nsecond \\ line')
79 self.assertEqual(payload["head_commit"]["id"], payload["after"])
80
81 self.git("push", "origin", "main:delete-me", cwd=work, env=env)
82 Receiver.events.pop(0)
83 self.git("push", "origin", ":delete-me", cwd=work, env=env)
84 event, _, payload = Receiver.events.pop(0)
85 self.assertEqual((event, payload["ref_type"], payload["ref"]), ("delete", "branch", "delete-me"))
86
87 def test_prepare_installs_only_named_repository_hooks(self):
88 owned = self.root / "repositories_owned"
89 database = self.root / "data/astheno.shale.db"
90 database.parent.mkdir()
91 with sqlite3.connect(database) as db:
92 db.executescript("""
93 CREATE TABLE users (provider TEXT, snowflake TEXT);
94 CREATE TABLE repositories (uuid TEXT, name TEXT);
95 INSERT INTO repositories VALUES ('snowcord-id', 'snowcord');
96 INSERT INTO repositories VALUES ('other-id', 'other');
97 """)
98 for name in ("snowcord-id", "other-id"):
99 self.git("init", "--bare", owned / name)
100 env = {**os.environ, "SNOWGLOBE_DOMAIN": "test"}
101 subprocess.run(
102 ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}),
103 env=env, check=True, text=True, capture_output=True,
104 )
105 hook = owned / "snowcord-id/hooks/post-receive"
106 self.assertEqual(os.readlink(hook), "/data/shale-webhook")
107 self.assertEqual((self.root / "data/shale-webhook").stat().st_mode & 0o777, 0o750)
108 self.assertEqual(
109 self.git("config", "--file", owned / "snowcord-id/config", "--get", "shale.snowglobe.repository"),
110 "snowcord",
111 )
112 self.assertFalse((owned / "other-id/hooks/post-receive").exists())
113
114 with sqlite3.connect(database) as db:
115 db.execute("UPDATE repositories SET name='renamed' WHERE uuid='snowcord-id'")
116 subprocess.run(
117 ["python3", PREPARE], input=json.dumps({"hostRoot": str(self.root), "uid": os.getuid()}),
118 env=env, check=True, text=True, capture_output=True,
119 )
120 self.assertFalse(os.path.lexists(hook))
121
122
123if __name__ == "__main__":
124 unittest.main()