diff --git a/config/shale.json b/config/shale.json index 8669e03b5264e833420068132ba418e91f9287cc..a1b954e2576117f2f7a83803615bcb6ebf6f1bc4 100644 --- a/config/shale.json +++ b/config/shale.json @@ -1 +1 @@ -{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}} +{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}} diff --git a/dashboard/src/auth.rs b/dashboard/src/auth.rs index 938817733489d4269d8891e51e6a3b6ba50cf9f4..7f9b5e0db99e87039835b7043af970ca5900d09c 100644 --- a/dashboard/src/auth.rs +++ b/dashboard/src/auth.rs @@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session"; const FLOW_COOKIE: &str = "__Host-snow-flow"; const CSRF_COOKIE: &str = "__Host-snow-csrf"; const SESSION_TTL: i64 = 30 * 86400; -const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"]; +const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"]; pub struct Store { pub db: Mutex, @@ -201,10 +201,13 @@ impl Store { oidc::initialise(&db)?; guest::initialise(&db)?; for name in GROUPS { - db.execute( + let added = db.execute( "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)", sql![uuid::Uuid::new_v4().to_string(), name], )?; + if *name == "knife-native" && added != 0 { + db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?; + } } Ok(Self { db: Mutex::new(db), diff --git a/dashboard/src/guest.rs b/dashboard/src/guest.rs index cb63a3bee535ad6912300b81b55cdff9853be695..1a2ad837201615e5659007260aa04f1a4f50dead 100644 --- a/dashboard/src/guest.rs +++ b/dashboard/src/guest.rs @@ -22,7 +22,9 @@ pub fn is_guest(user: &Value) -> bool { pub fn initialise(db: &Connection) -> Result<()> { db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL); CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id)); - CREATE TRIGGER IF NOT EXISTS guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END; + DROP TRIGGER IF EXISTS guest_no_groups; + CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; + CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END; CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?; Ok(()) @@ -934,7 +936,7 @@ mod tests { } #[test] - fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() { + fn identities_stay_separate_and_guests_can_only_join_native_access() { let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4())); let auth = auth::Store::new( &path, @@ -983,6 +985,9 @@ mod tests { db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first]) .is_err() ); + db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap(); + assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native"); + assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err()); assert!(auth::set_password(&db, &first, "a-password-hash").is_err()); assert!( db.execute( diff --git a/dashboard/src/knife-restricted.html b/dashboard/src/knife-restricted.html new file mode 100644 index 0000000000000000000000000000000000000000..3538a53cb93f2c30012a02f65a98f8ec521a0e51 --- /dev/null +++ b/dashboard/src/knife-restricted.html @@ -0,0 +1,28 @@ + + + + +The Knife's Native UI is restricted + +
+

The Knife's Native UI is restricted

+

This part of the code is sensitive. Please contact clover to request private access.

+Login +
+ diff --git a/dashboard/src/shale.rs b/dashboard/src/shale.rs index 3de986d0d5587fdbad631527265f74314b554e15..fe8229e02c67b20737cb3c5ebc8edc2ba1a53832 100644 --- a/dashboard/src/shale.rs +++ b/dashboard/src/shale.rs @@ -487,7 +487,7 @@ fn fields(query: &str) -> Result> { } Ok(fields) } -fn username(page: &str) -> Result { +pub(crate) fn username(page: &str) -> Result { let document = Html::parse_document(page); if document .select(&Selector::parse("body#page-user-settings").unwrap()) diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index f2380243b0af4af3dad703b635eea613dea15e85..d44ed182ffd6f74d809d06a65c553236b465ace6 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -347,6 +347,61 @@ fn strip_hop_headers(headers: &mut HeaderMap) { } } +fn knife_uri(path: &str) -> bool { + let mut decoded = path.to_owned(); + for _ in 0..5 { + if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) { + return true; + } + let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned(); + if next == decoded { break; } + decoded = next; + } + false +} + +fn restricted(authenticate: bool) -> Response { + (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "private, no-store"), + ("vary", "Cookie, Authorization"), + ("referrer-policy", "same-origin"), + ("x-content-type-options", "nosniff"), + ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"), + ("www-authenticate", "Basic realm=\"The Knife\""), + ], include_str!("knife-restricted.html")).into_response() +} + +fn redact_knife(html: &str, policy: &Value) -> Result { + let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect(); + let document = scraper::Html::parse_document(html); + let diff = policy["safeDiff"].as_str(); + if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 { + return Err(Error::new(403, "The Knife's Native UI is restricted")); + } + let links = scraper::Selector::parse("a[href]").unwrap(); + let rows: Vec = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| { + row.select(&links).any(|link| link.attr("href").is_some_and(|href| { + href.split(['/', '?', '#', '.']).any(|part| commits.contains(part)) + })) + }).collect(); + let mut row_index = 0; + Ok(lol_html::rewrite_str(html, RewriteStrSettings::new() + .append_element_content_handler(element!("#m-code", |element| { + if let Some(diff) = diff { + let diff = diff.replace('&', "&").replace('<', "<").replace('>', ">"); + element.set_inner_content(&format!("
{diff}
"), ContentType::Html); + } + Ok(()) + })) + .append_element_content_handler(element!("tr", |element| { + if *rows.get(row_index).unwrap_or(&true) { element.remove(); } + row_index += 1; + Ok(()) + })) + )?) +} + pub async fn proxy(app: Arc, request: Request) -> Result { let (mut parts, body) = request.into_parts(); let host = parts @@ -379,6 +434,46 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Err(Error::new(403, "Open Shale to continue.")); } let origin = url::Url::parse(&format!("https://{host}{uri}"))?; + let knife = knife_uri(uri.path()); + let mut target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; + let mut knife_policy = Value::Null; + if knife { + let mut account = String::new(); + if let Some(cookie) = parts.headers.get("cookie") { + let settings = match &app.internal { + Some((base, _)) => format!("{base}services/{service}/-/settings"), + None => format!("http://{upstream}/-/settings"), + }; + let mut identity = app.shale_http.get(settings).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3)); + if app.internal.is_none() { identity = identity.header("host", host); } + for name in ["x-forwarded-for", "x-real-ip", "user-agent"] { + if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); } + } + if let Ok(response) = identity.send().await + && response.status() == StatusCode::OK + && let Ok(body) = response.text().await + { + account = shale::username(&body).unwrap_or_default(); + } + } + knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await { + Ok(policy) => policy, + Err(_) => return Ok(restricted(false)), + }; + if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } + if let Some(uri) = knife_policy["uri"].as_str() { + target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; + } + if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() { + return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response()); + } + } let profiles = profiles(&app.auth)?; if matches!(parts.method, Method::GET | Method::HEAD) && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref()) @@ -393,10 +488,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { ) .into_response()); } - let target = match &app.internal { - Some((base, _)) => format!("{base}services/{service}{uri}"), - None => format!("http://{upstream}{uri}"), - }; if app.internal.is_some() { parts.headers.remove("host"); } @@ -424,9 +515,17 @@ pub async fn proxy(app: Arc, request: Request) -> Result { .body(reqwest::Body::wrap_stream(body.into_data_stream())) .send() .await?; - let status = upstream_response.status(); + let status = upstream_response.headers().get("status") + .and_then(|value| value.to_str().ok()).and_then(|value| value.split_whitespace().next()) + .and_then(|value| value.parse::().ok()).and_then(|value| StatusCode::from_u16(value).ok()) + .unwrap_or(upstream_response.status()); let mut headers = upstream_response.headers().clone(); strip_hop_headers(&mut headers); + headers.remove("status"); + if knife { + headers.insert("cache-control", "private, no-store".parse().unwrap()); + headers.insert("vary", "Cookie, Authorization".parse().unwrap()); + } let html = parts.method != Method::HEAD && headers .get("content-type") @@ -482,6 +581,14 @@ pub async fn proxy(app: Arc, request: Request) -> Result { profiles.insert(name.clone(), profile); } } + let redacted; + let text = if knife && knife_policy["allowed"] != true { + redacted = match redact_knife(text, &knife_policy) { + Ok(html) => html, + Err(_) => return Ok(restricted(false)), + }; + redacted.as_str() + } else { text }; let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?; for name in [ "content-length", @@ -498,6 +605,9 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Ok((status, headers, rewritten).into_response()); } } + if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) { + return Ok(restricted(false)); + } let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) .chain(upstream_response.bytes_stream()); Ok((status, headers, Body::from_stream(stream)).into_response()) @@ -509,6 +619,20 @@ mod tests { use rusqlite::params as sql; use scraper::{Html, Selector}; + #[test] + fn native_content_is_removed_from_both_diff_styles_and_history_rows() { + let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"}); + let html = "
PRIVATE MESSAGE
PUBLIC ROW
PRIVATE PATHS
knife/private.rs
PRIVATE SOURCE
src/public.rs
PUBLIC SOURCE
nested/Cargo.lock
PRIVATE LOCK
"; + let output = redact_knife(html, &policy).unwrap(); + assert!(!output.contains("PRIVATE"), "{output}"); + assert!(output.contains("PUBLIC ROW"), "{output}"); + assert!(output.contains("PUBLIC SOURCE"), "{output}"); + assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1); + assert!(redact_knife("
unknown diff markup
", &policy).is_err()); + for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); } + assert!(!knife_uri("/snowbound/commit/native")); + } + #[test] fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() { let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4())); diff --git a/dashboard/src/users.rs b/dashboard/src/users.rs index 61e8d32804b8a61004e6ed5386339d7ad453b981..86d19c7f121dd3f937d4a2fa9a9f435b28d0a5e2 100644 --- a/dashboard/src/users.rs +++ b/dashboard/src/users.rs @@ -158,10 +158,13 @@ pub async fn route( for id in &users { uuid(id)?; let user = auth::user(&transaction, id)?; - if guest::is_guest(&user) || user["kind"] == "ai" { + let guest_groups = if guest::is_guest(&user) { + add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true)) + } else { false }; + if guest_groups || user["kind"] == "ai" { return Err(Error::new( 400, - "Guests can use Shale only. Select regular accounts to edit groups.", + "Guests can only join knife-native.", )); } for group in &add { @@ -292,7 +295,7 @@ pub async fn route( let transaction = db.transaction()?; let mut user = auth::user(&transaction, id)?; if guest::is_guest(&user) - && matches!(parts, [_, "password"] | [_, "groups", _]) + && matches!(parts, [_, "password"]) && method != Method::GET { return Err(Error::new( @@ -342,6 +345,9 @@ pub async fn route( .optional()?; let name = name .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?; + if guest::is_guest(&user) && name != "knife-native" { + return Err(Error::new(400, "Guests can only join knife-native.")); + } if own && method == Method::DELETE && name == "infra-admin" { return Err(Error::new( 400, diff --git a/dashboard/web/pages/Users.tsx b/dashboard/web/pages/Users.tsx index 70c6141ab90e4e86057f5716d4183ff976fce7ca..290c67cab986898948353b76e45c5e828a8bc4fd 100644 --- a/dashboard/web/pages/Users.tsx +++ b/dashboard/web/pages/Users.tsx @@ -68,6 +68,7 @@ function adds(group: string, services: ServiceSummary[]) { /** "30 members · opens Jellyfin, Navidrome" */ function groupTip(group: string, d: Data) { + if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`; const more = adds(group, d.services); const members = d.users.filter((user) => names(user).includes(group)).length; const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : []; @@ -575,7 +576,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {
{(group) => ( - diff --git a/flake.nix b/flake.nix index 7ee41c2b59a24f7fb4f675bf9bad35eacae1f526..93df49be6c286f21ad03d87042504899e1ba7c91 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ outputs = { self, nixpkgs, ... }: { packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { }; + packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { }; packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image; packages.x86_64-linux.installer = self.nixosConfigurations.installer.config.system.build.isoImage; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index c5ebbbfe95dca0713004bee294f016780214acd1..d308f68053870ceaeb89177893204725ead167f4 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,9 +6,10 @@ let proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../tools/namespace-transition.py ../dashboard/src/namespace-transition.json ../guest/protocol.json ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../tools/namespace-transition.py ../dashboard/src/namespace-transition.json ../guest/protocol.json ]; }; nativePkl = pkgs.callPackage ./pkl.nix { }; + hostPython = pkgs.callPackage ./host-python.nix { }; screenPython = pkgs.python3.withPackages (packages: [ packages.pygobject3 packages.gst-python ]); screenPlugins = (with pkgs.gst_all_1; [ (lib.getLib gstreamer) gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-plugins-rs ]) ++ [ (lib.getLib pkgs.libnice) ]; secureOvmf = pkgs.OVMF.override { @@ -195,7 +196,7 @@ in SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables; }; serviceConfig = { - ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py"; + ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py"; Type = "notify"; Group = "snowglobe-dashboard"; StateDirectory = "snowglobe/host"; diff --git a/nixos/host-python.nix b/nixos/host-python.nix new file mode 100644 index 0000000000000000000000000000000000000000..5a5b85665df71d08ca9fc280068c36d37d9d7573 --- /dev/null +++ b/nixos/host-python.nix @@ -0,0 +1,2 @@ +{ python3 }: +python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ]) diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index e7625de03a885b88fc791dcb886b8cde0caf678b..f50919b50bb577ea871e2f555c390742cad58e76 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -92,6 +92,7 @@ class Host: "index.diff": {"dataset", "from", "to"}, "index.discard": {"dataset", "snapshot"}, "shale.metadata": {"repository", "accounts"}, + "shale.access": {"service", "uri", "account", "authorization"}, "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, @@ -105,6 +106,8 @@ class Host: raise Rejected("Use only the fields required by this host operation.") if operation == "copyparty.manage": return copyparty.handle(request) + if operation == "shale.access": + return importlib.import_module("shale-access").access(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index 0aed836a0303aa3d0b8cd100909ddf9d1eb0a14b..38137db63f30e420dfda977af2f3a909e9b5500a 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -18,6 +18,7 @@ import snowglobe OPERATIONS = frozenset({ + "shale.access", "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", "storage.snapshots", "vm.node", "vm.domains", "vm.library", "deploy.current", "deploy.main", "deploy.history", "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", @@ -34,6 +35,14 @@ class PreviewHost(boundary.Host): def handle(self, request): operation = request.get("operation") if isinstance(request, dict) else None + if operation == "shale.access": + service = request.get("service", "") + if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): + raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") + linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) + if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: + raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") + return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") if operation in {"files.snapshot", "files.snapshots", "files.discard"}: datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: @@ -159,11 +168,12 @@ def stage(): runtime = "/run/snowglobe-preview/" + identity host_environment = snowglobe.transition.host_environment() host_path = next(value for value in shlex.split(host_environment) if value.startswith("PATH=")) + host_python = snowglobe.command(*nix, "build", "--no-link", "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip() snowglobe.transition.stop_preview(identity) snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, "--property=Restart=on-failure", "--property=MemoryMax=256M", "--property=TasksMax=32", - "--setenv=" + host_path, - sys.executable, str(Path(__file__).resolve()), "--serve", identity) + "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], + host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity) for _ in range(100): if Path(runtime, "host.sock").exists(): break diff --git a/tools/release.py b/tools/release.py index 46e41a1d7654bb05caed20e99931f65afa626b57..fc6d8317fedda2044cbcf6e9deb4ecb2064180a5 100644 --- a/tools/release.py +++ b/tools/release.py @@ -63,7 +63,7 @@ def host_digest(root): digest = hashlib.sha256() paths = sorted(path for path, relative in files(root) if relative.parts[0] in {"nixos", "dashboard", "config", "service"} - or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/dashboard-run.py", "tools/release.py", "tools/namespace-transition.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "guest/protocol.json"}) + or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/namespace-transition.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "guest/protocol.json"}) for path in paths: digest.update(str(path.relative_to(root)).encode()) _hash_contents(digest, path) diff --git a/tools/router.py b/tools/router.py index dcf4144029a18ce02ab637f63d01b9738c0fe2b4..113adcfb37a50a2b7b3277ac696a21a991be5306 100644 --- a/tools/router.py +++ b/tools/router.py @@ -89,18 +89,6 @@ def shale_write_routes(host): " }", ' respond @shale_unsafe_origin "Forbidden" 403'] -def shale_git_routes(upstreams): - lines = [" @shale_git path */info/refs */git-upload-pack */git-receive-pack", - " handle @shale_git {", f" reverse_proxy {upstreams} {{"] - # Shale forwards CGI Status as a header instead of the HTTP status. - for status in (400, 403, 404, 405, 415, 500): - lines += [f' @cgi_{status} header Status "{status} *"', - f" handle_response @cgi_{status} {{", - " header {", " -Status", " defer", " }", - f" copy_response {status}", " }"] - return [*lines, " }", " }"] - - def shale_login_routes(host): # Shale parses the login Referer as an absolute URL after OAuth. return_url = rf"^https://{re.escape(host)}/(?:[A-Za-z0-9._~!$&'()*+,;=:@/?#\[\]-]|%[0-9A-Fa-f]{{2}})*$" @@ -304,8 +292,7 @@ def render(token): # Keep the Origin guard before every static/proxy handle; # otherwise Caddy sorts those handles ahead of respond. lines += [" route {", *shale_write_routes(host), - *shale_login_routes(host), - *shale_git_routes(" ".join(sorted(route["upstreams"])))] + *shale_login_routes(host)] if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])), "/srv/staging/" + service) diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py new file mode 100644 index 0000000000000000000000000000000000000000..cc1414670351ed9cd5cf3896a03dd8b8db82bb83 --- /dev/null +++ b/tools/shale-access-test.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +import base64 +import importlib +import json +import os +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +import pygit2 +from argon2 import PasswordHasher + +shale = importlib.import_module('shale-access') + + +class Access(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'}) + self.env.start() + self.addCleanup(self.env.stop) + (self.root / 'data').mkdir() + self.uuid = '0' * 26 + self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) + self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) + self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) + self.repo.create_reference('refs/heads/native', self.native) + self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native]) + self.repo.create_reference('refs/heads/main', self.public) + self.repo.set_head('refs/heads/main') + (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) + with sqlite3.connect(self.root / 'accounts.sqlite') as db: + db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') + for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): + db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) + if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') + db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) + for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): + db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name)) + db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret'))) + + def commit(self, files, parents=None): + index = pygit2.Index() + for path, content in files.items(): + index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB)) + tree = index.write_tree(self.repo) + author = pygit2.Signature('Fixture', 'fixture@example.invalid') + return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or []) + + def access(self, path, account='', authorization=''): + return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) + + def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): + public = self.access('/the-knife/tree/main/') + self.assertFalse(public['deny']) + self.assertIn(str(self.public), public['uri']) + for rev in ('native', str(self.native), str(self.native)[:12]): + for route in ('tree', 'plain', 'commit'): + self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev)) + self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny']) + self.repo.references.delete('refs/heads/native') + self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny']) + + def test_every_revision_blocks_files_even_when_absent(self): + for rev in ('main', str(self.base)): + for route in ('tree', 'plain'): + for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'): + self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path)) + self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny']) + + def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self): + result = self.access('/the-knife/commit/' + str(self.public)) + self.assertFalse(result['deny']) + self.assertNotIn('SECRET', result['safeDiff']) + self.assertIn('PUBLIC', result['safeDiff']) + for suffix in ('diff', 'patch'): + result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix) + self.assertFalse(result['deny']) + self.assertNotIn('SECRET', result['safeDiff']) + self.assertIn('PUBLIC', result['safeDiff']) + for suffix in ('stat', 'diffraw'): + self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny']) + + def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self): + path = '/the-knife/plain/native/knife/private.rs' + for name in ('snow', 'guest'): + self.assertTrue(self.access(path, name)['allowed']) + for name in ('stranger', 'disabled', 'ai', ''): + self.assertTrue(self.access(path, name)['deny']) + with sqlite3.connect(self.root / 'accounts.sqlite') as db: + db.execute("DELETE FROM memberships WHERE user_id='snow'") + self.assertTrue(self.access(path, 'snow')['deny']) + + def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): + path = '/the-knife/info/refs?service=git-upload-pack' + self.assertTrue(self.access(path, 'snow')['deny']) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access(path, authorization=token)['allowed']) + self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.execute('DELETE FROM personal_tokens') + self.assertTrue(self.access(path, authorization=token)['deny']) + + def test_native_push_during_classification_cannot_change_the_checked_revision(self): + newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) + repo = self.repo + class ConcurrentPush: + def __getattr__(self, name): return getattr(repo, name) + def __getitem__(self, key): return repo[key] + def walk(self, *args): + repo.lookup_reference('refs/heads/native').set_target(newer) + repo.create_reference('refs/heads/deadbeef', newer, force=True) + return repo.walk(*args) + for revision in ('native', 'deadbeef'): + repo.lookup_reference('refs/heads/native').set_target(self.native) + if 'refs/heads/deadbeef' in repo.references: + repo.references.delete('refs/heads/deadbeef') + with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()): + self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) + + def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") + self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny']) + + def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self): + for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'): + self.assertTrue(self.access(path)['deny'], path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/shale-access.py b/tools/shale-access.py new file mode 100644 index 0000000000000000000000000000000000000000..f5d141a17d056ac0f2659534f7c9aba2f5ed82f1 --- /dev/null +++ b/tools/shale-access.py @@ -0,0 +1,176 @@ +import base64 +from contextlib import closing +import importlib +import json +import os +from pathlib import Path +import posixpath +import re +import sqlite3 +import subprocess +from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit + +import pygit2 +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError + +Error = importlib.import_module('dashboard-run').Error +# The root host helper reads repositories owned by the Shale service UID. +pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) + + +def restricted_path(path): + parts = path.split('/') + return parts[0] == 'knife' or 'Cargo.lock' in parts + + +def allowed_account(db, user_id): + row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake " + "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m " + "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')", + (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone() + if not row: + return False + profile = json.loads(row[0]) + return (profile.get('enabled') is True and not profile.get('requiredActions') + and profile.get('kind') != 'ai') + + +def access(request, accounts=None): + service = request['service'] + if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): + raise Error(400, 'Choose a Shale instance.') + if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): + root = Path(fixture_root) + else: + site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) + root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) + uri = urlsplit(request['uri']) + path = uri.path + for _ in range(4): + decoded = unquote(path, errors='strict') + if decoded == path: + break + path = decoded + path = posixpath.normpath('/' + path.lstrip('/')) + parts = path.strip('/').split('/') + if parts[0].lower() not in ('the-knife', 'the-knife.git'): + return {'allowed': False, 'deny': True} + git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') + with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) + db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) + authorization = request['authorization'] + allowed = False + if authorization: + try: + scheme, value = authorization.split(' ', 1) + if scheme.lower() == 'basic': + name, secret = base64.b64decode(value, validate=True).decode().split(':', 1) + users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall() + if len(users) == 1 and allowed_account(db, users[0][0]): + for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]): + try: + if PasswordHasher().verify(hashed, secret): + allowed = True + break + except (VerificationError, InvalidHashError): + pass + except (ValueError, UnicodeError): + pass + elif request['account'] and not git: + users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall() + allowed = len(users) == 1 and allowed_account(db, users[0][0]) + if allowed: + return {'allowed': True, 'deny': False} + row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() + if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): + return {'allowed': False, 'deny': True} + repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) + policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] + head = repo.lookup_reference('HEAD').target + references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} + protected = set() + tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] + if 'refs/heads/native' in references: + tips.append(references['refs/heads/native']) + for tip in tips: + protected.update(str(commit.id) for commit in repo.walk(tip)) + result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} + if parts[0] != 'the-knife' or '%' in path or '\\' in path: + return result + route = parts[1] if len(parts) > 1 else '' + query = dict(parse_qsl(uri.query, keep_blank_values=True)) + if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): + return result + if route in ('', 'issues'): + if route == '' and query: + return result + result['htmlOnly'] = route != 'issues' + result['deny'] = False + return result + if route in ('heads', 'tags') and len(parts) == 2 and not query: + result['deny'] = False + return result + if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): + return result + if route not in ('log', 'tree', 'plain', 'commit'): + return result + revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' + suffix = '' + if route == 'commit': + revision, dot, suffix = revision.partition('.') + if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): + return result + if not revision or revision == '-': + revision = 'HEAD' + # Only named refs and object IDs have the same meaning in Shale and libgit2. + if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): + return result + try: + if revision == 'HEAD': + oid = references.get(head) if isinstance(head, str) else head + else: + candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} + if len(candidates) > 1: + return result + if candidates: + oid = candidates.pop() + elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): + oid = repo[revision].id + else: + return result + if oid is None: + return result + commit = repo[oid].peel(pygit2.Commit) + except (KeyError, ValueError, pygit2.GitError): + return result + if str(commit.id) in protected: + return result + if route in ('tree', 'plain'): + result['htmlOnly'] = route != 'plain' + file = '/'.join(parts[3:]) + if restricted_path(file): + return result + if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): + return result + if route == 'commit': + diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] + sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) + if sensitive: + if suffix in ('stat', 'diffraw'): + return result + result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" + if suffix: + result['rawDiff'] = True + result['deny'] = False + return result + if route == 'log': + result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) + else: + parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] + result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') + if query: + result['uri'] += '?' + urlencode(query) + result['deny'] = False + return result