diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index ee043bb5274e2c7daf53a145abde57f0216f7c7d..e34b96ecd2d18058431511bc9c826019bdef9bf5 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -412,6 +412,46 @@ fn redact_knife(html: &str, policy: &Value) -> Result { )?) } +/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch. +async fn clone(request: Value, body: Body) -> Result { + use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; + let (_, stream) = host::open(request).await?; + let (reader, mut writer) = stream.into_split(); + tokio::spawn(async move { + let mut body = body.into_data_stream(); + while let Some(Ok(chunk)) = body.next().await { + if writer.write_all(&chunk).await.is_err() { return; } + } + let _ = writer.shutdown().await; + }); + let mut reader = tokio::io::BufReader::new(reader); + let mut response = Response::builder(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).await? == 0 { + return Err(Error::new(502, "The Knife clone ended early.")); + } + let line = line.trim_end(); + if line.is_empty() { break; } + let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?; + let value = value.trim(); + if name.eq_ignore_ascii_case("status") { + response = response.status(value.split_whitespace().next().unwrap_or_default().parse::()?); + } else { + response = response.header(name, value); + } + } + let stream = futures::stream::unfold(reader, |mut reader| async move { + let mut buffer = vec![0; 64 * 1024]; + match reader.read(&mut buffer).await { + Ok(0) => None, + Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) } + Err(error) => Some((Err(error), reader)), + } + }); + Ok(response.body(Body::from_stream(stream))?) +} + pub async fn proxy(app: Arc, request: Request) -> Result { let (mut parts, body) = request.into_parts(); let host = parts @@ -475,6 +515,11 @@ pub async fn proxy(app: Arc, request: Request) -> Result { Err(_) => return Ok(restricted(false)), }; if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } + if knife_policy["clone"] == true { + let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default(); + let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")}); + return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true))); + } if let Some(uri) = knife_policy["uri"].as_str() { target = match &app.internal { Some((base, _)) => format!("{base}services/{service}{uri}"), diff --git a/nixos/host-python.nix b/nixos/host-python.nix index 5a5b85665df71d08ca9fc280068c36d37d9d7573..a9d47aee38f4142cd6d623f45c50c98f52b355c6 100644 --- a/nixos/host-python.nix +++ b/nixos/host-python.nix @@ -1,2 +1,6 @@ -{ python3 }: -python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ]) +{ python3, git }: +python3.buildEnv.override { + extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ]; + # shale-access serves anonymous Knife clones through git http-backend. + makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ]; +} diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index 1cdaeaa660c5b4616f966e65d851bb7bdfa945a4..4c47f424292dde11e7c3ed8054176d0699469405 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -16,6 +16,7 @@ import signal import socket import struct import subprocess +import sys import threading import time import vms @@ -41,6 +42,7 @@ MAX_REQUEST = PROTOCOL['request'] MAX_RESPONSE = PROTOCOL['response'] STREAM_SLOTS = threading.BoundedSemaphore(4) GUEST_SLOTS = threading.BoundedSemaphore(4) +CLONE_SLOTS = threading.BoundedSemaphore(2) VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" @@ -93,6 +95,7 @@ class Host: "index.discard": {"dataset", "snapshot"}, "shale.metadata": {"repository", "accounts"}, "shale.access": {"service", "uri", "account", "authorization"}, + "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, @@ -109,6 +112,8 @@ class Host: return copyparty.handle(request) if operation == "shale.access": return importlib.import_module("shale-access").access(request) + if operation == "shale.clone": + return importlib.import_module("shale-access").clone(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": @@ -463,6 +468,20 @@ def stream_upload(connection, request): Path(temporary).unlink(missing_ok=True) +def stream_clone(connection, plan): + owner = os.stat(plan["objects"]) + # Service identities have numeric owners without passwd entries, which systemd's User= rejects. + command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", + *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", + "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), + *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), + "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", + sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] + send_response(connection, {"value": None}) + connection.setblocking(True) + subprocess.run(command, stdin=connection, stdout=connection) + + def serve_connection(connection, host, allowed_uid, allowed_operations=None): with connection: connection.settimeout(65) @@ -501,6 +520,14 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): return finally: streams.release() + if isinstance(request, dict) and request.get("operation") == "shale.clone": + if not CLONE_SLOTS.acquire(blocking=False): + raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") + try: + stream_clone(connection, host.handle(request)) + return + finally: + CLONE_SLOTS.release() response = {"value": host.handle(request)} except Rejected as error: response = {"error": str(error), "status": 400} diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index 1eb47628058165c12a99c1a3270676bcdb97e5e2..fea28b22bf53fec95b68f23b9b5cc75932ba621d 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -20,7 +20,7 @@ import snowglobe OPERATIONS = frozenset({ - "shale.access", + "shale.access", "shale.clone", "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", @@ -55,13 +55,15 @@ class PreviewHost(boundary.Host): actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") - if operation == "shale.access": + if operation in {"shale.access", "shale.clone"}: service = request.get("service", "") if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") + if operation == "shale.clone": + return import_module("shale-access").clone(request) return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") if operation in {"files.snapshot", "files.snapshots", "files.discard"}: datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py index b9d7b9d5c5f0adfa5135473b075c330c80791f6a..d32ca65892524367159722b3e8a2e8ce7b571b32 100644 --- a/tools/shale-access-test.py +++ b/tools/shale-access-test.py @@ -5,8 +5,12 @@ import importlib import json import os from pathlib import Path +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import sqlite3 +import subprocess +import sys import tempfile +import threading import unittest from unittest.mock import patch @@ -16,6 +20,41 @@ from argon2 import PasswordHasher shale = importlib.import_module('shale-access') +class CGI(BaseHTTPRequestHandler): + """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" + def __init__(self, test, *args): + self.test = test + super().__init__(*args) + + def log_message(self, *args): + pass + + def do_GET(self): + self.relay() + + def do_POST(self): + self.relay() + + def relay(self): + try: + plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) + except shale.Error: + self.send_response(401) + self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') + self.end_headers() + return + body = self.rfile.read(int(self.headers.get('content-length', 0))) + output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, + env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout + head, _, payload = output.partition(b'\r\n\r\n') + headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) + self.send_response(int(headers.pop('Status', '200').split()[0])) + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(payload) + + class Access(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -114,8 +153,9 @@ class Access(unittest.TestCase): self.assertTrue(self.access(path, 'snow')['deny']) def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): - path = '/the-knife/info/refs?service=git-upload-pack' + path = '/the-knife/info/refs?service=git-receive-pack' self.assertTrue(self.access(path, 'snow')['deny']) + self.assertTrue(self.access(path)['authenticate']) token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() self.assertTrue(self.access(path, authorization=token)['allowed']) self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) @@ -123,6 +163,40 @@ class Access(unittest.TestCase): db.execute('DELETE FROM personal_tokens') self.assertTrue(self.access(path, authorization=token)['deny']) + def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): + for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): + self.assertTrue(self.access(path)['clone'], path) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access(path, authorization=token)['allowed'], path) + for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): + self.assertNotIn('clone', self.access(path), path) + + def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): + with self.assertRaises(shale.Error): + self.clone('/the-knife.git/info/refs?service=git-upload-pack') + self.repo.lookup_reference('refs/heads/main').set_target(self.base) + self.repo.create_reference('refs/heads/merge', self.public) + server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) + self.addCleanup(server.server_close) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.shutdown) + url = f'http://127.0.0.1:{server.server_port}/the-knife' + environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} + for version in ('0', '1', '2'): + checkout = self.root / ('clone-v' + version) + subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) + refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout + self.assertEqual(refs, f'refs/heads/main {self.base}\n') + for revision in (str(self.native), str(self.public)): + fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) + self.assertNotEqual(fetched.returncode, 0, (version, revision)) + self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) + pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) + self.assertNotEqual(pushed.returncode, 0) + + def clone(self, uri, method='GET', content_type='', content_encoding=''): + return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) + def test_native_push_during_classification_cannot_change_the_checked_revision(self): newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) repo = self.repo diff --git a/tools/shale-access.py b/tools/shale-access.py index 6e8d57fbc8f45131b80ca9adca4a951a59bac106..c1d88779af251ce2bc2192789abfe257b9a5b803 100644 --- a/tools/shale-access.py +++ b/tools/shale-access.py @@ -8,6 +8,8 @@ import posixpath import re import sqlite3 import subprocess +import sys +import tempfile from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit import pygit2 @@ -36,16 +38,17 @@ def allowed_account(db, user_id): and profile.get('kind') != 'ai') -def access(request, accounts=None): - service = request['service'] +def instance(service): if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): raise Error(400, 'Choose a Shale instance.') if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): - root = Path(fixture_root) - else: - site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) - root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) - uri = urlsplit(request['uri']) + return Path(fixture_root) + site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) + return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) + + +def target(request_uri): + uri = urlsplit(request_uri) path = uri.path for _ in range(4): decoded = unquote(path, errors='strict') @@ -53,10 +56,39 @@ def access(request, accounts=None): break path = decoded path = posixpath.normpath('/' + path.lstrip('/')) - parts = path.strip('/').split('/') + return uri, path, path.strip('/').split('/') + + +def upload_pack(uri, path, parts): + """Returns the CGI path of an anonymous fetch request, the only Git route served without a token.""" + if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path: + return None + if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query: + return '/the-knife.git/' + '/'.join(parts[1:]) + return None + + +def repository(root, db): + row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() + return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None + + +def protected_commits(repo, native): + policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] + tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] + if native: + tips.append(native) + return {str(commit.id) for tip in tips for commit in repo.walk(tip)} + + +def access(request, accounts=None): + root = instance(request['service']) + uri, path, parts = target(request['uri']) if parts[0].lower() not in ('the-knife', 'the-knife.git'): return {'allowed': False, 'deny': True} git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') + if not request['authorization'] and upload_pack(uri, path, parts): + return {'allowed': False, 'deny': False, 'clone': True} with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) @@ -83,20 +115,14 @@ def access(request, accounts=None): allowed = len(users) == 1 and allowed_account(db, users[0][0]) if allowed: return {'allowed': True, 'deny': False} - row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() - if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): + location = repository(root, db) + if not location: return {'allowed': False, 'deny': True} - repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) + repo = pygit2.Repository(str(location)) try: - policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] head = repo.lookup_reference('HEAD').target references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} - protected = set() - tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] - if 'refs/heads/native' in references: - tips.append(references['refs/heads/native']) - for tip in tips: - protected.update(str(commit.id) for commit in repo.walk(tip)) + protected = protected_commits(repo, references.get('refs/heads/native')) result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} if parts[0] != 'the-knife' or '%' in path or '\\' in path: return result @@ -177,3 +203,48 @@ def access(request, accounts=None): return result finally: repo.free() + + +def clone(request): + root = instance(request['service']) + uri, path, parts = target(request['uri']) + location = upload_pack(uri, path, parts) + if not location: + raise Error(404, 'Clone The Knife over HTTPS.') + with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + knife = repository(root, db) + if not knife: + raise Error(404, 'The Knife is unavailable.') + repo = pygit2.Repository(str(knife)) + try: + main = repo.lookup_reference('refs/heads/main').resolve().target + native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None + protected = protected_commits(repo, native) + if any(str(commit.id) in protected for commit in repo.walk(main)): + raise Error(403, "The Knife's main branch includes native history.") + finally: + repo.free() + return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': { + 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query, + 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'], + }} + + +def serve(objects, main): + """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects.""" + repository = Path(tempfile.mkdtemp()) / 'the-knife.git' + (repository / 'objects/info').mkdir(parents=True) + (repository / 'refs/heads').mkdir(parents=True) + (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n') + (repository / 'HEAD').write_text('ref: refs/heads/main\n') + (repository / 'refs/heads/main').write_text(main + '\n') + (repository / 'objects/info/alternates').write_text(objects + '\n') + # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main. + environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)} + os.execvpe('git', ['git', 'http-backend'], environment | { + 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', + }) + + +if __name__ == '__main__': + serve(*sys.argv[1:])