authoravatarclover caruso<git@paperclover.net>2026-10-24 21:36:19-07:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log2b3ff5841b47959fe8b22c78617bd88558c0ad1a
tree5fa045bf5554ca2491e785803f78c8afedf2e456
parentca421c2098f96de8070bb7d17dd1d3aab3b47487
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Serve anonymous HTTPS clones of The Knife main branch

Anonymous upload-pack requests get a scratch repository whose only ref is main, borrowing the real objects through alternates and pinned to protocol v0, which only serves objects reachable from advertised refs. Clones are refused while main reaches native history. Token requests keep the full repository, so owners fetch with http.proactiveAuth=basic. Assisted-by: claude-opus-5.5

6 files changed, 246 insertions(+), 23 deletions(-)

dashboard/src/shale_page.rs+45
......@@ -412,6 +412,46 @@ fn redact_knife(html: &str, policy: &Value) -> Result<String> {
412412 )?)
413413}
414414
415/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch.
416async fn clone(request: Value, body: Body) -> Result<Response> {
417 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
418 let (_, stream) = host::open(request).await?;
419 let (reader, mut writer) = stream.into_split();
420 tokio::spawn(async move {
421 let mut body = body.into_data_stream();
422 while let Some(Ok(chunk)) = body.next().await {
423 if writer.write_all(&chunk).await.is_err() { return; }
424 }
425 let _ = writer.shutdown().await;
426 });
427 let mut reader = tokio::io::BufReader::new(reader);
428 let mut response = Response::builder();
429 loop {
430 let mut line = String::new();
431 if reader.read_line(&mut line).await? == 0 {
432 return Err(Error::new(502, "The Knife clone ended early."));
433 }
434 let line = line.trim_end();
435 if line.is_empty() { break; }
436 let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?;
437 let value = value.trim();
438 if name.eq_ignore_ascii_case("status") {
439 response = response.status(value.split_whitespace().next().unwrap_or_default().parse::<u16>()?);
440 } else {
441 response = response.header(name, value);
442 }
443 }
444 let stream = futures::stream::unfold(reader, |mut reader| async move {
445 let mut buffer = vec![0; 64 * 1024];
446 match reader.read(&mut buffer).await {
447 Ok(0) => None,
448 Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) }
449 Err(error) => Some((Err(error), reader)),
450 }
451 });
452 Ok(response.body(Body::from_stream(stream))?)
453}
454
415455pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
416456 let (mut parts, body) = request.into_parts();
417457 let host = parts
......@@ -475,6 +515,11 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> {
475515 Err(_) => return Ok(restricted(false)),
476516 };
477517 if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); }
518 if knife_policy["clone"] == true {
519 let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default();
520 let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")});
521 return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true)));
522 }
478523 if let Some(uri) = knife_policy["uri"].as_str() {
479524 target = match &app.internal {
480525 Some((base, _)) => format!("{base}services/{service}{uri}"),
nixos/host-python.nix+6-2
......@@ -1,2 +1,6 @@
1{ python3 }:
2python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ])
1{ python3, git }:
2python3.buildEnv.override {
3 extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ];
4 # shale-access serves anonymous Knife clones through git http-backend.
5 makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ];
6}
tools/dashboard-host.py+27
......@@ -16,6 +16,7 @@ import signal
1616import socket
1717import struct
1818import subprocess
19import sys
1920import threading
2021import time
2122import vms
......@@ -41,6 +42,7 @@ MAX_REQUEST = PROTOCOL['request']
4142MAX_RESPONSE = PROTOCOL['response']
4243STREAM_SLOTS = threading.BoundedSemaphore(4)
4344GUEST_SLOTS = threading.BoundedSemaphore(4)
45CLONE_SLOTS = threading.BoundedSemaphore(2)
4446VM_PREPARATION_SLOT = threading.BoundedSemaphore(1)
4547INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?"
4648
......@@ -93,6 +95,7 @@ class Host:
9395 "index.discard": {"dataset", "snapshot"},
9496 "shale.metadata": {"repository", "accounts"},
9597 "shale.access": {"service", "uri", "account", "authorization"},
98 "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"},
9699 "shale.write": {"grant", "repository", "action", "arguments"},
97100 "host.sample": set(),
98101 "host.usage": {"refresh"},
......@@ -109,6 +112,8 @@ class Host:
109112 return copyparty.handle(request)
110113 if operation == "shale.access":
111114 return importlib.import_module("shale-access").access(request)
115 if operation == "shale.clone":
116 return importlib.import_module("shale-access").clone(request)
112117 if operation == "shale.metadata":
113118 return shale_labels.metadata(request)
114119 if operation == "shale.write":
......@@ -463,6 +468,20 @@ def stream_upload(connection, request):
463468 Path(temporary).unlink(missing_ok=True)
464469
465470
471def stream_clone(connection, plan):
472 owner = os.stat(plan["objects"])
473 # Service identities have numeric owners without passwd entries, which systemd's User= rejects.
474 command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec",
475 *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes",
476 "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")),
477 *(f"--setenv={name}={value}" for name, value in plan["environment"].items()),
478 "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--",
479 sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]]
480 send_response(connection, {"value": None})
481 connection.setblocking(True)
482 subprocess.run(command, stdin=connection, stdout=connection)
483
484
466485def serve_connection(connection, host, allowed_uid, allowed_operations=None):
467486 with connection:
468487 connection.settimeout(65)
......@@ -501,6 +520,14 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
501520 return
502521 finally:
503522 streams.release()
523 if isinstance(request, dict) and request.get("operation") == "shale.clone":
524 if not CLONE_SLOTS.acquire(blocking=False):
525 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
526 try:
527 stream_clone(connection, host.handle(request))
528 return
529 finally:
530 CLONE_SLOTS.release()
504531 response = {"value": host.handle(request)}
505532 except Rejected as error:
506533 response = {"error": str(error), "status": 400}
tools/dashboard-stage.py+4-2
......@@ -20,7 +20,7 @@ import snowglobe
2020
2121
2222OPERATIONS = frozenset({
23 "shale.access",
23 "shale.access", "shale.clone",
2424 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",
2525 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",
2626 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",
......@@ -55,13 +55,15 @@ class PreviewHost(boundary.Host):
5555 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}
5656 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:
5757 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")
58 if operation == "shale.access":
58 if operation in {"shale.access", "shale.clone"}:
5959 service = request.get("service", "")
6060 if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
6161 raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.")
6262 linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text())
6363 if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name:
6464 raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.")
65 if operation == "shale.clone":
66 return import_module("shale-access").clone(request)
6567 return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data")
6668 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:
6769 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]
tools/shale-access-test.py+75-1
......@@ -5,8 +5,12 @@ import importlib
55import json
66import os
77from pathlib import Path
8from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
89import sqlite3
10import subprocess
11import sys
912import tempfile
13import threading
1014import unittest
1115from unittest.mock import patch
1216
......@@ -16,6 +20,41 @@ from argon2 import PasswordHasher
1620shale = importlib.import_module('shale-access')
1721
1822
23class CGI(BaseHTTPRequestHandler):
24 """Plays the Shale gateway: every request passes through clone() before reaching git http-backend."""
25 def __init__(self, test, *args):
26 self.test = test
27 super().__init__(*args)
28
29 def log_message(self, *args):
30 pass
31
32 def do_GET(self):
33 self.relay()
34
35 def do_POST(self):
36 self.relay()
37
38 def relay(self):
39 try:
40 plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', ''))
41 except shale.Error:
42 self.send_response(401)
43 self.send_header('WWW-Authenticate', 'Basic realm="The Knife"')
44 self.end_headers()
45 return
46 body = self.rfile.read(int(self.headers.get('content-length', 0)))
47 output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True,
48 env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout
49 head, _, payload = output.partition(b'\r\n\r\n')
50 headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n'))
51 self.send_response(int(headers.pop('Status', '200').split()[0]))
52 for name, value in headers.items():
53 self.send_header(name, value)
54 self.end_headers()
55 self.wfile.write(payload)
56
57
1958class Access(unittest.TestCase):
2059 def setUp(self):
2160 self.temp = tempfile.TemporaryDirectory()
......@@ -114,8 +153,9 @@ class Access(unittest.TestCase):
114153 self.assertTrue(self.access(path, 'snow')['deny'])
115154
116155 def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self):
117 path = '/the-knife/info/refs?service=git-upload-pack'
156 path = '/the-knife/info/refs?service=git-receive-pack'
118157 self.assertTrue(self.access(path, 'snow')['deny'])
158 self.assertTrue(self.access(path)['authenticate'])
119159 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
120160 self.assertTrue(self.access(path, authorization=token)['allowed'])
121161 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
......@@ -123,6 +163,40 @@ class Access(unittest.TestCase):
123163 db.execute('DELETE FROM personal_tokens')
124164 self.assertTrue(self.access(path, authorization=token)['deny'])
125165
166 def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self):
167 for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'):
168 self.assertTrue(self.access(path)['clone'], path)
169 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
170 self.assertTrue(self.access(path, authorization=token)['allowed'], path)
171 for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'):
172 self.assertNotIn('clone', self.access(path), path)
173
174 def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self):
175 with self.assertRaises(shale.Error):
176 self.clone('/the-knife.git/info/refs?service=git-upload-pack')
177 self.repo.lookup_reference('refs/heads/main').set_target(self.base)
178 self.repo.create_reference('refs/heads/merge', self.public)
179 server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args))
180 self.addCleanup(server.server_close)
181 threading.Thread(target=server.serve_forever, daemon=True).start()
182 self.addCleanup(server.shutdown)
183 url = f'http://127.0.0.1:{server.server_port}/the-knife'
184 environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'}
185 for version in ('0', '1', '2'):
186 checkout = self.root / ('clone-v' + version)
187 subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment)
188 refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout
189 self.assertEqual(refs, f'refs/heads/main {self.base}\n')
190 for revision in (str(self.native), str(self.public)):
191 fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment)
192 self.assertNotEqual(fetched.returncode, 0, (version, revision))
193 self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0)
194 pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment)
195 self.assertNotEqual(pushed.returncode, 0)
196
197 def clone(self, uri, method='GET', content_type='', content_encoding=''):
198 return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding})
199
126200 def test_native_push_during_classification_cannot_change_the_checked_revision(self):
127201 newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native])
128202 repo = self.repo
tools/shale-access.py+89-18
......@@ -8,6 +8,8 @@ import posixpath
88import re
99import sqlite3
1010import subprocess
11import sys
12import tempfile
1113from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
1214
1315import pygit2
......@@ -36,16 +38,17 @@ def allowed_account(db, user_id):
3638 and profile.get('kind') != 'ai')
3739
3840
39def access(request, accounts=None):
40 service = request['service']
41def instance(service):
4142 if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service):
4243 raise Error(400, 'Choose a Shale instance.')
4344 if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'):
44 root = Path(fixture_root)
45 else:
46 site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5))
47 root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service)
48 uri = urlsplit(request['uri'])
45 return Path(fixture_root)
46 site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5))
47 return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service)
48
49
50def target(request_uri):
51 uri = urlsplit(request_uri)
4952 path = uri.path
5053 for _ in range(4):
5154 decoded = unquote(path, errors='strict')
......@@ -53,10 +56,39 @@ def access(request, accounts=None):
5356 break
5457 path = decoded
5558 path = posixpath.normpath('/' + path.lstrip('/'))
56 parts = path.strip('/').split('/')
59 return uri, path, path.strip('/').split('/')
60
61
62def upload_pack(uri, path, parts):
63 """Returns the CGI path of an anonymous fetch request, the only Git route served without a token."""
64 if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path:
65 return None
66 if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query:
67 return '/the-knife.git/' + '/'.join(parts[1:])
68 return None
69
70
71def repository(root, db):
72 row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone()
73 return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None
74
75
76def protected_commits(repo, native):
77 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
78 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
79 if native:
80 tips.append(native)
81 return {str(commit.id) for tip in tips for commit in repo.walk(tip)}
82
83
84def access(request, accounts=None):
85 root = instance(request['service'])
86 uri, path, parts = target(request['uri'])
5787 if parts[0].lower() not in ('the-knife', 'the-knife.git'):
5888 return {'allowed': False, 'deny': True}
5989 git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD')
90 if not request['authorization'] and upload_pack(uri, path, parts):
91 return {'allowed': False, 'deny': False, 'clone': True}
6092 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
6193 accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard'))
6294 db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',))
......@@ -83,20 +115,14 @@ def access(request, accounts=None):
83115 allowed = len(users) == 1 and allowed_account(db, users[0][0])
84116 if allowed:
85117 return {'allowed': True, 'deny': False}
86 row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone()
87 if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]):
118 location = repository(root, db)
119 if not location:
88120 return {'allowed': False, 'deny': True}
89 repo = pygit2.Repository(str(root / 'repositories_owned' / row[0]))
121 repo = pygit2.Repository(str(location))
90122 try:
91 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
92123 head = repo.lookup_reference('HEAD').target
93124 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
94 protected = set()
95 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
96 if 'refs/heads/native' in references:
97 tips.append(references['refs/heads/native'])
98 for tip in tips:
99 protected.update(str(commit.id) for commit in repo.walk(tip))
125 protected = protected_commits(repo, references.get('refs/heads/native'))
100126 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
101127 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
102128 return result
......@@ -177,3 +203,48 @@ def access(request, accounts=None):
177203 return result
178204 finally:
179205 repo.free()
206
207
208def clone(request):
209 root = instance(request['service'])
210 uri, path, parts = target(request['uri'])
211 location = upload_pack(uri, path, parts)
212 if not location:
213 raise Error(404, 'Clone The Knife over HTTPS.')
214 with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db:
215 knife = repository(root, db)
216 if not knife:
217 raise Error(404, 'The Knife is unavailable.')
218 repo = pygit2.Repository(str(knife))
219 try:
220 main = repo.lookup_reference('refs/heads/main').resolve().target
221 native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None
222 protected = protected_commits(repo, native)
223 if any(str(commit.id) in protected for commit in repo.walk(main)):
224 raise Error(403, "The Knife's main branch includes native history.")
225 finally:
226 repo.free()
227 return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': {
228 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query,
229 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'],
230 }}
231
232
233def serve(objects, main):
234 """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects."""
235 repository = Path(tempfile.mkdtemp()) / 'the-knife.git'
236 (repository / 'objects/info').mkdir(parents=True)
237 (repository / 'refs/heads').mkdir(parents=True)
238 (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n')
239 (repository / 'HEAD').write_text('ref: refs/heads/main\n')
240 (repository / 'refs/heads/main').write_text(main + '\n')
241 (repository / 'objects/info/alternates').write_text(objects + '\n')
242 # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main.
243 environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)}
244 os.execvpe('git', ['git', 'http-backend'], environment | {
245 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null',
246 })
247
248
249if __name__ == '__main__':
250 serve(*sys.argv[1:])