| author | |
| committer | |
| log | 2b3ff5841b47959fe8b22c78617bd88558c0ad1a |
| tree | 5fa045bf5554ca2491e785803f78c8afedf2e456 |
| parent | ca421c2098f96de8070bb7d17dd1d3aab3b47487 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Anonymous upload-pack requests get a scratch repository whose only ref is
main, borrowing the real objects through alternates and pinned to protocol
v0, which only serves objects reachable from advertised refs. Clones are
refused while main reaches native history. Token requests keep the full
repository, so owners fetch with http.proactiveAuth=basic.
Assisted-by: claude-opus-5.56 files changed, 246 insertions(+), 23 deletions(-)
dashboard/src/shale_page.rs+45| ... | @@ -412,6 +412,46 @@ fn redact_knife(html: &str, policy: &Value) -> Result<String> { | ... | @@ -412,6 +412,46 @@ fn redact_knife(html: &str, policy: &Value) -> Result<String> { |
| 412 | )?) | 412 | )?) |
| 413 | } | 413 | } |
| 414 | 414 | ||
| 415 | /// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch. | ||
| 416 | async fn clone(request: Value, body: Body) -> Result<Response> { | ||
| 417 | use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; | ||
| 418 | let (_, stream) = host::open(request).await?; | ||
| 419 | let (reader, mut writer) = stream.into_split(); | ||
| 420 | tokio::spawn(async move { | ||
| 421 | let mut body = body.into_data_stream(); | ||
| 422 | while let Some(Ok(chunk)) = body.next().await { | ||
| 423 | if writer.write_all(&chunk).await.is_err() { return; } | ||
| 424 | } | ||
| 425 | let _ = writer.shutdown().await; | ||
| 426 | }); | ||
| 427 | let mut reader = tokio::io::BufReader::new(reader); | ||
| 428 | let mut response = Response::builder(); | ||
| 429 | loop { | ||
| 430 | let mut line = String::new(); | ||
| 431 | if reader.read_line(&mut line).await? == 0 { | ||
| 432 | return Err(Error::new(502, "The Knife clone ended early.")); | ||
| 433 | } | ||
| 434 | let line = line.trim_end(); | ||
| 435 | if line.is_empty() { break; } | ||
| 436 | let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?; | ||
| 437 | let value = value.trim(); | ||
| 438 | if name.eq_ignore_ascii_case("status") { | ||
| 439 | response = response.status(value.split_whitespace().next().unwrap_or_default().parse::<u16>()?); | ||
| 440 | } else { | ||
| 441 | response = response.header(name, value); | ||
| 442 | } | ||
| 443 | } | ||
| 444 | let stream = futures::stream::unfold(reader, |mut reader| async move { | ||
| 445 | let mut buffer = vec![0; 64 * 1024]; | ||
| 446 | match reader.read(&mut buffer).await { | ||
| 447 | Ok(0) => None, | ||
| 448 | Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) } | ||
| 449 | Err(error) => Some((Err(error), reader)), | ||
| 450 | } | ||
| 451 | }); | ||
| 452 | Ok(response.body(Body::from_stream(stream))?) | ||
| 453 | } | ||
| 454 | |||
| 415 | pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | 455 | pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 416 | let (mut parts, body) = request.into_parts(); | 456 | let (mut parts, body) = request.into_parts(); |
| 417 | let host = parts | 457 | let host = parts |
| ... | @@ -475,6 +515,11 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { | ... | @@ -475,6 +515,11 @@ pub async fn proxy(app: Arc<App>, request: Request) -> Result<Response> { |
| 475 | Err(_) => return Ok(restricted(false)), | 515 | Err(_) => return Ok(restricted(false)), |
| 476 | }; | 516 | }; |
| 477 | if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } | 517 | if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } |
| 518 | if knife_policy["clone"] == true { | ||
| 519 | let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default(); | ||
| 520 | let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")}); | ||
| 521 | return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true))); | ||
| 522 | } | ||
| 478 | if let Some(uri) = knife_policy["uri"].as_str() { | 523 | if let Some(uri) = knife_policy["uri"].as_str() { |
| 479 | target = match &app.internal { | 524 | target = match &app.internal { |
| 480 | Some((base, _)) => format!("{base}services/{service}{uri}"), | 525 | Some((base, _)) => format!("{base}services/{service}{uri}"), |
nixos/host-python.nix+6-2| ... | @@ -1,2 +1,6 @@ | ... | @@ -1,2 +1,6 @@ |
| 1 | { python3 }: | 1 | { python3, git }: |
| 2 | python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ]) | 2 | python3.buildEnv.override { |
| 3 | extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ]; | ||
| 4 | # shale-access serves anonymous Knife clones through git http-backend. | ||
| 5 | makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ]; | ||
| 6 | } |
tools/dashboard-host.py+27| ... | @@ -16,6 +16,7 @@ import signal | ... | @@ -16,6 +16,7 @@ import signal |
| 16 | import socket | 16 | import socket |
| 17 | import struct | 17 | import struct |
| 18 | import subprocess | 18 | import subprocess |
| 19 | import sys | ||
| 19 | import threading | 20 | import threading |
| 20 | import time | 21 | import time |
| 21 | import vms | 22 | import vms |
| ... | @@ -41,6 +42,7 @@ MAX_REQUEST = PROTOCOL['request'] | ... | @@ -41,6 +42,7 @@ MAX_REQUEST = PROTOCOL['request'] |
| 41 | MAX_RESPONSE = PROTOCOL['response'] | 42 | MAX_RESPONSE = PROTOCOL['response'] |
| 42 | STREAM_SLOTS = threading.BoundedSemaphore(4) | 43 | STREAM_SLOTS = threading.BoundedSemaphore(4) |
| 43 | GUEST_SLOTS = threading.BoundedSemaphore(4) | 44 | GUEST_SLOTS = threading.BoundedSemaphore(4) |
| 45 | CLONE_SLOTS = threading.BoundedSemaphore(2) | ||
| 44 | VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) | 46 | VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) |
| 45 | INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" | 47 | INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" |
| 46 | 48 | ||
| ... | @@ -93,6 +95,7 @@ class Host: | ... | @@ -93,6 +95,7 @@ class Host: |
| 93 | "index.discard": {"dataset", "snapshot"}, | 95 | "index.discard": {"dataset", "snapshot"}, |
| 94 | "shale.metadata": {"repository", "accounts"}, | 96 | "shale.metadata": {"repository", "accounts"}, |
| 95 | "shale.access": {"service", "uri", "account", "authorization"}, | 97 | "shale.access": {"service", "uri", "account", "authorization"}, |
| 98 | "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, | ||
| 96 | "shale.write": {"grant", "repository", "action", "arguments"}, | 99 | "shale.write": {"grant", "repository", "action", "arguments"}, |
| 97 | "host.sample": set(), | 100 | "host.sample": set(), |
| 98 | "host.usage": {"refresh"}, | 101 | "host.usage": {"refresh"}, |
| ... | @@ -109,6 +112,8 @@ class Host: | ... | @@ -109,6 +112,8 @@ class Host: |
| 109 | return copyparty.handle(request) | 112 | return copyparty.handle(request) |
| 110 | if operation == "shale.access": | 113 | if operation == "shale.access": |
| 111 | return importlib.import_module("shale-access").access(request) | 114 | return importlib.import_module("shale-access").access(request) |
| 115 | if operation == "shale.clone": | ||
| 116 | return importlib.import_module("shale-access").clone(request) | ||
| 112 | if operation == "shale.metadata": | 117 | if operation == "shale.metadata": |
| 113 | return shale_labels.metadata(request) | 118 | return shale_labels.metadata(request) |
| 114 | if operation == "shale.write": | 119 | if operation == "shale.write": |
| ... | @@ -463,6 +468,20 @@ def stream_upload(connection, request): | ... | @@ -463,6 +468,20 @@ def stream_upload(connection, request): |
| 463 | Path(temporary).unlink(missing_ok=True) | 468 | Path(temporary).unlink(missing_ok=True) |
| 464 | 469 | ||
| 465 | 470 | ||
| 471 | def stream_clone(connection, plan): | ||
| 472 | owner = os.stat(plan["objects"]) | ||
| 473 | # Service identities have numeric owners without passwd entries, which systemd's User= rejects. | ||
| 474 | command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", | ||
| 475 | *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", | ||
| 476 | "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), | ||
| 477 | *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), | ||
| 478 | "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", | ||
| 479 | sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] | ||
| 480 | send_response(connection, {"value": None}) | ||
| 481 | connection.setblocking(True) | ||
| 482 | subprocess.run(command, stdin=connection, stdout=connection) | ||
| 483 | |||
| 484 | |||
| 466 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): | 485 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 467 | with connection: | 486 | with connection: |
| 468 | connection.settimeout(65) | 487 | connection.settimeout(65) |
| ... | @@ -501,6 +520,14 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): | ... | @@ -501,6 +520,14 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 501 | return | 520 | return |
| 502 | finally: | 521 | finally: |
| 503 | streams.release() | 522 | streams.release() |
| 523 | if isinstance(request, dict) and request.get("operation") == "shale.clone": | ||
| 524 | if not CLONE_SLOTS.acquire(blocking=False): | ||
| 525 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") | ||
| 526 | try: | ||
| 527 | stream_clone(connection, host.handle(request)) | ||
| 528 | return | ||
| 529 | finally: | ||
| 530 | CLONE_SLOTS.release() | ||
| 504 | response = {"value": host.handle(request)} | 531 | response = {"value": host.handle(request)} |
| 505 | except Rejected as error: | 532 | except Rejected as error: |
| 506 | response = {"error": str(error), "status": 400} | 533 | response = {"error": str(error), "status": 400} |
tools/dashboard-stage.py+4-2| ... | @@ -20,7 +20,7 @@ import snowglobe | ... | @@ -20,7 +20,7 @@ import snowglobe |
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | OPERATIONS = frozenset({ | 22 | OPERATIONS = frozenset({ |
| 23 | "shale.access", | 23 | "shale.access", "shale.clone", |
| 24 | "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", | 24 | "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", |
| 25 | "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", | 25 | "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", |
| 26 | "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", | 26 | "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", |
| ... | @@ -55,13 +55,15 @@ class PreviewHost(boundary.Host): | ... | @@ -55,13 +55,15 @@ class PreviewHost(boundary.Host): |
| 55 | actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} | 55 | actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} |
| 56 | if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: | 56 | if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: |
| 57 | raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") | 57 | raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") |
| 58 | if operation == "shale.access": | 58 | if operation in {"shale.access", "shale.clone"}: |
| 59 | service = request.get("service", "") | 59 | service = request.get("service", "") |
| 60 | if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): | 60 | if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): |
| 61 | raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") | 61 | raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") |
| 62 | linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) | 62 | linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) |
| 63 | if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: | 63 | if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: |
| 64 | raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") | 64 | raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") |
| 65 | if operation == "shale.clone": | ||
| 66 | return import_module("shale-access").clone(request) | ||
| 65 | return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") | 67 | return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") |
| 66 | if operation in {"files.snapshot", "files.snapshots", "files.discard"}: | 68 | if operation in {"files.snapshot", "files.snapshots", "files.discard"}: |
| 67 | datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] | 69 | datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] |
tools/shale-access-test.py+75-1| ... | @@ -5,8 +5,12 @@ import importlib | ... | @@ -5,8 +5,12 @@ import importlib |
| 5 | import json | 5 | import json |
| 6 | import os | 6 | import os |
| 7 | from pathlib import Path | 7 | from pathlib import Path |
| 8 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ||
| 8 | import sqlite3 | 9 | import sqlite3 |
| 10 | import subprocess | ||
| 11 | import sys | ||
| 9 | import tempfile | 12 | import tempfile |
| 13 | import threading | ||
| 10 | import unittest | 14 | import unittest |
| 11 | from unittest.mock import patch | 15 | from unittest.mock import patch |
| 12 | 16 | ||
| ... | @@ -16,6 +20,41 @@ from argon2 import PasswordHasher | ... | @@ -16,6 +20,41 @@ from argon2 import PasswordHasher |
| 16 | shale = importlib.import_module('shale-access') | 20 | shale = importlib.import_module('shale-access') |
| 17 | 21 | ||
| 18 | 22 | ||
| 23 | class CGI(BaseHTTPRequestHandler): | ||
| 24 | """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" | ||
| 25 | def __init__(self, test, *args): | ||
| 26 | self.test = test | ||
| 27 | super().__init__(*args) | ||
| 28 | |||
| 29 | def log_message(self, *args): | ||
| 30 | pass | ||
| 31 | |||
| 32 | def do_GET(self): | ||
| 33 | self.relay() | ||
| 34 | |||
| 35 | def do_POST(self): | ||
| 36 | self.relay() | ||
| 37 | |||
| 38 | def relay(self): | ||
| 39 | try: | ||
| 40 | plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) | ||
| 41 | except shale.Error: | ||
| 42 | self.send_response(401) | ||
| 43 | self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') | ||
| 44 | self.end_headers() | ||
| 45 | return | ||
| 46 | body = self.rfile.read(int(self.headers.get('content-length', 0))) | ||
| 47 | output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, | ||
| 48 | env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout | ||
| 49 | head, _, payload = output.partition(b'\r\n\r\n') | ||
| 50 | headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) | ||
| 51 | self.send_response(int(headers.pop('Status', '200').split()[0])) | ||
| 52 | for name, value in headers.items(): | ||
| 53 | self.send_header(name, value) | ||
| 54 | self.end_headers() | ||
| 55 | self.wfile.write(payload) | ||
| 56 | |||
| 57 | |||
| 19 | class Access(unittest.TestCase): | 58 | class Access(unittest.TestCase): |
| 20 | def setUp(self): | 59 | def setUp(self): |
| 21 | self.temp = tempfile.TemporaryDirectory() | 60 | self.temp = tempfile.TemporaryDirectory() |
| ... | @@ -114,8 +153,9 @@ class Access(unittest.TestCase): | ... | @@ -114,8 +153,9 @@ class Access(unittest.TestCase): |
| 114 | self.assertTrue(self.access(path, 'snow')['deny']) | 153 | self.assertTrue(self.access(path, 'snow')['deny']) |
| 115 | 154 | ||
| 116 | def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): | 155 | def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): |
| 117 | path = '/the-knife/info/refs?service=git-upload-pack' | 156 | path = '/the-knife/info/refs?service=git-receive-pack' |
| 118 | self.assertTrue(self.access(path, 'snow')['deny']) | 157 | self.assertTrue(self.access(path, 'snow')['deny']) |
| 158 | self.assertTrue(self.access(path)['authenticate']) | ||
| 119 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | 159 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
| 120 | self.assertTrue(self.access(path, authorization=token)['allowed']) | 160 | self.assertTrue(self.access(path, authorization=token)['allowed']) |
| 121 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) | 161 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) |
| ... | @@ -123,6 +163,40 @@ class Access(unittest.TestCase): | ... | @@ -123,6 +163,40 @@ class Access(unittest.TestCase): |
| 123 | db.execute('DELETE FROM personal_tokens') | 163 | db.execute('DELETE FROM personal_tokens') |
| 124 | self.assertTrue(self.access(path, authorization=token)['deny']) | 164 | self.assertTrue(self.access(path, authorization=token)['deny']) |
| 125 | 165 | ||
| 166 | def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): | ||
| 167 | for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): | ||
| 168 | self.assertTrue(self.access(path)['clone'], path) | ||
| 169 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | ||
| 170 | self.assertTrue(self.access(path, authorization=token)['allowed'], path) | ||
| 171 | for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): | ||
| 172 | self.assertNotIn('clone', self.access(path), path) | ||
| 173 | |||
| 174 | def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): | ||
| 175 | with self.assertRaises(shale.Error): | ||
| 176 | self.clone('/the-knife.git/info/refs?service=git-upload-pack') | ||
| 177 | self.repo.lookup_reference('refs/heads/main').set_target(self.base) | ||
| 178 | self.repo.create_reference('refs/heads/merge', self.public) | ||
| 179 | server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) | ||
| 180 | self.addCleanup(server.server_close) | ||
| 181 | threading.Thread(target=server.serve_forever, daemon=True).start() | ||
| 182 | self.addCleanup(server.shutdown) | ||
| 183 | url = f'http://127.0.0.1:{server.server_port}/the-knife' | ||
| 184 | environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} | ||
| 185 | for version in ('0', '1', '2'): | ||
| 186 | checkout = self.root / ('clone-v' + version) | ||
| 187 | subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) | ||
| 188 | refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout | ||
| 189 | self.assertEqual(refs, f'refs/heads/main {self.base}\n') | ||
| 190 | for revision in (str(self.native), str(self.public)): | ||
| 191 | fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) | ||
| 192 | self.assertNotEqual(fetched.returncode, 0, (version, revision)) | ||
| 193 | self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) | ||
| 194 | pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) | ||
| 195 | self.assertNotEqual(pushed.returncode, 0) | ||
| 196 | |||
| 197 | def clone(self, uri, method='GET', content_type='', content_encoding=''): | ||
| 198 | return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) | ||
| 199 | |||
| 126 | def test_native_push_during_classification_cannot_change_the_checked_revision(self): | 200 | def test_native_push_during_classification_cannot_change_the_checked_revision(self): |
| 127 | newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) | 201 | newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) |
| 128 | repo = self.repo | 202 | repo = self.repo |
tools/shale-access.py+89-18| ... | @@ -8,6 +8,8 @@ import posixpath | ... | @@ -8,6 +8,8 @@ import posixpath |
| 8 | import re | 8 | import re |
| 9 | import sqlite3 | 9 | import sqlite3 |
| 10 | import subprocess | 10 | import subprocess |
| 11 | import sys | ||
| 12 | import tempfile | ||
| 11 | from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit | 13 | from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit |
| 12 | 14 | ||
| 13 | import pygit2 | 15 | import pygit2 |
| ... | @@ -36,16 +38,17 @@ def allowed_account(db, user_id): | ... | @@ -36,16 +38,17 @@ def allowed_account(db, user_id): |
| 36 | and profile.get('kind') != 'ai') | 38 | and profile.get('kind') != 'ai') |
| 37 | 39 | ||
| 38 | 40 | ||
| 39 | def access(request, accounts=None): | 41 | def instance(service): |
| 40 | service = request['service'] | ||
| 41 | if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): | 42 | if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): |
| 42 | raise Error(400, 'Choose a Shale instance.') | 43 | raise Error(400, 'Choose a Shale instance.') |
| 43 | if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): | 44 | if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): |
| 44 | root = Path(fixture_root) | 45 | return Path(fixture_root) |
| 45 | else: | 46 | site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) |
| 46 | site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) | 47 | return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) |
| 47 | root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) | 48 | |
| 48 | uri = urlsplit(request['uri']) | 49 | |
| 50 | def target(request_uri): | ||
| 51 | uri = urlsplit(request_uri) | ||
| 49 | path = uri.path | 52 | path = uri.path |
| 50 | for _ in range(4): | 53 | for _ in range(4): |
| 51 | decoded = unquote(path, errors='strict') | 54 | decoded = unquote(path, errors='strict') |
| ... | @@ -53,10 +56,39 @@ def access(request, accounts=None): | ... | @@ -53,10 +56,39 @@ def access(request, accounts=None): |
| 53 | break | 56 | break |
| 54 | path = decoded | 57 | path = decoded |
| 55 | path = posixpath.normpath('/' + path.lstrip('/')) | 58 | path = posixpath.normpath('/' + path.lstrip('/')) |
| 56 | parts = path.strip('/').split('/') | 59 | return uri, path, path.strip('/').split('/') |
| 60 | |||
| 61 | |||
| 62 | def upload_pack(uri, path, parts): | ||
| 63 | """Returns the CGI path of an anonymous fetch request, the only Git route served without a token.""" | ||
| 64 | if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path: | ||
| 65 | return None | ||
| 66 | if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query: | ||
| 67 | return '/the-knife.git/' + '/'.join(parts[1:]) | ||
| 68 | return None | ||
| 69 | |||
| 70 | |||
| 71 | def repository(root, db): | ||
| 72 | row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() | ||
| 73 | return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None | ||
| 74 | |||
| 75 | |||
| 76 | def protected_commits(repo, native): | ||
| 77 | policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] | ||
| 78 | tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] | ||
| 79 | if native: | ||
| 80 | tips.append(native) | ||
| 81 | return {str(commit.id) for tip in tips for commit in repo.walk(tip)} | ||
| 82 | |||
| 83 | |||
| 84 | def access(request, accounts=None): | ||
| 85 | root = instance(request['service']) | ||
| 86 | uri, path, parts = target(request['uri']) | ||
| 57 | if parts[0].lower() not in ('the-knife', 'the-knife.git'): | 87 | if parts[0].lower() not in ('the-knife', 'the-knife.git'): |
| 58 | return {'allowed': False, 'deny': True} | 88 | return {'allowed': False, 'deny': True} |
| 59 | git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') | 89 | git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') |
| 90 | if not request['authorization'] and upload_pack(uri, path, parts): | ||
| 91 | return {'allowed': False, 'deny': False, 'clone': True} | ||
| 60 | with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: | 92 | with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: |
| 61 | accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) | 93 | accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) |
| 62 | db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) | 94 | db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) |
| ... | @@ -83,20 +115,14 @@ def access(request, accounts=None): | ... | @@ -83,20 +115,14 @@ def access(request, accounts=None): |
| 83 | allowed = len(users) == 1 and allowed_account(db, users[0][0]) | 115 | allowed = len(users) == 1 and allowed_account(db, users[0][0]) |
| 84 | if allowed: | 116 | if allowed: |
| 85 | return {'allowed': True, 'deny': False} | 117 | return {'allowed': True, 'deny': False} |
| 86 | row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() | 118 | location = repository(root, db) |
| 87 | if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): | 119 | if not location: |
| 88 | return {'allowed': False, 'deny': True} | 120 | return {'allowed': False, 'deny': True} |
| 89 | repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) | 121 | repo = pygit2.Repository(str(location)) |
| 90 | try: | 122 | try: |
| 91 | policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] | ||
| 92 | head = repo.lookup_reference('HEAD').target | 123 | head = repo.lookup_reference('HEAD').target |
| 93 | references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} | 124 | references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} |
| 94 | protected = set() | 125 | protected = protected_commits(repo, references.get('refs/heads/native')) |
| 95 | tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] | ||
| 96 | if 'refs/heads/native' in references: | ||
| 97 | tips.append(references['refs/heads/native']) | ||
| 98 | for tip in tips: | ||
| 99 | protected.update(str(commit.id) for commit in repo.walk(tip)) | ||
| 100 | result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} | 126 | result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} |
| 101 | if parts[0] != 'the-knife' or '%' in path or '\\' in path: | 127 | if parts[0] != 'the-knife' or '%' in path or '\\' in path: |
| 102 | return result | 128 | return result |
| ... | @@ -177,3 +203,48 @@ def access(request, accounts=None): | ... | @@ -177,3 +203,48 @@ def access(request, accounts=None): |
| 177 | return result | 203 | return result |
| 178 | finally: | 204 | finally: |
| 179 | repo.free() | 205 | repo.free() |
| 206 | |||
| 207 | |||
| 208 | def clone(request): | ||
| 209 | root = instance(request['service']) | ||
| 210 | uri, path, parts = target(request['uri']) | ||
| 211 | location = upload_pack(uri, path, parts) | ||
| 212 | if not location: | ||
| 213 | raise Error(404, 'Clone The Knife over HTTPS.') | ||
| 214 | with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: | ||
| 215 | knife = repository(root, db) | ||
| 216 | if not knife: | ||
| 217 | raise Error(404, 'The Knife is unavailable.') | ||
| 218 | repo = pygit2.Repository(str(knife)) | ||
| 219 | try: | ||
| 220 | main = repo.lookup_reference('refs/heads/main').resolve().target | ||
| 221 | native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None | ||
| 222 | protected = protected_commits(repo, native) | ||
| 223 | if any(str(commit.id) in protected for commit in repo.walk(main)): | ||
| 224 | raise Error(403, "The Knife's main branch includes native history.") | ||
| 225 | finally: | ||
| 226 | repo.free() | ||
| 227 | return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': { | ||
| 228 | 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query, | ||
| 229 | 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'], | ||
| 230 | }} | ||
| 231 | |||
| 232 | |||
| 233 | def serve(objects, main): | ||
| 234 | """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects.""" | ||
| 235 | repository = Path(tempfile.mkdtemp()) / 'the-knife.git' | ||
| 236 | (repository / 'objects/info').mkdir(parents=True) | ||
| 237 | (repository / 'refs/heads').mkdir(parents=True) | ||
| 238 | (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n') | ||
| 239 | (repository / 'HEAD').write_text('ref: refs/heads/main\n') | ||
| 240 | (repository / 'refs/heads/main').write_text(main + '\n') | ||
| 241 | (repository / 'objects/info/alternates').write_text(objects + '\n') | ||
| 242 | # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main. | ||
| 243 | environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)} | ||
| 244 | os.execvpe('git', ['git', 'http-backend'], environment | { | ||
| 245 | 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', | ||
| 246 | }) | ||
| 247 | |||
| 248 | |||
| 249 | if __name__ == '__main__': | ||
| 250 | serve(*sys.argv[1:]) |