authoravatarclover caruso<git@paperclover.net>2026-10-22 13:14:53-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logf4d8ce9b1e0defb90de5d60d85276bff2b7ff738
treee33312d42c4ac0fb5b005996a5462c700a2b961e
parentd7a4e059a91fedc10fe5726256c78d46f20499b5
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Release Shale repository handles after access checks

Close the libgit2 database on successful, rejected, and failed classifications so native pack mappings do not depend on Python collection. Keep disposable fixture databases and repositories closed as well. Assisted-by: gpt-6.1-sol

2 files changed, 103 insertions(+), 84 deletions(-)

tools/shale-access-test.py+21-5
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import base642import base64
3from contextlib import closing
3import importlib4import importlib
4import json5import json
5import os6import os
...@@ -26,6 +27,7 @@ class Access(unittest.TestCase):...@@ -26,6 +27,7 @@ class Access(unittest.TestCase):
26 (self.root / 'data').mkdir()27 (self.root / 'data').mkdir()
27 self.uuid = '0' * 2628 self.uuid = '0' * 26
28 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)29 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)
30 self.addCleanup(self.repo.free)
29 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})31 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})
30 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])32 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])
31 self.repo.create_reference('refs/heads/native', self.native)33 self.repo.create_reference('refs/heads/native', self.native)
...@@ -33,12 +35,12 @@ class Access(unittest.TestCase):...@@ -33,12 +35,12 @@ class Access(unittest.TestCase):
33 self.repo.create_reference('refs/heads/main', self.public)35 self.repo.create_reference('refs/heads/main', self.public)
34 self.repo.set_head('refs/heads/main')36 self.repo.set_head('refs/heads/main')
35 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))37 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))
36 with sqlite3.connect(self.root / 'accounts.sqlite') as db:38 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
37 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')39 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')
38 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):40 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):
39 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))41 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))
40 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))42 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))
41 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:43 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
42 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')44 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')
43 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))45 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))
44 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):46 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):
...@@ -56,6 +58,20 @@ class Access(unittest.TestCase):...@@ -56,6 +58,20 @@ class Access(unittest.TestCase):
56 def access(self, path, account='', authorization=''):58 def access(self, path, account='', authorization=''):
57 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})59 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})
5860
61 def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self):
62 released = []
63 class TrackedRepository(pygit2.Repository):
64 def free(self):
65 released.append(True)
66 super().free()
67 with patch.object(shale.pygit2, 'Repository', TrackedRepository):
68 self.assertFalse(self.access('/the-knife/')['deny'])
69 self.assertTrue(self.access('/the-knife/objects/x')['deny'])
70 with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')):
71 with self.assertRaises(ValueError):
72 self.access('/the-knife/')
73 self.assertEqual(len(released), 3)
74
59 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):75 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):
60 public = self.access('/the-knife/tree/main/')76 public = self.access('/the-knife/tree/main/')
61 self.assertFalse(public['deny'])77 self.assertFalse(public['deny'])
...@@ -93,7 +109,7 @@ class Access(unittest.TestCase):...@@ -93,7 +109,7 @@ class Access(unittest.TestCase):
93 self.assertTrue(self.access(path, name)['allowed'])109 self.assertTrue(self.access(path, name)['allowed'])
94 for name in ('stranger', 'disabled', 'ai', ''):110 for name in ('stranger', 'disabled', 'ai', ''):
95 self.assertTrue(self.access(path, name)['deny'])111 self.assertTrue(self.access(path, name)['deny'])
96 with sqlite3.connect(self.root / 'accounts.sqlite') as db:112 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
97 db.execute("DELETE FROM memberships WHERE user_id='snow'")113 db.execute("DELETE FROM memberships WHERE user_id='snow'")
98 self.assertTrue(self.access(path, 'snow')['deny'])114 self.assertTrue(self.access(path, 'snow')['deny'])
99115
...@@ -103,7 +119,7 @@ class Access(unittest.TestCase):...@@ -103,7 +119,7 @@ class Access(unittest.TestCase):
103 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()119 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
104 self.assertTrue(self.access(path, authorization=token)['allowed'])120 self.assertTrue(self.access(path, authorization=token)['allowed'])
105 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])121 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
106 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:122 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
107 db.execute('DELETE FROM personal_tokens')123 db.execute('DELETE FROM personal_tokens')
108 self.assertTrue(self.access(path, authorization=token)['deny'])124 self.assertTrue(self.access(path, authorization=token)['deny'])
109125
...@@ -125,7 +141,7 @@ class Access(unittest.TestCase):...@@ -125,7 +141,7 @@ class Access(unittest.TestCase):
125 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])141 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])
126142
127 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):143 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):
128 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:144 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
129 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")145 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")
130 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])146 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])
131 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()147 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
tools/shale-access.py+82-79
...@@ -87,90 +87,93 @@ def access(request, accounts=None):...@@ -87,90 +87,93 @@ def access(request, accounts=None):
87 if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]):87 if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]):
88 return {'allowed': False, 'deny': True}88 return {'allowed': False, 'deny': True}
89 repo = pygit2.Repository(str(root / 'repositories_owned' / row[0]))89 repo = pygit2.Repository(str(root / 'repositories_owned' / row[0]))
90 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']90 try:
91 head = repo.lookup_reference('HEAD').target91 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
92 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}92 head = repo.lookup_reference('HEAD').target
93 protected = set()93 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
94 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]94 protected = set()
95 if 'refs/heads/native' in references:95 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
96 tips.append(references['refs/heads/native'])96 if 'refs/heads/native' in references:
97 for tip in tips:97 tips.append(references['refs/heads/native'])
98 protected.update(str(commit.id) for commit in repo.walk(tip))98 for tip in tips:
99 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}99 protected.update(str(commit.id) for commit in repo.walk(tip))
100 if parts[0] != 'the-knife' or '%' in path or '\\' in path:100 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
101 return result101 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
102 route = parts[1] if len(parts) > 1 else ''
103 query = dict(parse_qsl(uri.query, keep_blank_values=True))
104 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
105 return result
106 if route in ('', 'issues'):
107 if route == '' and query:
108 return result102 return result
109 result['htmlOnly'] = route != 'issues'103 route = parts[1] if len(parts) > 1 else ''
110 result['deny'] = False104 query = dict(parse_qsl(uri.query, keep_blank_values=True))
111 return result105 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
112 if route in ('heads', 'tags') and len(parts) == 2 and not query:
113 result['deny'] = False
114 return result
115 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
116 return result
117 if route not in ('log', 'tree', 'plain', 'commit'):
118 return result
119 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
120 suffix = ''
121 if route == 'commit':
122 revision, dot, suffix = revision.partition('.')
123 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
124 return result106 return result
125 if not revision or revision == '-':107 if route in ('', 'issues'):
126 revision = 'HEAD'108 if route == '' and query:
127 # Only named refs and object IDs have the same meaning in Shale and libgit2.
128 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
129 return result
130 try:
131 if revision == 'HEAD':
132 oid = references.get(head) if isinstance(head, str) else head
133 else:
134 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
135 if len(candidates) > 1:
136 return result109 return result
137 if candidates:110 result['htmlOnly'] = route != 'issues'
138 oid = candidates.pop()111 result['deny'] = False
139 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):112 return result
140 oid = repo[revision].id113 if route in ('heads', 'tags') and len(parts) == 2 and not query:
114 result['deny'] = False
115 return result
116 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
117 return result
118 if route not in ('log', 'tree', 'plain', 'commit'):
119 return result
120 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
121 suffix = ''
122 if route == 'commit':
123 revision, dot, suffix = revision.partition('.')
124 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
125 return result
126 if not revision or revision == '-':
127 revision = 'HEAD'
128 # Only named refs and object IDs have the same meaning in Shale and libgit2.
129 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
130 return result
131 try:
132 if revision == 'HEAD':
133 oid = references.get(head) if isinstance(head, str) else head
141 else:134 else:
135 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
136 if len(candidates) > 1:
137 return result
138 if candidates:
139 oid = candidates.pop()
140 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):
141 oid = repo[revision].id
142 else:
143 return result
144 if oid is None:
142 return result145 return result
143 if oid is None:146 commit = repo[oid].peel(pygit2.Commit)
147 except (KeyError, ValueError, pygit2.GitError):
144 return result148 return result
145 commit = repo[oid].peel(pygit2.Commit)149 if str(commit.id) in protected:
146 except (KeyError, ValueError, pygit2.GitError):
147 return result
148 if str(commit.id) in protected:
149 return result
150 if route in ('tree', 'plain'):
151 result['htmlOnly'] = route != 'plain'
152 file = '/'.join(parts[3:])
153 if restricted_path(file):
154 return result150 return result
155 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):151 if route in ('tree', 'plain'):
156 return result152 result['htmlOnly'] = route != 'plain'
157 if route == 'commit':153 file = '/'.join(parts[3:])
158 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]154 if restricted_path(file):
159 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
160 if sensitive:
161 if suffix in ('stat', 'diffraw'):
162 return result155 return result
163 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"156 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):
164 if suffix:157 return result
165 result['rawDiff'] = True158 if route == 'commit':
166 result['deny'] = False159 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]
167 return result160 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
168 if route == 'log':161 if sensitive:
169 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})162 if suffix in ('stat', 'diffraw'):
170 else:163 return result
171 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]164 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"
172 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')165 if suffix:
173 if query:166 result['rawDiff'] = True
174 result['uri'] += '?' + urlencode(query)167 result['deny'] = False
175 result['deny'] = False168 return result
176 return result169 if route == 'log':
170 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})
171 else:
172 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]
173 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')
174 if query:
175 result['uri'] += '?' + urlencode(query)
176 result['deny'] = False
177 return result
178 finally:
179 repo.free()