| author | |
| committer | |
| log | f4d8ce9b1e0defb90de5d60d85276bff2b7ff738 |
| tree | e33312d42c4ac0fb5b005996a5462c700a2b961e |
| parent | d7a4e059a91fedc10fe5726256c78d46f20499b5 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Close the libgit2 database on successful, rejected, and failed classifications so native pack mappings do not depend on Python collection. Keep disposable fixture databases and repositories closed as well.
Assisted-by: gpt-6.1-sol2 files changed, 103 insertions(+), 84 deletions(-)
tools/shale-access-test.py+21-5| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | import base64 | 2 | import base64 |
| 3 | from contextlib import closing | ||
| 3 | import importlib | 4 | import importlib |
| 4 | import json | 5 | import json |
| 5 | import os | 6 | import os |
| ... | @@ -26,6 +27,7 @@ class Access(unittest.TestCase): | ... | @@ -26,6 +27,7 @@ class Access(unittest.TestCase): |
| 26 | (self.root / 'data').mkdir() | 27 | (self.root / 'data').mkdir() |
| 27 | self.uuid = '0' * 26 | 28 | self.uuid = '0' * 26 |
| 28 | self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) | 29 | self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) |
| 30 | self.addCleanup(self.repo.free) | ||
| 29 | self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) | 31 | self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) |
| 30 | self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) | 32 | self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) |
| 31 | self.repo.create_reference('refs/heads/native', self.native) | 33 | self.repo.create_reference('refs/heads/native', self.native) |
| ... | @@ -33,12 +35,12 @@ class Access(unittest.TestCase): | ... | @@ -33,12 +35,12 @@ class Access(unittest.TestCase): |
| 33 | self.repo.create_reference('refs/heads/main', self.public) | 35 | self.repo.create_reference('refs/heads/main', self.public) |
| 34 | self.repo.set_head('refs/heads/main') | 36 | self.repo.set_head('refs/heads/main') |
| 35 | (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) | 37 | (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) |
| 36 | with sqlite3.connect(self.root / 'accounts.sqlite') as db: | 38 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: |
| 37 | db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') | 39 | db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') |
| 38 | for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): | 40 | for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): |
| 39 | db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) | 41 | db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) |
| 40 | if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) | 42 | if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) |
| 41 | with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: | 43 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 42 | db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') | 44 | db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') |
| 43 | db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) | 45 | db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) |
| 44 | for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): | 46 | for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): |
| ... | @@ -56,6 +58,20 @@ class Access(unittest.TestCase): | ... | @@ -56,6 +58,20 @@ class Access(unittest.TestCase): |
| 56 | def access(self, path, account='', authorization=''): | 58 | def access(self, path, account='', authorization=''): |
| 57 | return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) | 59 | return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) |
| 58 | 60 | ||
| 61 | def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self): | ||
| 62 | released = [] | ||
| 63 | class TrackedRepository(pygit2.Repository): | ||
| 64 | def free(self): | ||
| 65 | released.append(True) | ||
| 66 | super().free() | ||
| 67 | with patch.object(shale.pygit2, 'Repository', TrackedRepository): | ||
| 68 | self.assertFalse(self.access('/the-knife/')['deny']) | ||
| 69 | self.assertTrue(self.access('/the-knife/objects/x')['deny']) | ||
| 70 | with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')): | ||
| 71 | with self.assertRaises(ValueError): | ||
| 72 | self.access('/the-knife/') | ||
| 73 | self.assertEqual(len(released), 3) | ||
| 74 | |||
| 59 | def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): | 75 | def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): |
| 60 | public = self.access('/the-knife/tree/main/') | 76 | public = self.access('/the-knife/tree/main/') |
| 61 | self.assertFalse(public['deny']) | 77 | self.assertFalse(public['deny']) |
| ... | @@ -93,7 +109,7 @@ class Access(unittest.TestCase): | ... | @@ -93,7 +109,7 @@ class Access(unittest.TestCase): |
| 93 | self.assertTrue(self.access(path, name)['allowed']) | 109 | self.assertTrue(self.access(path, name)['allowed']) |
| 94 | for name in ('stranger', 'disabled', 'ai', ''): | 110 | for name in ('stranger', 'disabled', 'ai', ''): |
| 95 | self.assertTrue(self.access(path, name)['deny']) | 111 | self.assertTrue(self.access(path, name)['deny']) |
| 96 | with sqlite3.connect(self.root / 'accounts.sqlite') as db: | 112 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: |
| 97 | db.execute("DELETE FROM memberships WHERE user_id='snow'") | 113 | db.execute("DELETE FROM memberships WHERE user_id='snow'") |
| 98 | self.assertTrue(self.access(path, 'snow')['deny']) | 114 | self.assertTrue(self.access(path, 'snow')['deny']) |
| 99 | 115 | ||
| ... | @@ -103,7 +119,7 @@ class Access(unittest.TestCase): | ... | @@ -103,7 +119,7 @@ class Access(unittest.TestCase): |
| 103 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | 119 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
| 104 | self.assertTrue(self.access(path, authorization=token)['allowed']) | 120 | self.assertTrue(self.access(path, authorization=token)['allowed']) |
| 105 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) | 121 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) |
| 106 | with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: | 122 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 107 | db.execute('DELETE FROM personal_tokens') | 123 | db.execute('DELETE FROM personal_tokens') |
| 108 | self.assertTrue(self.access(path, authorization=token)['deny']) | 124 | self.assertTrue(self.access(path, authorization=token)['deny']) |
| 109 | 125 | ||
| ... | @@ -125,7 +141,7 @@ class Access(unittest.TestCase): | ... | @@ -125,7 +141,7 @@ class Access(unittest.TestCase): |
| 125 | self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) | 141 | self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) |
| 126 | 142 | ||
| 127 | def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): | 143 | def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): |
| 128 | with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: | 144 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 129 | db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") | 145 | db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") |
| 130 | self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) | 146 | self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) |
| 131 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() | 147 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
tools/shale-access.py+82-79| ... | @@ -87,90 +87,93 @@ def access(request, accounts=None): | ... | @@ -87,90 +87,93 @@ def access(request, accounts=None): |
| 87 | if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): | 87 | if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): |
| 88 | return {'allowed': False, 'deny': True} | 88 | return {'allowed': False, 'deny': True} |
| 89 | repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) | 89 | repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) |
| 90 | policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] | 90 | try: |
| 91 | head = repo.lookup_reference('HEAD').target | 91 | policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] |
| 92 | references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} | 92 | head = repo.lookup_reference('HEAD').target |
| 93 | protected = set() | 93 | references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} |
| 94 | tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] | 94 | protected = set() |
| 95 | if 'refs/heads/native' in references: | 95 | tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] |
| 96 | tips.append(references['refs/heads/native']) | 96 | if 'refs/heads/native' in references: |
| 97 | for tip in tips: | 97 | tips.append(references['refs/heads/native']) |
| 98 | protected.update(str(commit.id) for commit in repo.walk(tip)) | 98 | for tip in tips: |
| 99 | result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} | 99 | protected.update(str(commit.id) for commit in repo.walk(tip)) |
| 100 | if parts[0] != 'the-knife' or '%' in path or '\\' in path: | 100 | result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} |
| 101 | return result | 101 | if parts[0] != 'the-knife' or '%' in path or '\\' in path: |
| 102 | route = parts[1] if len(parts) > 1 else '' | ||
| 103 | query = dict(parse_qsl(uri.query, keep_blank_values=True)) | ||
| 104 | if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): | ||
| 105 | return result | ||
| 106 | if route in ('', 'issues'): | ||
| 107 | if route == '' and query: | ||
| 108 | return result | 102 | return result |
| 109 | result['htmlOnly'] = route != 'issues' | 103 | route = parts[1] if len(parts) > 1 else '' |
| 110 | result['deny'] = False | 104 | query = dict(parse_qsl(uri.query, keep_blank_values=True)) |
| 111 | return result | 105 | if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): |
| 112 | if route in ('heads', 'tags') and len(parts) == 2 and not query: | ||
| 113 | result['deny'] = False | ||
| 114 | return result | ||
| 115 | if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): | ||
| 116 | return result | ||
| 117 | if route not in ('log', 'tree', 'plain', 'commit'): | ||
| 118 | return result | ||
| 119 | revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' | ||
| 120 | suffix = '' | ||
| 121 | if route == 'commit': | ||
| 122 | revision, dot, suffix = revision.partition('.') | ||
| 123 | if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): | ||
| 124 | return result | 106 | return result |
| 125 | if not revision or revision == '-': | 107 | if route in ('', 'issues'): |
| 126 | revision = 'HEAD' | 108 | if route == '' and query: |
| 127 | # Only named refs and object IDs have the same meaning in Shale and libgit2. | ||
| 128 | if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): | ||
| 129 | return result | ||
| 130 | try: | ||
| 131 | if revision == 'HEAD': | ||
| 132 | oid = references.get(head) if isinstance(head, str) else head | ||
| 133 | else: | ||
| 134 | candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} | ||
| 135 | if len(candidates) > 1: | ||
| 136 | return result | 109 | return result |
| 137 | if candidates: | 110 | result['htmlOnly'] = route != 'issues' |
| 138 | oid = candidates.pop() | 111 | result['deny'] = False |
| 139 | elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): | 112 | return result |
| 140 | oid = repo[revision].id | 113 | if route in ('heads', 'tags') and len(parts) == 2 and not query: |
| 114 | result['deny'] = False | ||
| 115 | return result | ||
| 116 | if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): | ||
| 117 | return result | ||
| 118 | if route not in ('log', 'tree', 'plain', 'commit'): | ||
| 119 | return result | ||
| 120 | revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' | ||
| 121 | suffix = '' | ||
| 122 | if route == 'commit': | ||
| 123 | revision, dot, suffix = revision.partition('.') | ||
| 124 | if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): | ||
| 125 | return result | ||
| 126 | if not revision or revision == '-': | ||
| 127 | revision = 'HEAD' | ||
| 128 | # Only named refs and object IDs have the same meaning in Shale and libgit2. | ||
| 129 | if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): | ||
| 130 | return result | ||
| 131 | try: | ||
| 132 | if revision == 'HEAD': | ||
| 133 | oid = references.get(head) if isinstance(head, str) else head | ||
| 141 | else: | 134 | else: |
| 135 | candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} | ||
| 136 | if len(candidates) > 1: | ||
| 137 | return result | ||
| 138 | if candidates: | ||
| 139 | oid = candidates.pop() | ||
| 140 | elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): | ||
| 141 | oid = repo[revision].id | ||
| 142 | else: | ||
| 143 | return result | ||
| 144 | if oid is None: | ||
| 142 | return result | 145 | return result |
| 143 | if oid is None: | 146 | commit = repo[oid].peel(pygit2.Commit) |
| 147 | except (KeyError, ValueError, pygit2.GitError): | ||
| 144 | return result | 148 | return result |
| 145 | commit = repo[oid].peel(pygit2.Commit) | 149 | if str(commit.id) in protected: |
| 146 | except (KeyError, ValueError, pygit2.GitError): | ||
| 147 | return result | ||
| 148 | if str(commit.id) in protected: | ||
| 149 | return result | ||
| 150 | if route in ('tree', 'plain'): | ||
| 151 | result['htmlOnly'] = route != 'plain' | ||
| 152 | file = '/'.join(parts[3:]) | ||
| 153 | if restricted_path(file): | ||
| 154 | return result | 150 | return result |
| 155 | if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): | 151 | if route in ('tree', 'plain'): |
| 156 | return result | 152 | result['htmlOnly'] = route != 'plain' |
| 157 | if route == 'commit': | 153 | file = '/'.join(parts[3:]) |
| 158 | diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] | 154 | if restricted_path(file): |
| 159 | sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) | ||
| 160 | if sensitive: | ||
| 161 | if suffix in ('stat', 'diffraw'): | ||
| 162 | return result | 155 | return result |
| 163 | result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" | 156 | if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): |
| 164 | if suffix: | 157 | return result |
| 165 | result['rawDiff'] = True | 158 | if route == 'commit': |
| 166 | result['deny'] = False | 159 | diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] |
| 167 | return result | 160 | sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) |
| 168 | if route == 'log': | 161 | if sensitive: |
| 169 | result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) | 162 | if suffix in ('stat', 'diffraw'): |
| 170 | else: | 163 | return result |
| 171 | parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] | 164 | result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" |
| 172 | result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') | 165 | if suffix: |
| 173 | if query: | 166 | result['rawDiff'] = True |
| 174 | result['uri'] += '?' + urlencode(query) | 167 | result['deny'] = False |
| 175 | result['deny'] = False | 168 | return result |
| 176 | return result | 169 | if route == 'log': |
| 170 | result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) | ||
| 171 | else: | ||
| 172 | parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] | ||
| 173 | result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') | ||
| 174 | if query: | ||
| 175 | result['uri'] += '?' + urlencode(query) | ||
| 176 | result['deny'] = False | ||
| 177 | return result | ||
| 178 | finally: | ||
| 179 | repo.free() |