| ... | ... | @@ -0,0 +1,437 @@ |
| 1 | #!/usr/bin/env python3 |
| 2 | """Import a verified personal Forgejo export into stopped Shale, without replacing native issues.""" |
| 3 | import argparse |
| 4 | from collections import defaultdict |
| 5 | from datetime import datetime, timezone |
| 6 | import hashlib |
| 7 | import grp |
| 8 | import json |
| 9 | import os |
| 10 | from pathlib import Path |
| 11 | import re |
| 12 | import shutil |
| 13 | import sqlite3 |
| 14 | import sys |
| 15 | import tempfile |
| 16 | import urllib.request |
| 17 | from urllib.parse import quote |
| 18 | |
| 19 | ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' |
| 20 | EPOCH = 1577836800 # Shale's ULIDs use 2020-01-01, rather than the Unix epoch. |
| 21 | CLOSED = ('done', 'not_planned', 'duplicate', 'invalid') |
| 22 | EVENTS = ['comment', 'reopened', 'closed', 'issue reference', 'commit reference', |
| 23 | 'comment reference', 'pull request reference', 'label changed', 'milestone changed', |
| 24 | 'assignee changed', 'title changed', 'branch deleted', 'time tracking started', |
| 25 | 'time tracking stopped', 'time added', 'time tracking canceled', 'deadline added', |
| 26 | 'deadline changed', 'deadline removed', 'dependency added', 'dependency removed', |
| 27 | 'code comment', 'review', 'locked', 'unlocked', 'target branch changed', |
| 28 | 'time deleted', 'review requested', 'merged', 'pull request updated', |
| 29 | 'project changed', 'project column changed', 'review dismissed', 'reference changed', |
| 30 | 'automatic merge scheduled', 'automatic merge canceled', 'pinned', 'unpinned'] |
| 31 | |
| 32 | |
| 33 | def timestamp(epoch): |
| 34 | return datetime.fromtimestamp(int(epoch), timezone.utc).isoformat(timespec='seconds') |
| 35 | |
| 36 | |
| 37 | def identifier(kind, key, epoch): |
| 38 | milliseconds = (int(epoch) - EPOCH) * 1000 |
| 39 | if not 0 <= milliseconds < 2 ** 48: |
| 40 | raise ValueError('Source creation time is outside Shale ULID range') |
| 41 | entropy = int.from_bytes(hashlib.sha256(f'personal-forgejo:{kind}:{key}'.encode()).digest()[:10], 'big') |
| 42 | value = (milliseconds << 80) | entropy |
| 43 | return ''.join(ALPHABET[(value >> (5 * i)) & 31] for i in range(25, -1, -1)) |
| 44 | |
| 45 | |
| 46 | def digest(path): |
| 47 | return hashlib.sha256(path.read_bytes()).hexdigest() |
| 48 | |
| 49 | |
| 50 | def stopped(job): |
| 51 | token = os.environ.get('NOMAD_TOKEN') or Path('/var/lib/studio/nomad.token').read_text().strip() |
| 52 | def read(path): |
| 53 | request = urllib.request.Request('http://127.0.0.1:4646/v1/' + path, |
| 54 | headers={'X-Nomad-Token': token}) |
| 55 | with urllib.request.urlopen(request, timeout=10) as response: |
| 56 | return json.load(response) |
| 57 | if not read('job/' + job).get('Stop') or any( |
| 58 | item['ClientStatus'] in ('pending', 'running') for item in read('job/' + job + '/allocations')): |
| 59 | raise ValueError('Stop the target Shale job and wait for its allocations before importing') |
| 60 | |
| 61 | |
| 62 | def migrate(database, source, attachments, target, evidence, before_commit=lambda: None): |
| 63 | db = sqlite3.connect(database) |
| 64 | db.row_factory = sqlite3.Row |
| 65 | db.execute('PRAGMA foreign_keys=ON') |
| 66 | if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': |
| 67 | raise ValueError('Shale SQLite integrity check failed') |
| 68 | source_hash = digest(source) |
| 69 | data = json.loads(source.read_text()) |
| 70 | for field in ['repositories', 'issues', 'comments', 'labels', 'issue_labels', 'authors', |
| 71 | 'attachments', 'assignees', 'milestones', 'pull_requests', 'history']: |
| 72 | if not isinstance(data.get(field), list): |
| 73 | raise ValueError(f'Missing source table: {field}') |
| 74 | owners = db.execute("SELECT * FROM users WHERE name='clover'").fetchall() |
| 75 | if len(owners) != 1 or owners[0]['snowflake'] != '8c909fab-98b0-4472-9171-9606ccebe9d5': |
| 76 | raise ValueError('Clover original identity is missing') |
| 77 | owner = owners[0]['id'] |
| 78 | repos = {r['name']: dict(r) for r in db.execute('SELECT * FROM repositories')} |
| 79 | sources = {r['id']: r for r in data['repositories']} |
| 80 | issue_sources = {r['id']: r for r in data['issues']} |
| 81 | if len(issue_sources) != len(data['issues']): |
| 82 | raise ValueError('Duplicate source issue IDs') |
| 83 | for r in data['repositories']: |
| 84 | if r['name'] not in repos or repos[r['name']]['owner'] != owner: |
| 85 | raise ValueError(f'Expected imported Clover repository: {r["name"]}') |
| 86 | if any(c['issue_id'] not in issue_sources for c in data['comments']): |
| 87 | raise ValueError('Orphan source comment') |
| 88 | with sqlite3.connect(evidence / 'before.db') as backup: |
| 89 | db.backup(backup) |
| 90 | originals = {table: [dict(r) for r in db.execute(f'SELECT * FROM {table}')] |
| 91 | for table in ['issues', 'issue_actions', 'issue_labels', 'issues__labels', 'users']} |
| 92 | db.execute('BEGIN IMMEDIATE') |
| 93 | db.execute('CREATE TABLE IF NOT EXISTS studio_forgejo_records(' |
| 94 | 'kind TEXT NOT NULL, source_id TEXT NOT NULL, target_id INTEGER, ' |
| 95 | 'source_json TEXT NOT NULL, PRIMARY KEY(kind,source_id))') |
| 96 | previous = db.execute("SELECT source_json FROM studio_forgejo_records WHERE kind='export' AND source_id='personal'").fetchone() |
| 97 | if previous and json.loads(previous[0])['sha256'] != source_hash: |
| 98 | raise ValueError('A different export was already imported; review the source before merging') |
| 99 | |
| 100 | def saved(kind, key): |
| 101 | row = db.execute('SELECT target_id FROM studio_forgejo_records WHERE kind=? AND source_id=?', (kind, str(key))).fetchone() |
| 102 | return row[0] if row else None |
| 103 | |
| 104 | def record(kind, key, target_id, row): |
| 105 | db.execute('INSERT OR IGNORE INTO studio_forgejo_records VALUES(?,?,?,?)', |
| 106 | (kind, str(key), target_id, json.dumps(row, ensure_ascii=False, sort_keys=True))) |
| 107 | |
| 108 | users = {1: owner} |
| 109 | names = {r['name'] for r in db.execute('SELECT name FROM users')} |
| 110 | actors = {int(r.get('poster_id') or 0) for r in [*data['issues'], *data['comments'], *data['history']]} |
| 111 | actors.update(int(r['assignee_id']) for r in data['assignees']) |
| 112 | author_source = {r['id']: r for r in data['authors']} |
| 113 | for key in sorted(actors - {1}): |
| 114 | row = author_source.get(key, {'id': key, 'name': 'forgejo-system' if key == 0 else f'forgejo-user-{key}', |
| 115 | 'created_unix': min(i['created_unix'] for i in data['issues']), |
| 116 | 'updated_unix': max(i['updated_unix'] for i in data['issues'])}) |
| 117 | existing = saved('user', key) |
| 118 | if existing is None: |
| 119 | name = row['name'] |
| 120 | if name in names: |
| 121 | name = 'forgejo-' + name |
| 122 | if name in names: |
| 123 | raise ValueError('Historical author name collision') |
| 124 | cursor = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)', |
| 125 | (identifier('user', key, row['created_unix']), 'personal-forgejo.invalid', |
| 126 | str(key), name, timestamp(row['created_unix']), timestamp(row['updated_unix']))) |
| 127 | existing = cursor.lastrowid |
| 128 | names.add(name) |
| 129 | record('user', key, existing, row) |
| 130 | users[key] = existing |
| 131 | record('user', 1, owner, author_source[1]) |
| 132 | |
| 133 | external = {'paperclover': owner} |
| 134 | external_rows = defaultdict(list) |
| 135 | for row in [*data['issues'], *data['comments']]: |
| 136 | if row.get('original_author'): |
| 137 | external_rows[row['original_author']].append(row) |
| 138 | for name, rows in external_rows.items(): |
| 139 | if name == 'paperclover': |
| 140 | continue |
| 141 | if not re.fullmatch(r'[A-Za-z0-9_.-]+', name): |
| 142 | raise ValueError('Unsupported external author name') |
| 143 | key = 'external:' + name |
| 144 | existing = saved('user', key) |
| 145 | if existing is None: |
| 146 | display = name if name not in names else 'github-' + name |
| 147 | if display in names: |
| 148 | raise ValueError('External author name collision') |
| 149 | created = min(row['created_unix'] for row in rows) |
| 150 | updated = max(row['updated_unix'] or row['created_unix'] for row in rows) |
| 151 | existing = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)', |
| 152 | (identifier('user', key, created), 'personal-forgejo-external.invalid', name, |
| 153 | display, timestamp(created), timestamp(updated))).lastrowid |
| 154 | names.add(display) |
| 155 | record('user', key, existing, {'original_author': name}) |
| 156 | external[name] = existing |
| 157 | |
| 158 | def actor(row): |
| 159 | return external[row['original_author']] if row.get('original_author') else users[row['poster_id']] |
| 160 | |
| 161 | labels = {} |
| 162 | for row in sorted(data['labels'], key=lambda r: r['id']): |
| 163 | if row['repo_id'] not in sources: |
| 164 | raise ValueError('Label has no source repository') |
| 165 | repo = repos[sources[row['repo_id']]['name']] |
| 166 | existing = saved('label', row['id']) |
| 167 | if existing is None: |
| 168 | color = row['color'].lstrip('#') |
| 169 | if not re.fullmatch('[0-9a-fA-F]{6}', color): |
| 170 | raise ValueError('Invalid source label color') |
| 171 | epoch = row['created_unix'] or sources[row['repo_id']]['created_unix'] |
| 172 | existing = db.execute('INSERT INTO issue_labels(uuid,repo,name,description,color,last_updated) VALUES(?,?,?,?,?,?)', |
| 173 | (identifier('label', row['id'], epoch), repo['id'], row['name'], row['description'], |
| 174 | '#' + color, timestamp(row['updated_unix'] or epoch))).lastrowid |
| 175 | record('label', row['id'], existing, row) |
| 176 | labels[row['id']] = existing |
| 177 | |
| 178 | # Preserve incoming numbers when free; collisions go above BOTH namespaces. |
| 179 | occupied = defaultdict(set) |
| 180 | ceilings = defaultdict(int) |
| 181 | for row in db.execute('SELECT repo,number FROM issues'): |
| 182 | occupied[row['repo']].add(row['number']) |
| 183 | ceilings[row['repo']] = max(ceilings[row['repo']], row['number']) |
| 184 | for row in data['issues']: |
| 185 | repo = repos[sources[row['repo_id']]['name']]['id'] |
| 186 | ceilings[repo] = max(ceilings[repo], row['index']) |
| 187 | issue_map = {} |
| 188 | collisions = [] |
| 189 | assignees = defaultdict(list) |
| 190 | for row in data['assignees']: |
| 191 | assignees[row['issue_id']].append(users[row['assignee_id']]) |
| 192 | for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])): |
| 193 | repo = repos[sources[row['repo_id']]['name']] |
| 194 | existing = saved('issue', row['id']) |
| 195 | number = row['index'] |
| 196 | if existing is None: |
| 197 | if number in occupied[repo['id']]: |
| 198 | ceilings[repo['id']] += 1 |
| 199 | number = ceilings[repo['id']] |
| 200 | occupied[repo['id']].add(number) |
| 201 | existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)', |
| 202 | (identifier('issue', row['id'], row['created_unix']), repo['id'], number, |
| 203 | actor(row), timestamp(row['updated_unix']), row['name'], |
| 204 | 'done' if row['is_closed'] else 'todo', |
| 205 | assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid |
| 206 | record('issue', row['id'], existing, row) |
| 207 | else: |
| 208 | number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0] |
| 209 | if number != row['index']: |
| 210 | collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number}) |
| 211 | issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'], |
| 212 | 'url': '/' + quote(repo['name'], safe='/') + '/issues/' + str(number)} |
| 213 | |
| 214 | # Preserve attachment authorization by checking the associated Shale issue |
| 215 | # before Caddy serves the copied file. No unauthenticated static directory. |
| 216 | attachment_map = {} |
| 217 | manifest = [] |
| 218 | for row in data['attachments']: |
| 219 | if row['issue_id'] not in issue_map: |
| 220 | raise ValueError('An attachment belongs to an unsupported release or missing issue') |
| 221 | if row['external_url']: |
| 222 | attachment_map[row['uuid']] = row['external_url'] |
| 223 | record('attachment', row['id'], None, row) |
| 224 | continue |
| 225 | key = row['uuid'] |
| 226 | if not re.fullmatch('[0-9a-f-]{36}', key): |
| 227 | raise ValueError('Unsafe attachment UUID') |
| 228 | source_file = attachments / key[0] / key[1] / key |
| 229 | if source_file.is_symlink() or not source_file.is_file() or source_file.stat().st_size != row['size']: |
| 230 | raise ValueError('Source attachment missing or size differs') |
| 231 | name = Path(row['name']).name |
| 232 | if name != row['name'] or not name or any(c in name for c in '\\"\r\n'): |
| 233 | raise ValueError('Unsafe attachment filename') |
| 234 | destination = target / 'forgejo-attachments' / key / name |
| 235 | destination.parent.mkdir(parents=True, exist_ok=True) |
| 236 | if destination.exists() and digest(destination) != digest(source_file): |
| 237 | raise ValueError('Attachment copy differs') |
| 238 | if not destination.exists(): |
| 239 | shutil.copyfile(source_file, destination) |
| 240 | checksum = digest(source_file) |
| 241 | if digest(destination) != checksum: |
| 242 | raise ValueError('Attachment checksum failed') |
| 243 | # Caddy serves these only after Shale authorizes the associated issue. |
| 244 | # Keep them unavailable to other local users. |
| 245 | caddy_group = grp.getgrnam('caddy').gr_gid |
| 246 | for directory in [target / 'forgejo-attachments', destination.parent]: |
| 247 | os.chown(directory, 0, caddy_group) |
| 248 | directory.chmod(0o750) |
| 249 | os.chown(destination, 0, caddy_group) |
| 250 | destination.chmod(0o640) |
| 251 | path = '/-/forgejo-attachments/' + key + '/' + quote(name, safe='') |
| 252 | attachment_map[key] = path |
| 253 | manifest.append({'path': path, 'file': key + '/' + name, |
| 254 | 'issue': issue_map[row['issue_id']]['url'], 'sha256': checksum}) |
| 255 | record('attachment', row['id'], None, row) |
| 256 | |
| 257 | source_urls = {} |
| 258 | for row in data['issues']: |
| 259 | repo = sources[row['repo_id']] |
| 260 | owner_name = repo['owner_name'] |
| 261 | for kind in ['issues', 'pulls']: |
| 262 | source_urls[f'/{owner_name}/{repo["name"]}/{kind}/{row["index"]}'] = issue_map[row['id']]['url'] |
| 263 | |
| 264 | def rewrite(text, repo_id): |
| 265 | def attachment(match): |
| 266 | key = match.group(1) |
| 267 | if key not in attachment_map: |
| 268 | raise ValueError('Issue references an attachment absent from the source export') |
| 269 | return attachment_map[key] |
| 270 | text = re.sub(r'(?:https?://(?:git|forgejo)\.paperclover\.net)?/attachments/([0-9a-f-]{36})', attachment, text) |
| 271 | def link(match): |
| 272 | return source_urls.get(match.group(1), match.group(0)) |
| 273 | text = re.sub(r'https?://(?:git|forgejo)\.paperclover\.net(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)', link, text) |
| 274 | text = re.sub(r'(?<=\]\()(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)(?=\))', link, text) |
| 275 | # Leave code, quoted text, and bare #references untouched; map explicit links only. |
| 276 | return text |
| 277 | |
| 278 | def action(kind, key, issue_id, actor, payload, added, updated=None, source_row=None): |
| 279 | existing = saved(kind, key) |
| 280 | if existing is not None: |
| 281 | return existing |
| 282 | created = db.execute('INSERT INTO issue_actions(uuid,issue,actor,kind,payload,added_on,last_updated) VALUES(?,?,?,?,?,?,?)', |
| 283 | (identifier(kind, key, added), issue_id, actor, kind if kind in ['comment', 'update_status', 'add_label', 'remove_label'] else 'comment', |
| 284 | payload, timestamp(added), timestamp(updated or added))).lastrowid |
| 285 | record(kind, key, created, source_row or {}) |
| 286 | return created |
| 287 | |
| 288 | milestone_source = {r['id']: r for r in data['milestones']} |
| 289 | pulls = {r['issue_id']: r for r in data['pull_requests']} |
| 290 | body_edits = defaultdict(int) |
| 291 | for revision in data['history']: |
| 292 | if not revision['comment_id'] and not revision['is_first_created']: |
| 293 | body_edits[revision['issue_id']] = max(body_edits[revision['issue_id']], revision['edited_unix']) |
| 294 | for row in data['issues']: |
| 295 | item = issue_map[row['id']] |
| 296 | body = rewrite(row['content'], row['repo_id']) |
| 297 | metadata = [] |
| 298 | if item['number'] != row['index']: |
| 299 | metadata.append(f'Original Forgejo issue #{row["index"]}.') |
| 300 | if row['is_pull']: |
| 301 | pr = pulls[row['id']] |
| 302 | metadata.append(f'Imported pull request: `{pr["head_branch"]}` → `{pr["base_branch"]}`.' + |
| 303 | (f' Merged as `{pr["merged_commit_id"]}`.' if pr['has_merged'] else '')) |
| 304 | if row['milestone_id']: |
| 305 | metadata.append('Milestone: ' + milestone_source[row['milestone_id']]['name'] + '.') |
| 306 | if row['is_locked']: |
| 307 | metadata.append('The original discussion was locked.') |
| 308 | if len(assignees[row['id']]) > 1: |
| 309 | original_assignees = [author_source[a['assignee_id']]['name'] for a in data['assignees'] if a['issue_id'] == row['id']] |
| 310 | metadata.append('Original assignees: ' + ', '.join(original_assignees) + '.') |
| 311 | if metadata: |
| 312 | body += '\n\n---\n\n' + '\n\n'.join(metadata) |
| 313 | action('issue-body', row['id'], item['id'], actor(row), body, |
| 314 | row['created_unix'], max(row['created_unix'], body_edits[row['id']]), row) |
| 315 | # Shale's status-change renderer requires an initial status action. |
| 316 | # Forgejo stores initial state on the issue, rather than as a comment. |
| 317 | action('update_status', 'initial-' + str(row['id']), item['id'], actor(row), |
| 318 | 'todo', row['created_unix']) |
| 319 | |
| 320 | for row in sorted(data['comments'], key=lambda r: (r['created_unix'], r['id'])): |
| 321 | item = issue_map[row['issue_id']] |
| 322 | kind = 'comment' |
| 323 | payload = rewrite(row['content'], issue_sources[row['issue_id']]['repo_id']) |
| 324 | if row['type'] in [1, 2]: |
| 325 | kind, payload = 'update_status', 'todo' if row['type'] == 1 else 'done' |
| 326 | elif row['type'] == 7 and row['label_id'] in labels: |
| 327 | kind = 'add_label' if row['content'] == '1' else 'remove_label' |
| 328 | payload = str(labels[row['label_id']]) |
| 329 | elif row['type'] != 0: |
| 330 | event = EVENTS[row['type']] if row['type'] < len(EVENTS) else f'event {row["type"]}' |
| 331 | detail = [] |
| 332 | for field in ['old_title', 'new_title', 'old_ref', 'new_ref', 'commit_sha', 'commit_id', 'tree_path', 'line']: |
| 333 | if row.get(field): |
| 334 | detail.append(f'{field}: {row[field]}') |
| 335 | for field in ['old_milestone_id', 'milestone_id']: |
| 336 | if row.get(field): |
| 337 | detail.append(field + ': ' + milestone_source.get(row[field], {}).get('name', str(row[field]))) |
| 338 | if row.get('dependent_issue_id'): |
| 339 | linked = issue_map.get(row['dependent_issue_id']) |
| 340 | detail.append('Dependency: ' + (f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})' if linked else str(row['dependent_issue_id']))) |
| 341 | if row.get('ref_issue_id') and row['ref_issue_id'] in issue_map: |
| 342 | linked = issue_map[row['ref_issue_id']] |
| 343 | detail.append(f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})') |
| 344 | payload = 'Forgejo: ' + event + '.' + ('\n\n' + '\n\n'.join(detail) if detail else '') + ('\n\n' + payload if payload else '') |
| 345 | existing = saved('forgejo-comment', row['id']) |
| 346 | if existing is None: |
| 347 | # Every original comment/event maps to exactly one Shale action. |
| 348 | action_id = action(kind, 'forgejo-' + str(row['id']), item['id'], actor(row), payload, |
| 349 | row['created_unix'], row['updated_unix'], row) |
| 350 | record('forgejo-comment', row['id'], action_id, row) |
| 351 | |
| 352 | for row in data['issue_labels']: |
| 353 | item = issue_map[row['issue_id']] |
| 354 | existing = saved('issue-label', row['id']) |
| 355 | if existing is None: |
| 356 | label = labels[row['label_id']] |
| 357 | epoch = issue_sources[row['issue_id']]['updated_unix'] |
| 358 | existing = db.execute('INSERT INTO issues__labels(uuid,issue,label) VALUES(?,?,?)', |
| 359 | (identifier('issue-label', row['id'], epoch), item['id'], label)).lastrowid |
| 360 | record('issue-label', row['id'], existing, row) |
| 361 | |
| 362 | for table in ['history', 'milestones', 'pull_requests', 'assignees']: |
| 363 | for row in data[table]: |
| 364 | record(table, row['id'], None, row) |
| 365 | for source_repo in data['repositories']: |
| 366 | repo = repos[source_repo['name']] |
| 367 | if any(i['repo_id'] == source_repo['id'] for i in data['issues']): |
| 368 | db.execute("UPDATE repositories SET access_issues=? WHERE id=? AND access_issues='off'", |
| 369 | ('private' if source_repo['is_private'] else 'public', repo['id'])) |
| 370 | for field in ['access_issues_submit', 'access_issues_comment']: |
| 371 | db.execute(f"UPDATE repositories SET {field}='private' WHERE id=? AND {field}='off'", (repo['id'],)) |
| 372 | slots = ','.join('?' for _ in CLOSED) |
| 373 | db.execute(f'UPDATE repositories SET open_issues=(SELECT count(*) FROM issues WHERE repo=repositories.id AND status NOT IN ({slots}))', CLOSED) |
| 374 | db.execute(f'UPDATE issue_labels SET open_issues=(SELECT count(*) FROM issues__labels il JOIN issues i ON i.id=il.issue WHERE il.label=issue_labels.id AND i.status NOT IN ({slots}))', CLOSED) |
| 375 | for table, rows in originals.items(): |
| 376 | for row in rows: |
| 377 | current = dict(db.execute(f'SELECT * FROM {table} WHERE id=?', (row['id'],)).fetchone()) |
| 378 | if table == 'issue_labels': |
| 379 | current.pop('open_issues');row = {k:v for k,v in row.items() if k != 'open_issues'} |
| 380 | if current != row: |
| 381 | raise ValueError(f'Existing Shale {table} row changed') |
| 382 | if db.execute('PRAGMA foreign_key_check').fetchall(): |
| 383 | raise ValueError('Imported data has invalid foreign keys') |
| 384 | for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']), |
| 385 | ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]: |
| 386 | count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0] |
| 387 | if count != len(rows): |
| 388 | raise ValueError(f'Incomplete import: {kind}') |
| 389 | record('export', 'personal', None, {'sha256': source_hash}) |
| 390 | before_commit() |
| 391 | db.commit() |
| 392 | db.close() |
| 393 | report = {'source_sha256': source_hash, 'issues': len(data['issues']), 'comments_and_events': len(data['comments']), |
| 394 | 'labels': len(labels), 'attachments': len(manifest), 'history_revisions': len(data['history']), |
| 395 | 'number_collisions': collisions, 'issue_mapping': issue_map} |
| 396 | (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n') |
| 397 | (target / 'forgejo-attachments.json').write_text(json.dumps(manifest, indent=2) + '\n') |
| 398 | print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2)) |
| 399 | |
| 400 | |
| 401 | def main(): |
| 402 | parser = argparse.ArgumentParser() |
| 403 | parser.add_argument('--target', type=Path, required=True) |
| 404 | parser.add_argument('--source', type=Path, required=True) |
| 405 | parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source') |
| 406 | parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments')) |
| 407 | parser.add_argument('--rehearsal', action='store_true') |
| 408 | args = parser.parse_args() |
| 409 | if os.geteuid() != 0: |
| 410 | parser.error('Run as root on Zenith') |
| 411 | os.umask(0o077) |
| 412 | target = args.target.resolve() |
| 413 | if 'evil' in str(target).lower() or target.is_relative_to('/mnt/storage1/apps'): |
| 414 | parser.error('Use a managed Shale target or isolated copy') |
| 415 | if args.rehearsal: |
| 416 | if target.is_relative_to('/srv/prod'): |
| 417 | parser.error('Rehearsal must use an isolated target') |
| 418 | elif target != Path('/srv/prod/shale'): |
| 419 | parser.error('Production imports must target /srv/prod/shale') |
| 420 | guard = lambda: None |
| 421 | if target == Path('/srv/prod/shale'): |
| 422 | guard = lambda: stopped('shale') |
| 423 | elif target.parent == Path('/srv/staging'): |
| 424 | if not re.fullmatch(r'shale-preview-[0-9a-f]{8}', target.name): |
| 425 | parser.error('Expected a managed Shale stage') |
| 426 | guard = lambda: stopped(target.name) |
| 427 | proof = args.proof or args.source.parent / 'source-proof.json' |
| 428 | if not proof.is_file() or json.loads(proof.read_text()).get('sourceSha256') != digest(args.source): |
| 429 | parser.error('Verified export proof is missing or the source checksum differs') |
| 430 | guard() |
| 431 | evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent))) |
| 432 | migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard) |
| 433 | print('Evidence:', evidence) |
| 434 | |
| 435 | |
| 436 | if __name__ == '__main__': |
| 437 | main() |
clover caruso <git@paperclover.net> 2026-10-05 00:27:24-07:00