diff --git a/service/shale/service.pkl b/service/shale/service.pkl index 3ca707cdc9b6400b7a4c11719e09717e4186f395..aab981d0f098c92c889819f579a3186bed3c092b 100644 --- a/service/shale/service.pkl +++ b/service/shale/service.pkl @@ -14,7 +14,7 @@ requirements { } container { - image = "docker.io/astheno/shale@sha256:ece987e25ebe67275fbe105b1b03e650a9b3adfe716928beb717cde1d32c9652" + image = "docker.io/astheno/shale@sha256:dfffceebe31fd3360b6dcf12caab664735415fdc32effd5082ac37b11864a232" extraHosts { "\(keycloak.container.http.hostname):host-gateway" } http { @@ -51,6 +51,8 @@ container { ["DOMAIN"] = module.container.http.hostname ["HOME"] = "/data" ["SERVER_TITLE"] = "clover's git" + // The older Markdown parser shares a capture buffer between request workers. + ["NPROC"] = "1" ["SESSION_SECRET"] = "${secret.own.session_secret}" ["OAUTH2_CLIENT"] = "oidc,\(keycloak.container.http.hostname)/realms/master|${secret.oidc.clientId}|${secret.oidc.clientSecret}" } diff --git a/tools/deploy.py b/tools/deploy.py index c925836080af9e94f57a7e38fbc787217d64a024..54cb7d4d747d06e361136be172ebe6a8422196ab 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -44,7 +44,7 @@ def sync_manager(release): def upload(main=False): - excluded = excluded_services(REPO) + excluded = set() if main else excluded_services(REPO) with tempfile.TemporaryDirectory() as temporary: snapshot = Path(temporary) revision = None @@ -58,6 +58,13 @@ def upload(main=False): if conflicted or not description.strip(): raise ValueError("main needs a commit description and no conflicts before deployment") revision = {"commit": commit, "description": description} + # Production must use main's exclusions too. A working staging + # change must not remove dependencies from the committed release. + excluded = set(json.loads(subprocess.run( + ["jj", "--ignore-working-copy", "file", "show", "-r", commit, + "config/excluded-services.json"], + cwd=REPO, check=True, capture_output=True, text=True, + ).stdout)) entries = subprocess.run( ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T", '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"', diff --git a/tools/import-forgejo-issues.py b/tools/import-forgejo-issues.py new file mode 100644 index 0000000000000000000000000000000000000000..010b225f923b5130372ffc6de311d0f851e293ff --- /dev/null +++ b/tools/import-forgejo-issues.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""Import a verified personal Forgejo export into stopped Shale, without replacing native issues.""" +import argparse +from collections import defaultdict +from datetime import datetime, timezone +import hashlib +import grp +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import sys +import tempfile +import urllib.request +from urllib.parse import quote + +ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' +EPOCH = 1577836800 # Shale's ULIDs use 2020-01-01, rather than the Unix epoch. +CLOSED = ('done', 'not_planned', 'duplicate', 'invalid') +EVENTS = ['comment', 'reopened', 'closed', 'issue reference', 'commit reference', + 'comment reference', 'pull request reference', 'label changed', 'milestone changed', + 'assignee changed', 'title changed', 'branch deleted', 'time tracking started', + 'time tracking stopped', 'time added', 'time tracking canceled', 'deadline added', + 'deadline changed', 'deadline removed', 'dependency added', 'dependency removed', + 'code comment', 'review', 'locked', 'unlocked', 'target branch changed', + 'time deleted', 'review requested', 'merged', 'pull request updated', + 'project changed', 'project column changed', 'review dismissed', 'reference changed', + 'automatic merge scheduled', 'automatic merge canceled', 'pinned', 'unpinned'] + + +def timestamp(epoch): + return datetime.fromtimestamp(int(epoch), timezone.utc).isoformat(timespec='seconds') + + +def identifier(kind, key, epoch): + milliseconds = (int(epoch) - EPOCH) * 1000 + if not 0 <= milliseconds < 2 ** 48: + raise ValueError('Source creation time is outside Shale ULID range') + entropy = int.from_bytes(hashlib.sha256(f'personal-forgejo:{kind}:{key}'.encode()).digest()[:10], 'big') + value = (milliseconds << 80) | entropy + return ''.join(ALPHABET[(value >> (5 * i)) & 31] for i in range(25, -1, -1)) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def stopped(job): + token = os.environ.get('NOMAD_TOKEN') or Path('/var/lib/studio/nomad.token').read_text().strip() + def read(path): + request = urllib.request.Request('http://127.0.0.1:4646/v1/' + path, + headers={'X-Nomad-Token': token}) + with urllib.request.urlopen(request, timeout=10) as response: + return json.load(response) + if not read('job/' + job).get('Stop') or any( + item['ClientStatus'] in ('pending', 'running') for item in read('job/' + job + '/allocations')): + raise ValueError('Stop the target Shale job and wait for its allocations before importing') + + +def migrate(database, source, attachments, target, evidence, before_commit=lambda: None): + db = sqlite3.connect(database) + db.row_factory = sqlite3.Row + db.execute('PRAGMA foreign_keys=ON') + if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': + raise ValueError('Shale SQLite integrity check failed') + source_hash = digest(source) + data = json.loads(source.read_text()) + for field in ['repositories', 'issues', 'comments', 'labels', 'issue_labels', 'authors', + 'attachments', 'assignees', 'milestones', 'pull_requests', 'history']: + if not isinstance(data.get(field), list): + raise ValueError(f'Missing source table: {field}') + owners = db.execute("SELECT * FROM users WHERE name='clover'").fetchall() + if len(owners) != 1 or owners[0]['snowflake'] != '8c909fab-98b0-4472-9171-9606ccebe9d5': + raise ValueError('Clover original identity is missing') + owner = owners[0]['id'] + repos = {r['name']: dict(r) for r in db.execute('SELECT * FROM repositories')} + sources = {r['id']: r for r in data['repositories']} + issue_sources = {r['id']: r for r in data['issues']} + if len(issue_sources) != len(data['issues']): + raise ValueError('Duplicate source issue IDs') + for r in data['repositories']: + if r['name'] not in repos or repos[r['name']]['owner'] != owner: + raise ValueError(f'Expected imported Clover repository: {r["name"]}') + if any(c['issue_id'] not in issue_sources for c in data['comments']): + raise ValueError('Orphan source comment') + with sqlite3.connect(evidence / 'before.db') as backup: + db.backup(backup) + originals = {table: [dict(r) for r in db.execute(f'SELECT * FROM {table}')] + for table in ['issues', 'issue_actions', 'issue_labels', 'issues__labels', 'users']} + db.execute('BEGIN IMMEDIATE') + db.execute('CREATE TABLE IF NOT EXISTS studio_forgejo_records(' + 'kind TEXT NOT NULL, source_id TEXT NOT NULL, target_id INTEGER, ' + 'source_json TEXT NOT NULL, PRIMARY KEY(kind,source_id))') + previous = db.execute("SELECT source_json FROM studio_forgejo_records WHERE kind='export' AND source_id='personal'").fetchone() + if previous and json.loads(previous[0])['sha256'] != source_hash: + raise ValueError('A different export was already imported; review the source before merging') + + def saved(kind, key): + row = db.execute('SELECT target_id FROM studio_forgejo_records WHERE kind=? AND source_id=?', (kind, str(key))).fetchone() + return row[0] if row else None + + def record(kind, key, target_id, row): + db.execute('INSERT OR IGNORE INTO studio_forgejo_records VALUES(?,?,?,?)', + (kind, str(key), target_id, json.dumps(row, ensure_ascii=False, sort_keys=True))) + + users = {1: owner} + names = {r['name'] for r in db.execute('SELECT name FROM users')} + actors = {int(r.get('poster_id') or 0) for r in [*data['issues'], *data['comments'], *data['history']]} + actors.update(int(r['assignee_id']) for r in data['assignees']) + author_source = {r['id']: r for r in data['authors']} + for key in sorted(actors - {1}): + row = author_source.get(key, {'id': key, 'name': 'forgejo-system' if key == 0 else f'forgejo-user-{key}', + 'created_unix': min(i['created_unix'] for i in data['issues']), + 'updated_unix': max(i['updated_unix'] for i in data['issues'])}) + existing = saved('user', key) + if existing is None: + name = row['name'] + if name in names: + name = 'forgejo-' + name + if name in names: + raise ValueError('Historical author name collision') + cursor = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)', + (identifier('user', key, row['created_unix']), 'personal-forgejo.invalid', + str(key), name, timestamp(row['created_unix']), timestamp(row['updated_unix']))) + existing = cursor.lastrowid + names.add(name) + record('user', key, existing, row) + users[key] = existing + record('user', 1, owner, author_source[1]) + + external = {'paperclover': owner} + external_rows = defaultdict(list) + for row in [*data['issues'], *data['comments']]: + if row.get('original_author'): + external_rows[row['original_author']].append(row) + for name, rows in external_rows.items(): + if name == 'paperclover': + continue + if not re.fullmatch(r'[A-Za-z0-9_.-]+', name): + raise ValueError('Unsupported external author name') + key = 'external:' + name + existing = saved('user', key) + if existing is None: + display = name if name not in names else 'github-' + name + if display in names: + raise ValueError('External author name collision') + created = min(row['created_unix'] for row in rows) + updated = max(row['updated_unix'] or row['created_unix'] for row in rows) + existing = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)', + (identifier('user', key, created), 'personal-forgejo-external.invalid', name, + display, timestamp(created), timestamp(updated))).lastrowid + names.add(display) + record('user', key, existing, {'original_author': name}) + external[name] = existing + + def actor(row): + return external[row['original_author']] if row.get('original_author') else users[row['poster_id']] + + labels = {} + for row in sorted(data['labels'], key=lambda r: r['id']): + if row['repo_id'] not in sources: + raise ValueError('Label has no source repository') + repo = repos[sources[row['repo_id']]['name']] + existing = saved('label', row['id']) + if existing is None: + color = row['color'].lstrip('#') + if not re.fullmatch('[0-9a-fA-F]{6}', color): + raise ValueError('Invalid source label color') + epoch = row['created_unix'] or sources[row['repo_id']]['created_unix'] + existing = db.execute('INSERT INTO issue_labels(uuid,repo,name,description,color,last_updated) VALUES(?,?,?,?,?,?)', + (identifier('label', row['id'], epoch), repo['id'], row['name'], row['description'], + '#' + color, timestamp(row['updated_unix'] or epoch))).lastrowid + record('label', row['id'], existing, row) + labels[row['id']] = existing + + # Preserve incoming numbers when free; collisions go above BOTH namespaces. + occupied = defaultdict(set) + ceilings = defaultdict(int) + for row in db.execute('SELECT repo,number FROM issues'): + occupied[row['repo']].add(row['number']) + ceilings[row['repo']] = max(ceilings[row['repo']], row['number']) + for row in data['issues']: + repo = repos[sources[row['repo_id']]['name']]['id'] + ceilings[repo] = max(ceilings[repo], row['index']) + issue_map = {} + collisions = [] + assignees = defaultdict(list) + for row in data['assignees']: + assignees[row['issue_id']].append(users[row['assignee_id']]) + for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])): + repo = repos[sources[row['repo_id']]['name']] + existing = saved('issue', row['id']) + number = row['index'] + if existing is None: + if number in occupied[repo['id']]: + ceilings[repo['id']] += 1 + number = ceilings[repo['id']] + occupied[repo['id']].add(number) + existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)', + (identifier('issue', row['id'], row['created_unix']), repo['id'], number, + actor(row), timestamp(row['updated_unix']), row['name'], + 'done' if row['is_closed'] else 'todo', + assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid + record('issue', row['id'], existing, row) + else: + number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0] + if number != row['index']: + collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number}) + issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'], + 'url': '/' + quote(repo['name'], safe='/') + '/issues/' + str(number)} + + # Preserve attachment authorization by checking the associated Shale issue + # before Caddy serves the copied file. No unauthenticated static directory. + attachment_map = {} + manifest = [] + for row in data['attachments']: + if row['issue_id'] not in issue_map: + raise ValueError('An attachment belongs to an unsupported release or missing issue') + if row['external_url']: + attachment_map[row['uuid']] = row['external_url'] + record('attachment', row['id'], None, row) + continue + key = row['uuid'] + if not re.fullmatch('[0-9a-f-]{36}', key): + raise ValueError('Unsafe attachment UUID') + source_file = attachments / key[0] / key[1] / key + if source_file.is_symlink() or not source_file.is_file() or source_file.stat().st_size != row['size']: + raise ValueError('Source attachment missing or size differs') + name = Path(row['name']).name + if name != row['name'] or not name or any(c in name for c in '\\"\r\n'): + raise ValueError('Unsafe attachment filename') + destination = target / 'forgejo-attachments' / key / name + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and digest(destination) != digest(source_file): + raise ValueError('Attachment copy differs') + if not destination.exists(): + shutil.copyfile(source_file, destination) + checksum = digest(source_file) + if digest(destination) != checksum: + raise ValueError('Attachment checksum failed') + # Caddy serves these only after Shale authorizes the associated issue. + # Keep them unavailable to other local users. + caddy_group = grp.getgrnam('caddy').gr_gid + for directory in [target / 'forgejo-attachments', destination.parent]: + os.chown(directory, 0, caddy_group) + directory.chmod(0o750) + os.chown(destination, 0, caddy_group) + destination.chmod(0o640) + path = '/-/forgejo-attachments/' + key + '/' + quote(name, safe='') + attachment_map[key] = path + manifest.append({'path': path, 'file': key + '/' + name, + 'issue': issue_map[row['issue_id']]['url'], 'sha256': checksum}) + record('attachment', row['id'], None, row) + + source_urls = {} + for row in data['issues']: + repo = sources[row['repo_id']] + owner_name = repo['owner_name'] + for kind in ['issues', 'pulls']: + source_urls[f'/{owner_name}/{repo["name"]}/{kind}/{row["index"]}'] = issue_map[row['id']]['url'] + + def rewrite(text, repo_id): + def attachment(match): + key = match.group(1) + if key not in attachment_map: + raise ValueError('Issue references an attachment absent from the source export') + return attachment_map[key] + text = re.sub(r'(?:https?://(?:git|forgejo)\.paperclover\.net)?/attachments/([0-9a-f-]{36})', attachment, text) + def link(match): + return source_urls.get(match.group(1), match.group(0)) + text = re.sub(r'https?://(?:git|forgejo)\.paperclover\.net(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)', link, text) + text = re.sub(r'(?<=\]\()(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)(?=\))', link, text) + # Leave code, quoted text, and bare #references untouched; map explicit links only. + return text + + def action(kind, key, issue_id, actor, payload, added, updated=None, source_row=None): + existing = saved(kind, key) + if existing is not None: + return existing + created = db.execute('INSERT INTO issue_actions(uuid,issue,actor,kind,payload,added_on,last_updated) VALUES(?,?,?,?,?,?,?)', + (identifier(kind, key, added), issue_id, actor, kind if kind in ['comment', 'update_status', 'add_label', 'remove_label'] else 'comment', + payload, timestamp(added), timestamp(updated or added))).lastrowid + record(kind, key, created, source_row or {}) + return created + + milestone_source = {r['id']: r for r in data['milestones']} + pulls = {r['issue_id']: r for r in data['pull_requests']} + body_edits = defaultdict(int) + for revision in data['history']: + if not revision['comment_id'] and not revision['is_first_created']: + body_edits[revision['issue_id']] = max(body_edits[revision['issue_id']], revision['edited_unix']) + for row in data['issues']: + item = issue_map[row['id']] + body = rewrite(row['content'], row['repo_id']) + metadata = [] + if item['number'] != row['index']: + metadata.append(f'Original Forgejo issue #{row["index"]}.') + if row['is_pull']: + pr = pulls[row['id']] + metadata.append(f'Imported pull request: `{pr["head_branch"]}` → `{pr["base_branch"]}`.' + + (f' Merged as `{pr["merged_commit_id"]}`.' if pr['has_merged'] else '')) + if row['milestone_id']: + metadata.append('Milestone: ' + milestone_source[row['milestone_id']]['name'] + '.') + if row['is_locked']: + metadata.append('The original discussion was locked.') + if len(assignees[row['id']]) > 1: + original_assignees = [author_source[a['assignee_id']]['name'] for a in data['assignees'] if a['issue_id'] == row['id']] + metadata.append('Original assignees: ' + ', '.join(original_assignees) + '.') + if metadata: + body += '\n\n---\n\n' + '\n\n'.join(metadata) + action('issue-body', row['id'], item['id'], actor(row), body, + row['created_unix'], max(row['created_unix'], body_edits[row['id']]), row) + # Shale's status-change renderer requires an initial status action. + # Forgejo stores initial state on the issue, rather than as a comment. + action('update_status', 'initial-' + str(row['id']), item['id'], actor(row), + 'todo', row['created_unix']) + + for row in sorted(data['comments'], key=lambda r: (r['created_unix'], r['id'])): + item = issue_map[row['issue_id']] + kind = 'comment' + payload = rewrite(row['content'], issue_sources[row['issue_id']]['repo_id']) + if row['type'] in [1, 2]: + kind, payload = 'update_status', 'todo' if row['type'] == 1 else 'done' + elif row['type'] == 7 and row['label_id'] in labels: + kind = 'add_label' if row['content'] == '1' else 'remove_label' + payload = str(labels[row['label_id']]) + elif row['type'] != 0: + event = EVENTS[row['type']] if row['type'] < len(EVENTS) else f'event {row["type"]}' + detail = [] + for field in ['old_title', 'new_title', 'old_ref', 'new_ref', 'commit_sha', 'commit_id', 'tree_path', 'line']: + if row.get(field): + detail.append(f'{field}: {row[field]}') + for field in ['old_milestone_id', 'milestone_id']: + if row.get(field): + detail.append(field + ': ' + milestone_source.get(row[field], {}).get('name', str(row[field]))) + if row.get('dependent_issue_id'): + linked = issue_map.get(row['dependent_issue_id']) + detail.append('Dependency: ' + (f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})' if linked else str(row['dependent_issue_id']))) + if row.get('ref_issue_id') and row['ref_issue_id'] in issue_map: + linked = issue_map[row['ref_issue_id']] + detail.append(f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})') + payload = 'Forgejo: ' + event + '.' + ('\n\n' + '\n\n'.join(detail) if detail else '') + ('\n\n' + payload if payload else '') + existing = saved('forgejo-comment', row['id']) + if existing is None: + # Every original comment/event maps to exactly one Shale action. + action_id = action(kind, 'forgejo-' + str(row['id']), item['id'], actor(row), payload, + row['created_unix'], row['updated_unix'], row) + record('forgejo-comment', row['id'], action_id, row) + + for row in data['issue_labels']: + item = issue_map[row['issue_id']] + existing = saved('issue-label', row['id']) + if existing is None: + label = labels[row['label_id']] + epoch = issue_sources[row['issue_id']]['updated_unix'] + existing = db.execute('INSERT INTO issues__labels(uuid,issue,label) VALUES(?,?,?)', + (identifier('issue-label', row['id'], epoch), item['id'], label)).lastrowid + record('issue-label', row['id'], existing, row) + + for table in ['history', 'milestones', 'pull_requests', 'assignees']: + for row in data[table]: + record(table, row['id'], None, row) + for source_repo in data['repositories']: + repo = repos[source_repo['name']] + if any(i['repo_id'] == source_repo['id'] for i in data['issues']): + db.execute("UPDATE repositories SET access_issues=? WHERE id=? AND access_issues='off'", + ('private' if source_repo['is_private'] else 'public', repo['id'])) + for field in ['access_issues_submit', 'access_issues_comment']: + db.execute(f"UPDATE repositories SET {field}='private' WHERE id=? AND {field}='off'", (repo['id'],)) + slots = ','.join('?' for _ in CLOSED) + db.execute(f'UPDATE repositories SET open_issues=(SELECT count(*) FROM issues WHERE repo=repositories.id AND status NOT IN ({slots}))', CLOSED) + db.execute(f'UPDATE issue_labels SET open_issues=(SELECT count(*) FROM issues__labels il JOIN issues i ON i.id=il.issue WHERE il.label=issue_labels.id AND i.status NOT IN ({slots}))', CLOSED) + for table, rows in originals.items(): + for row in rows: + current = dict(db.execute(f'SELECT * FROM {table} WHERE id=?', (row['id'],)).fetchone()) + if table == 'issue_labels': + current.pop('open_issues');row = {k:v for k,v in row.items() if k != 'open_issues'} + if current != row: + raise ValueError(f'Existing Shale {table} row changed') + if db.execute('PRAGMA foreign_key_check').fetchall(): + raise ValueError('Imported data has invalid foreign keys') + for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']), + ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]: + count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0] + if count != len(rows): + raise ValueError(f'Incomplete import: {kind}') + record('export', 'personal', None, {'sha256': source_hash}) + before_commit() + db.commit() + db.close() + report = {'source_sha256': source_hash, 'issues': len(data['issues']), 'comments_and_events': len(data['comments']), + 'labels': len(labels), 'attachments': len(manifest), 'history_revisions': len(data['history']), + 'number_collisions': collisions, 'issue_mapping': issue_map} + (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + (target / 'forgejo-attachments.json').write_text(json.dumps(manifest, indent=2) + '\n') + print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--target', type=Path, required=True) + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source') + parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments')) + parser.add_argument('--rehearsal', action='store_true') + args = parser.parse_args() + if os.geteuid() != 0: + parser.error('Run as root on Zenith') + os.umask(0o077) + target = args.target.resolve() + if 'evil' in str(target).lower() or target.is_relative_to('/mnt/storage1/apps'): + parser.error('Use a managed Shale target or isolated copy') + if args.rehearsal: + if target.is_relative_to('/srv/prod'): + parser.error('Rehearsal must use an isolated target') + elif target != Path('/srv/prod/shale'): + parser.error('Production imports must target /srv/prod/shale') + guard = lambda: None + if target == Path('/srv/prod/shale'): + guard = lambda: stopped('shale') + elif target.parent == Path('/srv/staging'): + if not re.fullmatch(r'shale-preview-[0-9a-f]{8}', target.name): + parser.error('Expected a managed Shale stage') + guard = lambda: stopped(target.name) + proof = args.proof or args.source.parent / 'source-proof.json' + if not proof.is_file() or json.loads(proof.read_text()).get('sourceSha256') != digest(args.source): + parser.error('Verified export proof is missing or the source checksum differs') + guard() + evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent))) + migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard) + print('Evidence:', evidence) + + +if __name__ == '__main__': + main() diff --git a/tools/router.py b/tools/router.py index 6e3daff95ff691bcc330bdc6f2e3d56741b93545..64436855afef933f41b8f9d2cc2216406c9d1e90 100644 --- a/tools/router.py +++ b/tools/router.py @@ -45,6 +45,44 @@ def shale_mcp_routes(port): *proxy(f"127.0.0.1:{port}", " "), " }"] +def shale_attachment_routes(upstreams, target="/srv/prod/shale"): + """Authorize retained attachments through their issue before serving bytes.""" + manifest = os.path.join(target, "forgejo-attachments.json") + if not os.path.exists(manifest): + return [] + with open(manifest) as source: + entries = json.load(source) + if not isinstance(entries, list) or len(entries) > 10000: + raise ValueError("invalid Shale attachment manifest") + lines = [] + for index, entry in enumerate(entries): + path, filename, issue = (entry[k] for k in ("path", "file", "issue")) + if (not re.fullmatch(r"[0-9a-f-]{36}/[^/\\\"\r\n]+", filename) + or path != "/-/forgejo-attachments/" + urllib.parse.quote(filename, safe="/") + or not re.fullmatch(r"/(?:[a-zA-Z0-9_.-]+/)+issues/[1-9][0-9]*", issue)): + raise ValueError("unsafe Shale attachment route") + # Request cookies go to Shale as usual. A private issue yields 403/404; + # only Shale's successful issue response permits the local file read. + name = f"@shale_attachment_{index}" + lines += [f" {name} path {json.dumps(path)}", f" handle {name} {{", + " header Cache-Control private,no-store", " header X-Content-Type-Options nosniff", + f" reverse_proxy {upstreams} {{", " method GET", f" rewrite {issue}", + " @authorized status 200", " handle_response @authorized {", + f" root * {json.dumps(os.path.join(target, 'forgejo-attachments'))}", + f" rewrite * {json.dumps('/' + urllib.parse.quote(filename, safe='/'))}", + " file_server", " }", " }", " }"] + return lines + + +def shale_write_routes(host): + # Older Shale releases predate form tokens. Browsers send Origin on POST; + # require the exact site origin for every cookie-authenticated mutation. + # HTTP Git clients use Basic authentication and do not carry this cookie. + return [" @shale_unsafe_origin {", " method POST PUT PATCH DELETE", + " header Cookie *SessionID=*", f" not header Origin https://{host}", + " }", ' respond @shale_unsafe_origin "Forbidden" 403'] + + def render(token): with open(os.environ["STUDIO_PROXY_TOKEN_FILE"]) as file: dashboard_proof = file.read().strip() @@ -136,7 +174,15 @@ def render(token): raise ValueError(f"invalid service name: {service}") if service == "keycloak" and host == auth_host: lines += webauthn + if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): + # Keep the Origin guard before every static/proxy handle; + # otherwise Caddy sorts those handles ahead of respond. + lines += [" route {", *shale_write_routes(host)] + if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): + lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])), + "/srv/staging/" + service) if service == "shale": + lines += shale_attachment_routes(" ".join(sorted(route["upstreams"]))) lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$", " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"] port = int(os.environ["STUDIO_DASHBOARD_PORT"]) @@ -241,6 +287,8 @@ def render(token): *proxy(upstreams, " ", uncompressed=True), " }", " }"] lines += [" handle {", *(f" request_header -{name}" for name in scrub), *proxy(upstreams, " "), " }", "}"] + if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): + lines.insert(len(lines) - 1, " }") else: lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"] if (80, auth_host) not in routes: diff --git a/tools/shale-migration.md b/tools/shale-migration.md index 8bca46b2b80e3e4d5f75ac9484de150d3bd7d8d0..d1860d91131fd989a84c40a6158ab007691b0436 100644 --- a/tools/shale-migration.md +++ b/tools/shale-migration.md @@ -34,6 +34,24 @@ A full-data test imported all 19 retained Forgejo histories into a disposable co After migration and authenticated browser checks, push the new infra `main` to `https://shale.paperclover.net/home-infra.git` with a Shale personal access token. The existing `home-infra` record and UUID are retained; the imported Forgejo branches remain available alongside the tested final `main`. +The October 5 issue migration uses `tools/import-forgejo-issues.py` and the verified personal PostgreSQL export at `/var/lib/studio/forgejo-issue-migration/source.json`. Its sibling `source-proof.json` records the source checksum. The export contains 452 issues, 1,429 comments and events, 60 labels, 24 attachments, and 312 history revisions. Pull request discussions become issues with their original branch and merge metadata. Historical authors retain their names through non-login identities; `paperclover` maps to the existing Clover account. Original creation, update, and comment timestamps are preserved. Every source row is retained in the private `studio_forgejo_records` ledger, including revision history that Shale cannot present as editable comments. + +Run this while the target job is stopped and its allocations have exited: + +```sh +python3 tools/import-forgejo-issues.py \ + --target /srv/prod/shale \ + --source /var/lib/studio/forgejo-issue-migration/source.json +``` + +For an isolated stage, add `--rehearsal` and use `/srv/staging/shale-preview-`. The importer verifies the export checksum, checks that the target job is stopped before writing and immediately before commit, backs up SQLite into a private evidence directory, verifies native issue records remain unchanged, and supports an idempotent rerun of the same export. Take a full service ZFS snapshot before the production import as well. Do not replace production with the stage clone: import again into the stopped production dataset so newer native issues survive. + +Six occupied numbers are remapped: `chat` #1→#6 and #2→#7; `home-infra` #1→#38, #2→#39, and #3→#40; `react-mutation` #1→#19. All other original numbers remain. Explicit Forgejo issue links are rewritten and remapped bodies identify their original number. Bare `#references` and code text remain untouched. Attachments are copied with verified SHA-256 hashes and served through an authorization check against their associated Shale issue. Their local files are readable only by root and Caddy; private issue attachments remain private. + +The writable ZFS rehearsal `shale-preview-0242cee6` starts from all 104 existing native issues and imports to 556 total. It preserves the original Clover OIDC identity. September's `r1616-ga87d2f5.zig.0.16.0` avoids the `r1758` anonymous deleted-comment crash, but its Markdown scanner uses a shared capture buffer and crashes under concurrent fenced-code rendering. The documented `NPROC=1` worker setting avoids that race. With this setting, 904 authenticated/anonymous imported-issue requests passed with eight clients, all 24 attachment hashes and access checks passed, and browser closing of a disposable copy of `chat` #1 succeeded. Production `chat` #1 remains untouched. + +This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict. + The October 4 transport check inspected the image pinned in [service.pkl](../service/shale/service.pkl) in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization. Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.