From c14c62225850a095a6e327e5525cea6966a92948 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 5 Oct 2026 02:12:10 -0700 Subject: [PATCH] Preserve Shale issue numbering after historical imports Insert imports by destination issue number and repair imported row ordering with guarded backups and all-table identity checks. Assisted-by: gpt-6 --- tools/import-forgejo-issues.py | 83 ++++++++++++++++++++++++++++++++-- tools/shale-migration.md | 2 + 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/tools/import-forgejo-issues.py b/tools/import-forgejo-issues.py index 010b225f923b5130372ffc6de311d0f851e293ff..ea01b6f991d307b51f2e1537eb91a86255837a19 100644 --- a/tools/import-forgejo-issues.py +++ b/tools/import-forgejo-issues.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Import a verified personal Forgejo export into stopped Shale, without replacing native issues.""" import argparse -from collections import defaultdict +from collections import Counter, defaultdict from datetime import datetime, timezone import hashlib import grp @@ -189,6 +189,7 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd assignees = defaultdict(list) for row in data['assignees']: assignees[row['issue_id']].append(users[row['assignee_id']]) + planned = [] for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])): repo = repos[sources[row['repo_id']]['name']] existing = saved('issue', row['id']) @@ -198,14 +199,18 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd ceilings[repo['id']] += 1 number = ceilings[repo['id']] occupied[repo['id']].add(number) + else: + number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0] + planned.append((repo, number, existing, row)) + # Shale allocates the next number from the issue with the highest row ID. + for repo, number, existing, row in sorted(planned, key=lambda item: (item[0]['id'], item[1])): + if existing is None: existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)', (identifier('issue', row['id'], row['created_unix']), repo['id'], number, actor(row), timestamp(row['updated_unix']), row['name'], 'done' if row['is_closed'] else 'todo', assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid record('issue', row['id'], existing, row) - else: - number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0] if number != row['index']: collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number}) issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'], @@ -381,6 +386,9 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd raise ValueError(f'Existing Shale {table} row changed') if db.execute('PRAGMA foreign_key_check').fetchall(): raise ValueError('Imported data has invalid foreign keys') + if db.execute('SELECT 1 FROM issues i WHERE i.id=(SELECT max(id) FROM issues WHERE repo=i.repo) ' + 'AND i.number<>(SELECT max(number) FROM issues WHERE repo=i.repo)').fetchone(): + raise ValueError('Issue row order would reuse a number; repair it before importing') for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']), ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]: count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0] @@ -398,6 +406,69 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2)) +def repair_issue_order(database, evidence, before_commit): + with sqlite3.connect(database) as db: + db.row_factory = sqlite3.Row + db.execute('PRAGMA foreign_keys=ON') + with sqlite3.connect(evidence / 'before.db') as backup: + db.backup(backup) + db.execute('BEGIN IMMEDIATE') + db.execute('PRAGMA defer_foreign_keys=ON') + if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok' or db.execute('PRAGMA foreign_key_check').fetchall(): + raise ValueError('Shale database integrity failed') + if db.execute('SELECT 1 FROM issues GROUP BY repo,number HAVING count(*)>1').fetchone(): + raise ValueError('Duplicate issue numbers require separate review') + tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")] + if any(not re.fullmatch('[a-z_]+', table) for table in tables): + raise ValueError('Unexpected table name') + references = {table: [r['from'] for r in db.execute(f'PRAGMA foreign_key_list({table})') if r['table'] == 'issues'] for table in tables} + for table in tables: + for column in db.execute(f'PRAGMA table_info({table})'): + if re.fullmatch('(.*_)?issue(_id)?', column['name']) and column['name'] not in references[table]: + raise ValueError('Undeclared issue reference requires review') + imported = {r[0] for r in db.execute("SELECT target_id FROM studio_forgejo_records WHERE kind='issue'")} + swaps = {} + for repo in db.execute('SELECT DISTINCT repo FROM issues'): + newest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY id DESC LIMIT 1', repo).fetchone() + highest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY number DESC LIMIT 1', repo).fetchone() + if newest['number'] != highest['number']: + if newest['id'] not in imported or highest['id'] not in imported: + raise ValueError('Repair would change a native issue ID') + swaps[newest['id']], swaps[highest['id']] = highest['id'], newest['id'] + + def contents(): + identities = dict(db.execute('SELECT id,uuid FROM issues')) + result = {} + for table in tables: + rows = [] + for record in db.execute(f'SELECT * FROM {table}'): + row = dict(record) + if table == 'issues': + row['id'] = identities[row['id']] if row['id'] in imported else row['id'] + for column in references[table]: + row[column] = identities[row[column]] + if table == 'studio_forgejo_records' and row['kind'] == 'issue': + row['target_id'] = identities[row['target_id']] + rows.append(tuple(row.items())) + result[table] = Counter(rows) + return result + + before = contents() + for old, new in [*((old, -old) for old in swaps), *((-old, new) for old, new in swaps.items())]: + db.execute('UPDATE issues SET id=? WHERE id=?', (new, old)) + for table, columns in references.items(): + for column in columns: + db.execute(f'UPDATE {table} SET {column}=? WHERE {column}=?', (new, old)) + db.execute("UPDATE studio_forgejo_records SET target_id=? WHERE kind='issue' AND target_id=?", (new, old)) + if before != contents() or db.execute('PRAGMA foreign_key_check').fetchall(): + raise ValueError('Repair changed issue content or references') + before_commit() + db.commit() + report = {'surrogate_id_swaps': swaps, 'all_table_contents_preserved': True} + (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report)) + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--target', type=Path, required=True) @@ -405,6 +476,7 @@ def main(): parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source') parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments')) parser.add_argument('--rehearsal', action='store_true') + parser.add_argument('--repair-issue-order', action='store_true') args = parser.parse_args() if os.geteuid() != 0: parser.error('Run as root on Zenith') @@ -429,7 +501,10 @@ def main(): parser.error('Verified export proof is missing or the source checksum differs') guard() evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent))) - migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard) + if args.repair_issue_order: + repair_issue_order(target / 'data/astheno.shale.db', evidence, guard) + else: + migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard) print('Evidence:', evidence) diff --git a/tools/shale-migration.md b/tools/shale-migration.md index b96a3d3247290ad28229b7ed2976a870a5c69643..3a39ff1cddd82233c00081521ad077ad78c85b3a 100644 --- a/tools/shale-migration.md +++ b/tools/shale-migration.md @@ -56,6 +56,8 @@ This older build predates hidden form CSRF tokens. The router requires the exact The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing. +Shale allocates an issue number from the issue with the highest SQLite row ID. The importer now inserts by destination number, including remapped collisions. The already-imported database needs `--repair-issue-order` once while its job is stopped: the guarded repair saves a SQLite backup, swaps six imported surrogate IDs, rewrites foreign keys and ledger references, and verifies every table's contents using issue UUIDs. Native IDs, public numbers, timestamps, comments, labels, attachments and history remain intact. It refuses duplicate numbers or changes to native issue IDs. On the repaired native-auth clone, the actual Rust Backend created `home-infra` #41, `react-mutation` #20 and `chat` #8, then commented on and closed the disposable `home-infra` issue. Fresh import, repeated import, repeated repair, and refusal checks passed. + The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization. Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected. -- 2.54.0