authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 02:12:10-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 15:36:02-07:00
logc14c62225850a095a6e327e5525cea6966a92948
tree1a31620ead71a83bb598237f00f7638f6f9da89c
parent4af52cc7885ec42021a98021be0e06555a5c81d4
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Preserve Shale issue numbering after historical imports

Insert imports by destination issue number and repair imported row ordering with guarded backups and all-table identity checks. Assisted-by: gpt-6

2 files changed, 81 insertions(+), 4 deletions(-)

tools/import-forgejo-issues.py+79-4
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2"""Import a verified personal Forgejo export into stopped Shale, without replacing native issues."""2"""Import a verified personal Forgejo export into stopped Shale, without replacing native issues."""
3import argparse3import argparse
4from collections import defaultdict4from collections import Counter, defaultdict
5from datetime import datetime, timezone5from datetime import datetime, timezone
6import hashlib6import hashlib
7import grp7import grp
...@@ -189,6 +189,7 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd...@@ -189,6 +189,7 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
189 assignees = defaultdict(list)189 assignees = defaultdict(list)
190 for row in data['assignees']:190 for row in data['assignees']:
191 assignees[row['issue_id']].append(users[row['assignee_id']])191 assignees[row['issue_id']].append(users[row['assignee_id']])
192 planned = []
192 for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])):193 for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])):
193 repo = repos[sources[row['repo_id']]['name']]194 repo = repos[sources[row['repo_id']]['name']]
194 existing = saved('issue', row['id'])195 existing = saved('issue', row['id'])
...@@ -198,14 +199,18 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd...@@ -198,14 +199,18 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
198 ceilings[repo['id']] += 1199 ceilings[repo['id']] += 1
199 number = ceilings[repo['id']]200 number = ceilings[repo['id']]
200 occupied[repo['id']].add(number)201 occupied[repo['id']].add(number)
202 else:
203 number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0]
204 planned.append((repo, number, existing, row))
205 # Shale allocates the next number from the issue with the highest row ID.
206 for repo, number, existing, row in sorted(planned, key=lambda item: (item[0]['id'], item[1])):
207 if existing is None:
201 existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)',208 existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)',
202 (identifier('issue', row['id'], row['created_unix']), repo['id'], number,209 (identifier('issue', row['id'], row['created_unix']), repo['id'], number,
203 actor(row), timestamp(row['updated_unix']), row['name'],210 actor(row), timestamp(row['updated_unix']), row['name'],
204 'done' if row['is_closed'] else 'todo',211 'done' if row['is_closed'] else 'todo',
205 assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid212 assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid
206 record('issue', row['id'], existing, row)213 record('issue', row['id'], existing, row)
207 else:
208 number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0]
209 if number != row['index']:214 if number != row['index']:
210 collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number})215 collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number})
211 issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'],216 issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'],
...@@ -381,6 +386,9 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd...@@ -381,6 +386,9 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
381 raise ValueError(f'Existing Shale {table} row changed')386 raise ValueError(f'Existing Shale {table} row changed')
382 if db.execute('PRAGMA foreign_key_check').fetchall():387 if db.execute('PRAGMA foreign_key_check').fetchall():
383 raise ValueError('Imported data has invalid foreign keys')388 raise ValueError('Imported data has invalid foreign keys')
389 if db.execute('SELECT 1 FROM issues i WHERE i.id=(SELECT max(id) FROM issues WHERE repo=i.repo) '
390 'AND i.number<>(SELECT max(number) FROM issues WHERE repo=i.repo)').fetchone():
391 raise ValueError('Issue row order would reuse a number; repair it before importing')
384 for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']),392 for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']),
385 ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]:393 ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]:
386 count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0]394 count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0]
...@@ -398,6 +406,69 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd...@@ -398,6 +406,69 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
398 print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2))406 print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2))
399407
400408
409def repair_issue_order(database, evidence, before_commit):
410 with sqlite3.connect(database) as db:
411 db.row_factory = sqlite3.Row
412 db.execute('PRAGMA foreign_keys=ON')
413 with sqlite3.connect(evidence / 'before.db') as backup:
414 db.backup(backup)
415 db.execute('BEGIN IMMEDIATE')
416 db.execute('PRAGMA defer_foreign_keys=ON')
417 if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok' or db.execute('PRAGMA foreign_key_check').fetchall():
418 raise ValueError('Shale database integrity failed')
419 if db.execute('SELECT 1 FROM issues GROUP BY repo,number HAVING count(*)>1').fetchone():
420 raise ValueError('Duplicate issue numbers require separate review')
421 tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
422 if any(not re.fullmatch('[a-z_]+', table) for table in tables):
423 raise ValueError('Unexpected table name')
424 references = {table: [r['from'] for r in db.execute(f'PRAGMA foreign_key_list({table})') if r['table'] == 'issues'] for table in tables}
425 for table in tables:
426 for column in db.execute(f'PRAGMA table_info({table})'):
427 if re.fullmatch('(.*_)?issue(_id)?', column['name']) and column['name'] not in references[table]:
428 raise ValueError('Undeclared issue reference requires review')
429 imported = {r[0] for r in db.execute("SELECT target_id FROM studio_forgejo_records WHERE kind='issue'")}
430 swaps = {}
431 for repo in db.execute('SELECT DISTINCT repo FROM issues'):
432 newest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY id DESC LIMIT 1', repo).fetchone()
433 highest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY number DESC LIMIT 1', repo).fetchone()
434 if newest['number'] != highest['number']:
435 if newest['id'] not in imported or highest['id'] not in imported:
436 raise ValueError('Repair would change a native issue ID')
437 swaps[newest['id']], swaps[highest['id']] = highest['id'], newest['id']
438
439 def contents():
440 identities = dict(db.execute('SELECT id,uuid FROM issues'))
441 result = {}
442 for table in tables:
443 rows = []
444 for record in db.execute(f'SELECT * FROM {table}'):
445 row = dict(record)
446 if table == 'issues':
447 row['id'] = identities[row['id']] if row['id'] in imported else row['id']
448 for column in references[table]:
449 row[column] = identities[row[column]]
450 if table == 'studio_forgejo_records' and row['kind'] == 'issue':
451 row['target_id'] = identities[row['target_id']]
452 rows.append(tuple(row.items()))
453 result[table] = Counter(rows)
454 return result
455
456 before = contents()
457 for old, new in [*((old, -old) for old in swaps), *((-old, new) for old, new in swaps.items())]:
458 db.execute('UPDATE issues SET id=? WHERE id=?', (new, old))
459 for table, columns in references.items():
460 for column in columns:
461 db.execute(f'UPDATE {table} SET {column}=? WHERE {column}=?', (new, old))
462 db.execute("UPDATE studio_forgejo_records SET target_id=? WHERE kind='issue' AND target_id=?", (new, old))
463 if before != contents() or db.execute('PRAGMA foreign_key_check').fetchall():
464 raise ValueError('Repair changed issue content or references')
465 before_commit()
466 db.commit()
467 report = {'surrogate_id_swaps': swaps, 'all_table_contents_preserved': True}
468 (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
469 print(json.dumps(report))
470
471
401def main():472def main():
402 parser = argparse.ArgumentParser()473 parser = argparse.ArgumentParser()
403 parser.add_argument('--target', type=Path, required=True)474 parser.add_argument('--target', type=Path, required=True)
...@@ -405,6 +476,7 @@ def main():...@@ -405,6 +476,7 @@ def main():
405 parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source')476 parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source')
406 parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments'))477 parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments'))
407 parser.add_argument('--rehearsal', action='store_true')478 parser.add_argument('--rehearsal', action='store_true')
479 parser.add_argument('--repair-issue-order', action='store_true')
408 args = parser.parse_args()480 args = parser.parse_args()
409 if os.geteuid() != 0:481 if os.geteuid() != 0:
410 parser.error('Run as root on Zenith')482 parser.error('Run as root on Zenith')
...@@ -429,7 +501,10 @@ def main():...@@ -429,7 +501,10 @@ def main():
429 parser.error('Verified export proof is missing or the source checksum differs')501 parser.error('Verified export proof is missing or the source checksum differs')
430 guard()502 guard()
431 evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent)))503 evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent)))
432 migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard)504 if args.repair_issue_order:
505 repair_issue_order(target / 'data/astheno.shale.db', evidence, guard)
506 else:
507 migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard)
433 print('Evidence:', evidence)508 print('Evidence:', evidence)
434509
435510
tools/shale-migration.md+2
...@@ -56,6 +56,8 @@ This older build predates hidden form CSRF tokens. The router requires the exact...@@ -56,6 +56,8 @@ This older build predates hidden form CSRF tokens. The router requires the exact
5656
57The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing.57The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing.
5858
59Shale allocates an issue number from the issue with the highest SQLite row ID. The importer now inserts by destination number, including remapped collisions. The already-imported database needs `--repair-issue-order` once while its job is stopped: the guarded repair saves a SQLite backup, swaps six imported surrogate IDs, rewrites foreign keys and ledger references, and verifies every table's contents using issue UUIDs. Native IDs, public numbers, timestamps, comments, labels, attachments and history remain intact. It refuses duplicate numbers or changes to native issue IDs. On the repaired native-auth clone, the actual Rust Backend created `home-infra` #41, `react-mutation` #20 and `chat` #8, then commented on and closed the disposable `home-infra` issue. Fresh import, repeated import, repeated repair, and refusal checks passed.
60
59The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.61The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.
6062
61Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.63Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.