authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 02:12:10-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 15:36:02-07:00
logc14c62225850a095a6e327e5525cea6966a92948
tree1a31620ead71a83bb598237f00f7638f6f9da89c
parent4af52cc7885ec42021a98021be0e06555a5c81d4
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Preserve Shale issue numbering after historical imports

Insert imports by destination issue number and repair imported row ordering with guarded backups and all-table identity checks. Assisted-by: gpt-6

2 files changed, 81 insertions(+), 4 deletions(-)

tools/import-forgejo-issues.py+79-4
......@@ -1,7 +1,7 @@
11#!/usr/bin/env python3
22"""Import a verified personal Forgejo export into stopped Shale, without replacing native issues."""
33import argparse
4from collections import defaultdict
4from collections import Counter, defaultdict
55from datetime import datetime, timezone
66import hashlib
77import grp
......@@ -189,6 +189,7 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
189189 assignees = defaultdict(list)
190190 for row in data['assignees']:
191191 assignees[row['issue_id']].append(users[row['assignee_id']])
192 planned = []
192193 for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])):
193194 repo = repos[sources[row['repo_id']]['name']]
194195 existing = saved('issue', row['id'])
......@@ -198,14 +199,18 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
198199 ceilings[repo['id']] += 1
199200 number = ceilings[repo['id']]
200201 occupied[repo['id']].add(number)
202 else:
203 number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0]
204 planned.append((repo, number, existing, row))
205 # Shale allocates the next number from the issue with the highest row ID.
206 for repo, number, existing, row in sorted(planned, key=lambda item: (item[0]['id'], item[1])):
207 if existing is None:
201208 existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)',
202209 (identifier('issue', row['id'], row['created_unix']), repo['id'], number,
203210 actor(row), timestamp(row['updated_unix']), row['name'],
204211 'done' if row['is_closed'] else 'todo',
205212 assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid
206213 record('issue', row['id'], existing, row)
207 else:
208 number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0]
209214 if number != row['index']:
210215 collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number})
211216 issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'],
......@@ -381,6 +386,9 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
381386 raise ValueError(f'Existing Shale {table} row changed')
382387 if db.execute('PRAGMA foreign_key_check').fetchall():
383388 raise ValueError('Imported data has invalid foreign keys')
389 if db.execute('SELECT 1 FROM issues i WHERE i.id=(SELECT max(id) FROM issues WHERE repo=i.repo) '
390 'AND i.number<>(SELECT max(number) FROM issues WHERE repo=i.repo)').fetchone():
391 raise ValueError('Issue row order would reuse a number; repair it before importing')
384392 for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']),
385393 ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]:
386394 count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0]
......@@ -398,6 +406,69 @@ def migrate(database, source, attachments, target, evidence, before_commit=lambd
398406 print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2))
399407
400408
409def repair_issue_order(database, evidence, before_commit):
410 with sqlite3.connect(database) as db:
411 db.row_factory = sqlite3.Row
412 db.execute('PRAGMA foreign_keys=ON')
413 with sqlite3.connect(evidence / 'before.db') as backup:
414 db.backup(backup)
415 db.execute('BEGIN IMMEDIATE')
416 db.execute('PRAGMA defer_foreign_keys=ON')
417 if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok' or db.execute('PRAGMA foreign_key_check').fetchall():
418 raise ValueError('Shale database integrity failed')
419 if db.execute('SELECT 1 FROM issues GROUP BY repo,number HAVING count(*)>1').fetchone():
420 raise ValueError('Duplicate issue numbers require separate review')
421 tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
422 if any(not re.fullmatch('[a-z_]+', table) for table in tables):
423 raise ValueError('Unexpected table name')
424 references = {table: [r['from'] for r in db.execute(f'PRAGMA foreign_key_list({table})') if r['table'] == 'issues'] for table in tables}
425 for table in tables:
426 for column in db.execute(f'PRAGMA table_info({table})'):
427 if re.fullmatch('(.*_)?issue(_id)?', column['name']) and column['name'] not in references[table]:
428 raise ValueError('Undeclared issue reference requires review')
429 imported = {r[0] for r in db.execute("SELECT target_id FROM studio_forgejo_records WHERE kind='issue'")}
430 swaps = {}
431 for repo in db.execute('SELECT DISTINCT repo FROM issues'):
432 newest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY id DESC LIMIT 1', repo).fetchone()
433 highest = db.execute('SELECT id,number FROM issues WHERE repo=? ORDER BY number DESC LIMIT 1', repo).fetchone()
434 if newest['number'] != highest['number']:
435 if newest['id'] not in imported or highest['id'] not in imported:
436 raise ValueError('Repair would change a native issue ID')
437 swaps[newest['id']], swaps[highest['id']] = highest['id'], newest['id']
438
439 def contents():
440 identities = dict(db.execute('SELECT id,uuid FROM issues'))
441 result = {}
442 for table in tables:
443 rows = []
444 for record in db.execute(f'SELECT * FROM {table}'):
445 row = dict(record)
446 if table == 'issues':
447 row['id'] = identities[row['id']] if row['id'] in imported else row['id']
448 for column in references[table]:
449 row[column] = identities[row[column]]
450 if table == 'studio_forgejo_records' and row['kind'] == 'issue':
451 row['target_id'] = identities[row['target_id']]
452 rows.append(tuple(row.items()))
453 result[table] = Counter(rows)
454 return result
455
456 before = contents()
457 for old, new in [*((old, -old) for old in swaps), *((-old, new) for old, new in swaps.items())]:
458 db.execute('UPDATE issues SET id=? WHERE id=?', (new, old))
459 for table, columns in references.items():
460 for column in columns:
461 db.execute(f'UPDATE {table} SET {column}=? WHERE {column}=?', (new, old))
462 db.execute("UPDATE studio_forgejo_records SET target_id=? WHERE kind='issue' AND target_id=?", (new, old))
463 if before != contents() or db.execute('PRAGMA foreign_key_check').fetchall():
464 raise ValueError('Repair changed issue content or references')
465 before_commit()
466 db.commit()
467 report = {'surrogate_id_swaps': swaps, 'all_table_contents_preserved': True}
468 (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
469 print(json.dumps(report))
470
471
401472def main():
402473 parser = argparse.ArgumentParser()
403474 parser.add_argument('--target', type=Path, required=True)
......@@ -405,6 +476,7 @@ def main():
405476 parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source')
406477 parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments'))
407478 parser.add_argument('--rehearsal', action='store_true')
479 parser.add_argument('--repair-issue-order', action='store_true')
408480 args = parser.parse_args()
409481 if os.geteuid() != 0:
410482 parser.error('Run as root on Zenith')
......@@ -429,7 +501,10 @@ def main():
429501 parser.error('Verified export proof is missing or the source checksum differs')
430502 guard()
431503 evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent)))
432 migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard)
504 if args.repair_issue_order:
505 repair_issue_order(target / 'data/astheno.shale.db', evidence, guard)
506 else:
507 migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard)
433508 print('Evidence:', evidence)
434509
435510
tools/shale-migration.md+2
......@@ -56,6 +56,8 @@ This older build predates hidden form CSRF tokens. The router requires the exact
5656
5757The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing.
5858
59Shale allocates an issue number from the issue with the highest SQLite row ID. The importer now inserts by destination number, including remapped collisions. The already-imported database needs `--repair-issue-order` once while its job is stopped: the guarded repair saves a SQLite backup, swaps six imported surrogate IDs, rewrites foreign keys and ledger references, and verifies every table's contents using issue UUIDs. Native IDs, public numbers, timestamps, comments, labels, attachments and history remain intact. It refuses duplicate numbers or changes to native issue IDs. On the repaired native-auth clone, the actual Rust Backend created `home-infra` #41, `react-mutation` #20 and `chat` #8, then commented on and closed the disposable `home-infra` issue. Fresh import, repeated import, repeated repair, and refusal checks passed.
60
5961The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.
6062
6163Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.