authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 00:27:24-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 15:36:02-07:00
log4dd72bcd7c26fc5de14d5198985891f111f1f29f
treeffad87e48fcc0c6f07b0a7e4a9d6d6590a9cfb23
parentbc6dafcf6a1a26d0c997fae4d9aa887de5a7085c
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Migrate personal Forgejo issues and deploy stable Shale from committed main

5 files changed, 514 insertions(+), 2 deletions(-)

service/shale/service.pkl+3-1
...@@ -14,7 +14,7 @@ requirements {...@@ -14,7 +14,7 @@ requirements {
14}14}
1515
16container {16container {
17 image = "docker.io/astheno/shale@sha256:ece987e25ebe67275fbe105b1b03e650a9b3adfe716928beb717cde1d32c9652"17 image = "docker.io/astheno/shale@sha256:dfffceebe31fd3360b6dcf12caab664735415fdc32effd5082ac37b11864a232"
18 extraHosts { "\(keycloak.container.http.hostname):host-gateway" }18 extraHosts { "\(keycloak.container.http.hostname):host-gateway" }
1919
20 http {20 http {
...@@ -51,6 +51,8 @@ container {...@@ -51,6 +51,8 @@ container {
51 ["DOMAIN"] = module.container.http.hostname51 ["DOMAIN"] = module.container.http.hostname
52 ["HOME"] = "/data"52 ["HOME"] = "/data"
53 ["SERVER_TITLE"] = "clover's git"53 ["SERVER_TITLE"] = "clover's git"
54 // The older Markdown parser shares a capture buffer between request workers.
55 ["NPROC"] = "1"
54 ["SESSION_SECRET"] = "${secret.own.session_secret}"56 ["SESSION_SECRET"] = "${secret.own.session_secret}"
55 ["OAUTH2_CLIENT"] = "oidc,\(keycloak.container.http.hostname)/realms/master|${secret.oidc.clientId}|${secret.oidc.clientSecret}"57 ["OAUTH2_CLIENT"] = "oidc,\(keycloak.container.http.hostname)/realms/master|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
56 }58 }
tools/deploy.py+8-1
...@@ -44,7 +44,7 @@ def sync_manager(release):...@@ -44,7 +44,7 @@ def sync_manager(release):
4444
4545
46def upload(main=False):46def upload(main=False):
47 excluded = excluded_services(REPO)47 excluded = set() if main else excluded_services(REPO)
48 with tempfile.TemporaryDirectory() as temporary:48 with tempfile.TemporaryDirectory() as temporary:
49 snapshot = Path(temporary)49 snapshot = Path(temporary)
50 revision = None50 revision = None
...@@ -58,6 +58,13 @@ def upload(main=False):...@@ -58,6 +58,13 @@ def upload(main=False):
58 if conflicted or not description.strip():58 if conflicted or not description.strip():
59 raise ValueError("main needs a commit description and no conflicts before deployment")59 raise ValueError("main needs a commit description and no conflicts before deployment")
60 revision = {"commit": commit, "description": description}60 revision = {"commit": commit, "description": description}
61 # Production must use main's exclusions too. A working staging
62 # change must not remove dependencies from the committed release.
63 excluded = set(json.loads(subprocess.run(
64 ["jj", "--ignore-working-copy", "file", "show", "-r", commit,
65 "config/excluded-services.json"],
66 cwd=REPO, check=True, capture_output=True, text=True,
67 ).stdout))
61 entries = subprocess.run(68 entries = subprocess.run(
62 ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T",69 ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T",
63 '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"',70 '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"',
tools/import-forgejo-issues.py created+437
...@@ -0,0 +1,437 @@
1#!/usr/bin/env python3
2"""Import a verified personal Forgejo export into stopped Shale, without replacing native issues."""
3import argparse
4from collections import defaultdict
5from datetime import datetime, timezone
6import hashlib
7import grp
8import json
9import os
10from pathlib import Path
11import re
12import shutil
13import sqlite3
14import sys
15import tempfile
16import urllib.request
17from urllib.parse import quote
18
19ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'
20EPOCH = 1577836800 # Shale's ULIDs use 2020-01-01, rather than the Unix epoch.
21CLOSED = ('done', 'not_planned', 'duplicate', 'invalid')
22EVENTS = ['comment', 'reopened', 'closed', 'issue reference', 'commit reference',
23 'comment reference', 'pull request reference', 'label changed', 'milestone changed',
24 'assignee changed', 'title changed', 'branch deleted', 'time tracking started',
25 'time tracking stopped', 'time added', 'time tracking canceled', 'deadline added',
26 'deadline changed', 'deadline removed', 'dependency added', 'dependency removed',
27 'code comment', 'review', 'locked', 'unlocked', 'target branch changed',
28 'time deleted', 'review requested', 'merged', 'pull request updated',
29 'project changed', 'project column changed', 'review dismissed', 'reference changed',
30 'automatic merge scheduled', 'automatic merge canceled', 'pinned', 'unpinned']
31
32
33def timestamp(epoch):
34 return datetime.fromtimestamp(int(epoch), timezone.utc).isoformat(timespec='seconds')
35
36
37def identifier(kind, key, epoch):
38 milliseconds = (int(epoch) - EPOCH) * 1000
39 if not 0 <= milliseconds < 2 ** 48:
40 raise ValueError('Source creation time is outside Shale ULID range')
41 entropy = int.from_bytes(hashlib.sha256(f'personal-forgejo:{kind}:{key}'.encode()).digest()[:10], 'big')
42 value = (milliseconds << 80) | entropy
43 return ''.join(ALPHABET[(value >> (5 * i)) & 31] for i in range(25, -1, -1))
44
45
46def digest(path):
47 return hashlib.sha256(path.read_bytes()).hexdigest()
48
49
50def stopped(job):
51 token = os.environ.get('NOMAD_TOKEN') or Path('/var/lib/studio/nomad.token').read_text().strip()
52 def read(path):
53 request = urllib.request.Request('http://127.0.0.1:4646/v1/' + path,
54 headers={'X-Nomad-Token': token})
55 with urllib.request.urlopen(request, timeout=10) as response:
56 return json.load(response)
57 if not read('job/' + job).get('Stop') or any(
58 item['ClientStatus'] in ('pending', 'running') for item in read('job/' + job + '/allocations')):
59 raise ValueError('Stop the target Shale job and wait for its allocations before importing')
60
61
62def migrate(database, source, attachments, target, evidence, before_commit=lambda: None):
63 db = sqlite3.connect(database)
64 db.row_factory = sqlite3.Row
65 db.execute('PRAGMA foreign_keys=ON')
66 if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
67 raise ValueError('Shale SQLite integrity check failed')
68 source_hash = digest(source)
69 data = json.loads(source.read_text())
70 for field in ['repositories', 'issues', 'comments', 'labels', 'issue_labels', 'authors',
71 'attachments', 'assignees', 'milestones', 'pull_requests', 'history']:
72 if not isinstance(data.get(field), list):
73 raise ValueError(f'Missing source table: {field}')
74 owners = db.execute("SELECT * FROM users WHERE name='clover'").fetchall()
75 if len(owners) != 1 or owners[0]['snowflake'] != '8c909fab-98b0-4472-9171-9606ccebe9d5':
76 raise ValueError('Clover original identity is missing')
77 owner = owners[0]['id']
78 repos = {r['name']: dict(r) for r in db.execute('SELECT * FROM repositories')}
79 sources = {r['id']: r for r in data['repositories']}
80 issue_sources = {r['id']: r for r in data['issues']}
81 if len(issue_sources) != len(data['issues']):
82 raise ValueError('Duplicate source issue IDs')
83 for r in data['repositories']:
84 if r['name'] not in repos or repos[r['name']]['owner'] != owner:
85 raise ValueError(f'Expected imported Clover repository: {r["name"]}')
86 if any(c['issue_id'] not in issue_sources for c in data['comments']):
87 raise ValueError('Orphan source comment')
88 with sqlite3.connect(evidence / 'before.db') as backup:
89 db.backup(backup)
90 originals = {table: [dict(r) for r in db.execute(f'SELECT * FROM {table}')]
91 for table in ['issues', 'issue_actions', 'issue_labels', 'issues__labels', 'users']}
92 db.execute('BEGIN IMMEDIATE')
93 db.execute('CREATE TABLE IF NOT EXISTS studio_forgejo_records('
94 'kind TEXT NOT NULL, source_id TEXT NOT NULL, target_id INTEGER, '
95 'source_json TEXT NOT NULL, PRIMARY KEY(kind,source_id))')
96 previous = db.execute("SELECT source_json FROM studio_forgejo_records WHERE kind='export' AND source_id='personal'").fetchone()
97 if previous and json.loads(previous[0])['sha256'] != source_hash:
98 raise ValueError('A different export was already imported; review the source before merging')
99
100 def saved(kind, key):
101 row = db.execute('SELECT target_id FROM studio_forgejo_records WHERE kind=? AND source_id=?', (kind, str(key))).fetchone()
102 return row[0] if row else None
103
104 def record(kind, key, target_id, row):
105 db.execute('INSERT OR IGNORE INTO studio_forgejo_records VALUES(?,?,?,?)',
106 (kind, str(key), target_id, json.dumps(row, ensure_ascii=False, sort_keys=True)))
107
108 users = {1: owner}
109 names = {r['name'] for r in db.execute('SELECT name FROM users')}
110 actors = {int(r.get('poster_id') or 0) for r in [*data['issues'], *data['comments'], *data['history']]}
111 actors.update(int(r['assignee_id']) for r in data['assignees'])
112 author_source = {r['id']: r for r in data['authors']}
113 for key in sorted(actors - {1}):
114 row = author_source.get(key, {'id': key, 'name': 'forgejo-system' if key == 0 else f'forgejo-user-{key}',
115 'created_unix': min(i['created_unix'] for i in data['issues']),
116 'updated_unix': max(i['updated_unix'] for i in data['issues'])})
117 existing = saved('user', key)
118 if existing is None:
119 name = row['name']
120 if name in names:
121 name = 'forgejo-' + name
122 if name in names:
123 raise ValueError('Historical author name collision')
124 cursor = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)',
125 (identifier('user', key, row['created_unix']), 'personal-forgejo.invalid',
126 str(key), name, timestamp(row['created_unix']), timestamp(row['updated_unix'])))
127 existing = cursor.lastrowid
128 names.add(name)
129 record('user', key, existing, row)
130 users[key] = existing
131 record('user', 1, owner, author_source[1])
132
133 external = {'paperclover': owner}
134 external_rows = defaultdict(list)
135 for row in [*data['issues'], *data['comments']]:
136 if row.get('original_author'):
137 external_rows[row['original_author']].append(row)
138 for name, rows in external_rows.items():
139 if name == 'paperclover':
140 continue
141 if not re.fullmatch(r'[A-Za-z0-9_.-]+', name):
142 raise ValueError('Unsupported external author name')
143 key = 'external:' + name
144 existing = saved('user', key)
145 if existing is None:
146 display = name if name not in names else 'github-' + name
147 if display in names:
148 raise ValueError('External author name collision')
149 created = min(row['created_unix'] for row in rows)
150 updated = max(row['updated_unix'] or row['created_unix'] for row in rows)
151 existing = db.execute('INSERT INTO users(uuid,provider,snowflake,name,joined_on,last_updated) VALUES(?,?,?,?,?,?)',
152 (identifier('user', key, created), 'personal-forgejo-external.invalid', name,
153 display, timestamp(created), timestamp(updated))).lastrowid
154 names.add(display)
155 record('user', key, existing, {'original_author': name})
156 external[name] = existing
157
158 def actor(row):
159 return external[row['original_author']] if row.get('original_author') else users[row['poster_id']]
160
161 labels = {}
162 for row in sorted(data['labels'], key=lambda r: r['id']):
163 if row['repo_id'] not in sources:
164 raise ValueError('Label has no source repository')
165 repo = repos[sources[row['repo_id']]['name']]
166 existing = saved('label', row['id'])
167 if existing is None:
168 color = row['color'].lstrip('#')
169 if not re.fullmatch('[0-9a-fA-F]{6}', color):
170 raise ValueError('Invalid source label color')
171 epoch = row['created_unix'] or sources[row['repo_id']]['created_unix']
172 existing = db.execute('INSERT INTO issue_labels(uuid,repo,name,description,color,last_updated) VALUES(?,?,?,?,?,?)',
173 (identifier('label', row['id'], epoch), repo['id'], row['name'], row['description'],
174 '#' + color, timestamp(row['updated_unix'] or epoch))).lastrowid
175 record('label', row['id'], existing, row)
176 labels[row['id']] = existing
177
178 # Preserve incoming numbers when free; collisions go above BOTH namespaces.
179 occupied = defaultdict(set)
180 ceilings = defaultdict(int)
181 for row in db.execute('SELECT repo,number FROM issues'):
182 occupied[row['repo']].add(row['number'])
183 ceilings[row['repo']] = max(ceilings[row['repo']], row['number'])
184 for row in data['issues']:
185 repo = repos[sources[row['repo_id']]['name']]['id']
186 ceilings[repo] = max(ceilings[repo], row['index'])
187 issue_map = {}
188 collisions = []
189 assignees = defaultdict(list)
190 for row in data['assignees']:
191 assignees[row['issue_id']].append(users[row['assignee_id']])
192 for row in sorted(data['issues'], key=lambda r: (r['repo_id'], r['index'])):
193 repo = repos[sources[row['repo_id']]['name']]
194 existing = saved('issue', row['id'])
195 number = row['index']
196 if existing is None:
197 if number in occupied[repo['id']]:
198 ceilings[repo['id']] += 1
199 number = ceilings[repo['id']]
200 occupied[repo['id']].add(number)
201 existing = db.execute('INSERT INTO issues(uuid,repo,number,author,last_updated,title,status,assignee) VALUES(?,?,?,?,?,?,?,?)',
202 (identifier('issue', row['id'], row['created_unix']), repo['id'], number,
203 actor(row), timestamp(row['updated_unix']), row['name'],
204 'done' if row['is_closed'] else 'todo',
205 assignees[row['id']][0] if len(assignees[row['id']]) == 1 else None)).lastrowid
206 record('issue', row['id'], existing, row)
207 else:
208 number = db.execute('SELECT number FROM issues WHERE id=?', (existing,)).fetchone()[0]
209 if number != row['index']:
210 collisions.append({'repo': repo['name'], 'forgejo': row['index'], 'shale': number})
211 issue_map[row['id']] = {'id': existing, 'number': number, 'repo': repo['name'],
212 'url': '/' + quote(repo['name'], safe='/') + '/issues/' + str(number)}
213
214 # Preserve attachment authorization by checking the associated Shale issue
215 # before Caddy serves the copied file. No unauthenticated static directory.
216 attachment_map = {}
217 manifest = []
218 for row in data['attachments']:
219 if row['issue_id'] not in issue_map:
220 raise ValueError('An attachment belongs to an unsupported release or missing issue')
221 if row['external_url']:
222 attachment_map[row['uuid']] = row['external_url']
223 record('attachment', row['id'], None, row)
224 continue
225 key = row['uuid']
226 if not re.fullmatch('[0-9a-f-]{36}', key):
227 raise ValueError('Unsafe attachment UUID')
228 source_file = attachments / key[0] / key[1] / key
229 if source_file.is_symlink() or not source_file.is_file() or source_file.stat().st_size != row['size']:
230 raise ValueError('Source attachment missing or size differs')
231 name = Path(row['name']).name
232 if name != row['name'] or not name or any(c in name for c in '\\"\r\n'):
233 raise ValueError('Unsafe attachment filename')
234 destination = target / 'forgejo-attachments' / key / name
235 destination.parent.mkdir(parents=True, exist_ok=True)
236 if destination.exists() and digest(destination) != digest(source_file):
237 raise ValueError('Attachment copy differs')
238 if not destination.exists():
239 shutil.copyfile(source_file, destination)
240 checksum = digest(source_file)
241 if digest(destination) != checksum:
242 raise ValueError('Attachment checksum failed')
243 # Caddy serves these only after Shale authorizes the associated issue.
244 # Keep them unavailable to other local users.
245 caddy_group = grp.getgrnam('caddy').gr_gid
246 for directory in [target / 'forgejo-attachments', destination.parent]:
247 os.chown(directory, 0, caddy_group)
248 directory.chmod(0o750)
249 os.chown(destination, 0, caddy_group)
250 destination.chmod(0o640)
251 path = '/-/forgejo-attachments/' + key + '/' + quote(name, safe='')
252 attachment_map[key] = path
253 manifest.append({'path': path, 'file': key + '/' + name,
254 'issue': issue_map[row['issue_id']]['url'], 'sha256': checksum})
255 record('attachment', row['id'], None, row)
256
257 source_urls = {}
258 for row in data['issues']:
259 repo = sources[row['repo_id']]
260 owner_name = repo['owner_name']
261 for kind in ['issues', 'pulls']:
262 source_urls[f'/{owner_name}/{repo["name"]}/{kind}/{row["index"]}'] = issue_map[row['id']]['url']
263
264 def rewrite(text, repo_id):
265 def attachment(match):
266 key = match.group(1)
267 if key not in attachment_map:
268 raise ValueError('Issue references an attachment absent from the source export')
269 return attachment_map[key]
270 text = re.sub(r'(?:https?://(?:git|forgejo)\.paperclover\.net)?/attachments/([0-9a-f-]{36})', attachment, text)
271 def link(match):
272 return source_urls.get(match.group(1), match.group(0))
273 text = re.sub(r'https?://(?:git|forgejo)\.paperclover\.net(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)', link, text)
274 text = re.sub(r'(?<=\]\()(/[^\s)<>]+/(?:issues|pulls)/[0-9]+)(?=\))', link, text)
275 # Leave code, quoted text, and bare #references untouched; map explicit links only.
276 return text
277
278 def action(kind, key, issue_id, actor, payload, added, updated=None, source_row=None):
279 existing = saved(kind, key)
280 if existing is not None:
281 return existing
282 created = db.execute('INSERT INTO issue_actions(uuid,issue,actor,kind,payload,added_on,last_updated) VALUES(?,?,?,?,?,?,?)',
283 (identifier(kind, key, added), issue_id, actor, kind if kind in ['comment', 'update_status', 'add_label', 'remove_label'] else 'comment',
284 payload, timestamp(added), timestamp(updated or added))).lastrowid
285 record(kind, key, created, source_row or {})
286 return created
287
288 milestone_source = {r['id']: r for r in data['milestones']}
289 pulls = {r['issue_id']: r for r in data['pull_requests']}
290 body_edits = defaultdict(int)
291 for revision in data['history']:
292 if not revision['comment_id'] and not revision['is_first_created']:
293 body_edits[revision['issue_id']] = max(body_edits[revision['issue_id']], revision['edited_unix'])
294 for row in data['issues']:
295 item = issue_map[row['id']]
296 body = rewrite(row['content'], row['repo_id'])
297 metadata = []
298 if item['number'] != row['index']:
299 metadata.append(f'Original Forgejo issue #{row["index"]}.')
300 if row['is_pull']:
301 pr = pulls[row['id']]
302 metadata.append(f'Imported pull request: `{pr["head_branch"]}` → `{pr["base_branch"]}`.' +
303 (f' Merged as `{pr["merged_commit_id"]}`.' if pr['has_merged'] else ''))
304 if row['milestone_id']:
305 metadata.append('Milestone: ' + milestone_source[row['milestone_id']]['name'] + '.')
306 if row['is_locked']:
307 metadata.append('The original discussion was locked.')
308 if len(assignees[row['id']]) > 1:
309 original_assignees = [author_source[a['assignee_id']]['name'] for a in data['assignees'] if a['issue_id'] == row['id']]
310 metadata.append('Original assignees: ' + ', '.join(original_assignees) + '.')
311 if metadata:
312 body += '\n\n---\n\n' + '\n\n'.join(metadata)
313 action('issue-body', row['id'], item['id'], actor(row), body,
314 row['created_unix'], max(row['created_unix'], body_edits[row['id']]), row)
315 # Shale's status-change renderer requires an initial status action.
316 # Forgejo stores initial state on the issue, rather than as a comment.
317 action('update_status', 'initial-' + str(row['id']), item['id'], actor(row),
318 'todo', row['created_unix'])
319
320 for row in sorted(data['comments'], key=lambda r: (r['created_unix'], r['id'])):
321 item = issue_map[row['issue_id']]
322 kind = 'comment'
323 payload = rewrite(row['content'], issue_sources[row['issue_id']]['repo_id'])
324 if row['type'] in [1, 2]:
325 kind, payload = 'update_status', 'todo' if row['type'] == 1 else 'done'
326 elif row['type'] == 7 and row['label_id'] in labels:
327 kind = 'add_label' if row['content'] == '1' else 'remove_label'
328 payload = str(labels[row['label_id']])
329 elif row['type'] != 0:
330 event = EVENTS[row['type']] if row['type'] < len(EVENTS) else f'event {row["type"]}'
331 detail = []
332 for field in ['old_title', 'new_title', 'old_ref', 'new_ref', 'commit_sha', 'commit_id', 'tree_path', 'line']:
333 if row.get(field):
334 detail.append(f'{field}: {row[field]}')
335 for field in ['old_milestone_id', 'milestone_id']:
336 if row.get(field):
337 detail.append(field + ': ' + milestone_source.get(row[field], {}).get('name', str(row[field])))
338 if row.get('dependent_issue_id'):
339 linked = issue_map.get(row['dependent_issue_id'])
340 detail.append('Dependency: ' + (f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})' if linked else str(row['dependent_issue_id'])))
341 if row.get('ref_issue_id') and row['ref_issue_id'] in issue_map:
342 linked = issue_map[row['ref_issue_id']]
343 detail.append(f'[{linked["repo"]}#{linked["number"]}]({linked["url"]})')
344 payload = 'Forgejo: ' + event + '.' + ('\n\n' + '\n\n'.join(detail) if detail else '') + ('\n\n' + payload if payload else '')
345 existing = saved('forgejo-comment', row['id'])
346 if existing is None:
347 # Every original comment/event maps to exactly one Shale action.
348 action_id = action(kind, 'forgejo-' + str(row['id']), item['id'], actor(row), payload,
349 row['created_unix'], row['updated_unix'], row)
350 record('forgejo-comment', row['id'], action_id, row)
351
352 for row in data['issue_labels']:
353 item = issue_map[row['issue_id']]
354 existing = saved('issue-label', row['id'])
355 if existing is None:
356 label = labels[row['label_id']]
357 epoch = issue_sources[row['issue_id']]['updated_unix']
358 existing = db.execute('INSERT INTO issues__labels(uuid,issue,label) VALUES(?,?,?)',
359 (identifier('issue-label', row['id'], epoch), item['id'], label)).lastrowid
360 record('issue-label', row['id'], existing, row)
361
362 for table in ['history', 'milestones', 'pull_requests', 'assignees']:
363 for row in data[table]:
364 record(table, row['id'], None, row)
365 for source_repo in data['repositories']:
366 repo = repos[source_repo['name']]
367 if any(i['repo_id'] == source_repo['id'] for i in data['issues']):
368 db.execute("UPDATE repositories SET access_issues=? WHERE id=? AND access_issues='off'",
369 ('private' if source_repo['is_private'] else 'public', repo['id']))
370 for field in ['access_issues_submit', 'access_issues_comment']:
371 db.execute(f"UPDATE repositories SET {field}='private' WHERE id=? AND {field}='off'", (repo['id'],))
372 slots = ','.join('?' for _ in CLOSED)
373 db.execute(f'UPDATE repositories SET open_issues=(SELECT count(*) FROM issues WHERE repo=repositories.id AND status NOT IN ({slots}))', CLOSED)
374 db.execute(f'UPDATE issue_labels SET open_issues=(SELECT count(*) FROM issues__labels il JOIN issues i ON i.id=il.issue WHERE il.label=issue_labels.id AND i.status NOT IN ({slots}))', CLOSED)
375 for table, rows in originals.items():
376 for row in rows:
377 current = dict(db.execute(f'SELECT * FROM {table} WHERE id=?', (row['id'],)).fetchone())
378 if table == 'issue_labels':
379 current.pop('open_issues');row = {k:v for k,v in row.items() if k != 'open_issues'}
380 if current != row:
381 raise ValueError(f'Existing Shale {table} row changed')
382 if db.execute('PRAGMA foreign_key_check').fetchall():
383 raise ValueError('Imported data has invalid foreign keys')
384 for kind, rows in [('issue', data['issues']), ('forgejo-comment', data['comments']), ('label', data['labels']),
385 ('issue-label', data['issue_labels']), ('attachment', data['attachments']), ('history', data['history'])]:
386 count = db.execute('SELECT count(*) FROM studio_forgejo_records WHERE kind=?', (kind,)).fetchone()[0]
387 if count != len(rows):
388 raise ValueError(f'Incomplete import: {kind}')
389 record('export', 'personal', None, {'sha256': source_hash})
390 before_commit()
391 db.commit()
392 db.close()
393 report = {'source_sha256': source_hash, 'issues': len(data['issues']), 'comments_and_events': len(data['comments']),
394 'labels': len(labels), 'attachments': len(manifest), 'history_revisions': len(data['history']),
395 'number_collisions': collisions, 'issue_mapping': issue_map}
396 (evidence / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
397 (target / 'forgejo-attachments.json').write_text(json.dumps(manifest, indent=2) + '\n')
398 print(json.dumps({k:v for k,v in report.items() if k != 'issue_mapping'}, indent=2))
399
400
401def main():
402 parser = argparse.ArgumentParser()
403 parser.add_argument('--target', type=Path, required=True)
404 parser.add_argument('--source', type=Path, required=True)
405 parser.add_argument('--proof', type=Path, help='Verified export proof; defaults to source-proof.json beside the source')
406 parser.add_argument('--attachments', type=Path, default=Path('/mnt/storage1/apps/forgejo/work/attachments'))
407 parser.add_argument('--rehearsal', action='store_true')
408 args = parser.parse_args()
409 if os.geteuid() != 0:
410 parser.error('Run as root on Zenith')
411 os.umask(0o077)
412 target = args.target.resolve()
413 if 'evil' in str(target).lower() or target.is_relative_to('/mnt/storage1/apps'):
414 parser.error('Use a managed Shale target or isolated copy')
415 if args.rehearsal:
416 if target.is_relative_to('/srv/prod'):
417 parser.error('Rehearsal must use an isolated target')
418 elif target != Path('/srv/prod/shale'):
419 parser.error('Production imports must target /srv/prod/shale')
420 guard = lambda: None
421 if target == Path('/srv/prod/shale'):
422 guard = lambda: stopped('shale')
423 elif target.parent == Path('/srv/staging'):
424 if not re.fullmatch(r'shale-preview-[0-9a-f]{8}', target.name):
425 parser.error('Expected a managed Shale stage')
426 guard = lambda: stopped(target.name)
427 proof = args.proof or args.source.parent / 'source-proof.json'
428 if not proof.is_file() or json.loads(proof.read_text()).get('sourceSha256') != digest(args.source):
429 parser.error('Verified export proof is missing or the source checksum differs')
430 guard()
431 evidence = Path(tempfile.mkdtemp(prefix='issue-import-', dir=str(args.source.parent)))
432 migrate(target / 'data/astheno.shale.db', args.source, args.attachments, target, evidence, guard)
433 print('Evidence:', evidence)
434
435
436if __name__ == '__main__':
437 main()
tools/router.py+48
...@@ -45,6 +45,44 @@ def shale_mcp_routes(port):...@@ -45,6 +45,44 @@ def shale_mcp_routes(port):
45 *proxy(f"127.0.0.1:{port}", " "), " }"]45 *proxy(f"127.0.0.1:{port}", " "), " }"]
4646
4747
48def shale_attachment_routes(upstreams, target="/srv/prod/shale"):
49 """Authorize retained attachments through their issue before serving bytes."""
50 manifest = os.path.join(target, "forgejo-attachments.json")
51 if not os.path.exists(manifest):
52 return []
53 with open(manifest) as source:
54 entries = json.load(source)
55 if not isinstance(entries, list) or len(entries) > 10000:
56 raise ValueError("invalid Shale attachment manifest")
57 lines = []
58 for index, entry in enumerate(entries):
59 path, filename, issue = (entry[k] for k in ("path", "file", "issue"))
60 if (not re.fullmatch(r"[0-9a-f-]{36}/[^/\\\"\r\n]+", filename)
61 or path != "/-/forgejo-attachments/" + urllib.parse.quote(filename, safe="/")
62 or not re.fullmatch(r"/(?:[a-zA-Z0-9_.-]+/)+issues/[1-9][0-9]*", issue)):
63 raise ValueError("unsafe Shale attachment route")
64 # Request cookies go to Shale as usual. A private issue yields 403/404;
65 # only Shale's successful issue response permits the local file read.
66 name = f"@shale_attachment_{index}"
67 lines += [f" {name} path {json.dumps(path)}", f" handle {name} {{",
68 " header Cache-Control private,no-store", " header X-Content-Type-Options nosniff",
69 f" reverse_proxy {upstreams} {{", " method GET", f" rewrite {issue}",
70 " @authorized status 200", " handle_response @authorized {",
71 f" root * {json.dumps(os.path.join(target, 'forgejo-attachments'))}",
72 f" rewrite * {json.dumps('/' + urllib.parse.quote(filename, safe='/'))}",
73 " file_server", " }", " }", " }"]
74 return lines
75
76
77def shale_write_routes(host):
78 # Older Shale releases predate form tokens. Browsers send Origin on POST;
79 # require the exact site origin for every cookie-authenticated mutation.
80 # HTTP Git clients use Basic authentication and do not carry this cookie.
81 return [" @shale_unsafe_origin {", " method POST PUT PATCH DELETE",
82 " header Cookie *SessionID=*", f" not header Origin https://{host}",
83 " }", ' respond @shale_unsafe_origin "Forbidden" 403']
84
85
48def render(token):86def render(token):
49 with open(os.environ["STUDIO_PROXY_TOKEN_FILE"]) as file:87 with open(os.environ["STUDIO_PROXY_TOKEN_FILE"]) as file:
50 dashboard_proof = file.read().strip()88 dashboard_proof = file.read().strip()
...@@ -136,7 +174,15 @@ def render(token):...@@ -136,7 +174,15 @@ def render(token):
136 raise ValueError(f"invalid service name: {service}")174 raise ValueError(f"invalid service name: {service}")
137 if service == "keycloak" and host == auth_host:175 if service == "keycloak" and host == auth_host:
138 lines += webauthn176 lines += webauthn
177 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
178 # Keep the Origin guard before every static/proxy handle;
179 # otherwise Caddy sorts those handles ahead of respond.
180 lines += [" route {", *shale_write_routes(host)]
181 if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
182 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])),
183 "/srv/staging/" + service)
139 if service == "shale":184 if service == "shale":
185 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])))
140 lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$",186 lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$",
141 " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"]187 " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"]
142 port = int(os.environ["STUDIO_DASHBOARD_PORT"])188 port = int(os.environ["STUDIO_DASHBOARD_PORT"])
...@@ -241,6 +287,8 @@ def render(token):...@@ -241,6 +287,8 @@ def render(token):
241 *proxy(upstreams, " ", uncompressed=True), " }", " }"]287 *proxy(upstreams, " ", uncompressed=True), " }", " }"]
242 lines += [" handle {", *(f" request_header -{name}" for name in scrub),288 lines += [" handle {", *(f" request_header -{name}" for name in scrub),
243 *proxy(upstreams, " "), " }", "}"]289 *proxy(upstreams, " "), " }", "}"]
290 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
291 lines.insert(len(lines) - 1, " }")
244 else:292 else:
245 lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"]293 lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"]
246 if (80, auth_host) not in routes:294 if (80, auth_host) not in routes:
tools/shale-migration.md+18
...@@ -34,6 +34,24 @@ A full-data test imported all 19 retained Forgejo histories into a disposable co...@@ -34,6 +34,24 @@ A full-data test imported all 19 retained Forgejo histories into a disposable co
3434
35After migration and authenticated browser checks, push the new infra `main` to `https://shale.paperclover.net/home-infra.git` with a Shale personal access token. The existing `home-infra` record and UUID are retained; the imported Forgejo branches remain available alongside the tested final `main`.35After migration and authenticated browser checks, push the new infra `main` to `https://shale.paperclover.net/home-infra.git` with a Shale personal access token. The existing `home-infra` record and UUID are retained; the imported Forgejo branches remain available alongside the tested final `main`.
3636
37The October 5 issue migration uses `tools/import-forgejo-issues.py` and the verified personal PostgreSQL export at `/var/lib/studio/forgejo-issue-migration/source.json`. Its sibling `source-proof.json` records the source checksum. The export contains 452 issues, 1,429 comments and events, 60 labels, 24 attachments, and 312 history revisions. Pull request discussions become issues with their original branch and merge metadata. Historical authors retain their names through non-login identities; `paperclover` maps to the existing Clover account. Original creation, update, and comment timestamps are preserved. Every source row is retained in the private `studio_forgejo_records` ledger, including revision history that Shale cannot present as editable comments.
38
39Run this while the target job is stopped and its allocations have exited:
40
41```sh
42python3 tools/import-forgejo-issues.py \
43 --target /srv/prod/shale \
44 --source /var/lib/studio/forgejo-issue-migration/source.json
45```
46
47For an isolated stage, add `--rehearsal` and use `/srv/staging/shale-preview-<id>`. The importer verifies the export checksum, checks that the target job is stopped before writing and immediately before commit, backs up SQLite into a private evidence directory, verifies native issue records remain unchanged, and supports an idempotent rerun of the same export. Take a full service ZFS snapshot before the production import as well. Do not replace production with the stage clone: import again into the stopped production dataset so newer native issues survive.
48
49Six occupied numbers are remapped: `chat` #1→#6 and #2→#7; `home-infra` #1→#38, #2→#39, and #3→#40; `react-mutation` #1→#19. All other original numbers remain. Explicit Forgejo issue links are rewritten and remapped bodies identify their original number. Bare `#references` and code text remain untouched. Attachments are copied with verified SHA-256 hashes and served through an authorization check against their associated Shale issue. Their local files are readable only by root and Caddy; private issue attachments remain private.
50
51The writable ZFS rehearsal `shale-preview-0242cee6` starts from all 104 existing native issues and imports to 556 total. It preserves the original Clover OIDC identity. September's `r1616-ga87d2f5.zig.0.16.0` avoids the `r1758` anonymous deleted-comment crash, but its Markdown scanner uses a shared capture buffer and crashes under concurrent fenced-code rendering. The documented `NPROC=1` worker setting avoids that race. With this setting, 904 authenticated/anonymous imported-issue requests passed with eight clients, all 24 attachment hashes and access checks passed, and browser closing of a disposable copy of `chat` #1 succeeded. Production `chat` #1 remains untouched.
52
53This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict.
54
37The October 4 transport check inspected the image pinned in [service.pkl](../service/shale/service.pkl) in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.55The October 4 transport check inspected the image pinned in [service.pkl](../service/shale/service.pkl) in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.
3856
39Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.57Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.