1#!/usr/bin/env python3
2import argparse
3from datetime import datetime, timezone
4import hashlib
5import json
6import os
7from pathlib import Path
8import re
9import secrets
10import shutil
11import sqlite3
12import tempfile
13import urllib.error
14import urllib.request
15
16
17SOURCE = Path('/mnt/storage1/apps/forgejo/git/repositories')
18ACCESS = ('access_git_webui', 'access_git_proto', 'access_git_proto_push', 'access_issues',
19 'access_issues_submit', 'access_issues_comment', 'access_readme')
20
21
22def sha256(path):
23 digest = hashlib.sha256()
24 with path.open('rb') as source:
25 for block in iter(lambda: source.read(1024 * 1024), b''):
26 digest.update(block)
27 return digest.hexdigest()
28
29
30def refs(repository):
31 result = {}
32 packed = repository / 'packed-refs'
33 if packed.exists():
34 for line in packed.read_text().splitlines():
35 if line and line[0] not in '#^':
36 value, name = line.split(' ')
37 result[name] = value
38 for path in (repository / 'refs').rglob('*'):
39 if path.is_symlink():
40 raise ValueError('Repository refs contain a link. Resolve it before importing')
41 if path.is_file():
42 result[str(path.relative_to(repository))] = path.read_text().strip()
43 for name, value in result.items():
44 if not name.startswith('refs/') or any(part in ('', '.', '..') for part in name.split('/')):
45 raise ValueError('Repository has an unsafe ref name. Review its retained refs')
46 if not re.fullmatch('[0-9a-f]{40}', value):
47 raise ValueError('Repository has a symbolic or unsupported ref. Resolve it before importing')
48 return result
49
50
51def stopped():
52 token = os.environ.get('NOMAD_TOKEN') or Path('/var/lib/studio/nomad.token').read_text().strip()
53 def read(path):
54 request = urllib.request.Request('http://127.0.0.1:4646/v1/' + path,
55 headers={'X-Nomad-Token': token})
56 with urllib.request.urlopen(request, timeout=10) as response:
57 return json.load(response)
58 try:
59 job = read('job/shale')
60 except urllib.error.HTTPError as error:
61 if error.code == 404:
62 return
63 raise
64 if not job.get('Stop') or any(item['ClientStatus'] in ('pending', 'running')
65 for item in read('job/shale/allocations')):
66 raise ValueError('Stop Shale and wait for its allocations before importing')
67
68
69def main():
70 parser = argparse.ArgumentParser(description='Import retained personal Forgejo histories into stopped Shale.')
71 parser.add_argument('--target', type=Path, required=True, help='Copied Shale service root')
72 parser.add_argument('--metadata', type=Path, required=True, help='Personal Forgejo repository metadata JSON')
73 args = parser.parse_args()
74 if os.geteuid() != 0:
75 parser.error('Run as root on Zenith')
76 os.umask(0o077)
77 target = args.target.resolve()
78 retained = Path('/mnt/storage1/apps')
79 if target.is_relative_to(retained) or retained.is_relative_to(target):
80 raise ValueError('Target overlaps retained app data. Use the managed Shale copy')
81 database = target / 'data/astheno.shale.db'
82 if not database.is_file():
83 raise ValueError('Shale database is missing. Import its retained state first')
84 metadata = json.loads(args.metadata.read_text())
85 required = {'owner_name', 'lower_name', 'name', 'is_private', 'default_branch', 'is_empty', 'is_mirror'}
86 if not isinstance(metadata, list) or any(not isinstance(row, dict) or not required.issubset(row) for row in metadata):
87 raise ValueError('Forgejo metadata is incomplete. Export the documented personal repository fields')
88 if any(not re.fullmatch(r'[A-Za-z0-9_.-]+', row[field]) or row[field] in ('.', '..')
89 for row in metadata for field in ('owner_name', 'lower_name', 'name')):
90 raise ValueError('Repository metadata has unsafe names. Review the personal Forgejo export')
91 indexed = {(row['owner_name'], row['lower_name']): row for row in metadata}
92 if len(indexed) != len(metadata) or any(type(row[field]) is not bool for row in metadata
93 for field in ('is_private', 'is_empty', 'is_mirror')):
94 raise ValueError('Forgejo metadata has duplicate repositories or missing visibility')
95 sources = sorted(SOURCE.glob('*/*.git'))
96 if not sources:
97 raise ValueError('No retained personal Forgejo repositories were found')
98 if any(path.is_file() for path in (SOURCE.parent / 'lfs').rglob('*')):
99 raise ValueError('Forgejo has LFS data. Preserve its serving path before migrating repositories')
100 for source in sources:
101 if source.is_symlink() or (source.parent.name, source.name[:-4]) not in indexed:
102 raise ValueError('A retained repository has no verified metadata. Export its personal Forgejo database entry')
103 if any((source / folder).is_symlink() for folder in ('objects', 'refs')):
104 raise ValueError('Repository storage contains a link. Preserve its contents before importing')
105 if (source / 'shallow').exists() or (source / 'objects/info/alternates').exists():
106 raise ValueError('Repository history depends on external objects. Preserve them before importing')
107 if any((source / 'objects/pack').glob('*.promisor')):
108 raise ValueError('Repository has promised objects. Retrieve the complete history before importing')
109 expected_paths = {(row['owner_name'], row['lower_name']) for row in metadata if not row['is_empty']}
110 if not expected_paths.issubset({(source.parent.name, source.name[:-4]) for source in sources}):
111 raise ValueError('A nonempty Forgejo repository is missing from retained storage')
112 stopped()
113 evidence = Path(tempfile.mkdtemp(prefix='shale-forgejo-import-', dir='/var/lib/studio'))
114 db = sqlite3.connect(database)
115 db.row_factory = sqlite3.Row
116 if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
117 raise ValueError('Shale database failed integrity checking. Recover the retained copy')
118 owners = db.execute("SELECT id FROM users WHERE name='snow'").fetchall()
119 if len(owners) != 1:
120 raise ValueError('Shale needs one existing snow account. Preserve its original identity before importing')
121 owner = owners[0]['id']
122 stat = database.stat()
123 with sqlite3.connect(evidence / 'before.db') as backup:
124 db.backup(backup)
125 existing = {row['name'].casefold(): dict(row) for row in db.execute(
126 'SELECT id,uuid,owner,name,' + ','.join(ACCESS) + ' FROM repositories')}
127 records = []
128 now = datetime.now(timezone.utc).isoformat(timespec='seconds')
129 try:
130 for key, row in indexed.items():
131 name = row['name'] if key[0] == 'clo' else key[0] + '/' + row['name']
132 entry = existing.get(name.casefold())
133 if entry and entry['owner'] != owner:
134 raise ValueError('Repository belongs to another Shale account. Review its ownership before merging')
135 if entry and row['is_private']:
136 for field in ACCESS:
137 db.execute(f"UPDATE repositories SET {field}='private' WHERE id=? AND {field} IN ('public','unlisted')", (entry['id'],))
138 db.commit()
139 db.execute('BEGIN IMMEDIATE')
140 for key, row in sorted(indexed.items()):
141 source = SOURCE / key[0] / (key[1] + '.git')
142 name = row['name'] if key[0] == 'clo' else key[0] + '/' + row['name']
143 entry = existing.get(name.casefold())
144 if entry:
145 directory = target / 'repositories_owned' / entry['uuid']
146 if not directory.is_dir():
147 raise ValueError('An existing Shale repository directory is missing. Recover it before merging')
148 else:
149 alphabet = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'
150 number = secrets.randbits(128)
151 repo_id = ''.join(alphabet[(number >> (5 * position)) & 31] for position in range(25, -1, -1))
152 directory = target / 'repositories_owned' / repo_id
153 directory.mkdir(mode=0o700)
154 os.chown(directory, stat.st_uid, stat.st_gid)
155 for folder in ('objects', 'refs'):
156 (directory / folder).mkdir(mode=0o700)
157 os.chown(directory / folder, stat.st_uid, stat.st_gid)
158 (directory / 'config').write_text('[core]\nrepositoryformatversion = 0\nbare = true\n[http]\nreceivepack = true\n')
159 access = 'private' if row['is_private'] else 'public'
160 cursor = db.execute(
161 'INSERT INTO repositories(uuid,owner,created_on,name,description,' + ','.join(ACCESS) + ',last_updated) '
162 'VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)',
163 (repo_id, owner, now, name, '', access, access, 'private', 'off', 'off', 'off', access, now))
164 entry = {'id': cursor.lastrowid, 'uuid': repo_id, 'name': name}
165 existing[name.casefold()] = entry
166 previous = refs(directory)
167 original = refs(source) if source.exists() else {}
168 objects = {}
169 for path in sorted((source / 'objects').rglob('*')) if source.exists() else ():
170 if path.is_symlink():
171 raise ValueError('Repository objects contain a link. Preserve its contents before importing')
172 relative = path.relative_to(source / 'objects')
173 if not path.is_file() or relative.parts[0] == 'info':
174 continue
175 checksum = sha256(path)
176 destination = directory / 'objects' / relative
177 destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
178 if destination.exists():
179 if sha256(destination) != checksum:
180 raise ValueError('Object files differ under the same name. Keep Shale stopped and review the copies')
181 else:
182 shutil.copyfile(path, destination)
183 os.chmod(destination, 0o600)
184 os.chown(destination, stat.st_uid, stat.st_gid)
185 if sha256(destination) != checksum:
186 raise RuntimeError('Copied objects failed verification. Keep Shale stopped')
187 objects[str(relative)] = checksum
188 mapped = {}
189 combined = dict(previous)
190 for reference, value in original.items():
191 destination = reference
192 if destination in combined and combined[destination] != value:
193 category = 'heads' if reference.startswith('refs/heads/') else 'tags' if reference.startswith('refs/tags/') else None
194 destination = ('refs/' + category + '/forgejo/' + key[0] + '/' + reference.split('/', 2)[2]
195 if category else 'refs/forgejo/' + key[0] + '/' + reference[5:])
196 if destination in combined and combined[destination] != value:
197 destination += '-'+value
198 path = directory / destination
199 path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
200 path.write_text(value + '\n')
201 os.chown(path, stat.st_uid, stat.st_gid)
202 combined[destination] = value
203 mapped[reference] = destination
204 if not (directory / 'HEAD').exists():
205 desired = 'refs/heads/' + row['default_branch']
206 if original and desired not in original:
207 desired = next((reference for reference in sorted(original) if reference.startswith('refs/heads/')), desired)
208 (directory / 'HEAD').write_text('ref: ' + desired + '\n')
209 os.chown(directory / 'HEAD', stat.st_uid, stat.st_gid)
210 os.chown(directory / 'config', stat.st_uid, stat.st_gid)
211 final = refs(directory)
212 if any(final.get(reference) != value for reference, value in previous.items()) or any(
213 final.get(mapped[reference]) != value for reference, value in original.items()
214 ):
215 raise RuntimeError('Ref verification failed. Keep Shale stopped and review the retained refs')
216 if source.exists() and refs(source) != original:
217 raise RuntimeError('Source refs changed during import. Keep Shale stopped')
218 for parent in (directory / 'objects', directory / 'refs'):
219 for path in parent.rglob('*'):
220 if path.is_dir():
221 os.chown(path, stat.st_uid, stat.st_gid)
222 records.append({'source': '/'.join(key), 'target': entry['name'], 'uuid': entry['uuid'],
223 'private': row['is_private'], 'sourceMirror': row['is_mirror'], 'objects': objects,
224 'originalRefs': original, 'refMapping': mapped, 'previousRefs': previous,
225 'head': (directory / 'HEAD').read_text().strip()})
226 stopped()
227 if db.execute('PRAGMA foreign_key_check').fetchall():
228 raise RuntimeError('Shale database references are invalid. Keep it stopped')
229 db.commit()
230 finally:
231 db.close()
232 (evidence / 'repositories.json').write_text(json.dumps(records, indent=2) + '\n')
233 print(f'Imported and verified {len(records)} personal repositories. Evidence and previous database: {evidence}')
234
235
236if __name__ == '__main__':
237 try:
238 main()
239 except (ValueError, RuntimeError, OSError, sqlite3.Error, urllib.error.HTTPError) as error:
240 raise SystemExit(str(error)) from None