1#!/usr/bin/env python3
2import argparse
3from contextlib import contextmanager
4from datetime import datetime, timezone
5import hashlib
6import json
7import os
8from pathlib import Path
9import secrets
10import shutil
11import subprocess
12import tempfile
13
14
15def run(command, *, user=None, output=None):
16 result = subprocess.run(command, stdout=output or subprocess.PIPE,
17 stderr=subprocess.PIPE, user=user, group=user,
18 extra_groups=[] if user is not None else None)
19 if result.returncode:
20 with (destination / 'diagnostics.log').open('ab') as log:
21 log.write(result.stderr)
22 raise RuntimeError(f'{Path(command[0]).name} failed; see private diagnostics.log')
23 return result.stdout
24
25
26def sha256(path):
27 digest = hashlib.sha256()
28 with path.open('rb') as source:
29 for block in iter(lambda: source.read(1024 * 1024), b''):
30 digest.update(block)
31 return digest.hexdigest()
32
33
34def tree_hash(path):
35 digest = hashlib.sha256()
36 for entry in sorted(path.rglob('*')):
37 digest.update(str(entry.relative_to(path)).encode() + b'\0')
38 if entry.is_symlink():
39 raise ValueError(f'External cluster link requires separate backup: {entry}')
40 if entry.is_file():
41 digest.update(bytes.fromhex(sha256(entry)))
42 return digest.hexdigest()
43
44
45def identifier(value):
46 return '"' + value.replace('"', '""') + '"'
47
48
49@contextmanager
50def server(runtime, folder, *, source=None):
51 folder.mkdir(mode=0o700)
52 data = folder / 'data'
53 socket = folder / 'socket'
54 socket.mkdir(mode=0o700)
55 os.chown(folder, 65534, 65534)
56 os.chown(socket, 65534, 65534)
57 if source:
58 run(['rsync', '-aH', '--numeric-ids', str(source) + '/', str(data) + '/'])
59 (data / 'postmaster.pid').unlink(missing_ok=True)
60 for entry in [data, *data.rglob('*')]:
61 os.chown(entry, 65534, 65534)
62 else:
63 run([str(runtime / 'bin/initdb'), '-D', str(data), '-U', 'infra2_backup_verifier',
64 '--auth=trust', '--locale=C.UTF-8'], user=65534)
65 config = folder / 'postgresql.conf'
66 config.write_text("listen_addresses = ''\nssl = off\nlogging_collector = off\n"
67 "shared_preload_libraries = ''\n"
68 f"unix_socket_directories = '{socket}'\n"
69 f"hba_file = '{folder / 'pg_hba.conf'}'\n")
70 (folder / 'pg_hba.conf').write_text('local all all trust\n')
71 os.chown(config, 65534, 65534)
72 os.chown(folder / 'pg_hba.conf', 65534, 65534)
73 control = [str(runtime / 'bin/pg_ctl'), '-D', str(data)]
74 try:
75 run(control + ['-l', str(folder / 'server.log'), '-o', f'-c config_file={config}',
76 '-w', 'start'], user=65534)
77 yield socket
78 finally:
79 if (data / 'postmaster.pid').exists():
80 run(control + ['-m', 'fast', '-w', 'stop'], user=65534)
81
82
83def sql(runtime, socket, database, query, *, role='postgres'):
84 return run([str(runtime / 'bin/psql'), '-X', '-h', str(socket), '-U', role,
85 '-d', database, '-v', 'ON_ERROR_STOP=1', '-At', '-c', query])
86
87
88def fingerprint(runtime, socket, database):
89 relations = json.loads(sql(runtime, socket, database, """
90 SELECT coalesce(json_agg(x ORDER BY n, c), '[]') FROM (
91 SELECT n.nspname n, c.relname c, c.relkind k
92 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
93 WHERE n.nspname NOT LIKE 'pg_%' AND n.nspname <> 'information_schema'
94 AND c.relkind IN ('r','m','S')
95 ) x
96 """))
97 result = {}
98 for relation in relations:
99 table = identifier(relation['n']) + '.' + identifier(relation['c'])
100 query = (f'SELECT last_value,is_called FROM {table}' if relation['k'] == 'S' else
101 f"COPY (SELECT row_to_json(t)::text FROM {table} t "
102 'ORDER BY row_to_json(t)::text COLLATE "C") TO STDOUT')
103 content = sql(runtime, socket, database, query)
104 result[table] = {'rows': len(content.splitlines()),
105 'sha256': hashlib.sha256(content).hexdigest()}
106 for name, query in {
107 'large_objects': 'COPY (SELECT loid,pageno,encode(data,\'hex\') FROM pg_largeobject ORDER BY loid,pageno) TO STDOUT',
108 'large_object_owners': 'COPY (SELECT oid,pg_get_userbyid(lomowner),lomacl::text FROM pg_largeobject_metadata ORDER BY oid) TO STDOUT',
109 }.items():
110 content = sql(runtime, socket, database, query)
111 result[name] = {'rows': len(content.splitlines()),
112 'sha256': hashlib.sha256(content).hexdigest()}
113 return result
114
115
116parser = argparse.ArgumentParser(description='Export and restore-check stopped PostgreSQL clusters from private copies.')
117parser.add_argument('--cluster', nargs=2, action='append', required=True, metavar=('SOURCE', 'RUNTIME'))
118parser.add_argument('--locale-archive', type=Path, required=True)
119args = parser.parse_args()
120if os.geteuid() != 0:
121 parser.error('Run as root on the installer')
122os.umask(0o077)
123os.environ['LOCALE_ARCHIVE'] = str(args.locale_archive)
124os.environ.pop('PGOPTIONS', None)
125root = Path('/mnt/storage1/apps/studio-handoff')
126mounted = subprocess.check_output(['findmnt', '-n', '-o', 'SOURCE,FSTYPE', '--mountpoint', str(root.parent)], text=True).split()
127if len(mounted) != 2 or mounted[1] != 'zfs' or not mounted[0].endswith('/apps'):
128 parser.error('Expected retained apps dataset mounted at /mnt/storage1/apps')
129if subprocess.run(['pgrep', '-x', 'postgres'], stdout=subprocess.DEVNULL).returncode == 0:
130 parser.error('Stop PostgreSQL before taking an offline copy')
131handoff_id = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ-') + secrets.token_hex(3)
132root.mkdir(mode=0o700, exist_ok=True)
133destination = root / ('.pending-' + handoff_id)
134destination.mkdir(mode=0o700)
135manifest = {'id': handoff_id, 'clusters': {}, 'databases': {}}
136work = Path(tempfile.mkdtemp(prefix='infra2-pg-export-', dir='/run'))
137os.chown(work, 65534, 65534)
138try:
139 for source_arg, runtime_arg in args.cluster:
140 source, runtime = Path(source_arg).resolve(), Path(runtime_arg).resolve()
141 if not source.is_relative_to(root.parent) or not (source / 'PG_VERSION').is_file():
142 raise ValueError('Source must be a retained cluster inside the apps dataset')
143 major = (source / 'PG_VERSION').read_text().strip()
144 version = run([str(runtime / 'bin/postgres'), '--version']).decode().strip()
145 if version.split()[-1].split('.')[0] != major or major in manifest['clusters']:
146 raise ValueError('Runtime major must match its unique source cluster')
147 before = tree_hash(source)
148 output = destination / ('pg' + major)
149 output.mkdir(mode=0o700)
150 run(['tar', '-C', str(source), '-czf', str(output / 'cluster.tar.gz'), '.'])
151 cluster = {'source': str(source), 'runtime': version, 'sourceSha256': before, 'databases': {}}
152 with server(runtime, work / ('source' + major), source=source) as socket:
153 databases = json.loads(sql(runtime, socket, 'postgres', """
154 SELECT json_agg(x ORDER BY name) FROM (
155 SELECT datname name, datallowconn connect, datistemplate template,
156 pg_get_userbyid(datdba) owner FROM pg_database WHERE datname <> 'template0'
157 ) x
158 """))
159 tablespaces = json.loads(sql(runtime, socket, 'postgres', "SELECT coalesce(json_agg(spcname), '[]') FROM pg_tablespace WHERE spcname NOT IN ('pg_default','pg_global')"))
160 if tablespaces:
161 raise ValueError('Custom tablespaces require an explicit restore layout')
162 with (output / 'globals.sql').open('wb') as file:
163 run([str(runtime / 'bin/pg_dumpall'), '-h', str(socket), '-U', 'postgres', '--globals-only'], output=file)
164 for index, database in enumerate(databases):
165 name = database['name']
166 if not database['connect']:
167 sql(runtime, socket, 'postgres', f'ALTER DATABASE {identifier(name)} ALLOW_CONNECTIONS true')
168 dump = output / f'{index:03d}.dump'
169 with dump.open('wb') as file:
170 run([str(runtime / 'bin/pg_dump'), '-h', str(socket), '-U', 'postgres', '-d', name,
171 '-Fc', '--create'], output=file)
172 entry = {**database, 'file': str(dump.relative_to(destination)), 'bytes': dump.stat().st_size,
173 'sha256': sha256(dump), 'contents': fingerprint(runtime, socket, name)}
174 cluster['databases'][name] = entry
175 if name in {'evil-forgejo', 'evil-hedgedoc'} and major == '18':
176 query = ('SELECT (SELECT count(*) FROM "user"), (SELECT count(*) FROM repository)' if name == 'evil-forgejo' else
177 'SELECT (SELECT count(*) FROM "Notes"), (SELECT count(*) FROM "Users"), (SELECT count(*) FROM "Revisions"), (SELECT count(*) FROM "Authors")')
178 entry['counts'] = [int(n) for n in sql(runtime, socket, name, query).decode().strip().split('|')]
179 if tree_hash(source) != before:
180 raise RuntimeError('Original cluster changed during export; export is not publishable')
181 with server(runtime, work / ('restore' + major)) as socket:
182 bootstrap = 'infra2_backup_verifier'
183 sql(runtime, socket, 'postgres', f'CREATE DATABASE {bootstrap}', role=bootstrap)
184 run([str(runtime / 'bin/psql'), '-X', '-h', str(socket), '-U', bootstrap, '-d', bootstrap,
185 '-v', 'ON_ERROR_STOP=1', '-f', str(output / 'globals.sql')])
186 sql(runtime, socket, bootstrap, 'ALTER DATABASE template1 IS_TEMPLATE false', role=bootstrap)
187 for name, entry in cluster['databases'].items():
188 run([str(runtime / 'bin/pg_restore'), '-h', str(socket), '-U', bootstrap, '-d', bootstrap,
189 '--clean', '--if-exists', '--create', '--exit-on-error', str(destination / entry['file'])])
190 if fingerprint(runtime, socket, name) != entry['contents']:
191 raise RuntimeError(f'Restored data differs: PostgreSQL {major} database {name}')
192 print(f'PostgreSQL {major}: restored and verified {name}', flush=True)
193 cluster['restoreVerified'] = True
194 manifest['clusters'][major] = cluster
195 for name in ('evil-forgejo', 'evil-hedgedoc'):
196 entry = manifest['clusters']['18']['databases'][name]
197 (destination / (name + '.dump')).symlink_to(entry['file'])
198 manifest['databases'][name] = {key: entry[key] for key in ('bytes', 'sha256', 'counts')}
199 manifest['databases'][name]['file'] = name + '.dump'
200 keys = {'lfs_jwt': 'EVIL_FORGEJO_SERVER_LFS_JWT_SECRET', 'oauth_jwt': 'EVIL_FORGEJO_OAUTH2_JWT_SECRET',
201 'security_key': 'EVIL_FORGEJO_SECURITY_SECRET_KEY', 'internal_token': 'EVIL_FORGEJO_SECURITY_INTERNAL_TOKEN',
202 'anubis_key': 'ANUBIS_PRIVATE_KEY', 'mailer_address': 'MAILER_ADDRESS',
203 'mailer_username': 'MAILER_USERNAME', 'mailer_password': 'MAILER_PASSWORD'}
204 legacy_env = (root.parent / 'home-infra/.env').read_text().splitlines()
205 fingerprints = {}
206 for target, key in keys.items():
207 values = [line.split('=', 1)[1] for line in legacy_env if line.startswith(key + '=')]
208 if len(values) != 1 or not values[0]:
209 raise ValueError(f'Legacy secret unavailable: {key}')
210 fingerprints[target] = hashlib.sha256(values[0].encode()).hexdigest()
211 manifest['databases']['evil-forgejo']['secretSha256'] = fingerprints
212 manifest['files'] = {str(path.relative_to(destination)): {'bytes': path.stat().st_size, 'sha256': sha256(path)}
213 for path in destination.rglob('*') if path.is_file() and not path.is_symlink()}
214 (destination / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
215 final = root / handoff_id
216 destination.rename(final)
217 os.sync()
218 print(f'Verified PostgreSQL handoff: {final}', flush=True)
219finally:
220 shutil.rmtree(work)