1#!/usr/bin/env python3
2"""Builds, signs and publishes Snowbound for each desktop platform; see tools/RELEASE.md."""
3import argparse
4import base64
5from datetime import datetime
6import hashlib
7import json
8import os
9from pathlib import Path
10import re
11import shutil
12import subprocess
13import sys
14import tempfile
15import time
16import zipfile
17from zoneinfo import ZoneInfo
18
19ROOT = Path(__file__).resolve().parents[1]
20PUBLISHED = Path('/Volumes/clover/Documents/Public/Snowbound')
21URL = 'https://file.paperclover.net/shr/snowbound/'
22KEY = Path.home() / '.config/snowbound/release-key'
23# Its public half, which the app checks updates against.
24MINISIGN = ROOT / 'minisign.pub'
25ZONE = ZoneInfo('America/Los_Angeles')
26PLATFORMS = ['macos-aarch64', 'macos-x86_64', 'macos-10.6', 'linux-x86_64', 'linux-aarch64',
27 'windows-x86_64', 'windows-aarch64']
28# Windows 7 to 11 on x86_64 (nightly's tier-3 win7 target), and Windows 11 on Arm.
29WINDOWS = {'x86_64': 'x86_64-win7-windows-gnu', 'aarch64': 'aarch64-pc-windows-gnullvm'}
30# Clover's Developer ID Application certificate, by its SHA-1 hash: its name is the account
31# holder's legal name, which nothing here prints or stores.
32IDENTITY = 'BA308AA3591299E053E8824CEF1651F686F8908E'
33# The App Store Connect API key that notarizes it: {"key": P8 PATH, "key_id": ID, "issuer": ID}.
34NOTARY = Path('~/.config/snowbound/notary.json').expanduser()
35# The first published build's commit, where changes start when no build was published before.
36FIRST = '354f001dec3d731a4d1a6fac0a25d9e28550d781'
37KINDS = {'feat': 'feature', 'fix': 'fix'}
38
39
40def derive(release, commits):
41 """The version of the commit made at `release`, given when each commit leading to it was
42 made, itself included: its day in Los Angeles, and how many of those were made that day."""
43 day = release.astimezone(ZONE).date()
44 return day.isoformat(), sum(1 for made in commits if made.astimezone(ZONE).date() == day)
45
46
47def name(version):
48 return f'{version[0]}-r{version[1]}'
49
50
51def folder(version):
52 """The build's top-level folder."""
53 return f'{version[0]}.r{version[1]}'
54
55
56def parse(text):
57 date, revision = text.split('-r')
58 return date, int(revision)
59
60
61def newest(latest, archives, version):
62 """`latest.json`'s map with each of `archives`' platforms moved to `version` where that is
63 newer than what it names."""
64 return {**latest, **{platform: name(version) for platform in archives
65 if platform not in latest or parse(latest[platform]) < version}}
66
67
68def builds(published):
69 """The versions of the builds `published` holds, oldest first."""
70 return sorted(parse(entry.name.replace('.r', '-r')) for entry in published.iterdir()
71 if re.fullmatch(r'\d{4}-\d{2}-\d{2}\.r\d+', entry.name))
72
73
74def jj(*args):
75 return subprocess.check_output(['jj', *args], cwd=ROOT, text=True)
76
77
78def clean(dry_run, moment):
79 """Refuses to go on, or in a dry run warns, where the working copy isn't `main`."""
80 changed = jj('diff', '--from', 'main', '--to', '@', '--summary').strip()
81 if changed and not dry_run:
82 sys.exit(f'The working copy differs from main {moment}:\n{changed}')
83 if changed:
84 print(f'Dry run: the working copy differs from main {moment}.', file=sys.stderr)
85
86
87def run(command, **kwargs):
88 print('+', ' '.join(map(str, command)), flush=True)
89 subprocess.run(command, cwd=ROOT, check=True, **kwargs)
90
91
92def history():
93 """Every commit on `main` by id: its parents' ids, when it was made, and its message."""
94 template = ('commit_id ++ "\\x1f" ++ parents.map(|parent| parent.commit_id()).join(" ") ++ "\\x1f" ++ '
95 'committer.timestamp().utc().format("%Y-%m-%dT%H:%M:%S+00:00") ++ "\\x1f" ++ description ++ "\\x1e"')
96 commits = {}
97 for record in jj('log', '--no-graph', '-r', '::main', '-T', template).split('\x1e')[:-1]:
98 commit, parents, made, description = record.split('\x1f')
99 commits[commit] = (parents.split(), datetime.fromisoformat(made), description)
100 return commits
101
102
103def ancestors(commits, commit):
104 """`commit` and every commit before it."""
105 found, stack = set(), [commit]
106 while stack:
107 commit = stack.pop()
108 if commit not in found:
109 found.add(commit)
110 stack.extend(commits[commit][0])
111 return found
112
113
114def version_of(commits, commit):
115 return derive(commits[commit][1], [commits[each][1] for each in ancestors(commits, commit)])
116
117
118def entries(description):
119 """What a commit brings, as (kind, title): one entry of its prefix's kind, or one per item where
120 its body has a top-level bulleted list, each of the prefix's kind."""
121 subject, _, body = description.strip().partition('\n')
122 prefix = re.match(r'(\w+)(\([^)]*\))?!?:\s*', subject)
123 kind = KINDS.get(prefix[1].lower(), 'other') if prefix else 'other'
124 items, open_item = [], False
125 for line in body.splitlines():
126 if line.startswith(('- ', '* ')):
127 items.append(line[2:].strip())
128 open_item = True
129 elif open_item and line[:1].isspace() and line.strip():
130 items[-1] += ' ' + line.strip()
131 else:
132 open_item = False
133 titles = [title.rstrip('.') for title in items or [subject[prefix.end():] if prefix else subject]]
134 # Capitalized as a sentence, except a word like macOS or iCloud.
135 return [(kind, title if re.match(r'\S+[A-Z]', title) else title[:1].upper() + title[1:])
136 for title in titles if title]
137
138
139def changes(commits, commit, since):
140 """Every entry the commits after `since` up to `commit` bring, oldest first, each with the
141 version of the commit that brought it."""
142 versions = {each: version_of(commits, each)
143 for each in ancestors(commits, commit) - ancestors(commits, since)}
144 return [{'version': name(versions[each]), 'kind': kind, 'title': title}
145 for each in sorted(versions, key=versions.get) for kind, title in entries(commits[each][2])]
146
147
148def sign(files):
149 output = subprocess.check_output(
150 ['cargo', 'run', '--quiet', '--release', '-p', 'snowbound', '--example', 'release_sign', '--', KEY, *files],
151 cwd=ROOT, text=True)
152 return output.split()
153
154
155def minisign(files):
156 """Writes FILE.minisig beside each of `files` as `minisign -S` would with the release key:
157 a signature of the file's BLAKE2b-512, then one of that and the trusted comment."""
158 key_id = base64.b64decode(MINISIGN.read_text().splitlines()[1])[2:10]
159 comments = [f'timestamp:{int(time.time())}\tfile:{file.name}\thashed' for file in files]
160 with tempfile.TemporaryDirectory() as scratch:
161 def signed(messages):
162 paths = [Path(scratch) / str(index) for index in range(len(messages))]
163 for path, message in zip(paths, messages):
164 path.write_bytes(message)
165 return [bytes.fromhex(signature) for signature in sign(paths)]
166 signatures = signed([hashlib.blake2b(file.read_bytes()).digest() for file in files])
167 global_signatures = signed([signature + comment.encode() for signature, comment in zip(signatures, comments)])
168 for file, signature, comment, global_signature in zip(files, signatures, comments, global_signatures):
169 Path(f'{file}.minisig').write_text(
170 'untrusted comment: signature from the Snowbound release key\n'
171 f'{base64.b64encode(b"ED" + key_id + signature).decode()}\n'
172 f'trusted comment: {comment}\n{base64.b64encode(global_signature).decode()}\n')
173
174
175def split_debug(executable, debug):
176 """Moves `executable`'s debug info to `debug`, which its debug link then names."""
177 sysroot = subprocess.check_output(['rustc', '--print', 'sysroot'], text=True).strip()
178 host = re.search(r'^host: (\S+)$', subprocess.check_output(['rustc', '-vV'], text=True), re.M)[1]
179 objcopy = Path(sysroot) / 'lib/rustlib' / host / 'bin/rust-objcopy'
180 run([objcopy, '--only-keep-debug', executable, debug])
181 run([objcopy, '--strip-debug', f'--add-gnu-debuglink={debug}', executable])
182
183
184def zip_bundle(bundle, archive):
185 run(['ditto', '-c', '-k', '--norsrc', '--noextattr', '--noqtn', '--noacl', '--keepParent', bundle, archive])
186
187
188def notary():
189 """notarytool's credential arguments from NOTARY, if they sign in."""
190 if not NOTARY.exists():
191 return None
192 key = json.loads(NOTARY.read_text())
193 arguments = ['--key', str(Path(key['key']).expanduser()), '--key-id', key['key_id'], '--issuer', key['issuer']]
194 signs_in = subprocess.run(['xcrun', 'notarytool', 'history', *arguments], capture_output=True).returncode == 0
195 return arguments if signs_in else None
196
197
198def build_mac(platform, folder, developer_id, notarize, symbols):
199 """The zipped app, which build_macos.py signs; 10.6's stays unsigned, as it predates Developer ID.
200 Its zipped dSYM goes to `symbols`."""
201 bundle = folder / 'Snowbound.app'
202 if platform == 'macos-10.6':
203 signing = ['--snow-leopard']
204 elif developer_id:
205 signing = ['--sign', 'developer-id', '--sign-identity', IDENTITY]
206 else:
207 signing = ['--sign', 'ad-hoc']
208 if platform != 'macos-10.6':
209 signing += ['--arch', platform.removeprefix('macos-')]
210 dsym = folder / 'Snowbound.dSYM'
211 run([sys.executable, ROOT / 'tools/canvas/build_macos.py', '--release', '--output', bundle, '--dsym', dsym, *signing])
212 zip_bundle(dsym, symbols)
213 archive = folder / 'archive.zip'
214 if notarize and platform != 'macos-10.6':
215 zip_bundle(bundle, archive)
216 run(['xcrun', 'notarytool', 'submit', archive, *notarize, '--wait'])
217 run(['xcrun', 'stapler', 'staple', bundle])
218 archive.unlink()
219 zip_bundle(bundle, archive)
220 return archive
221
222
223def build_linux(architectures):
224 """The executables, each all of Snowbound for its architecture."""
225 run(['sh', ROOT / 'crates/snowbound/linux/package.sh', *architectures])
226 return {f'linux-{arch}': ROOT / f'target/{arch}-unknown-linux-gnu/release/snowbound' for arch in architectures}
227
228
229def build_windows(architectures):
230 """The executables, each all of Snowbound for its architecture, one running on every
231 Windows it supports."""
232 built = {}
233 for arch in architectures:
234 run(['sh', ROOT / 'platform/windows/cargo.sh', arch, 'build', '--release', '-p', 'snowbound'])
235 built[f'windows-{arch}'] = ROOT / f'target/windows/{WINDOWS[arch]}/release/snowbound.exe'
236 return built
237
238
239def copy_download(source, destination):
240 shutil.copyfile(source, destination)
241 try:
242 shutil.copymode(source, destination)
243 except PermissionError as error:
244 print(f'Warning: copied {destination}, but could not preserve file permissions: {error}',
245 file=sys.stderr)
246
247
248def main():
249 parser = argparse.ArgumentParser(description=__doc__)
250 parser.add_argument('--dry-run', action='store_true',
251 help='Publish into a new temporary folder instead, even from a changed working copy')
252 parser.add_argument('--platforms', nargs='+', choices=PLATFORMS, default=PLATFORMS)
253 parser.add_argument('--ad-hoc', action='store_true',
254 help='Sign the macOS app ad hoc instead of with Developer ID, unnotarized')
255 args = parser.parse_args()
256 developer_id = not args.ad_hoc and any(platform in ('macos-aarch64', 'macos-x86_64') for platform in args.platforms)
257 identities = subprocess.run(['security', 'find-identity', '-v', '-p', 'codesigning'],
258 capture_output=True, text=True).stdout
259 if developer_id and IDENTITY not in identities:
260 sys.exit(f'The keychain has no signing identity {IDENTITY}; release with --ad-hoc, or add it.')
261 notarize = notary() if developer_id else None
262 if developer_id and not notarize:
263 print(f'Not notarizing: no API key in {NOTARY} that signs in (see tools/RELEASE.md).',
264 file=sys.stderr)
265
266 commit = jj('log', '--no-graph', '-r', 'main', '-T', 'commit_id').strip()
267 clean(args.dry_run, 'to release it')
268 commits = history()
269 version = version_of(commits, commit)
270 print(f'Snowbound build {version[0]} revision {version[1]}, commit {commit}', flush=True)
271
272 published = Path(tempfile.mkdtemp(prefix='snowbound-release-')) if args.dry_run else PUBLISHED
273 if not published.is_dir():
274 sys.exit(f'{published} is not mounted.')
275 target = published / folder(version)
276 if target.exists():
277 build = json.loads((target / 'build.json').read_text())
278 if build['commit'] != commit:
279 sys.exit(f'{target} holds commit {build["commit"]}, not {commit}.')
280 print(f'{target} is already published.')
281 else:
282 run([sys.executable, ROOT / 'tools/ci.py', '--rev', commit])
283 clean(args.dry_run, 'after the checks')
284 stage = ROOT / 'target/release-stage' / name(version)
285 shutil.rmtree(stage, ignore_errors=True)
286 stage.mkdir(parents=True)
287 # The app reads its version from this as it compiles.
288 os.environ['SNOWBOUND_BUILD'] = name(version)
289 # For the symbol files; the executables shed it.
290 os.environ['CARGO_PROFILE_RELEASE_DEBUG'] = 'line-tables-only'
291 built = {}
292 symbols = []
293 for platform in args.platforms:
294 if platform.startswith('macos'):
295 work = stage / platform
296 work.mkdir()
297 symbols.append(stage / f'Snowbound-{name(version)}-{platform}.dSYM.zip')
298 built[platform] = build_mac(platform, work, developer_id, notarize, symbols[-1])
299 linux = [platform.removeprefix('linux-') for platform in args.platforms if platform.startswith('linux')]
300 if linux:
301 built |= build_linux(linux)
302 windows = [platform.removeprefix('windows-') for platform in args.platforms if platform.startswith('windows')]
303 if windows:
304 built |= build_windows(windows)
305 clean(args.dry_run, 'after the build')
306 files = {}
307 for platform, source in built.items():
308 prefix = 'Snowbound' if platform.startswith('macos') else 'snowbound'
309 files[platform] = stage / f'{prefix}-{name(version)}-{platform}{source.suffix}'
310 shutil.copy2(source, files[platform])
311 if not platform.startswith('macos'):
312 debug = stage / f'{prefix}-{name(version)}-{platform}.debug'
313 split_debug(files[platform], debug)
314 symbols.append(debug.with_name(f'{debug.name}.zip'))
315 with zipfile.ZipFile(symbols[-1], 'w', zipfile.ZIP_DEFLATED) as archive:
316 archive.write(debug, debug.name)
317 signatures = sign(files.values())
318 # A dry run's changes too start after the newest build the share holds.
319 before = [each for each in builds(PUBLISHED) if each < version] if PUBLISHED.is_dir() else []
320 since = json.loads((PUBLISHED / folder(before[-1]) / 'build.json').read_text())['commit'] if before else FIRST
321 build = {
322 'version': name(version),
323 'commit': commit,
324 'published': datetime.now(ZONE).isoformat(timespec='seconds'),
325 'changes': changes(commits, commit, since),
326 'archives': {platform: {
327 'file': file.name,
328 'size': file.stat().st_size,
329 'sha256': hashlib.sha256(file.read_bytes()).hexdigest(),
330 # Older apps check it; newer ones trust the sha256 build.json.sig vouches for.
331 'signature': signature,
332 } for (platform, file), signature in zip(files.items(), signatures)},
333 }
334 (stage / 'build.json').write_text(json.dumps(build, indent=2) + '\n')
335 (stage / 'build.json.sig').write_text(sign([stage / 'build.json'])[0] + '\n')
336 downloads = [*files.values(), *symbols, stage / 'build.json']
337 minisign(downloads)
338 partial = target.with_name(f'.{target.name}.partial')
339 shutil.rmtree(partial, ignore_errors=True)
340 partial.mkdir()
341 for file in [*downloads, *(Path(f'{file}.minisig') for file in downloads), stage / 'build.json.sig']:
342 copy_download(file, partial / file.name)
343 partial.rename(target)
344 shutil.rmtree(stage)
345 print(f'Published {target}')
346
347 history_file = published / 'history.json'
348 partial = history_file.with_name('.history.json.partial')
349 partial.write_text(json.dumps([name(each) for each in builds(published)], indent=2) + '\n')
350 os.replace(partial, history_file)
351 latest_file = published / 'latest.json'
352 latest = json.loads(latest_file.read_text()) if latest_file.exists() else {}
353 build = json.loads((target / 'build.json').read_text())
354 moved = newest(latest, build['archives'], version)
355 if moved != latest:
356 partial = latest_file.with_name('.latest.json.partial')
357 partial.write_text(json.dumps(moved, indent=2, sort_keys=True) + '\n')
358 os.replace(partial, latest_file)
359 print(f'{latest_file}: {json.dumps(moved, sort_keys=True)}')
360 # Stable names for the readme's download links, always the newest build of each platform.
361 downloads = published / 'latest'
362 downloads.mkdir(exist_ok=True)
363 for platform, newest_name in moved.items():
364 if newest_name != name(version):
365 continue
366 file = build['archives'][platform]['file']
367 for published_name in (file, f'{file}.minisig'):
368 stable = downloads / published_name.replace(f'-{name(version)}', '')
369 partial = stable.with_name(f'.{stable.name}.partial')
370 copy_download(target / published_name, partial)
371 os.replace(partial, stable)
372 if not args.dry_run:
373 print(f'{URL}{folder(version)}/')
374
375
376if __name__ == '__main__':
377 main()