1#!/usr/bin/env python3
2import argparse
3import base64
4import ctypes
5import ctypes.util
6import contextlib
7import datetime
8import email.utils
9import fcntl
10import functools
11import hashlib
12import html
13from html.parser import HTMLParser
14import http.server
15import json
16import os
17from pathlib import Path
18import re
19import shutil
20import stat
21import subprocess
22import tempfile
23import threading
24import time
25import urllib.error
26import urllib.parse
27import urllib.request
28
29PATTERN = re.compile(r'^zig(?:|-bootstrap|-win64|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$')
30VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z')
31PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U'
32OFFICIAL = 'https://ziglang.org'
33NEKTRO = 'http://mirrors.nektro.net/zig/'
34SIGNATURE_NOTE = '<span style="font-style:italic;color:orange">Don\'t forget to verify your build signatures!</span> <span style="opacity:0.7">&nbsp;(the 🔐 is a link to the minisig)'
35MAX_ARTIFACT = 2 * 1024**3
36_index_cache = (float('-inf'), {})
37_index_lock = threading.Lock()
38
39
40class Unavailable(Exception):
41 pass
42
43
44@functools.lru_cache(maxsize=1)
45def comment_verifier():
46 name = ctypes.util.find_library('sodium')
47 if not name:
48 raise Unavailable('Minisign comment verification unavailable')
49 library = ctypes.CDLL(name)
50 if library.sodium_init() < 0:
51 raise Unavailable('Minisign comment verification unavailable')
52 verify = library.crypto_sign_verify_detached
53 verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p]
54 verify.restype = ctypes.c_int
55 return verify
56
57
58def metadata_open(path, final_flags=os.O_RDONLY):
59 path = path.absolute()
60 descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0))
61 try:
62 for index, part in enumerate(path.parts[1:]):
63 flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0)
64 if index < len(path.parts) - 2:
65 flags |= os.O_DIRECTORY
66 next_descriptor = os.open(part, flags, dir_fd=descriptor)
67 os.close(descriptor)
68 descriptor = next_descriptor
69 return descriptor
70 except BaseException:
71 os.close(descriptor)
72 raise
73
74
75def signed_time(signature):
76 with os.fdopen(metadata_open(signature), 'rb') as stream:
77 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
78 raise Unavailable('signature is not a regular file')
79 body = stream.read(4097)
80 lines = body.splitlines()
81 try:
82 if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '):
83 raise ValueError()
84 key = base64.b64decode(PUBLIC_KEY, validate=True)
85 sig = base64.b64decode(lines[1], validate=True)
86 global_sig = base64.b64decode(lines[3], validate=True)
87 comment = lines[2][len(b'trusted comment: '):]
88 if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]:
89 raise ValueError()
90 # Minisign authenticates the artifact signature followed by the exact trusted comment.
91 message = sig[10:] + comment
92 if comment_verifier()(global_sig, message, len(message), key[10:]) != 0:
93 raise ValueError()
94 match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment)
95 if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'):
96 raise ValueError()
97 return int(match[1]) * 1_000_000_000
98 except (ValueError, IndexError) as error:
99 raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error
100
101
102@functools.lru_cache(maxsize=128)
103def stable_time(version):
104 try:
105 entries = index()
106 if version not in entries:
107 entries = index(refresh=True)
108 date = datetime.date.fromisoformat(entries[version]['date'])
109 return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000
110 except (KeyError, ValueError) as error:
111 raise Unavailable('official release date unavailable: ' + version) from error
112
113
114def artifact_time(artifact):
115 version = filename(artifact.name)
116 if '-dev.' in version or version == '0.15.0':
117 return signed_time(Path(str(artifact) + '.minisig'))
118 return stable_time(version)
119
120
121def children(directory):
122 descriptor = metadata_open(directory)
123 try:
124 with os.scandir(descriptor) as entries:
125 return [(directory / entry.name, entry.stat(follow_symlinks=False))
126 for entry in entries if not entry.name.startswith('.')]
127 finally:
128 os.close(descriptor)
129
130
131def set_mtime(path, timestamp):
132 descriptor = metadata_open(path)
133 try:
134 before = os.fstat(descriptor)
135 if before.st_mtime_ns != timestamp:
136 os.utime(descriptor, ns=(before.st_atime_ns, timestamp))
137 finally:
138 os.close(descriptor)
139
140
141def file_state(value):
142 return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')}
143
144
145def version_readme(version, artifacts, development=False):
146 systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows',
147 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'}
148 source = []
149 targets = {}
150
151 def link(artifact):
152 name = artifact.name
153 url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/')
154 label = 'tar.xz' if name.endswith('.tar.xz') else 'zip'
155 result = f'[{label}]({url})'
156 signature = Path(str(artifact) + '.minisig')
157 if signature.is_file() and not signature.is_symlink():
158 result += f' <a href="{url}.minisig" style="background:none!important;border:none!important;font-size:13px;margin-left:-5px;margin-right:-3px" aria-label="Signature for {html.escape(name, quote=True)}">&#128272;</a>'
159 return result
160
161 for artifact in sorted(artifacts):
162 name = artifact.name
163 extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip'
164 target = name[len('zig'): -len('-' + version + extension)]
165 if target in ('', '-bootstrap'):
166 source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact))
167 continue
168 if target == '-win64':
169 architecture, system = 'x86_64', 'windows'
170 else:
171 first, second = target.removeprefix('-').split('-')
172 system, architecture = (first, second) if first in systems else (second, first)
173 targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact))
174 columns = list(systems)
175 columns += sorted({system for row in targets.values() for system in row} - set(columns))
176 heading = '<br>'.join(source) or 'Architecture'
177 lines = [f'### Zig {version}', '']
178 if not artifacts and development:
179 return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode()
180 if artifacts and '-dev.' not in version:
181 release = index().get(version, {})
182 for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]:
183 url = release.get(key)
184 if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url):
185 lines.append(f'- [{label}]({url})')
186 if lines[-1]:
187 lines.append('')
188 if '-dev.' in version:
189 lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', '']
190 if not artifacts:
191 return '\n'.join(lines).encode()
192 lines += ['<div role="region" aria-label="Zig downloads" tabindex="0" style="max-width:100%;overflow-x:auto;white-space:nowrap;word-break:normal;overflow-wrap:normal">', '',
193 '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |',
194 '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |']
195 priority = ('x86_64', 'aarch64', 'x86', 'arm')
196 for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)):
197 lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |')
198 lines += ['', '</div>']
199 if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts):
200 lines += ['', SIGNATURE_NOTE, '']
201 else:
202 lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', '']
203 return '\n'.join(lines).encode()
204
205
206def filename(name):
207 match = PATTERN.fullmatch(name)
208 if len(name) > 240 or not match:
209 raise ValueError('invalid Zig artifact name')
210 if name.startswith('zig-win64-') and name not in {'zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip'}:
211 raise ValueError('invalid legacy Zig artifact name')
212 version = match[1]
213 major, minor, patch = version.split('-')[0].split('.')
214 if any(part != str(int(part)) for part in (major, minor, patch)):
215 raise ValueError('invalid Zig version')
216 return version
217
218
219def relative(name):
220 version = filename(name)
221 if '-dev.' in version:
222 return Path(version.split('-')[0]) / 'dev' / version / name
223 return Path(version) / name
224
225
226def upstream(name):
227 version = filename(name)
228 prefix = '/builds/' if '-dev.' in version else '/download/' + version + '/'
229 return OFFICIAL + prefix + urllib.parse.quote(name)
230
231
232def fetch(url, target, limit):
233 request = urllib.request.Request(url, headers={'User-Agent': 'snowglobe-zig-mirror/1'})
234 try:
235 with urllib.request.urlopen(request, timeout=180) as response, target.open('wb') as output:
236 count = 0
237 while chunk := response.read(1024 * 1024):
238 count += len(chunk)
239 if count > limit:
240 raise Unavailable('upstream artifact exceeds size limit')
241 output.write(chunk)
242 output.flush()
243 os.fsync(output.fileno())
244 except (OSError, urllib.error.URLError) as error:
245 raise Unavailable('upstream download failed') from error
246
247
248def index(refresh=False):
249 global _index_cache
250 with _index_lock:
251 if refresh or time.monotonic() - _index_cache[0] >= 3600:
252 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:
253 entries = json.load(response)
254 _index_cache = (time.monotonic(), entries)
255 return _index_cache[1]
256
257
258def digest(path):
259 checksum = hashlib.sha256()
260 with path.open('rb') as stream:
261 while chunk := stream.read(1024 * 1024):
262 checksum.update(chunk)
263 return checksum.hexdigest()
264
265
266class Mirror:
267 def __init__(self, root):
268 self.root = Path(root).absolute()
269 if self.root.is_symlink():
270 raise ValueError('mirror root is a symlink')
271 self.root = self.root.resolve()
272 self.root.mkdir(parents=True, exist_ok=True)
273 self.slots = threading.BoundedSemaphore(4)
274 self.refresh_lock = threading.Lock()
275
276 def path(self, name):
277 path = self.root / relative(name)
278 if not path.resolve().is_relative_to(self.root.resolve()):
279 raise ValueError('artifact escapes mirror')
280 return path
281
282 def lock_directory(self):
283 locks = self.root / '.mirror-locks'
284 descriptor = metadata_open(self.root)
285 try:
286 fcntl.flock(descriptor, fcntl.LOCK_EX)
287 timestamp = os.fstat(descriptor).st_mtime_ns
288 try:
289 locks.mkdir()
290 except FileExistsError:
291 pass
292 else:
293 set_mtime(self.root, timestamp)
294 finally:
295 os.close(descriptor)
296 descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY)
297 os.close(descriptor)
298 return locks
299
300 @contextlib.contextmanager
301 def publishing(self):
302 locks = self.lock_directory()
303 with (locks / '.timestamps').open('a') as lock:
304 fcntl.flock(lock, fcntl.LOCK_EX)
305 yield
306
307 def parent_times(self, artifact):
308 def latest(directory):
309 values = []
310 for path, value in children(directory):
311 if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path:
312 values.append(value.st_mtime_ns)
313 elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'):
314 values.append(latest(path))
315 return max(values, default=0)
316
317 directory = artifact.parent
318 while True:
319 timestamp = latest(directory)
320 if timestamp:
321 set_mtime(directory, timestamp)
322 if directory == self.root:
323 break
324 directory = directory.parent
325
326 def ensure(self, name, source=None, checksum=None):
327 target = self.path(name)
328 artifact_name = name.removesuffix('.minisig')
329 artifact = self.path(artifact_name)
330 if target.is_file():
331 return target
332 if not self.slots.acquire(timeout=1):
333 raise BlockingIOError('mirror download capacity reached')
334 try:
335 if shutil.disk_usage(self.root).free < 16 * 1024**3:
336 raise Unavailable('mirror free-space reserve reached')
337 locks = self.lock_directory()
338 with (locks / artifact_name).open('a') as lock:
339 fcntl.flock(lock, fcntl.LOCK_EX)
340 if target.is_file():
341 return target
342 version = filename(artifact_name)
343 signed = '-dev.' in version or tuple(map(int, version.split('.'))) >= (0, 6, 0)
344 if not signed and not checksum:
345 checksum = next((item['shasum'] for item in index().get(version, {}).values()
346 if isinstance(item, dict) and 'tarball' in item
347 and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None)
348 if not checksum:
349 raise Unavailable('unsigned artifact has no official SHA-256')
350 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
351 pending = Path(temporary) / artifact_name
352 signature = Path(str(pending) + '.minisig')
353 url = source or upstream(artifact_name)
354 fetch(url, pending, MAX_ARTIFACT)
355 if checksum and digest(pending) != checksum:
356 raise Unavailable('artifact SHA-256 mismatch')
357 if signed:
358 fetch(url + '.minisig', signature, 4096)
359 result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY],
360 capture_output=True)
361 if result.returncode:
362 raise Unavailable('artifact signature verification failed')
363 timestamp = artifact_time(pending)
364 os.chmod(pending, 0o664)
365 set_mtime(pending, timestamp)
366 if signature.exists():
367 os.chmod(signature, 0o664)
368 set_mtime(signature, timestamp)
369 with self.publishing():
370 self.path(name)
371 artifact.parent.mkdir(parents=True, exist_ok=True)
372 os.replace(pending, artifact)
373 if signature.exists():
374 os.replace(signature, Path(str(artifact) + '.minisig'))
375 try:
376 self.write_readmes(artifact.parent)
377 finally:
378 self.parent_times(artifact)
379 if name.endswith('.minisig') and not target.exists():
380 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
381 pending = Path(temporary) / name
382 fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096)
383 os.chmod(pending, 0o664)
384 set_mtime(pending, artifact.stat().st_mtime_ns)
385 with self.publishing():
386 os.replace(pending, target)
387 try:
388 self.write_readmes(artifact.parent)
389 finally:
390 self.parent_times(artifact)
391 return target
392 finally:
393 self.slots.release()
394
395 def readme_change(self, directory, artifacts, content=None):
396 path = directory / 'preadme.md'
397 if content is None:
398 content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir())
399 before = None
400 previous = b''
401 if path.exists() or path.is_symlink():
402 descriptor = metadata_open(path)
403 with os.fdopen(descriptor, 'rb') as stream:
404 value = os.fstat(stream.fileno())
405 if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024:
406 raise Unavailable('version readme is not a small regular file')
407 before = file_state(value)
408 previous = stream.read(512 * 1024 + 1)
409 required = {artifact.name for artifact in artifacts}
410 required |= {artifact.name + '.minisig' for artifact in artifacts
411 if Path(str(artifact) + '.minisig').is_file()}
412 present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1])
413 for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())}
414 linked = {name for name in present if PATTERN.fullmatch(name)}
415 introduction = content.split(b'<div ', 1)[0]
416 preserved = previous
417 if '-dev.' not in directory.name:
418 preserved = preserved.replace(f'\n[Development builds](/mirrors/zig/{directory.name}/dev/)\n'.encode(), b'')
419 if artifacts and linked == required and preserved.startswith(introduction) and b'https://file.paperclover.net' not in preserved:
420 content = preserved.replace(
421 b'\n<span style="font-style:italic;color:orange">Verify your build signatures.</span> <span style="opacity:0.7">The &#128272; links open the Minisign signatures.</span>\n',
422 b'\n' + SIGNATURE_NOTE.encode() + b'\n', 1)
423 if previous == content:
424 return None
425 return {'path': str(path.relative_to(self.root)), 'before': before,
426 'previous': base64.b64encode(previous).decode(), 'content': content.decode()}
427
428 def artifacts(self, directory):
429 return [path for path, value in children(directory)
430 if stat.S_ISREG(value.st_mode) and PATTERN.fullmatch(path.name)
431 and not path.name.endswith('.minisig') and self.path(path.name) == path]
432
433 def directory_readme(self, directory):
434 if directory.name == 'dev':
435 content = f'### Zig {directory.parent.name} development builds\n'.encode()
436 return self.readme_change(directory, [], content)
437 artifacts = self.artifacts(directory)
438 if artifacts or (directory / 'dev').is_dir():
439 return self.readme_change(directory, artifacts)
440
441 def write_readmes(self, directory):
442 directories = [directory]
443 if directory.parent.name == 'dev':
444 directories += [directory.parent, directory.parent.parent]
445 for current in directories:
446 change = self.directory_readme(current)
447 if change:
448 self.apply_readme(change)
449
450 def apply_readme(self, change):
451 path = self.root / change['path']
452 if change['before'] is None:
453 parent = metadata_open(path.parent)
454 try:
455 descriptor = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o664, dir_fd=parent)
456 os.fchown(descriptor, -1, os.fstat(parent).st_gid)
457 finally:
458 os.close(parent)
459 before = os.fstat(descriptor)
460 else:
461 descriptor = metadata_open(path, os.O_WRONLY)
462 before = os.fstat(descriptor)
463 if any(file_state(before)[key] != value for key, value in change['before'].items() if key != 'atime_ns'):
464 os.close(descriptor)
465 raise Unavailable('version readme changed since plan')
466 try:
467 os.ftruncate(descriptor, 0)
468 with os.fdopen(os.dup(descriptor), 'wb') as output:
469 output.write(change['content'].encode())
470 output.flush()
471 os.fsync(output.fileno())
472 os.utime(descriptor, ns=(before.st_atime_ns, before.st_mtime_ns))
473 finally:
474 os.close(descriptor)
475
476 def metadata_plan(self):
477 files = {}
478 folders = {}
479 errors = []
480 pending = [self.root]
481 while pending:
482 for artifact, value in children(pending.pop()):
483 if stat.S_ISDIR(value.st_mode):
484 pending.append(artifact)
485 continue
486 if not stat.S_ISREG(value.st_mode) or not PATTERN.fullmatch(artifact.name) or artifact.name.endswith('.minisig'):
487 continue
488 try:
489 if artifact.relative_to(self.root) != relative(artifact.name):
490 raise Unavailable('misplaced Zig artifact')
491 timestamp = artifact_time(artifact)
492 files[artifact] = timestamp
493 signature = Path(str(artifact) + '.minisig')
494 if signature.exists():
495 with os.fdopen(metadata_open(signature), 'rb') as stream:
496 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
497 raise Unavailable('signature is not a regular file')
498 files[signature] = timestamp
499 directory = artifact.parent
500 while True:
501 folders[directory] = max(folders.get(directory, 0), timestamp)
502 if directory == self.root:
503 break
504 directory = directory.parent
505 except (Unavailable, OSError, ValueError) as error:
506 errors.append({'path': str(artifact.relative_to(self.root)), 'error': str(error)})
507 readmes = []
508 for directory in sorted(set(folders) - {self.root}):
509 try:
510 change = self.directory_readme(directory)
511 if change:
512 readmes.append(change)
513 except (Unavailable, OSError, ValueError) as error:
514 errors.append({'path': str(directory.relative_to(self.root)), 'error': str(error)})
515 entries = []
516 for path, timestamp in sorted({**files, **folders}.items()):
517 descriptor = metadata_open(path)
518 try:
519 entries.append({'path': str(path.relative_to(self.root)), 'before': file_state(os.fstat(descriptor)), 'mtime_ns': timestamp})
520 finally:
521 os.close(descriptor)
522 return {'root': str(self.root), 'files': len(files), 'folders': len(folders),
523 'changed': sum(entry['before']['mtime_ns'] != entry['mtime_ns'] for entry in entries),
524 'provenance': {'stable': OFFICIAL + '/download/index.json UTC date', 'development': 'pinned Minisign authenticated trusted-comment timestamp', 'withdrawn0.15.0': 'authenticated signing time; release retracted before 0.15.1', 'folders': 'newest contained artifact timestamp'},
525 'errors': errors, 'entries': entries, 'readmes': readmes}
526
527 def reconcile(self, apply=False, journal=None):
528 with self.publishing() if apply else contextlib.nullcontext():
529 plan = self.metadata_plan()
530 if apply:
531 if plan['errors']:
532 raise Unavailable('timestamp plan has unresolved dates')
533 if journal is None or journal.resolve().is_relative_to(self.root):
534 raise ValueError('private journal outside the mirror is required')
535 journal = journal.resolve()
536 descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
537 with os.fdopen(descriptor, 'w') as output:
538 json.dump(plan, output)
539 output.write('\n')
540 output.flush()
541 os.fsync(output.fileno())
542 descriptor = metadata_open(journal.parent)
543 try:
544 os.fsync(descriptor)
545 finally:
546 os.close(descriptor)
547 for change in plan['readmes']:
548 self.apply_readme(change)
549 for entry in plan['entries']:
550 if entry['before']['mtime_ns'] == entry['mtime_ns'] and not stat.S_ISDIR(entry['before']['mode']):
551 continue
552 descriptor = metadata_open(self.root / entry['path'])
553 try:
554 current = file_state(os.fstat(descriptor))
555 expected = entry['before']
556 fields = ('dev', 'ino', 'mode', 'uid', 'gid') if stat.S_ISDIR(current['mode']) else tuple(key for key in expected if key != 'atime_ns')
557 if any(current[key] != expected[key] for key in fields):
558 raise Unavailable('file changed since timestamp plan: ' + entry['path'])
559 if current['mtime_ns'] != entry['mtime_ns']:
560 os.utime(descriptor, ns=(current['atime_ns'], entry['mtime_ns']))
561 finally:
562 os.close(descriptor)
563 applied = {}
564 paths = {entry['path'] for entry in plan['entries']} | {change['path'] for change in plan['readmes']}
565 for path in sorted(paths):
566 descriptor = metadata_open(self.root / path)
567 try:
568 applied[path] = file_state(os.fstat(descriptor))
569 finally:
570 os.close(descriptor)
571 with os.fdopen(metadata_open(journal, os.O_WRONLY | os.O_APPEND), 'w') as output:
572 json.dump({'applied': applied}, output)
573 output.write('\n')
574 output.flush()
575 os.fsync(output.fileno())
576 return plan
577
578 def verify(self):
579 entries = index()
580 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
581 for release in entries.values() for item in release.values()
582 if isinstance(item, dict) and 'tarball' in item}
583 checked = 0
584 for artifact in sorted(self.root.rglob('zig-*')):
585 if any(part.startswith('.') for part in artifact.relative_to(self.root).parts):
586 continue
587 if not artifact.is_file() or artifact.name.endswith('.minisig'):
588 continue
589 filename(artifact.name)
590 if not artifact.resolve().is_relative_to(self.root.resolve()):
591 raise Unavailable('artifact escapes mirror')
592 checksum = hashes.get(artifact.name)
593 if checksum:
594 if digest(artifact) != checksum:
595 raise Unavailable('artifact SHA-256 mismatch: ' + str(artifact))
596 else:
597 result = subprocess.run(['minisign', '-Vm', str(artifact), '-x', str(artifact) + '.minisig', '-P', PUBLIC_KEY], capture_output=True)
598 if result.returncode:
599 raise Unavailable('artifact signature verification failed: ' + str(artifact))
600 checked += 1
601 print(json.dumps({'verifiedArtifacts': checked}))
602
603 def refresh(self):
604 if not self.refresh_lock.acquire(blocking=False):
605 return
606 try:
607 entries = index(refresh=True)
608 latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))]
609 for release in (latest, entries['master']):
610 artifact = None
611 for item in release.values():
612 if isinstance(item, dict) and 'tarball' in item:
613 name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1])
614 artifact = self.ensure(name, checksum=item['shasum'])
615 if artifact is not None:
616 with self.publishing():
617 try:
618 self.write_readmes(artifact.parent)
619 finally:
620 self.parent_times(artifact)
621 finally:
622 self.refresh_lock.release()
623
624
625class Handler(http.server.BaseHTTPRequestHandler):
626 protocol_version = 'HTTP/1.1'
627 timeout = 30
628
629 def do_HEAD(self):
630 self.do_GET()
631
632 def do_GET(self):
633 path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path)
634 if path == '/health':
635 self.send_response(200)
636 self.send_header('Content-Length', '0')
637 self.end_headers()
638 return
639 name = path.removeprefix('/mirrors/zig/').removeprefix('/')
640 try:
641 filename(name)
642 target = self.server.mirror.ensure(name)
643 size = target.stat().st_size
644 first, last, status = 0, size - 1, 200
645 if self.headers.get('Range'):
646 match = re.fullmatch(r'bytes=(\d*)-(\d*)', self.headers['Range'])
647 if not match or not any(match.groups()):
648 raise IndexError()
649 if not match[1]:
650 first = max(0, size - int(match[2]))
651 else:
652 first = int(match[1])
653 if match[2]:
654 last = min(last, int(match[2]))
655 if first > last or first >= size:
656 raise IndexError()
657 status = 206
658 self.send_response(status)
659 self.send_header('Content-Type', 'application/octet-stream')
660 self.send_header('Content-Length', str(last - first + 1))
661 self.send_header('Accept-Ranges', 'bytes')
662 self.send_header('Cache-Control', 'public, max-age=31536000, immutable')
663 self.send_header('Last-Modified', email.utils.formatdate(target.stat().st_mtime, usegmt=True))
664 if status == 206:
665 self.send_header('Content-Range', f'bytes {first}-{last}/{size}')
666 self.end_headers()
667 if self.command != 'HEAD':
668 with target.open('rb') as stream:
669 stream.seek(first)
670 remaining = last - first + 1
671 while remaining and (chunk := stream.read(min(1024 * 1024, remaining))):
672 self.wfile.write(chunk)
673 remaining -= len(chunk)
674 except ValueError:
675 self.send_error(404)
676 except IndexError:
677 self.send_response(416)
678 self.send_header('Content-Range', f'bytes */{size}')
679 self.send_header('Content-Length', '0')
680 self.end_headers()
681 except BlockingIOError:
682 self.send_error(429)
683 except (Unavailable, OSError):
684 self.send_error(504)
685
686 def do_POST(self):
687 self.send_error(405)
688
689
690class Links(HTMLParser):
691 def __init__(self):
692 super().__init__()
693 self.names = []
694
695 def handle_starttag(self, tag, attrs):
696 if tag == 'a':
697 href = dict(attrs).get('href', '')
698 if '/' not in href.rstrip('/') and not href.startswith('.'):
699 self.names.append(urllib.parse.unquote(href))
700
701
702def listing(url):
703 with urllib.request.urlopen(url, timeout=30) as response:
704 links = Links()
705 links.feed(response.read(4 * 1024**2).decode())
706 return links.names
707
708
709def backfill(mirror, budget, apply):
710 entries = index()
711 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
712 for release in entries.values() for item in release.values()
713 if isinstance(item, dict) and 'tarball' in item}
714 copied = 0
715 for version in listing(NEKTRO):
716 version = version.rstrip('/')
717 if not VERSION.fullmatch(version):
718 continue
719 url = NEKTRO + urllib.parse.quote(version) + '/'
720 for name in listing(url):
721 try:
722 filename(name)
723 except ValueError:
724 continue
725 if name.endswith('.minisig') or mirror.path(name).exists():
726 continue
727 if not apply:
728 print(url + urllib.parse.quote(name))
729 continue
730 if copied + MAX_ARTIFACT > budget:
731 raise SystemExit('backfill budget reached; rerun to resume')
732 path = mirror.ensure(name, url + urllib.parse.quote(name), hashes.get(name))
733 copied += path.stat().st_size
734 print(json.dumps({'downloadedBytes': copied, 'apply': apply}))
735
736
737def main():
738 parser = argparse.ArgumentParser()
739 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify', 'reconcile'])
740 parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror'))
741 parser.add_argument('--budget-gib', type=int, default=32)
742 parser.add_argument('--apply', action='store_true')
743 parser.add_argument('--journal', type=Path)
744 args = parser.parse_args()
745 os.umask(0o002)
746 if args.command == 'reconcile' and not Path(args.root).is_dir():
747 raise ValueError('existing mirror root is required')
748 mirror = Mirror(args.root)
749 if args.command == 'verify':
750 mirror.verify()
751 elif args.command == 'reconcile':
752 print(json.dumps(mirror.reconcile(args.apply, args.journal)))
753 elif args.command == 'refresh':
754 mirror.refresh()
755 elif args.command == 'backfill':
756 backfill(mirror, args.budget_gib * 1024**3, args.apply)
757 else:
758 interval = int(os.environ.get('ZIG_MIRROR_REFRESH', '21600'))
759 if interval:
760 def refresh():
761 while True:
762 try:
763 mirror.refresh()
764 except Exception as error:
765 print(type(error).__name__ + ': refresh failed', flush=True)
766 time.sleep(interval)
767 threading.Thread(target=refresh, daemon=True).start()
768 server = http.server.ThreadingHTTPServer(('0.0.0.0', 8080), Handler)
769 server.mirror = mirror
770 server.serve_forever()
771
772
773if __name__ == '__main__':
774 main()