| author | |
| committer | |
| log | a4c36c6d8f5893c91e75def1bc06dad8917475bc |
| tree | 6ad7df4cddc8c9bb09a4fdff1a5ec7b27625c737 |
| parent | a061380e5790ff217adfe1d0ca9dd0426d126739 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Generate published documentation links and version navigation, authenticate development build dates, and preserve root and ancestor timestamps on publication. Simplify anonymous mirror browsing in Copyparty. Run reconciliation explicitly with a private recovery journal.
Assisted-by: gpt-6.1-sol13 files changed, 815 insertions(+), 31 deletions(-)
config/Service.pkl+1| ... | @@ -85,6 +85,7 @@ class Container { | ... | @@ -85,6 +85,7 @@ class Container { |
| 85 | secretFiles: Mapping<String, String> = new {} | 85 | secretFiles: Mapping<String, String> = new {} |
| 86 | args: Listing<String> = new {} | 86 | args: Listing<String> = new {} |
| 87 | capAdd: Listing<String> = new {} | 87 | capAdd: Listing<String> = new {} |
| 88 | readOnlyRoot: Boolean = false | ||
| 88 | devices: Listing<String> = new {} | 89 | devices: Listing<String> = new {} |
| 89 | extraHosts: Listing<String> = new {} | 90 | extraHosts: Listing<String> = new {} |
| 90 | hostNetwork: Boolean = false | 91 | hostNetwork: Boolean = false |
service/copyparty/copyparty.conf+1| ... | @@ -24,6 +24,7 @@ | ... | @@ -24,6 +24,7 @@ |
| 24 | idp-logout: /auth/file/sign-out | 24 | idp-logout: /auth/file/sign-out |
| 25 | idp-login-t: with snow globe | 25 | idp-login-t: with snow globe |
| 26 | html-head: <link rel="stylesheet" href="/.static/copyparty.css"> | 26 | html-head: <link rel="stylesheet" href="/.static/copyparty.css"> |
| 27 | js-browser: /.static/mirrors.js | ||
| 27 | 28 | ||
| 28 | [/] | 29 | [/] |
| 29 | /w | 30 | /w |
service/copyparty/static/copyparty.css+16| ... | @@ -7,3 +7,19 @@ | ... | @@ -7,3 +7,19 @@ |
| 7 | --font-serif: "Name Sans", serif; | 7 | --font-serif: "Name Sans", serif; |
| 8 | --font-mono: "Name Mono", monospace; | 8 | --font-mono: "Name Mono", monospace; |
| 9 | } | 9 | } |
| 10 | .mirror-anonymous #files tr > :first-child, | ||
| 11 | .mirror-anonymous #tree, | ||
| 12 | .mirror-anonymous #entree, | ||
| 13 | .mirror-anonymous #opa_auc, | ||
| 14 | .mirror-anonymous #op_player, | ||
| 15 | .mirror-anonymous #widget { | ||
| 16 | display: none !important; | ||
| 17 | } | ||
| 18 | .mirror-anonymous #path { | ||
| 19 | display: inline-block !important; | ||
| 20 | } | ||
| 21 | .mirror-anonymous #wrap { | ||
| 22 | margin-left: 1.5em !important; | ||
| 23 | padding-bottom: 1.5em; | ||
| 24 | overflow-x: auto; | ||
| 25 | } |
service/copyparty/static/mirrors.js created+12| ... | @@ -0,0 +1,12 @@ | ||
| 1 | (function () { | ||
| 2 | var render = treectl.gentab; | ||
| 3 | treectl.gentab = function (top, listing) { | ||
| 4 | var anonymous = (listing.acct === undefined ? acct : listing.acct) === "*" && /^\/mirrors(?:\/|$)/.test(top); | ||
| 5 | document.body.classList.toggle("mirror-anonymous", anonymous); | ||
| 6 | if (anonymous) | ||
| 7 | read_dsort("-ts"); | ||
| 8 | return render.call(this, top, listing); | ||
| 9 | }; | ||
| 10 | if (ls0 !== null) | ||
| 11 | treectl.gentab(get_evpath(), ls0); | ||
| 12 | })(); | ||
service/zig-mirror/build/mirror.py+446-15| ... | @@ -1,8 +1,15 @@ | ... | @@ -1,8 +1,15 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | import argparse | 2 | import argparse |
| 3 | import base64 | ||
| 4 | import ctypes | ||
| 5 | import ctypes.util | ||
| 6 | import contextlib | ||
| 7 | import datetime | ||
| 3 | import email.utils | 8 | import email.utils |
| 4 | import fcntl | 9 | import fcntl |
| 10 | import functools | ||
| 5 | import hashlib | 11 | import hashlib |
| 12 | import html | ||
| 6 | from html.parser import HTMLParser | 13 | from html.parser import HTMLParser |
| 7 | import http.server | 14 | import http.server |
| 8 | import json | 15 | import json |
| ... | @@ -10,6 +17,7 @@ import os | ... | @@ -10,6 +17,7 @@ import os |
| 10 | from pathlib import Path | 17 | from pathlib import Path |
| 11 | import re | 18 | import re |
| 12 | import shutil | 19 | import shutil |
| 20 | import stat | ||
| 13 | import subprocess | 21 | import subprocess |
| 14 | import tempfile | 22 | import tempfile |
| 15 | import threading | 23 | import threading |
| ... | @@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' | ... | @@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' |
| 24 | OFFICIAL = 'https://ziglang.org' | 32 | OFFICIAL = 'https://ziglang.org' |
| 25 | NEKTRO = 'http://mirrors.nektro.net/zig/' | 33 | NEKTRO = 'http://mirrors.nektro.net/zig/' |
| 26 | MAX_ARTIFACT = 2 * 1024**3 | 34 | MAX_ARTIFACT = 2 * 1024**3 |
| 35 | _index_cache = (float('-inf'), {}) | ||
| 36 | _index_lock = threading.Lock() | ||
| 27 | 37 | ||
| 28 | 38 | ||
| 29 | class Unavailable(Exception): | 39 | class Unavailable(Exception): |
| 30 | pass | 40 | pass |
| 31 | 41 | ||
| 32 | 42 | ||
| 43 | @functools.lru_cache(maxsize=1) | ||
| 44 | def comment_verifier(): | ||
| 45 | name = ctypes.util.find_library('sodium') | ||
| 46 | if not name: | ||
| 47 | raise Unavailable('Minisign comment verification unavailable') | ||
| 48 | library = ctypes.CDLL(name) | ||
| 49 | if library.sodium_init() < 0: | ||
| 50 | raise Unavailable('Minisign comment verification unavailable') | ||
| 51 | verify = library.crypto_sign_verify_detached | ||
| 52 | verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p] | ||
| 53 | verify.restype = ctypes.c_int | ||
| 54 | return verify | ||
| 55 | |||
| 56 | |||
| 57 | def metadata_open(path, final_flags=os.O_RDONLY): | ||
| 58 | path = path.absolute() | ||
| 59 | descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0)) | ||
| 60 | try: | ||
| 61 | for index, part in enumerate(path.parts[1:]): | ||
| 62 | flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0) | ||
| 63 | if index < len(path.parts) - 2: | ||
| 64 | flags |= os.O_DIRECTORY | ||
| 65 | next_descriptor = os.open(part, flags, dir_fd=descriptor) | ||
| 66 | os.close(descriptor) | ||
| 67 | descriptor = next_descriptor | ||
| 68 | return descriptor | ||
| 69 | except BaseException: | ||
| 70 | os.close(descriptor) | ||
| 71 | raise | ||
| 72 | |||
| 73 | |||
| 74 | def signed_time(signature): | ||
| 75 | with os.fdopen(metadata_open(signature), 'rb') as stream: | ||
| 76 | if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): | ||
| 77 | raise Unavailable('signature is not a regular file') | ||
| 78 | body = stream.read(4097) | ||
| 79 | lines = body.splitlines() | ||
| 80 | try: | ||
| 81 | if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '): | ||
| 82 | raise ValueError() | ||
| 83 | key = base64.b64decode(PUBLIC_KEY, validate=True) | ||
| 84 | sig = base64.b64decode(lines[1], validate=True) | ||
| 85 | global_sig = base64.b64decode(lines[3], validate=True) | ||
| 86 | comment = lines[2][len(b'trusted comment: '):] | ||
| 87 | if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]: | ||
| 88 | raise ValueError() | ||
| 89 | # Minisign authenticates the artifact signature followed by the exact trusted comment. | ||
| 90 | message = sig[10:] + comment | ||
| 91 | if comment_verifier()(global_sig, message, len(message), key[10:]) != 0: | ||
| 92 | raise ValueError() | ||
| 93 | match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment) | ||
| 94 | if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'): | ||
| 95 | raise ValueError() | ||
| 96 | return int(match[1]) * 1_000_000_000 | ||
| 97 | except (ValueError, IndexError) as error: | ||
| 98 | raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error | ||
| 99 | |||
| 100 | |||
| 101 | @functools.lru_cache(maxsize=128) | ||
| 102 | def stable_time(version): | ||
| 103 | try: | ||
| 104 | entries = index() | ||
| 105 | if version not in entries: | ||
| 106 | entries = index(refresh=True) | ||
| 107 | date = datetime.date.fromisoformat(entries[version]['date']) | ||
| 108 | return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000 | ||
| 109 | except (KeyError, ValueError) as error: | ||
| 110 | raise Unavailable('official release date unavailable: ' + version) from error | ||
| 111 | |||
| 112 | |||
| 113 | def artifact_time(artifact): | ||
| 114 | version = filename(artifact.name) | ||
| 115 | if '-dev.' in version or version == '0.15.0': | ||
| 116 | return signed_time(Path(str(artifact) + '.minisig')) | ||
| 117 | return stable_time(version) | ||
| 118 | |||
| 119 | |||
| 120 | def children(directory): | ||
| 121 | descriptor = metadata_open(directory) | ||
| 122 | try: | ||
| 123 | with os.scandir(descriptor) as entries: | ||
| 124 | return [(directory / entry.name, entry.stat(follow_symlinks=False)) | ||
| 125 | for entry in entries if not entry.name.startswith('.')] | ||
| 126 | finally: | ||
| 127 | os.close(descriptor) | ||
| 128 | |||
| 129 | |||
| 130 | def set_mtime(path, timestamp): | ||
| 131 | descriptor = metadata_open(path) | ||
| 132 | try: | ||
| 133 | before = os.fstat(descriptor) | ||
| 134 | if before.st_mtime_ns != timestamp: | ||
| 135 | os.utime(descriptor, ns=(before.st_atime_ns, timestamp)) | ||
| 136 | finally: | ||
| 137 | os.close(descriptor) | ||
| 138 | |||
| 139 | |||
| 140 | def file_state(value): | ||
| 141 | return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')} | ||
| 142 | |||
| 143 | |||
| 144 | def version_readme(version, artifacts, development=False): | ||
| 145 | systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows', | ||
| 146 | 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'} | ||
| 147 | source = [] | ||
| 148 | targets = {} | ||
| 149 | |||
| 150 | def link(artifact): | ||
| 151 | name = artifact.name | ||
| 152 | url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/') | ||
| 153 | label = 'tar.xz' if name.endswith('.tar.xz') else 'zip' | ||
| 154 | result = f'[{label}]({url})' | ||
| 155 | signature = Path(str(artifact) + '.minisig') | ||
| 156 | if signature.is_file() and not signature.is_symlink(): | ||
| 157 | result += f' <a href="{url}.minisig" style="background:none!important;border:none!important;font-size:13px;margin-left:-5px;margin-right:-3px" aria-label="Signature for {html.escape(name, quote=True)}">&#128272;</a>' | ||
| 158 | return result | ||
| 159 | |||
| 160 | for artifact in sorted(artifacts): | ||
| 161 | name = artifact.name | ||
| 162 | extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip' | ||
| 163 | target = name[len('zig'): -len('-' + version + extension)] | ||
| 164 | if target in ('', '-bootstrap'): | ||
| 165 | source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact)) | ||
| 166 | continue | ||
| 167 | if target == '-win64': | ||
| 168 | architecture, system = 'x86_64', 'windows' | ||
| 169 | else: | ||
| 170 | first, second = target.removeprefix('-').split('-') | ||
| 171 | system, architecture = (first, second) if first in systems else (second, first) | ||
| 172 | targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact)) | ||
| 173 | columns = list(systems) | ||
| 174 | columns += sorted({system for row in targets.values() for system in row} - set(columns)) | ||
| 175 | heading = '<br>'.join(source) or 'Architecture' | ||
| 176 | lines = [f'### Zig {version}', ''] | ||
| 177 | if not artifacts and development: | ||
| 178 | return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode() | ||
| 179 | if artifacts and '-dev.' not in version: | ||
| 180 | release = index().get(version, {}) | ||
| 181 | for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]: | ||
| 182 | url = release.get(key) | ||
| 183 | if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url): | ||
| 184 | lines.append(f'- [{label}]({url})') | ||
| 185 | if lines[-1]: | ||
| 186 | lines.append('') | ||
| 187 | if development: | ||
| 188 | lines += [f'[Development builds](/mirrors/zig/{version}/dev/)', ''] | ||
| 189 | elif '-dev.' in version: | ||
| 190 | lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', ''] | ||
| 191 | if not artifacts: | ||
| 192 | return '\n'.join(lines).encode() | ||
| 193 | lines += ['<div role="region" aria-label="Zig downloads" tabindex="0" style="max-width:100%;overflow-x:auto;white-space:nowrap;word-break:normal;overflow-wrap:normal">', '', | ||
| 194 | '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |', | ||
| 195 | '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |'] | ||
| 196 | priority = ('x86_64', 'aarch64', 'x86', 'arm') | ||
| 197 | for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)): | ||
| 198 | lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |') | ||
| 199 | lines += ['', '</div>'] | ||
| 200 | if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts): | ||
| 201 | lines += ['', '<span style="font-style:italic;color:orange">Verify your build signatures.</span> <span style="opacity:0.7">The &#128272; links open the Minisign signatures.</span>', ''] | ||
| 202 | else: | ||
| 203 | lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', ''] | ||
| 204 | return '\n'.join(lines).encode() | ||
| 205 | |||
| 206 | |||
| 33 | def filename(name): | 207 | def filename(name): |
| 34 | match = PATTERN.fullmatch(name) | 208 | match = PATTERN.fullmatch(name) |
| 35 | if len(name) > 240 or not match: | 209 | if len(name) > 240 or not match: |
| ... | @@ -72,9 +246,14 @@ def fetch(url, target, limit): | ... | @@ -72,9 +246,14 @@ def fetch(url, target, limit): |
| 72 | raise Unavailable('upstream download failed') from error | 246 | raise Unavailable('upstream download failed') from error |
| 73 | 247 | ||
| 74 | 248 | ||
| 75 | def index(): | 249 | def index(refresh=False): |
| 76 | with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: | 250 | global _index_cache |
| 77 | return json.load(response) | 251 | with _index_lock: |
| 252 | if refresh or time.monotonic() - _index_cache[0] >= 3600: | ||
| 253 | with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: | ||
| 254 | entries = json.load(response) | ||
| 255 | _index_cache = (time.monotonic(), entries) | ||
| 256 | return _index_cache[1] | ||
| 78 | 257 | ||
| 79 | 258 | ||
| 80 | def digest(path): | 259 | def digest(path): |
| ... | @@ -87,7 +266,10 @@ def digest(path): | ... | @@ -87,7 +266,10 @@ def digest(path): |
| 87 | 266 | ||
| 88 | class Mirror: | 267 | class Mirror: |
| 89 | def __init__(self, root): | 268 | def __init__(self, root): |
| 90 | self.root = Path(root) | 269 | self.root = Path(root).absolute() |
| 270 | if self.root.is_symlink(): | ||
| 271 | raise ValueError('mirror root is a symlink') | ||
| 272 | self.root = self.root.resolve() | ||
| 91 | self.root.mkdir(parents=True, exist_ok=True) | 273 | self.root.mkdir(parents=True, exist_ok=True) |
| 92 | self.slots = threading.BoundedSemaphore(4) | 274 | self.slots = threading.BoundedSemaphore(4) |
| 93 | self.refresh_lock = threading.Lock() | 275 | self.refresh_lock = threading.Lock() |
| ... | @@ -98,6 +280,50 @@ class Mirror: | ... | @@ -98,6 +280,50 @@ class Mirror: |
| 98 | raise ValueError('artifact escapes mirror') | 280 | raise ValueError('artifact escapes mirror') |
| 99 | return path | 281 | return path |
| 100 | 282 | ||
| 283 | def lock_directory(self): | ||
| 284 | locks = self.root / '.mirror-locks' | ||
| 285 | descriptor = metadata_open(self.root) | ||
| 286 | try: | ||
| 287 | fcntl.flock(descriptor, fcntl.LOCK_EX) | ||
| 288 | timestamp = os.fstat(descriptor).st_mtime_ns | ||
| 289 | try: | ||
| 290 | locks.mkdir() | ||
| 291 | except FileExistsError: | ||
| 292 | pass | ||
| 293 | else: | ||
| 294 | set_mtime(self.root, timestamp) | ||
| 295 | finally: | ||
| 296 | os.close(descriptor) | ||
| 297 | descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY) | ||
| 298 | os.close(descriptor) | ||
| 299 | return locks | ||
| 300 | |||
| 301 | @contextlib.contextmanager | ||
| 302 | def publishing(self): | ||
| 303 | locks = self.lock_directory() | ||
| 304 | with (locks / '.timestamps').open('a') as lock: | ||
| 305 | fcntl.flock(lock, fcntl.LOCK_EX) | ||
| 306 | yield | ||
| 307 | |||
| 308 | def parent_times(self, artifact): | ||
| 309 | def latest(directory): | ||
| 310 | values = [] | ||
| 311 | for path, value in children(directory): | ||
| 312 | if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path: | ||
| 313 | values.append(value.st_mtime_ns) | ||
| 314 | elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'): | ||
| 315 | values.append(latest(path)) | ||
| 316 | return max(values, default=0) | ||
| 317 | |||
| 318 | directory = artifact.parent | ||
| 319 | while True: | ||
| 320 | timestamp = latest(directory) | ||
| 321 | if timestamp: | ||
| 322 | set_mtime(directory, timestamp) | ||
| 323 | if directory == self.root: | ||
| 324 | break | ||
| 325 | directory = directory.parent | ||
| 326 | |||
| 101 | def ensure(self, name, source=None, checksum=None): | 327 | def ensure(self, name, source=None, checksum=None): |
| 102 | target = self.path(name) | 328 | target = self.path(name) |
| 103 | artifact_name = name.removesuffix('.minisig') | 329 | artifact_name = name.removesuffix('.minisig') |
| ... | @@ -109,8 +335,7 @@ class Mirror: | ... | @@ -109,8 +335,7 @@ class Mirror: |
| 109 | try: | 335 | try: |
| 110 | if shutil.disk_usage(self.root).free < 16 * 1024**3: | 336 | if shutil.disk_usage(self.root).free < 16 * 1024**3: |
| 111 | raise Unavailable('mirror free-space reserve reached') | 337 | raise Unavailable('mirror free-space reserve reached') |
| 112 | locks = self.root / '.mirror-locks' | 338 | locks = self.lock_directory() |
| 113 | locks.mkdir(exist_ok=True) | ||
| 114 | with (locks / artifact_name).open('a') as lock: | 339 | with (locks / artifact_name).open('a') as lock: |
| 115 | fcntl.flock(lock, fcntl.LOCK_EX) | 340 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 116 | if target.is_file(): | 341 | if target.is_file(): |
| ... | @@ -123,8 +348,7 @@ class Mirror: | ... | @@ -123,8 +348,7 @@ class Mirror: |
| 123 | and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) | 348 | and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) |
| 124 | if not checksum: | 349 | if not checksum: |
| 125 | raise Unavailable('unsigned artifact has no official SHA-256') | 350 | raise Unavailable('unsigned artifact has no official SHA-256') |
| 126 | artifact.parent.mkdir(parents=True, exist_ok=True) | 351 | with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: |
| 127 | with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: | ||
| 128 | pending = Path(temporary) / artifact_name | 352 | pending = Path(temporary) / artifact_name |
| 129 | signature = Path(str(pending) + '.minisig') | 353 | signature = Path(str(pending) + '.minisig') |
| 130 | url = source or upstream(artifact_name) | 354 | url = source or upstream(artifact_name) |
| ... | @@ -137,21 +361,216 @@ class Mirror: | ... | @@ -137,21 +361,216 @@ class Mirror: |
| 137 | capture_output=True) | 361 | capture_output=True) |
| 138 | if result.returncode: | 362 | if result.returncode: |
| 139 | raise Unavailable('artifact signature verification failed') | 363 | raise Unavailable('artifact signature verification failed') |
| 364 | timestamp = artifact_time(pending) | ||
| 140 | os.chmod(pending, 0o664) | 365 | os.chmod(pending, 0o664) |
| 141 | os.replace(pending, artifact) | 366 | set_mtime(pending, timestamp) |
| 142 | if signature.exists(): | 367 | if signature.exists(): |
| 143 | os.chmod(signature, 0o664) | 368 | os.chmod(signature, 0o664) |
| 144 | os.replace(signature, Path(str(artifact) + '.minisig')) | 369 | set_mtime(signature, timestamp) |
| 370 | with self.publishing(): | ||
| 371 | self.path(name) | ||
| 372 | artifact.parent.mkdir(parents=True, exist_ok=True) | ||
| 373 | os.replace(pending, artifact) | ||
| 374 | if signature.exists(): | ||
| 375 | os.replace(signature, Path(str(artifact) + '.minisig')) | ||
| 376 | try: | ||
| 377 | self.write_readmes(artifact.parent) | ||
| 378 | finally: | ||
| 379 | self.parent_times(artifact) | ||
| 145 | if name.endswith('.minisig') and not target.exists(): | 380 | if name.endswith('.minisig') and not target.exists(): |
| 146 | with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: | 381 | with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: |
| 147 | pending = Path(temporary) / name | 382 | pending = Path(temporary) / name |
| 148 | fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) | 383 | fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) |
| 149 | os.chmod(pending, 0o664) | 384 | os.chmod(pending, 0o664) |
| 150 | os.replace(pending, target) | 385 | set_mtime(pending, artifact.stat().st_mtime_ns) |
| 386 | with self.publishing(): | ||
| 387 | os.replace(pending, target) | ||
| 388 | try: | ||
| 389 | self.write_readmes(artifact.parent) | ||
| 390 | finally: | ||
| 391 | self.parent_times(artifact) | ||
| 151 | return target | 392 | return target |
| 152 | finally: | 393 | finally: |
| 153 | self.slots.release() | 394 | self.slots.release() |
| 154 | 395 | ||
| 396 | def readme_change(self, directory, artifacts, content=None): | ||
| 397 | path = directory / 'preadme.md' | ||
| 398 | if content is None: | ||
| 399 | content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir()) | ||
| 400 | before = None | ||
| 401 | previous = b'' | ||
| 402 | if path.exists() or path.is_symlink(): | ||
| 403 | descriptor = metadata_open(path) | ||
| 404 | with os.fdopen(descriptor, 'rb') as stream: | ||
| 405 | value = os.fstat(stream.fileno()) | ||
| 406 | if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024: | ||
| 407 | raise Unavailable('version readme is not a small regular file') | ||
| 408 | before = file_state(value) | ||
| 409 | previous = stream.read(512 * 1024 + 1) | ||
| 410 | required = {artifact.name for artifact in artifacts} | ||
| 411 | required |= {artifact.name + '.minisig' for artifact in artifacts | ||
| 412 | if Path(str(artifact) + '.minisig').is_file()} | ||
| 413 | present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1]) | ||
| 414 | for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())} | ||
| 415 | linked = {name for name in present if PATTERN.fullmatch(name)} | ||
| 416 | introduction = content.split(b'<div ', 1)[0] | ||
| 417 | if artifacts and linked == required and previous.startswith(introduction) and b'https://file.paperclover.net' not in previous: | ||
| 418 | return None | ||
| 419 | if previous == content: | ||
| 420 | return None | ||
| 421 | return {'path': str(path.relative_to(self.root)), 'before': before, | ||
| 422 | 'previous': base64.b64encode(previous).decode(), 'content': content.decode()} | ||
| 423 | |||
| 424 | def artifacts(self, directory): | ||
| 425 | return [path for path, value in children(directory) | ||
| 426 | if stat.S_ISREG(value.st_mode) and PATTERN.fullmatch(path.name) | ||
| 427 | and not path.name.endswith('.minisig') and self.path(path.name) == path] | ||
| 428 | |||
| 429 | def directory_readme(self, directory): | ||
| 430 | if directory.name == 'dev': | ||
| 431 | content = f'### Zig {directory.parent.name} development builds\n'.encode() | ||
| 432 | return self.readme_change(directory, [], content) | ||
| 433 | artifacts = self.artifacts(directory) | ||
| 434 | if artifacts or (directory / 'dev').is_dir(): | ||
| 435 | return self.readme_change(directory, artifacts) | ||
| 436 | |||
| 437 | def write_readmes(self, directory): | ||
| 438 | directories = [directory] | ||
| 439 | if directory.parent.name == 'dev': | ||
| 440 | directories += [directory.parent, directory.parent.parent] | ||
| 441 | for current in directories: | ||
| 442 | change = self.directory_readme(current) | ||
| 443 | if change: | ||
| 444 | self.apply_readme(change) | ||
| 445 | |||
| 446 | def apply_readme(self, change): | ||
| 447 | path = self.root / change['path'] | ||
| 448 | if change['before'] is None: | ||
| 449 | parent = metadata_open(path.parent) | ||
| 450 | try: | ||
| 451 | descriptor = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o664, dir_fd=parent) | ||
| 452 | os.fchown(descriptor, -1, os.fstat(parent).st_gid) | ||
| 453 | finally: | ||
| 454 | os.close(parent) | ||
| 455 | before = os.fstat(descriptor) | ||
| 456 | else: | ||
| 457 | descriptor = metadata_open(path, os.O_WRONLY) | ||
| 458 | before = os.fstat(descriptor) | ||
| 459 | if any(file_state(before)[key] != value for key, value in change['before'].items() if key != 'atime_ns'): | ||
| 460 | os.close(descriptor) | ||
| 461 | raise Unavailable('version readme changed since plan') | ||
| 462 | try: | ||
| 463 | os.ftruncate(descriptor, 0) | ||
| 464 | with os.fdopen(os.dup(descriptor), 'wb') as output: | ||
| 465 | output.write(change['content'].encode()) | ||
| 466 | output.flush() | ||
| 467 | os.fsync(output.fileno()) | ||
| 468 | os.utime(descriptor, ns=(before.st_atime_ns, before.st_mtime_ns)) | ||
| 469 | finally: | ||
| 470 | os.close(descriptor) | ||
| 471 | |||
| 472 | def metadata_plan(self): | ||
| 473 | files = {} | ||
| 474 | folders = {} | ||
| 475 | errors = [] | ||
| 476 | pending = [self.root] | ||
| 477 | while pending: | ||
| 478 | for artifact, value in children(pending.pop()): | ||
| 479 | if stat.S_ISDIR(value.st_mode): | ||
| 480 | pending.append(artifact) | ||
| 481 | continue | ||
| 482 | if not stat.S_ISREG(value.st_mode) or not PATTERN.fullmatch(artifact.name) or artifact.name.endswith('.minisig'): | ||
| 483 | continue | ||
| 484 | try: | ||
| 485 | if artifact.relative_to(self.root) != relative(artifact.name): | ||
| 486 | raise Unavailable('misplaced Zig artifact') | ||
| 487 | timestamp = artifact_time(artifact) | ||
| 488 | files[artifact] = timestamp | ||
| 489 | signature = Path(str(artifact) + '.minisig') | ||
| 490 | if signature.exists(): | ||
| 491 | with os.fdopen(metadata_open(signature), 'rb') as stream: | ||
| 492 | if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): | ||
| 493 | raise Unavailable('signature is not a regular file') | ||
| 494 | files[signature] = timestamp | ||
| 495 | directory = artifact.parent | ||
| 496 | while True: | ||
| 497 | folders[directory] = max(folders.get(directory, 0), timestamp) | ||
| 498 | if directory == self.root: | ||
| 499 | break | ||
| 500 | directory = directory.parent | ||
| 501 | except (Unavailable, OSError, ValueError) as error: | ||
| 502 | errors.append({'path': str(artifact.relative_to(self.root)), 'error': str(error)}) | ||
| 503 | readmes = [] | ||
| 504 | for directory in sorted(set(folders) - {self.root}): | ||
| 505 | try: | ||
| 506 | change = self.directory_readme(directory) | ||
| 507 | if change: | ||
| 508 | readmes.append(change) | ||
| 509 | except (Unavailable, OSError, ValueError) as error: | ||
| 510 | errors.append({'path': str(directory.relative_to(self.root)), 'error': str(error)}) | ||
| 511 | entries = [] | ||
| 512 | for path, timestamp in sorted({**files, **folders}.items()): | ||
| 513 | descriptor = metadata_open(path) | ||
| 514 | try: | ||
| 515 | entries.append({'path': str(path.relative_to(self.root)), 'before': file_state(os.fstat(descriptor)), 'mtime_ns': timestamp}) | ||
| 516 | finally: | ||
| 517 | os.close(descriptor) | ||
| 518 | return {'root': str(self.root), 'files': len(files), 'folders': len(folders), | ||
| 519 | 'changed': sum(entry['before']['mtime_ns'] != entry['mtime_ns'] for entry in entries), | ||
| 520 | 'provenance': {'stable': OFFICIAL + '/download/index.json UTC date', 'development': 'pinned Minisign authenticated trusted-comment timestamp', 'withdrawn0.15.0': 'authenticated signing time; release retracted before 0.15.1', 'folders': 'newest contained artifact timestamp'}, | ||
| 521 | 'errors': errors, 'entries': entries, 'readmes': readmes} | ||
| 522 | |||
| 523 | def reconcile(self, apply=False, journal=None): | ||
| 524 | with self.publishing() if apply else contextlib.nullcontext(): | ||
| 525 | plan = self.metadata_plan() | ||
| 526 | if apply: | ||
| 527 | if plan['errors']: | ||
| 528 | raise Unavailable('timestamp plan has unresolved dates') | ||
| 529 | if journal is None or journal.resolve().is_relative_to(self.root): | ||
| 530 | raise ValueError('private journal outside the mirror is required') | ||
| 531 | journal = journal.resolve() | ||
| 532 | descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) | ||
| 533 | with os.fdopen(descriptor, 'w') as output: | ||
| 534 | json.dump(plan, output) | ||
| 535 | output.write('\n') | ||
| 536 | output.flush() | ||
| 537 | os.fsync(output.fileno()) | ||
| 538 | descriptor = metadata_open(journal.parent) | ||
| 539 | try: | ||
| 540 | os.fsync(descriptor) | ||
| 541 | finally: | ||
| 542 | os.close(descriptor) | ||
| 543 | for change in plan['readmes']: | ||
| 544 | self.apply_readme(change) | ||
| 545 | for entry in plan['entries']: | ||
| 546 | if entry['before']['mtime_ns'] == entry['mtime_ns'] and not stat.S_ISDIR(entry['before']['mode']): | ||
| 547 | continue | ||
| 548 | descriptor = metadata_open(self.root / entry['path']) | ||
| 549 | try: | ||
| 550 | current = file_state(os.fstat(descriptor)) | ||
| 551 | expected = entry['before'] | ||
| 552 | fields = ('dev', 'ino', 'mode', 'uid', 'gid') if stat.S_ISDIR(current['mode']) else tuple(key for key in expected if key != 'atime_ns') | ||
| 553 | if any(current[key] != expected[key] for key in fields): | ||
| 554 | raise Unavailable('file changed since timestamp plan: ' + entry['path']) | ||
| 555 | if current['mtime_ns'] != entry['mtime_ns']: | ||
| 556 | os.utime(descriptor, ns=(current['atime_ns'], entry['mtime_ns'])) | ||
| 557 | finally: | ||
| 558 | os.close(descriptor) | ||
| 559 | applied = {} | ||
| 560 | paths = {entry['path'] for entry in plan['entries']} | {change['path'] for change in plan['readmes']} | ||
| 561 | for path in sorted(paths): | ||
| 562 | descriptor = metadata_open(self.root / path) | ||
| 563 | try: | ||
| 564 | applied[path] = file_state(os.fstat(descriptor)) | ||
| 565 | finally: | ||
| 566 | os.close(descriptor) | ||
| 567 | with os.fdopen(metadata_open(journal, os.O_WRONLY | os.O_APPEND), 'w') as output: | ||
| 568 | json.dump({'applied': applied}, output) | ||
| 569 | output.write('\n') | ||
| 570 | output.flush() | ||
| 571 | os.fsync(output.fileno()) | ||
| 572 | return plan | ||
| 573 | |||
| 155 | def verify(self): | 574 | def verify(self): |
| 156 | entries = index() | 575 | entries = index() |
| 157 | hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] | 576 | hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] |
| ... | @@ -181,13 +600,20 @@ class Mirror: | ... | @@ -181,13 +600,20 @@ class Mirror: |
| 181 | if not self.refresh_lock.acquire(blocking=False): | 600 | if not self.refresh_lock.acquire(blocking=False): |
| 182 | return | 601 | return |
| 183 | try: | 602 | try: |
| 184 | entries = index() | 603 | entries = index(refresh=True) |
| 185 | latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))] | 604 | latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))] |
| 186 | for release in (latest, entries['master']): | 605 | for release in (latest, entries['master']): |
| 606 | artifact = None | ||
| 187 | for item in release.values(): | 607 | for item in release.values(): |
| 188 | if isinstance(item, dict) and 'tarball' in item: | 608 | if isinstance(item, dict) and 'tarball' in item: |
| 189 | name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]) | 609 | name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]) |
| 190 | self.ensure(name, checksum=item['shasum']) | 610 | artifact = self.ensure(name, checksum=item['shasum']) |
| 611 | if artifact is not None: | ||
| 612 | with self.publishing(): | ||
| 613 | try: | ||
| 614 | self.write_readmes(artifact.parent) | ||
| 615 | finally: | ||
| 616 | self.parent_times(artifact) | ||
| 191 | finally: | 617 | finally: |
| 192 | self.refresh_lock.release() | 618 | self.refresh_lock.release() |
| 193 | 619 | ||
| ... | @@ -306,15 +732,20 @@ def backfill(mirror, budget, apply): | ... | @@ -306,15 +732,20 @@ def backfill(mirror, budget, apply): |
| 306 | 732 | ||
| 307 | def main(): | 733 | def main(): |
| 308 | parser = argparse.ArgumentParser() | 734 | parser = argparse.ArgumentParser() |
| 309 | parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify']) | 735 | parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify', 'reconcile']) |
| 310 | parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror')) | 736 | parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror')) |
| 311 | parser.add_argument('--budget-gib', type=int, default=32) | 737 | parser.add_argument('--budget-gib', type=int, default=32) |
| 312 | parser.add_argument('--apply', action='store_true') | 738 | parser.add_argument('--apply', action='store_true') |
| 739 | parser.add_argument('--journal', type=Path) | ||
| 313 | args = parser.parse_args() | 740 | args = parser.parse_args() |
| 314 | os.umask(0o002) | 741 | os.umask(0o002) |
| 742 | if args.command == 'reconcile' and not Path(args.root).is_dir(): | ||
| 743 | raise ValueError('existing mirror root is required') | ||
| 315 | mirror = Mirror(args.root) | 744 | mirror = Mirror(args.root) |
| 316 | if args.command == 'verify': | 745 | if args.command == 'verify': |
| 317 | mirror.verify() | 746 | mirror.verify() |
| 747 | elif args.command == 'reconcile': | ||
| 748 | print(json.dumps(mirror.reconcile(args.apply, args.journal))) | ||
| 318 | elif args.command == 'refresh': | 749 | elif args.command == 'refresh': |
| 319 | mirror.refresh() | 750 | mirror.refresh() |
| 320 | elif args.command == 'backfill': | 751 | elif args.command == 'backfill': |
service/zig-mirror/fixture/zig-x86_64-linux-0.15.0.tar.xz.minisig created+4| ... | @@ -0,0 +1,4 @@ | ||
| 1 | untrusted comment: signature from minisign secret key | ||
| 2 | RUSGOq2NVecA2RV/TllU7u4sMW2JwbgaUIX6upgmh5dqHIle3ERNdeh58+iO14pz84aSsJzW8yWISzgmbf/AKPvj09piCUZmGws= | ||
| 3 | trusted comment: timestamp:1755639580	file:zig-x86_64-linux-0.15.0.tar.xz	hashed | ||
| 4 | RO3woTlcv5vkx53pBuILPCuk3u74YyiPQqWvbLJzcOacBRhZ6nQcBM7RvBCyNts7v2Jt3CmbEULL0yfppsrpAA== | ||
service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig created+4| ... | @@ -0,0 +1,4 @@ | ||
| 1 | untrusted comment: signature from minisign secret key | ||
| 2 | RUSGOq2NVecA2RwFmP6WKntiiWhQqtOhDbzNAVDP/x3uN6v1EcAMLi5uCCw1kisYZ1Dl6SOJTU6/6FT4pJDgRK093OhyF8QHfQg= | ||
| 3 | trusted comment: timestamp:1791194091	file:zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip	hashed | ||
| 4 | m6Yh4Pw9wtV1WB26pz7pMTQ6gdjUABVuOTN66QXuwvuoKn0ixv6DO01+QtJQnaZfcQg1EnJGp854sGjb627pDA== | ||
service/zig-mirror/prepare.py deleted-14| ... | @@ -1,14 +0,0 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import json | ||
| 3 | import os | ||
| 4 | from pathlib import Path | ||
| 5 | import sys | ||
| 6 | |||
| 7 | data = json.load(sys.stdin) | ||
| 8 | if Path(data['hostRoot']).name != 'zig-mirror': | ||
| 9 | root = Path(data['hostRoot']) / 'mirror' | ||
| 10 | artifact = root / '0.0.0/zig-0.0.0.tar.xz' | ||
| 11 | artifact.parent.mkdir(parents=True, exist_ok=True) | ||
| 12 | artifact.write_bytes(b'Synthetic Zig mirror preview.\n') | ||
| 13 | for path in (artifact.parent, artifact): | ||
| 14 | os.chown(path, data['uid'], data['uid']) | ||
service/zig-mirror/service.pkl+2-1| ... | @@ -5,10 +5,11 @@ import "../../config/site.pkl" as site | ... | @@ -5,10 +5,11 @@ import "../../config/site.pkl" as site |
| 5 | name = "Zig mirror" | 5 | name = "Zig mirror" |
| 6 | launcher = false | 6 | launcher = false |
| 7 | stageIsolation = "fresh" | 7 | stageIsolation = "fresh" |
| 8 | prepare = "prepare.py" | ||
| 9 | 8 | ||
| 10 | container { | 9 | container { |
| 11 | build = "build" | 10 | build = "build" |
| 11 | readOnlyRoot = true | ||
| 12 | capAdd { "FOWNER" } | ||
| 12 | http { | 13 | http { |
| 13 | containerPort = 8080 | 14 | containerPort = 8080 |
| 14 | subdomain = "zig" | 15 | subdomain = "zig" |
tools/snowglobe-secret-files-test.py+12| ... | @@ -43,6 +43,18 @@ class SecretFiles(unittest.TestCase): | ... | @@ -43,6 +43,18 @@ class SecretFiles(unittest.TestCase): |
| 43 | self.assertNotIn('template {', rendered) | 43 | self.assertNotIn('template {', rendered) |
| 44 | self.assertNotIn('secrets/', rendered) | 44 | self.assertNotIn('secrets/', rendered) |
| 45 | 45 | ||
| 46 | def test_read_only_root_is_opt_in_and_preserves_writable_data_mounts(self): | ||
| 47 | self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data})) | ||
| 48 | self.task["readOnlyRoot"] = True | ||
| 49 | self.task["volumes"] = {"/mirror": {"src": "/fixture/mirror", "readOnly": False}} | ||
| 50 | self.task["capAdd"] = ["FOWNER"] | ||
| 51 | rendered = snowglobe.render(self.data, {"fixture": self.data}) | ||
| 52 | self.assertIn('readonly_rootfs = true', rendered) | ||
| 53 | self.assertIn('cap_add = ["FOWNER"]', rendered) | ||
| 54 | self.assertIn('/fixture/mirror:/mirror"', rendered) | ||
| 55 | del self.task["readOnlyRoot"] | ||
| 56 | self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data})) | ||
| 57 | |||
| 46 | def test_unsafe_and_colliding_targets_are_rejected(self): | 58 | def test_unsafe_and_colliding_targets_are_rejected(self): |
| 47 | for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"): | 59 | for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"): |
| 48 | with self.subTest(target=target): | 60 | with self.subTest(target=target): |
tools/snowglobe.py+2| ... | @@ -398,6 +398,8 @@ def render(data, definitions): | ... | @@ -398,6 +398,8 @@ def render(data, definitions): |
| 398 | lines.append(f" args = {q(task['args'])}") | 398 | lines.append(f" args = {q(task['args'])}") |
| 399 | if task["hostNetwork"]: | 399 | if task["hostNetwork"]: |
| 400 | lines.append(' network_mode = "host"') | 400 | lines.append(' network_mode = "host"') |
| 401 | if task.get("readOnlyRoot"): | ||
| 402 | lines.append(' readonly_rootfs = true') | ||
| 401 | task_ports = [label for (owner, _), label in ports.items() if owner == task_name] | 403 | task_ports = [label for (owner, _), label in ports.items() if owner == task_name] |
| 402 | if task_ports and not task["hostNetwork"]: | 404 | if task_ports and not task["hostNetwork"]: |
| 403 | lines.append(f" ports = {q(task_ports)}") | 405 | lines.append(f" ports = {q(task_ports)}") |
tools/zig-mirror-test.py+257-1| ... | @@ -3,7 +3,10 @@ import concurrent.futures | ... | @@ -3,7 +3,10 @@ import concurrent.futures |
| 3 | import importlib.util | 3 | import importlib.util |
| 4 | import io | 4 | import io |
| 5 | import contextlib | 5 | import contextlib |
| 6 | import ctypes.util | ||
| 7 | import datetime | ||
| 6 | import json | 8 | import json |
| 9 | import os | ||
| 7 | from pathlib import Path | 10 | from pathlib import Path |
| 8 | from types import SimpleNamespace | 11 | from types import SimpleNamespace |
| 9 | import tempfile | 12 | import tempfile |
| ... | @@ -19,6 +22,50 @@ mirror = importlib.util.module_from_spec(spec) | ... | @@ -19,6 +22,50 @@ mirror = importlib.util.module_from_spec(spec) |
| 19 | spec.loader.exec_module(mirror) | 22 | spec.loader.exec_module(mirror) |
| 20 | 23 | ||
| 21 | 24 | ||
| 25 | class IndexTests(unittest.TestCase): | ||
| 26 | def setUp(self): | ||
| 27 | cache = patch.object(mirror, '_index_cache', (float('-inf'), {})) | ||
| 28 | cache.start() | ||
| 29 | self.addCleanup(cache.stop) | ||
| 30 | mirror.stable_time.cache_clear() | ||
| 31 | |||
| 32 | def test_index_coalesces_concurrent_reads_expires_and_forces_refresh(self): | ||
| 33 | snapshot = {'0.17.0': {'docs': 'https://ziglang.org/documentation/0.17.0/'}} | ||
| 34 | def response(*args, **kwargs): | ||
| 35 | time.sleep(.01) | ||
| 36 | return io.BytesIO(json.dumps(snapshot).encode()) | ||
| 37 | with patch.object(mirror.urllib.request, 'urlopen', side_effect=response) as request, patch.object(mirror.time, 'monotonic', return_value=0) as clock: | ||
| 38 | with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers: | ||
| 39 | entries = list(workers.map(lambda _: mirror.index(), range(8))) | ||
| 40 | self.assertTrue(all(entry == snapshot for entry in entries)) | ||
| 41 | self.assertEqual(request.call_count, 1) | ||
| 42 | clock.return_value = 3599 | ||
| 43 | mirror.index() | ||
| 44 | self.assertEqual(request.call_count, 1) | ||
| 45 | clock.return_value = 3600 | ||
| 46 | mirror.index() | ||
| 47 | self.assertEqual(request.call_count, 2) | ||
| 48 | mirror.index(refresh=True) | ||
| 49 | self.assertEqual(request.call_count, 3) | ||
| 50 | self.assertTrue(all(call.args[0] == 'https://ziglang.org/download/index.json' for call in request.call_args_list)) | ||
| 51 | |||
| 52 | def test_new_release_miss_refreshes_stale_index_before_publication(self): | ||
| 53 | release = {'date': '2027-04-01', 'docs': 'https://ziglang.org/documentation/0.19.0/'} | ||
| 54 | responses = [{}, {'0.19.0': release}] | ||
| 55 | with tempfile.TemporaryDirectory() as root, patch.object(mirror.urllib.request, 'urlopen', side_effect=lambda *args, **kwargs: io.BytesIO(json.dumps(responses.pop(0)).encode())) as request: | ||
| 56 | mirror.index() | ||
| 57 | cache = mirror.Mirror(root) | ||
| 58 | with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)), patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')), patch.object(mirror.subprocess, 'run', return_value=SimpleNamespace(returncode=0)): | ||
| 59 | artifact = cache.ensure('zig-x86_64-linux-0.19.0.tar.xz') | ||
| 60 | self.assertEqual(request.call_count, 2) | ||
| 61 | self.assertIn('[Language Reference](' + release['docs'] + ')', (artifact.parent / 'preadme.md').read_text()) | ||
| 62 | expected = int(datetime.datetime(2027, 4, 1, tzinfo=datetime.timezone.utc).timestamp()) * 10**9 | ||
| 63 | self.assertEqual(artifact.stat().st_mtime_ns, expected) | ||
| 64 | self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, expected) | ||
| 65 | self.assertEqual(artifact.parent.stat().st_mtime_ns, expected) | ||
| 66 | self.assertEqual(cache.root.stat().st_mtime_ns, expected) | ||
| 67 | |||
| 68 | |||
| 22 | class MirrorTests(unittest.TestCase): | 69 | class MirrorTests(unittest.TestCase): |
| 23 | def setUp(self): | 70 | def setUp(self): |
| 24 | self.temporary = tempfile.TemporaryDirectory() | 71 | self.temporary = tempfile.TemporaryDirectory() |
| ... | @@ -27,6 +74,16 @@ class MirrorTests(unittest.TestCase): | ... | @@ -27,6 +74,16 @@ class MirrorTests(unittest.TestCase): |
| 27 | disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)) | 74 | disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)) |
| 28 | disk.start() | 75 | disk.start() |
| 29 | self.addCleanup(disk.stop) | 76 | self.addCleanup(disk.stop) |
| 77 | mirror.stable_time.cache_clear() | ||
| 78 | dates = {version: {'date': date} for version, date in [('0.1.1', '2017-10-17'), ('0.2.0', '2018-03-15'), | ||
| 79 | ('0.3.0', '2018-09-28'), ('0.4.0', '2019-04-08'), ('0.5.0', '2019-09-30'), ('0.17.0', '2026-10-01')]} | ||
| 80 | metadata = patch.object(mirror, 'index', return_value=dates) | ||
| 81 | metadata.start() | ||
| 82 | self.addCleanup(metadata.stop) | ||
| 83 | |||
| 84 | def dev_fixture(self): | ||
| 85 | fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig' | ||
| 86 | return fixture.name.removesuffix('.minisig'), fixture.read_bytes() | ||
| 30 | 87 | ||
| 31 | def test_spec_filenames_and_existing_layout(self): | 88 | def test_spec_filenames_and_existing_layout(self): |
| 32 | cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), | 89 | cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), |
| ... | @@ -150,7 +207,7 @@ class MirrorTests(unittest.TestCase): | ... | @@ -150,7 +207,7 @@ class MirrorTests(unittest.TestCase): |
| 150 | def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self): | 207 | def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self): |
| 151 | names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz'] | 208 | names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz'] |
| 152 | body = b'official historical archive' | 209 | body = b'official historical archive' |
| 153 | entries = {mirror.filename(name): {'platform': {'tarball': mirror.upstream(name), | 210 | entries = {mirror.filename(name): {'date': '2017-10-17', 'platform': {'tarball': mirror.upstream(name), |
| 154 | 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}} | 211 | 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}} |
| 155 | for i, name in enumerate(names)} | 212 | for i, name in enumerate(names)} |
| 156 | def fetch(url, target, limit): | 213 | def fetch(url, target, limit): |
| ... | @@ -179,6 +236,205 @@ class MirrorTests(unittest.TestCase): | ... | @@ -179,6 +236,205 @@ class MirrorTests(unittest.TestCase): |
| 179 | self.assertEqual(response.read(), body) | 236 | self.assertEqual(response.read(), body) |
| 180 | self.assertFalse(list(self.cache.root.rglob('.download-*'))) | 237 | self.assertFalse(list(self.cache.root.rglob('.download-*'))) |
| 181 | 238 | ||
| 239 | @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required') | ||
| 240 | def test_real_minisign_comment_and_tampering(self): | ||
| 241 | name, body = self.dev_fixture() | ||
| 242 | signature = self.cache.root / (name + '.minisig') | ||
| 243 | signature.write_bytes(body) | ||
| 244 | self.assertEqual(mirror.signed_time(signature), 1791194091 * 10**9) | ||
| 245 | for changed in [body.replace(b'1791194091', b'1791194092'), body.replace(b'RUSGO', b'RUSGP'), | ||
| 246 | body.replace(b'trusted comment: timestamp:', b'trusted comment: forged:')]: | ||
| 247 | signature.write_bytes(changed) | ||
| 248 | with self.assertRaises(mirror.Unavailable): | ||
| 249 | mirror.signed_time(signature) | ||
| 250 | signature.unlink() | ||
| 251 | os.mkfifo(signature) | ||
| 252 | with self.assertRaises(mirror.Unavailable): | ||
| 253 | mirror.signed_time(signature) | ||
| 254 | |||
| 255 | @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required') | ||
| 256 | def test_publisher_dates_pairs_and_folders_after_cleanup(self): | ||
| 257 | name, signature = self.dev_fixture() | ||
| 258 | def fetch(url, target, limit): | ||
| 259 | target.write_bytes(signature if url.endswith('.minisig') and '-dev.' in url else b'fixture') | ||
| 260 | with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify: | ||
| 261 | verify.return_value.returncode = 0 | ||
| 262 | artifact = self.cache.ensure(name) | ||
| 263 | stamp = 1791194091 * 10**9 | ||
| 264 | self.assertEqual(artifact.stat().st_mtime_ns, stamp) | ||
| 265 | self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, stamp) | ||
| 266 | for directory in [artifact.parent, artifact.parent.parent, artifact.parent.parent.parent, self.cache.root]: | ||
| 267 | self.assertEqual(directory.stat().st_mtime_ns, stamp) | ||
| 268 | (self.cache.root / '0.99.0').mkdir() | ||
| 269 | older = self.cache.ensure('zig-0.17.0.tar.xz') | ||
| 270 | self.assertEqual(older.stat().st_mtime_ns, 1790812800 * 10**9) | ||
| 271 | self.assertEqual(self.cache.root.stat().st_mtime_ns, stamp) | ||
| 272 | self.assertFalse(list(self.cache.root.rglob('.download-*'))) | ||
| 273 | self.assertIn('Signature for '+name, (artifact.parent/'preadme.md').read_text()) | ||
| 274 | self.assertIn('this release only has [development builds]', (artifact.parent.parent.parent/'preadme.md').read_text()) | ||
| 275 | |||
| 276 | @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required') | ||
| 277 | def test_reconcile_is_metadata_only_idempotent_and_journalled(self): | ||
| 278 | name, signature = self.dev_fixture() | ||
| 279 | artifact = self.cache.path(name);artifact.parent.mkdir(parents=True) | ||
| 280 | artifact.write_bytes(b'public compiler fixture') | ||
| 281 | Path(str(artifact)+'.minisig').write_bytes(signature) | ||
| 282 | stable = self.cache.path('zig-win64-0.1.1.zip');stable.parent.mkdir();stable.write_bytes(b'legacy') | ||
| 283 | root_readme = self.cache.root/'preadme.md';root_readme.write_text('owner root notes') | ||
| 284 | version_readme = artifact.parent/'preadme.md';version_readme.write_text('old public version notes') | ||
| 285 | hidden = self.cache.root/'.hist';hidden.mkdir();(hidden/'zig-0.17.0.tar.xz').write_bytes(b'private history') | ||
| 286 | original = {path: (path.stat().st_ino,path.stat().st_uid,path.stat().st_gid,path.stat().st_mode,path.stat().st_size) | ||
| 287 | for path in [artifact,Path(str(artifact)+'.minisig'),stable,root_readme,hidden/'zig-0.17.0.tar.xz']} | ||
| 288 | plan = self.cache.reconcile() | ||
| 289 | self.assertFalse(plan['errors']) | ||
| 290 | self.assertFalse((self.cache.root/'.mirror-locks').exists()) | ||
| 291 | with tempfile.TemporaryDirectory() as private: | ||
| 292 | journal=Path(private)/'undo.json' | ||
| 293 | self.cache.reconcile(True,journal) | ||
| 294 | self.assertEqual(journal.stat().st_mode&0o777,0o600) | ||
| 295 | records = [json.loads(line) for line in journal.read_text().splitlines()] | ||
| 296 | self.assertTrue(records[0]['readmes']) | ||
| 297 | self.assertEqual(set(records[1]['applied']), | ||
| 298 | {entry['path'] for entry in records[0]['entries']} | | ||
| 299 | {change['path'] for change in records[0]['readmes']}) | ||
| 300 | with self.assertRaises(FileExistsError): | ||
| 301 | self.cache.reconcile(True, journal) | ||
| 302 | with self.assertRaises(ValueError): | ||
| 303 | self.cache.reconcile(True, self.cache.root / 'public-undo.json') | ||
| 304 | for path,state in original.items(): | ||
| 305 | actual=path.stat();self.assertEqual((actual.st_ino,actual.st_uid,actual.st_gid,actual.st_mode,actual.st_size),state) | ||
| 306 | self.assertEqual(root_readme.read_text(),'owner root notes') | ||
| 307 | self.assertEqual(artifact.read_bytes(),b'public compiler fixture') | ||
| 308 | self.assertEqual(self.cache.reconcile()['changed'],0) | ||
| 309 | self.assertEqual(self.cache.reconcile()['readmes'],[]) | ||
| 310 | |||
| 311 | def test_hidden_lock_creation_preserves_visible_root_date(self): | ||
| 312 | os.utime(self.cache.root, ns=(1234, 5678)) | ||
| 313 | with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers: | ||
| 314 | list(workers.map(lambda _: self.cache.lock_directory(), range(12))) | ||
| 315 | self.assertEqual(self.cache.root.stat().st_mtime_ns, 5678) | ||
| 316 | |||
| 317 | @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required') | ||
| 318 | def test_withdrawn_0150_uses_authenticated_signing_time(self): | ||
| 319 | name = 'zig-x86_64-linux-0.15.0.tar.xz' | ||
| 320 | fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture' / (name + '.minisig') | ||
| 321 | artifact = self.cache.path(name) | ||
| 322 | artifact.parent.mkdir() | ||
| 323 | artifact.write_bytes(b'already verified archive') | ||
| 324 | signature = Path(str(artifact) + '.minisig') | ||
| 325 | signature.write_bytes(fixture.read_bytes()) | ||
| 326 | with patch.object(mirror, 'index', side_effect=AssertionError('withdrawn release must not use another release date')): | ||
| 327 | self.assertEqual(mirror.artifact_time(artifact), mirror.signed_time(signature)) | ||
| 328 | signature.write_bytes(fixture.read_bytes().replace(b'timestamp:', b'tampered:')) | ||
| 329 | with self.assertRaises(mirror.Unavailable): | ||
| 330 | mirror.artifact_time(artifact) | ||
| 331 | |||
| 332 | def test_missing_release_date_never_publishes(self): | ||
| 333 | with patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')): | ||
| 334 | with self.assertRaises(mirror.Unavailable): | ||
| 335 | self.cache.ensure('zig-0.4.1.tar.xz', checksum=mirror.hashlib.sha256(b'fixture').hexdigest()) | ||
| 336 | self.assertFalse(self.cache.path('zig-0.4.1.tar.xz').exists()) | ||
| 337 | |||
| 338 | def test_metadata_traversal_rejects_intermediate_symlinks(self): | ||
| 339 | with tempfile.TemporaryDirectory() as outside: | ||
| 340 | target=Path(outside)/'file';target.write_bytes(b'untouched') | ||
| 341 | (self.cache.root/'alias').symlink_to(outside) | ||
| 342 | with self.assertRaises(OSError): | ||
| 343 | mirror.set_mtime(self.cache.root/'alias/file',1) | ||
| 344 | |||
| 345 | def test_dev_index_replaces_duplicate_list_with_heading(self): | ||
| 346 | artifact = self.cache.path('zig-x86_64-linux-0.18.0-dev.4+5a23bf4b2.tar.xz') | ||
| 347 | artifact.parent.mkdir(parents=True) | ||
| 348 | artifact.write_bytes(b'public compiler fixture') | ||
| 349 | directory = artifact.parent.parent | ||
| 350 | readme = directory / 'preadme.md' | ||
| 351 | readme.write_text('### Zig 0.18.0 development builds\n\n- [0.18.0-dev.4+5a23bf4b2](old-link/)\n') | ||
| 352 | previous = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, directory, directory.parent, self.cache.root]} | ||
| 353 | change = self.cache.directory_readme(directory) | ||
| 354 | self.assertEqual(change['content'], '### Zig 0.18.0 development builds\n') | ||
| 355 | self.cache.apply_readme(change) | ||
| 356 | self.assertIsNone(self.cache.directory_readme(directory)) | ||
| 357 | self.assertEqual(readme.read_text(), change['content']) | ||
| 358 | for path, timestamp in previous.items(): | ||
| 359 | self.assertEqual(path.stat().st_mtime_ns, timestamp) | ||
| 360 | |||
| 361 | def test_development_only_base_has_exact_sentence_without_release_docs(self): | ||
| 362 | directory = self.cache.root / '0.18.0' | ||
| 363 | (directory / 'dev').mkdir(parents=True) | ||
| 364 | with patch.object(mirror, 'index', side_effect=AssertionError('development navigation needs no release metadata')): | ||
| 365 | change = self.cache.directory_readme(directory) | ||
| 366 | self.assertEqual(change['content'], '### Zig 0.18.0\n\nthis release only has [development builds](/mirrors/zig/0.18.0/dev/) out.\n') | ||
| 367 | self.cache.apply_readme(change) | ||
| 368 | self.assertIsNone(self.cache.directory_readme(directory)) | ||
| 369 | |||
| 370 | def test_release_links_follow_index_presence_and_precede_artifact_table(self): | ||
| 371 | artifact = self.cache.path('zig-x86_64-linux-0.19.0.tar.xz') | ||
| 372 | release = {'notes': 'https://ziglang.org/download/0.19.0/release-notes.html', | ||
| 373 | 'docs': 'https://ziglang.org/documentation/0.19.0/', | ||
| 374 | 'stdDocs': 'https://ziglang.org/documentation/0.19.0/std/'} | ||
| 375 | labels = {'notes': 'Release Notes', 'docs': 'Language Reference', 'stdDocs': 'Standard Library Documentation'} | ||
| 376 | for absent in [None, *release]: | ||
| 377 | with self.subTest(absent=absent), patch.object(mirror, 'index', return_value={'0.19.0': {key: url for key, url in release.items() if key != absent}}): | ||
| 378 | text = mirror.version_readme('0.19.0', [artifact], True).decode() | ||
| 379 | for key, url in release.items(): | ||
| 380 | if key == absent: | ||
| 381 | self.assertNotIn(f'[{labels[key]}]({url})', text) | ||
| 382 | else: | ||
| 383 | self.assertIn(f'- [{labels[key]}]({url})', text) | ||
| 384 | self.assertLess(text.index(url), text.index('[Development builds]')) | ||
| 385 | self.assertLess(text.index('[Development builds]'), text.index('<div ')) | ||
| 386 | with patch.object(mirror, 'index', return_value={'0.19.0': {'notes': 'https://example.org/notes', 'docs': None, 'stdDocs': 'https://ziglang.org/bad)link'}}): | ||
| 387 | text = mirror.version_readme('0.19.0', [artifact]).decode() | ||
| 388 | for label in labels.values(): | ||
| 389 | self.assertNotIn(label, text) | ||
| 390 | self.assertIn(artifact.name, text) | ||
| 391 | |||
| 392 | def test_refresh_updates_cached_release_links_without_changing_dates(self): | ||
| 393 | artifact = self.cache.path('zig-x86_64-linux-0.17.0.tar.xz') | ||
| 394 | artifact.parent.mkdir() | ||
| 395 | artifact.write_bytes(b'already verified archive') | ||
| 396 | stamp = mirror.stable_time('0.17.0') | ||
| 397 | for path in [artifact, artifact.parent, self.cache.root]: | ||
| 398 | mirror.set_mtime(path, stamp) | ||
| 399 | self.cache.write_readmes(artifact.parent) | ||
| 400 | mirror.set_mtime(artifact.parent, stamp) | ||
| 401 | readme = artifact.parent / 'preadme.md' | ||
| 402 | original = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, self.cache.root, readme]} | ||
| 403 | release = {'date': '2026-10-01', 'docs': 'https://ziglang.org/documentation/0.17.0/', | ||
| 404 | 'src': {'tarball': mirror.upstream(artifact.name), 'shasum': 'unused-cached-hash'}} | ||
| 405 | with patch.object(mirror, 'index', return_value={'0.17.0': release, 'master': {}}) as metadata, patch.object(mirror, 'fetch', side_effect=AssertionError('cached refresh must not redownload')): | ||
| 406 | self.cache.refresh() | ||
| 407 | self.assertIn('[Language Reference](' + release['docs'] + ')', readme.read_text()) | ||
| 408 | self.assertIsNone(self.cache.directory_readme(artifact.parent)) | ||
| 409 | metadata.assert_any_call(refresh=True) | ||
| 410 | release['notes'] = 'https://ziglang.org/download/0.17.0/release-notes.html' | ||
| 411 | self.cache.refresh() | ||
| 412 | self.assertIn('[Release Notes](' + release['notes'] + ')', readme.read_text()) | ||
| 413 | self.assertIsNone(self.cache.directory_readme(artifact.parent)) | ||
| 414 | del release['docs'] | ||
| 415 | self.cache.refresh() | ||
| 416 | self.assertNotIn('Language Reference', readme.read_text()) | ||
| 417 | readme.write_text(readme.read_text() + '\nCustom table notes.\n') | ||
| 418 | mirror.set_mtime(readme, original[readme]) | ||
| 419 | self.assertIsNone(self.cache.directory_readme(artifact.parent)) | ||
| 420 | for path, timestamp in original.items(): | ||
| 421 | self.assertEqual(path.stat().st_mtime_ns, timestamp) | ||
| 422 | |||
| 423 | def test_readme_covers_legacy_bootstrap_both_target_orders_and_dynamic_os(self): | ||
| 424 | names=['zig-0.1.1.tar.xz','zig-bootstrap-0.1.1.zip','zig-win64-0.1.1.zip', | ||
| 425 | 'zig-linux-aarch64-0.1.1.tar.xz','zig-x86_64-macos-0.1.1.tar.xz','zig-newarch-newos-0.1.1.zip'] | ||
| 426 | artifacts=[self.cache.path(name) for name in names] | ||
| 427 | for artifact in artifacts:artifact.parent.mkdir(parents=True,exist_ok=True);artifact.write_bytes(b'fixture') | ||
| 428 | text=mirror.version_readme('0.1.1',artifacts).decode() | ||
| 429 | for name in names:self.assertIn(name,text) | ||
| 430 | self.assertIn('Bootstrap',text);self.assertIn('newos',text);self.assertNotIn('&#128272;',text) | ||
| 431 | self.assertIn('role="region" aria-label="Zig downloads" tabindex="0"',text) | ||
| 432 | self.assertIn('>\n\n| Source ',text) | ||
| 433 | self.assertLess(text.index('</div>'),text.index('This historical release')) | ||
| 434 | self.assertNotIn('https://file.paperclover.net',text) | ||
| 435 | empty=mirror.version_readme('0.17.0',[],True).decode() | ||
| 436 | self.assertIn('[development builds]',empty);self.assertNotIn('| Linux',empty);self.assertNotIn('role="region"',empty) | ||
| 437 | |||
| 182 | 438 | ||
| 183 | if __name__ == '__main__': | 439 | if __name__ == '__main__': |
| 184 | unittest.main() | 440 | unittest.main() |
tools/zig-mirror.md+58| ... | @@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after | ... | @@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after |
| 23 | SHA-256 and/or Zig Minisign verification. Existing archive files need the | 23 | SHA-256 and/or Zig Minisign verification. Existing archive files need the |
| 24 | one-time `verify` operation before first public exposure. | 24 | one-time `verify` operation before first public exposure. |
| 25 | 25 | ||
| 26 | ## Release dates and navigation | ||
| 27 | |||
| 28 | Indexed stable releases use the official download index's date at UTC midnight. | ||
| 29 | The retracted 0.15.0 artifacts use authenticated signing time instead: the | ||
| 30 | [official retraction](https://github.com/ziglang/www.ziglang.org/commit/1be0f80b15d256d42411a21f10e208f9b269a5e2) | ||
| 31 | removed that release from the index. Other unknown stable releases fail closed. | ||
| 32 | Development archives use the build timestamp from their Minisign trusted comment; | ||
| 33 | its global Ed25519 signature is checked against Zig's pinned key before the date | ||
| 34 | is accepted. The signature file receives the archive's date. Each visible folder, | ||
| 35 | including the mirror root, uses its newest contained archive date. Empty folders, | ||
| 36 | README files and hidden operational/history files do not contribute to that date. | ||
| 37 | Every cache miss, periodic refresh and backfill uses this same publication path. | ||
| 38 | |||
| 39 | Version folders contain an architecture/OS download table built only from present | ||
| 40 | canonical artifacts and signatures. Released versions list Release Notes, | ||
| 41 | Language Reference and Standard Library Documentation below their heading only | ||
| 42 | when the official index publishes the corresponding `notes`, `docs` or `stdDocs` | ||
| 43 | URL. No documentation URL is synthesized or probed per artifact. The index has | ||
| 44 | one process-wide, lock-protected snapshot, expiring after an hour; periodic/manual | ||
| 45 | refresh bypasses it, as does a cache miss for an unindexed stable version's date. | ||
| 46 | Refresh also updates the current release/build navigation when archives are | ||
| 47 | already cached, preserving artifact and folder dates. | ||
| 48 | |||
| 49 | Base versions link to their development index; a base folder with only development | ||
| 50 | builds says “this release only has development builds out.” Each development | ||
| 51 | version links back to its index, whose heading leaves the native directory listing | ||
| 52 | to show the builds. Complete existing relative-link tables are retained when | ||
| 53 | their introduction matches the published links and navigation; incomplete or | ||
| 54 | production-only links are replaced. The owner's root `preadme.md` and hidden | ||
| 55 | history remain untouched. | ||
| 56 | |||
| 57 | Existing verified archives need a one-time metadata reconciliation. It reads | ||
| 58 | signature comments and file metadata, not archive bodies, and preserves atime, | ||
| 59 | ownership and modes. Preview it first: | ||
| 60 | |||
| 61 | ```sh | ||
| 62 | python3 /app/mirror.py reconcile > /alloc/metadata-proposal.json | ||
| 63 | python3 /app/mirror.py reconcile --apply --journal /alloc/private-metadata-undo.jsonl | ||
| 64 | python3 /app/mirror.py reconcile > /alloc/metadata-after.json | ||
| 65 | ``` | ||
| 66 | |||
| 67 | Retain proposal and undo files privately outside `/mirror`; never publish them | ||
| 68 | through Copyparty. Apply fails closed on missing dates, unsafe paths or changed | ||
| 69 | files. The journal is created exclusively with mode0600 and fsynced before any | ||
| 70 | change. Its first record retains old metadata and replaced public README bytes; | ||
| 71 | a second record captures completed inode/metadata states. A journal without the | ||
| 72 | completion record requires recovery review. A guarded one-time undo must verify | ||
| 73 | those completed identities and generated README bytes before restoring old | ||
| 74 | metadata/docs; later writes must cause it to stop. Keep that recovery helper with | ||
| 75 | the private deployment evidence, rather than adding an automatic rollback hook. | ||
| 76 | |||
| 77 | The worker has only `FOWNER` beyond its normal capability set, allowing explicit | ||
| 78 | historic dates on the existing UID3000-owned NAS folders without changing their | ||
| 79 | ownership. Its root filesystem is read-only and its only writable external mount | ||
| 80 | is the canonical Zig subtree. Download temporaries and locks stay beneath hidden | ||
| 81 | `.mirror-locks`; they cannot change visible folder dates. Fresh worker previews | ||
| 82 | start empty, disable polling and fetch public artifacts only when requested. | ||
| 83 | |||
| 26 | ## Production cutover | 84 | ## Production cutover |
| 27 | 85 | ||
| 28 | The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files | 86 | The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files |