authoravatarclover caruso<git@paperclover.net>2026-10-17 12:43:21-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
loga4c36c6d8f5893c91e75def1bc06dad8917475bc
tree6ad7df4cddc8c9bb09a4fdff1a5ec7b27625c737
parenta061380e5790ff217adfe1d0ca9dd0426d126739
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Show Zig release navigation and preserve archive dates

Generate published documentation links and version navigation, authenticate development build dates, and preserve root and ancestor timestamps on publication. Simplify anonymous mirror browsing in Copyparty. Run reconciliation explicitly with a private recovery journal. Assisted-by: gpt-6.1-sol

13 files changed, 815 insertions(+), 31 deletions(-)

config/Service.pkl+1
......@@ -85,6 +85,7 @@ class Container {
8585 secretFiles: Mapping<String, String> = new {}
8686 args: Listing<String> = new {}
8787 capAdd: Listing<String> = new {}
88 readOnlyRoot: Boolean = false
8889 devices: Listing<String> = new {}
8990 extraHosts: Listing<String> = new {}
9091 hostNetwork: Boolean = false
service/copyparty/copyparty.conf+1
......@@ -24,6 +24,7 @@
2424 idp-logout: /auth/file/sign-out
2525 idp-login-t: with snow globe
2626 html-head: <link rel="stylesheet" href="/.static/copyparty.css">
27 js-browser: /.static/mirrors.js
2728
2829[/]
2930 /w
service/copyparty/static/copyparty.css+16
......@@ -7,3 +7,19 @@
77 --font-serif: "Name Sans", serif;
88 --font-mono: "Name Mono", monospace;
99}
10.mirror-anonymous #files tr > :first-child,
11.mirror-anonymous #tree,
12.mirror-anonymous #entree,
13.mirror-anonymous #opa_auc,
14.mirror-anonymous #op_player,
15.mirror-anonymous #widget {
16 display: none !important;
17}
18.mirror-anonymous #path {
19 display: inline-block !important;
20}
21.mirror-anonymous #wrap {
22 margin-left: 1.5em !important;
23 padding-bottom: 1.5em;
24 overflow-x: auto;
25}
service/copyparty/static/mirrors.js created+12
......@@ -0,0 +1,12 @@
1(function () {
2 var render = treectl.gentab;
3 treectl.gentab = function (top, listing) {
4 var anonymous = (listing.acct === undefined ? acct : listing.acct) === "*" && /^\/mirrors(?:\/|$)/.test(top);
5 document.body.classList.toggle("mirror-anonymous", anonymous);
6 if (anonymous)
7 read_dsort("-ts");
8 return render.call(this, top, listing);
9 };
10 if (ls0 !== null)
11 treectl.gentab(get_evpath(), ls0);
12})();
service/zig-mirror/build/mirror.py+446-15
......@@ -1,8 +1,15 @@
11#!/usr/bin/env python3
22import argparse
3import base64
4import ctypes
5import ctypes.util
6import contextlib
7import datetime
38import email.utils
49import fcntl
10import functools
511import hashlib
12import html
613from html.parser import HTMLParser
714import http.server
815import json
......@@ -10,6 +17,7 @@ import os
1017from pathlib import Path
1118import re
1219import shutil
20import stat
1321import subprocess
1422import tempfile
1523import threading
......@@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U'
2432OFFICIAL = 'https://ziglang.org'
2533NEKTRO = 'http://mirrors.nektro.net/zig/'
2634MAX_ARTIFACT = 2 * 1024**3
35_index_cache = (float('-inf'), {})
36_index_lock = threading.Lock()
2737
2838
2939class Unavailable(Exception):
3040 pass
3141
3242
43@functools.lru_cache(maxsize=1)
44def comment_verifier():
45 name = ctypes.util.find_library('sodium')
46 if not name:
47 raise Unavailable('Minisign comment verification unavailable')
48 library = ctypes.CDLL(name)
49 if library.sodium_init() < 0:
50 raise Unavailable('Minisign comment verification unavailable')
51 verify = library.crypto_sign_verify_detached
52 verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p]
53 verify.restype = ctypes.c_int
54 return verify
55
56
57def metadata_open(path, final_flags=os.O_RDONLY):
58 path = path.absolute()
59 descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0))
60 try:
61 for index, part in enumerate(path.parts[1:]):
62 flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0)
63 if index < len(path.parts) - 2:
64 flags |= os.O_DIRECTORY
65 next_descriptor = os.open(part, flags, dir_fd=descriptor)
66 os.close(descriptor)
67 descriptor = next_descriptor
68 return descriptor
69 except BaseException:
70 os.close(descriptor)
71 raise
72
73
74def signed_time(signature):
75 with os.fdopen(metadata_open(signature), 'rb') as stream:
76 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
77 raise Unavailable('signature is not a regular file')
78 body = stream.read(4097)
79 lines = body.splitlines()
80 try:
81 if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '):
82 raise ValueError()
83 key = base64.b64decode(PUBLIC_KEY, validate=True)
84 sig = base64.b64decode(lines[1], validate=True)
85 global_sig = base64.b64decode(lines[3], validate=True)
86 comment = lines[2][len(b'trusted comment: '):]
87 if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]:
88 raise ValueError()
89 # Minisign authenticates the artifact signature followed by the exact trusted comment.
90 message = sig[10:] + comment
91 if comment_verifier()(global_sig, message, len(message), key[10:]) != 0:
92 raise ValueError()
93 match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment)
94 if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'):
95 raise ValueError()
96 return int(match[1]) * 1_000_000_000
97 except (ValueError, IndexError) as error:
98 raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error
99
100
101@functools.lru_cache(maxsize=128)
102def stable_time(version):
103 try:
104 entries = index()
105 if version not in entries:
106 entries = index(refresh=True)
107 date = datetime.date.fromisoformat(entries[version]['date'])
108 return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000
109 except (KeyError, ValueError) as error:
110 raise Unavailable('official release date unavailable: ' + version) from error
111
112
113def artifact_time(artifact):
114 version = filename(artifact.name)
115 if '-dev.' in version or version == '0.15.0':
116 return signed_time(Path(str(artifact) + '.minisig'))
117 return stable_time(version)
118
119
120def children(directory):
121 descriptor = metadata_open(directory)
122 try:
123 with os.scandir(descriptor) as entries:
124 return [(directory / entry.name, entry.stat(follow_symlinks=False))
125 for entry in entries if not entry.name.startswith('.')]
126 finally:
127 os.close(descriptor)
128
129
130def set_mtime(path, timestamp):
131 descriptor = metadata_open(path)
132 try:
133 before = os.fstat(descriptor)
134 if before.st_mtime_ns != timestamp:
135 os.utime(descriptor, ns=(before.st_atime_ns, timestamp))
136 finally:
137 os.close(descriptor)
138
139
140def file_state(value):
141 return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')}
142
143
144def version_readme(version, artifacts, development=False):
145 systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows',
146 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'}
147 source = []
148 targets = {}
149
150 def link(artifact):
151 name = artifact.name
152 url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/')
153 label = 'tar.xz' if name.endswith('.tar.xz') else 'zip'
154 result = f'[{label}]({url})'
155 signature = Path(str(artifact) + '.minisig')
156 if signature.is_file() and not signature.is_symlink():
157 result += f' <a href="{url}.minisig" style="background:none!important;border:none!important;font-size:13px;margin-left:-5px;margin-right:-3px" aria-label="Signature for {html.escape(name, quote=True)}">&#128272;</a>'
158 return result
159
160 for artifact in sorted(artifacts):
161 name = artifact.name
162 extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip'
163 target = name[len('zig'): -len('-' + version + extension)]
164 if target in ('', '-bootstrap'):
165 source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact))
166 continue
167 if target == '-win64':
168 architecture, system = 'x86_64', 'windows'
169 else:
170 first, second = target.removeprefix('-').split('-')
171 system, architecture = (first, second) if first in systems else (second, first)
172 targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact))
173 columns = list(systems)
174 columns += sorted({system for row in targets.values() for system in row} - set(columns))
175 heading = '<br>'.join(source) or 'Architecture'
176 lines = [f'### Zig {version}', '']
177 if not artifacts and development:
178 return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode()
179 if artifacts and '-dev.' not in version:
180 release = index().get(version, {})
181 for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]:
182 url = release.get(key)
183 if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url):
184 lines.append(f'- [{label}]({url})')
185 if lines[-1]:
186 lines.append('')
187 if development:
188 lines += [f'[Development builds](/mirrors/zig/{version}/dev/)', '']
189 elif '-dev.' in version:
190 lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', '']
191 if not artifacts:
192 return '\n'.join(lines).encode()
193 lines += ['<div role="region" aria-label="Zig downloads" tabindex="0" style="max-width:100%;overflow-x:auto;white-space:nowrap;word-break:normal;overflow-wrap:normal">', '',
194 '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |',
195 '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |']
196 priority = ('x86_64', 'aarch64', 'x86', 'arm')
197 for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)):
198 lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |')
199 lines += ['', '</div>']
200 if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts):
201 lines += ['', '<span style="font-style:italic;color:orange">Verify your build signatures.</span> <span style="opacity:0.7">The &#128272; links open the Minisign signatures.</span>', '']
202 else:
203 lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', '']
204 return '\n'.join(lines).encode()
205
206
33207def filename(name):
34208 match = PATTERN.fullmatch(name)
35209 if len(name) > 240 or not match:
......@@ -72,9 +246,14 @@ def fetch(url, target, limit):
72246 raise Unavailable('upstream download failed') from error
73247
74248
75def index():
76 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:
77 return json.load(response)
249def index(refresh=False):
250 global _index_cache
251 with _index_lock:
252 if refresh or time.monotonic() - _index_cache[0] >= 3600:
253 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:
254 entries = json.load(response)
255 _index_cache = (time.monotonic(), entries)
256 return _index_cache[1]
78257
79258
80259def digest(path):
......@@ -87,7 +266,10 @@ def digest(path):
87266
88267class Mirror:
89268 def __init__(self, root):
90 self.root = Path(root)
269 self.root = Path(root).absolute()
270 if self.root.is_symlink():
271 raise ValueError('mirror root is a symlink')
272 self.root = self.root.resolve()
91273 self.root.mkdir(parents=True, exist_ok=True)
92274 self.slots = threading.BoundedSemaphore(4)
93275 self.refresh_lock = threading.Lock()
......@@ -98,6 +280,50 @@ class Mirror:
98280 raise ValueError('artifact escapes mirror')
99281 return path
100282
283 def lock_directory(self):
284 locks = self.root / '.mirror-locks'
285 descriptor = metadata_open(self.root)
286 try:
287 fcntl.flock(descriptor, fcntl.LOCK_EX)
288 timestamp = os.fstat(descriptor).st_mtime_ns
289 try:
290 locks.mkdir()
291 except FileExistsError:
292 pass
293 else:
294 set_mtime(self.root, timestamp)
295 finally:
296 os.close(descriptor)
297 descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY)
298 os.close(descriptor)
299 return locks
300
301 @contextlib.contextmanager
302 def publishing(self):
303 locks = self.lock_directory()
304 with (locks / '.timestamps').open('a') as lock:
305 fcntl.flock(lock, fcntl.LOCK_EX)
306 yield
307
308 def parent_times(self, artifact):
309 def latest(directory):
310 values = []
311 for path, value in children(directory):
312 if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path:
313 values.append(value.st_mtime_ns)
314 elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'):
315 values.append(latest(path))
316 return max(values, default=0)
317
318 directory = artifact.parent
319 while True:
320 timestamp = latest(directory)
321 if timestamp:
322 set_mtime(directory, timestamp)
323 if directory == self.root:
324 break
325 directory = directory.parent
326
101327 def ensure(self, name, source=None, checksum=None):
102328 target = self.path(name)
103329 artifact_name = name.removesuffix('.minisig')
......@@ -109,8 +335,7 @@ class Mirror:
109335 try:
110336 if shutil.disk_usage(self.root).free < 16 * 1024**3:
111337 raise Unavailable('mirror free-space reserve reached')
112 locks = self.root / '.mirror-locks'
113 locks.mkdir(exist_ok=True)
338 locks = self.lock_directory()
114339 with (locks / artifact_name).open('a') as lock:
115340 fcntl.flock(lock, fcntl.LOCK_EX)
116341 if target.is_file():
......@@ -123,8 +348,7 @@ class Mirror:
123348 and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None)
124349 if not checksum:
125350 raise Unavailable('unsigned artifact has no official SHA-256')
126 artifact.parent.mkdir(parents=True, exist_ok=True)
127 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:
351 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
128352 pending = Path(temporary) / artifact_name
129353 signature = Path(str(pending) + '.minisig')
130354 url = source or upstream(artifact_name)
......@@ -137,21 +361,216 @@ class Mirror:
137361 capture_output=True)
138362 if result.returncode:
139363 raise Unavailable('artifact signature verification failed')
364 timestamp = artifact_time(pending)
140365 os.chmod(pending, 0o664)
141 os.replace(pending, artifact)
366 set_mtime(pending, timestamp)
142367 if signature.exists():
143368 os.chmod(signature, 0o664)
144 os.replace(signature, Path(str(artifact) + '.minisig'))
369 set_mtime(signature, timestamp)
370 with self.publishing():
371 self.path(name)
372 artifact.parent.mkdir(parents=True, exist_ok=True)
373 os.replace(pending, artifact)
374 if signature.exists():
375 os.replace(signature, Path(str(artifact) + '.minisig'))
376 try:
377 self.write_readmes(artifact.parent)
378 finally:
379 self.parent_times(artifact)
145380 if name.endswith('.minisig') and not target.exists():
146 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:
381 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
147382 pending = Path(temporary) / name
148383 fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096)
149384 os.chmod(pending, 0o664)
150 os.replace(pending, target)
385 set_mtime(pending, artifact.stat().st_mtime_ns)
386 with self.publishing():
387 os.replace(pending, target)
388 try:
389 self.write_readmes(artifact.parent)
390 finally:
391 self.parent_times(artifact)
151392 return target
152393 finally:
153394 self.slots.release()
154395
396 def readme_change(self, directory, artifacts, content=None):
397 path = directory / 'preadme.md'
398 if content is None:
399 content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir())
400 before = None
401 previous = b''
402 if path.exists() or path.is_symlink():
403 descriptor = metadata_open(path)
404 with os.fdopen(descriptor, 'rb') as stream:
405 value = os.fstat(stream.fileno())
406 if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024:
407 raise Unavailable('version readme is not a small regular file')
408 before = file_state(value)
409 previous = stream.read(512 * 1024 + 1)
410 required = {artifact.name for artifact in artifacts}
411 required |= {artifact.name + '.minisig' for artifact in artifacts
412 if Path(str(artifact) + '.minisig').is_file()}
413 present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1])
414 for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())}
415 linked = {name for name in present if PATTERN.fullmatch(name)}
416 introduction = content.split(b'<div ', 1)[0]
417 if artifacts and linked == required and previous.startswith(introduction) and b'https://file.paperclover.net' not in previous:
418 return None
419 if previous == content:
420 return None
421 return {'path': str(path.relative_to(self.root)), 'before': before,
422 'previous': base64.b64encode(previous).decode(), 'content': content.decode()}
423
424 def artifacts(self, directory):
425 return [path for path, value in children(directory)
426 if stat.S_ISREG(value.st_mode) and PATTERN.fullmatch(path.name)
427 and not path.name.endswith('.minisig') and self.path(path.name) == path]
428
429 def directory_readme(self, directory):
430 if directory.name == 'dev':
431 content = f'### Zig {directory.parent.name} development builds\n'.encode()
432 return self.readme_change(directory, [], content)
433 artifacts = self.artifacts(directory)
434 if artifacts or (directory / 'dev').is_dir():
435 return self.readme_change(directory, artifacts)
436
437 def write_readmes(self, directory):
438 directories = [directory]
439 if directory.parent.name == 'dev':
440 directories += [directory.parent, directory.parent.parent]
441 for current in directories:
442 change = self.directory_readme(current)
443 if change:
444 self.apply_readme(change)
445
446 def apply_readme(self, change):
447 path = self.root / change['path']
448 if change['before'] is None:
449 parent = metadata_open(path.parent)
450 try:
451 descriptor = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o664, dir_fd=parent)
452 os.fchown(descriptor, -1, os.fstat(parent).st_gid)
453 finally:
454 os.close(parent)
455 before = os.fstat(descriptor)
456 else:
457 descriptor = metadata_open(path, os.O_WRONLY)
458 before = os.fstat(descriptor)
459 if any(file_state(before)[key] != value for key, value in change['before'].items() if key != 'atime_ns'):
460 os.close(descriptor)
461 raise Unavailable('version readme changed since plan')
462 try:
463 os.ftruncate(descriptor, 0)
464 with os.fdopen(os.dup(descriptor), 'wb') as output:
465 output.write(change['content'].encode())
466 output.flush()
467 os.fsync(output.fileno())
468 os.utime(descriptor, ns=(before.st_atime_ns, before.st_mtime_ns))
469 finally:
470 os.close(descriptor)
471
472 def metadata_plan(self):
473 files = {}
474 folders = {}
475 errors = []
476 pending = [self.root]
477 while pending:
478 for artifact, value in children(pending.pop()):
479 if stat.S_ISDIR(value.st_mode):
480 pending.append(artifact)
481 continue
482 if not stat.S_ISREG(value.st_mode) or not PATTERN.fullmatch(artifact.name) or artifact.name.endswith('.minisig'):
483 continue
484 try:
485 if artifact.relative_to(self.root) != relative(artifact.name):
486 raise Unavailable('misplaced Zig artifact')
487 timestamp = artifact_time(artifact)
488 files[artifact] = timestamp
489 signature = Path(str(artifact) + '.minisig')
490 if signature.exists():
491 with os.fdopen(metadata_open(signature), 'rb') as stream:
492 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
493 raise Unavailable('signature is not a regular file')
494 files[signature] = timestamp
495 directory = artifact.parent
496 while True:
497 folders[directory] = max(folders.get(directory, 0), timestamp)
498 if directory == self.root:
499 break
500 directory = directory.parent
501 except (Unavailable, OSError, ValueError) as error:
502 errors.append({'path': str(artifact.relative_to(self.root)), 'error': str(error)})
503 readmes = []
504 for directory in sorted(set(folders) - {self.root}):
505 try:
506 change = self.directory_readme(directory)
507 if change:
508 readmes.append(change)
509 except (Unavailable, OSError, ValueError) as error:
510 errors.append({'path': str(directory.relative_to(self.root)), 'error': str(error)})
511 entries = []
512 for path, timestamp in sorted({**files, **folders}.items()):
513 descriptor = metadata_open(path)
514 try:
515 entries.append({'path': str(path.relative_to(self.root)), 'before': file_state(os.fstat(descriptor)), 'mtime_ns': timestamp})
516 finally:
517 os.close(descriptor)
518 return {'root': str(self.root), 'files': len(files), 'folders': len(folders),
519 'changed': sum(entry['before']['mtime_ns'] != entry['mtime_ns'] for entry in entries),
520 'provenance': {'stable': OFFICIAL + '/download/index.json UTC date', 'development': 'pinned Minisign authenticated trusted-comment timestamp', 'withdrawn0.15.0': 'authenticated signing time; release retracted before 0.15.1', 'folders': 'newest contained artifact timestamp'},
521 'errors': errors, 'entries': entries, 'readmes': readmes}
522
523 def reconcile(self, apply=False, journal=None):
524 with self.publishing() if apply else contextlib.nullcontext():
525 plan = self.metadata_plan()
526 if apply:
527 if plan['errors']:
528 raise Unavailable('timestamp plan has unresolved dates')
529 if journal is None or journal.resolve().is_relative_to(self.root):
530 raise ValueError('private journal outside the mirror is required')
531 journal = journal.resolve()
532 descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
533 with os.fdopen(descriptor, 'w') as output:
534 json.dump(plan, output)
535 output.write('\n')
536 output.flush()
537 os.fsync(output.fileno())
538 descriptor = metadata_open(journal.parent)
539 try:
540 os.fsync(descriptor)
541 finally:
542 os.close(descriptor)
543 for change in plan['readmes']:
544 self.apply_readme(change)
545 for entry in plan['entries']:
546 if entry['before']['mtime_ns'] == entry['mtime_ns'] and not stat.S_ISDIR(entry['before']['mode']):
547 continue
548 descriptor = metadata_open(self.root / entry['path'])
549 try:
550 current = file_state(os.fstat(descriptor))
551 expected = entry['before']
552 fields = ('dev', 'ino', 'mode', 'uid', 'gid') if stat.S_ISDIR(current['mode']) else tuple(key for key in expected if key != 'atime_ns')
553 if any(current[key] != expected[key] for key in fields):
554 raise Unavailable('file changed since timestamp plan: ' + entry['path'])
555 if current['mtime_ns'] != entry['mtime_ns']:
556 os.utime(descriptor, ns=(current['atime_ns'], entry['mtime_ns']))
557 finally:
558 os.close(descriptor)
559 applied = {}
560 paths = {entry['path'] for entry in plan['entries']} | {change['path'] for change in plan['readmes']}
561 for path in sorted(paths):
562 descriptor = metadata_open(self.root / path)
563 try:
564 applied[path] = file_state(os.fstat(descriptor))
565 finally:
566 os.close(descriptor)
567 with os.fdopen(metadata_open(journal, os.O_WRONLY | os.O_APPEND), 'w') as output:
568 json.dump({'applied': applied}, output)
569 output.write('\n')
570 output.flush()
571 os.fsync(output.fileno())
572 return plan
573
155574 def verify(self):
156575 entries = index()
157576 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
......@@ -181,13 +600,20 @@ class Mirror:
181600 if not self.refresh_lock.acquire(blocking=False):
182601 return
183602 try:
184 entries = index()
603 entries = index(refresh=True)
185604 latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))]
186605 for release in (latest, entries['master']):
606 artifact = None
187607 for item in release.values():
188608 if isinstance(item, dict) and 'tarball' in item:
189609 name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1])
190 self.ensure(name, checksum=item['shasum'])
610 artifact = self.ensure(name, checksum=item['shasum'])
611 if artifact is not None:
612 with self.publishing():
613 try:
614 self.write_readmes(artifact.parent)
615 finally:
616 self.parent_times(artifact)
191617 finally:
192618 self.refresh_lock.release()
193619
......@@ -306,15 +732,20 @@ def backfill(mirror, budget, apply):
306732
307733def main():
308734 parser = argparse.ArgumentParser()
309 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify'])
735 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify', 'reconcile'])
310736 parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror'))
311737 parser.add_argument('--budget-gib', type=int, default=32)
312738 parser.add_argument('--apply', action='store_true')
739 parser.add_argument('--journal', type=Path)
313740 args = parser.parse_args()
314741 os.umask(0o002)
742 if args.command == 'reconcile' and not Path(args.root).is_dir():
743 raise ValueError('existing mirror root is required')
315744 mirror = Mirror(args.root)
316745 if args.command == 'verify':
317746 mirror.verify()
747 elif args.command == 'reconcile':
748 print(json.dumps(mirror.reconcile(args.apply, args.journal)))
318749 elif args.command == 'refresh':
319750 mirror.refresh()
320751 elif args.command == 'backfill':
service/zig-mirror/fixture/zig-x86_64-linux-0.15.0.tar.xz.minisig created+4
......@@ -0,0 +1,4 @@
1untrusted comment: signature from minisign secret key
2RUSGOq2NVecA2RV/TllU7u4sMW2JwbgaUIX6upgmh5dqHIle3ERNdeh58+iO14pz84aSsJzW8yWISzgmbf/AKPvj09piCUZmGws=
3trusted comment: timestamp:1755639580 file:zig-x86_64-linux-0.15.0.tar.xz hashed
4RO3woTlcv5vkx53pBuILPCuk3u74YyiPQqWvbLJzcOacBRhZ6nQcBM7RvBCyNts7v2Jt3CmbEULL0yfppsrpAA==
service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig created+4
......@@ -0,0 +1,4 @@
1untrusted comment: signature from minisign secret key
2RUSGOq2NVecA2RwFmP6WKntiiWhQqtOhDbzNAVDP/x3uN6v1EcAMLi5uCCw1kisYZ1Dl6SOJTU6/6FT4pJDgRK093OhyF8QHfQg=
3trusted comment: timestamp:1791194091 file:zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip hashed
4m6Yh4Pw9wtV1WB26pz7pMTQ6gdjUABVuOTN66QXuwvuoKn0ixv6DO01+QtJQnaZfcQg1EnJGp854sGjb627pDA==
service/zig-mirror/prepare.py deleted-14
......@@ -1,14 +0,0 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import sys
6
7data = json.load(sys.stdin)
8if Path(data['hostRoot']).name != 'zig-mirror':
9 root = Path(data['hostRoot']) / 'mirror'
10 artifact = root / '0.0.0/zig-0.0.0.tar.xz'
11 artifact.parent.mkdir(parents=True, exist_ok=True)
12 artifact.write_bytes(b'Synthetic Zig mirror preview.\n')
13 for path in (artifact.parent, artifact):
14 os.chown(path, data['uid'], data['uid'])
service/zig-mirror/service.pkl+2-1
......@@ -5,10 +5,11 @@ import "../../config/site.pkl" as site
55name = "Zig mirror"
66launcher = false
77stageIsolation = "fresh"
8prepare = "prepare.py"
98
109container {
1110 build = "build"
11 readOnlyRoot = true
12 capAdd { "FOWNER" }
1213 http {
1314 containerPort = 8080
1415 subdomain = "zig"
tools/snowglobe-secret-files-test.py+12
......@@ -43,6 +43,18 @@ class SecretFiles(unittest.TestCase):
4343 self.assertNotIn('template {', rendered)
4444 self.assertNotIn('secrets/', rendered)
4545
46 def test_read_only_root_is_opt_in_and_preserves_writable_data_mounts(self):
47 self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data}))
48 self.task["readOnlyRoot"] = True
49 self.task["volumes"] = {"/mirror": {"src": "/fixture/mirror", "readOnly": False}}
50 self.task["capAdd"] = ["FOWNER"]
51 rendered = snowglobe.render(self.data, {"fixture": self.data})
52 self.assertIn('readonly_rootfs = true', rendered)
53 self.assertIn('cap_add = ["FOWNER"]', rendered)
54 self.assertIn('/fixture/mirror:/mirror"', rendered)
55 del self.task["readOnlyRoot"]
56 self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data}))
57
4658 def test_unsafe_and_colliding_targets_are_rejected(self):
4759 for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"):
4860 with self.subTest(target=target):
tools/snowglobe.py+2
......@@ -398,6 +398,8 @@ def render(data, definitions):
398398 lines.append(f" args = {q(task['args'])}")
399399 if task["hostNetwork"]:
400400 lines.append(' network_mode = "host"')
401 if task.get("readOnlyRoot"):
402 lines.append(' readonly_rootfs = true')
401403 task_ports = [label for (owner, _), label in ports.items() if owner == task_name]
402404 if task_ports and not task["hostNetwork"]:
403405 lines.append(f" ports = {q(task_ports)}")
tools/zig-mirror-test.py+257-1
......@@ -3,7 +3,10 @@ import concurrent.futures
33import importlib.util
44import io
55import contextlib
6import ctypes.util
7import datetime
68import json
9import os
710from pathlib import Path
811from types import SimpleNamespace
912import tempfile
......@@ -19,6 +22,50 @@ mirror = importlib.util.module_from_spec(spec)
1922spec.loader.exec_module(mirror)
2023
2124
25class IndexTests(unittest.TestCase):
26 def setUp(self):
27 cache = patch.object(mirror, '_index_cache', (float('-inf'), {}))
28 cache.start()
29 self.addCleanup(cache.stop)
30 mirror.stable_time.cache_clear()
31
32 def test_index_coalesces_concurrent_reads_expires_and_forces_refresh(self):
33 snapshot = {'0.17.0': {'docs': 'https://ziglang.org/documentation/0.17.0/'}}
34 def response(*args, **kwargs):
35 time.sleep(.01)
36 return io.BytesIO(json.dumps(snapshot).encode())
37 with patch.object(mirror.urllib.request, 'urlopen', side_effect=response) as request, patch.object(mirror.time, 'monotonic', return_value=0) as clock:
38 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers:
39 entries = list(workers.map(lambda _: mirror.index(), range(8)))
40 self.assertTrue(all(entry == snapshot for entry in entries))
41 self.assertEqual(request.call_count, 1)
42 clock.return_value = 3599
43 mirror.index()
44 self.assertEqual(request.call_count, 1)
45 clock.return_value = 3600
46 mirror.index()
47 self.assertEqual(request.call_count, 2)
48 mirror.index(refresh=True)
49 self.assertEqual(request.call_count, 3)
50 self.assertTrue(all(call.args[0] == 'https://ziglang.org/download/index.json' for call in request.call_args_list))
51
52 def test_new_release_miss_refreshes_stale_index_before_publication(self):
53 release = {'date': '2027-04-01', 'docs': 'https://ziglang.org/documentation/0.19.0/'}
54 responses = [{}, {'0.19.0': release}]
55 with tempfile.TemporaryDirectory() as root, patch.object(mirror.urllib.request, 'urlopen', side_effect=lambda *args, **kwargs: io.BytesIO(json.dumps(responses.pop(0)).encode())) as request:
56 mirror.index()
57 cache = mirror.Mirror(root)
58 with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)), patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')), patch.object(mirror.subprocess, 'run', return_value=SimpleNamespace(returncode=0)):
59 artifact = cache.ensure('zig-x86_64-linux-0.19.0.tar.xz')
60 self.assertEqual(request.call_count, 2)
61 self.assertIn('[Language Reference](' + release['docs'] + ')', (artifact.parent / 'preadme.md').read_text())
62 expected = int(datetime.datetime(2027, 4, 1, tzinfo=datetime.timezone.utc).timestamp()) * 10**9
63 self.assertEqual(artifact.stat().st_mtime_ns, expected)
64 self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, expected)
65 self.assertEqual(artifact.parent.stat().st_mtime_ns, expected)
66 self.assertEqual(cache.root.stat().st_mtime_ns, expected)
67
68
2269class MirrorTests(unittest.TestCase):
2370 def setUp(self):
2471 self.temporary = tempfile.TemporaryDirectory()
......@@ -27,6 +74,16 @@ class MirrorTests(unittest.TestCase):
2774 disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3))
2875 disk.start()
2976 self.addCleanup(disk.stop)
77 mirror.stable_time.cache_clear()
78 dates = {version: {'date': date} for version, date in [('0.1.1', '2017-10-17'), ('0.2.0', '2018-03-15'),
79 ('0.3.0', '2018-09-28'), ('0.4.0', '2019-04-08'), ('0.5.0', '2019-09-30'), ('0.17.0', '2026-10-01')]}
80 metadata = patch.object(mirror, 'index', return_value=dates)
81 metadata.start()
82 self.addCleanup(metadata.stop)
83
84 def dev_fixture(self):
85 fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig'
86 return fixture.name.removesuffix('.minisig'), fixture.read_bytes()
3087
3188 def test_spec_filenames_and_existing_layout(self):
3289 cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'),
......@@ -150,7 +207,7 @@ class MirrorTests(unittest.TestCase):
150207 def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self):
151208 names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz']
152209 body = b'official historical archive'
153 entries = {mirror.filename(name): {'platform': {'tarball': mirror.upstream(name),
210 entries = {mirror.filename(name): {'date': '2017-10-17', 'platform': {'tarball': mirror.upstream(name),
154211 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}}
155212 for i, name in enumerate(names)}
156213 def fetch(url, target, limit):
......@@ -179,6 +236,205 @@ class MirrorTests(unittest.TestCase):
179236 self.assertEqual(response.read(), body)
180237 self.assertFalse(list(self.cache.root.rglob('.download-*')))
181238
239 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
240 def test_real_minisign_comment_and_tampering(self):
241 name, body = self.dev_fixture()
242 signature = self.cache.root / (name + '.minisig')
243 signature.write_bytes(body)
244 self.assertEqual(mirror.signed_time(signature), 1791194091 * 10**9)
245 for changed in [body.replace(b'1791194091', b'1791194092'), body.replace(b'RUSGO', b'RUSGP'),
246 body.replace(b'trusted comment: timestamp:', b'trusted comment: forged:')]:
247 signature.write_bytes(changed)
248 with self.assertRaises(mirror.Unavailable):
249 mirror.signed_time(signature)
250 signature.unlink()
251 os.mkfifo(signature)
252 with self.assertRaises(mirror.Unavailable):
253 mirror.signed_time(signature)
254
255 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
256 def test_publisher_dates_pairs_and_folders_after_cleanup(self):
257 name, signature = self.dev_fixture()
258 def fetch(url, target, limit):
259 target.write_bytes(signature if url.endswith('.minisig') and '-dev.' in url else b'fixture')
260 with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify:
261 verify.return_value.returncode = 0
262 artifact = self.cache.ensure(name)
263 stamp = 1791194091 * 10**9
264 self.assertEqual(artifact.stat().st_mtime_ns, stamp)
265 self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, stamp)
266 for directory in [artifact.parent, artifact.parent.parent, artifact.parent.parent.parent, self.cache.root]:
267 self.assertEqual(directory.stat().st_mtime_ns, stamp)
268 (self.cache.root / '0.99.0').mkdir()
269 older = self.cache.ensure('zig-0.17.0.tar.xz')
270 self.assertEqual(older.stat().st_mtime_ns, 1790812800 * 10**9)
271 self.assertEqual(self.cache.root.stat().st_mtime_ns, stamp)
272 self.assertFalse(list(self.cache.root.rglob('.download-*')))
273 self.assertIn('Signature for '+name, (artifact.parent/'preadme.md').read_text())
274 self.assertIn('this release only has [development builds]', (artifact.parent.parent.parent/'preadme.md').read_text())
275
276 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
277 def test_reconcile_is_metadata_only_idempotent_and_journalled(self):
278 name, signature = self.dev_fixture()
279 artifact = self.cache.path(name);artifact.parent.mkdir(parents=True)
280 artifact.write_bytes(b'public compiler fixture')
281 Path(str(artifact)+'.minisig').write_bytes(signature)
282 stable = self.cache.path('zig-win64-0.1.1.zip');stable.parent.mkdir();stable.write_bytes(b'legacy')
283 root_readme = self.cache.root/'preadme.md';root_readme.write_text('owner root notes')
284 version_readme = artifact.parent/'preadme.md';version_readme.write_text('old public version notes')
285 hidden = self.cache.root/'.hist';hidden.mkdir();(hidden/'zig-0.17.0.tar.xz').write_bytes(b'private history')
286 original = {path: (path.stat().st_ino,path.stat().st_uid,path.stat().st_gid,path.stat().st_mode,path.stat().st_size)
287 for path in [artifact,Path(str(artifact)+'.minisig'),stable,root_readme,hidden/'zig-0.17.0.tar.xz']}
288 plan = self.cache.reconcile()
289 self.assertFalse(plan['errors'])
290 self.assertFalse((self.cache.root/'.mirror-locks').exists())
291 with tempfile.TemporaryDirectory() as private:
292 journal=Path(private)/'undo.json'
293 self.cache.reconcile(True,journal)
294 self.assertEqual(journal.stat().st_mode&0o777,0o600)
295 records = [json.loads(line) for line in journal.read_text().splitlines()]
296 self.assertTrue(records[0]['readmes'])
297 self.assertEqual(set(records[1]['applied']),
298 {entry['path'] for entry in records[0]['entries']} |
299 {change['path'] for change in records[0]['readmes']})
300 with self.assertRaises(FileExistsError):
301 self.cache.reconcile(True, journal)
302 with self.assertRaises(ValueError):
303 self.cache.reconcile(True, self.cache.root / 'public-undo.json')
304 for path,state in original.items():
305 actual=path.stat();self.assertEqual((actual.st_ino,actual.st_uid,actual.st_gid,actual.st_mode,actual.st_size),state)
306 self.assertEqual(root_readme.read_text(),'owner root notes')
307 self.assertEqual(artifact.read_bytes(),b'public compiler fixture')
308 self.assertEqual(self.cache.reconcile()['changed'],0)
309 self.assertEqual(self.cache.reconcile()['readmes'],[])
310
311 def test_hidden_lock_creation_preserves_visible_root_date(self):
312 os.utime(self.cache.root, ns=(1234, 5678))
313 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers:
314 list(workers.map(lambda _: self.cache.lock_directory(), range(12)))
315 self.assertEqual(self.cache.root.stat().st_mtime_ns, 5678)
316
317 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
318 def test_withdrawn_0150_uses_authenticated_signing_time(self):
319 name = 'zig-x86_64-linux-0.15.0.tar.xz'
320 fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture' / (name + '.minisig')
321 artifact = self.cache.path(name)
322 artifact.parent.mkdir()
323 artifact.write_bytes(b'already verified archive')
324 signature = Path(str(artifact) + '.minisig')
325 signature.write_bytes(fixture.read_bytes())
326 with patch.object(mirror, 'index', side_effect=AssertionError('withdrawn release must not use another release date')):
327 self.assertEqual(mirror.artifact_time(artifact), mirror.signed_time(signature))
328 signature.write_bytes(fixture.read_bytes().replace(b'timestamp:', b'tampered:'))
329 with self.assertRaises(mirror.Unavailable):
330 mirror.artifact_time(artifact)
331
332 def test_missing_release_date_never_publishes(self):
333 with patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')):
334 with self.assertRaises(mirror.Unavailable):
335 self.cache.ensure('zig-0.4.1.tar.xz', checksum=mirror.hashlib.sha256(b'fixture').hexdigest())
336 self.assertFalse(self.cache.path('zig-0.4.1.tar.xz').exists())
337
338 def test_metadata_traversal_rejects_intermediate_symlinks(self):
339 with tempfile.TemporaryDirectory() as outside:
340 target=Path(outside)/'file';target.write_bytes(b'untouched')
341 (self.cache.root/'alias').symlink_to(outside)
342 with self.assertRaises(OSError):
343 mirror.set_mtime(self.cache.root/'alias/file',1)
344
345 def test_dev_index_replaces_duplicate_list_with_heading(self):
346 artifact = self.cache.path('zig-x86_64-linux-0.18.0-dev.4+5a23bf4b2.tar.xz')
347 artifact.parent.mkdir(parents=True)
348 artifact.write_bytes(b'public compiler fixture')
349 directory = artifact.parent.parent
350 readme = directory / 'preadme.md'
351 readme.write_text('### Zig 0.18.0 development builds\n\n- [0.18.0-dev.4+5a23bf4b2](old-link/)\n')
352 previous = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, directory, directory.parent, self.cache.root]}
353 change = self.cache.directory_readme(directory)
354 self.assertEqual(change['content'], '### Zig 0.18.0 development builds\n')
355 self.cache.apply_readme(change)
356 self.assertIsNone(self.cache.directory_readme(directory))
357 self.assertEqual(readme.read_text(), change['content'])
358 for path, timestamp in previous.items():
359 self.assertEqual(path.stat().st_mtime_ns, timestamp)
360
361 def test_development_only_base_has_exact_sentence_without_release_docs(self):
362 directory = self.cache.root / '0.18.0'
363 (directory / 'dev').mkdir(parents=True)
364 with patch.object(mirror, 'index', side_effect=AssertionError('development navigation needs no release metadata')):
365 change = self.cache.directory_readme(directory)
366 self.assertEqual(change['content'], '### Zig 0.18.0\n\nthis release only has [development builds](/mirrors/zig/0.18.0/dev/) out.\n')
367 self.cache.apply_readme(change)
368 self.assertIsNone(self.cache.directory_readme(directory))
369
370 def test_release_links_follow_index_presence_and_precede_artifact_table(self):
371 artifact = self.cache.path('zig-x86_64-linux-0.19.0.tar.xz')
372 release = {'notes': 'https://ziglang.org/download/0.19.0/release-notes.html',
373 'docs': 'https://ziglang.org/documentation/0.19.0/',
374 'stdDocs': 'https://ziglang.org/documentation/0.19.0/std/'}
375 labels = {'notes': 'Release Notes', 'docs': 'Language Reference', 'stdDocs': 'Standard Library Documentation'}
376 for absent in [None, *release]:
377 with self.subTest(absent=absent), patch.object(mirror, 'index', return_value={'0.19.0': {key: url for key, url in release.items() if key != absent}}):
378 text = mirror.version_readme('0.19.0', [artifact], True).decode()
379 for key, url in release.items():
380 if key == absent:
381 self.assertNotIn(f'[{labels[key]}]({url})', text)
382 else:
383 self.assertIn(f'- [{labels[key]}]({url})', text)
384 self.assertLess(text.index(url), text.index('[Development builds]'))
385 self.assertLess(text.index('[Development builds]'), text.index('<div '))
386 with patch.object(mirror, 'index', return_value={'0.19.0': {'notes': 'https://example.org/notes', 'docs': None, 'stdDocs': 'https://ziglang.org/bad)link'}}):
387 text = mirror.version_readme('0.19.0', [artifact]).decode()
388 for label in labels.values():
389 self.assertNotIn(label, text)
390 self.assertIn(artifact.name, text)
391
392 def test_refresh_updates_cached_release_links_without_changing_dates(self):
393 artifact = self.cache.path('zig-x86_64-linux-0.17.0.tar.xz')
394 artifact.parent.mkdir()
395 artifact.write_bytes(b'already verified archive')
396 stamp = mirror.stable_time('0.17.0')
397 for path in [artifact, artifact.parent, self.cache.root]:
398 mirror.set_mtime(path, stamp)
399 self.cache.write_readmes(artifact.parent)
400 mirror.set_mtime(artifact.parent, stamp)
401 readme = artifact.parent / 'preadme.md'
402 original = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, self.cache.root, readme]}
403 release = {'date': '2026-10-01', 'docs': 'https://ziglang.org/documentation/0.17.0/',
404 'src': {'tarball': mirror.upstream(artifact.name), 'shasum': 'unused-cached-hash'}}
405 with patch.object(mirror, 'index', return_value={'0.17.0': release, 'master': {}}) as metadata, patch.object(mirror, 'fetch', side_effect=AssertionError('cached refresh must not redownload')):
406 self.cache.refresh()
407 self.assertIn('[Language Reference](' + release['docs'] + ')', readme.read_text())
408 self.assertIsNone(self.cache.directory_readme(artifact.parent))
409 metadata.assert_any_call(refresh=True)
410 release['notes'] = 'https://ziglang.org/download/0.17.0/release-notes.html'
411 self.cache.refresh()
412 self.assertIn('[Release Notes](' + release['notes'] + ')', readme.read_text())
413 self.assertIsNone(self.cache.directory_readme(artifact.parent))
414 del release['docs']
415 self.cache.refresh()
416 self.assertNotIn('Language Reference', readme.read_text())
417 readme.write_text(readme.read_text() + '\nCustom table notes.\n')
418 mirror.set_mtime(readme, original[readme])
419 self.assertIsNone(self.cache.directory_readme(artifact.parent))
420 for path, timestamp in original.items():
421 self.assertEqual(path.stat().st_mtime_ns, timestamp)
422
423 def test_readme_covers_legacy_bootstrap_both_target_orders_and_dynamic_os(self):
424 names=['zig-0.1.1.tar.xz','zig-bootstrap-0.1.1.zip','zig-win64-0.1.1.zip',
425 'zig-linux-aarch64-0.1.1.tar.xz','zig-x86_64-macos-0.1.1.tar.xz','zig-newarch-newos-0.1.1.zip']
426 artifacts=[self.cache.path(name) for name in names]
427 for artifact in artifacts:artifact.parent.mkdir(parents=True,exist_ok=True);artifact.write_bytes(b'fixture')
428 text=mirror.version_readme('0.1.1',artifacts).decode()
429 for name in names:self.assertIn(name,text)
430 self.assertIn('Bootstrap',text);self.assertIn('newos',text);self.assertNotIn('&#128272;',text)
431 self.assertIn('role="region" aria-label="Zig downloads" tabindex="0"',text)
432 self.assertIn('>\n\n| Source ',text)
433 self.assertLess(text.index('</div>'),text.index('This historical release'))
434 self.assertNotIn('https://file.paperclover.net',text)
435 empty=mirror.version_readme('0.17.0',[],True).decode()
436 self.assertIn('[development builds]',empty);self.assertNotIn('| Linux',empty);self.assertNotIn('role="region"',empty)
437
182438
183439if __name__ == '__main__':
184440 unittest.main()
tools/zig-mirror.md+58
......@@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after
2323SHA-256 and/or Zig Minisign verification. Existing archive files need the
2424one-time `verify` operation before first public exposure.
2525
26## Release dates and navigation
27
28Indexed stable releases use the official download index's date at UTC midnight.
29The retracted 0.15.0 artifacts use authenticated signing time instead: the
30[official retraction](https://github.com/ziglang/www.ziglang.org/commit/1be0f80b15d256d42411a21f10e208f9b269a5e2)
31removed that release from the index. Other unknown stable releases fail closed.
32Development archives use the build timestamp from their Minisign trusted comment;
33its global Ed25519 signature is checked against Zig's pinned key before the date
34is accepted. The signature file receives the archive's date. Each visible folder,
35including the mirror root, uses its newest contained archive date. Empty folders,
36README files and hidden operational/history files do not contribute to that date.
37Every cache miss, periodic refresh and backfill uses this same publication path.
38
39Version folders contain an architecture/OS download table built only from present
40canonical artifacts and signatures. Released versions list Release Notes,
41Language Reference and Standard Library Documentation below their heading only
42when the official index publishes the corresponding `notes`, `docs` or `stdDocs`
43URL. No documentation URL is synthesized or probed per artifact. The index has
44one process-wide, lock-protected snapshot, expiring after an hour; periodic/manual
45refresh bypasses it, as does a cache miss for an unindexed stable version's date.
46Refresh also updates the current release/build navigation when archives are
47already cached, preserving artifact and folder dates.
48
49Base versions link to their development index; a base folder with only development
50builds says “this release only has development builds out.” Each development
51version links back to its index, whose heading leaves the native directory listing
52to show the builds. Complete existing relative-link tables are retained when
53their introduction matches the published links and navigation; incomplete or
54production-only links are replaced. The owner's root `preadme.md` and hidden
55history remain untouched.
56
57Existing verified archives need a one-time metadata reconciliation. It reads
58signature comments and file metadata, not archive bodies, and preserves atime,
59ownership and modes. Preview it first:
60
61```sh
62python3 /app/mirror.py reconcile > /alloc/metadata-proposal.json
63python3 /app/mirror.py reconcile --apply --journal /alloc/private-metadata-undo.jsonl
64python3 /app/mirror.py reconcile > /alloc/metadata-after.json
65```
66
67Retain proposal and undo files privately outside `/mirror`; never publish them
68through Copyparty. Apply fails closed on missing dates, unsafe paths or changed
69files. The journal is created exclusively with mode0600 and fsynced before any
70change. Its first record retains old metadata and replaced public README bytes;
71a second record captures completed inode/metadata states. A journal without the
72completion record requires recovery review. A guarded one-time undo must verify
73those completed identities and generated README bytes before restoring old
74metadata/docs; later writes must cause it to stop. Keep that recovery helper with
75the private deployment evidence, rather than adding an automatic rollback hook.
76
77The worker has only `FOWNER` beyond its normal capability set, allowing explicit
78historic dates on the existing UID3000-owned NAS folders without changing their
79ownership. Its root filesystem is read-only and its only writable external mount
80is the canonical Zig subtree. Download temporaries and locks stay beneath hidden
81`.mirror-locks`; they cannot change visible folder dates. Fresh worker previews
82start empty, disable polling and fetch public artifacts only when requested.
83
2684## Production cutover
2785
2886The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files