From 248649cf074333f5aec822a0616f916ccab97dac Mon Sep 17 00:00:00 2001 From: clover caruso Date: Fri, 16 Oct 2026 23:59:48 -0700 Subject: [PATCH] Serve verified Zig artifacts through the public mirrors share Add a bounded cache with on-demand upstream fetches, periodic current-build refresh and resumable Nektro backfill. Keep preview data isolated and retain the existing tree until the documented copy and checksum cutover is approved. Verified the public preview download, ranges, anonymous write denial and private-share boundary. IPv6 routing and production cutover remain separately controlled. Assisted-by: gpt-6.1-sol --- service/copyparty/copyparty.conf | 8 + service/copyparty/prepare.py | 11 + service/copyparty/service.pkl | 9 +- service/zig-mirror/build/Dockerfile | 4 + service/zig-mirror/build/mirror.py | 326 ++++++++++++++++++++++++++++ service/zig-mirror/prepare.py | 14 ++ service/zig-mirror/service.pkl | 24 ++ tools/zig-mirror-test.py | 135 ++++++++++++ tools/zig-mirror.md | 91 ++++++++ 9 files changed, 618 insertions(+), 4 deletions(-) create mode 100644 service/zig-mirror/build/Dockerfile create mode 100644 service/zig-mirror/build/mirror.py create mode 100644 service/zig-mirror/prepare.py create mode 100644 service/zig-mirror/service.pkl create mode 100644 tools/zig-mirror-test.py create mode 100644 tools/zig-mirror.md diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index 15dcdbe7cf23b4159eada845d91fc83d77a1b9b2..7cf0ed8176c97bcf258ba33732513c6883dbbfe5 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -49,3 +49,11 @@ accs: rwmda.: snow g: * +[/mirrors] + /w/media/Software/Public + accs: + r: * + A: snow + flags: + chmod_f: 664 + chmod_d: 775 diff --git a/service/copyparty/prepare.py b/service/copyparty/prepare.py index 8ca419eea55204de75d7c3714dd2db07a58bc5c2..78d6ce035d16d53b72a04afa51194f00de781368 100644 --- a/service/copyparty/prepare.py +++ b/service/copyparty/prepare.py @@ -44,3 +44,14 @@ if not target.exists() or target.read_bytes() != rendered: copyparty.atomic(target, rendered, (data['uid'], data['uid'])) os.chmod(target, 0o600) os.chown(target, data['uid'], data['uid']) + +if service != 'copyparty': + public = Path(data['hostRoot']) / 'w/media/Software/Public/zig/0.0.0' + public.mkdir(parents=True, exist_ok=True) + fixture = public / 'zig-0.0.0.tar.xz' + fixture.write_bytes(b'Synthetic Zig mirror preview.\n') + for directory in [public, *public.parents]: + if directory == Path(data['hostRoot']): + break + os.chown(directory, data['uid'], data['uid']) + os.chown(fixture, data['uid'], data['uid']) diff --git a/service/copyparty/service.pkl b/service/copyparty/service.pkl index 069dcdb85bb96585a52677deabdd60f203ddd559..3819e3588a3a0cdb13c346d7b202a30c53f6b652 100644 --- a/service/copyparty/service.pkl +++ b/service/copyparty/service.pkl @@ -4,6 +4,7 @@ import "../../config/site.pkl" as site name = "Copyparty" prepare = "prepare.py" +stageIsolation = "fresh" secrets { ["idp_header"] { bytes = 16 } @@ -31,9 +32,9 @@ container { ["/cfg"] {} ["/w"] {} ["/hooks/reject-apple-cruft.py"] { config = "reject-apple-cruft.py" } - ["/w/clover"] { src = site.cloverRoot; readOnly = site.cloverReadOnly } - ["/w/clover/Media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - ["/w/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - ["/w/logs"] { src = "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly } + ["/w/clover"] { src = if (site.preview) null else site.cloverRoot; readOnly = site.cloverReadOnly } + ["/w/clover/Media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly } + ["/w/media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly } + ["/w/logs"] { src = if (site.preview) null else "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly } } } diff --git a/service/zig-mirror/build/Dockerfile b/service/zig-mirror/build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7afc07cf384bd50ec086c203a3fa21019e1cc466 --- /dev/null +++ b/service/zig-mirror/build/Dockerfile @@ -0,0 +1,4 @@ +FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332 +RUN apt-get update && apt-get install -y --no-install-recommends minisign ca-certificates && rm -rf /var/lib/apt/lists/* +COPY mirror.py /app/mirror.py +CMD ["python3", "-u", "/app/mirror.py", "serve"] diff --git a/service/zig-mirror/build/mirror.py b/service/zig-mirror/build/mirror.py new file mode 100644 index 0000000000000000000000000000000000000000..407b1bfe7b6e98344ffa10ac86261b8a46ee3069 --- /dev/null +++ b/service/zig-mirror/build/mirror.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +import argparse +import email.utils +import fcntl +import hashlib +from html.parser import HTMLParser +import http.server +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +PATTERN = re.compile(r'^zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') +VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') +PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' +OFFICIAL = 'https://ziglang.org' +NEKTRO = 'http://mirrors.nektro.net/zig/' +MAX_ARTIFACT = 2 * 1024**3 + + +class Unavailable(Exception): + pass + + +def filename(name): + match = PATTERN.fullmatch(name) + if len(name) > 240 or not match: + raise ValueError('invalid Zig artifact name') + version = match[1] + major, minor, patch = version.split('-')[0].split('.') + if any(part != str(int(part)) for part in (major, minor, patch)): + raise ValueError('invalid Zig version') + return version + + +def relative(name): + version = filename(name) + if '-dev.' in version: + return Path(version.split('-')[0]) / 'dev' / version / name + return Path(version) / name + + +def upstream(name): + version = filename(name) + prefix = '/builds/' if '-dev.' in version else '/download/' + version + '/' + return OFFICIAL + prefix + urllib.parse.quote(name) + + +def fetch(url, target, limit): + request = urllib.request.Request(url, headers={'User-Agent': 'snowglobe-zig-mirror/1'}) + try: + with urllib.request.urlopen(request, timeout=180) as response, target.open('wb') as output: + count = 0 + while chunk := response.read(1024 * 1024): + count += len(chunk) + if count > limit: + raise Unavailable('upstream artifact exceeds size limit') + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + except (OSError, urllib.error.URLError) as error: + raise Unavailable('upstream download failed') from error + + +def index(): + with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: + return json.load(response) + + +def digest(path): + checksum = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024 * 1024): + checksum.update(chunk) + return checksum.hexdigest() + + +class Mirror: + def __init__(self, root): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + self.locks = self.root / '.mirror-locks' + self.slots = threading.BoundedSemaphore(4) + self.refresh_lock = threading.Lock() + + def path(self, name): + path = self.root / relative(name) + if not path.resolve().is_relative_to(self.root.resolve()): + raise ValueError('artifact escapes mirror') + return path + + def ensure(self, name, source=None, checksum=None): + target = self.path(name) + artifact_name = name.removesuffix('.minisig') + artifact = self.path(artifact_name) + if target.is_file(): + return target + if not self.slots.acquire(timeout=1): + raise BlockingIOError('mirror download capacity reached') + try: + if shutil.disk_usage(self.root).free < 16 * 1024**3: + raise Unavailable('mirror free-space reserve reached') + self.locks.mkdir(exist_ok=True) + with (self.locks / artifact_name).open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if target.is_file(): + return target + artifact.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + pending = Path(temporary) / artifact_name + signature = Path(str(pending) + '.minisig') + url = source or upstream(artifact_name) + fetch(url, pending, MAX_ARTIFACT) + if checksum and digest(pending) != checksum: + raise Unavailable('artifact SHA-256 mismatch') + if not checksum or tuple(map(int, filename(artifact_name).split('-')[0].split('.'))) >= (0, 6, 0): + fetch(url + '.minisig', signature, 4096) + result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], + capture_output=True) + if result.returncode: + raise Unavailable('artifact signature verification failed') + os.chmod(pending, 0o664) + os.replace(pending, artifact) + if signature.exists(): + os.chmod(signature, 0o664) + os.replace(signature, Path(str(artifact) + '.minisig')) + if name.endswith('.minisig') and not target.exists(): + with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + pending = Path(temporary) / name + fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) + os.chmod(pending, 0o664) + os.replace(pending, target) + return target + finally: + self.slots.release() + + def verify(self): + entries = index() + hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] + for release in entries.values() for item in release.values() + if isinstance(item, dict) and 'tarball' in item} + checked = 0 + for artifact in self.root.rglob('zig-*'): + if not artifact.is_file() or artifact.name.endswith('.minisig'): + continue + filename(artifact.name) + if not artifact.resolve().is_relative_to(self.root.resolve()): + raise Unavailable('artifact escapes mirror') + checksum = hashes.get(artifact.name) + if checksum: + if digest(artifact) != checksum: + raise Unavailable('artifact SHA-256 mismatch: ' + str(artifact)) + else: + result = subprocess.run(['minisign', '-Vm', str(artifact), '-x', str(artifact) + '.minisig', '-P', PUBLIC_KEY], capture_output=True) + if result.returncode: + raise Unavailable('artifact signature verification failed: ' + str(artifact)) + checked += 1 + print(json.dumps({'verifiedArtifacts': checked})) + + def refresh(self): + if not self.refresh_lock.acquire(blocking=False): + return + try: + entries = index() + latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))] + for release in (latest, entries['master']): + for item in release.values(): + if isinstance(item, dict) and 'tarball' in item: + name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]) + self.ensure(name, checksum=item['shasum']) + finally: + self.refresh_lock.release() + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + timeout = 30 + + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) + if path == '/health': + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + return + name = path.removeprefix('/mirrors/zig/').removeprefix('/') + try: + filename(name) + target = self.server.mirror.ensure(name) + size = target.stat().st_size + first, last, status = 0, size - 1, 200 + if self.headers.get('Range'): + match = re.fullmatch(r'bytes=(\d*)-(\d*)', self.headers['Range']) + if not match or not any(match.groups()): + raise IndexError() + if not match[1]: + first = max(0, size - int(match[2])) + else: + first = int(match[1]) + if match[2]: + last = min(last, int(match[2])) + if first > last or first >= size: + raise IndexError() + status = 206 + self.send_response(status) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', str(last - first + 1)) + self.send_header('Accept-Ranges', 'bytes') + self.send_header('Cache-Control', 'public, max-age=31536000, immutable') + self.send_header('Last-Modified', email.utils.formatdate(target.stat().st_mtime, usegmt=True)) + if status == 206: + self.send_header('Content-Range', f'bytes {first}-{last}/{size}') + self.end_headers() + if self.command != 'HEAD': + with target.open('rb') as stream: + stream.seek(first) + remaining = last - first + 1 + while remaining and (chunk := stream.read(min(1024 * 1024, remaining))): + self.wfile.write(chunk) + remaining -= len(chunk) + except ValueError: + self.send_error(404) + except IndexError: + self.send_response(416) + self.send_header('Content-Range', f'bytes */{size}') + self.send_header('Content-Length', '0') + self.end_headers() + except BlockingIOError: + self.send_error(429) + except (Unavailable, OSError): + self.send_error(504) + + def do_POST(self): + self.send_error(405) + + +class Links(HTMLParser): + def __init__(self): + super().__init__() + self.names = [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + href = dict(attrs).get('href', '') + if '/' not in href.rstrip('/') and not href.startswith('.'): + self.names.append(urllib.parse.unquote(href)) + + +def listing(url): + with urllib.request.urlopen(url, timeout=30) as response: + links = Links() + links.feed(response.read(4 * 1024**2).decode()) + return links.names + + +def backfill(mirror, budget, apply): + entries = index() + hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] + for release in entries.values() for item in release.values() + if isinstance(item, dict) and 'tarball' in item} + copied = 0 + for version in listing(NEKTRO): + version = version.rstrip('/') + if not VERSION.fullmatch(version): + continue + url = NEKTRO + urllib.parse.quote(version) + '/' + for name in listing(url): + try: + filename(name) + except ValueError: + continue + if name.endswith('.minisig') or mirror.path(name).exists(): + continue + if not apply: + print(url + urllib.parse.quote(name)) + continue + if copied + MAX_ARTIFACT > budget: + raise SystemExit('backfill budget reached; rerun to resume') + path = mirror.ensure(name, url + urllib.parse.quote(name), hashes.get(name)) + copied += path.stat().st_size + print(json.dumps({'downloadedBytes': copied, 'apply': apply})) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify']) + parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror')) + parser.add_argument('--budget-gib', type=int, default=32) + parser.add_argument('--apply', action='store_true') + args = parser.parse_args() + mirror = Mirror(args.root) + if args.command == 'verify': + mirror.verify() + elif args.command == 'refresh': + mirror.refresh() + elif args.command == 'backfill': + backfill(mirror, args.budget_gib * 1024**3, args.apply) + else: + interval = int(os.environ.get('ZIG_MIRROR_REFRESH', '21600')) + if interval: + def refresh(): + while True: + try: + mirror.refresh() + except Exception as error: + print(type(error).__name__ + ': refresh failed', flush=True) + time.sleep(interval) + threading.Thread(target=refresh, daemon=True).start() + server = http.server.ThreadingHTTPServer(('0.0.0.0', 8080), Handler) + server.mirror = mirror + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/service/zig-mirror/prepare.py b/service/zig-mirror/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..de46a55841da7e191f3461032a08e36e27e7bfb0 --- /dev/null +++ b/service/zig-mirror/prepare.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +data = json.load(sys.stdin) +if Path(data['hostRoot']).name != 'zig-mirror': + root = Path(data['hostRoot']) / 'mirror' + artifact = root / '0.0.0/zig-0.0.0.tar.xz' + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(b'Synthetic Zig mirror preview.\n') + for path in (artifact.parent, artifact): + os.chown(path, data['uid'], data['uid']) diff --git a/service/zig-mirror/service.pkl b/service/zig-mirror/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8d55600734f34e28507b884c74228a9e7e8f8e6a --- /dev/null +++ b/service/zig-mirror/service.pkl @@ -0,0 +1,24 @@ +amends "../../config/Service.pkl" + +import "../../config/site.pkl" as site + +name = "Zig mirror" +launcher = false +stageIsolation = "fresh" +prepare = "prepare.py" + +container { + build = "build" + http { + containerPort = 8080 + subdomain = "zig" + checkPath = "/health" + } + volumes { + ["/mirror"] { src = if (site.preview) null else "\(site.mediaRoot)/Software/Public/zig" } + } + env { + ["ZIG_MIRROR_ROOT"] = "/mirror" + ["ZIG_MIRROR_REFRESH"] = if (site.preview) "0" else "21600" + } +} diff --git a/tools/zig-mirror-test.py b/tools/zig-mirror-test.py new file mode 100644 index 0000000000000000000000000000000000000000..eb084cae5f4faf6c1acb44cd57e8c8dcfa54a05d --- /dev/null +++ b/tools/zig-mirror-test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import concurrent.futures +import importlib.util +import io +import contextlib +import json +from pathlib import Path +from types import SimpleNamespace +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('mirror', Path(__file__).resolve().parents[1] / 'service/zig-mirror/build/mirror.py') +mirror = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mirror) + + +class MirrorTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.cache = mirror.Mirror(self.temporary.name) + disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)) + disk.start() + self.addCleanup(disk.stop) + + def test_spec_filenames_and_existing_layout(self): + cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), + ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'), + ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')] + for name, prefix in cases: + self.assertEqual(str(mirror.relative(name)), prefix + name) + self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name)) + for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz']: + with self.assertRaises(ValueError): + mirror.filename(name) + + def test_concurrent_miss_downloads_once_and_atomic_signature_pair(self): + calls = [] + def fetch(url, target, limit): + calls.append(url) + time.sleep(.03) + target.write_bytes(b'signed data' if not url.endswith('.minisig') else b'signature') + with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify: + verify.return_value.returncode = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers: + paths = list(workers.map(self.cache.ensure, ['zig-0.17.0.tar.xz'] * 4)) + self.assertEqual(len(calls), 2) + self.assertTrue(all(path.read_bytes() == b'signed data' for path in paths)) + self.assertEqual(Path(str(paths[0]) + '.minisig').read_bytes(), b'signature') + self.assertFalse(list(self.cache.root.rglob('.download-*'))) + + def test_bad_hash_and_signature_never_publish(self): + def fetch(url, target, limit): + target.write_bytes(b'bad data') + with patch.object(mirror, 'fetch', side_effect=fetch): + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz', checksum='0' * 64) + with patch.object(mirror.subprocess, 'run') as verify: + verify.return_value.returncode = 1 + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz') + self.assertFalse(self.cache.path('zig-0.17.0.tar.xz').exists()) + self.assertFalse(list(self.cache.root.rglob('.download-*'))) + + def test_symlink_cannot_escape_cache(self): + (self.cache.root / '0.17.0').symlink_to(self.cache.root.parent) + with self.assertRaises(ValueError): + self.cache.ensure('zig-0.17.0.tar.xz') + + def test_reserve_rejects_new_download_but_retains_cached_reads(self): + with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=0)), patch.object(mirror, 'fetch') as fetch: + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz') + fetch.assert_not_called() + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'cached') + self.assertEqual(self.cache.ensure(target.name), target) + + def test_backfill_dry_run_and_budget_do_not_download(self): + versions = ['0.17.0/', '../', 'not-a-version/'] + artifacts = ['zig-0.17.0.tar.xz', 'zig-0.17.0.tar.xz.minisig', 'private.txt'] + with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure, contextlib.redirect_stdout(io.StringIO()) as output: + mirror.backfill(self.cache, 0, False) + ensure.assert_not_called() + self.assertIn('http://mirrors.nektro.net/zig/0.17.0/zig-0.17.0.tar.xz', output.getvalue()) + self.assertNotIn('private.txt', output.getvalue()) + with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure: + with self.assertRaises(SystemExit): + mirror.backfill(self.cache, 0, True) + ensure.assert_not_called() + + def test_verify_rejects_corrupted_existing_archive(self): + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'corrupt') + entries = {'0.17.0': {'src': {'tarball': 'https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz', 'shasum': '0' * 64}}} + with patch.object(mirror, 'index', return_value=entries): + with self.assertRaises(mirror.Unavailable): + self.cache.verify() + + def test_http_query_range_head_denials_and_timeout(self): + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'0123456789') + server = mirror.http.server.ThreadingHTTPServer(('127.0.0.1', 0), mirror.Handler) + server.mirror = self.cache + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base = 'http://127.0.0.1:' + str(server.server_port) + with urllib.request.urlopen(base + '/mirrors/zig/zig-0.17.0.tar.xz?source=spec-test') as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.read(), b'0123456789') + for method, value, expected in [('GET', 'bytes=2-5', b'2345'), ('GET', 'bytes=-3', b'789'), ('HEAD', 'bytes=2-5', b'')]: + with urllib.request.urlopen(urllib.request.Request(base + '/zig-0.17.0.tar.xz', method=method, headers={'Range': value})) as response: + self.assertEqual(response.status, 206) + self.assertEqual(response.read(), expected) + for path, headers, status in [('/zig-0.17.0.tar.xz', {'Range': 'bytes=100-'}, 416), ('/../secret', {}, 404), ('/0.17.0/zig-0.17.0.tar.xz', {}, 404)]: + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(urllib.request.Request(base + path, headers=headers)) + self.assertEqual(error.exception.code, status) + with patch.object(self.cache, 'ensure', side_effect=mirror.Unavailable()): + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(base + '/zig-0.17.0.tar.xz') + self.assertEqual(error.exception.code, 504) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/zig-mirror.md b/tools/zig-mirror.md new file mode 100644 index 0000000000000000000000000000000000000000..3905a6e36060f2e90dfa52d17e1ec7d544b7a9d6 --- /dev/null +++ b/tools/zig-mirror.md @@ -0,0 +1,91 @@ +# Zig mirror + +The canonical artifact tree is `/srv/clover/Media/Software/Public/zig`. +Copyparty publishes only `Software/Public` as anonymous read-only `/mirrors`; +its existing authenticated `/media` and `/clover` mounts keep their permissions. +The mirror container shares only the Zig subtree. Flat protocol URLs fetch +missing files synchronously, then serve them locally; nested release/dev paths +remain browsable through Copyparty. + +[Zig's mirror contract](https://codeberg.org/ziglang/ziglang.org/src/branch/main/MIRRORS.md) +requires HTTPS on **both IPv4 and IPv6**. Do not submit this mirror to Zig's list +until both address families work from outside the LAN. On 2026-10-17 the host +had a public IPv6 RA address/default route, but outbound IPv6 to Zig and Google +failed and `file.paperclover.net` had no AAAA record. DDNS currently updates A +records only. A working delegated prefix/router firewall and matching AAAA +publication must be verified before enabling dual-stack discovery. + +The worker polls the official index every six hours and caches the latest stable +release and current development build. A request for any uncached supported name +checks its exact official URL immediately; a newly published release needs no +manual trigger or wait for the poll. `refresh` runs the periodic operation on +demand. Missing upstream files/timeouts return 504; saturated downloads return +429. Downloads have four slots, a 2 GiB per-artifact ceiling, and a 16 GiB +free-space reserve. Completed archives and signatures publish atomically after +SHA-256 and/or Zig Minisign verification. Existing archive files need the +one-time `verify` operation before first public exposure. + +## Production cutover + +This is a reviewable runbook, **not an executed migration**. The 2026-10-17 +inventory found 267 GiB / 10,063 files in `/srv/mirrors/zig`, on `globe/mirrors`. +The destination Zig directory existed empty on `globe/clover/Media`, with about +9.13 TiB available. These are different filesystems: an atomic rename cannot +move the tree. Preserve the old tree until copy verification and public checks +finish. Both roots use group `chloe` (GID 3000). + +After production approval, confirm the host machine ID +`691734bfb3c24b19ae66e5cacd17f5e2`, mounts, free space, and that no writer is +changing the old tree. Then copy without deleting the source: + +```sh +findmnt -T /srv/mirrors/zig +findmnt -T /srv/clover/Media/Software/Public/zig +rsync -aH --info=progress2 /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/ +rsync -anrc --delete /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/ +``` + +The checksum dry run must report no differences. Preserve existing numeric +ownership and ACLs; do not recursively chown the Media dataset. The destination +root must retain group traversal/write access for the mirror's declared Clover +group. Run the mirror image's `verify` command against the copied destination +before publishing it. A disposable container can bind that exact subtree +read-only. Retain +any old `.hist` metadata privately; anonymous users lack dot-file permission. + +Promote the reviewed configuration through `tools/deploy.py prod`. Do not replace +containers by hand. Confirm anonymous GET/HEAD/Range on flat and nested URLs, +anonymous upload/delete denial, authenticated existing shares, and that parent +folders remain inaccessible. The source can be retired separately after the +verified copy and service behavior are accepted; it is not an active second +mirror and no alias is required. + +## Backfill and manual refresh + +Inside the running mirror task, the same script exposes: + +```sh +python3 /app/mirror.py verify +python3 /app/mirror.py refresh +python3 /app/mirror.py backfill +python3 /app/mirror.py backfill --apply --budget-gib 32 +``` + +Use the normal authenticated Nomad allocation exec path to run these. The +no-`--apply` backfill lists missing artifact URLs without downloading them. +The source is [Meghan's mirror](http://mirrors.nektro.net/zig/), which listed +893 version directories on 2026-10-17. Stable files also check the official +index hash; development files require Zig's signature. Invalid bytes never +become public. The 32 GiB budget reserves the full per-artifact ceiling before +each download and stops conservatively; rerun to resume, preserving cached +files. Missing historical builds cannot be recovered from Zig once removed, +so failed verification/downloads stop the batch for review. + +## Staging + +Both services use fresh previews. Copyparty mounts disposable directories instead +of the live NAS and generates one clearly synthetic public archive. Mirror +previews disable periodic downloads. A flat cache miss may fetch a public Zig +artifact on demand for verification. The preview association lives in the +Copyparty stage's `mirrorStage` metadata; an unpaired preview returns 503 for +flat protocol URLs and never falls back to production. -- 2.54.1