diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index 15dcdbe7cf23b4159eada845d91fc83d77a1b9b2..7cf0ed8176c97bcf258ba33732513c6883dbbfe5 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -49,3 +49,11 @@ accs: rwmda.: snow g: * +[/mirrors] + /w/media/Software/Public + accs: + r: * + A: snow + flags: + chmod_f: 664 + chmod_d: 775 diff --git a/service/copyparty/prepare.py b/service/copyparty/prepare.py index 8ca419eea55204de75d7c3714dd2db07a58bc5c2..78d6ce035d16d53b72a04afa51194f00de781368 100644 --- a/service/copyparty/prepare.py +++ b/service/copyparty/prepare.py @@ -44,3 +44,14 @@ if not target.exists() or target.read_bytes() != rendered: copyparty.atomic(target, rendered, (data['uid'], data['uid'])) os.chmod(target, 0o600) os.chown(target, data['uid'], data['uid']) + +if service != 'copyparty': + public = Path(data['hostRoot']) / 'w/media/Software/Public/zig/0.0.0' + public.mkdir(parents=True, exist_ok=True) + fixture = public / 'zig-0.0.0.tar.xz' + fixture.write_bytes(b'Synthetic Zig mirror preview.\n') + for directory in [public, *public.parents]: + if directory == Path(data['hostRoot']): + break + os.chown(directory, data['uid'], data['uid']) + os.chown(fixture, data['uid'], data['uid']) diff --git a/service/copyparty/service.pkl b/service/copyparty/service.pkl index 069dcdb85bb96585a52677deabdd60f203ddd559..3819e3588a3a0cdb13c346d7b202a30c53f6b652 100644 --- a/service/copyparty/service.pkl +++ b/service/copyparty/service.pkl @@ -4,6 +4,7 @@ import "../../config/site.pkl" as site name = "Copyparty" prepare = "prepare.py" +stageIsolation = "fresh" secrets { ["idp_header"] { bytes = 16 } @@ -31,9 +32,9 @@ container { ["/cfg"] {} ["/w"] {} ["/hooks/reject-apple-cruft.py"] { config = "reject-apple-cruft.py" } - ["/w/clover"] { src = site.cloverRoot; readOnly = site.cloverReadOnly } - ["/w/clover/Media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - ["/w/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - ["/w/logs"] { src = "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly } + ["/w/clover"] { src = if (site.preview) null else site.cloverRoot; readOnly = site.cloverReadOnly } + ["/w/clover/Media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly } + ["/w/media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly } + ["/w/logs"] { src = if (site.preview) null else "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly } } } diff --git a/service/zig-mirror/build/Dockerfile b/service/zig-mirror/build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7afc07cf384bd50ec086c203a3fa21019e1cc466 --- /dev/null +++ b/service/zig-mirror/build/Dockerfile @@ -0,0 +1,4 @@ +FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332 +RUN apt-get update && apt-get install -y --no-install-recommends minisign ca-certificates && rm -rf /var/lib/apt/lists/* +COPY mirror.py /app/mirror.py +CMD ["python3", "-u", "/app/mirror.py", "serve"] diff --git a/service/zig-mirror/build/mirror.py b/service/zig-mirror/build/mirror.py new file mode 100644 index 0000000000000000000000000000000000000000..407b1bfe7b6e98344ffa10ac86261b8a46ee3069 --- /dev/null +++ b/service/zig-mirror/build/mirror.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +import argparse +import email.utils +import fcntl +import hashlib +from html.parser import HTMLParser +import http.server +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +PATTERN = re.compile(r'^zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') +VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') +PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' +OFFICIAL = 'https://ziglang.org' +NEKTRO = 'http://mirrors.nektro.net/zig/' +MAX_ARTIFACT = 2 * 1024**3 + + +class Unavailable(Exception): + pass + + +def filename(name): + match = PATTERN.fullmatch(name) + if len(name) > 240 or not match: + raise ValueError('invalid Zig artifact name') + version = match[1] + major, minor, patch = version.split('-')[0].split('.') + if any(part != str(int(part)) for part in (major, minor, patch)): + raise ValueError('invalid Zig version') + return version + + +def relative(name): + version = filename(name) + if '-dev.' in version: + return Path(version.split('-')[0]) / 'dev' / version / name + return Path(version) / name + + +def upstream(name): + version = filename(name) + prefix = '/builds/' if '-dev.' in version else '/download/' + version + '/' + return OFFICIAL + prefix + urllib.parse.quote(name) + + +def fetch(url, target, limit): + request = urllib.request.Request(url, headers={'User-Agent': 'snowglobe-zig-mirror/1'}) + try: + with urllib.request.urlopen(request, timeout=180) as response, target.open('wb') as output: + count = 0 + while chunk := response.read(1024 * 1024): + count += len(chunk) + if count > limit: + raise Unavailable('upstream artifact exceeds size limit') + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + except (OSError, urllib.error.URLError) as error: + raise Unavailable('upstream download failed') from error + + +def index(): + with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: + return json.load(response) + + +def digest(path): + checksum = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024 * 1024): + checksum.update(chunk) + return checksum.hexdigest() + + +class Mirror: + def __init__(self, root): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + self.locks = self.root / '.mirror-locks' + self.slots = threading.BoundedSemaphore(4) + self.refresh_lock = threading.Lock() + + def path(self, name): + path = self.root / relative(name) + if not path.resolve().is_relative_to(self.root.resolve()): + raise ValueError('artifact escapes mirror') + return path + + def ensure(self, name, source=None, checksum=None): + target = self.path(name) + artifact_name = name.removesuffix('.minisig') + artifact = self.path(artifact_name) + if target.is_file(): + return target + if not self.slots.acquire(timeout=1): + raise BlockingIOError('mirror download capacity reached') + try: + if shutil.disk_usage(self.root).free < 16 * 1024**3: + raise Unavailable('mirror free-space reserve reached') + self.locks.mkdir(exist_ok=True) + with (self.locks / artifact_name).open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if target.is_file(): + return target + artifact.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + pending = Path(temporary) / artifact_name + signature = Path(str(pending) + '.minisig') + url = source or upstream(artifact_name) + fetch(url, pending, MAX_ARTIFACT) + if checksum and digest(pending) != checksum: + raise Unavailable('artifact SHA-256 mismatch') + if not checksum or tuple(map(int, filename(artifact_name).split('-')[0].split('.'))) >= (0, 6, 0): + fetch(url + '.minisig', signature, 4096) + result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], + capture_output=True) + if result.returncode: + raise Unavailable('artifact signature verification failed') + os.chmod(pending, 0o664) + os.replace(pending, artifact) + if signature.exists(): + os.chmod(signature, 0o664) + os.replace(signature, Path(str(artifact) + '.minisig')) + if name.endswith('.minisig') and not target.exists(): + with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + pending = Path(temporary) / name + fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) + os.chmod(pending, 0o664) + os.replace(pending, target) + return target + finally: + self.slots.release() + + def verify(self): + entries = index() + hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] + for release in entries.values() for item in release.values() + if isinstance(item, dict) and 'tarball' in item} + checked = 0 + for artifact in self.root.rglob('zig-*'): + if not artifact.is_file() or artifact.name.endswith('.minisig'): + continue + filename(artifact.name) + if not artifact.resolve().is_relative_to(self.root.resolve()): + raise Unavailable('artifact escapes mirror') + checksum = hashes.get(artifact.name) + if checksum: + if digest(artifact) != checksum: + raise Unavailable('artifact SHA-256 mismatch: ' + str(artifact)) + else: + result = subprocess.run(['minisign', '-Vm', str(artifact), '-x', str(artifact) + '.minisig', '-P', PUBLIC_KEY], capture_output=True) + if result.returncode: + raise Unavailable('artifact signature verification failed: ' + str(artifact)) + checked += 1 + print(json.dumps({'verifiedArtifacts': checked})) + + def refresh(self): + if not self.refresh_lock.acquire(blocking=False): + return + try: + entries = index() + latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))] + for release in (latest, entries['master']): + for item in release.values(): + if isinstance(item, dict) and 'tarball' in item: + name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]) + self.ensure(name, checksum=item['shasum']) + finally: + self.refresh_lock.release() + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + timeout = 30 + + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) + if path == '/health': + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + return + name = path.removeprefix('/mirrors/zig/').removeprefix('/') + try: + filename(name) + target = self.server.mirror.ensure(name) + size = target.stat().st_size + first, last, status = 0, size - 1, 200 + if self.headers.get('Range'): + match = re.fullmatch(r'bytes=(\d*)-(\d*)', self.headers['Range']) + if not match or not any(match.groups()): + raise IndexError() + if not match[1]: + first = max(0, size - int(match[2])) + else: + first = int(match[1]) + if match[2]: + last = min(last, int(match[2])) + if first > last or first >= size: + raise IndexError() + status = 206 + self.send_response(status) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', str(last - first + 1)) + self.send_header('Accept-Ranges', 'bytes') + self.send_header('Cache-Control', 'public, max-age=31536000, immutable') + self.send_header('Last-Modified', email.utils.formatdate(target.stat().st_mtime, usegmt=True)) + if status == 206: + self.send_header('Content-Range', f'bytes {first}-{last}/{size}') + self.end_headers() + if self.command != 'HEAD': + with target.open('rb') as stream: + stream.seek(first) + remaining = last - first + 1 + while remaining and (chunk := stream.read(min(1024 * 1024, remaining))): + self.wfile.write(chunk) + remaining -= len(chunk) + except ValueError: + self.send_error(404) + except IndexError: + self.send_response(416) + self.send_header('Content-Range', f'bytes */{size}') + self.send_header('Content-Length', '0') + self.end_headers() + except BlockingIOError: + self.send_error(429) + except (Unavailable, OSError): + self.send_error(504) + + def do_POST(self): + self.send_error(405) + + +class Links(HTMLParser): + def __init__(self): + super().__init__() + self.names = [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + href = dict(attrs).get('href', '') + if '/' not in href.rstrip('/') and not href.startswith('.'): + self.names.append(urllib.parse.unquote(href)) + + +def listing(url): + with urllib.request.urlopen(url, timeout=30) as response: + links = Links() + links.feed(response.read(4 * 1024**2).decode()) + return links.names + + +def backfill(mirror, budget, apply): + entries = index() + hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] + for release in entries.values() for item in release.values() + if isinstance(item, dict) and 'tarball' in item} + copied = 0 + for version in listing(NEKTRO): + version = version.rstrip('/') + if not VERSION.fullmatch(version): + continue + url = NEKTRO + urllib.parse.quote(version) + '/' + for name in listing(url): + try: + filename(name) + except ValueError: + continue + if name.endswith('.minisig') or mirror.path(name).exists(): + continue + if not apply: + print(url + urllib.parse.quote(name)) + continue + if copied + MAX_ARTIFACT > budget: + raise SystemExit('backfill budget reached; rerun to resume') + path = mirror.ensure(name, url + urllib.parse.quote(name), hashes.get(name)) + copied += path.stat().st_size + print(json.dumps({'downloadedBytes': copied, 'apply': apply})) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify']) + parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror')) + parser.add_argument('--budget-gib', type=int, default=32) + parser.add_argument('--apply', action='store_true') + args = parser.parse_args() + mirror = Mirror(args.root) + if args.command == 'verify': + mirror.verify() + elif args.command == 'refresh': + mirror.refresh() + elif args.command == 'backfill': + backfill(mirror, args.budget_gib * 1024**3, args.apply) + else: + interval = int(os.environ.get('ZIG_MIRROR_REFRESH', '21600')) + if interval: + def refresh(): + while True: + try: + mirror.refresh() + except Exception as error: + print(type(error).__name__ + ': refresh failed', flush=True) + time.sleep(interval) + threading.Thread(target=refresh, daemon=True).start() + server = http.server.ThreadingHTTPServer(('0.0.0.0', 8080), Handler) + server.mirror = mirror + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/service/zig-mirror/prepare.py b/service/zig-mirror/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..de46a55841da7e191f3461032a08e36e27e7bfb0 --- /dev/null +++ b/service/zig-mirror/prepare.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +data = json.load(sys.stdin) +if Path(data['hostRoot']).name != 'zig-mirror': + root = Path(data['hostRoot']) / 'mirror' + artifact = root / '0.0.0/zig-0.0.0.tar.xz' + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(b'Synthetic Zig mirror preview.\n') + for path in (artifact.parent, artifact): + os.chown(path, data['uid'], data['uid']) diff --git a/service/zig-mirror/service.pkl b/service/zig-mirror/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8d55600734f34e28507b884c74228a9e7e8f8e6a --- /dev/null +++ b/service/zig-mirror/service.pkl @@ -0,0 +1,24 @@ +amends "../../config/Service.pkl" + +import "../../config/site.pkl" as site + +name = "Zig mirror" +launcher = false +stageIsolation = "fresh" +prepare = "prepare.py" + +container { + build = "build" + http { + containerPort = 8080 + subdomain = "zig" + checkPath = "/health" + } + volumes { + ["/mirror"] { src = if (site.preview) null else "\(site.mediaRoot)/Software/Public/zig" } + } + env { + ["ZIG_MIRROR_ROOT"] = "/mirror" + ["ZIG_MIRROR_REFRESH"] = if (site.preview) "0" else "21600" + } +} diff --git a/tools/zig-mirror-test.py b/tools/zig-mirror-test.py new file mode 100644 index 0000000000000000000000000000000000000000..eb084cae5f4faf6c1acb44cd57e8c8dcfa54a05d --- /dev/null +++ b/tools/zig-mirror-test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import concurrent.futures +import importlib.util +import io +import contextlib +import json +from pathlib import Path +from types import SimpleNamespace +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('mirror', Path(__file__).resolve().parents[1] / 'service/zig-mirror/build/mirror.py') +mirror = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mirror) + + +class MirrorTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.cache = mirror.Mirror(self.temporary.name) + disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)) + disk.start() + self.addCleanup(disk.stop) + + def test_spec_filenames_and_existing_layout(self): + cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), + ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'), + ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')] + for name, prefix in cases: + self.assertEqual(str(mirror.relative(name)), prefix + name) + self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name)) + for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz']: + with self.assertRaises(ValueError): + mirror.filename(name) + + def test_concurrent_miss_downloads_once_and_atomic_signature_pair(self): + calls = [] + def fetch(url, target, limit): + calls.append(url) + time.sleep(.03) + target.write_bytes(b'signed data' if not url.endswith('.minisig') else b'signature') + with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify: + verify.return_value.returncode = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers: + paths = list(workers.map(self.cache.ensure, ['zig-0.17.0.tar.xz'] * 4)) + self.assertEqual(len(calls), 2) + self.assertTrue(all(path.read_bytes() == b'signed data' for path in paths)) + self.assertEqual(Path(str(paths[0]) + '.minisig').read_bytes(), b'signature') + self.assertFalse(list(self.cache.root.rglob('.download-*'))) + + def test_bad_hash_and_signature_never_publish(self): + def fetch(url, target, limit): + target.write_bytes(b'bad data') + with patch.object(mirror, 'fetch', side_effect=fetch): + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz', checksum='0' * 64) + with patch.object(mirror.subprocess, 'run') as verify: + verify.return_value.returncode = 1 + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz') + self.assertFalse(self.cache.path('zig-0.17.0.tar.xz').exists()) + self.assertFalse(list(self.cache.root.rglob('.download-*'))) + + def test_symlink_cannot_escape_cache(self): + (self.cache.root / '0.17.0').symlink_to(self.cache.root.parent) + with self.assertRaises(ValueError): + self.cache.ensure('zig-0.17.0.tar.xz') + + def test_reserve_rejects_new_download_but_retains_cached_reads(self): + with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=0)), patch.object(mirror, 'fetch') as fetch: + with self.assertRaises(mirror.Unavailable): + self.cache.ensure('zig-0.17.0.tar.xz') + fetch.assert_not_called() + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'cached') + self.assertEqual(self.cache.ensure(target.name), target) + + def test_backfill_dry_run_and_budget_do_not_download(self): + versions = ['0.17.0/', '../', 'not-a-version/'] + artifacts = ['zig-0.17.0.tar.xz', 'zig-0.17.0.tar.xz.minisig', 'private.txt'] + with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure, contextlib.redirect_stdout(io.StringIO()) as output: + mirror.backfill(self.cache, 0, False) + ensure.assert_not_called() + self.assertIn('http://mirrors.nektro.net/zig/0.17.0/zig-0.17.0.tar.xz', output.getvalue()) + self.assertNotIn('private.txt', output.getvalue()) + with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure: + with self.assertRaises(SystemExit): + mirror.backfill(self.cache, 0, True) + ensure.assert_not_called() + + def test_verify_rejects_corrupted_existing_archive(self): + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'corrupt') + entries = {'0.17.0': {'src': {'tarball': 'https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz', 'shasum': '0' * 64}}} + with patch.object(mirror, 'index', return_value=entries): + with self.assertRaises(mirror.Unavailable): + self.cache.verify() + + def test_http_query_range_head_denials_and_timeout(self): + target = self.cache.path('zig-0.17.0.tar.xz') + target.parent.mkdir() + target.write_bytes(b'0123456789') + server = mirror.http.server.ThreadingHTTPServer(('127.0.0.1', 0), mirror.Handler) + server.mirror = self.cache + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base = 'http://127.0.0.1:' + str(server.server_port) + with urllib.request.urlopen(base + '/mirrors/zig/zig-0.17.0.tar.xz?source=spec-test') as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.read(), b'0123456789') + for method, value, expected in [('GET', 'bytes=2-5', b'2345'), ('GET', 'bytes=-3', b'789'), ('HEAD', 'bytes=2-5', b'')]: + with urllib.request.urlopen(urllib.request.Request(base + '/zig-0.17.0.tar.xz', method=method, headers={'Range': value})) as response: + self.assertEqual(response.status, 206) + self.assertEqual(response.read(), expected) + for path, headers, status in [('/zig-0.17.0.tar.xz', {'Range': 'bytes=100-'}, 416), ('/../secret', {}, 404), ('/0.17.0/zig-0.17.0.tar.xz', {}, 404)]: + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(urllib.request.Request(base + path, headers=headers)) + self.assertEqual(error.exception.code, status) + with patch.object(self.cache, 'ensure', side_effect=mirror.Unavailable()): + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(base + '/zig-0.17.0.tar.xz') + self.assertEqual(error.exception.code, 504) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/zig-mirror.md b/tools/zig-mirror.md new file mode 100644 index 0000000000000000000000000000000000000000..3905a6e36060f2e90dfa52d17e1ec7d544b7a9d6 --- /dev/null +++ b/tools/zig-mirror.md @@ -0,0 +1,91 @@ +# Zig mirror + +The canonical artifact tree is `/srv/clover/Media/Software/Public/zig`. +Copyparty publishes only `Software/Public` as anonymous read-only `/mirrors`; +its existing authenticated `/media` and `/clover` mounts keep their permissions. +The mirror container shares only the Zig subtree. Flat protocol URLs fetch +missing files synchronously, then serve them locally; nested release/dev paths +remain browsable through Copyparty. + +[Zig's mirror contract](https://codeberg.org/ziglang/ziglang.org/src/branch/main/MIRRORS.md) +requires HTTPS on **both IPv4 and IPv6**. Do not submit this mirror to Zig's list +until both address families work from outside the LAN. On 2026-10-17 the host +had a public IPv6 RA address/default route, but outbound IPv6 to Zig and Google +failed and `file.paperclover.net` had no AAAA record. DDNS currently updates A +records only. A working delegated prefix/router firewall and matching AAAA +publication must be verified before enabling dual-stack discovery. + +The worker polls the official index every six hours and caches the latest stable +release and current development build. A request for any uncached supported name +checks its exact official URL immediately; a newly published release needs no +manual trigger or wait for the poll. `refresh` runs the periodic operation on +demand. Missing upstream files/timeouts return 504; saturated downloads return +429. Downloads have four slots, a 2 GiB per-artifact ceiling, and a 16 GiB +free-space reserve. Completed archives and signatures publish atomically after +SHA-256 and/or Zig Minisign verification. Existing archive files need the +one-time `verify` operation before first public exposure. + +## Production cutover + +This is a reviewable runbook, **not an executed migration**. The 2026-10-17 +inventory found 267 GiB / 10,063 files in `/srv/mirrors/zig`, on `globe/mirrors`. +The destination Zig directory existed empty on `globe/clover/Media`, with about +9.13 TiB available. These are different filesystems: an atomic rename cannot +move the tree. Preserve the old tree until copy verification and public checks +finish. Both roots use group `chloe` (GID 3000). + +After production approval, confirm the host machine ID +`691734bfb3c24b19ae66e5cacd17f5e2`, mounts, free space, and that no writer is +changing the old tree. Then copy without deleting the source: + +```sh +findmnt -T /srv/mirrors/zig +findmnt -T /srv/clover/Media/Software/Public/zig +rsync -aH --info=progress2 /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/ +rsync -anrc --delete /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/ +``` + +The checksum dry run must report no differences. Preserve existing numeric +ownership and ACLs; do not recursively chown the Media dataset. The destination +root must retain group traversal/write access for the mirror's declared Clover +group. Run the mirror image's `verify` command against the copied destination +before publishing it. A disposable container can bind that exact subtree +read-only. Retain +any old `.hist` metadata privately; anonymous users lack dot-file permission. + +Promote the reviewed configuration through `tools/deploy.py prod`. Do not replace +containers by hand. Confirm anonymous GET/HEAD/Range on flat and nested URLs, +anonymous upload/delete denial, authenticated existing shares, and that parent +folders remain inaccessible. The source can be retired separately after the +verified copy and service behavior are accepted; it is not an active second +mirror and no alias is required. + +## Backfill and manual refresh + +Inside the running mirror task, the same script exposes: + +```sh +python3 /app/mirror.py verify +python3 /app/mirror.py refresh +python3 /app/mirror.py backfill +python3 /app/mirror.py backfill --apply --budget-gib 32 +``` + +Use the normal authenticated Nomad allocation exec path to run these. The +no-`--apply` backfill lists missing artifact URLs without downloading them. +The source is [Meghan's mirror](http://mirrors.nektro.net/zig/), which listed +893 version directories on 2026-10-17. Stable files also check the official +index hash; development files require Zig's signature. Invalid bytes never +become public. The 32 GiB budget reserves the full per-artifact ceiling before +each download and stops conservatively; rerun to resume, preserving cached +files. Missing historical builds cannot be recovered from Zig once removed, +so failed verification/downloads stop the batch for review. + +## Staging + +Both services use fresh previews. Copyparty mounts disposable directories instead +of the live NAS and generates one clearly synthetic public archive. Mirror +previews disable periodic downloads. A flat cache miss may fetch a public Zig +artifact on demand for verification. The preview association lives in the +Copyparty stage's `mirrorStage` metadata; an unpaired preview returns 503 for +flat protocol URLs and never falls back to production.