| author | |
| committer | |
| log | 803ad5b24699b4589c6a829aad7e0b66b4fb719f |
| tree | a00f29b6ccb454fde2a91c97b11e33b165aacbc8 |
| parent | 8b177d2982cee8727b59b7ea06cd3154c80f54de |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
4 files changed, 53 insertions(+), 5 deletions(-)
service/zig-mirror/build/mirror.py+12-2| ... | @@ -18,7 +18,7 @@ import urllib.error | ... | @@ -18,7 +18,7 @@ import urllib.error |
| 18 | import urllib.parse | 18 | import urllib.parse |
| 19 | import urllib.request | 19 | import urllib.request |
| 20 | 20 | ||
| 21 | PATTERN = re.compile(r'^zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') | 21 | PATTERN = re.compile(r'^zig(?:|-bootstrap|-win64|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') |
| 22 | VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') | 22 | VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') |
| 23 | PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' | 23 | PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' |
| 24 | OFFICIAL = 'https://ziglang.org' | 24 | OFFICIAL = 'https://ziglang.org' |
| ... | @@ -34,6 +34,8 @@ def filename(name): | ... | @@ -34,6 +34,8 @@ def filename(name): |
| 34 | match = PATTERN.fullmatch(name) | 34 | match = PATTERN.fullmatch(name) |
| 35 | if len(name) > 240 or not match: | 35 | if len(name) > 240 or not match: |
| 36 | raise ValueError('invalid Zig artifact name') | 36 | raise ValueError('invalid Zig artifact name') |
| 37 | if name.startswith('zig-win64-') and name not in {'zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip'}: | ||
| 38 | raise ValueError('invalid legacy Zig artifact name') | ||
| 37 | version = match[1] | 39 | version = match[1] |
| 38 | major, minor, patch = version.split('-')[0].split('.') | 40 | major, minor, patch = version.split('-')[0].split('.') |
| 39 | if any(part != str(int(part)) for part in (major, minor, patch)): | 41 | if any(part != str(int(part)) for part in (major, minor, patch)): |
| ... | @@ -113,6 +115,14 @@ class Mirror: | ... | @@ -113,6 +115,14 @@ class Mirror: |
| 113 | fcntl.flock(lock, fcntl.LOCK_EX) | 115 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 114 | if target.is_file(): | 116 | if target.is_file(): |
| 115 | return target | 117 | return target |
| 118 | version = filename(artifact_name) | ||
| 119 | signed = '-dev.' in version or tuple(map(int, version.split('.'))) >= (0, 6, 0) | ||
| 120 | if not signed and not checksum: | ||
| 121 | checksum = next((item['shasum'] for item in index().get(version, {}).values() | ||
| 122 | if isinstance(item, dict) and 'tarball' in item | ||
| 123 | and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) | ||
| 124 | if not checksum: | ||
| 125 | raise Unavailable('unsigned artifact has no official SHA-256') | ||
| 116 | artifact.parent.mkdir(parents=True, exist_ok=True) | 126 | artifact.parent.mkdir(parents=True, exist_ok=True) |
| 117 | with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: | 127 | with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: |
| 118 | pending = Path(temporary) / artifact_name | 128 | pending = Path(temporary) / artifact_name |
| ... | @@ -121,7 +131,7 @@ class Mirror: | ... | @@ -121,7 +131,7 @@ class Mirror: |
| 121 | fetch(url, pending, MAX_ARTIFACT) | 131 | fetch(url, pending, MAX_ARTIFACT) |
| 122 | if checksum and digest(pending) != checksum: | 132 | if checksum and digest(pending) != checksum: |
| 123 | raise Unavailable('artifact SHA-256 mismatch') | 133 | raise Unavailable('artifact SHA-256 mismatch') |
| 124 | if not checksum or tuple(map(int, filename(artifact_name).split('-')[0].split('.'))) >= (0, 6, 0): | 134 | if signed: |
| 125 | fetch(url + '.minisig', signature, 4096) | 135 | fetch(url + '.minisig', signature, 4096) |
| 126 | result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], | 136 | result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], |
| 127 | capture_output=True) | 137 | capture_output=True) |
tools/router-zig-test.py+5-1| ... | @@ -83,7 +83,11 @@ def main(): | ... | @@ -83,7 +83,11 @@ def main(): |
| 83 | assert path == archive + '?download=1' | 83 | assert path == archive + '?download=1' |
| 84 | assert 'Cookie' not in headers and 'Authorization' not in headers | 84 | assert 'Cookie' not in headers and 'Authorization' not in headers |
| 85 | assert request(host, archive, method='HEAD') == (200, b'') | 85 | assert request(host, archive, method='HEAD') == (200, b'') |
| 86 | for path in ['/mirrors/zig/', '/mirrors/zig/releases/', '/mirrors/zig/dev/', '/mirrors/zig/releases/zig-0.0.0.tar.xz']: | 86 | for name in ['zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip', 'zig-future-0.99.0.zip']: |
| 87 | assert request(host, '/mirrors/zig/' + name) == (200, b'hello') | ||
| 88 | assert request(host, '/mirrors/zig/' + name, method='HEAD') == (200, b'') | ||
| 89 | for path in ['/mirrors/zig/', '/mirrors/zig/releases/', '/mirrors/zig/dev/', '/mirrors/zig/releases/zig-0.0.0.tar.xz', | ||
| 90 | '/mirrors/zig/preadme.md', '/mirrors/zig/not-zig.zip']: | ||
| 87 | assert request(host, path) == (204, b'') | 91 | assert request(host, path) == (204, b'') |
| 88 | assert request(host, archive, method='POST') == (204, b'') | 92 | assert request(host, archive, method='POST') == (204, b'') |
| 89 | count = len(Mirror.requests) | 93 | count = len(Mirror.requests) |
tools/router.py+1-1| ... | @@ -118,7 +118,7 @@ def zig_mirror_routes(service, routes): | ... | @@ -118,7 +118,7 @@ def zig_mirror_routes(service, routes): |
| 118 | except FileNotFoundError: | 118 | except FileNotFoundError: |
| 119 | mirror = None | 119 | mirror = None |
| 120 | upstreams = set().union(*(route["upstreams"] for route in routes.values() if mirror in route["services"])) | 120 | upstreams = set().union(*(route["upstreams"] for route in routes.values() if mirror in route["services"])) |
| 121 | pattern = r"^/mirrors/zig/zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-([0-9]+\.[0-9]+\.[0-9]+(?:-dev\.[0-9]+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$" | 121 | pattern = r"^/mirrors/zig/zig[^/]*\.(?:tar\.xz|zip)(?:\.minisig)?$" |
| 122 | lines = [" @zig_mirror {", " method GET HEAD", f" path_regexp zig_mirror {pattern}", " }", | 122 | lines = [" @zig_mirror {", " method GET HEAD", f" path_regexp zig_mirror {pattern}", " }", |
| 123 | " handle @zig_mirror {"] | 123 | " handle @zig_mirror {"] |
| 124 | if upstreams: | 124 | if upstreams: |
tools/zig-mirror-test.py+35-1| ... | @@ -30,12 +30,14 @@ class MirrorTests(unittest.TestCase): | ... | @@ -30,12 +30,14 @@ class MirrorTests(unittest.TestCase): |
| 30 | 30 | ||
| 31 | def test_spec_filenames_and_existing_layout(self): | 31 | def test_spec_filenames_and_existing_layout(self): |
| 32 | cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), | 32 | cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), |
| 33 | ('zig-win64-0.1.1.zip', '0.1.1/'), ('zig-win64-0.2.0.zip', '0.2.0/'), | ||
| 33 | ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'), | 34 | ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'), |
| 34 | ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')] | 35 | ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')] |
| 35 | for name, prefix in cases: | 36 | for name, prefix in cases: |
| 36 | self.assertEqual(str(mirror.relative(name)), prefix + name) | 37 | self.assertEqual(str(mirror.relative(name)), prefix + name) |
| 37 | self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name)) | 38 | self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name)) |
| 38 | for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz']: | 39 | for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz', |
| 40 | 'zig-win64-0.3.0.zip', 'zig-win64-0.1.1.tar.xz', 'zig-win64-0.1.1.zip.minisig', 'zig-linux-0.1.1.zip']: | ||
| 39 | with self.assertRaises(ValueError): | 41 | with self.assertRaises(ValueError): |
| 40 | mirror.filename(name) | 42 | mirror.filename(name) |
| 41 | 43 | ||
| ... | @@ -130,6 +132,38 @@ class MirrorTests(unittest.TestCase): | ... | @@ -130,6 +132,38 @@ class MirrorTests(unittest.TestCase): |
| 130 | urllib.request.urlopen(base + '/zig-0.17.0.tar.xz') | 132 | urllib.request.urlopen(base + '/zig-0.17.0.tar.xz') |
| 131 | self.assertEqual(error.exception.code, 504) | 133 | self.assertEqual(error.exception.code, 504) |
| 132 | 134 | ||
| 135 | def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self): | ||
| 136 | names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz'] | ||
| 137 | body = b'official historical archive' | ||
| 138 | entries = {mirror.filename(name): {'platform': {'tarball': mirror.upstream(name), | ||
| 139 | 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}} | ||
| 140 | for i, name in enumerate(names)} | ||
| 141 | def fetch(url, target, limit): | ||
| 142 | self.assertNotIn('.minisig', url) | ||
| 143 | self.assertEqual(url, mirror.upstream(target.name)) | ||
| 144 | target.write_bytes(body) | ||
| 145 | server = mirror.http.server.ThreadingHTTPServer(('127.0.0.1', 0), mirror.Handler) | ||
| 146 | server.mirror = self.cache | ||
| 147 | threading.Thread(target=server.serve_forever, daemon=True).start() | ||
| 148 | self.addCleanup(server.server_close) | ||
| 149 | self.addCleanup(server.shutdown) | ||
| 150 | base = 'http://127.0.0.1:' + str(server.server_port) + '/mirrors/zig/' | ||
| 151 | with patch.object(mirror, 'index', return_value=entries) as index, patch.object(mirror, 'fetch', side_effect=fetch) as download, patch.object(mirror.subprocess, 'run') as signature: | ||
| 152 | for name in names[:2]: | ||
| 153 | with urllib.request.urlopen(base + name) as response: | ||
| 154 | self.assertEqual(response.read(), body) | ||
| 155 | for name in [names[2], 'zig-linux-x86_64-0.3.0.tar.xz']: | ||
| 156 | with self.assertRaises(urllib.error.HTTPError) as error: | ||
| 157 | urllib.request.urlopen(base + name) | ||
| 158 | self.assertEqual(error.exception.code, 504) | ||
| 159 | self.assertFalse(self.cache.path(name).exists()) | ||
| 160 | self.assertEqual(download.call_count, 3) | ||
| 161 | signature.assert_not_called() | ||
| 162 | index.side_effect = mirror.Unavailable('offline') | ||
| 163 | with urllib.request.urlopen(base + names[0]) as response: | ||
| 164 | self.assertEqual(response.read(), body) | ||
| 165 | self.assertFalse(list(self.cache.root.rglob('.download-*'))) | ||
| 166 | |||
| 133 | 167 | ||
| 134 | if __name__ == '__main__': | 168 | if __name__ == '__main__': |
| 135 | unittest.main() | 169 | unittest.main() |