1#!/usr/bin/env python3
2import argparse
3from contextlib import ExitStack
4from html.parser import HTMLParser
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import os
8from pathlib import Path
9import socket
10import sqlite3
11import subprocess
12import tempfile
13import threading
14import time
15import urllib.error
16import urllib.request
17
18import router
19
20
21class Links(HTMLParser):
22 def __init__(self, html):
23 super().__init__()
24 self.links = []
25 self.feed(html)
26
27 def handle_starttag(self, tag, attrs):
28 if tag == 'a':
29 self.links.append(dict(attrs))
30
31
32def port():
33 with socket.socket() as sock:
34 sock.bind(('127.0.0.1', 0))
35 return sock.getsockname()[1]
36
37
38def main():
39 parser = argparse.ArgumentParser()
40 parser.add_argument('--caddy', required=True, type=Path)
41 parser.add_argument('--binary', type=Path, default=Path(__file__).resolve().parent.parent / 'dashboard/target/debug/home-dashboard')
42 parser.add_argument('--page', type=Path)
43 parser.add_argument('--browser-ready-file', type=Path)
44 args = parser.parse_args()
45 repo = Path(__file__).resolve().parent.parent
46 observations = []
47 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'
48 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'
49 blob = bytes(range(256)) * 65536
50
51 class Fixture(BaseHTTPRequestHandler):
52 protocol_version = 'HTTP/1.1'
53
54 def do_GET(self):
55 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
56 if self.path == '/binary':
57 return self.respond(blob, 'application/octet-stream')
58 if self.path == '/large':
59 return self.respond(b' ' * (9 * 1024 * 1024), 'text/html')
60 if self.path == '/-/login':
61 callback = 'https://' + self.headers['Host'] + '/-/callback'
62 return self.respond(b'', 'text/plain', 302, [('Location', 'https://snowglobe.studio.test/auth/oidc/authorize?redirect_uri=' + urllib.parse.quote(callback, safe=''))])
63 if self.path == '/redirect':
64 return self.respond(b'', 'text/plain', 302, [('Location', '/snowbound/issues/26'), ('Set-Cookie', 'SessionID=new; Path=/; HttpOnly'), ('Set-Cookie', 'other=kept; Path=/')])
65 if self.path.startswith('/-/') and args.page:
66 with urllib.request.urlopen('https://shale.paperclover.net' + self.path, timeout=15) as response:
67 return self.respond(response.read(), response.headers['Content-Type'])
68 return self.respond(issue + sample, 'text/html; charset=utf-8', headers=[('ETag', '"stale"'), ('Last-Modified', 'Mon, 05 Oct 2026 00:00:00 GMT')])
69
70 def do_HEAD(self):
71 self.respond(b'', 'text/html')
72
73 def do_POST(self):
74 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
75 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
76 self.respond(body, 'application/octet-stream', 201)
77
78 def respond(self, body, content_type, status=200, headers=()):
79 self.send_response(status)
80 self.send_header('Content-Type', content_type)
81 self.send_header('Content-Length', str(len(body)))
82 for key, value in headers:
83 self.send_header(key, value)
84 self.end_headers()
85 self.wfile.write(body)
86
87 def log_message(self, *args):
88 pass
89
90 class NoRedirect(urllib.request.HTTPRedirectHandler):
91 def redirect_request(self, *args):
92 return None
93
94 opener = urllib.request.build_opener(NoRedirect)
95 with tempfile.TemporaryDirectory(prefix='shale-page-') as temporary, ExitStack() as stack:
96 root = Path(temporary).resolve()
97 fixture = ThreadingHTTPServer(('127.0.0.1', 0), Fixture)
98 stack.callback(fixture.server_close)
99 stack.callback(fixture.shutdown)
100 threading.Thread(target=fixture.serve_forever, daemon=True).start()
101 dashboard_port, gateway_port, internal_port = port(), port(), port()
102 proof = 'a' * 64
103 token = root / 'proxy.token'
104 token.write_text(proof)
105 certificate, gateway_key = root / 'gateway.pem', root / 'gateway.key'
106 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
107 data = root / 'data'
108 environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_INTERNAL_URL': f'https://localhost:{internal_port}', 'STUDIO_CA_BUNDLE': str(certificate), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')}
109 for key in ['STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']:
110 environment.pop(key, None)
111 log = stack.enter_context((root / 'dashboard.log').open('w'))
112 dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log)
113 stack.callback(lambda: dashboard.wait(timeout=10))
114 stack.callback(dashboard.terminate)
115 deadline = time.monotonic() + 15
116 while True:
117 assert dashboard.poll() is None, (root / 'dashboard.log').read_text()
118 try:
119 with socket.create_connection(('127.0.0.1', dashboard_port), timeout=.1):
120 break
121 except OSError:
122 assert time.monotonic() < deadline
123 time.sleep(.05)
124 with sqlite3.connect(data / 'accounts.sqlite') as db:
125 for provider, subject, suffix, name in [('github', '24465214', '24465214', 'paperclover'), ('astheno', 'pairwise-astheno-subject', 'abc', 'Astheno user'), ('github', '777', '777', None), ('astheno', 'unmapped-pairwise-subject', 'unknown', 'Unmapped guest')]:
126 username = f'guest-{provider}-{suffix}'
127 profile = {'kind': 'guest', 'enabled': True, 'username': username, 'guestProvider': provider, 'firstName': name}
128 db.execute('INSERT INTO users(id,profile) VALUES (?,?)', [username, json.dumps(profile)])
129 db.execute('INSERT INTO external_identities VALUES (?,?,?)', [provider, subject, username])
130 os.environ.update(STUDIO_DOMAIN='studio.test', STUDIO_DASHBOARD_PORT=str(dashboard_port), STUDIO_PROXY_TOKEN_FILE=str(token), STUDIO_INTERNAL_PORT=str(internal_port))
131 router.ROUTE_DIR = str(root / 'routes')
132 Path(router.ROUTE_DIR).mkdir()
133 (Path(router.ROUTE_DIR) / 'shale.json').write_text(json.dumps({'headHtml': {'shale.studio.test': {'/snowbound/*': '<meta name="rewrite-test" content="kept">'}}}))
134
135 def nomad(path, token):
136 if path == '/v1/services':
137 return [{'Namespace': 'default', 'Services': [{'ServiceName': 'shale'}, {'ServiceName': 'shale-preview-12345678'}]}]
138 if path.startswith('/v1/service/'):
139 service = path.rsplit('/', 1)[1]
140 host = 'shale.studio.test' if service == 'shale' else service + '.studio.test'
141 return [{'Address': '127.0.0.1', 'Port': fixture.server_port, 'ServiceName': service, 'AllocID': service, 'JobID': service, 'Tags': ['caddy-host=' + host]}]
142 return {'ready': {'Status': 'success'}}
143
144 router.nomad = nomad
145 rendered = router.render('fixture')
146 full_config = root / 'routes.caddy'
147 full_config.write_text('{\n admin off\n auto_https off\n}\n' + rendered)
148 caddy_env = {**os.environ, 'XDG_DATA_HOME': str(root / 'caddy-data'), 'XDG_CONFIG_HOME': str(root / 'caddy-config')}
149 subprocess.run([str(args.caddy), 'validate', '--config', str(full_config), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
150 def local_site(host, listener):
151 start = rendered.index(host + ' {')
152 end = rendered.index('\n}', start) + 2
153 return rendered[start:end].replace(host + ' {', f'http://127.0.0.1:{listener} {{', 1).replace(' tls internal\n', '').replace(' route {\n', f' route {{\n request_header Host {host}\n', 1)
154
155 preview_port = port()
156 config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.studio.test', gateway_port) + '\n' + local_site('shale-preview-12345678.studio.test', preview_port)
157 internal_host = f'dashboard.internal.studio.test:{internal_port}'
158 start = rendered.index(internal_host + ' {')
159 config += '\n' + rendered[start:].replace(internal_host + ' {', f'https://localhost:{internal_port} {{', 1).replace(' tls internal\n', f' tls {certificate} {gateway_key}\n', 1)
160 config_path = root / 'Caddyfile'
161 config_path.write_text(config)
162 subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
163 caddy_log = stack.enter_context((root / 'caddy.log').open('w'))
164 caddy = subprocess.Popen([str(args.caddy), 'run', '--config', str(config_path), '--adapter', 'caddyfile'], stdout=caddy_log, stderr=caddy_log, env=caddy_env)
165 stack.callback(lambda: caddy.wait(timeout=10))
166 stack.callback(caddy.terminate)
167 origin = f'http://127.0.0.1:{gateway_port}'
168
169 def request(path, *, headers=None, body=None, direct=False):
170 supplied = {'Cookie': 'SessionID=preserved', 'Origin': 'https://shale.studio.test', **(headers or {})}
171 url = (f'http://127.0.0.1:{dashboard_port}' if direct else origin) + path
172 try:
173 response = opener.open(urllib.request.Request(url, headers=supplied, data=body), timeout=15)
174 except urllib.error.HTTPError as error:
175 response = error
176 with response:
177 return response.status, response.headers, response.read()
178
179 deadline = time.monotonic() + 15
180 while True:
181 try:
182 status, headers, body = request('/snowbound/issues/26?query=kept')
183 break
184 except urllib.error.URLError:
185 assert time.monotonic() < deadline and caddy.poll() is None
186 time.sleep(.05)
187 assert status == 200, (status, body, (root / 'dashboard.log').read_text(), (root / 'caddy.log').read_text())
188 links = Links(body.decode()).links
189 link = next(link for link in links if link.get('href') == 'https://github.com/paperclover')
190 assert link['target'] == '_blank' and link['rel'] == 'noreferrer', link
191 assert any(link.get('href') == '/~guest-github-unknown' for link in links)
192 assert any(link.get('href') == '/~guest-github-24465214/repo' for link in links)
193 assert b'Astheno user' in body and not any(link.get('href') == '/~guest-astheno-abc' for link in links)
194 assert b'identity.astheno.software/user/' not in body and b'pairwise-astheno-subject' not in body
195 assert b'https://avatars.githubusercontent.com/u/24465214?s=64' in body
196 assert b'rewrite-test' in body
197 assert headers['Cache-Control'] == 'no-store' and headers.get('ETag') is None and headers.get('Last-Modified') is None
198 assert observations[-1][1] == '/snowbound/issues/26?query=kept', observations[-1]
199 assert observations[-1][2]['host'] == 'shale.studio.test', observations[-1]
200 assert observations[-1][2]['cookie'] == 'SessionID=preserved', observations[-1]
201 assert not any(key.lower().startswith('studio-') for key in observations[-1][2]), observations[-1]
202 status, headers, body = request('/~guest-github-24465214')
203 assert status == 302 and headers['Location'] == 'https://github.com/paperclover' and headers['Referrer-Policy'] == 'no-referrer'
204 assert request('/~guest-astheno-abc/')[0] == 200
205 assert observations[-1][1] == '/~guest-astheno-abc/'
206 assert request('/~guest-astheno-unknown')[0] == 200
207 status, headers, body = request('/-/login')
208 assert status == 302 and urllib.parse.parse_qs(urllib.parse.urlparse(headers['Location']).query)['redirect_uri'] == ['https://shale.studio.test/-/callback']
209 status, headers, body = request('/redirect')
210 assert status == 302 and headers['Location'] == '/snowbound/issues/26' and len(headers.get_all('Set-Cookie')) == 2
211 with opener.open(f'http://127.0.0.1:{preview_port}/snowbound/issues/26') as response:
212 assert response.status == 200
213 assert observations[-1][2]['host'] == 'shale-preview-12345678.studio.test'
214 assert any(link.get('href') == 'https://github.com/paperclover' for link in Links(response.read().decode()).links)
215 assert b'/~guest-github-24465214' in request('/repo/info/refs')[2]
216 assert request('/binary')[2] == blob
217 assert len(request('/large')[2]) == 9 * 1024 * 1024
218 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')
219 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'
220 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403
221 assert request('/auth/shale/page', direct=True)[0] == 403
222 assert request('/auth/shale/page', direct=True, headers={'Studio-Proxy-Token': proof, 'Studio-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Studio-Shale-Uri': '/', 'Host': 'snowglobe.studio.test'})[0] == 403
223 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
224 if args.browser_ready_file:
225 args.browser_ready_file.write_text(json.dumps({'origin': origin}))
226 stop = args.browser_ready_file.with_suffix('.stop')
227 deadline = time.monotonic() + 600
228 while not stop.exists() and time.monotonic() < deadline:
229 time.sleep(.2)
230 args.browser_ready_file.unlink(missing_ok=True)
231 stop.unlink(missing_ok=True)
232
233
234if __name__ == '__main__':
235 main()