1#!/usr/bin/env python3
2import json
3import grp
4import os
5import re
6import subprocess
7import sys
8import time
9import urllib.parse
10import urllib.request
11
12ROUTES = "/var/lib/caddy/routes.caddy"
13TOKEN = "/var/lib/studio/router.token"
14ROUTE_DIR = "/var/lib/studio/routes"
15HOST = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?\Z")
16
17
18def nomad(path, token):
19 request = urllib.request.Request(
20 "http://127.0.0.1:4646" + path,
21 headers={"X-Nomad-Token": token},
22 )
23 with urllib.request.urlopen(request, timeout=5) as response:
24 return json.load(response)
25
26
27def proxy(upstreams, indent, uncompressed=False, host=None):
28 lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s",
29 f"{indent} fail_duration 30s"]
30 if uncompressed:
31 lines.append(f"{indent} header_up Accept-Encoding identity")
32 if host:
33 lines.append(f"{indent} header_up Host {host}")
34 return [*lines, f"{indent}}}"]
35
36
37def shale_mcp_routes(port):
38 return [" @shale_mcp_link path_regexp shale_mcp_link ^/-/studio-mcp/([A-Za-z0-9_-]{43})$",
39 " handle @shale_mcp_link {", " rewrite * /oauth/shale/link/{re.shale_mcp_link.1}",
40 " request_header -User-Name", " request_header -User-Groups", " request_header -Studio-Proxy-Token",
41 *proxy(f"127.0.0.1:{port}", " "), " }",
42 " @shale_mcp_callback {", " path /-/callback", " header Cookie *studio_mcp_shale_link=*", " }",
43 " handle @shale_mcp_callback {", " rewrite * /oauth/shale/callback",
44 " request_header -User-Name", " request_header -User-Groups", " request_header -Studio-Proxy-Token",
45 *proxy(f"127.0.0.1:{port}", " "), " }"]
46
47
48def shale_attachment_routes(upstreams, target="/srv/prod/shale"):
49 """Authorize retained attachments through their issue before serving bytes."""
50 manifest = os.path.join(target, "forgejo-attachments.json")
51 if not os.path.exists(manifest):
52 return []
53 with open(manifest) as source:
54 entries = json.load(source)
55 if not isinstance(entries, list) or len(entries) > 10000:
56 raise ValueError("invalid Shale attachment manifest")
57 lines = []
58 for index, entry in enumerate(entries):
59 path, filename, issue = (entry[k] for k in ("path", "file", "issue"))
60 if (not re.fullmatch(r"[0-9a-f-]{36}/[^/\\\"\r\n]+", filename)
61 or path != "/-/forgejo-attachments/" + urllib.parse.quote(filename, safe="/")
62 or not re.fullmatch(r"/(?:[a-zA-Z0-9_.-]+/)+issues/[1-9][0-9]*", issue)):
63 raise ValueError("unsafe Shale attachment route")
64 # Request cookies go to Shale as usual. A private issue yields 403/404;
65 # only Shale's successful issue response permits the local file read.
66 name = f"@shale_attachment_{index}"
67 lines += [f" {name} path {json.dumps(path)}", f" handle {name} {{",
68 " header Cache-Control private,no-store", " header X-Content-Type-Options nosniff",
69 f" reverse_proxy {upstreams} {{", " method GET", f" rewrite {issue}",
70 " @authorized status 200", " handle_response @authorized {",
71 f" root * {json.dumps(os.path.join(target, 'forgejo-attachments'))}",
72 f" rewrite * {json.dumps('/' + urllib.parse.quote(filename, safe='/'))}",
73 " file_server", " }", " }", " }"]
74 return lines
75
76
77def shale_write_routes(host):
78 # Older Shale releases predate form tokens. Browsers send Origin on POST;
79 # require the exact site origin for every cookie-authenticated mutation.
80 # HTTP Git clients use Basic authentication and do not carry this cookie.
81 return [" @shale_unsafe_origin {", " method POST PUT PATCH DELETE",
82 " header Cookie *SessionID=*", f" not header Origin https://{host}",
83 " }", ' respond @shale_unsafe_origin "Forbidden" 403']
84
85
86def shale_git_routes(upstreams):
87 lines = [" @shale_git path */info/refs */git-upload-pack */git-receive-pack",
88 " handle @shale_git {", f" reverse_proxy {upstreams} {{"]
89 # Shale forwards CGI Status as a header instead of the HTTP status.
90 for status in (400, 403, 404, 405, 415, 500):
91 lines += [f' @cgi_{status} header Status "{status} *"',
92 f" handle_response @cgi_{status} {{",
93 " header {", " -Status", " defer", " }",
94 f" copy_response {status}", " }"]
95 return [*lines, " }", " }"]
96
97
98def render(token):
99 with open(os.environ["STUDIO_PROXY_TOKEN_FILE"]) as file:
100 dashboard_proof = file.read().strip()
101 if not re.fullmatch(r"[0-9a-fA-F]{64}", dashboard_proof):
102 raise ValueError("invalid dashboard proxy token")
103 auth_host = "auth." + os.environ["STUDIO_DOMAIN"]
104 legacy_host = "keycloak." + os.environ["STUDIO_DOMAIN"]
105 origins = json.dumps({"origins": ["https://snowglobe." + os.environ["STUDIO_DOMAIN"]]}, separators=(",", ":"))
106 webauthn = [" handle /.well-known/webauthn {", ' header Content-Type application/json',
107 f" respond {json.dumps(origins)} 200", " }"]
108 routes = {}
109 internal_services = {}
110 auth_upstreams = set()
111 allocation_checks = {}
112 for namespace in nomad("/v1/services", token):
113 if namespace["Namespace"] != "default":
114 continue
115 for service in namespace["Services"]:
116 name = urllib.parse.quote(service["ServiceName"], safe="")
117 for instance in nomad(f"/v1/service/{name}", token):
118 tags = instance.get("Tags") or []
119 if instance["ServiceName"] != "forward-auth" and not any(tag.startswith("caddy-") for tag in tags):
120 continue
121 address = instance["Address"]
122 port = instance["Port"]
123 if not re.fullmatch(r"[0-9a-fA-F:.]+", address) or type(port) is not int or not 1 <= port <= 65535:
124 continue
125 upstream = f"[{address}]:{port}" if ":" in address else f"{address}:{port}"
126 if not re.fullmatch(r"[a-z][a-z0-9-]*", instance["ServiceName"]):
127 raise ValueError("invalid internal service name")
128 internal_services.setdefault(instance["ServiceName"], set()).add(upstream)
129 alloc = instance["AllocID"]
130 if alloc not in allocation_checks:
131 allocation_checks[alloc] = nomad(f"/v1/allocation/{alloc}/checks", token)
132 service_checks = list(allocation_checks[alloc].values())
133 if not service_checks or any(check["Status"] != "success" for check in service_checks):
134 continue
135 if instance["ServiceName"] == "forward-auth":
136 auth_upstreams.add(upstream)
137 access = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-auth-role=")]
138 user_headers = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-user-header=")]
139 if len(access) > 1 or (access and not re.fullmatch(r"[a-z][a-z0-9-]*", access[0])):
140 raise ValueError("invalid route access role")
141 if len(user_headers) > 1 or (user_headers and (not access or not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", user_headers[0]))):
142 raise ValueError("invalid authenticated user header")
143 if access and not any(tag.startswith("caddy-host=") for tag in tags):
144 raise ValueError("route access role has no host")
145 for tag in tags:
146 if tag.startswith("caddy-host="):
147 host = tag.split("=", 1)[1]
148 if not HOST.fullmatch(host):
149 raise ValueError(f"invalid Caddy host: {host!r}")
150 route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False})
151 if access and route["authRole"] not in (None, access[0]):
152 raise ValueError("conflicting route access roles")
153 if user_headers and route["userHeader"] not in (None, user_headers[0]):
154 raise ValueError("conflicting authenticated user headers")
155 if access:
156 route["authRole"] = access[0]
157 if user_headers:
158 route["userHeader"] = user_headers[0]
159 if "caddy-internal=true" in tags:
160 route["internal"] = True
161 if "caddy-real-ip=true" in tags:
162 route["realIp"] = True
163 route["upstreams"].add(upstream)
164 route["services"].add(instance.get("JobID") or instance["ServiceName"])
165 elif tag.startswith("caddy-port="):
166 listener = int(tag.split("=", 1)[1])
167 if not 1 <= listener <= 65535:
168 raise ValueError(f"invalid Caddy port: {listener}")
169 route = routes.setdefault((listener, ""), {"upstreams": set(), "services": set()})
170 route["upstreams"].add(upstream)
171
172 lines = []
173 for (listener, host), route in sorted(routes.items()):
174 if host:
175 if host in {auth_host, legacy_host}:
176 continue
177 role = route["authRole"]
178 if role and not auth_upstreams:
179 continue
180 lines.append(f"{host if listener == 80 else f'{host}:{listener}'} {{")
181 if host.endswith(".test") or route["internal"]:
182 lines.append(" tls internal")
183 if route["realIp"]:
184 lines.append(" request_header X-Real-Ip {remote_host}")
185 if len(route["services"]) != 1:
186 raise ValueError(f"multiple services claim {host}")
187 service = next(iter(route["services"]))
188 if not re.fullmatch(r"[a-z][a-z0-9-]*", service):
189 raise ValueError(f"invalid service name: {service}")
190 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
191 # Keep the Origin guard before every static/proxy handle;
192 # otherwise Caddy sorts those handles ahead of respond.
193 lines += [" route {", *shale_write_routes(host),
194 *shale_git_routes(" ".join(sorted(route["upstreams"])))]
195 if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
196 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])),
197 "/srv/staging/" + service)
198 if service == "shale":
199 lines += [" handle /-/logout {", f" reverse_proxy {' '.join(sorted(route['upstreams']))} {{",
200 f" header_down Location https://snowglobe.{os.environ['STUDIO_DOMAIN']}/auth/shale/sign-out",
201 " @logged_out status 200 302 303 404", " handle_response @logged_out {",
202 " copy_response 303", " }", " }", " }"]
203 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])))
204 lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$",
205 " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"]
206 port = int(os.environ["STUDIO_DASHBOARD_PORT"])
207 if not 1 <= port <= 65535:
208 raise ValueError("invalid dashboard port for Shale linking")
209 lines += shale_mcp_routes(port)
210 lines += [" tracing {", f" span {service}", " span_attributes {",
211 f" studio.service {service}", " studio.kind edge", " }", " }"]
212 manifest = os.path.join(ROUTE_DIR, service + ".json")
213 assets = {}
214 headers = {}
215 head_html = {}
216 proof = None
217 scrub = []
218 if os.path.exists(manifest):
219 with open(manifest) as file:
220 assets = json.load(file)
221 identity = assets.get("identity", {})
222 headers = identity.get(host, {})
223 head_html = assets.get("headHtml", {}).get(host, {})
224 proof = assets.get("identityProof", {}).get(host)
225 scrub = sorted({name for configured in identity.values() for name in configured} | set(assets.get("identityProof", {}).values()))
226 if any(not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", name) for name in scrub):
227 raise ValueError(f"invalid identity header: {host}")
228 if proof and not headers:
229 raise ValueError(f"identity proof has no identity headers: {host}")
230 if any(source not in {"X-Auth-Request-User", "X-Auth-Request-Groups", "X-Auth-Request-Preferred-Username"} for source in headers.values()):
231 raise ValueError(f"invalid identity claim: {host}")
232 if role and headers:
233 raise ValueError(f"route has both required and optional auth: {host}")
234 if role and (assets.get("files") or assets.get("dirs")):
235 raise ValueError(f"authenticated route has static overrides: {host}")
236 if head_html and (role or headers):
237 raise ValueError(f"authenticated route has HTML injection: {host}")
238 for request, markup in head_html.items():
239 if not re.fullmatch(r"/[A-Za-z0-9/._*-]*", request) or not markup:
240 raise ValueError(f"invalid HTML injection: {host} {request}")
241 for request, path in assets.get("files", {}).items():
242 if not request.startswith("/") or " " in request or "\n" in request:
243 raise ValueError(f"invalid static path: {request}")
244 lines += [f" handle {request} {{", f" root * {json.dumps(os.path.dirname(path))}",
245 f" rewrite * /{os.path.basename(path)}", ' header Cache-Control "no-store"',
246 " file_server", " }"]
247 for request, path in assets.get("dirs", {}).items():
248 if not request.startswith("/") or " " in request or "\n" in request:
249 raise ValueError(f"invalid static prefix: {request}")
250 lines += [f" handle_path {request}* {{", f" root * {json.dumps(path)}",
251 " file_server", " }"]
252 metrics_path = assets.get("metricsPaths", {}).get(host)
253 if metrics_path:
254 if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path):
255 raise ValueError(f"invalid metrics path: {host}")
256 lines += [f" handle {metrics_path} {{", " respond 404", " }"]
257 upstreams = " ".join(sorted(route["upstreams"]))
258 if role:
259 auth = " ".join(sorted(auth_upstreams))
260 lines += [" handle /snow.oauth2/* {", *proxy(auth, " "), " }", " handle {"]
261 if route["userHeader"]:
262 lines.append(f" request_header -{route['userHeader']}")
263 lines += [
264 f" reverse_proxy {auth} {{", " lb_try_duration 5s", " fail_duration 30s", " method GET",
265 " rewrite /snow.oauth2/auth", " header_up X-Forwarded-Method {method}",
266 " header_up X-Forwarded-Uri {uri}", " @unauthorized status 401",
267 " handle_response @unauthorized {",
268 " redir * /snow.oauth2/sign_in?rd={scheme}://{host}{uri}", " }",
269 f" @allowed header X-Auth-Request-Groups *role:{role}*",
270 " handle_response @allowed {", " method {method}", " rewrite {uri}",
271 ]
272 if route["userHeader"]:
273 lines.append(f" request_header {route['userHeader']} {{rp.header.X-Auth-Request-Preferred-Username}}")
274 lines += [
275 *proxy(upstreams, " "), " }",
276 " handle_response {", " respond 403", " }", " }", " }", "}",
277 ]
278 elif headers and (auth_upstreams or service == "copyparty"):
279 native = service == "copyparty"
280 auth = f"127.0.0.1:{int(os.environ['STUDIO_DASHBOARD_PORT'])}" if native else " ".join(sorted(auth_upstreams))
281 if native:
282 lines += [" handle /auth/file/* {", " request_header -User-Name", " request_header -User-Groups",
283 f" request_header Studio-Proxy-Token {dashboard_proof}",
284 " request_header X-Studio-Client-IP {remote_host}", *proxy(auth," "), " }"]
285 else:
286 lines += [" handle /snow.oauth2/* {", *proxy(auth," "), " }"]
287 lines += [" handle {"]
288 lines += [f" request_header -{name}" for name in scrub]
289 lines += [f" reverse_proxy {auth} {{", " lb_try_duration 5s", " fail_duration 30s", " method GET", " rewrite " + ("/auth/file/check" if native else "/snow.oauth2/auth"),
290 *([f" header_up Studio-Proxy-Token {dashboard_proof}"] if native else []),
291 " header_up X-Forwarded-Method {method}", " header_up X-Forwarded-Uri {uri}",
292 " @authenticated status 2xx", " handle_response @authenticated {"]
293 lines += [f" request_header {name} {{rp.header.{source}}}" for name, source in headers.items()]
294 if proof:
295 lines.append(f" request_header {proof} 1")
296 lines += [" }", " @anonymous status 4xx", " handle_response @anonymous {",
297 f" request_header -{scrub[0]}", " }", " }",
298 *proxy(upstreams, " "), " }", "}"]
299 else:
300 page_upstream = upstreams
301 shale_page = service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service)
302 page_headers = []
303 if shale_page:
304 page_upstream = f"127.0.0.1:{int(os.environ['STUDIO_DASHBOARD_PORT'])}"
305 page_headers = [f" request_header Studio-Proxy-Token {dashboard_proof}",
306 f" request_header Studio-Shale-Upstream {json.dumps(sorted(route['upstreams'])[0])}",
307 " request_header Studio-Shale-Uri {uri}",
308 " rewrite * /auth/shale/page"]
309 for index, (request, markup) in enumerate(head_html.items()):
310 name = f"@studio_head_{index}"
311 lines += [f" {name} path {request}", f" handle {name} {{", " route {",
312 f" replace </head> {json.dumps(markup + '</head>')} {{",
313 " match {", " header Content-Type text/html*", " }", " }",
314 *[" " + line for line in page_headers],
315 *proxy(page_upstream, " ", uncompressed=True), " }", " }"]
316 page_handler = [*page_headers, *proxy(page_upstream, " ")]
317 if shale_page:
318 page_handler = [" route {", *[" " + line for line in page_handler], " }"]
319 lines += [" handle {", *(f" request_header -{name}" for name in scrub),
320 *page_handler, " }", "}"]
321 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
322 lines.insert(len(lines) - 1, " }")
323 else:
324 lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"]
325 dashboard_host = "snowglobe." + os.environ["STUDIO_DOMAIN"]
326 if any(not HOST.fullmatch(host) for host in [auth_host, legacy_host, dashboard_host]):
327 raise ValueError("invalid login domain")
328 for host in [auth_host, legacy_host]:
329 lines += [f"{host} {{", *([" tls internal"] if host.endswith(".test") else []),
330 *(webauthn if host == auth_host else []), " handle {",
331 f" redir https://{dashboard_host}/ 308", " }", "}"]
332 traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None)
333 if traces:
334 lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1",
335 *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"]
336 dashboard_port = int(os.environ["STUDIO_DASHBOARD_PORT"])
337 if not 1 <= dashboard_port <= 65535:
338 raise ValueError("invalid dashboard route")
339 if (80, dashboard_host) in routes:
340 raise ValueError(f"dashboard route conflicts with a Nomad service: {dashboard_host}")
341 lines.append(f"{dashboard_host} {{")
342 if dashboard_host.endswith(".test"):
343 lines.append(" tls internal")
344 lines += [" encode zstd gzip", " tracing {", " span globe", " span_attributes {", " studio.kind edge", " }", " }"]
345 lines += [" handle {", " request_header -User-Name", " request_header -User-Groups",
346 f" request_header Studio-Proxy-Token {dashboard_proof}",
347 " request_header X-Studio-Client-IP {remote_host}",
348 *proxy(f"127.0.0.1:{dashboard_port}", " "), " }", "}"]
349 internal_port = os.environ.get("STUDIO_INTERNAL_PORT")
350 if internal_port is not None:
351 internal_host = "dashboard.internal." + os.environ["STUDIO_DOMAIN"]
352 if not HOST.fullmatch(internal_host) or not internal_port.isdecimal() or not 1 <= int(internal_port) <= 65535:
353 raise ValueError("invalid internal dashboard route")
354 if (80, internal_host) in routes or (int(internal_port), "") in routes:
355 raise ValueError("internal dashboard route conflicts with a Nomad service")
356 lines += [f"{internal_host}:{internal_port} {{", " tls internal",
357 f" @dashboard header Studio-Proxy-Token {dashboard_proof}",
358 " handle @dashboard {", " request_header -Studio-Proxy-Token",
359 " request_header -User-Name", " request_header -User-Groups",
360 " handle_path /nomad/* {", *proxy("127.0.0.1:4646", " ", host="{upstream_hostport}"), " }"]
361 for service, upstreams in sorted(internal_services.items()):
362 host = f"{service}.{os.environ['STUDIO_DOMAIN']}" if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service) else "{upstream_hostport}"
363 lines += [f" handle_path /services/{service}/* {{",
364 *proxy(" ".join(sorted(upstreams)), " ", host=host), " }"]
365 lines += [" handle {", " respond 404", " }", " }", " handle {", " respond 403", " }", "}"]
366 return "\n".join(lines) + "\n"
367
368
369def update(content):
370 try:
371 with open(ROUTES) as file:
372 previous = file.read()
373 except FileNotFoundError:
374 previous = ""
375 if content == previous:
376 return
377 pending = ROUTES + ".pending"
378 def replace(value):
379 with open(pending, "w") as file:
380 file.write(value)
381 os.chown(pending, -1, grp.getgrnam("caddy").gr_gid)
382 os.chmod(pending, 0o640)
383 os.replace(pending, ROUTES)
384
385 replace(content)
386 result = subprocess.run(["systemctl", "reload", "caddy"], capture_output=True, text=True)
387 if result.returncode:
388 replace(previous)
389 raise RuntimeError(result.stderr.strip())
390 print("Caddy routes updated", flush=True)
391
392
393def main():
394 with open(TOKEN) as file:
395 token = file.read().strip()
396 while True:
397 try:
398 update(render(token))
399 except Exception as error:
400 print(f"Caddy route update failed: {error}", file=sys.stderr, flush=True)
401 time.sleep(5)
402
403
404if __name__ == "__main__":
405 main()