Commit 4af52cc7885ec42021a98021be0e06555a5c81d4 | home-infra

dashboard/src/shale.rs+5-12

 @@ -9,8 +9,7 @@ use rmcp::{
 };
 use scraper::{Html, Selector};
-/// Shale rotates the session token while rendering comment deletion forms.
+/// r1758 rotates CSRF tokens per deletion form; the final token also works on r1763.
-/// The final deletion form therefore carries the token accepted by every issue form.
 fn issue_csrf(document: &Html) -> Result<Option<String>> {
     let forms = Selector::parse("ul.timeline li.comment form[method=post]").unwrap();
     let kind = Selector::parse("input[name=t]").unwrap();
 @@ -651,13 +650,10 @@ pub async fn manage(
                     .and_then(|v| v.to_str().ok())
                     .unwrap_or_default(),
             )?;
-            let issuer = url::Url::parse(&env(
+            let issuer = &app.auth.origin;
-                "STUDIO_KEYCLOAK_URL",
-                &format!("https://auth.{}", env("STUDIO_DOMAIN", "studio.test")),
-            ))?;
             let parameters = fields(authorization.query().unwrap_or_default())?;
             if authorization.origin() != issuer.origin()
-                || authorization.path() != "/realms/master/protocol/openid-connect/auth"
+                || authorization.path() != "/auth/oidc/authorize"
                 || !authorization.username().is_empty()
                 || authorization.password().is_some()
                 || authorization.fragment().is_some()
 @@ -772,10 +768,7 @@ pub async fn oauth(app: Arc<App>, request: Request) -> Response {
         if parameters.get("code").is_none_or(|code| code.is_empty() || code.len() > 4096) || parameters.contains_key("error") {
             return Err(Error::new(400, "Shale sign-in was declined or incomplete. Start linking again."));
         }
-        let identity = host::call(json!({"operation":"iam.request", "path":format!("/users/{}/shale-username", string(&link["owner"])), "method":"GET", "body":null})).await?;
+        let expected_username = oidc::username(&app.auth, string(&link["owner"]), "shale")?;
-        if identity["body"]["enabled"] != true {
-            return Err(Error::new(403, "This dashboard account is disabled. Contact its administrator."));
-        }
         let (status, headers, _) = app.shale.get(&format!("/-/callback?{query}"), None).await?;
         if !status.is_redirection() {
             return Err(Error::new(502, "Shale couldn't finish sign-in. Link it again."));
 @@ -783,7 +776,7 @@ pub async fn oauth(app: Arc<App>, request: Request) -> Response {
         let session = session_cookie(&headers)?;
         let verified: Result<()> = async {
             let (status, _, body) = app.shale.get("/-/settings", Some(&session)).await?;
-            if status != StatusCode::OK || username(&body)? != identity["body"]["username"] {
+            if status != StatusCode::OK || username(&body)? != expected_username {
                 return Err(Error::new(403, "Sign in to Shale with the same account as your dashboard, then link it again."));
             }
             let owner = string(&link["owner"]);

dashboard/web/types/model.ts+3-3

nixos/configuration.nix+2-3

service/dawarich/service.pkl+5-4

...	@@ -1,7 +1,7 @@	...	@@ -1,7 +1,7 @@
1	extends "../../config/Service.pkl"	1	extends "../../config/Service.pkl"
2		2
3	import "../../config/Service.pkl" as service	3	import "../../config/Service.pkl" as service
4	import "../keycloak/service.pkl" as keycloak	4	import "../../config/OpenID.pkl" as sso
5	import "../postgres/service.pkl" as postgres	5	import "../postgres/service.pkl" as postgres
6		6
7	local dawarichImage = "docker.io/freikin/dawarich@sha256:76ec5fa62f414a5ca9e6dd71a9a5b09088c0011075c41644ba35bbb67627a943"	7	local dawarichImage = "docker.io/freikin/dawarich@sha256:76ec5fa62f414a5ca9e6dd71a9a5b09088c0011075c41644ba35bbb67627a943"
...	@@ -48,9 +48,10 @@ healthyDeadline = "20m"	...	@@ -48,9 +48,10 @@ healthyDeadline = "20m"
48		48
49	requirements {	49	requirements {
50	database	50	database
51	new keycloak.OpenIDClient {	51	new sso.Client {
52	clientId = module.id	52	clientId = module.id
53	name = module.meta.name	53	name = module.meta.name
		54	redirectUris { "https://\(module.containers["web"].http.hostname)/users/auth/openid_connect/callback" }
54	}	55	}
55	}	56	}
56		57
...	@@ -74,7 +75,7 @@ containers {	...	@@ -74,7 +75,7 @@ containers {
74		75
75	["web"] {	76	["web"] {
76	image = dawarichImage	77	image = dawarichImage
77	extraHosts { "\(keycloak.container.http.hostname):host-gateway" }	78	extraHosts { "\(sso.hostname):host-gateway" }
78	entrypoint = "web-entrypoint.sh"	79	entrypoint = "web-entrypoint.sh"
79	args { "bin/rails"; "server"; "-p"; "3000"; "-b"; "::" }	80	args { "bin/rails"; "server"; "-p"; "3000"; "-b"; "::" }
80	imageUser = true	81	imageUser = true
...	@@ -94,7 +95,7 @@ containers {	...	@@ -94,7 +95,7 @@ containers {
94	["WEB_CONCURRENCY"] = "1"	95	["WEB_CONCURRENCY"] = "1"
95	["OIDC_CLIENT_ID"] = "${secret.oidc.clientId}"	96	["OIDC_CLIENT_ID"] = "${secret.oidc.clientId}"
96	["OIDC_CLIENT_SECRET"] = "${secret.oidc.clientSecret}"	97	["OIDC_CLIENT_SECRET"] = "${secret.oidc.clientSecret}"
97	["OIDC_ISSUER"] = "https://\(keycloak.container.http.hostname)/realms/master"	98	["OIDC_ISSUER"] = sso.issuer
98	["OIDC_REDIRECT_URI"] = "https://\(module.containers["web"].http.hostname)/users/auth/openid_connect/callback"	99	["OIDC_REDIRECT_URI"] = "https://\(module.containers["web"].http.hostname)/users/auth/openid_connect/callback"
99	["ALLOW_EMAIL_PASSWORD_REGISTRATION"] = "false"	100	["ALLOW_EMAIL_PASSWORD_REGISTRATION"] = "false"
100	}	101	}

service/forward-auth/service.pkl+18-6

 @@ -1,15 +1,23 @@
 amends "../../config/Service.pkl"
 import "../../config/site.pkl" as site
-import "../keycloak/service.pkl" as keycloak
+import "../../config/OpenID.pkl" as sso
+local isPreview = read?("prop:preview") == "true"
 meta { name = "Forward Auth" }
 rollout = "overlapped"
 requirements {
-  new keycloak.OpenIDClient {
+  new sso.Client {
     clientId = module.id
     name = module.meta.name
+    redirectUris {
+      when (isPreview) { "https://\(module.id).\(site.domain)/snow.oauth2/callback" }
+      when (!isPreview) { for (host in new Listing<String> { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt" }) {
+        "https://\(host).\(site.domain)/snow.oauth2/callback"
+      } }
+    }
   }
 }
 @@ -21,11 +29,12 @@ container {
   image = "quay.io/oauth2-proxy/oauth2-proxy@sha256:56e3daedf765c7a1eea6e366fbe684be7d3084830ade14b6174570d3c7960954"
   cpu = 100
   memory = 256
-  extraHosts { "\(keycloak.container.http.hostname):host-gateway" }
+  extraHosts { "\(sso.hostname):host-gateway" }
   http {
     containerPort = 4180
     checkPath = "/ping"
+    subdomain = if (isPreview) module.id else null
   }
   volumes {
 @@ -39,9 +48,11 @@ container {
     ["OAUTH2_PROXY_CLIENT_ID"] = "${secret.oidc.clientId}"
     ["OAUTH2_PROXY_CLIENT_SECRET"] = "${secret.oidc.clientSecret}"
     ["OAUTH2_PROXY_COOKIE_SECRET"] = "${secret.own.cookie}"
-    ["OAUTH2_PROXY_PROVIDER"] = "keycloak-oidc"
+    ["OAUTH2_PROXY_PROVIDER"] = "oidc"
-    ["OAUTH2_PROXY_OIDC_ISSUER_URL"] = "https://\(keycloak.container.http.hostname)/realms/master"
+    ["OAUTH2_PROXY_OIDC_ISSUER_URL"] = sso.issuer
-    // OAuth2 Proxy requires this claim even when Keycloak accounts have no email.
+    ["OAUTH2_PROXY_SCOPE"] = "openid profile email groups"
+    ["OAUTH2_PROXY_OIDC_GROUPS_CLAIM"] = "groups"
+    // Existing accounts may have no email.
     ["OAUTH2_PROXY_OIDC_EMAIL_CLAIM"] = "preferred_username"
     ["OAUTH2_PROXY_CODE_CHALLENGE_METHOD"] = "S256"
     ["OAUTH2_PROXY_EMAIL_DOMAINS"] = "*"
 @@ -51,6 +62,7 @@ container {
     ["OAUTH2_PROXY_REVERSE_PROXY"] = "true"
     ["OAUTH2_PROXY_WHITELIST_DOMAINS"] = "*.\(site.domain)"
     ["OAUTH2_PROXY_COOKIE_DOMAINS"] = ".\(site.domain)"
+    when (isPreview) { ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_stage_\(module.id)" }
     ["OAUTH2_PROXY_SET_XAUTHREQUEST"] = "true"
     ["OAUTH2_PROXY_PASS_USER_HEADERS"] = "true"
     ["OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL"] = "true"

service/jellyfin/service.pkl+7-3

 @@ -1,7 +1,7 @@
 amends "../../config/Service.pkl"
 import "../../config/site.pkl" as site
-import "../keycloak/service.pkl" as keycloak
+import "../../config/OpenID.pkl" as sso
 meta { name = "Jellyfin" }
 // SQLite under /config requires one writer during rollout.
 @@ -14,9 +14,13 @@ setup = "configure.py"
 secrets { ["admin_password"] {} }
 requirements {
-  new keycloak.OpenIDClient {
+  new sso.Client {
     clientId = module.id
     name = module.meta.name
+    redirectUris {
+      "https://\(module.container.http.hostname)/sso/OID/r/snow"
+      "https://\(module.container.http.hostname)/sso/OID/redirect/snow"
+    }
   }
 }
 @@ -24,7 +28,7 @@ container {
   image = "docker.io/jellyfin/jellyfin@sha256:aefb67e6a7ff1debdd154a78a7bbb780fd0c873d8639210a7f6a2016ad2b35db"
   cpu = 500
   memory = 3072
-  extraHosts { "\(keycloak.container.http.hostname):host-gateway" }
+  extraHosts { "\(sso.hostname):host-gateway" }
   http {
     containerPort = 8096

service/shale/prepare.py created+22

...	@@ -0,0 +1,22 @@
1	#!/usr/bin/env python3
2	"""Keep Shale identities attached to their existing accounts when the issuer moves."""
3	import json
4	import os
5	from pathlib import Path
6	import sqlite3
7	import sys
8
9	data = json.load(sys.stdin)
10	path = Path(data["hostRoot"]) / "data/astheno.shale.db"
11	if path.exists():
12	domain = os.environ["STUDIO_DOMAIN"]
13	old, new = "auth." + domain + "/realms/master", "snowglobe." + domain
14	db = sqlite3.connect(path, timeout=30)
15	db.execute("BEGIN IMMEDIATE")
16	if db.execute("SELECT 1 FROM users old JOIN users new ON old.snowflake=new.snowflake WHERE old.provider=? AND new.provider=?", (old,new)).fetchone():
17	raise ValueError("duplicate Shale identity; resolve it before moving the issuer")
18	count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount
19	db.commit()
20	assert db.execute("PRAGMA quick_check").fetchone() == ("ok",)
21	db.close()
22	print("Moved", count, "existing Shale identity bindings to Snowglobe")

service/shale/readme/issue-forms.js deleted-23

...	@@ -1,23 +0,0 @@
1	// Shale r1758 rotates the session's CSRF token while rendering each comment's
2	// delete form, leaving the surrounding issue forms with the earlier token.
3	(() => {
4	function normalize(root, initial = false) {
5	const forms = [...root.querySelectorAll('form[method="post" i]')];
6	const tokens = forms.flatMap(form => {
7	if (initial && form.querySelector('input[name="t"]')?.value !== 'delete') return [];
8	const input = form.querySelector('input[name="csrf_token"]');
9	return input?.value ? [input.value] : [];
10	});
11	const token = tokens.at(-1);
12	if (!token) return;
13	for (const input of document.querySelectorAll('form[method="post" i] input[name="csrf_token"]')) {
14	input.value = token;
15	}
16	}
17	function start() {
18	normalize(document, true);
19	document.body.addEventListener('htmx:afterSwap', event => normalize(event.detail.target));
20	}
21	if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', start, { once: true });
22	else start();
23	})();

service/shale/service.pkl+10-9

 @@ -1,21 +1,25 @@
 amends "../../config/Service.pkl"
-import "../keycloak/service.pkl" as keycloak
+import "../../config/OpenID.pkl" as sso
 meta {
   name = "Shale"
 }
 requirements {
-  new keycloak.OpenIDClient {
+  new sso.Client {
     clientId = module.id
     name = module.meta.name
+    redirectUris { "https://\(module.container.http.hostname)/-/callback" }
     usernameAliases { ["snow"] = "clover" }
+    allowGuests = true
+    usernameRequired = true
   }
 }
+prepare = "prepare.py"
 container {
-  image = "docker.io/astheno/shale@sha256:dfffceebe31fd3360b6dcf12caab664735415fdc32effd5082ac37b11864a232"
+  image = "docker.io/astheno/shale@sha256:d89f4d8fe0ee4bd8f57ef35e3cf19c91be158ee131c222bbc7d47920ac198b6f"
-  extraHosts { "\(keycloak.container.http.hostname):host-gateway" }
+  extraHosts { "\(sso.hostname):host-gateway" }
   http {
     containerPort = 8000
 @@ -26,9 +30,6 @@ container {
       ["/-/studio-readme/"] = "readme"
     }
     headHtml {
-      ["/*/issues/*"] = """
-        <script defer src="/-/studio-readme/issue-forms.js"></script>
-        """
       ["/snowbound/"] = """
         <script defer src="/-/studio-readme/markdown-it.min.js"></script><script defer src="/-/studio-readme/purify.min.js"></script><script defer src="/-/studio-readme/readme.js"></script>
         """
 @@ -51,10 +52,10 @@ container {
     ["DOMAIN"] = module.container.http.hostname
     ["HOME"] = "/data"
     ["SERVER_TITLE"] = "clover's git"
-    // The older Markdown parser shares a capture buffer between request workers.
+    // Concurrent Markdown rendering shares capture state between request workers.
     ["NPROC"] = "1"
     ["SESSION_SECRET"] = "${secret.own.session_secret}"
-    ["OAUTH2_CLIENT"] = "oidc,\(keycloak.container.http.hostname)/realms/master|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
+    ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}"
   }
 }

tools/dashboard-run.py-137

...	@@ -13,14 +13,12 @@ import sys	...	@@ -13,14 +13,12 @@ import sys
13	import time	13	import time
14	import uuid	14	import uuid
15	import urllib.error	15	import urllib.error
16	import urllib.parse
17	import urllib.request	16	import urllib.request
18		17
19	import release	18	import release
20		19
21		20
22	FIELDS = {	21	FIELDS = {
23	"iam.request": {"path", "method", "body"},
24	"deploy.current": set(), "deploy.main": set(), "deploy.history": set(), "deploy.stages": set(), "deploy.managed": set(),	22	"deploy.current": set(), "deploy.main": set(), "deploy.history": set(), "deploy.stages": set(), "deploy.managed": set(),
25	"deploy.release": {"release"}, "deploy.output": {"kind", "target"},	23	"deploy.release": {"release"}, "deploy.output": {"kind", "target"},
26	"deploy.last": set(), "deploy.run": {"id"}, "deploy.start": {"action", "target"},	24	"deploy.last": set(), "deploy.run": {"id"}, "deploy.start": {"action", "target"},
...	@@ -31,8 +29,6 @@ ACTIONS = {"deploy", "destroy", "rollback", "start", "stop", "restart", "secret-	...	@@ -31,8 +29,6 @@ ACTIONS = {"deploy", "destroy", "rollback", "start", "stop", "restart", "secret-
31	MAX_LOG = 1024 * 1024	29	MAX_LOG = 1024 * 1024
32	MAX_METADATA = 65536	30	MAX_METADATA = 65536
33	HOST_STATE = Path(os.environ.get("STUDIO_HOST_STATE_ROOT", "/var/lib/studio/host"))	31	HOST_STATE = Path(os.environ.get("STUDIO_HOST_STATE_ROOT", "/var/lib/studio/host"))
34	IAM_ROLES = {"infra-admin", "media", "media-manage"}
35	IAM_ACTIONS = {"UPDATE_PASSWORD", "VERIFY_EMAIL", "UPDATE_PROFILE", "CONFIGURE_TOTP", "webauthn-register", "webauthn-register-passwordless"}
36		32
37		33
38	class Error(Exception):	34	class Error(Exception):
...	@@ -255,23 +251,6 @@ def start(action, target, key=None, value=None):	...	@@ -255,23 +251,6 @@ def start(action, target, key=None, value=None):
255		251
256	def handle(request):	252	def handle(request):
257	operation = request["operation"]	253	operation = request["operation"]
258	if operation == "iam.request":
259	iam_validate(request)
260	process = subprocess.run([
261	"systemd-run", "--pipe", "--wait", "--collect", "--quiet",
262	"--unit=studio-iam-" + str(uuid.uuid4()), "--property=RuntimeMaxSec=25",
263	"--property=MemoryMax=128M", "--property=TasksMax=8", "--property=ProtectSystem=strict",
264	"--property=ProtectHome=yes", "--property=NoNewPrivileges=yes", "--property=CapabilityBoundingSet=",
265	"--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX", "--property=IPAddressDeny=any",
266	"--property=IPAddressAllow=localhost", "--setenv=STUDIO_DOMAIN=" + os.environ["STUDIO_DOMAIN"],
267	"--setenv=STUDIO_API_TIMEOUT=5", "--", sys.executable, str(Path(__file__)), "--iam"],
268	input=json.dumps(request), capture_output=True, text=True, timeout=30, check=True)
269	response = json.loads(process.stdout)
270	if "error" in response:
271	raise Error(response["status"], response["error"])
272	return response["value"]
273	if operation.startswith("deploy.secret.") and request["service"] == "keycloak":
274	raise Error(403, "Keycloak credentials are managed by the host.")
275	if operation == "deploy.secret.get":	254	if operation == "deploy.secret.get":
276	if not isinstance(request["key"], str):	255	if not isinstance(request["key"], str):
277	raise Error(400, "Choose a secret from this service's list.")	256	raise Error(400, "Choose a secret from this service's list.")
...	@@ -373,108 +352,6 @@ def handle(request):	...	@@ -373,108 +352,6 @@ def handle(request):
373	return start(request["action"], request["target"])	352	return start(request["action"], request["target"])
374		353
375		354
376	def iam_validate(request):
377	path, method, body = request["path"], request["method"], request["body"]
378	if not isinstance(path, str) or not isinstance(method, str):
379	raise Error(400, "Choose a supported user operation.")
380	allowed = {
381	"/roles": {"GET"}, "/users?max=1000": {"GET"}, "/users": {"POST"},
382	"": {"GET", "PUT", "DELETE"}, "/sessions": {"GET"}, "/credentials": {"GET"},
383	"/role-mappings/realm": {"GET", "POST", "DELETE"}, "/logout": {"POST"},
384	"/shale-username": {"GET"},
385	"/execute-actions-email": {"PUT"}, "/reset-password": {"PUT"},
386	}
387	user = re.fullmatch(r"/users/([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})(.*)", path)
388	suffix = user[2] if user else path
389	if user and suffix not in {"", "/sessions", "/credentials", "/role-mappings/realm", "/logout", "/execute-actions-email", "/reset-password", "/shale-username"}:
390	raise Error(400, "Choose a supported user operation.")
391	if path.startswith("/users?username="):
392	try:
393	query = urllib.parse.parse_qs(path.partition("?")[2], keep_blank_values=True, strict_parsing=True)
394	except ValueError:
395	raise Error(400, "Choose a username.") from None
396	if (set(query) != {"username", "exact"} or query["exact"] != ["true"]
397	or len(query["username"]) != 1 or not re.fullmatch(r"[a-z0-9][a-z0-9._@-]{0,254}", query["username"][0])):
398	raise Error(400, "Choose a username.")
399	suffix = "/users?max=1000"
400	if (method not in allowed.get(suffix, set()) or not user and suffix == ""
401	or method in {"GET", "DELETE", "POST"} and suffix not in {"/users", "/role-mappings/realm"} and body is not None
402	or method == "GET" and body is not None or suffix == "/shale-username" and not user):
403	raise Error(400, "Choose a supported user operation.")
404	if method == "PUT" and suffix == "/reset-password":
405	if (not isinstance(body, dict) or set(body) != {"type", "value", "temporary"}
406	or body["type"] != "password" or not isinstance(body["value"], str)
407	or not 8 <= len(body["value"]) <= 8192 or type(body["temporary"]) is not bool):
408	raise Error(400, "Enter a password and choose whether it is temporary.")
409	elif method == "PUT" and suffix == "/execute-actions-email":
410	if not isinstance(body, list) or not body or not all(isinstance(action, str) and action in IAM_ACTIONS for action in body):
411	raise Error(400, "Choose a sign-in action.")
412	elif suffix == "/role-mappings/realm" and method != "GET":
413	if (not isinstance(body, list) or len(body) != 1 or not isinstance(body[0], dict)
414	or not isinstance(body[0].get("name"), str) or body[0]["name"] not in IAM_ROLES or not isinstance(body[0].get("id"), str)):
415	raise Error(403, "Choose a dashboard group.")
416	elif method == "POST" and suffix == "/users" or method == "PUT" and suffix == "":
417	if not isinstance(body, dict) or not body or not set(body) <= {"username", "email", "firstName", "lastName", "enabled", "emailVerified", "requiredActions", "attributes"}:
418	raise Error(400, "Enter a user profile.")
419	for key, value in body.items():
420	if key in {"enabled", "emailVerified"}:
421	valid = type(value) is bool
422	elif key == "requiredActions":
423	valid = isinstance(value, list) and all(isinstance(action, str) and action in IAM_ACTIONS for action in value)
424	elif key == "attributes":
425	valid = isinstance(value, dict) and set(value) == {"picture"} and (value["picture"] is None or isinstance(value["picture"], list) and len(value["picture"]) == 1 and isinstance(value["picture"][0], str) and len(value["picture"][0]) <= 8192)
426	else:
427	valid = value is None and key != "username" or isinstance(value, str) and len(value) <= 8192
428	if key == "username":
429	valid = isinstance(value, str) and bool(re.fullmatch(r"[a-z0-9][a-z0-9._@-]{0,254}", value)) and value != "admin"
430	if not valid:
431	raise Error(400, "Enter a valid user profile.")
432	return user
433
434
435	def iam(request):
436	user = iam_validate(request)
437	sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "service/keycloak"))
438	from api import Keycloak
439	client = Keycloak("keycloak." + os.environ["STUDIO_DOMAIN"], secret("get", "keycloak", "password"),
440	attempts=1, cafile="/var/lib/studio/ca-bundle.crt")
441	path, method, body = request["path"], request["method"], request["body"]
442	if user:
443	profile = client.request("/admin/realms/master/users/" + user[1])
444	if profile["username"] == "admin":
445	raise Error(403, "The Keycloak administrator is managed outside the dashboard.")
446	if user[2] == "/shale-username":
447	clients = client.request("/admin/realms/master/clients?clientId=shale")
448	clients = [item for item in clients if item["clientId"] == "shale"]
449	if len(clients) != 1:
450	raise Error(502, "Shale's sign-in client is unavailable.")
451	aliases = client.request(f"/admin/realms/master/users/{user[1]}/role-mappings/clients/{clients[0]['id']}/composite")
452	if len(aliases) > 1:
453	raise Error(502, "This account has multiple Shale usernames.")
454	return {"body": {"username": aliases[0]["name"] if aliases else profile["username"], "enabled": profile["enabled"]}}
455	if isinstance(body, dict) and "attributes" in body:
456	attributes = dict(profile.get("attributes", {}))
457	picture = body["attributes"]["picture"]
458	if picture is None:
459	attributes.pop("picture", None)
460	else:
461	attributes["picture"] = picture
462	body = {**{key: profile[key] for key in ["username", "email", "firstName", "lastName"] if key in profile},
463	**body, "attributes": attributes}
464	if path.endswith("/role-mappings/realm") and method != "GET":
465	roles = client.request("/admin/realms/master/roles")
466	role = next((role for role in roles if role["name"] in IAM_ROLES and role["id"] == body[0]["id"] and role["name"] == body[0]["name"]), None)
467	if role is None:
468	raise Error(403, "Choose a dashboard group.")
469	body = [{"id": role["id"], "name": role["name"]}]
470	result = client.request("/admin/realms/master" + path, method, body, full=True)
471	if method == "GET" and path.startswith("/users?"):
472	result["body"] = [user for user in result["body"] if user["username"] != "admin"]
473	elif method == "GET" and (path == "/roles" or path.endswith("/role-mappings/realm")):
474	result["body"] = [role for role in result["body"] if role["name"] in IAM_ROLES]
475	return result
476
477
478	def worker(identity, action, target, key=None):	355	def worker(identity, action, target, key=None):
479	if not re.fullmatch(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", identity):	356	if not re.fullmatch(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", identity):
480	raise ValueError("incorrect deployment run ID")	357	raise ValueError("incorrect deployment run ID")
...	@@ -544,20 +421,6 @@ def worker(identity, action, target, key=None):	...	@@ -544,20 +421,6 @@ def worker(identity, action, target, key=None):
544		421
545		422
546	if __name__ == "__main__":	423	if __name__ == "__main__":
547	if sys.argv[1:] == ["--iam"]:
548	try:
549	payload = sys.stdin.read(MAX_METADATA + 1)
550	if len(payload.encode()) > MAX_METADATA:
551	raise Error(400, "The user request is too large. Narrow the selection.")
552	result = {"value": iam(json.loads(payload))}
553	except Error as error:
554	result = {"error": str(error), "status": error.status}
555	except urllib.error.HTTPError as error:
556	result = {"error": "Keycloak refused this change. Reload the page and retry.", "status": error.code if error.code in {404, 409} else 502}
557	except Exception:
558	result = {"error": "Keycloak is unavailable. Check its service logs.", "status": 502}
559	print(json.dumps(result))
560	raise SystemExit(0)
561	if len(sys.argv) == 5 and sys.argv[1] == "--secret" and sys.argv[2] in {"get", "set", "rotate"}:	424	if len(sys.argv) == 5 and sys.argv[1] == "--secret" and sys.argv[2] in {"get", "set", "rotate"}:
562	action, target, key = sys.argv[2:]	425	action, target, key = sys.argv[2:]
563	try:	426	try:

tools/shale-migration.md+4-2

...	@@ -28,7 +28,7 @@ The importer preserves existing Shale identities and repository records. Clover'	...	@@ -28,7 +28,7 @@ The importer preserves existing Shale identities and repository records. Clover'
28		28
29	New repository access follows verified Forgejo visibility, with pushes restricted to the owner and issue submission disabled. When private Forgejo history joins an existing repository, public or unlisted permissions are tightened to private before objects are copied; existing `off` permissions remain off. An import failure must leave Shale stopped. Restoring only the previous database can re-expose imported private objects through its old public permissions, so recovery must preserve the tightened access or restore the complete service dataset.	29	New repository access follows verified Forgejo visibility, with pushes restricted to the owner and issue submission disabled. When private Forgejo history joins an existing repository, public or unlisted permissions are tightened to private before objects are copied; existing `off` permissions remain off. An import failure must leave Shale stopped. Restoring only the previous database can re-expose imported private objects through its old public permissions, so recovery must preserve the tightened access or restore the complete service dataset.
30		30
31	The existing `snow` account's original OIDC provider and subject must survive the cutover. Keep the canonical `auth.paperclover.net/realms/master` issuer. Shale caches repository metadata and access at startup, so finish SQLite changes before starting the application. An isolated pinned-image test proved direct registration of a new repository: private anonymous web access returned 404 and Git discovery returned 401; after public permissions and a restart, its page and Git advertisement returned 200 with the original branch object ID. The test used copied data, `--network none`, and no published ports.	31	Existing Shale account IDs and OIDC subjects must survive the cutover. A deliberate issuer move uses `service/shale/prepare.py` to rebind the provider before startup; duplicate bindings stop the migration. Shale caches repository metadata and access at startup, so finish SQLite changes before starting the application. An isolated pinned-image test proved direct registration of a new repository: private anonymous web access returned 404 and Git discovery returned 401; after public permissions and a restart, its page and Git advertisement returned 200 with the original branch object ID. The test used copied data, `--network none`, and no published ports.
32		32
33	A full-data test imported all 19 retained Forgejo histories into a disposable copy of the existing Shale state, yielding 21 repository records. Its conservative fixture metadata marked every incoming repository private. Every copied object file and every source ref passed verification. The pinned application then denied anonymous access to a new private repository and a formerly public collision, while preserving an unaffected public repository. On that isolated copy, public test permissions proved HTTP pages and Git advertisements for `bgds`, `home-infra`, and `nix/config`; `home-infra` advertised the original Forgejo `main` and `vllm`. An actual smart HTTP fetch returned a 53-object pack with a verified pack checksum. Production visibility must come from the restored Forgejo metadata, not this test fixture.	33	A full-data test imported all 19 retained Forgejo histories into a disposable copy of the existing Shale state, yielding 21 repository records. Its conservative fixture metadata marked every incoming repository private. Every copied object file and every source ref passed verification. The pinned application then denied anonymous access to a new private repository and a formerly public collision, while preserving an unaffected public repository. On that isolated copy, public test permissions proved HTTP pages and Git advertisements for `bgds`, `home-infra`, and `nix/config`; `home-infra` advertised the original Forgejo `main` and `vllm`. An actual smart HTTP fetch returned a 53-object pack with a verified pack checksum. Production visibility must come from the restored Forgejo metadata, not this test fixture.
34		34
...	@@ -54,7 +54,9 @@ Production issue import completed on October 5 under release `dda941e618934ad9`,	...	@@ -54,7 +54,9 @@ Production issue import completed on October 5 under release `dda941e618934ad9`,
54		54
55	This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict.	55	This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict.
56		56
57	The October 4 transport check inspected the image pinned in [service.pkl](../service/shale/service.pkl) in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.	57	The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing.
		58
		59	The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization.
58		60
59	Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.	61	Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected.
60		62

author	clover caruso <git@paperclover.net> 2026-10-05 01:55:47-07:00
committer	clover caruso <git@paperclover.net> 2026-10-05 15:36:02-07:00
log	4af52cc7885ec42021a98021be0e06555a5c81d4
tree	9086317fac82feecda17cfd2602c8c14a7ca091e
parent	d61a846db111f203963efe7cda8ac7af96da1880
signature	Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

...	@@ -20,7 +20,7 @@ export interface Me {	...	@@ -20,7 +20,7 @@ export interface Me {
20	/** Cookie holding the comma-separated groups an admin previews the dashboard as. */	20	/** Cookie holding the comma-separated groups an admin previews the dashboard as. */
21	export const VIEW_AS = "view-as";	21	export const VIEW_AS = "view-as";
22		22
23	/** The Keycloak group that opens each dashboard section; null opens it to everyone. */	23	/** The account group that opens each dashboard section; null opens it to everyone. */
24	const SECTION_GROUPS = { launcher: null, admin: "infra-admin", metrics: "metrics", media: "media-manage", vms: "vm" } as const;	24	const SECTION_GROUPS = { launcher: null, admin: "infra-admin", metrics: "metrics", media: "media-manage", vms: "vm" } as const;
25	export type Section = keyof typeof SECTION_GROUPS;	25	export type Section = keyof typeof SECTION_GROUPS;
26		26
...	@@ -38,7 +38,7 @@ export interface ServiceSummary {	...	@@ -38,7 +38,7 @@ export interface ServiceSummary {
38	url: string \| null;	38	url: string \| null;
39	/** Versioned image URLs per color scheme; the same URL twice when the service has one image. */	39	/** Versioned image URLs per color scheme; the same URL twice when the service has one image. */
40	icon: { light: string; dark: string } \| null;	40	icon: { light: string; dark: string } \| null;
41	/** Keycloak group that sees this service in the launcher; null means everyone. */	41	/** Account group that sees this service in the launcher; null means everyone. */
42	access: string \| null;	42	access: string \| null;
43	/** What the app is for, in a few words, for people who don't know it by name. */	43	/** What the app is for, in a few words, for people who don't know it by name. */
44	tagline: string \| null;	44	tagline: string \| null;
...	@@ -127,7 +127,7 @@ export interface ServiceDefinition {	...	@@ -127,7 +127,7 @@ export interface ServiceDefinition {
127	port: string;	127	port: string;
128	/** Hostnames the router sends here; empty for a service only other services reach. */	128	/** Hostnames the router sends here; empty for a service only other services reach. */
129	hostnames: string[];	129	hostnames: string[];
130	/** The Keycloak group that must sign in first; null for no sign-in gate. */	130	/** The account group that must sign in first; null for no sign-in gate. */
131	authRole: string \| null;	131	authRole: string \| null;
132	/** `restartAfter` failures in a row restart `task`, counted once `grace` has passed since it started. */	132	/** `restartAfter` failures in a row restart `task`, counted once `grace` has passed since it started. */
133	check: {	133	check: {

...	@@ -6,7 +6,7 @@ let	...	@@ -6,7 +6,7 @@ let
6	proxyToken = "/var/lib/studio/dashboard-proxy.token";	6	proxyToken = "/var/lib/studio/dashboard-proxy.token";
7	hostTools = lib.fileset.toSource {	7	hostTools = lib.fileset.toSource {
8	root = ../.;	8	root = ../.;
9	fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ../service/keycloak/api.py ];	9	fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ];
10	};	10	};
11	nativePkl = pkgs.callPackage ./pkl.nix { };	11	nativePkl = pkgs.callPackage ./pkl.nix { };
12	in	12	in
...	@@ -162,7 +162,6 @@ in	...	@@ -162,7 +162,6 @@ in
162	STUDIO_AUTH_RP_ID = "auth.${config.environment.variables.STUDIO_DOMAIN}";	162	STUDIO_AUTH_RP_ID = "auth.${config.environment.variables.STUDIO_DOMAIN}";
163	STUDIO_FILE_ORIGIN = "https://file.${config.environment.variables.STUDIO_DOMAIN}";	163	STUDIO_FILE_ORIGIN = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
164	STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}";	164	STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}";
165	STUDIO_KEYCLOAK_URL = "https://auth.${config.environment.variables.STUDIO_DOMAIN}";
166	STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}";	165	STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}";
167	STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}";	166	STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}";
168	STUDIO_PUBLISHED_ROOT = "/srv/clover/Published";	167	STUDIO_PUBLISHED_ROOT = "/srv/clover/Published";
...	@@ -210,7 +209,7 @@ in	...	@@ -210,7 +209,7 @@ in
210	--mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \	209	--mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \
211	--mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \	210	--mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \
212	${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \	211	${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \
213	${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "keycloak" "jelly" "db" "shale" ]} \	212	${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "jelly" "db" "shale" ]} \
214	"''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}	213	"''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}
215	'';	214	'';
216	ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard";	215	ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard";

Move all service sign-in to Snowglobe and upgrade Shale