From 4af52cc7885ec42021a98021be0e06555a5c81d4 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 5 Oct 2026 01:55:47 -0700 Subject: [PATCH] Move all service sign-in to Snowglobe and upgrade Shale Preserve existing subjects and Shale account IDs during issuer migration. Assisted-by: gpt-6 --- dashboard/src/shale.rs | 17 +--- dashboard/web/types/model.ts | 6 +- nixos/configuration.nix | 5 +- service/dawarich/service.pkl | 9 +- service/forward-auth/service.pkl | 24 +++-- service/jellyfin/service.pkl | 10 +- service/shale/prepare.py | 22 +++++ service/shale/readme/issue-forms.js | 23 ----- service/shale/service.pkl | 19 ++-- tools/dashboard-run.py | 137 ---------------------------- tools/shale-migration.md | 6 +- 11 files changed, 76 insertions(+), 202 deletions(-) create mode 100644 service/shale/prepare.py delete mode 100644 service/shale/readme/issue-forms.js diff --git a/dashboard/src/shale.rs b/dashboard/src/shale.rs index e454af7f42b3dfdda362f36a1db29a2e3d04fb18..df66a472331d41abf10aeba87decdffecb024456 100644 --- a/dashboard/src/shale.rs +++ b/dashboard/src/shale.rs @@ -9,8 +9,7 @@ use rmcp::{ }; use scraper::{Html, Selector}; -/// Shale rotates the session token while rendering comment deletion forms. -/// The final deletion form therefore carries the token accepted by every issue form. +/// r1758 rotates CSRF tokens per deletion form; the final token also works on r1763. fn issue_csrf(document: &Html) -> Result> { let forms = Selector::parse("ul.timeline li.comment form[method=post]").unwrap(); let kind = Selector::parse("input[name=t]").unwrap(); @@ -651,13 +650,10 @@ pub async fn manage( .and_then(|v| v.to_str().ok()) .unwrap_or_default(), )?; - let issuer = url::Url::parse(&env( - "STUDIO_KEYCLOAK_URL", - &format!("https://auth.{}", env("STUDIO_DOMAIN", "studio.test")), - ))?; + let issuer = &app.auth.origin; let parameters = fields(authorization.query().unwrap_or_default())?; if authorization.origin() != issuer.origin() - || authorization.path() != "/realms/master/protocol/openid-connect/auth" + || authorization.path() != "/auth/oidc/authorize" || !authorization.username().is_empty() || authorization.password().is_some() || authorization.fragment().is_some() @@ -772,10 +768,7 @@ pub async fn oauth(app: Arc, request: Request) -> Response { if parameters.get("code").is_none_or(|code| code.is_empty() || code.len() > 4096) || parameters.contains_key("error") { return Err(Error::new(400, "Shale sign-in was declined or incomplete. Start linking again.")); } - let identity = host::call(json!({"operation":"iam.request", "path":format!("/users/{}/shale-username", string(&link["owner"])), "method":"GET", "body":null})).await?; - if identity["body"]["enabled"] != true { - return Err(Error::new(403, "This dashboard account is disabled. Contact its administrator.")); - } + let expected_username = oidc::username(&app.auth, string(&link["owner"]), "shale")?; let (status, headers, _) = app.shale.get(&format!("/-/callback?{query}"), None).await?; if !status.is_redirection() { return Err(Error::new(502, "Shale couldn't finish sign-in. Link it again.")); @@ -783,7 +776,7 @@ pub async fn oauth(app: Arc, request: Request) -> Response { let session = session_cookie(&headers)?; let verified: Result<()> = async { let (status, _, body) = app.shale.get("/-/settings", Some(&session)).await?; - if status != StatusCode::OK || username(&body)? != identity["body"]["username"] { + if status != StatusCode::OK || username(&body)? != expected_username { return Err(Error::new(403, "Sign in to Shale with the same account as your dashboard, then link it again.")); } let owner = string(&link["owner"]); diff --git a/dashboard/web/types/model.ts b/dashboard/web/types/model.ts index 4b36742aa34752289a9360e8ed1d0db64a164122..d167d888c544fbfa5330b5b2c5dba106fcfa1ad5 100644 --- a/dashboard/web/types/model.ts +++ b/dashboard/web/types/model.ts @@ -20,7 +20,7 @@ export interface Me { /** Cookie holding the comma-separated groups an admin previews the dashboard as. */ export const VIEW_AS = "view-as"; -/** The Keycloak group that opens each dashboard section; null opens it to everyone. */ +/** The account group that opens each dashboard section; null opens it to everyone. */ const SECTION_GROUPS = { launcher: null, admin: "infra-admin", metrics: "metrics", media: "media-manage", vms: "vm" } as const; export type Section = keyof typeof SECTION_GROUPS; @@ -38,7 +38,7 @@ export interface ServiceSummary { url: string | null; /** Versioned image URLs per color scheme; the same URL twice when the service has one image. */ icon: { light: string; dark: string } | null; - /** Keycloak group that sees this service in the launcher; null means everyone. */ + /** Account group that sees this service in the launcher; null means everyone. */ access: string | null; /** What the app is for, in a few words, for people who don't know it by name. */ tagline: string | null; @@ -127,7 +127,7 @@ export interface ServiceDefinition { port: string; /** Hostnames the router sends here; empty for a service only other services reach. */ hostnames: string[]; - /** The Keycloak group that must sign in first; null for no sign-in gate. */ + /** The account group that must sign in first; null for no sign-in gate. */ authRole: string | null; /** `restartAfter` failures in a row restart `task`, counted once `grace` has passed since it started. */ check: { diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 4f2ddff445585f8e1827371377dece67b32c3f77..f09a392e58930074f28a30c0d0802621c3ecf705 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,7 +6,7 @@ let proxyToken = "/var/lib/studio/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ../service/keycloak/api.py ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ]; }; nativePkl = pkgs.callPackage ./pkl.nix { }; in @@ -162,7 +162,6 @@ in STUDIO_AUTH_RP_ID = "auth.${config.environment.variables.STUDIO_DOMAIN}"; STUDIO_FILE_ORIGIN = "https://file.${config.environment.variables.STUDIO_DOMAIN}"; STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}"; - STUDIO_KEYCLOAK_URL = "https://auth.${config.environment.variables.STUDIO_DOMAIN}"; STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}"; STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}"; STUDIO_PUBLISHED_ROOT = "/srv/clover/Published"; @@ -210,7 +209,7 @@ in --mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \ --mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \ ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \ - ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "keycloak" "jelly" "db" "shale" ]} \ + ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "jelly" "db" "shale" ]} \ "''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"} ''; ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard"; diff --git a/service/dawarich/service.pkl b/service/dawarich/service.pkl index 1bee28fd24082b550b61558443ce48a8bc907ebe..e15f6080db2d426df112c6080b9c369ac72269d0 100644 --- a/service/dawarich/service.pkl +++ b/service/dawarich/service.pkl @@ -1,7 +1,7 @@ extends "../../config/Service.pkl" import "../../config/Service.pkl" as service -import "../keycloak/service.pkl" as keycloak +import "../../config/OpenID.pkl" as sso import "../postgres/service.pkl" as postgres local dawarichImage = "docker.io/freikin/dawarich@sha256:76ec5fa62f414a5ca9e6dd71a9a5b09088c0011075c41644ba35bbb67627a943" @@ -48,9 +48,10 @@ healthyDeadline = "20m" requirements { database - new keycloak.OpenIDClient { + new sso.Client { clientId = module.id name = module.meta.name + redirectUris { "https://\(module.containers["web"].http.hostname)/users/auth/openid_connect/callback" } } } @@ -74,7 +75,7 @@ containers { ["web"] { image = dawarichImage - extraHosts { "\(keycloak.container.http.hostname):host-gateway" } + extraHosts { "\(sso.hostname):host-gateway" } entrypoint = "web-entrypoint.sh" args { "bin/rails"; "server"; "-p"; "3000"; "-b"; "::" } imageUser = true @@ -94,7 +95,7 @@ containers { ["WEB_CONCURRENCY"] = "1" ["OIDC_CLIENT_ID"] = "${secret.oidc.clientId}" ["OIDC_CLIENT_SECRET"] = "${secret.oidc.clientSecret}" - ["OIDC_ISSUER"] = "https://\(keycloak.container.http.hostname)/realms/master" + ["OIDC_ISSUER"] = sso.issuer ["OIDC_REDIRECT_URI"] = "https://\(module.containers["web"].http.hostname)/users/auth/openid_connect/callback" ["ALLOW_EMAIL_PASSWORD_REGISTRATION"] = "false" } diff --git a/service/forward-auth/service.pkl b/service/forward-auth/service.pkl index 3e829f26e3bd517996717070b741872fe83787de..dc4a7d57009e9aad8c492f50bf5a8a2871794fd1 100644 --- a/service/forward-auth/service.pkl +++ b/service/forward-auth/service.pkl @@ -1,15 +1,23 @@ amends "../../config/Service.pkl" import "../../config/site.pkl" as site -import "../keycloak/service.pkl" as keycloak +import "../../config/OpenID.pkl" as sso + +local isPreview = read?("prop:preview") == "true" meta { name = "Forward Auth" } rollout = "overlapped" requirements { - new keycloak.OpenIDClient { + new sso.Client { clientId = module.id name = module.meta.name + redirectUris { + when (isPreview) { "https://\(module.id).\(site.domain)/snow.oauth2/callback" } + when (!isPreview) { for (host in new Listing { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt" }) { + "https://\(host).\(site.domain)/snow.oauth2/callback" + } } + } } } @@ -21,11 +29,12 @@ container { image = "quay.io/oauth2-proxy/oauth2-proxy@sha256:56e3daedf765c7a1eea6e366fbe684be7d3084830ade14b6174570d3c7960954" cpu = 100 memory = 256 - extraHosts { "\(keycloak.container.http.hostname):host-gateway" } + extraHosts { "\(sso.hostname):host-gateway" } http { containerPort = 4180 checkPath = "/ping" + subdomain = if (isPreview) module.id else null } volumes { @@ -39,9 +48,11 @@ container { ["OAUTH2_PROXY_CLIENT_ID"] = "${secret.oidc.clientId}" ["OAUTH2_PROXY_CLIENT_SECRET"] = "${secret.oidc.clientSecret}" ["OAUTH2_PROXY_COOKIE_SECRET"] = "${secret.own.cookie}" - ["OAUTH2_PROXY_PROVIDER"] = "keycloak-oidc" - ["OAUTH2_PROXY_OIDC_ISSUER_URL"] = "https://\(keycloak.container.http.hostname)/realms/master" - // OAuth2 Proxy requires this claim even when Keycloak accounts have no email. + ["OAUTH2_PROXY_PROVIDER"] = "oidc" + ["OAUTH2_PROXY_OIDC_ISSUER_URL"] = sso.issuer + ["OAUTH2_PROXY_SCOPE"] = "openid profile email groups" + ["OAUTH2_PROXY_OIDC_GROUPS_CLAIM"] = "groups" + // Existing accounts may have no email. ["OAUTH2_PROXY_OIDC_EMAIL_CLAIM"] = "preferred_username" ["OAUTH2_PROXY_CODE_CHALLENGE_METHOD"] = "S256" ["OAUTH2_PROXY_EMAIL_DOMAINS"] = "*" @@ -51,6 +62,7 @@ container { ["OAUTH2_PROXY_REVERSE_PROXY"] = "true" ["OAUTH2_PROXY_WHITELIST_DOMAINS"] = "*.\(site.domain)" ["OAUTH2_PROXY_COOKIE_DOMAINS"] = ".\(site.domain)" + when (isPreview) { ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_stage_\(module.id)" } ["OAUTH2_PROXY_SET_XAUTHREQUEST"] = "true" ["OAUTH2_PROXY_PASS_USER_HEADERS"] = "true" ["OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL"] = "true" diff --git a/service/jellyfin/service.pkl b/service/jellyfin/service.pkl index ab0e477a3fa2a2aae0e848ca71b9a516f38ce23d..fd6b9ccc6cc71e9d4969fccdd509f1c3ca76b39d 100644 --- a/service/jellyfin/service.pkl +++ b/service/jellyfin/service.pkl @@ -1,7 +1,7 @@ amends "../../config/Service.pkl" import "../../config/site.pkl" as site -import "../keycloak/service.pkl" as keycloak +import "../../config/OpenID.pkl" as sso meta { name = "Jellyfin" } // SQLite under /config requires one writer during rollout. @@ -14,9 +14,13 @@ setup = "configure.py" secrets { ["admin_password"] {} } requirements { - new keycloak.OpenIDClient { + new sso.Client { clientId = module.id name = module.meta.name + redirectUris { + "https://\(module.container.http.hostname)/sso/OID/r/snow" + "https://\(module.container.http.hostname)/sso/OID/redirect/snow" + } } } @@ -24,7 +28,7 @@ container { image = "docker.io/jellyfin/jellyfin@sha256:aefb67e6a7ff1debdd154a78a7bbb780fd0c873d8639210a7f6a2016ad2b35db" cpu = 500 memory = 3072 - extraHosts { "\(keycloak.container.http.hostname):host-gateway" } + extraHosts { "\(sso.hostname):host-gateway" } http { containerPort = 8096 diff --git a/service/shale/prepare.py b/service/shale/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..3fe3560d4de8a012a932961ea0b331062bb77ae6 --- /dev/null +++ b/service/shale/prepare.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Keep Shale identities attached to their existing accounts when the issuer moves.""" +import json +import os +from pathlib import Path +import sqlite3 +import sys + +data = json.load(sys.stdin) +path = Path(data["hostRoot"]) / "data/astheno.shale.db" +if path.exists(): + domain = os.environ["STUDIO_DOMAIN"] + old, new = "auth." + domain + "/realms/master", "snowglobe." + domain + db = sqlite3.connect(path, timeout=30) + db.execute("BEGIN IMMEDIATE") + if db.execute("SELECT 1 FROM users old JOIN users new ON old.snowflake=new.snowflake WHERE old.provider=? AND new.provider=?", (old,new)).fetchone(): + raise ValueError("duplicate Shale identity; resolve it before moving the issuer") + count = db.execute("UPDATE users SET provider=? WHERE provider=?", (new,old)).rowcount + db.commit() + assert db.execute("PRAGMA quick_check").fetchone() == ("ok",) + db.close() + print("Moved", count, "existing Shale identity bindings to Snowglobe") diff --git a/service/shale/readme/issue-forms.js b/service/shale/readme/issue-forms.js deleted file mode 100644 index 080f192fb94d52625ac9980a6899f1b38638bb4e..0000000000000000000000000000000000000000 --- a/service/shale/readme/issue-forms.js +++ /dev/null @@ -1,23 +0,0 @@ -// Shale r1758 rotates the session's CSRF token while rendering each comment's -// delete form, leaving the surrounding issue forms with the earlier token. -(() => { - function normalize(root, initial = false) { - const forms = [...root.querySelectorAll('form[method="post" i]')]; - const tokens = forms.flatMap(form => { - if (initial && form.querySelector('input[name="t"]')?.value !== 'delete') return []; - const input = form.querySelector('input[name="csrf_token"]'); - return input?.value ? [input.value] : []; - }); - const token = tokens.at(-1); - if (!token) return; - for (const input of document.querySelectorAll('form[method="post" i] input[name="csrf_token"]')) { - input.value = token; - } - } - function start() { - normalize(document, true); - document.body.addEventListener('htmx:afterSwap', event => normalize(event.detail.target)); - } - if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', start, { once: true }); - else start(); -})(); diff --git a/service/shale/service.pkl b/service/shale/service.pkl index aab981d0f098c92c889819f579a3186bed3c092b..df942fc7ba21552920ba6cb5c36befd44798d9f8 100644 --- a/service/shale/service.pkl +++ b/service/shale/service.pkl @@ -1,21 +1,25 @@ amends "../../config/Service.pkl" -import "../keycloak/service.pkl" as keycloak +import "../../config/OpenID.pkl" as sso meta { name = "Shale" } requirements { - new keycloak.OpenIDClient { + new sso.Client { clientId = module.id name = module.meta.name + redirectUris { "https://\(module.container.http.hostname)/-/callback" } usernameAliases { ["snow"] = "clover" } + allowGuests = true + usernameRequired = true } } +prepare = "prepare.py" container { - image = "docker.io/astheno/shale@sha256:dfffceebe31fd3360b6dcf12caab664735415fdc32effd5082ac37b11864a232" - extraHosts { "\(keycloak.container.http.hostname):host-gateway" } + image = "docker.io/astheno/shale@sha256:d89f4d8fe0ee4bd8f57ef35e3cf19c91be158ee131c222bbc7d47920ac198b6f" + extraHosts { "\(sso.hostname):host-gateway" } http { containerPort = 8000 @@ -26,9 +30,6 @@ container { ["/-/studio-readme/"] = "readme" } headHtml { - ["/*/issues/*"] = """ - - """ ["/snowbound/"] = """ """ @@ -51,10 +52,10 @@ container { ["DOMAIN"] = module.container.http.hostname ["HOME"] = "/data" ["SERVER_TITLE"] = "clover's git" - // The older Markdown parser shares a capture buffer between request workers. + // Concurrent Markdown rendering shares capture state between request workers. ["NPROC"] = "1" ["SESSION_SECRET"] = "${secret.own.session_secret}" - ["OAUTH2_CLIENT"] = "oidc,\(keycloak.container.http.hostname)/realms/master|${secret.oidc.clientId}|${secret.oidc.clientSecret}" + ["OAUTH2_CLIENT"] = "oidc,\(sso.hostname)|${secret.oidc.clientId}|${secret.oidc.clientSecret}" } } diff --git a/tools/dashboard-run.py b/tools/dashboard-run.py index 66daf37c1387c951973fdaf4408a3612d48f08eb..05998255771de00aa81c69114deab046b46efffc 100644 --- a/tools/dashboard-run.py +++ b/tools/dashboard-run.py @@ -13,14 +13,12 @@ import sys import time import uuid import urllib.error -import urllib.parse import urllib.request import release FIELDS = { - "iam.request": {"path", "method", "body"}, "deploy.current": set(), "deploy.main": set(), "deploy.history": set(), "deploy.stages": set(), "deploy.managed": set(), "deploy.release": {"release"}, "deploy.output": {"kind", "target"}, "deploy.last": set(), "deploy.run": {"id"}, "deploy.start": {"action", "target"}, @@ -31,8 +29,6 @@ ACTIONS = {"deploy", "destroy", "rollback", "start", "stop", "restart", "secret- MAX_LOG = 1024 * 1024 MAX_METADATA = 65536 HOST_STATE = Path(os.environ.get("STUDIO_HOST_STATE_ROOT", "/var/lib/studio/host")) -IAM_ROLES = {"infra-admin", "media", "media-manage"} -IAM_ACTIONS = {"UPDATE_PASSWORD", "VERIFY_EMAIL", "UPDATE_PROFILE", "CONFIGURE_TOTP", "webauthn-register", "webauthn-register-passwordless"} class Error(Exception): @@ -255,23 +251,6 @@ def start(action, target, key=None, value=None): def handle(request): operation = request["operation"] - if operation == "iam.request": - iam_validate(request) - process = subprocess.run([ - "systemd-run", "--pipe", "--wait", "--collect", "--quiet", - "--unit=studio-iam-" + str(uuid.uuid4()), "--property=RuntimeMaxSec=25", - "--property=MemoryMax=128M", "--property=TasksMax=8", "--property=ProtectSystem=strict", - "--property=ProtectHome=yes", "--property=NoNewPrivileges=yes", "--property=CapabilityBoundingSet=", - "--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX", "--property=IPAddressDeny=any", - "--property=IPAddressAllow=localhost", "--setenv=STUDIO_DOMAIN=" + os.environ["STUDIO_DOMAIN"], - "--setenv=STUDIO_API_TIMEOUT=5", "--", sys.executable, str(Path(__file__)), "--iam"], - input=json.dumps(request), capture_output=True, text=True, timeout=30, check=True) - response = json.loads(process.stdout) - if "error" in response: - raise Error(response["status"], response["error"]) - return response["value"] - if operation.startswith("deploy.secret.") and request["service"] == "keycloak": - raise Error(403, "Keycloak credentials are managed by the host.") if operation == "deploy.secret.get": if not isinstance(request["key"], str): raise Error(400, "Choose a secret from this service's list.") @@ -373,108 +352,6 @@ def handle(request): return start(request["action"], request["target"]) -def iam_validate(request): - path, method, body = request["path"], request["method"], request["body"] - if not isinstance(path, str) or not isinstance(method, str): - raise Error(400, "Choose a supported user operation.") - allowed = { - "/roles": {"GET"}, "/users?max=1000": {"GET"}, "/users": {"POST"}, - "": {"GET", "PUT", "DELETE"}, "/sessions": {"GET"}, "/credentials": {"GET"}, - "/role-mappings/realm": {"GET", "POST", "DELETE"}, "/logout": {"POST"}, - "/shale-username": {"GET"}, - "/execute-actions-email": {"PUT"}, "/reset-password": {"PUT"}, - } - user = re.fullmatch(r"/users/([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})(.*)", path) - suffix = user[2] if user else path - if user and suffix not in {"", "/sessions", "/credentials", "/role-mappings/realm", "/logout", "/execute-actions-email", "/reset-password", "/shale-username"}: - raise Error(400, "Choose a supported user operation.") - if path.startswith("/users?username="): - try: - query = urllib.parse.parse_qs(path.partition("?")[2], keep_blank_values=True, strict_parsing=True) - except ValueError: - raise Error(400, "Choose a username.") from None - if (set(query) != {"username", "exact"} or query["exact"] != ["true"] - or len(query["username"]) != 1 or not re.fullmatch(r"[a-z0-9][a-z0-9._@-]{0,254}", query["username"][0])): - raise Error(400, "Choose a username.") - suffix = "/users?max=1000" - if (method not in allowed.get(suffix, set()) or not user and suffix == "" - or method in {"GET", "DELETE", "POST"} and suffix not in {"/users", "/role-mappings/realm"} and body is not None - or method == "GET" and body is not None or suffix == "/shale-username" and not user): - raise Error(400, "Choose a supported user operation.") - if method == "PUT" and suffix == "/reset-password": - if (not isinstance(body, dict) or set(body) != {"type", "value", "temporary"} - or body["type"] != "password" or not isinstance(body["value"], str) - or not 8 <= len(body["value"]) <= 8192 or type(body["temporary"]) is not bool): - raise Error(400, "Enter a password and choose whether it is temporary.") - elif method == "PUT" and suffix == "/execute-actions-email": - if not isinstance(body, list) or not body or not all(isinstance(action, str) and action in IAM_ACTIONS for action in body): - raise Error(400, "Choose a sign-in action.") - elif suffix == "/role-mappings/realm" and method != "GET": - if (not isinstance(body, list) or len(body) != 1 or not isinstance(body[0], dict) - or not isinstance(body[0].get("name"), str) or body[0]["name"] not in IAM_ROLES or not isinstance(body[0].get("id"), str)): - raise Error(403, "Choose a dashboard group.") - elif method == "POST" and suffix == "/users" or method == "PUT" and suffix == "": - if not isinstance(body, dict) or not body or not set(body) <= {"username", "email", "firstName", "lastName", "enabled", "emailVerified", "requiredActions", "attributes"}: - raise Error(400, "Enter a user profile.") - for key, value in body.items(): - if key in {"enabled", "emailVerified"}: - valid = type(value) is bool - elif key == "requiredActions": - valid = isinstance(value, list) and all(isinstance(action, str) and action in IAM_ACTIONS for action in value) - elif key == "attributes": - valid = isinstance(value, dict) and set(value) == {"picture"} and (value["picture"] is None or isinstance(value["picture"], list) and len(value["picture"]) == 1 and isinstance(value["picture"][0], str) and len(value["picture"][0]) <= 8192) - else: - valid = value is None and key != "username" or isinstance(value, str) and len(value) <= 8192 - if key == "username": - valid = isinstance(value, str) and bool(re.fullmatch(r"[a-z0-9][a-z0-9._@-]{0,254}", value)) and value != "admin" - if not valid: - raise Error(400, "Enter a valid user profile.") - return user - - -def iam(request): - user = iam_validate(request) - sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "service/keycloak")) - from api import Keycloak - client = Keycloak("keycloak." + os.environ["STUDIO_DOMAIN"], secret("get", "keycloak", "password"), - attempts=1, cafile="/var/lib/studio/ca-bundle.crt") - path, method, body = request["path"], request["method"], request["body"] - if user: - profile = client.request("/admin/realms/master/users/" + user[1]) - if profile["username"] == "admin": - raise Error(403, "The Keycloak administrator is managed outside the dashboard.") - if user[2] == "/shale-username": - clients = client.request("/admin/realms/master/clients?clientId=shale") - clients = [item for item in clients if item["clientId"] == "shale"] - if len(clients) != 1: - raise Error(502, "Shale's sign-in client is unavailable.") - aliases = client.request(f"/admin/realms/master/users/{user[1]}/role-mappings/clients/{clients[0]['id']}/composite") - if len(aliases) > 1: - raise Error(502, "This account has multiple Shale usernames.") - return {"body": {"username": aliases[0]["name"] if aliases else profile["username"], "enabled": profile["enabled"]}} - if isinstance(body, dict) and "attributes" in body: - attributes = dict(profile.get("attributes", {})) - picture = body["attributes"]["picture"] - if picture is None: - attributes.pop("picture", None) - else: - attributes["picture"] = picture - body = {**{key: profile[key] for key in ["username", "email", "firstName", "lastName"] if key in profile}, - **body, "attributes": attributes} - if path.endswith("/role-mappings/realm") and method != "GET": - roles = client.request("/admin/realms/master/roles") - role = next((role for role in roles if role["name"] in IAM_ROLES and role["id"] == body[0]["id"] and role["name"] == body[0]["name"]), None) - if role is None: - raise Error(403, "Choose a dashboard group.") - body = [{"id": role["id"], "name": role["name"]}] - result = client.request("/admin/realms/master" + path, method, body, full=True) - if method == "GET" and path.startswith("/users?"): - result["body"] = [user for user in result["body"] if user["username"] != "admin"] - elif method == "GET" and (path == "/roles" or path.endswith("/role-mappings/realm")): - result["body"] = [role for role in result["body"] if role["name"] in IAM_ROLES] - return result - - def worker(identity, action, target, key=None): if not re.fullmatch(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", identity): raise ValueError("incorrect deployment run ID") @@ -544,20 +421,6 @@ def worker(identity, action, target, key=None): if __name__ == "__main__": - if sys.argv[1:] == ["--iam"]: - try: - payload = sys.stdin.read(MAX_METADATA + 1) - if len(payload.encode()) > MAX_METADATA: - raise Error(400, "The user request is too large. Narrow the selection.") - result = {"value": iam(json.loads(payload))} - except Error as error: - result = {"error": str(error), "status": error.status} - except urllib.error.HTTPError as error: - result = {"error": "Keycloak refused this change. Reload the page and retry.", "status": error.code if error.code in {404, 409} else 502} - except Exception: - result = {"error": "Keycloak is unavailable. Check its service logs.", "status": 502} - print(json.dumps(result)) - raise SystemExit(0) if len(sys.argv) == 5 and sys.argv[1] == "--secret" and sys.argv[2] in {"get", "set", "rotate"}: action, target, key = sys.argv[2:] try: diff --git a/tools/shale-migration.md b/tools/shale-migration.md index bad757fc4bceb839737a72926d70b20e19d008ce..b96a3d3247290ad28229b7ed2976a870a5c69643 100644 --- a/tools/shale-migration.md +++ b/tools/shale-migration.md @@ -28,7 +28,7 @@ The importer preserves existing Shale identities and repository records. Clover' New repository access follows verified Forgejo visibility, with pushes restricted to the owner and issue submission disabled. When private Forgejo history joins an existing repository, public or unlisted permissions are tightened to private **before objects are copied**; existing `off` permissions remain off. An import failure must leave Shale stopped. Restoring only the previous database can re-expose imported private objects through its old public permissions, so recovery must preserve the tightened access or restore the complete service dataset. -The existing `snow` account's original OIDC provider and subject must survive the cutover. Keep the canonical `auth.paperclover.net/realms/master` issuer. Shale caches repository metadata and access at startup, so finish SQLite changes before starting the application. An isolated pinned-image test proved direct registration of a new repository: private anonymous web access returned 404 and Git discovery returned 401; after public permissions and a restart, its page and Git advertisement returned 200 with the original branch object ID. The test used copied data, `--network none`, and no published ports. +Existing Shale account IDs and OIDC subjects must survive the cutover. A deliberate issuer move uses `service/shale/prepare.py` to rebind the provider before startup; duplicate bindings stop the migration. Shale caches repository metadata and access at startup, so finish SQLite changes before starting the application. An isolated pinned-image test proved direct registration of a new repository: private anonymous web access returned 404 and Git discovery returned 401; after public permissions and a restart, its page and Git advertisement returned 200 with the original branch object ID. The test used copied data, `--network none`, and no published ports. A full-data test imported all 19 retained Forgejo histories into a disposable copy of the existing Shale state, yielding 21 repository records. Its conservative fixture metadata marked every incoming repository private. Every copied object file and every source ref passed verification. The pinned application then denied anonymous access to a new private repository and a formerly public collision, while preserving an unaffected public repository. On that isolated copy, public test permissions proved HTTP pages and Git advertisements for `bgds`, `home-infra`, and `nix/config`; `home-infra` advertised the original Forgejo `main` and `vllm`. An actual smart HTTP fetch returned a 53-object pack with a verified pack checksum. Production visibility must come from the restored Forgejo metadata, not this test fixture. @@ -54,7 +54,9 @@ Production issue import completed on October 5 under release `dda941e618934ad9`, This older build predates hidden form CSRF tokens. The router requires the exact HTTPS Origin for every Shale request using the `SessionID` cookie and a mutating method. The guard precedes all Shale handlers inside an explicit Caddy `route`; otherwise default directive ordering can bypass it. Missing, wrong, and suffix-forged origins returned 403 without a database change on the clone. The valid site origin allowed a status change, while Basic-auth Git requests still reached Shale. The MCP adapter permits tokenless issue forms only with the exact verified `r1616` structural footer and no CSRF-token input anywhere on the page. Newer or mixed-token markup remains strict. -The October 4 transport check inspected the image pinned in [service.pkl](../service/shale/service.pkl) in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization. +The October 5 `r1763-g9f5b1c7.zig.0.16.0` upgrade rehearsal passed anonymous reads of the formerly crashing issues, Unicode issue creation and comments, issue closing with a comment Delete form present, access denial, restart persistence, token revocation, and logout. The Rust adapter parsed its actual owner issue markup and accepted its status/comment CSRF fields. All migrated SQLite rows remained identical. Eight-worker concurrent Markdown rendering still crashed; keep `NPROC=1`, which passed 280 concurrent fenced-code requests. The injected issue-form script is no longer needed. Private evidence lives at `/var/lib/studio/shale-upgrade-0680d28c`; its disposable containers were removed after testing. + +The October 4 transport check inspected the then-pinned image in disposable containers without mounting real app data. Its embedded Git endpoint and account settings use HTTP and personal access tokens; no SSH listener, authorized-key interface, or forced-command handler was found. The [official installation](https://astheno.software/shale/installation/) and [configuration reference](https://astheno.software/shale/reference/environment/) also expose HTTP serving and OAuth login without SSH configuration. A `git` account must either use a Shale-aware SSH bridge or await native SSH support. Direct filesystem Git commands would bypass Shale's authorization. Zenith's Shale app directory contains a small SQLite database and 419 MB of owned repositories. `bash tools/import-shale.sh shale-preview-4eea0e3b` copied `data`, `repositories_owned`, and `repositories_mirrors` opaquely from the read-only `storage1/apps@hourly-2026-09-26_05-00` snapshot. It verified checksums and SQLite integrity, then restarted the preview. Both sides had 11 top-level owned repository directories; the preview had one healthy Nomad allocation and returned HTTPS 200. Repository contents were not inspected. -- 2.54.0