diff --git a/config/Service.pkl b/config/Service.pkl index 6be9d0890fb198a952ec306a728312b0cd71b56e..7132b2b923b6f6e5bc0826c78c6612bbee5e2ed4 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -26,6 +26,8 @@ class Http { identityHeaders: Mapping = new {} /// Set X-Real-Ip from Caddy's observed client address before proxying. forwardRealIp: Boolean = false + /// Require HTTPS when browsers send cookies issued by this route. + secureCookies: Boolean = false /// Generated secret whose value becomes the private proxy header name. identityProofSecret: String? /// Additional hosts routed to this container without SSO headers. diff --git a/service/knife/bundle.json b/service/knife/bundle.json new file mode 100644 index 0000000000000000000000000000000000000000..6beb982f12b73115d26596e037c9fed41adced6c --- /dev/null +++ b/service/knife/bundle.json @@ -0,0 +1,5 @@ +{ + "url": "https://file.paperclover.net/shr/the-knife/knife-server-linux-amd64.tar.gz", + "sha256": "02d9dda61a4de554c5640e6039d2fe9bc5ebcb71d3967efa1930ffc953614102", + "image": "sha256:027c96783e914e8a7b4ab3351e89e2c2f75845e889bf81e19aa5b2e87c8a01a6" +} diff --git a/service/knife/prepare.py b/service/knife/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..6e45e375082e6ea72d3f714d75cad7168e22b43e --- /dev/null +++ b/service/knife/prepare.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import urllib.request + + +bundle = json.loads(Path(__file__).with_name("bundle.json").read_text()) +if subprocess.run(["podman", "image", "exists", bundle["image"]]).returncode: + with tempfile.TemporaryDirectory(prefix="studio-knife-") as temporary: + archive = Path(temporary) / "bundle.tar.gz" + digest = hashlib.sha256() + with urllib.request.urlopen(bundle["url"], timeout=120) as source, archive.open("wb") as target: + while block := source.read(1024 * 1024): + digest.update(block) + target.write(block) + if digest.hexdigest() != bundle["sha256"]: + raise ValueError("Knife server bundle failed checksum verification") + image = Path(temporary) / "image.tar" + with tarfile.open(archive) as source, image.open("wb") as target: + member = source.getmember("image.tar") + if not member.isfile(): + raise ValueError("Knife server bundle has no image file") + with source.extractfile(member) as contents: + shutil.copyfileobj(contents, target) + subprocess.run(["podman", "load", "--quiet", "--input", str(image)], check=True) + subprocess.run(["podman", "image", "exists", bundle["image"]], check=True) diff --git a/service/knife/service.pkl b/service/knife/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a827182904f7731065f16969f9ad8eb81c78b79e --- /dev/null +++ b/service/knife/service.pkl @@ -0,0 +1,46 @@ +amends "../../config/Service.pkl" + +import "pkl:json" + +local bundle = (new json.Parser {}).parse(read("bundle.json")) + +meta { name = "The Knife"; access = "infra-admin" } +stageIsolation = "fresh" +prepare = "prepare.py" +setup = "setup.py" + +container { + image = bundle.image + cpu = 1000 + memory = 4096 + entrypoint = "/usr/bin/tini" + args { + "--" + "/bin/sh" + "-ec" + "node /opt/knife/scripts/container-runtime.ts; exec knife start --host 0.0.0.0 --port 3773 --no-browser" + } + http { + containerPort = 3773 + subdomain = "knife" + loopback = true + secureCookies = true + checkPath = "/.well-known/t3/environment" + } + volumes { + ["/data"] {} + ["/home/node"] {} + ["/archives"] {} + ["/workspaces"] {} + } + env { + ["HOME"] = "/home/node" + ["KNIFE_UPDATE_URL"] = "https://file.paperclover.net/shr/the-knife/" + ["T3CODE_HOME"] = "/data" + ["T3CODE_ARCHIVE_DIR"] = "/archives" + ["T3CODE_ARCHIVE_OFFLOAD"] = "true" + ["T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD"] = "false" + // Info-level startup logs include a live owner pairing credential. + ["T3CODE_LOG_LEVEL"] = "Warn" + } +} diff --git a/service/knife/setup.py b/service/knife/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..ec5115381302ce30667280bd462919499a492877 --- /dev/null +++ b/service/knife/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import subprocess +import sys + + +data = json.load(sys.stdin) +allocations = json.loads(subprocess.check_output( + ["nomad", "job", "allocs", "-json", data["serviceId"]], + env={**os.environ, "NOMAD_TOKEN": Path("/var/lib/studio/nomad.token").read_text().strip()}, +)) +running = [item["ID"] for item in allocations if item["ClientStatus"] == "running" and item["DesiredStatus"] == "run"] +if len(running) != 1: + raise ValueError("Knife setup requires one running allocation") +subprocess.run([ + "podman", "exec", "app-" + running[0], "knife", "shared", "hub", "configure", "https://" + data["host"], +], check=True, capture_output=True) diff --git a/tools/knife.md b/tools/knife.md new file mode 100644 index 0000000000000000000000000000000000000000..47f718a4018820a1013ffdf3bd639456879975bb --- /dev/null +++ b/tools/knife.md @@ -0,0 +1,184 @@ +# Knife setup and operation + +`https://knife.paperclover.net` runs the private Knife authority, archive +receiver, and optional environment hub. Native single-use pairing controls +access; there is no public registration. The service's four directories live +on the managed NAS dataset `/srv/prod/knife`: `data`, `archives`, `home/node`, +and `workspaces`. Provider credentials can stay on each laptop. + +## Pair the owner + +Run these commands in your terminal. SSH access to Zenith is the recovery +path; no owner credential is written to application or deployment logs. + +```sh +ssh -p 22 root@10.0.0.1 +export NOMAD_TOKEN="$(cat /var/lib/studio/nomad.token)" +knife_allocation=$(nomad job allocs -json knife | python3 -c 'import json,sys; a=[r["ID"] for r in json.load(sys.stdin) if r["ClientStatus"]=="running" and r["DesiredStatus"]=="run"]; assert len(a)==1; print(a[0])') +knife_container="app-$knife_allocation" +podman exec "$knife_container" knife auth pairing create \ + --ttl 5m --label 'Clover owner' --base-url https://knife.paperclover.net \ + --scope orchestration:read --scope orchestration:operate \ + --scope settings:write --scope providers:manage \ + --scope environment:maintain --scope preview:operate \ + --scope diagnostics:read --scope terminal:read --scope terminal:operate \ + --scope source-control:write --scope filesystem:read --scope filesystem:write \ + --scope relay:read --scope relay:write --scope access:read --scope access:write +``` + +Open the printed pairing link privately within five minutes. It is single-use. +An owner pairing needs the full scope list: supplying `--scope` replaces the +CLI defaults, and default pairing omits access management. The browser session +lasts 30 days. Repeat the command for another owner device or browser. + +In the desktop app, choose **Self-hosted server**, enter +`https://knife.paperclover.net`, and paste a fresh pairing code. The same entry +is under **Settings → Environments**. Choose this path to avoid T3 Connect. +A browser paired to the NAS controls the NAS environment; laptop transcripts +remain in their originating environments until those laptops are connected. + +The NAS needs no provider login for archive storage or hub routing. Configure +providers on the NAS only if agents should actually run there. Its writable +workspace is `/workspaces`; the personal file archive and host credentials are +not mounted into the container. + +## Enable laptop archive offload + +UI pairing and archive offload are separate connections. Repeat these steps for +each laptop, using a separate, revocable token and label. + +From the Mac, create a private local token file by capturing SSH output directly. +The current API requires orchestration read/operate and provider-management +permissions for offload plus the `shared connect` account-list check. These also +permit thread/provider operations on the NAS; keep the token private. No provider +accounts need to be imported. + +```sh +umask 077 +mkdir -p "$HOME/.config/knife" +ssh -p 22 root@10.0.0.1 'export NOMAD_TOKEN="$(cat /var/lib/studio/nomad.token)"; allocation=$(nomad job allocs -json knife | python3 -c '\''import json,sys; a=[r["ID"] for r in json.load(sys.stdin) if r["ClientStatus"]=="running" and r["DesiredStatus"]=="run"]; assert len(a)==1; print(a[0])'\''); podman exec "app-$allocation" knife auth session issue --ttl 30d --label "Clover Mac archive" --scope orchestration:read --scope orchestration:operate --scope providers:manage --token-only' > "$HOME/.config/knife/nas-token" +chmod 600 "$HOME/.config/knife/nas-token" +"$HOME/.t3/bin/t3" shared connect https://knife.paperclover.net \ + --token-file "$HOME/.config/knife/nas-token" +``` + +The packaged Knife desktop installs `~/.t3/bin/t3`. If a custom `T3CODE_HOME` +is used, use its `bin/t3` and pass the matching `--base-dir` to CLI commands. +The local Knife backend must already be running for `shared connect`. + +Quit Knife, then launch its executable with archive offload enabled: + +```sh +T3CODE_ARCHIVE_OFFLOAD=true '/Applications/The Knife.app/Contents/MacOS/The Knife' +``` + +The setting must reach the backend on every launch. For Finder launches, +`launchctl setenv T3CODE_ARCHIVE_OFFLOAD true` supplies it for this login session; +repeat after logging in again. Fully quit the existing app before relaunching. +Windows WSL backends currently do not forward this variable from the desktop; +enable it inside the WSL backend environment before using offload there. + +Check the connection and archive queue: + +```sh +"$HOME/.t3/bin/t3" shared status +"$HOME/.t3/bin/t3" shared archive status +``` + +Archive one disposable conversation first. Within roughly a minute, status +should report `enabled: true`, an increased `offloaded` count, `pending: 0`, +and no failures. Open its transcript and unarchive it to check retrieval and +restoration before relying on the connection for real history. + +Offload replaces archived T3 transcript bodies with local metadata after the +NAS acknowledges the object. Failed uploads retain the local bodies. Reading +cold history requires the NAS and its valid token; unarchive restores local +bodies. Existing SQLite files can retain allocated pages, and native Claude or +Codex history files remain: this feature does not compact all existing laptop +storage. Keep the NAS state and archive objects together in backups. + +The archive token expires after 30 days without automatic renewal. Issue a +replacement into the private file and run `shared connect` again before it +expires. `shared archive retry` retries failed uploads. Choose a longer `--ttl` +explicitly if monthly renewal is unsuitable; retain a separate label per laptop. + +## Route laptop environments through the NAS + +Archive offload only needs an outbound laptop connection to the NAS. Routing +interactive access through the hub additionally requires Zenith to reach the +laptop backend over a private LAN or VPN address. This fork has no reverse +connection that makes an unreachable laptop accessible through the hub. + +The NAS public hub URL is configured automatically during deployment. To enroll +a laptop, get its environment ID from its private +`/.well-known/t3/environment` descriptor. Issue a separate, labeled administrative +session on that laptop with `~/.t3/bin/t3 auth session issue --ttl 30d +--label "Clover NAS hub" --token-only`, capturing it into a mode-0600 file rather +than displaying it. +Transfer that file privately to Zenith. In the SSH session above: + +```sh +podman cp /root/knife-laptop-enrollment-token "$knife_container":/data/laptop-enrollment-token +podman exec "$knife_container" knife shared hub enroll \ + https://YOUR-PRIVATE-LAPTOP-ADDRESS ENVIRONMENT-ID \ + --token-file /data/laptop-enrollment-token +podman exec "$knife_container" rm /data/laptop-enrollment-token +rm /root/knife-laptop-enrollment-token +``` + +Use a private endpoint already secured by Knife authentication; Tailscale Serve +is one option. No public laptop port is required. Do not disable certificate +verification. The hub retains this administrative credential in its private +state to delegate later connections; removing the transfer files does not revoke +it. Renew and re-enroll before its 30-day expiry, or revoke the labeled laptop +session to withdraw hub access. A sleeping or unreachable laptop cannot serve +an interactive session. Its uploaded archive objects remain on the NAS. + +## Revoke access and update the service + +The owner session can manage pairing links and clients in **Settings → This machine → Authorized clients**. +SSH recovery also works without a live browser session: + +```sh +podman exec "$knife_container" knife auth session list +podman exec "$knife_container" knife auth session revoke SESSION-ID +podman exec "$knife_container" knife auth pairing list +podman exec "$knife_container" knife auth pairing revoke PAIRING-ID +``` + +Keep pairing links and archive tokens private. Access is possession-based: +anyone holding a live code or token has its granted scopes. The proxy adds +`Secure` to browser cookies; the app supplies `HttpOnly` and `SameSite=Lax`. +The container's published port binds only to loopback. + +As requested, each container restart checks the release feed and starts its +newest compatible server/UI/assets after SHA-256 verification. The running +process keeps its current code until restarted. This quiet startup path does +not prefetch in the background. A temporary feed outage starts the installed +runtime; corrupt downloads or a release requiring a newer base image stop +startup instead of executing unverified or incompatible code. + +For a new base image, review the Linux bundle, update +`service/knife/bundle.json` with its checksum and image ID, and use the normal +stage → main → prod workflow. The loader rejects changed bytes at the existing +bundle URL until its recorded checksum is updated. Application code cached +under `/data/runtime/knife-container` survives image/infra rollback; restoring +an older runtime requires an explicit application recovery choice. Code +rollback does not roll back state or archive objects. + +## 2026-10-16 verification + +Zenith fresh preview verified anonymous project/archive/pairing denial, +single-use pairing, hostile cookie Origin rejection, session revocation, +hash-checked archive upload, identical download, byte ranges, and corrupt-object +rejection. A disposable local client imported a two-message synthetic Claude +conversation, offloaded it to the NAS, restarted, read both cold transcript +items, and restored both bodies on unarchive. Temporary client credentials +were revoked and local fixture stores removed. + +Browser verification covered first run, self-hosted connection entry, settings, +access management, archive/history controls, keyboard navigation, and reload +persistence. Real Caddy checks verified Secure on multiple cookies and logout +responses while leaving default routes unchanged. Warn-level startup emitted +no pairing credential. All four service mounts used disposable managed preview +storage with the assigned service UID. diff --git a/tools/router-cookie-test.py b/tools/router-cookie-test.py new file mode 100644 index 0000000000000000000000000000000000000000..6a555a2648844889bce45ec2af319f05b0faf7fe --- /dev/null +++ b/tools/router-cookie-test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import argparse +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import threading +import time +from unittest.mock import patch + +import router + + +class Cookies(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(204 if self.path == '/logout' else 200) + self.send_header('Set-Cookie', 'session=fixture; HttpOnly; Path=/') + self.send_header('Set-Cookie', 'other=fixture; SameSite=Lax') + self.end_headers() + + def log_message(self, *_): + pass + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--caddy', required=True, type=Path) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix='studio-router-cookies-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Cookies) as backend: + root = Path(temporary) + proof = root / 'proof' + proof.write_text('a' * 64) + environment = {**os.environ, 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DASHBOARD_PORT': '3000', + 'STUDIO_PROXY_TOKEN_FILE': str(proof), 'XDG_DATA_HOME': str(root / 'data'), + 'XDG_CONFIG_HOME': str(root / 'config')} + environment.pop('STUDIO_INTERNAL_PORT', None) + + def nomad(path, _): + if path == '/v1/services': + return [{'Namespace': 'default', 'Services': [{'ServiceName': 'knife'}, {'ServiceName': 'plain'}]}] + if path.startswith('/v1/service/'): + service = path.rsplit('/', 1)[1] + return [{'ServiceName': service, 'JobID': service, 'AllocID': service, 'Address': '127.0.0.1', + 'Port': backend.server_port, 'Tags': ['caddy-host=' + service + '.studio.test'] + + (['caddy-secure-cookies=true'] if service == 'knife' else [])}] + return {'http': {'Status': 'success'}} + + with patch.dict(os.environ, environment, clear=True), patch.object(router, 'nomad', nomad), patch.object(router, 'ROUTE_DIR', str(root / 'routes')): + rendered = router.render('fixture') + assert rendered.count('header >Set-Cookie') == 1 + configuration = root / 'routes.caddy' + configuration.write_text('{\n admin off\n auto_https off\n skip_install_trust\n}\n' + rendered) + adapted = subprocess.run([str(args.caddy), 'adapt', '--config', str(configuration), '--adapter', 'caddyfile'], check=True, capture_output=True, text=True, env=environment) + assert '"deferred":true' in adapted.stdout.replace(' ', '') + subprocess.run([str(args.caddy), 'validate', '--config', str(configuration), '--adapter', 'caddyfile'], check=True, capture_output=True, env=environment) + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + runtime = root / 'Caddyfile' + sites = rendered.split('auth.studio.test {', 1)[0] + sites = sites.replace('knife.studio.test {', f'http://127.0.0.1:{port} {{').replace('plain.studio.test {', f'http://localhost:{port} {{').replace(' tls internal\n', '') + runtime.write_text('{\n admin off\n auto_https off\n}\n' + sites) + threading.Thread(target=backend.serve_forever, daemon=True).start() + with (root / 'caddy.log').open('w') as log: + caddy = subprocess.Popen([str(args.caddy), 'run', '--config', str(runtime), '--adapter', 'caddyfile'], stdout=log, stderr=log, env=environment) + try: + deadline = time.monotonic() + 10 + while True: + try: + connection = HTTPConnection('127.0.0.1', port, timeout=1) + connection.request('GET', '/') + response = connection.getresponse() + response.read() + connection.close() + break + except OSError: + assert caddy.poll() is None and time.monotonic() < deadline, (root / 'caddy.log').read_text() + time.sleep(0.05) + for host in ('127.0.0.1', 'localhost'): + for path in ('/', '/logout'): + connection = HTTPConnection('127.0.0.1', port, timeout=2) + connection.request('GET', path, headers={'Host': f'{host}:{port}'}) + response = connection.getresponse() + cookies = [value for key, value in response.getheaders() if key.lower() == 'set-cookie'] + assert response.status == (204 if path == '/logout' else 200) + assert cookies == ['session=fixture; HttpOnly; Path=/' + ('; Secure' if host == '127.0.0.1' else ''), + 'other=fixture; SameSite=Lax' + ('; Secure' if host == '127.0.0.1' else '')], cookies + response.read() + connection.close() + finally: + caddy.terminate() + caddy.wait(timeout=10) + backend.shutdown() + print(json.dumps({'generated_config': 'validated', 'multiple_cookies': 'secure', 'logout_cookies': 'secure', 'default_route': 'unchanged'})) + + +if __name__ == '__main__': + main() diff --git a/tools/router.py b/tools/router.py index 25872312f190dd4251d4b44fbc9bdd60e5e379b7..6e76921123d708c54d09473768269a0b1136e3b6 100644 --- a/tools/router.py +++ b/tools/router.py @@ -151,7 +151,7 @@ def render(token): host = tag.split("=", 1)[1] if not HOST.fullmatch(host): raise ValueError(f"invalid Caddy host: {host!r}") - route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False}) + route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False, "secureCookies": False}) if access and route["authRole"] not in (None, access[0]): raise ValueError("conflicting route access roles") if user_headers and route["userHeader"] not in (None, user_headers[0]): @@ -164,6 +164,8 @@ def render(token): route["internal"] = True if "caddy-real-ip=true" in tags: route["realIp"] = True + if "caddy-secure-cookies=true" in tags: + route["secureCookies"] = True route["upstreams"].add(upstream) route["services"].add(instance.get("JobID") or instance["ServiceName"]) elif tag.startswith("caddy-port="): @@ -184,6 +186,8 @@ def render(token): lines.append(f"{host if listener == 80 else f'{host}:{listener}'} {{") if host.endswith(".test") or route["internal"]: lines.append(" tls internal") + if route["secureCookies"]: + lines.append(' header >Set-Cookie "(.*)" "$1; Secure"') if route["realIp"]: lines.append(" request_header X-Real-Ip {remote_host}") if len(route["services"]) != 1: diff --git a/tools/studio.py b/tools/studio.py index 89cea5db9e1687f0887fc90265612d2d3230e20d..a1b379ed9f37729f9dcf6c56ab78bb0e0c536469 100644 --- a/tools/studio.py +++ b/tools/studio.py @@ -342,6 +342,8 @@ def render(data, definitions): tags.append("caddy-user-header=" + endpoint["userHeader"]) if endpoint["forwardRealIp"]: tags.append("caddy-real-ip=true") + if endpoint.get("secureCookies"): + tags.append("caddy-secure-cookies=true") if endpoint["tlsInternal"]: tags.append("caddy-internal=true") if tags: