diff --git a/service/cliproxy/build-source.json b/service/cliproxy/build-source.json deleted file mode 100644 index d4b71993ed4ec21b7fa7da19f0581eeb13ee7891..0000000000000000000000000000000000000000 --- a/service/cliproxy/build-source.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "sources": [ - { - "source": "../cliproxyapi", - "revision": "91b6ce009bc93dbefd319e7a5b84812f333363ba", - "dockerfile": "Dockerfile.snowglobe", - "include": [ - "go.mod", - "go.sum", - "cmd", - "internal", - "sdk" - ] - }, - { - "source": "../cliproxy-panel", - "revision": "c355dc7b07797c42e8cb7e191e3b176b5442da37", - "destination": "management", - "include": [ - "package.json", - "bun.lock", - "index.html", - "tsconfig.json", - "tsconfig.node.json", - "vite.config.ts", - "src" - ] - } - ] -} diff --git a/service/cliproxy/service.pkl b/service/cliproxy/service.pkl deleted file mode 100644 index 093abffc0e801396d71fb5460c8d291372664707..0000000000000000000000000000000000000000 --- a/service/cliproxy/service.pkl +++ /dev/null @@ -1,58 +0,0 @@ -amends "../../config/Service.pkl" - -import "../../config/site.pkl" as site - -name = "CLIProxyAPI" -tagline = "Codex and Claude API access" -access = "infra-admin" -stageIsolation = "fresh" - -secrets { - ["api_key"] {} - ["management_key"] {} -} - -container { - build = "build" - killTimeout = "30s" - entrypoint = "/bin/sh" - args { "/snowglobe/start.sh" } - readOnlyRoot = true - tmpfs { "/tmp:rw,nosuid,nodev,noexec,size=64m" } - http { - containerPort = 8317 - subdomain = "cpa" - authRole = "infra-admin" - checkPath = "/healthz" - plainHostnames { "cpa-api.\(site.domain)" } - } - volumes { - ["/data"] {} - ["/snowglobe/start.sh"] { config = "start.sh" } - ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true } - } - env { - ["HOME"] = "/data" - ["WRITABLE_PATH"] = "/data" - ["MANAGEMENT_STATIC_PATH"] = "/CLIProxyAPI/static" - ["CLIPROXY_SHUTDOWN_DRAIN"] = "25s" - ["TZ"] = "America/Los_Angeles" - } - secretFiles { - ["/run/secrets/cliproxy-seed.json"] = """ - {{ with nomadVar "nomad/jobs/\(module.id)" }} - { - "config-version": 8, - "server": {"host": "0.0.0.0", "port": 8317, "commercial-mode": true, "max-request-body-bytes": 33554432, "discovery": {"enabled": false}}, - "access": {"api-keys": [{{ .api_key.Value | toJSON }}]}, - "management": {"allow-remote": true, "secret-key": {{ .management_key.Value | toJSON }}, "disable-auto-update-panel": true}, - "oauth": {"auth-dir": "/data/auth"}, - "routing": {"strategy": "reset-soonest", "session-affinity": true, "session-affinity-reclaim": true, "session-affinity-ttl": "1h", "cooldown": {"save-cooldown-status": true}}, - "client": {"codex": {"codex-models-only": true}}, - "plugins": {"enabled": false, "dir": "/data/plugins"}, - "observability": {"history": {"enabled": true}, "logs": {"debug": false, "logging-to-file": false, "request-log": false}, "usage": {"usage-statistics-enabled": false}, "pprof": {"enable": false}} - } - {{ end }} - """ - } -} diff --git a/service/cliproxy/start.sh b/service/cliproxy/start.sh deleted file mode 100644 index 18857c32dd49350c8be1a6fe6ed6fed1dd8d2664..0000000000000000000000000000000000000000 --- a/service/cliproxy/start.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -eu -umask 077 - -if [ -L /data/config.yaml ] || { [ -e /data/config.yaml ] && [ ! -f /data/config.yaml ]; }; then - echo 'CLIProxyAPI configuration must be a regular file' >&2 - exit 1 -fi -if [ ! -e /data/config.yaml ]; then - pending=$(mktemp /data/.config.XXXXXX) - trap 'rm -f "$pending"' EXIT HUP INT TERM - cat /run/secrets/cliproxy-seed.json > "$pending" - ln "$pending" /data/config.yaml || test -f /data/config.yaml - rm -f "$pending" - trap - EXIT HUP INT TERM -fi -test -s /data/config.yaml -chmod 600 /data/config.yaml -mkdir -p /data/auth /data/plugins -chmod 700 /data /data/auth /data/plugins -if [ ! -e /data/.snowglobe-fork-v1 ]; then - /CLIProxyAPI/CLIProxyAPI --config /data/config.yaml --fork-migrate - touch /data/.snowglobe-fork-v1 -fi -exec /CLIProxyAPI/CLIProxyAPI --config /data/config.yaml diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index 2466309442cbea7b5b22d32101276f07e00e6fc8..33ef775a953f645048c219ea790113df88f11e62 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -6,6 +6,7 @@ df: SNOWGLOBE_MIN_FREE_GB ver theme: 2 + localtime name: clover's nas stats, nos-dup dav-auth diff --git a/service/forward-auth/service.pkl b/service/forward-auth/service.pkl index 8c8a69f4acdffd51c523e71572a529da65525be4..2a8a4f84766fe92940c2df23a5f028c9b54242de 100644 --- a/service/forward-auth/service.pkl +++ b/service/forward-auth/service.pkl @@ -14,7 +14,7 @@ requirements { name = module.name redirectUris { when (isPreview) { "https://\(module.id).\(site.domain)/snow.oauth2/callback" } - when (!isPreview) { for (host in new Listing { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw"; "cpa" }) { + when (!isPreview) { for (host in new Listing { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw" }) { "https://\(host).\(site.domain)/snow.oauth2/callback" } } } diff --git a/service/substation/build-source.json b/service/substation/build-source.json new file mode 100644 index 0000000000000000000000000000000000000000..15d436858c10d4b2607f53457dadfa7b143d061a --- /dev/null +++ b/service/substation/build-source.json @@ -0,0 +1,25 @@ +{ + "source": "../substation", + "revision": "b3083b6316d928fbc2e01f8941941c9e3769bdce", + "dockerfile": "Dockerfile.snowglobe", + "includeManifest": true, + "include": [ + ".gitignore", + "package.json", + "README.md", + "build.ts", + "build-info.ts", + "running.ts", + "settings.ts", + "substation.ts", + "tests.ts", + "test-settings.ts", + "update.ts", + "restore.ts", + "hub", + "dashboard", + "ops", + "m0", + "docs" + ] +} diff --git a/service/substation/service.pkl b/service/substation/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a00f7091b16a7e695408ee827e7f2464e68ba13d --- /dev/null +++ b/service/substation/service.pkl @@ -0,0 +1,111 @@ +amends "../../config/Service.pkl" + +import "../../config/OpenID.pkl" as sso +import "../forward-auth/service.pkl" as forwardAuth + +name = "Substation" +tagline = "Claude and Codex subscriptions" +access = "infra-admin" +stageIsolation = "fresh" + +requirements { + new sso.Client { + clientId = module.id + name = module.name + redirectUris { "https://\(module.containers["app"].http.hostname)/snow.oauth2/callback" } + } +} + +secrets { + ["api_key"] {} + ["management_key"] {} + ["cookie"] = forwardAuth.secrets["cookie"] +} + +local runtime = new Container { + build = "build" + hostNetwork = true + readOnlyRoot = true + tmpfs { "/tmp:rw,nosuid,nodev,noexec,size=128m" } + volumes { + ["/data/home"] {} + ["/data/run"] {} + ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true } + } + env { + ["HOME"] = "/data/home" + ["SUBSTATION_RUN_DIR"] = "/data/run" + ["TZ"] = "America/Los_Angeles" + } +} + +containers { + ["hub"] = (runtime) { + lifecycle = "prestartSidecar" + args { "hub" } + killTimeout = "90s" + memory = 1024 + tcp { new { name = "api"; containerPort = 8317 } } + volumes { + ["/data/auth"] {} + ["/data/hub"] {} + } + env { + ["SUBSTATION_HOST"] = "127.0.0.1" + ["SUBSTATION_AUTH_DIR"] = "/data/auth" + ["SUBSTATION_STATE_DIR"] = "/data/hub" + ["SUBSTATION_REFRESH"] = "1" + ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}" + ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}" + ["DISABLE_AUTOUPDATER"] = "1" + } + envTemplate = "SUBSTATION_PORT={{ env \"NOMAD_PORT_hub_api\" }}" + } + ["dashboard"] = (runtime) { + lifecycle = "prestartSidecar" + args { "dashboard" } + tcp { new { name = "web"; containerPort = 8319 } } + volumes { ["/data/dashboard"] {} } + extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" } + env { + ["DASHBOARD_HOST"] = "127.0.0.1" + ["DASHBOARD_AUTH"] = "proxy" + ["DASHBOARD_AUTH_HEADER"] = "X-Forwarded-User" + ["DASHBOARD_TRUSTED_PROXY"] = "127.0.0.1,::1" + ["DASHBOARD_ALLOWED_HOSTS"] = module.containers["app"].http.hostname!! + ["DASHBOARD_DB"] = "/data/dashboard/dashboard.sqlite" + ["SUBSTATION_STATE_DIR"] = "/data/dashboard" + ["SUBSTATION_HUB_KEY"] = "${secret.own.management_key}" + ["SUBSTATION_HUB_URL"] = "https://\(module.containers["app"].http.hostname)" + } + envTemplate = """ + DASHBOARD_PORT={{ env "NOMAD_PORT_dashboard_web" }} + """ + } + ["app"] = (forwardAuth.container!!) { + hostNetwork = true + http { + containerPort = 4180 + loopback = true + subdomain = "substation.ai" + } + envTemplate = """ + OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }} + OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ + """ + env { + ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)" + ["OAUTH2_PROXY_COOKIE_DOMAINS"] = module.containers["app"].http.hostname!! + ["OAUTH2_PROXY_WHITELIST_DOMAINS"] = "\(module.containers["app"].http.hostname),\(sso.hostname)" + ["OAUTH2_PROXY_REDIRECT_URL"] = "https://\(module.containers["app"].http.hostname)/snow.oauth2/callback" + ["OAUTH2_PROXY_ALLOWED_GROUPS"] = "role:infra-admin" + ["OAUTH2_PROXY_AUTH_LOGGING"] = "false" + ["OAUTH2_PROXY_REQUEST_LOGGING"] = "false" + ["OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER"] = "false" + ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false" + ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false" + ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s" + ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/" + } + } +} diff --git a/tools/cliproxy-test.py b/tools/cliproxy-test.py deleted file mode 100644 index a0870e2ad152d01031602e5d99fe1465897ed99c..0000000000000000000000000000000000000000 --- a/tools/cliproxy-test.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import base64 -from contextlib import contextmanager -import hashlib -import http.client -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import json -import os -from pathlib import Path -import re -import socket -import subprocess -import tempfile -import threading -import time -import unittest -from unittest.mock import patch - -import snowglobe -import router - - -SERVICE = Path(__file__).resolve().parents[1] / "service/cliproxy" -CADDY = None -API_KEY = "a1" * 32 -MANAGEMENT_KEY = "b2" * 32 -ADMIN_HOST = "cliproxy-preview-12345678.paperclover.net" -API_HOST = "cliproxy-preview-12345678-cpa-api.paperclover.net" - - -class ServiceTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - result = subprocess.run([os.environ.get("PKL", "pkl"), "eval", "-x", "((module) { enabled = true }).output.text", "-p", "uid=3150", "-p", "serviceId=cliproxy", "-p", "domain=paperclover.net", str(SERVICE / "service.pkl")], check=True, capture_output=True, text=True) - cls.data = json.loads(result.stdout) - cls.data.update(uid=3150, sourceId="cliproxy", inputs={}) - cls.data.pop("requirements") - cls.data['containers']['app']['image'] = 'localhost/cliproxy:test' - - def test_native_defaults_keep_credentials_private_and_requests_unlogged(self): - app = self.data["containers"]["app"] - seed = app["secretFiles"]["/run/secrets/cliproxy-seed.json"] - seed = re.sub(r"{{ with nomadVar [^}]+}}|{{ end }}", "", seed) - seed = seed.replace("{{ .api_key.Value | toJSON }}", json.dumps(API_KEY)) - seed = seed.replace("{{ .management_key.Value | toJSON }}", json.dumps(MANAGEMENT_KEY)) - config = json.loads(seed) - self.assertEqual(config["access"]["api-keys"], [API_KEY]) - self.assertNotEqual(config["access"]["api-keys"][0], config["management"]["secret-key"]) - self.assertTrue(config["server"]["commercial-mode"]) - self.assertFalse(config["observability"]["logs"]["request-log"]) - self.assertFalse(config["observability"]["pprof"]["enable"]) - self.assertFalse(config["plugins"]["enabled"]) - self.assertEqual(config["oauth"]["auth-dir"], "/data/auth") - self.assertTrue(config["management"]["disable-auto-update-panel"]) - self.assertEqual(app["http"]["authRole"], "infra-admin") - self.assertEqual(app["http"]["plainHostnames"], ['cpa-api.paperclover.net']) - self.assertEqual(self.data["stageIsolation"], "fresh") - - def test_rendered_job_has_only_nonroot_private_managed_storage(self): - rendered = snowglobe.render(self.data, {"cliproxy": self.data}) - self.assertEqual(rendered.count('user = "3150:3150"'), 1) - self.assertEqual(rendered.count('readonly_rootfs = true'), 1) - self.assertEqual(rendered.count('perms = "0400"'), 1) - self.assertIn('secrets/cliproxy-seed.json:/run/secrets/cliproxy-seed.json:ro', rendered) - self.assertIn('kill_timeout = "30s"', rendered) - self.assertIn('/CLIProxyAPI/static', rendered) - self.assertNotIn("NOMAD_SERVICE", rendered) - self.assertNotIn("NOMAD_VARIABLE", rendered) - self.assertNotIn("/srv/clover", rendered) - self.assertNotIn("host_port", rendered) - self.assertNotIn('env = true', rendered) - - def test_bootstrap_preserves_config_after_the_first_migration(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - data = root / "data" - data.mkdir() - seed = root / "seed.json" - seed.write_text(json.dumps({"access": {"api-keys": [API_KEY]}})) - binary = root / "binary" - binary.write_text('#!/usr/bin/env python3\nimport pathlib,sys\nassert sys.argv[1] == "--config" and pathlib.Path(sys.argv[2]).stat().st_size\n') - binary.chmod(0o700) - script = (SERVICE / "start.sh").read_text().replace('/run/secrets/cliproxy-seed.json', str(seed)).replace('/CLIProxyAPI/CLIProxyAPI', str(binary)).replace('/data', str(data)) - self.assertEqual(subprocess.run(["sh", "-ec", script], timeout=5).returncode, 0) - config = data / "config.yaml" - self.assertEqual(json.loads(config.read_text())["access"]["api-keys"], [API_KEY]) - self.assertEqual(config.stat().st_mode & 0o777, 0o600) - self.assertEqual((data / "auth").stat().st_mode & 0o777, 0o700) - config.write_text('access:\n api-keys: [edited-key]\nprovider-setting: retained\n') - credential = data / "auth/fixture.json" - credential.write_text('{"synthetic":true}') - seed.write_text('different seed') - self.assertEqual(subprocess.run(["sh", "-ec", script], timeout=5).returncode, 0) - self.assertIn('provider-setting: retained', config.read_text()) - self.assertEqual(credential.read_text(), '{"synthetic":true}') - config.unlink() - config.symlink_to(seed) - result = subprocess.run(["sh", "-ec", script], capture_output=True) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(seed.read_text(), 'different seed') - - - -@unittest.skipUnless(CADDY, "pass --caddy to test the real router") -class GatewayTests(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory(prefix='cliproxy-caddy-') - self.addCleanup(temporary.cleanup) - root = Path(temporary.name) - self.observed = [] - self.auth_checks = [] - self.websocket_started = threading.Event() - self.websocket_release = threading.Event() - observed, auth_checks = self.observed, self.auth_checks - websocket_started, websocket_release = self.websocket_started, self.websocket_release - - class Backend(BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' - - def do_GET(self): - observed.append((self.command, self.path, dict(self.headers))) - management = self.path.startswith(('/v0/management/', '/v8/management/')) - if management: - valid = self.headers.get('Authorization') == 'Bearer ' + MANAGEMENT_KEY or self.headers.get('X-Management-Key') == MANAGEMENT_KEY - elif self.path.startswith('/v1/'): - valid = self.headers.get('Authorization') == 'Bearer ' + API_KEY or self.headers.get('X-Api-Key') == API_KEY - else: - valid = True - if self.headers.get('Upgrade', '').lower() == 'websocket' and valid: - accept = base64.b64encode(hashlib.sha1((self.headers['Sec-WebSocket-Key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest()).decode() - self.send_response(101) - self.send_header('Upgrade', 'websocket'); self.send_header('Connection', 'Upgrade') - self.send_header('Sec-WebSocket-Accept', accept); self.end_headers() - header, mask, body = self.rfile.read(2), self.rfile.read(4), self.rfile.read(2) - if header != b'\x81\x82' or bytes(value ^ mask[index % 4] for index, value in enumerate(body)) != b'hi': - raise AssertionError('incorrect client WebSocket frame') - self.wfile.write(b'\x81\x02hi'); self.wfile.flush() - if self.headers.get('X-Synthetic-Hold'): - websocket_started.set() - if not websocket_release.wait(5): - raise AssertionError('WebSocket release was not signaled') - self.wfile.write(b'\x81\x02ok'); self.wfile.flush() - self.close_connection = True - return - self.send_response(200 if valid else 401) - self.send_header('Content-Type', 'application/json') - self.send_header('Access-Control-Allow-Origin', '*') - if self.headers.get('X-Synthetic-Stream') and valid: - self.send_header('Connection', 'close'); self.end_headers() - self.wfile.write(b'data: first\n\n'); self.wfile.flush() - time.sleep(.5) - self.wfile.write(b'data: last\n\n'); self.close_connection = True - else: - body = b'{}' - self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) - - do_POST = do_GET - do_PUT = do_GET - - def log_message(self, *args): - pass - - class Auth(BaseHTTPRequestHandler): - def do_GET(self): - auth_checks.append(dict(self.headers)) - cookie = self.headers.get('Cookie', '') - self.send_response(202 if cookie else 401) - if cookie: - self.send_header('X-Auth-Request-Groups', 'role:infra-admin' if cookie == 'fixture=admin' else 'role:media') - self.end_headers() - - def log_message(self, *args): - pass - - for handler, attribute in [(Backend, 'backend'), (Auth, 'auth')]: - server = ThreadingHTTPServer(('127.0.0.1', 0), handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - self.addCleanup(server.server_close); self.addCleanup(server.shutdown) - setattr(self, attribute, server) - with socket.socket() as reservation: - reservation.bind(('127.0.0.1', 0)); self.port = reservation.getsockname()[1] - proof = root / 'proof'; proof.write_text('f' * 64) - (root / 'cliproxy.json').write_text(json.dumps({'cliproxy': {'api_key': API_KEY, 'management_key': MANAGEMENT_KEY}})) - def nomad(path, token): - if path == '/v1/services': - return [{'Namespace': 'default', 'Services': [{'ServiceName': name} for name in ('cliproxy', 'forward-auth')]}] - if path.startswith('/v1/service/'): - service = path.rsplit('/', 1)[1] - return [{'ServiceName': service, 'JobID': service, 'AllocID': service, 'Address': '127.0.0.1', - 'Port': self.backend.server_port if service == 'cliproxy' else self.auth.server_port, - 'Tags': ['caddy-host=' + ADMIN_HOST, 'caddy-plain-host=' + API_HOST, 'caddy-auth-role=infra-admin'] if service == 'cliproxy' else []}] - return {'http': {'Status': 'success'}} - environment = {'SNOWGLOBE_DOMAIN': 'paperclover.net', 'SNOWGLOBE_DASHBOARD_PORT': '28001', - 'SNOWGLOBE_PROXY_TOKEN_FILE': str(proof)} - with patch.dict(os.environ, environment), patch.object(router, 'ROUTE_DIR', str(root)), patch.object(router, 'nomad', nomad): - rendered = router.render('fixture') - with socket.socket() as admin: - admin.bind(('127.0.0.1', 0)); self.admin_port = admin.getsockname()[1] - config = '{\n admin 127.0.0.1:' + str(self.admin_port) + '\n auto_https off\n}\n' - for host in (ADMIN_HOST, API_HOST): - block = rendered.split(host + ' {', 1)[1].split('\n}\n', 1)[0] - config += f'http://{host}:{self.port} {{' + block + '\n}\n' - path = root / 'Caddyfile'; path.write_text(config) - self.config_path = path - self.log = root / 'caddy.log' - log = self.log.open('wb'); self.addCleanup(log.close) - process = subprocess.Popen([CADDY, 'run', '--config', str(path), '--adapter', 'caddyfile'], stdout=log, stderr=log) - def stop(): - process.terminate(); process.wait(timeout=5) - self.addCleanup(stop) - for _ in range(100): - if process.poll() is not None: - self.fail('Caddy failed: ' + self.log.read_text()) - try: - with socket.create_connection(('127.0.0.1', self.port), timeout=.1): - break - except OSError: - time.sleep(.02) - else: - self.fail('Caddy did not listen') - - @contextmanager - def request(self, method, path, headers=None): - connection = http.client.HTTPConnection('127.0.0.1', self.port, timeout=3) - connection.request(method, path, headers={'Host': ADMIN_HOST, **(headers or {})}) - response = connection.getresponse() - try: - yield response - finally: - response.close(); connection.close() - - def test_native_keys_bypass_sso_on_both_hosts(self): - for host in (ADMIN_HOST, API_HOST): - for headers in ({'Authorization': 'Bearer ' + MANAGEMENT_KEY}, {'X-Management-Key': MANAGEMENT_KEY}): - with self.request('GET', '/v8/management/config', {'Host': host, **headers}) as response: - self.assertEqual(response.status, 200); response.read() - for headers in ({'Authorization': 'Bearer ' + API_KEY}, {'X-Api-Key': API_KEY}): - with self.request('GET', '/v1/models', {'Host': host, **headers}) as response: - self.assertEqual(response.status, 200); response.read() - self.assertEqual(self.auth_checks, []) - self.assertTrue(all('Cookie' not in headers for _, _, headers in self.observed)) - - def test_wrong_or_absent_keys_cannot_gain_management_access(self): - for host in (ADMIN_HOST, API_HOST): - for headers in ({}, {'Authorization': 'Bearer invalid'}, {'Authorization': 'Bearer ' + API_KEY}): - with self.request('GET', '/v8/management/config', {'Host': host, **headers}) as response: - self.assertEqual(response.status, 401); response.read() - - def test_dashboard_requires_sso_even_with_management_key(self): - for path in ('/', '/management.html'): - with self.request('GET', path, {'Authorization': 'Bearer ' + MANAGEMENT_KEY}) as response: - self.assertEqual(response.status, 302) - self.assertTrue(response.getheader('Location').startswith('/snow.oauth2/sign_in')); response.read() - with self.request('GET', path, {'Host': API_HOST}) as response: - self.assertEqual(response.status, 404); response.read() - with self.request('GET', '/management.html', {'Cookie': 'fixture=member'}) as response: - self.assertEqual(response.status, 403); response.read() - with self.request('GET', '/management.html', {'Cookie': 'fixture=admin'}) as response: - self.assertEqual(response.status, 200); response.read() - - def test_sso_panel_injects_private_credentials_and_guards_origins(self): - for path in ('/v8/management/config', '/v1/models'): - with self.request('GET', path, {'Cookie': 'fixture=admin'}) as response: - self.assertEqual(response.status, 200) - self.assertEqual(response.getheader('Cache-Control'), 'no-store') - self.assertIsNone(response.getheader('Access-Control-Allow-Origin')); response.read() - self.assertNotIn('Cookie', self.observed[-1][2]) - for origin in ('', 'https://evil.example', 'null'): - with self.request('POST', '/v8/management/config', {'Cookie': 'fixture=admin', **({'Origin': origin} if origin else {})}) as response: - self.assertEqual(response.status, 403); response.read() - with self.request('POST', '/v8/management/config', {'Cookie': 'fixture=admin', 'Origin': 'https://' + ADMIN_HOST}) as response: - self.assertEqual(response.status, 200); response.read() - with self.request('POST', '/v8/management/config', {'Authorization': 'Bearer ' + MANAGEMENT_KEY}) as response: - self.assertEqual(response.status, 200); response.read() - - def test_native_websocket_upgrade(self): - key = base64.b64encode(b'synthetic-16-key').decode() - with socket.create_connection(('127.0.0.1', self.port), timeout=3) as connection: - connection.sendall(('GET /v1/responses HTTP/1.1\r\nHost: ' + ADMIN_HOST + '\r\nAuthorization: Bearer ' + API_KEY + '\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ' + key + '\r\n\r\n').encode()) - headers = b'' - while not headers.endswith(b'\r\n\r\n'): - headers += connection.recv(1) - self.assertIn(b' 101 ', headers.split(b'\r\n')[0]) - connection.sendall(b'\x81\x82\x01\x02\x03\x04ik') - self.assertEqual(connection.recv(4), b'\x81\x02hi') - self.assertEqual(self.auth_checks, []) - - def test_websocket_survives_caddy_configuration_reload(self): - for host in (ADMIN_HOST, API_HOST): - self.websocket_started.clear(); self.websocket_release.clear() - key = base64.b64encode(b'synthetic-16-key').decode() - with socket.create_connection(('127.0.0.1', self.port), timeout=3) as connection: - connection.sendall(('GET /v1/responses HTTP/1.1\r\nHost: ' + host + '\r\nAuthorization: Bearer ' + API_KEY + '\r\nX-Synthetic-Hold: yes\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ' + key + '\r\n\r\n').encode()) - headers = b'' - while not headers.endswith(b'\r\n\r\n'): - headers += connection.recv(1) - self.assertIn(b' 101 ', headers) - connection.sendall(b'\x81\x82\x01\x02\x03\x04ik') - self.assertEqual(connection.recv(4), b'\x81\x02hi') - self.assertTrue(self.websocket_started.wait(1)) - subprocess.run([CADDY, 'reload', '--address', f'127.0.0.1:{self.admin_port}', '--config', str(self.config_path), '--adapter', 'caddyfile', '--force'], check=True, capture_output=True) - self.websocket_release.set() - self.assertEqual(connection.recv(4), b'\x81\x02ok') - - def test_native_response_streams(self): - started = time.monotonic() - with self.request('POST', '/v1/responses', {'Authorization': 'Bearer ' + API_KEY, 'X-Synthetic-Stream': 'yes'}) as response: - self.assertEqual(response.status, 200) - self.assertEqual(response.readline(), b'data: first\n') - self.assertLess(time.monotonic() - started, .4) - self.assertIn(b'data: last', response.read()) - log = self.log.read_text() - self.assertNotIn(API_KEY, log); self.assertNotIn(MANAGEMENT_KEY, log) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--caddy', type=Path) - options, remaining = parser.parse_known_args() - CADDY = str(options.caddy.resolve()) if options.caddy else None - GatewayTests.__unittest_skip__ = not CADDY - unittest.main(argv=[__file__, *remaining]) diff --git a/tools/cliproxy.md b/tools/cliproxy.md deleted file mode 100644 index ca392793cf2c6465b4511379a2cd81163f64ae8c..0000000000000000000000000000000000000000 --- a/tools/cliproxy.md +++ /dev/null @@ -1,114 +0,0 @@ -# CLIProxyAPI - -`https://cpa.paperclover.net` serves the Snow Globe CPA fork’s inference and management -APIs. Caddy protects the dashboard with Snow's `infra-admin` role. The -`cpa-api.paperclover.net` alias serves the same native APIs, with dashboard -paths disabled. - -CPA validates its two independent keys: - -- The client `api_key` authenticates inference and model listing through - `Authorization: Bearer …` or the provider-compatible key headers. -- The `management_key` authenticates `/v8/management/*` through - `Authorization: Bearer …` or `X-Management-Key`. - -Nomad variable `nomad/jobs/cliproxy` owns both keys. Keep their values in -restricted files or a password manager, never in chat, arguments, or tracked -configuration. An inference key does not grant management access. A management -key can modify configuration and use stored provider credentials. - -For OpenAI-compatible clients, use `https://cpa.paperclover.net/v1`; for -Claude-compatible clients, use `https://cpa.paperclover.net`. Management callers -use `https://cpa.paperclover.net/v8/management` with the management key. Native -requests carrying a key bypass SSO; CPA rejects missing or invalid credentials. -The dashboard still requires SSO even when a management key is supplied. - -## Dashboard - -Snow sign-in opens the panel automatically. Caddy supplies the management key -only after an `infra-admin` SSO check, and the client key for the panel's model -list. Cookie-authenticated writes require the exact dashboard Origin; reads -reject a foreign Origin. Keys stay on the server, and cookies are removed before -proxying to CPA. Native requests with explicit key headers retain CPA's own -authentication and do not receive injected credentials. - -Opening the panel clears remembered native passwords and validates access -through the config API. Logout clears the shared forward-auth cookie and opens -Snow sign-in. The source panel implements this SSO adaptation; the image contains -one immutable panel build. - -## Providers and persistence - -Use the panel's OAuth section to connect providers. For a remote flow ending at -an unreachable localhost URL, submit the entire callback URL through the panel's -callback form. Treat it as a temporary credential. API-key providers can also be -configured in the native panel. - -`/data/config.yaml` is seeded once from a private Nomad template. Configuration, -OAuth accounts, and refreshed tokens persist in `/srv/prod/cliproxy/data`. -The first fork start atomically migrates the native configuration, preserving -provider credentials and unrelated settings. It selects reset-soonest routing, -affinity with priority reclaim, retained history, Codex-only model discovery, -and saved cooldowns; panel auto-updates and request logging stay disabled. -`/data/.snowglobe-fork-v1` makes this a one-time migration, so later operator -changes survive restarts. The container publishes one -CPA port through Nomad's loopback binding; Caddy is the only public entry point. -There is no nginx container or second CPA listener. - -Changing a Nomad key does not overwrite existing native configuration. Rotate it -in CPA and Nomad together; Caddy's private route manifest is refreshed during -provisioning. Fresh previews have separate storage and keys and must use -synthetic providers only. Shared forward-auth does not register arbitrary preview -callback URLs; a fresh preview cannot complete interactive sign-in through it. - -## Fork sources and history - -[Backend](https://shale.paperclover.net/cliproxyapi/) and -[panel](https://shale.paperclover.net/cliproxy-panel/) fork CPA v8.0.32 and panel -v1.26.5. `service/cliproxy/build-source.json` pins the exact source revisions; -the deployment exporter builds both into one image using -`Dockerfile.snowglobe`. Upstream panel auto-update cannot replace the baked file. -Cursor's private agent protocol is excluded. - -`observability.history.enabled` enables retained monthly JSON usage and quota -files under `/data/history`, beside the auth directory. They contain token -counts, provider/model, hashed sessions and hashed account identities, never -prompts. Quota is sampled each minute; idle subscription usage is polled every -ten minutes. Usage-plan calls share a two-minute cache, with up to thirty minutes -of stale successful data after a 429. The sample retains its original observation -time so stale data cannot masquerade as fresh quota. - -`GET /v8/management/history/{usage,quota,accounts}` reads history; `since` accepts -an RFC3339 timestamp. Accounts are masked with a unique suffix; `reveal=true` -reveals email addresses to an authenticated management caller. `POST` to usage or -quota accepts up to 10,000 bounded rows; stable row IDs make repeated imports -idempotent. `tools/export-history.py` in the backend exports only transcript -paths explicitly supplied by the operator, requires a cutoff, and never exports -message contents. Do not point it at personal transcripts without authorization. - -API-equivalent estimates use models.dev, then LiteLLM, then explicit -`observability.history.prices` overrides (USD per million tokens: `input`, -`output`, `cache-read`, `cache-write`). Catalogs are cached for a day; overrides -apply immediately to new records and imports. Missing catalog cache rates fall back to -input rates; explicit zero rates remain zero. Estimates are neither subscription -bills nor guaranteed provider charges. Existing retained rows keep the price -observed when they were recorded. Restart after enabling history on an instance -that started with it disabled. - -## Deployment and checks - -Stage with `python3 tools/deploy.py stage cliproxy`. A preview retains its own -configuration/history across updates and never promotes that data to production. -Production uses `python3 tools/deploy.py prod` from a described, conflict-free -main. The fork closes its listener before draining active requests for up to -25 seconds, flushes usage, and exits within Nomad's 30-second kill timeout. -Caddy retries new connections briefly and preserves existing WebSockets across -route reloads. A single writer still has a short replacement window; a turn -longer than the drain budget can be interrupted. - -Run `python3 tools/cliproxy-test.py --caddy /path/to/caddy` for native key -separation, SSO roles, dashboard protection, Origin checks, streams, WebSockets -including Caddy reloads, and configuration preservation. It uses disposable -HTTP fixtures, synthetic keys, and a separate Caddy process. -`python3 tools/deploy-test.py` covers composed source exports. The backend uses -`go test ./...` and `go build ./cmd/server`; the panel uses `bun run verify`. diff --git a/tools/deploy.py b/tools/deploy.py index 2aac68b721adc550115878fbce497cfd51802a81..4a7551141495b437b684c056cdc22bbb01f427e3 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -141,6 +141,8 @@ def upload(main=False): shutil.copy2(origin, destination) if not (context / "Dockerfile").is_file(): raise ValueError(f"missing build source Dockerfile: {manifest.parent.name}") + if spec.get("includeManifest"): + shutil.copy2(manifest, context / ".snowglobe-source.json") digest = tree_digest(snapshot) release = digest[:16] remote_release = REMOTE / "releases" / release diff --git a/tools/icloud-export.md b/tools/icloud-export.md new file mode 100644 index 0000000000000000000000000000000000000000..586307ddc2daca433229459f6b110a387c47646c --- /dev/null +++ b/tools/icloud-export.md @@ -0,0 +1,866 @@ +# Export iCloud data to Snow Globe + +Export **9 GB of Voice Memos first** inside the dedicated macOS VM, then +trial optimized native Photos exports of **about 1.7 TB** to Snow Globe. +These are the only requested categories. Keep original media, edited versions, and +metadata in ordinary Windows-readable files, organized by year with mtimes +set from recording dates. The remote friend supplies Apple sign-in and +two-factor approval; the iPhone never needs to hold the whole library or +connect to Clover's Mac by cable. + +The storage split is confirmed by the friend's iCloud storage screenshot. +Apple sign-in, two-factor approval, and encrypted-data authorization are complete +in the dedicated VM. Advanced Data Protection remains on. Unrelated app sync +is disabled in this guest. The source iPhone reports **348 All Recordings, +1 favorite, and 2 Recently Deleted**. + +The Mac catalog contains 358 rows. On macOS 26, the native playable predicate +`flags != nil AND flags & 6 != 0` recognizes **348 active recordings**, matching +the phone, plus two deleted recordings. The native favorite count is one. +Two active recordings are shorter than 0.1 seconds; duration filtering would +exclude them and incorrectly include two stale catalog entries. Use the native +predicate when selecting recordings, and preserve all 358 catalog rows. + +The macOS 15 cache exposed 161 audio files. A fresh macOS 26 import retrieved +350 files totaling 9,612,921,523 bytes: 161 `.m4a` and 189 `.qta`, including the +two deleted recordings. All 348 active recordings now have audio references. +The complete host and native checks recovered **310 active recordings** passing full +stereo decoding and catalog-duration checks. The remaining 38 older `.m4a` +recordings comprise five empty files, 14 headers without a readable container, +and 19 decodable files shorter than their catalog duration. Both independent +native downloads retrieved the same legacy bytes. Keep these originals and +their metadata; do not claim a complete export or delete cloud data. + +The `.qta` files contain stereo AAC alongside newer audio tracks. Native macOS +26 validation checked every audio track without playback. Portable stereo +`.m4a` copies can reuse the original AAC packets without re-encoding; retain +the byte-identical `.qta` originals to preserve spatial/layer and other metadata. + +Three pilot files totaling 107,836,545 bytes passed source/host/NAS SHA-256, +full decoding, duration reconciliation, and exact recording-time mtime checks. +The macOS 15 local-file pass found 123 files passing full decoding and catalog +and decoded duration checks; 19 decode but are shorter than their catalog +duration, and 14 nonempty files contain headers without a `moov` atom. The +five zero-byte files are excluded from the portable set. Failed and incomplete +files remain in the native archive, with private per-record validation results. + +The public delivery root `/srv/clover/Share/iCloud Export - 2026-11-04` +contains only `Voice Memos//...`: **310 validated recordings**, delivered +as 310 Windows `.m4a` files and 187 original `.qta` files, totaling +12,163,467,515 bytes. The first 123 recordings remain included. Pilot, +Available, native databases, manifests, and helper folders are absent. +The original `.qta` files preserve every track and embedded field. Their stereo +copies preserve identical AAC packets without re-encoding. Both audio tracks +of every active `.qta` passed native macOS 26 decoding: 187 recordings, +374 tracks. Source/host/NAS hashes and recording-time mtimes match. + +Native files and metadata are outside the share at +`/srv/clover/.icloud-export-private/2026-11-04`, behind a mode-0700 parent: + +- `Native initial snapshot`: 232 source/support files totaling 1,746,210,872 + bytes, including a consistent SQLite backup with all 358 rows. +- `Native macOS 26 snapshot`: 351 files totaling 9,615,002,291 bytes, + including all 350 media files and a consistent 358-row SQLite backup. +- `Portable metadata`: complete recording index, original titles, source and + portable inventories, native validation results, and the recovery index. + +Both native databases pass `quick_check` on Mac and NAS. The existing +password-protected read-only share URL is unchanged. Live HTTP tests verify +an initial file, a new original and stereo copy, recording-time headers, +range requests, and denial of writes and parent access. TAR downloads preserve +whole-second recording mtimes and the `Voice Memos/` layout. The friend +has **not yet tested Windows playback or Date modified**. + +A separate protected recovery share at +`/srv/clover/Share/iCloud Export Recovery - 2026-11-04` contains instructions +identifying the 38 exceptions and an `Uploads` inbox. Its URL is in +`private/recovery-share.json`; it uses the same password as the delivery share. +Instruction names are generated mechanically for the friend and never printed. +Uploads and authenticated downloads pass; deletion and parent access are denied. +The friend checks the originals on the iPhone, exports available recordings as +Rendered audio, and names each upload using the checklist's `recovery-NNN` +identifier. The private recovery index maps that identifier to the original +recording date, catalog duration, and title for later verification and replacement. +The recovery share now also has `Partial recordings//...`: 19 untouched +short originals and 28 reconstructed candidates covering the other 14 nonempty +exceptions. Both 44.1 kHz and 48 kHz headers are retained for those 14 because +the source header is missing. All 47 files pass decoding, NAS hash, mtime, and +live HTTP checks; every reconstructed encoded packet occurs in its untouched +source. These 33 logical partial recordings do not count toward the 310 complete +recordings. Candidate audio rate still needs comparison with the original source. +Never publish incomplete files as successfully recovered recordings. + +The audio sync worker was restarted in this guest after the native snapshot; +counts and bytes remained unchanged. The normal share manager rolled back because its success-text check does not +recognize Copyparty’s current reload response. A read-only password-protected +link was instead created through Copyparty’s native HTTP API for the dated +export root. Unauthenticated reads return 401; authenticated media hashes and +HTTP mtimes match; range requests work; writes and parent access return 403. +The URL and password-file path are in `private/export-share.json`. Evidence remains in the VM's restricted +`evidence/*-nas-validation.json`; private per-file manifests are not operator +readable output. + +## Privacy boundary + +Expose only basic metadata and aggregate integrity results: counts, byte +sizes, dates, durations, and opaque identifiers. Do not display or listen to +media, inspect transcripts or thumbnails, or reveal filenames, recording +titles, GPS coordinates, or other location information in chat or operator +logs. Keep media apps closed during authentication and exclude profile +images and media windows from authentication captures. + +Automated workers may copy, hash, and validate files without exposing their +contents. Preserve original bytes and all metadata, including location data, +inside the friend's restricted archive; withholding metadata from the operator +must not strip it from the export. Names and metadata may be handled +mechanically inside private workers and manifests without returning their +values. The friend performs any visual or listening review on Windows. +These constraints take precedence over manual sample inspection below. + +## Agreed order + +1. Create a fresh macOS VM on this Mac, prove its console and guest control, + then sign into the friend's Apple Account and enable Voice Memos only. + A separate host macOS account remains an alternative if the VM pilot fails. +2. Prove a small memo export, then download and reconcile the full 9 GB, + preserve native sources, and share the dated portable files from Snow Globe. + Check the result on the friend's Windows PC. +3. Trial native Photos exports using the VM's existing Apple session. + Check originals, edits, companions, counts, dates, and cache space before + the full run. The web downloader requires separate authentication and + remains an alternative when the friend is available. + +## Capacity and cleanup + +Read-only measurements on 4 November 2026, in decimal GB and TB: + +| Storage | Used | Available | Allocation | +| --- | ---: | ---: | --- | +| Project | 567.5 GB | Shared below | APFS volume on the 2 TB WD Blue SN5000 | +| Documents | 1183.3 GB | Shared below | Same APFS container as Project | +| Project and Documents container | About 1751 GB | **249.1 GB total** | 2000.2 GB capacity; no volume quotas or reserves | +| Mac startup container | About 895 GB | 99.2 GB | Separate 994.7 GB APFS container | +| Snow Globe personal storage | — | About 7.79 TB | ZFS availability for `/srv/clover` | + +Project and Documents already share free space. Shrinking one APFS volume +does not give the other more space; moving or deleting verified data frees +their common pool. Neither volume currently has APFS snapshots. These are +volumes on a physical disk, not sparse disk images that can be compacted. + +The **9 GB memo run needs no disk cleanup**. For the VM route, reserve about +100 GB of the Documents container's available 249 GB for a 64 GB guest disk, +the restore-image download, and transfer/scratch space. This is a planning +budget from before installation. After the macOS 26 upgrade, backups, and +full memo export, the shared container still has about **134.3 GB free**; +no personal files were removed. A host-account fallback +would instead use the startup disk's available 99 GB for the memo store. + +The current Photos route keeps an optimized local library and sends exported +files to Snow Globe. It reuses the existing Apple login and supplies a native +catalog for reconciliation. Original downloads temporarily consume guest and +host disk space; the trial must establish a workable batch size and cache +reclamation behavior. + +For a full Download Originals library, budget **about 1.85 TB free** in the shared +Project/Documents container: 1.7 TB of photos plus an initial 150 GB allowance +for databases, caches, and temporary files. That means freeing **about 1.60 TB +more** against the pre-VM measurement above. A retained memo VM consumes space +in this same pool, so add its actual allocation and remeasure before cleanup. +The iCloud size is rounded and the allowance is a planning target. Export +directly to Snow Globe to avoid a second full local media set. + +Any eventual cleanup must follow the recovery requirements in +[AGENTS.md](../AGENTS.md). NAS space must then cover both relocated Clover +data and the friend's export, including edited versions. + +## Dedicated VM route + +Apple supports iCloud in macOS VMs created using a macOS 15 or later restore +image on a macOS 15 or later Apple silicon host. Upgrading a VM originally +created with an older OS does not add this support. VM identities are tied +to the host's Secure Enclave; moving a VM or running another clone can require +reauthentication. +[Apple's iCloud virtualization requirements](https://developer.apple.com/documentation/virtualization/using-icloud-with-macos-virtual-machines). + +This host runs macOS 15.7.5 and has 256 GiB RAM. Its existing Tart guest is +the running `gui-agent`, exposed as `m3-devbox`. The installed +Tart 2.41.1 supports fresh IPSW installation and a specified guest disk size. +Apple's Sequoia license, section 2B(iii), permits two additional virtual macOS +instances for the listed purposes, including personal noncommercial use. +[macOS Sequoia license](https://www.apple.com/legal/sla/docs/macOSSequoia.pdf). + +The pilot has four vCPUs and 8 GiB RAM. Its initially 64 GB sparse disk has +been enlarged to 128 GB for the macOS 26 compatibility test. It was created +from Apple's macOS 15.6.1 restore image, build 24G90, with verified SHA-256 +`3d87686b691ac765eb6a6b3082b2334e2af9710096a00432dd519af89ff2ea78`. +The Apple CDN URL and installation evidence are in its private `evidence` +directory. The initial macOS 15.8.2 guest recognized only 161 of 348 as playable. +The guest has now booted macOS 26.7.2 for the compatibility test, matching the +phone’s iOS 26.6.1 generation. + +Reuse the installed signed Tart binary and the control approach documented +in [the devbox runbook](mac-agent-vm.md), with a separate VM state directory +under `/Volumes/Documents/VM/macOS Code Sandbox/icloud-export`. Its existing +ancestor ACL protects this tree on Documents, where ownership is disabled. +Keep its disk, credentials, and transfer directory there. Keep the devbox +guest and its active work separate. A fresh install avoids copying devbox +accounts, credentials, and workspace mounts, and establishes the required +installation provenance; cloning is not categorically incompatible with iCloud. + +[Tart's fresh-install procedure](https://tart.run/quick-start/) begins with +`tart create --from-ipsw IMAGE --disk-size 64 icloud-export`, followed by guest +Setup Assistant. Use the isolated state directory for every command. Keep +only a dedicated transfer folder available to the guest, then copy verified +portable output to the NAS from the host. + +Console access, input, capture, guest storage, and the transfer path have been +proved before Apple sign-in. The existing dashboard bridge selects `m3-devbox` +specifically; it does not automatically control another guest. A dedicated +guest console/control connection can run while Clover stays in the existing +host login. This avoids creating or switching to another host account. +The friend still provides Apple authentication and two-factor approval. +Supported iCloud sign-in is a prerequisite; the pilot must also establish +actual Voice Memos sync, complete audio downloads, and a verified export. + +### Resume this pilot + +The guest has been upgraded to macOS 26.7.2, matching the phone's iOS 26.6.1 +generation. It was cleanly shut down first. APFS clones of the stopped +`disk.img`, `nvram.bin`, and `config.json` remain in `private/pre-tahoe`; all +64 GB of the old disk passed byte comparison. Never run this backup +concurrently or alter its VM identity. + +The guest disk is now 128 GB. Its Recovery partition was copied to the new +end without changing partition identifiers; all 5,368,668,160 Recovery bytes +passed hash verification, and both GPT copies pass CRC checks. Apple's +`diskutil apfs resizeContainer` then expanded the guest system container, +leaving about 97.8 GB free. No host partitions or devbox disks changed. A temporary Virtualization VNC +server was closed before input because it listens on all host interfaces; +normal runs remain headless with no host VNC listener. +`evidence/pre-tahoe-backup.json` and `recovery-relocation.json` record these +checks. Apple Software Update downloaded the 26.7.2 installer, and the in-place guest +upgrade completed and the guest has booted macOS 26.7.2. The macOS 26 desktop is open. A one-boot automatic login bypassed unreliable +Screen Sharing password entry, then its root cleanup job removed the setting, +`/etc/kcpassword`, and its own plist. Automatic login is disabled; the guest now reports FileVault enabled. The local +password, keychain, and Apple Account are unchanged. The one-boot setup helper +was removed to prevent replay; evidence is `one-boot-login-validation.json`. +`sysadminctl`'s prompted form returned error 22, so the temporary credential was +written to its normal root-only file over stdin; no secret appeared in argv or +logs. Executable signatures verify; the installer intentionally +uses legacy resource omission rules rejected by a generic Gatekeeper app +assessment. No security policy was changed. `startosinstall` uses the guest +local volume-owner password over stdin, never command arguments. The completed login is verified by console UID 501, the login-done flag, and +Finder/Dock processes; a GUI launchd domain alone does not establish login. The +lock-state guard uses pinned guest SSH because the login-session Tart agent +is unavailable before console login. + +All machine-specific state is under +`/Volumes/Documents/VM/macOS Code Sandbox/icloud-export`. `state` is its +isolated Tart home; the VM name is `icloud-export`. A symlink in +`~/.tart/vms/icloud-export` names this same guest. The local guest account is +`admin`; its generated password and separate SSH key remain in `private`. +The friend's Apple credentials belong only in `private/apple-account.json`, +with fields `apple_account` and `password`. The guest does not +mount that directory. + +`private/run-vm.py` runs only this guest. Launch it detached through the +existing localhost SSH route so it survives the controlling shell. It mounts +`transfer/tooling` read-only and `transfer/output` read/write. The guest sees +these as `/Volumes/My Shared Files/tooling` and `output`. Host paths outside +these two directories are not shared with it. + +`private/vm-ui.py` takes JSON on stdin for guest screenshots and input. +It uses vncdotool 1.3.0's Apple Screen Sharing authentication over SSH, with +a guest-specific key and pinned host key. Its Unix viewer +socket stays inside `private`. The physical Mac can remain locked. The +temporary Tart installer VNC listener is stopped, its credentials are removed, +and normal pilot starts omit `--vnc-experimental`. The temporary legacy VNC +password was also disabled and removed; the console uses the local account. + +Native Screen Sharing maps `super` to Command. Its `clear` operation uses +Command–A then Backspace; individual character events are paced. A public +test containing upper/lowercase letters, underscore, hyphen, plus, and an +email address was reproduced exactly. The sign-in field and guest clipboard +were cleared after the check. Guest login and console reconnection passed +after the OS update and the final VM restart. Automatic login remains disabled. The native console +controller is now closed; the headless guest desktop remains active and SSH +metadata access remains available. No host VNC listener is enabled. + +For guest commands, use the installed Tart binary with +`TART_HOME="/Volumes/Documents/VM/macOS Code Sandbox/icloud-export/state"` +and `exec icloud-export COMMAND`. The guest agent runs only its command RPC; +clipboard integration is off. SSH offers a second command route and supplies +the private screen-sharing tunnel. Never point the existing devbox installer +or its lifecycle commands at this pilot. + +The synthetic transfer fixture was removed after passing its hash and mtime +checks. Pilot evidence is in `evidence/pilot-nas-validation.json`. + +The iPhone passcode file was deleted after successful authorization. Never +reset encrypted iCloud data. Read credentials only inside the input worker; +never print or display their files. Authentication captures use restricted +regions and mask inputs before writing images. Never capture a media window. + +`private/vm-ui.py --stream` maintains one guest console connection across JSON +lines. Do not infer the guest lock state from the VNC connection status. Verify +it through `ioreg`; automatic login remains disabled. Controller +capture defaults to false. A menubar-only crop can verify the active app +without exposing recording names. Start the controller PTY with `stty -icanon -echo`; macOS canonical input +truncates long JSON lines. Counted Up/Down steps are supported, and counted +Backspace requires the locked-guest guard. Explicit paced Shift combinations +handle the US keyboard layout without exposing credentials. + +Voice Memos' migrated local store was preserved intact at guest +`/Users/admin/iCloudExportRecovery/Recordings-before-refetch`. A fresh +`Recordings` store has imported all 358 catalog rows. Its first stable pass +retrieved 103 media paths, including 18 `.qta` recordings absent from the +macOS 15 store. This changes only the local cache, not iCloud records. +The fresh assets require native decoding/export for formats unsupported by the +host FFmpeg; a decode failure alone does not establish source corruption. +Launch GUI apps through the guest agent in the completed desktop session. +OSLaunchdErrorDomain 125 was observed before console login completed. +Keep all credential input on stdin. The optional post-upgrade assistant was +closed without approving extra cloud services. + +`private/memo-export.py` reports aggregate status without arguments; `recovered` +copies and validates stable files from the current native store. Selection uses +the native playable predicate, including sub-0.1-second recordings. Existing +validation is reused only for identical hashes, recording times, and catalog +durations. Initial `pilot` and `available` inventories are historical evidence. +`native-audio-check.py` validates every QTA audio track with the native decoder; +PCM scratch is discarded after checking frame counts and duration. +`build-portable.py` preserves original media and builds packet-identical stereo +copies. `publish-macOS26.py` has completed its sealed native snapshot publication; +do not replay it against the existing native destination. Future updates must +preserve the flat public layout and private recovery material. Report 310 of +348 complete recordings until the remaining 38 reconcile. + +### Photos without another Apple login + +The friend is unavailable for another verification code. The separate web-login +worker has stopped, its pending request was denied in the VM, and its unused +NAS password copy and cookies were removed. No additional code was requested. +Use Photos in the already signed-in VM with Optimize Mac Storage, exporting +requested originals and edits in batches through native Photos/PhotoKit or +OSXPhotos. +Photos is now enabled in that same Apple session and has begun importing its +catalog. Optimize Mac Storage is selected and independently verified through +its native settings control. Keep the library on the guest's local APFS disk +and send exported files to Snow Globe. Apple advises against putting a Photos +library on a network share. + +The catalog holds 38,142 assets: 37,804 visible (1,823,889,190,154 original +bytes; 4,544 videos, 1,348 edited, 363 Live Photos), 303 hidden burst members, +and 35 in Recently Deleted. No shared albums exist. OSXPhotos 0.78.0 runs in the +guest from `~/.local/bin` (installed with a checksum-verified uv 0.9.7). + +PhotoKit needs Photos access from a responsible app with a usage string; CLI +processes under SSH or Terminal are refused or time out before the prompt can be +answered. `transfer/tooling/PhotoExport.app` is an ad-hoc-signed launcher +(`net.paperclover.icloud-export.photos`) that spawns its arguments, and holds +full Photos access in the guest. Terminal also received access while this was +diagnosed. Prompts are `UserNotificationCenter` alerts; answer them with +`run_steps` by button name, never with screenshots. + +Guest `~/PhotoExport` holds the private inventory, 110 date-ordered UUID batches +(about 15 GB each; `b0000` is a 30-asset pilot), and per-batch logs, reports, +and export databases. `driver.py` (from `private/photos-driver.py`), started by +`open -n ~/Applications/PhotoExport.app --args /bin/sh ~/PhotoExport/run.command`, +exports each batch with `--download-missing --use-photokit`, year directories, +`{date}_{uuid}` names, XMP sidecars, and `--touch-file`, then repeats it once +with `--update`. It keeps at most one unshipped batch and pauses below 25 GB +free; `~/PhotoExport/STOP` halts it between batches. On the host, +`private/photos-ship.py` streams each finished batch to Snow Globe, verifying +every file's size, SHA-256, and mtime. Media goes to `Photos/` in the +delivery share; sidecars, reports, and export databases go to the private +`Photos metadata/`. Only then is guest staging deleted. Logs: +`evidence/photos-ship.log` and guest `driver.log`. + +The pilot exported 223 files (30 assets and their burst members) with no +missing or failed items, and passed mtime, year-folder, and file-signature +checks before shipping. + +Downloaded originals stay in the Photos cache until space pressure. In this +guest the per-user `deleted` (CacheDelete) agent had a stale volume object +("unable to validate volume"), so nothing was purgeable and Photos never evicted, +even at 3 GB free. Killing that agent, then quitting and reopening Photos, made +the cache purgeable; pressure at about 6 GB free then evicted 9.5 GB within +seconds. Before each batch below 40 GB free, the driver preallocates a temporary +balloon down to 6 GB, waits 30 s, and deletes it. If purges stop freeing space, +check `NSURLVolumeAvailableCapacityForImportantUsageKey` and the `deleted` log first. + +A full local download of the 1.7 TB is not a prerequisite for this optimized +route. Originals are temporarily downloaded by Photos as exports request them; +Photos manages eviction. Cache reclamation is asynchronous, so verify bounded +space use in a trial, monitor free space, and pause before exhausting the guest +or host disk. Do not use OSXPhotos `--cleanup` to evict cloud originals: it +removes files from the export destination. +[Apple Photos and iCloud](https://support.apple.com/en-euro/guide/photos/phtf5e48489c/mac) · +[OSXPhotos maintainer on optimized storage](https://github.com/RhetTbull/osxphotos/discussions/1926) · +[Apple library-storage requirements](https://support.apple.com/en-us/108345). + +The alternative web downloader remains installed privately on Snow Globe: +upstream `icloudpd` 1.33.0 at +`/srv/clover/.icloud-export-private/2026-11-04/tools/icloudpd-1.33.0`, pinned to +commit `3caeae584c3d8bc6fcc51a1c9f7e5413b1422fd3`. It requires its own Apple web +session. Keep its cookies and credentials private; leave ADP enabled. It is +not the current route while the friend cannot supply another code. + +### Remaining memo recovery checks + +The cached CloudKit records use LZFSE/LZVN compression. After decompression, +all 38 exception assets' recorded sizes match the local files, including the +five empty files. Their latest cloud modification predates this export: +24 February 2026. Each has one audio-future generation in the cached index. +No additional Voice Memos audio was found in its app/group stores. Native Core +Audio decoding of all 33 nonempty exceptions recovers 19 short streams but +none of the full catalog durations; the other 14 fail native decoding too. +Some short files contain additional `mdat`/`moof` fragments, retained for repair +experiments. The audio was never uploaded. The 38 cloud assets hold 1,221,488 bytes +for 9.7 catalog hours, and each record's own local duration is 0–22 seconds, +whereas healthy recordings match their catalog duration. No Mac-side or iCloud +copy can supply the rest; only the iPhone (or a local backup of it) can, if its +copy is complete. A normal app export has now been attempted for all 38: all 19 short files +remain short, 18 supply no audio, and one empty item cannot be copied through +the UI. Repair with a pinned build of `anthwlock/untrunc` produced decodable +fragments for all 14 header-only files (16.831 seconds across their longest +candidates), but no full catalog durations. The original 19 short files remain +better than their reconstructed variants and are retained untouched. +Evidence is `recovery-cloud-metadata-validation.json`, with private atom, +packet, and native-decoder inventories. Never display media, names, or locations. + +## What each person does + +Once the export login is controllable, the agent can configure the export, +oversee sync and retries, read the authorized memo database, restore names +and dates, verify files, and manage the restricted NAS delivery. + +Guest installation and its local account can be prepared from Clover's +existing host session. The separate host-account alternative requires Clover +to authenticate account creation: the host shell runs as `clo` without +passwordless sudo. + +The friend keeps the subscription active, enters Apple credentials and any +encrypted-data authentication, approves two-factor requests on the iPhone, +checks source counts, and tests Windows playback. Remote sign-in needs both +credentials and two-factor approval; approving a notification is insufficient +on its own. + +Current Cua Driver checks pass for Accessibility, Screen Recording permission, +AX access, and the WindowServer connection in Clover's login. The locked host +desktop refused GUI control; the pilot's independent guest console works. +For the host-account alternative, first login and +the GUI controller's access must be established in that login, including any +per-user permissions. Keep the existing Knife session available in Clover's +account. A host account switch must not log it out. +These setup and authentication steps can be done through a trusted remote +connection while the Mac is powered on and reachable; they do not inherently +require a visit to the machine. + +Use the restricted credential file or a private remote-control session into +the export VM for password entry. Device-passcode entry and two-factor +approval may still require the friend. Credentials, recovery keys, and verification codes stay out of +chat, shell arguments, repository files, and export logs. The temporary Mac +becomes an authorized device with access to the account; the friend should +understand that before signing in. + +## Before downloading + +1. Confirm the friend's iOS version and whether Advanced Data Protection is + enabled. The requested storage split is 9 GB of memos and about 1.7 TB of + Photos; other account categories are outside the export. +2. Record Photos' photo/video counts with filters cleared. Inventory Hidden, + personal and Shared Photo Libraries, Shared Albums, bursts, RAW pairs, + edited media, and anything wanted from Recently Deleted. Do the same for + Voice Memos, including custom folders and Recently Deleted. Do not restore + deleted items without the friend's choice. +3. Confirm both Photos and Voice Memos are enabled in iCloud on the iPhone + and have finished uploading. Phone-only items cannot be fetched from + iCloud. Resolve upload errors on the source first. +4. Agree how future photos and recordings will be protected after migration. + A one-time export does not upload tomorrow's recordings. + +## Host account alternative + +Use a new standard user dedicated to this export, with its normal home under +`/Users` on the startup disk. The 9 GB memo store does not justify a custom +home directory or another APFS volume. Clover authenticates account creation +through the Mac's administrator prompt; do not put passwords in command +arguments. Test first login and the account's GUI control before sign-in. + +Sign into the friend's Apple Account in that user, leaving Clover's normal +account signed in separately. Review the iCloud app switches during setup and +enable Voice Memos only. Keep Photos and Drive sync off for this phase. Leave +Find My, Passwords and Keychain, Messages, and purchase-account sign-in out +of this setup. Decline merging unrelated local data. + +Only for the native Photos fallback, create the friend's Photos library in +a private folder on the existing Project/Documents APFS disk after freeing +space. Make it that user's System Photo Library, enable iCloud Photos, and +select **Download Originals to this Mac**. Keep the Mac powered, awake, and +connected during downloads. Do not put the live `.photoslibrary` on the NAS: +Apple requires appropriate local Mac-formatted storage and warns against +network storage for live libraries. +[Apple's library storage requirements](https://support.apple.com/en-us/108345). + +A cable import would require the iCloud originals to be downloaded onto the +iPhone first. It is therefore unsuitable for this 1.7 TB photo library; a cable +copy does not inherently mean all embedded camera metadata is lost. +[Apple's PC import requirements](https://support.apple.com/en-us/120267). + +Layered Voice Memos require at least macOS 15.2. The initial macOS 15 guest +proved insufficient for this catalog; the phone runs iOS 26.6.1, and a macOS 26 +compatibility test is underway. +[Apple's Voice Memos playback requirements](https://support.apple.com/en-euro/guide/voice-memos/vma2c8c0a040/mac). + +## Prove a small export first + +For the memo pilot, test ordinary, renamed, trimmed, long, and layered +recordings where present. Include a recording close to New Year or a timezone +boundary. + +For the subsequent Photos trial, test an old and a new photo, duplicate +original filenames, an edited picture, a Live Photo, a video, a burst, a RAW +pair if present, a hidden item, and a date corrected in Photos. + +The friend opens the pilot on Windows and checks appearance, sound, edits, +duration, filenames, and dates. An original plus its edited rendition are two +intentional versions. A Live Photo's image and video must both survive. +Full-resolution originals and all companions must exist; thumbnails and a +matching item count are insufficient. + +## Native Photos fallback + +Use [OSXPhotos](https://rhettbull.github.io/osxphotos/index.html) to export the +downloaded library. Its year and UUID templates give ordinary directories +without filename collisions. Keep original names and album membership in +metadata rather than duplicating media into every album. +[Template reference](https://rhettbull.github.io/osxphotos/template_help.html). + +Run the export as the operator with read access to the friend's downloaded +library and write access to the destination. Using Clover's operator account +for this phase keeps NAS credentials outside the temporary Apple login. +Grant access only to the agreed source tree; macOS privacy permissions may +also be required. Always pass the explicit library path. Do not let the tool +fall back to Clover's own Photos library. + +Example paths below are placeholders. Confirm the NAS mount and its backing +host before creating the destination. Keep export state on the Mac; only +validated portable media belongs in the public share. Keep catalog metadata +and native recovery files in restricted staging outside it. + +```sh +uv tool install 'osxphotos==0.78.0' + +LIBRARY='/Volumes/Documents/iCloud Export/Photos Library.photoslibrary' +WORK="$HOME/Library/Application Support/iCloud Export" +mkdir -p "$WORK/photo-staging" +chmod 700 "$WORK" + +osxphotos info --library "$LIBRARY" --json > "$WORK/photos-info.json" +osxphotos query --library "$LIBRARY" --json > "$WORK/photos.json" + +osxphotos export "$WORK/photo-staging" \ + --library "$LIBRARY" \ + --directory '{created.year}' \ + --filename '{created.strftime,%Y-%m-%d_%H%M%S}_{uuid}' \ + --touch-file \ + --sidecar xmp \ + --keyword-template '{folder_album(>)}' \ + --exportdb "$WORK/photos-export.db" \ + --update \ + --report "$WORK/photos-export.csv" +``` + +Capture dry-run and tool diagnostics privately; return only aggregate progress +and integrity results. Never display private paths or media metadata. Add +`--limit 30` for a pilot, then remove the limit for the full export. Retain the +export database and options for retries and the final update. Keep dated +copies of reports before subsequent runs replace them. Defaults export +originals, edited versions, Live Photo videos, burst members, and associated +RAW images; edited files receive an `_edited` suffix. `--touch-file` sets the +media mtime from the Photos date. Do not enable overwrite or cleanup options. +[Export command reference](https://rhettbull.github.io/osxphotos/cli). + +Wait for full hydration and inspect missing/error reports. For exceptions, +OSXPhotos has `--download-missing`, but its documentation says this does not +retrieve every burst member. Resolve every missing companion explicitly. +For network exports, a local export database avoids putting its SQLite writes +on SMB. XMP sidecars preserve catalog metadata without rewriting original +media bytes; the JSON inventory keeps the broader catalog information. Retain +these sidecars and inventories outside the public media-only share. +[Export and metadata tutorial](https://rhettbull.github.io/osxphotos/tutorial.html). + +Apple's built-in fallback is **File → Export → Export Unmodified Original** +with **Export IPTC as XMP**, plus a separate export of edited versions. A Live +Photo exports as an image and a video. This fallback still needs inventory +matching and timestamp correction for the proposed layout. +[Apple's export procedure](https://support.apple.com/en-gb/guide/photos/pht6e157c5f/mac). + +## Export Voice Memos + +The database contains metadata; audio lives in separate files. Enable Voice +Memos sync in the temporary user, open the app, and reconcile its recordings +with the iPhone. A listed recording is not proof its audio is downloaded. +[Apple's sync procedure](https://support.apple.com/en-ie/guide/voice-memos/vma6cc4d0571/mac). + +The current common store is: + +```text +~/Library/Group Containers/group.com.apple.VoiceMemos.shared/Recordings/ + CloudRecordings.db + CloudRecordings.db-wal when present + CloudRecordings.db-shm when present + media and supporting files +``` + +Preserve a private copy of the entire native store, including editing and +supporting files. Use SQLite's backup API through a read-only source connection +to obtain a consistent database snapshot including committed WAL data. Copying +only the live `.db`, or opening it with `immutable=1`, can miss recent changes. +Quiesce app edits during the copy and reconcile again afterward. +[SQLite's backup API](https://www.sqlite.org/backup.html). + +Inspect the copied schema before implementing extraction. Published exporters +use `ZCLOUDRECORDING`, `ZUNIQUEID`, `ZPATH`, `ZDATE`, and `ZDURATION`; one +documents `ZEVICTIONDATE` for deleted state. These are reverse-engineered +details, so verify their meaning against this account's app views. Titles can +occupy different fields; preserve them mechanically in private metadata without +displaying them. Native `.m4a` and `.qta` files both occur. +[Exporter schema and SQLite handling](https://github.com/grapeot/voice-memos-skill/blob/master/docs/rfc.md). + +The extraction contract is: + +```text +for each wanted logical recording in the database: + require a local, nonempty, decodable media file + reconcile duration, active/deleted state, and recording ID + preserve the original title privately without operator inspection + recording instant = ZDATE + 978307200 seconds since Unix epoch + copy native media without changing its bytes + export a rendered copy when edits/layers/effects need rendering + name portable output YYYY-MM-DD_HHMMSSZ_unique-id.ext + set mtime from the recording instant + record ID, full title, date, duration, folder, variant, and output path +``` + +The epoch conversion applies only after confirming `ZDATE` uses Apple's 2001 +epoch in this store. Use UTC for memo filenames and year folders if historical +recording timezones are unavailable; preserve any known timezone separately. +Opaque UUID filenames avoid Windows-invalid characters and private titles or +locations in paths. Preserve full titles privately. Do not merge duplicate +titles or infer dates from download times. + +An ordinary native M4A is suitable when its playback matches the app. Compare +trimmed recordings too: raw file copying alone must not silently discard edits. +For special recordings, export both the native editable source and a rendered +M4A. Apple documents rendered sharing as mixing effects and layers into one +file. The remote friend can select recordings on their iPhone, choose **Share +→ Options → Rendered**, save a manageable batch locally, and upload it through +a temporary folder-scoped HTTPS upload link. Export editable versions too when +the source is not already preserved on the Mac. No physical proximity is +needed; do not ask the phone to download the entire photo library. +[Apple's multiple-recording sharing options](https://support.apple.com/en-mt/guide/iphone/iph3d6dc359/ios). + +Match these uploads to the database inventory before renaming or assigning +dates; resolve duplicate titles explicitly. Export existing transcriptions as +text when wanted, and retain their native source. Do not generate replacement +transcripts as part of copying recordings. + +## Windows files and dates + +The agreed public layout contains media only. Photos is added after its trial: + +```text +iCloud Export - 2026-11-04/ + Voice Memos/ + 2025/ + 2025-06-12_212233Z_.m4a + 2026/ + 2026-06-12_212233Z_.m4a + 2026-06-12_212233Z_.qta + Photos/ + 2024/ + 2024-06-12_142233_.heic + 2024-06-12_142233_.mov + 2024-06-12_142233__edited.jpeg +``` + +Native sources, catalog snapshots, names, album mappings, sidecars, validation +manifests, and recovery tools stay outside the shared root. No Pilot, Available, +Metadata, native-store, or tool folders belong in the public delivery. + +Preserve HEIC, JPEG, RAW, MOV, MP4, and native audio formats. HEIC/HEVC playback +depends on the Windows apps and installed codecs; test the actual PC. +[Microsoft's HEIF and HEVC guidance](https://support.microsoft.com/en-us/onedrive/upload-heif-and-hevc-photos-and-videos-to-onedrive). +For audio and video, [VLC](https://www.videolan.org/vlc/features.html) is a +useful pilot player. If needed, add separate JPEG or H.264 viewing copies with +the same asset IDs and dates. Preserve the originals; JPEG, stereo mixes, +and video transcodes do not retain every RAW, HDR, depth, or spatial property. +Measure derivative space before converting a whole library. + +Use the Photos catalog's creation date and timezone for photo folders and +mtime, including dates the friend corrected in Photos. Preserve the original +embedded EXIF even when it disagrees; XMP and the inventory carry catalog +corrections. Mtime is a UTC instant, while Windows displays it in the viewer's +timezone. Windows Date created is a separate field and may reflect download +time. Other documents retain their actual modification dates. + +In the restricted recovery store, create a manifest record for every delivered +file with `relative_path`, +`bytes`, `sha256`, and `mtime_utc` as ISO 8601 with an explicit UTC offset. +Link media to logical IDs and variant roles in the inventories. Exclude the +manifest itself from its own listing. Keep paths relative and check case +collisions and Windows path limits. Generate and verify the manifest only +after files and metadata are final. + +The public share’s TAR download preserves UTC recording timestamps to whole +seconds in both archive metadata and a real extraction test. Its URL is the normal share URL +plus `Voice%20Memos/?tar`; the archive contains `Voice Memos//...` only. +Copyparty’s generated ZIP carries UTC DOS dates and a legacy Unix extra field +that the tested extractor did not honor, shifting extracted mtimes. Use TAR +for the handoff and have the friend verify the resulting Windows dates. +Private evidence: `evidence/tar-time-validation.json`. + +Individual browser downloads may reset filesystem timestamps. Prefer the +verified TAR route and check the friend's actual Windows extraction. Downloads +should use manageable batches or individual-file range requests when necessary. + +## Snow Globe sharing + +Place portable output under `/srv/clover/Share/iCloud Export - 2026-11-04`, visible +as `/Volumes/clover/Share/iCloud Export - 2026-11-04` on this Mac. Use the existing +[Shares manager and Copyparty](copyparty.md), with a password and read-only +permission scoped to that folder. Keep native app databases, credentials, and +temporary writable uploads outside the read-only delivery link. Retain existing +numeric owners; do not recursively change ownership of Clover's files. + +Create the delivery link after validation. Test its access from a signed-out +client, including denial of parent and sibling folders, a large download, +resume behavior, and the Windows timestamp procedure. For remote uploads of +exceptional memos, use a separate restricted upload folder and revoke that +access after reconciliation. This uses existing services and needs no new +permanent iCloud container. + +## Direct Photos trial + +[The linked project](https://github.com/boredazfcuk/docker-icloudpd) wraps +`icloudpd` in an Alpine container, adding scheduled downloads, credential and +session storage, notifications, and optional conversions. Its scope is Photos +and videos, including Live Photo components; it does not export Voice Memos +or the whole account. + +Its README still says ADP must be disabled. In contrast, current upstream +`icloudpd` documents `--adp-web-access`, introduced in 1.33.0, with approval +from a trusted device and iCloud web access enabled. This is an upstream +capability, not proof the wrapper image supports that option. Use a pinned +version and prove the actual account flow; the plan does not require disabling +ADP. +[Upstream authentication documentation](https://icloud-photos-downloader.github.io/icloud_photos_downloader/authentication.html). + +For a direct-to-NAS pilot, run upstream in one-time copy mode with private +authentication state. Retrieve original and adjusted assets, RAW alternatives, +and Live Photo components from every wanted library; enumerate Shared Albums +separately. Reconcile asset IDs and all variants. Shared Albums can contain +reduced-resolution copies; obtain originals from their owner where needed. +Regular album membership is not a reason to store the same original repeatedly. +[Upstream features and modes](https://github.com/icloud-photos-downloader/icloud_photos_downloader). + +With upstream **1.33.0**, pass these trial options to the parser inside the +private worker. Account fields come from the restricted file into process memory; +keep credentials out of OS command arguments and keep all tool output private. +The sample and sidecars stay outside the delivery share until verified. +`--recent 30` is only for the trial; remove it for the full run and final +reconciliation. Repeat for each library returned by `--list-libraries`, with a +distinct destination per library. + +```sh +--adp-web-access \ + --cookie-directory '/srv/clover/.icloud-export-private/2026-11-04/photos-session' \ + --directory '/srv/clover/.icloud-export-private/2026-11-04/Photos trial' \ + --folder-structure '{:%Y}' \ + --file-match-policy name-id7 \ + --live-photo-mov-filename-policy original \ + --size original --size adjusted --size alternative \ + --live-photo-size original \ + --xmp-sidecar \ + --log-level error \ + --recent 30 +``` + +The repeated sizes request originals and available edits/RAW alternatives. +The ID naming policy distinguishes repeated original filenames. Verify all +IDs and variants, reported creation instants, mtimes, XMP contents, and year +boundaries before accepting this route. Preserve the downloaded names unless +the inventory supplies an unambiguous mapping to a new layout. +[Asset sizes](https://icloud-photos-downloader.github.io/icloud_photos_downloader/size.html) · +[File naming](https://icloud-photos-downloader.github.io/icloud_photos_downloader/naming.html) · +[Command reference](https://icloud-photos-downloader.github.io/icloud_photos_downloader/reference.html). + +Keep cloud-deleting and local-mirroring features disabled: `auto_delete`, +`delete_after_download`, `keep_icloud_recent_only`, and `delete_accompanying`; +leave `keep_icloud_recent_days` unset. The wrapper configuration warns about +filename collisions with flat or year-only download directories. Upstream +documents automatic name de-duplication, so verify the pinned behavior with +duplicate names instead of assuming either statement describes every build. +Normalize to the final year/ID layout only after inventory reconciliation. +Do not rewrite EXIF on archived originals. +[Wrapper configuration](https://github.com/boredazfcuk/docker-icloudpd/blob/master/CONFIGURATION.md). + +## Finish before canceling + +1. Reconcile every wanted logical item and its expected files, including edits, + RAW pairs, bursts, Live Photos, hidden items, shared content, and special + recordings. Resolve all missing, empty, corrupt, and failed exports. +2. Probe all audio/video files and decode-check images; inspect representative + playback on Windows, including long videos and trimmed or layered memos. + Have the friend compare their important items with the source app. +3. Verify every delivered file's size and SHA-256 on Snow Globe and on the + friend's Windows copy. Check all media mtimes against the inventories, + allowing only the destination filesystem's timestamp precision. +4. Take a final incremental pass after a short pause in new recordings and + edits, then reconcile new items and regenerate manifests. Keep the native + Mac stores until handoff succeeds. +5. Establish independently verified copies on separate devices before cloud + deletion. With the native route, retain the Mac source, NAS delivery, and + friend's Windows copy through the deletion. ZFS redundancy is not another + copy, and this repository does not configure automatic snapshot retention. + Before destructive storage cleanup, satisfy [AGENTS.md](../AGENTS.md): + verify application/data recovery and any needed keys from two restricted + off-server copies stored separately. Apply this to relocated Clover data + too; do not count an unverified NAS move as permission to erase its source. +6. The friend chooses when to stop syncing, remove cloud content, and downgrade. + Confirm remaining use fits the chosen plan and that future backups have a + destination. Never use the downloader to delete automatically during export. + The screenshot also shows 35.7 GB of backups; exporting the two media + categories does not by itself make the account fit the free 5 GB tier. +7. Sign out inside the export VM or fallback host account. Remove the exact + export device from the friend's trusted devices and revoke downloader + sessions and temporary upload access. Remove the export VM or temporary + host account and native stores after the verified handoff and the recovery + requirements above are satisfied. + +**Deleting synced media can delete it on the friend's other devices.** Apple +also warns that exceeding the downgraded capacity stops iCloud updates and +backups. Keep cancellation and cloud cleanup after verification. +[Synced deletion behavior](https://support.apple.com/en-us/108922) · +[Downgrade consequences](https://support.apple.com/en-us/108318). + +## Planning verification + +The Mac version, physical disks, APFS allocation, volume quotas/reserves, +snapshot lists, and Snow Globe ZFS availability were read without opening +personal content. OSXPhotos 0.78.0 was run in an isolated temporary tool +environment for `export --help`, `query --help`, and `info --help`; the flags +above exist in that build. Upstream icloudpd 1.33.0 was also checked with +`--help` in isolation, including ADP, asset-size, naming, and XMP options. +The current Cua Driver is 0.38.0; its status and read-only health checks pass +in Clover's login. Passwordless host sudo is unavailable. The devbox was +observed running with Tart 2.41.1; no lifecycle action was taken on it. +Apple's iCloud requirements and license were checked for the VM route. +The dedicated guest has passed command, screen capture/input, read-only +tooling mount, writable transfer, AAC decode, and NAS hash/mtime checks. +Apple sign-in, native sync, the friend's memo schema, and NAS byte/timestamp +checks have passed. Full library reconciliation and the friend's Windows +playback/date check remain required; Clover reports the Windows test has not +yet been performed. diff --git a/tools/router.py b/tools/router.py index c884ff71dd3591c9f32d68e84d10e0ba4333fda9..113adcfb37a50a2b7b3277ac696a21a991be5306 100644 --- a/tools/router.py +++ b/tools/router.py @@ -40,55 +40,6 @@ def proxy(upstreams, indent, uncompressed=False, host=None, keep_streams=False): return [*lines, f"{indent}}}"] -def cliproxy_routes(host, upstreams, auth, keys, role): - native = [" request_header -Cookie", *proxy(upstreams, " ", keep_streams=True)] - if not role: - return [" @dashboard path / /management.html /sign-out /snow.oauth2/*", - " respond @dashboard 404", " request_header -Cookie", *proxy(upstreams, " ", keep_streams=True)] - if any(not re.fullmatch(r"[0-9a-f]{64}", keys.get(name, "")) for name in ("api_key", "management_key")): - raise ValueError("missing CLIProxyAPI gateway credentials") - return [ - " route {", - " handle /snow.oauth2/* {", *proxy(auth, " "), " }", - " @credentials {", - " not path / /management.html /sign-out /snow.oauth2/*", - ' expression `{http.request.header.Authorization} != "" || {http.request.header.X-Management-Key} != "" || {http.request.header.X-Api-Key} != "" || {http.request.header.X-Goog-Api-Key} != ""`', - " }", - " handle @credentials {", *native, " }", - " @session path / /management.html /sign-out /v0/management/* /v8/management/* /v1/models", - " handle @session {", - f" reverse_proxy {auth} {{", - " method GET", " rewrite /snow.oauth2/auth", - " header_up X-Forwarded-Method {method}", " header_up X-Forwarded-Uri {uri}", - " @unauthorized status 401", - " handle_response @unauthorized {", - " @dashboard path / /management.html /sign-out", - " route {", - " redir @dashboard /snow.oauth2/sign_in?rd={scheme}://{host}{uri}", - " respond 401", " }", " }", - f" @allowed header X-Auth-Request-Groups *role:{role}*", - " handle_response @allowed {", - " method {method}", " rewrite {uri}", - " @foreign_origin {", " header Origin *", f" not header Origin https://{host}", " }", - " @unsafe_origin {", " method POST PUT PATCH DELETE", f" not header Origin https://{host}", " }", - " route {", - " respond @foreign_origin 403", " respond @unsafe_origin 403", - " redir / /management.html 302", - f" redir /sign-out /snow.oauth2/sign_out?rd=https%3A%2F%2Fsnowglobe.{os.environ['SNOWGLOBE_DOMAIN']}%2Fsign-in 303", - " @management path /v0/management/* /v8/management/*", - f' request_header @management Authorization "Bearer {keys["management_key"]}"', - " @models path /v1/models", - f' request_header @models Authorization "Bearer {keys["api_key"]}"', - " request_header -Cookie", - f" reverse_proxy {upstreams} {{", - " lb_try_duration 5s", " stream_close_delay 8h", - " header_down -Access-Control-Allow-Origin", ' header_down Cache-Control "no-store"', - " }", " }", " }", - " handle_response {", " respond 403", " }", " }", " }", - " handle {", *native, " }", " }", - ] - - def shale_mcp_routes(port): return [" @shale_mcp_link path_regexp shale_mcp_link ^/-/snowglobe-mcp/([A-Za-z0-9_-]{43})$", " handle @shale_mcp_link {", " rewrite * /oauth/shale/link/{re.shale_mcp_link.1}", @@ -337,12 +288,6 @@ def render(token): service = next(iter(route["services"])) if not re.fullmatch(r"[a-z][a-z0-9-]*", service): raise ValueError(f"invalid service name: {service}") - if service == "cliproxy" or re.fullmatch(r"cliproxy-preview-[0-9a-f]{8}", service): - with open(os.path.join(ROUTE_DIR, service + ".json")) as source: - keys = json.load(source).get("cliproxy", {}) - lines += [*cliproxy_routes(host, " ".join(sorted(route["upstreams"])), - " ".join(sorted(auth_upstreams)), keys, role), "}"] - continue if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): # Keep the Origin guard before every static/proxy handle; # otherwise Caddy sorts those handles ahead of respond. diff --git a/tools/snowglobe.py b/tools/snowglobe.py index 7d84ed0d4dd3a7b94d52b98194947faa661a93e0..18801a587db22acf0de3e79d59c0b8f47832a658 100644 --- a/tools/snowglobe.py +++ b/tools/snowglobe.py @@ -626,8 +626,6 @@ def provision(data): write_private(route_dir / f"{data['id']}.json", json.dumps({ "files": static, "dirs": directories, "identity": identity, "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, - **({"cliproxy": {name: values[name] for name in ("api_key", "management_key")}} - if data["sourceId"] == "cliproxy" else {}), })) diff --git a/tools/substation.md b/tools/substation.md new file mode 100644 index 0000000000000000000000000000000000000000..3e6bb4170f02d79c52af36821fea70375d82e053 --- /dev/null +++ b/tools/substation.md @@ -0,0 +1,96 @@ +# Substation + +`https://substation.ai.paperclover.net` serves the Substation dashboard and its +native APIs. Snow OAuth protects the dashboard and requires `infra-admin`. +Substation's trusted-proxy mode accepts the identity header from the local +OAuth proxy. Hub requests authenticate with Substation's own keys. + +## Set up accounts + +1. Open the dashboard and sign in through Snow. +2. Open **Settings → Hub accounts** and add a Claude or Codex account. +3. Follow the provider's sign-in link. For Codex, enter the displayed device + code at the provider's page; for Claude, paste the returned code into + Substation. Repeat for each account. +4. Check that each account shows its quota and can take requests. + +The container includes both providers' login tools. Login scratch directories +are disposable; the hub saves and refreshes the resulting account files in its +own store. Production starts empty and imports no CPA accounts or history. + +## Connect clients + +The client key is saved on Sandwich at +`~/.config/substation/client-key`, readable only by Clover. SSO mode deliberately +does not reveal or rotate keys through Substation's dashboard. Nomad variable +`nomad/jobs/substation` owns the initial `api_key` and `management_key`. + +For Claude Code: + +```sh +export SUBSTATION_CLIENT_KEY="$(cat ~/.config/substation/client-key)" +export ANTHROPIC_BASE_URL=https://substation.ai.paperclover.net +export ANTHROPIC_AUTH_TOKEN="$SUBSTATION_CLIENT_KEY" +claude +``` + +For Codex, use a separate `CODEX_HOME` to keep its normal sign-in and settings: + +```sh +mkdir -p ~/.codex-substation +``` + +Save this as `~/.codex-substation/config.toml`: + +```toml +model = "gpt-6.1-sol" +model_provider = "substation" + +[model_providers.substation] +name = "Substation" +base_url = "https://substation.ai.paperclover.net/v1" +env_key = "SUBSTATION_CLIENT_KEY" +wire_api = "responses" +``` + +Then run: + +```sh +export SUBSTATION_CLIENT_KEY="$(cat ~/.config/substation/client-key)" +CODEX_HOME="$HOME/.codex-substation" codex +``` + +The [official OpenAI configuration documentation](https://learn.chatgpt.com/docs/config-file/config-advanced) +describes custom providers. Choose a model the connected account supports. +Substation serves Anthropic Messages and OpenAI Responses; it does not implement +`/v1/chat/completions`. + +T3 Code's hub quota integration uses the same base address and the separate +management key from `~/.config/substation/management-key` on Sandwich. The +management API is `/v0/management`; it remains behind its own bearer key. The +public reverse proxy reaches the hub over loopback, so the hub's remote +management switch is unnecessary. A client key grants no management access. + +## Persistence and deployment + +Production state lives under `/srv/prod/substation/data`: `auth` holds provider +accounts, `hub` holds routing state, `dashboard` holds SQLite history and +settings, `home` holds the login tools' home, and `run` holds build identities. +Nomad owns the generated keys and OAuth client credentials. Existing numeric +service identities remain reserved when a service is retired. + +`service/substation/build-source.json` pins a local Jujutsu revision of +`../substation`. That checkout's `Dockerfile.snowglobe` builds the program and +includes pinned Claude and Codex login tools. The exporter includes its source +manifest as build provenance. Snow Globe receives source and builds locally; +it receives no GitHub SSH key or repository credentials. + +Stage with `python3 tools/deploy.py stage substation`. Previews start with empty +storage, their own keys, and a separate Snow OAuth client. Updates preserve +that preview's state. Use disposable accounts and prompts when testing. +Production uses described, conflict-free main and `python3 tools/deploy.py prod`. +The checkout-dependent updater is unavailable inside the immutable image. + +CPA's live dataset, preview, keys, and routing are retired after the replacement +passes production checks. Restricted off-server recovery copies retain the old +store; restoring them requires a separate explicit request.