authoravatarclover caruso<git@paperclover.net>2026-11-04 11:52:40-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
logc742fe629c76859587fee2f9bb699800813fec62
treebda65f5bef037088c70dec491d8a6bf35a391327
parent34decd157cba7bd22c0b167fb47fb5b1f998a24c
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Replace CLIProxyAPI with Substation

Build the private pinned source on Snow Globe without GitHub credentials. Serve the dashboard through Snow OAuth and native inference/management APIs through independent Substation keys at substation.ai.paperclover.net. Start with fresh account and history stores; allow long inference replies. Opt in the source manifest for build provenance, preserving other build contexts. Remove CPA key injection, service assets, tests and runbook. Document account setup and Claude/Codex/T3 client configuration. Verified 61 Linux runtime suites, nine deployment export tests, live preview OAuth roles, native key separation, cross-site browser guards, restart persistence, both provider login flows, keyboard and narrow layouts, and 35-second delayed/streaming responses through both proxies. Two restricted off-server CPA copies passed isolated application and production-key recovery checks before retirement. Assisted-by: gpt-6.1-sol

14 files changed, 1102 insertions(+), 608 deletions(-)

service/cliproxy/build-source.json deleted-30
......@@ -1,30 +0,0 @@
1{
2 "sources": [
3 {
4 "source": "../cliproxyapi",
5 "revision": "91b6ce009bc93dbefd319e7a5b84812f333363ba",
6 "dockerfile": "Dockerfile.snowglobe",
7 "include": [
8 "go.mod",
9 "go.sum",
10 "cmd",
11 "internal",
12 "sdk"
13 ]
14 },
15 {
16 "source": "../cliproxy-panel",
17 "revision": "c355dc7b07797c42e8cb7e191e3b176b5442da37",
18 "destination": "management",
19 "include": [
20 "package.json",
21 "bun.lock",
22 "index.html",
23 "tsconfig.json",
24 "tsconfig.node.json",
25 "vite.config.ts",
26 "src"
27 ]
28 }
29 ]
30}
service/cliproxy/service.pkl deleted-58
......@@ -1,58 +0,0 @@
1amends "../../config/Service.pkl"
2
3import "../../config/site.pkl" as site
4
5name = "CLIProxyAPI"
6tagline = "Codex and Claude API access"
7access = "infra-admin"
8stageIsolation = "fresh"
9
10secrets {
11 ["api_key"] {}
12 ["management_key"] {}
13}
14
15container {
16 build = "build"
17 killTimeout = "30s"
18 entrypoint = "/bin/sh"
19 args { "/snowglobe/start.sh" }
20 readOnlyRoot = true
21 tmpfs { "/tmp:rw,nosuid,nodev,noexec,size=64m" }
22 http {
23 containerPort = 8317
24 subdomain = "cpa"
25 authRole = "infra-admin"
26 checkPath = "/healthz"
27 plainHostnames { "cpa-api.\(site.domain)" }
28 }
29 volumes {
30 ["/data"] {}
31 ["/snowglobe/start.sh"] { config = "start.sh" }
32 ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true }
33 }
34 env {
35 ["HOME"] = "/data"
36 ["WRITABLE_PATH"] = "/data"
37 ["MANAGEMENT_STATIC_PATH"] = "/CLIProxyAPI/static"
38 ["CLIPROXY_SHUTDOWN_DRAIN"] = "25s"
39 ["TZ"] = "America/Los_Angeles"
40 }
41 secretFiles {
42 ["/run/secrets/cliproxy-seed.json"] = """
43 {{ with nomadVar "nomad/jobs/\(module.id)" }}
44 {
45 "config-version": 8,
46 "server": {"host": "0.0.0.0", "port": 8317, "commercial-mode": true, "max-request-body-bytes": 33554432, "discovery": {"enabled": false}},
47 "access": {"api-keys": [{{ .api_key.Value | toJSON }}]},
48 "management": {"allow-remote": true, "secret-key": {{ .management_key.Value | toJSON }}, "disable-auto-update-panel": true},
49 "oauth": {"auth-dir": "/data/auth"},
50 "routing": {"strategy": "reset-soonest", "session-affinity": true, "session-affinity-reclaim": true, "session-affinity-ttl": "1h", "cooldown": {"save-cooldown-status": true}},
51 "client": {"codex": {"codex-models-only": true}},
52 "plugins": {"enabled": false, "dir": "/data/plugins"},
53 "observability": {"history": {"enabled": true}, "logs": {"debug": false, "logging-to-file": false, "request-log": false}, "usage": {"usage-statistics-enabled": false}, "pprof": {"enable": false}}
54 }
55 {{ end }}
56 """
57 }
58}
service/cliproxy/start.sh deleted-25
......@@ -1,25 +0,0 @@
1#!/bin/sh
2set -eu
3umask 077
4
5if [ -L /data/config.yaml ] || { [ -e /data/config.yaml ] && [ ! -f /data/config.yaml ]; }; then
6 echo 'CLIProxyAPI configuration must be a regular file' >&2
7 exit 1
8fi
9if [ ! -e /data/config.yaml ]; then
10 pending=$(mktemp /data/.config.XXXXXX)
11 trap 'rm -f "$pending"' EXIT HUP INT TERM
12 cat /run/secrets/cliproxy-seed.json > "$pending"
13 ln "$pending" /data/config.yaml || test -f /data/config.yaml
14 rm -f "$pending"
15 trap - EXIT HUP INT TERM
16fi
17test -s /data/config.yaml
18chmod 600 /data/config.yaml
19mkdir -p /data/auth /data/plugins
20chmod 700 /data /data/auth /data/plugins
21if [ ! -e /data/.snowglobe-fork-v1 ]; then
22 /CLIProxyAPI/CLIProxyAPI --config /data/config.yaml --fork-migrate
23 touch /data/.snowglobe-fork-v1
24fi
25exec /CLIProxyAPI/CLIProxyAPI --config /data/config.yaml
service/copyparty/copyparty.conf+1
......@@ -6,6 +6,7 @@
66 df: SNOWGLOBE_MIN_FREE_GB
77 ver
88 theme: 2
9 localtime
910 name: clover's nas
1011 stats, nos-dup
1112 dav-auth
service/forward-auth/service.pkl+1-1
......@@ -14,7 +14,7 @@ requirements {
1414 name = module.name
1515 redirectUris {
1616 when (isPreview) { "https://\(module.id).\(site.domain)/snow.oauth2/callback" }
17 when (!isPreview) { for (host in new Listing<String> { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw"; "cpa" }) {
17 when (!isPreview) { for (host in new Listing<String> { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw" }) {
1818 "https://\(host).\(site.domain)/snow.oauth2/callback"
1919 } }
2020 }
service/substation/build-source.json created+25
......@@ -0,0 +1,25 @@
1{
2 "source": "../substation",
3 "revision": "b3083b6316d928fbc2e01f8941941c9e3769bdce",
4 "dockerfile": "Dockerfile.snowglobe",
5 "includeManifest": true,
6 "include": [
7 ".gitignore",
8 "package.json",
9 "README.md",
10 "build.ts",
11 "build-info.ts",
12 "running.ts",
13 "settings.ts",
14 "substation.ts",
15 "tests.ts",
16 "test-settings.ts",
17 "update.ts",
18 "restore.ts",
19 "hub",
20 "dashboard",
21 "ops",
22 "m0",
23 "docs"
24 ]
25}
service/substation/service.pkl created+111
......@@ -0,0 +1,111 @@
1amends "../../config/Service.pkl"
2
3import "../../config/OpenID.pkl" as sso
4import "../forward-auth/service.pkl" as forwardAuth
5
6name = "Substation"
7tagline = "Claude and Codex subscriptions"
8access = "infra-admin"
9stageIsolation = "fresh"
10
11requirements {
12 new sso.Client {
13 clientId = module.id
14 name = module.name
15 redirectUris { "https://\(module.containers["app"].http.hostname)/snow.oauth2/callback" }
16 }
17}
18
19secrets {
20 ["api_key"] {}
21 ["management_key"] {}
22 ["cookie"] = forwardAuth.secrets["cookie"]
23}
24
25local runtime = new Container {
26 build = "build"
27 hostNetwork = true
28 readOnlyRoot = true
29 tmpfs { "/tmp:rw,nosuid,nodev,noexec,size=128m" }
30 volumes {
31 ["/data/home"] {}
32 ["/data/run"] {}
33 ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true }
34 }
35 env {
36 ["HOME"] = "/data/home"
37 ["SUBSTATION_RUN_DIR"] = "/data/run"
38 ["TZ"] = "America/Los_Angeles"
39 }
40}
41
42containers {
43 ["hub"] = (runtime) {
44 lifecycle = "prestartSidecar"
45 args { "hub" }
46 killTimeout = "90s"
47 memory = 1024
48 tcp { new { name = "api"; containerPort = 8317 } }
49 volumes {
50 ["/data/auth"] {}
51 ["/data/hub"] {}
52 }
53 env {
54 ["SUBSTATION_HOST"] = "127.0.0.1"
55 ["SUBSTATION_AUTH_DIR"] = "/data/auth"
56 ["SUBSTATION_STATE_DIR"] = "/data/hub"
57 ["SUBSTATION_REFRESH"] = "1"
58 ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}"
59 ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}"
60 ["DISABLE_AUTOUPDATER"] = "1"
61 }
62 envTemplate = "SUBSTATION_PORT={{ env \"NOMAD_PORT_hub_api\" }}"
63 }
64 ["dashboard"] = (runtime) {
65 lifecycle = "prestartSidecar"
66 args { "dashboard" }
67 tcp { new { name = "web"; containerPort = 8319 } }
68 volumes { ["/data/dashboard"] {} }
69 extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" }
70 env {
71 ["DASHBOARD_HOST"] = "127.0.0.1"
72 ["DASHBOARD_AUTH"] = "proxy"
73 ["DASHBOARD_AUTH_HEADER"] = "X-Forwarded-User"
74 ["DASHBOARD_TRUSTED_PROXY"] = "127.0.0.1,::1"
75 ["DASHBOARD_ALLOWED_HOSTS"] = module.containers["app"].http.hostname!!
76 ["DASHBOARD_DB"] = "/data/dashboard/dashboard.sqlite"
77 ["SUBSTATION_STATE_DIR"] = "/data/dashboard"
78 ["SUBSTATION_HUB_KEY"] = "${secret.own.management_key}"
79 ["SUBSTATION_HUB_URL"] = "https://\(module.containers["app"].http.hostname)"
80 }
81 envTemplate = """
82 DASHBOARD_PORT={{ env "NOMAD_PORT_dashboard_web" }}
83 """
84 }
85 ["app"] = (forwardAuth.container!!) {
86 hostNetwork = true
87 http {
88 containerPort = 4180
89 loopback = true
90 subdomain = "substation.ai"
91 }
92 envTemplate = """
93 OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }}
94 OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/
95 """
96 env {
97 ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)"
98 ["OAUTH2_PROXY_COOKIE_DOMAINS"] = module.containers["app"].http.hostname!!
99 ["OAUTH2_PROXY_WHITELIST_DOMAINS"] = "\(module.containers["app"].http.hostname),\(sso.hostname)"
100 ["OAUTH2_PROXY_REDIRECT_URL"] = "https://\(module.containers["app"].http.hostname)/snow.oauth2/callback"
101 ["OAUTH2_PROXY_ALLOWED_GROUPS"] = "role:infra-admin"
102 ["OAUTH2_PROXY_AUTH_LOGGING"] = "false"
103 ["OAUTH2_PROXY_REQUEST_LOGGING"] = "false"
104 ["OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER"] = "false"
105 ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false"
106 ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false"
107 ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s"
108 ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/"
109 }
110 }
111}
tools/cliproxy-test.py deleted-323
......@@ -1,323 +0,0 @@
1#!/usr/bin/env python3
2import argparse
3import base64
4from contextlib import contextmanager
5import hashlib
6import http.client
7from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
8import json
9import os
10from pathlib import Path
11import re
12import socket
13import subprocess
14import tempfile
15import threading
16import time
17import unittest
18from unittest.mock import patch
19
20import snowglobe
21import router
22
23
24SERVICE = Path(__file__).resolve().parents[1] / "service/cliproxy"
25CADDY = None
26API_KEY = "a1" * 32
27MANAGEMENT_KEY = "b2" * 32
28ADMIN_HOST = "cliproxy-preview-12345678.paperclover.net"
29API_HOST = "cliproxy-preview-12345678-cpa-api.paperclover.net"
30
31
32class ServiceTests(unittest.TestCase):
33 @classmethod
34 def setUpClass(cls):
35 result = subprocess.run([os.environ.get("PKL", "pkl"), "eval", "-x", "((module) { enabled = true }).output.text", "-p", "uid=3150", "-p", "serviceId=cliproxy", "-p", "domain=paperclover.net", str(SERVICE / "service.pkl")], check=True, capture_output=True, text=True)
36 cls.data = json.loads(result.stdout)
37 cls.data.update(uid=3150, sourceId="cliproxy", inputs={})
38 cls.data.pop("requirements")
39 cls.data['containers']['app']['image'] = 'localhost/cliproxy:test'
40
41 def test_native_defaults_keep_credentials_private_and_requests_unlogged(self):
42 app = self.data["containers"]["app"]
43 seed = app["secretFiles"]["/run/secrets/cliproxy-seed.json"]
44 seed = re.sub(r"{{ with nomadVar [^}]+}}|{{ end }}", "", seed)
45 seed = seed.replace("{{ .api_key.Value | toJSON }}", json.dumps(API_KEY))
46 seed = seed.replace("{{ .management_key.Value | toJSON }}", json.dumps(MANAGEMENT_KEY))
47 config = json.loads(seed)
48 self.assertEqual(config["access"]["api-keys"], [API_KEY])
49 self.assertNotEqual(config["access"]["api-keys"][0], config["management"]["secret-key"])
50 self.assertTrue(config["server"]["commercial-mode"])
51 self.assertFalse(config["observability"]["logs"]["request-log"])
52 self.assertFalse(config["observability"]["pprof"]["enable"])
53 self.assertFalse(config["plugins"]["enabled"])
54 self.assertEqual(config["oauth"]["auth-dir"], "/data/auth")
55 self.assertTrue(config["management"]["disable-auto-update-panel"])
56 self.assertEqual(app["http"]["authRole"], "infra-admin")
57 self.assertEqual(app["http"]["plainHostnames"], ['cpa-api.paperclover.net'])
58 self.assertEqual(self.data["stageIsolation"], "fresh")
59
60 def test_rendered_job_has_only_nonroot_private_managed_storage(self):
61 rendered = snowglobe.render(self.data, {"cliproxy": self.data})
62 self.assertEqual(rendered.count('user = "3150:3150"'), 1)
63 self.assertEqual(rendered.count('readonly_rootfs = true'), 1)
64 self.assertEqual(rendered.count('perms = "0400"'), 1)
65 self.assertIn('secrets/cliproxy-seed.json:/run/secrets/cliproxy-seed.json:ro', rendered)
66 self.assertIn('kill_timeout = "30s"', rendered)
67 self.assertIn('/CLIProxyAPI/static', rendered)
68 self.assertNotIn("NOMAD_SERVICE", rendered)
69 self.assertNotIn("NOMAD_VARIABLE", rendered)
70 self.assertNotIn("/srv/clover", rendered)
71 self.assertNotIn("host_port", rendered)
72 self.assertNotIn('env = true', rendered)
73
74 def test_bootstrap_preserves_config_after_the_first_migration(self):
75 with tempfile.TemporaryDirectory() as temporary:
76 root = Path(temporary)
77 data = root / "data"
78 data.mkdir()
79 seed = root / "seed.json"
80 seed.write_text(json.dumps({"access": {"api-keys": [API_KEY]}}))
81 binary = root / "binary"
82 binary.write_text('#!/usr/bin/env python3\nimport pathlib,sys\nassert sys.argv[1] == "--config" and pathlib.Path(sys.argv[2]).stat().st_size\n')
83 binary.chmod(0o700)
84 script = (SERVICE / "start.sh").read_text().replace('/run/secrets/cliproxy-seed.json', str(seed)).replace('/CLIProxyAPI/CLIProxyAPI', str(binary)).replace('/data', str(data))
85 self.assertEqual(subprocess.run(["sh", "-ec", script], timeout=5).returncode, 0)
86 config = data / "config.yaml"
87 self.assertEqual(json.loads(config.read_text())["access"]["api-keys"], [API_KEY])
88 self.assertEqual(config.stat().st_mode & 0o777, 0o600)
89 self.assertEqual((data / "auth").stat().st_mode & 0o777, 0o700)
90 config.write_text('access:\n api-keys: [edited-key]\nprovider-setting: retained\n')
91 credential = data / "auth/fixture.json"
92 credential.write_text('{"synthetic":true}')
93 seed.write_text('different seed')
94 self.assertEqual(subprocess.run(["sh", "-ec", script], timeout=5).returncode, 0)
95 self.assertIn('provider-setting: retained', config.read_text())
96 self.assertEqual(credential.read_text(), '{"synthetic":true}')
97 config.unlink()
98 config.symlink_to(seed)
99 result = subprocess.run(["sh", "-ec", script], capture_output=True)
100 self.assertNotEqual(result.returncode, 0)
101 self.assertEqual(seed.read_text(), 'different seed')
102
103
104
105@unittest.skipUnless(CADDY, "pass --caddy to test the real router")
106class GatewayTests(unittest.TestCase):
107 def setUp(self):
108 temporary = tempfile.TemporaryDirectory(prefix='cliproxy-caddy-')
109 self.addCleanup(temporary.cleanup)
110 root = Path(temporary.name)
111 self.observed = []
112 self.auth_checks = []
113 self.websocket_started = threading.Event()
114 self.websocket_release = threading.Event()
115 observed, auth_checks = self.observed, self.auth_checks
116 websocket_started, websocket_release = self.websocket_started, self.websocket_release
117
118 class Backend(BaseHTTPRequestHandler):
119 protocol_version = 'HTTP/1.1'
120
121 def do_GET(self):
122 observed.append((self.command, self.path, dict(self.headers)))
123 management = self.path.startswith(('/v0/management/', '/v8/management/'))
124 if management:
125 valid = self.headers.get('Authorization') == 'Bearer ' + MANAGEMENT_KEY or self.headers.get('X-Management-Key') == MANAGEMENT_KEY
126 elif self.path.startswith('/v1/'):
127 valid = self.headers.get('Authorization') == 'Bearer ' + API_KEY or self.headers.get('X-Api-Key') == API_KEY
128 else:
129 valid = True
130 if self.headers.get('Upgrade', '').lower() == 'websocket' and valid:
131 accept = base64.b64encode(hashlib.sha1((self.headers['Sec-WebSocket-Key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest()).decode()
132 self.send_response(101)
133 self.send_header('Upgrade', 'websocket'); self.send_header('Connection', 'Upgrade')
134 self.send_header('Sec-WebSocket-Accept', accept); self.end_headers()
135 header, mask, body = self.rfile.read(2), self.rfile.read(4), self.rfile.read(2)
136 if header != b'\x81\x82' or bytes(value ^ mask[index % 4] for index, value in enumerate(body)) != b'hi':
137 raise AssertionError('incorrect client WebSocket frame')
138 self.wfile.write(b'\x81\x02hi'); self.wfile.flush()
139 if self.headers.get('X-Synthetic-Hold'):
140 websocket_started.set()
141 if not websocket_release.wait(5):
142 raise AssertionError('WebSocket release was not signaled')
143 self.wfile.write(b'\x81\x02ok'); self.wfile.flush()
144 self.close_connection = True
145 return
146 self.send_response(200 if valid else 401)
147 self.send_header('Content-Type', 'application/json')
148 self.send_header('Access-Control-Allow-Origin', '*')
149 if self.headers.get('X-Synthetic-Stream') and valid:
150 self.send_header('Connection', 'close'); self.end_headers()
151 self.wfile.write(b'data: first\n\n'); self.wfile.flush()
152 time.sleep(.5)
153 self.wfile.write(b'data: last\n\n'); self.close_connection = True
154 else:
155 body = b'{}'
156 self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body)
157
158 do_POST = do_GET
159 do_PUT = do_GET
160
161 def log_message(self, *args):
162 pass
163
164 class Auth(BaseHTTPRequestHandler):
165 def do_GET(self):
166 auth_checks.append(dict(self.headers))
167 cookie = self.headers.get('Cookie', '')
168 self.send_response(202 if cookie else 401)
169 if cookie:
170 self.send_header('X-Auth-Request-Groups', 'role:infra-admin' if cookie == 'fixture=admin' else 'role:media')
171 self.end_headers()
172
173 def log_message(self, *args):
174 pass
175
176 for handler, attribute in [(Backend, 'backend'), (Auth, 'auth')]:
177 server = ThreadingHTTPServer(('127.0.0.1', 0), handler)
178 threading.Thread(target=server.serve_forever, daemon=True).start()
179 self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
180 setattr(self, attribute, server)
181 with socket.socket() as reservation:
182 reservation.bind(('127.0.0.1', 0)); self.port = reservation.getsockname()[1]
183 proof = root / 'proof'; proof.write_text('f' * 64)
184 (root / 'cliproxy.json').write_text(json.dumps({'cliproxy': {'api_key': API_KEY, 'management_key': MANAGEMENT_KEY}}))
185 def nomad(path, token):
186 if path == '/v1/services':
187 return [{'Namespace': 'default', 'Services': [{'ServiceName': name} for name in ('cliproxy', 'forward-auth')]}]
188 if path.startswith('/v1/service/'):
189 service = path.rsplit('/', 1)[1]
190 return [{'ServiceName': service, 'JobID': service, 'AllocID': service, 'Address': '127.0.0.1',
191 'Port': self.backend.server_port if service == 'cliproxy' else self.auth.server_port,
192 'Tags': ['caddy-host=' + ADMIN_HOST, 'caddy-plain-host=' + API_HOST, 'caddy-auth-role=infra-admin'] if service == 'cliproxy' else []}]
193 return {'http': {'Status': 'success'}}
194 environment = {'SNOWGLOBE_DOMAIN': 'paperclover.net', 'SNOWGLOBE_DASHBOARD_PORT': '28001',
195 'SNOWGLOBE_PROXY_TOKEN_FILE': str(proof)}
196 with patch.dict(os.environ, environment), patch.object(router, 'ROUTE_DIR', str(root)), patch.object(router, 'nomad', nomad):
197 rendered = router.render('fixture')
198 with socket.socket() as admin:
199 admin.bind(('127.0.0.1', 0)); self.admin_port = admin.getsockname()[1]
200 config = '{\n admin 127.0.0.1:' + str(self.admin_port) + '\n auto_https off\n}\n'
201 for host in (ADMIN_HOST, API_HOST):
202 block = rendered.split(host + ' {', 1)[1].split('\n}\n', 1)[0]
203 config += f'http://{host}:{self.port} {{' + block + '\n}\n'
204 path = root / 'Caddyfile'; path.write_text(config)
205 self.config_path = path
206 self.log = root / 'caddy.log'
207 log = self.log.open('wb'); self.addCleanup(log.close)
208 process = subprocess.Popen([CADDY, 'run', '--config', str(path), '--adapter', 'caddyfile'], stdout=log, stderr=log)
209 def stop():
210 process.terminate(); process.wait(timeout=5)
211 self.addCleanup(stop)
212 for _ in range(100):
213 if process.poll() is not None:
214 self.fail('Caddy failed: ' + self.log.read_text())
215 try:
216 with socket.create_connection(('127.0.0.1', self.port), timeout=.1):
217 break
218 except OSError:
219 time.sleep(.02)
220 else:
221 self.fail('Caddy did not listen')
222
223 @contextmanager
224 def request(self, method, path, headers=None):
225 connection = http.client.HTTPConnection('127.0.0.1', self.port, timeout=3)
226 connection.request(method, path, headers={'Host': ADMIN_HOST, **(headers or {})})
227 response = connection.getresponse()
228 try:
229 yield response
230 finally:
231 response.close(); connection.close()
232
233 def test_native_keys_bypass_sso_on_both_hosts(self):
234 for host in (ADMIN_HOST, API_HOST):
235 for headers in ({'Authorization': 'Bearer ' + MANAGEMENT_KEY}, {'X-Management-Key': MANAGEMENT_KEY}):
236 with self.request('GET', '/v8/management/config', {'Host': host, **headers}) as response:
237 self.assertEqual(response.status, 200); response.read()
238 for headers in ({'Authorization': 'Bearer ' + API_KEY}, {'X-Api-Key': API_KEY}):
239 with self.request('GET', '/v1/models', {'Host': host, **headers}) as response:
240 self.assertEqual(response.status, 200); response.read()
241 self.assertEqual(self.auth_checks, [])
242 self.assertTrue(all('Cookie' not in headers for _, _, headers in self.observed))
243
244 def test_wrong_or_absent_keys_cannot_gain_management_access(self):
245 for host in (ADMIN_HOST, API_HOST):
246 for headers in ({}, {'Authorization': 'Bearer invalid'}, {'Authorization': 'Bearer ' + API_KEY}):
247 with self.request('GET', '/v8/management/config', {'Host': host, **headers}) as response:
248 self.assertEqual(response.status, 401); response.read()
249
250 def test_dashboard_requires_sso_even_with_management_key(self):
251 for path in ('/', '/management.html'):
252 with self.request('GET', path, {'Authorization': 'Bearer ' + MANAGEMENT_KEY}) as response:
253 self.assertEqual(response.status, 302)
254 self.assertTrue(response.getheader('Location').startswith('/snow.oauth2/sign_in')); response.read()
255 with self.request('GET', path, {'Host': API_HOST}) as response:
256 self.assertEqual(response.status, 404); response.read()
257 with self.request('GET', '/management.html', {'Cookie': 'fixture=member'}) as response:
258 self.assertEqual(response.status, 403); response.read()
259 with self.request('GET', '/management.html', {'Cookie': 'fixture=admin'}) as response:
260 self.assertEqual(response.status, 200); response.read()
261
262 def test_sso_panel_injects_private_credentials_and_guards_origins(self):
263 for path in ('/v8/management/config', '/v1/models'):
264 with self.request('GET', path, {'Cookie': 'fixture=admin'}) as response:
265 self.assertEqual(response.status, 200)
266 self.assertEqual(response.getheader('Cache-Control'), 'no-store')
267 self.assertIsNone(response.getheader('Access-Control-Allow-Origin')); response.read()
268 self.assertNotIn('Cookie', self.observed[-1][2])
269 for origin in ('', 'https://evil.example', 'null'):
270 with self.request('POST', '/v8/management/config', {'Cookie': 'fixture=admin', **({'Origin': origin} if origin else {})}) as response:
271 self.assertEqual(response.status, 403); response.read()
272 with self.request('POST', '/v8/management/config', {'Cookie': 'fixture=admin', 'Origin': 'https://' + ADMIN_HOST}) as response:
273 self.assertEqual(response.status, 200); response.read()
274 with self.request('POST', '/v8/management/config', {'Authorization': 'Bearer ' + MANAGEMENT_KEY}) as response:
275 self.assertEqual(response.status, 200); response.read()
276
277 def test_native_websocket_upgrade(self):
278 key = base64.b64encode(b'synthetic-16-key').decode()
279 with socket.create_connection(('127.0.0.1', self.port), timeout=3) as connection:
280 connection.sendall(('GET /v1/responses HTTP/1.1\r\nHost: ' + ADMIN_HOST + '\r\nAuthorization: Bearer ' + API_KEY + '\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ' + key + '\r\n\r\n').encode())
281 headers = b''
282 while not headers.endswith(b'\r\n\r\n'):
283 headers += connection.recv(1)
284 self.assertIn(b' 101 ', headers.split(b'\r\n')[0])
285 connection.sendall(b'\x81\x82\x01\x02\x03\x04ik')
286 self.assertEqual(connection.recv(4), b'\x81\x02hi')
287 self.assertEqual(self.auth_checks, [])
288
289 def test_websocket_survives_caddy_configuration_reload(self):
290 for host in (ADMIN_HOST, API_HOST):
291 self.websocket_started.clear(); self.websocket_release.clear()
292 key = base64.b64encode(b'synthetic-16-key').decode()
293 with socket.create_connection(('127.0.0.1', self.port), timeout=3) as connection:
294 connection.sendall(('GET /v1/responses HTTP/1.1\r\nHost: ' + host + '\r\nAuthorization: Bearer ' + API_KEY + '\r\nX-Synthetic-Hold: yes\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ' + key + '\r\n\r\n').encode())
295 headers = b''
296 while not headers.endswith(b'\r\n\r\n'):
297 headers += connection.recv(1)
298 self.assertIn(b' 101 ', headers)
299 connection.sendall(b'\x81\x82\x01\x02\x03\x04ik')
300 self.assertEqual(connection.recv(4), b'\x81\x02hi')
301 self.assertTrue(self.websocket_started.wait(1))
302 subprocess.run([CADDY, 'reload', '--address', f'127.0.0.1:{self.admin_port}', '--config', str(self.config_path), '--adapter', 'caddyfile', '--force'], check=True, capture_output=True)
303 self.websocket_release.set()
304 self.assertEqual(connection.recv(4), b'\x81\x02ok')
305
306 def test_native_response_streams(self):
307 started = time.monotonic()
308 with self.request('POST', '/v1/responses', {'Authorization': 'Bearer ' + API_KEY, 'X-Synthetic-Stream': 'yes'}) as response:
309 self.assertEqual(response.status, 200)
310 self.assertEqual(response.readline(), b'data: first\n')
311 self.assertLess(time.monotonic() - started, .4)
312 self.assertIn(b'data: last', response.read())
313 log = self.log.read_text()
314 self.assertNotIn(API_KEY, log); self.assertNotIn(MANAGEMENT_KEY, log)
315
316
317if __name__ == '__main__':
318 parser = argparse.ArgumentParser()
319 parser.add_argument('--caddy', type=Path)
320 options, remaining = parser.parse_known_args()
321 CADDY = str(options.caddy.resolve()) if options.caddy else None
322 GatewayTests.__unittest_skip__ = not CADDY
323 unittest.main(argv=[__file__, *remaining])
tools/cliproxy.md deleted-114
......@@ -1,114 +0,0 @@
1# CLIProxyAPI
2
3`https://cpa.paperclover.net` serves the Snow Globe CPA fork’s inference and management
4APIs. Caddy protects the dashboard with Snow's `infra-admin` role. The
5`cpa-api.paperclover.net` alias serves the same native APIs, with dashboard
6paths disabled.
7
8CPA validates its two independent keys:
9
10- The client `api_key` authenticates inference and model listing through
11 `Authorization: Bearer …` or the provider-compatible key headers.
12- The `management_key` authenticates `/v8/management/*` through
13 `Authorization: Bearer …` or `X-Management-Key`.
14
15Nomad variable `nomad/jobs/cliproxy` owns both keys. Keep their values in
16restricted files or a password manager, never in chat, arguments, or tracked
17configuration. An inference key does not grant management access. A management
18key can modify configuration and use stored provider credentials.
19
20For OpenAI-compatible clients, use `https://cpa.paperclover.net/v1`; for
21Claude-compatible clients, use `https://cpa.paperclover.net`. Management callers
22use `https://cpa.paperclover.net/v8/management` with the management key. Native
23requests carrying a key bypass SSO; CPA rejects missing or invalid credentials.
24The dashboard still requires SSO even when a management key is supplied.
25
26## Dashboard
27
28Snow sign-in opens the panel automatically. Caddy supplies the management key
29only after an `infra-admin` SSO check, and the client key for the panel's model
30list. Cookie-authenticated writes require the exact dashboard Origin; reads
31reject a foreign Origin. Keys stay on the server, and cookies are removed before
32proxying to CPA. Native requests with explicit key headers retain CPA's own
33authentication and do not receive injected credentials.
34
35Opening the panel clears remembered native passwords and validates access
36through the config API. Logout clears the shared forward-auth cookie and opens
37Snow sign-in. The source panel implements this SSO adaptation; the image contains
38one immutable panel build.
39
40## Providers and persistence
41
42Use the panel's OAuth section to connect providers. For a remote flow ending at
43an unreachable localhost URL, submit the entire callback URL through the panel's
44callback form. Treat it as a temporary credential. API-key providers can also be
45configured in the native panel.
46
47`/data/config.yaml` is seeded once from a private Nomad template. Configuration,
48OAuth accounts, and refreshed tokens persist in `/srv/prod/cliproxy/data`.
49The first fork start atomically migrates the native configuration, preserving
50provider credentials and unrelated settings. It selects reset-soonest routing,
51affinity with priority reclaim, retained history, Codex-only model discovery,
52and saved cooldowns; panel auto-updates and request logging stay disabled.
53`/data/.snowglobe-fork-v1` makes this a one-time migration, so later operator
54changes survive restarts. The container publishes one
55CPA port through Nomad's loopback binding; Caddy is the only public entry point.
56There is no nginx container or second CPA listener.
57
58Changing a Nomad key does not overwrite existing native configuration. Rotate it
59in CPA and Nomad together; Caddy's private route manifest is refreshed during
60provisioning. Fresh previews have separate storage and keys and must use
61synthetic providers only. Shared forward-auth does not register arbitrary preview
62callback URLs; a fresh preview cannot complete interactive sign-in through it.
63
64## Fork sources and history
65
66[Backend](https://shale.paperclover.net/cliproxyapi/) and
67[panel](https://shale.paperclover.net/cliproxy-panel/) fork CPA v8.0.32 and panel
68v1.26.5. `service/cliproxy/build-source.json` pins the exact source revisions;
69the deployment exporter builds both into one image using
70`Dockerfile.snowglobe`. Upstream panel auto-update cannot replace the baked file.
71Cursor's private agent protocol is excluded.
72
73`observability.history.enabled` enables retained monthly JSON usage and quota
74files under `/data/history`, beside the auth directory. They contain token
75counts, provider/model, hashed sessions and hashed account identities, never
76prompts. Quota is sampled each minute; idle subscription usage is polled every
77ten minutes. Usage-plan calls share a two-minute cache, with up to thirty minutes
78of stale successful data after a 429. The sample retains its original observation
79time so stale data cannot masquerade as fresh quota.
80
81`GET /v8/management/history/{usage,quota,accounts}` reads history; `since` accepts
82an RFC3339 timestamp. Accounts are masked with a unique suffix; `reveal=true`
83reveals email addresses to an authenticated management caller. `POST` to usage or
84quota accepts up to 10,000 bounded rows; stable row IDs make repeated imports
85idempotent. `tools/export-history.py` in the backend exports only transcript
86paths explicitly supplied by the operator, requires a cutoff, and never exports
87message contents. Do not point it at personal transcripts without authorization.
88
89API-equivalent estimates use models.dev, then LiteLLM, then explicit
90`observability.history.prices` overrides (USD per million tokens: `input`,
91`output`, `cache-read`, `cache-write`). Catalogs are cached for a day; overrides
92apply immediately to new records and imports. Missing catalog cache rates fall back to
93input rates; explicit zero rates remain zero. Estimates are neither subscription
94bills nor guaranteed provider charges. Existing retained rows keep the price
95observed when they were recorded. Restart after enabling history on an instance
96that started with it disabled.
97
98## Deployment and checks
99
100Stage with `python3 tools/deploy.py stage cliproxy`. A preview retains its own
101configuration/history across updates and never promotes that data to production.
102Production uses `python3 tools/deploy.py prod` from a described, conflict-free
103main. The fork closes its listener before draining active requests for up to
10425 seconds, flushes usage, and exits within Nomad's 30-second kill timeout.
105Caddy retries new connections briefly and preserves existing WebSockets across
106route reloads. A single writer still has a short replacement window; a turn
107longer than the drain budget can be interrupted.
108
109Run `python3 tools/cliproxy-test.py --caddy /path/to/caddy` for native key
110separation, SSO roles, dashboard protection, Origin checks, streams, WebSockets
111including Caddy reloads, and configuration preservation. It uses disposable
112HTTP fixtures, synthetic keys, and a separate Caddy process.
113`python3 tools/deploy-test.py` covers composed source exports. The backend uses
114`go test ./...` and `go build ./cmd/server`; the panel uses `bun run verify`.
tools/deploy.py+2
......@@ -141,6 +141,8 @@ def upload(main=False):
141141 shutil.copy2(origin, destination)
142142 if not (context / "Dockerfile").is_file():
143143 raise ValueError(f"missing build source Dockerfile: {manifest.parent.name}")
144 if spec.get("includeManifest"):
145 shutil.copy2(manifest, context / ".snowglobe-source.json")
144146 digest = tree_digest(snapshot)
145147 release = digest[:16]
146148 remote_release = REMOTE / "releases" / release
tools/icloud-export.md created+866
......@@ -0,0 +1,866 @@
1# Export iCloud data to Snow Globe
2
3Export **9 GB of Voice Memos first** inside the dedicated macOS VM, then
4trial optimized native Photos exports of **about 1.7 TB** to Snow Globe.
5These are the only requested categories. Keep original media, edited versions, and
6metadata in ordinary Windows-readable files, organized by year with mtimes
7set from recording dates. The remote friend supplies Apple sign-in and
8two-factor approval; the iPhone never needs to hold the whole library or
9connect to Clover's Mac by cable.
10
11The storage split is confirmed by the friend's iCloud storage screenshot.
12Apple sign-in, two-factor approval, and encrypted-data authorization are complete
13in the dedicated VM. Advanced Data Protection remains on. Unrelated app sync
14is disabled in this guest. The source iPhone reports **348 All Recordings,
151 favorite, and 2 Recently Deleted**.
16
17The Mac catalog contains 358 rows. On macOS 26, the native playable predicate
18`flags != nil AND flags & 6 != 0` recognizes **348 active recordings**, matching
19the phone, plus two deleted recordings. The native favorite count is one.
20Two active recordings are shorter than 0.1 seconds; duration filtering would
21exclude them and incorrectly include two stale catalog entries. Use the native
22predicate when selecting recordings, and preserve all 358 catalog rows.
23
24The macOS 15 cache exposed 161 audio files. A fresh macOS 26 import retrieved
25350 files totaling 9,612,921,523 bytes: 161 `.m4a` and 189 `.qta`, including the
26two deleted recordings. All 348 active recordings now have audio references.
27The complete host and native checks recovered **310 active recordings** passing full
28stereo decoding and catalog-duration checks. The remaining 38 older `.m4a`
29recordings comprise five empty files, 14 headers without a readable container,
30and 19 decodable files shorter than their catalog duration. Both independent
31native downloads retrieved the same legacy bytes. Keep these originals and
32their metadata; do not claim a complete export or delete cloud data.
33
34The `.qta` files contain stereo AAC alongside newer audio tracks. Native macOS
3526 validation checked every audio track without playback. Portable stereo
36`.m4a` copies can reuse the original AAC packets without re-encoding; retain
37the byte-identical `.qta` originals to preserve spatial/layer and other metadata.
38
39Three pilot files totaling 107,836,545 bytes passed source/host/NAS SHA-256,
40full decoding, duration reconciliation, and exact recording-time mtime checks.
41The macOS 15 local-file pass found 123 files passing full decoding and catalog
42and decoded duration checks; 19 decode but are shorter than their catalog
43duration, and 14 nonempty files contain headers without a `moov` atom. The
44five zero-byte files are excluded from the portable set. Failed and incomplete
45files remain in the native archive, with private per-record validation results.
46
47The public delivery root `/srv/clover/Share/iCloud Export - 2026-11-04`
48contains only `Voice Memos/<year>/...`: **310 validated recordings**, delivered
49as 310 Windows `.m4a` files and 187 original `.qta` files, totaling
5012,163,467,515 bytes. The first 123 recordings remain included. Pilot,
51Available, native databases, manifests, and helper folders are absent.
52The original `.qta` files preserve every track and embedded field. Their stereo
53copies preserve identical AAC packets without re-encoding. Both audio tracks
54of every active `.qta` passed native macOS 26 decoding: 187 recordings,
55374 tracks. Source/host/NAS hashes and recording-time mtimes match.
56
57Native files and metadata are outside the share at
58`/srv/clover/.icloud-export-private/2026-11-04`, behind a mode-0700 parent:
59
60- `Native initial snapshot`: 232 source/support files totaling 1,746,210,872
61 bytes, including a consistent SQLite backup with all 358 rows.
62- `Native macOS 26 snapshot`: 351 files totaling 9,615,002,291 bytes,
63 including all 350 media files and a consistent 358-row SQLite backup.
64- `Portable metadata`: complete recording index, original titles, source and
65 portable inventories, native validation results, and the recovery index.
66
67Both native databases pass `quick_check` on Mac and NAS. The existing
68password-protected read-only share URL is unchanged. Live HTTP tests verify
69an initial file, a new original and stereo copy, recording-time headers,
70range requests, and denial of writes and parent access. TAR downloads preserve
71whole-second recording mtimes and the `Voice Memos/<year>` layout. The friend
72has **not yet tested Windows playback or Date modified**.
73
74A separate protected recovery share at
75`/srv/clover/Share/iCloud Export Recovery - 2026-11-04` contains instructions
76identifying the 38 exceptions and an `Uploads` inbox. Its URL is in
77`private/recovery-share.json`; it uses the same password as the delivery share.
78Instruction names are generated mechanically for the friend and never printed.
79Uploads and authenticated downloads pass; deletion and parent access are denied.
80The friend checks the originals on the iPhone, exports available recordings as
81Rendered audio, and names each upload using the checklist's `recovery-NNN`
82identifier. The private recovery index maps that identifier to the original
83recording date, catalog duration, and title for later verification and replacement.
84The recovery share now also has `Partial recordings/<year>/...`: 19 untouched
85short originals and 28 reconstructed candidates covering the other 14 nonempty
86exceptions. Both 44.1 kHz and 48 kHz headers are retained for those 14 because
87the source header is missing. All 47 files pass decoding, NAS hash, mtime, and
88live HTTP checks; every reconstructed encoded packet occurs in its untouched
89source. These 33 logical partial recordings do not count toward the 310 complete
90recordings. Candidate audio rate still needs comparison with the original source.
91Never publish incomplete files as successfully recovered recordings.
92
93The audio sync worker was restarted in this guest after the native snapshot;
94counts and bytes remained unchanged. The normal share manager rolled back because its success-text check does not
95recognize Copyparty’s current reload response. A read-only password-protected
96link was instead created through Copyparty’s native HTTP API for the dated
97export root. Unauthenticated reads return 401; authenticated media hashes and
98HTTP mtimes match; range requests work; writes and parent access return 403.
99The URL and password-file path are in `private/export-share.json`. Evidence remains in the VM's restricted
100`evidence/*-nas-validation.json`; private per-file manifests are not operator
101readable output.
102
103## Privacy boundary
104
105Expose only basic metadata and aggregate integrity results: counts, byte
106sizes, dates, durations, and opaque identifiers. Do not display or listen to
107media, inspect transcripts or thumbnails, or reveal filenames, recording
108titles, GPS coordinates, or other location information in chat or operator
109logs. Keep media apps closed during authentication and exclude profile
110images and media windows from authentication captures.
111
112Automated workers may copy, hash, and validate files without exposing their
113contents. Preserve original bytes and all metadata, including location data,
114inside the friend's restricted archive; withholding metadata from the operator
115must not strip it from the export. Names and metadata may be handled
116mechanically inside private workers and manifests without returning their
117values. The friend performs any visual or listening review on Windows.
118These constraints take precedence over manual sample inspection below.
119
120## Agreed order
121
1221. Create a fresh macOS VM on this Mac, prove its console and guest control,
123 then sign into the friend's Apple Account and enable Voice Memos only.
124 A separate host macOS account remains an alternative if the VM pilot fails.
1252. Prove a small memo export, then download and reconcile the full 9 GB,
126 preserve native sources, and share the dated portable files from Snow Globe.
127 Check the result on the friend's Windows PC.
1283. Trial native Photos exports using the VM's existing Apple session.
129 Check originals, edits, companions, counts, dates, and cache space before
130 the full run. The web downloader requires separate authentication and
131 remains an alternative when the friend is available.
132
133## Capacity and cleanup
134
135Read-only measurements on 4 November 2026, in decimal GB and TB:
136
137| Storage | Used | Available | Allocation |
138| --- | ---: | ---: | --- |
139| Project | 567.5 GB | Shared below | APFS volume on the 2 TB WD Blue SN5000 |
140| Documents | 1183.3 GB | Shared below | Same APFS container as Project |
141| Project and Documents container | About 1751 GB | **249.1 GB total** | 2000.2 GB capacity; no volume quotas or reserves |
142| Mac startup container | About 895 GB | 99.2 GB | Separate 994.7 GB APFS container |
143| Snow Globe personal storage | — | About 7.79 TB | ZFS availability for `/srv/clover` |
144
145Project and Documents already share free space. Shrinking one APFS volume
146does not give the other more space; moving or deleting verified data frees
147their common pool. Neither volume currently has APFS snapshots. These are
148volumes on a physical disk, not sparse disk images that can be compacted.
149
150The **9 GB memo run needs no disk cleanup**. For the VM route, reserve about
151100 GB of the Documents container's available 249 GB for a 64 GB guest disk,
152the restore-image download, and transfer/scratch space. This is a planning
153budget from before installation. After the macOS 26 upgrade, backups, and
154full memo export, the shared container still has about **134.3 GB free**;
155no personal files were removed. A host-account fallback
156would instead use the startup disk's available 99 GB for the memo store.
157
158The current Photos route keeps an optimized local library and sends exported
159files to Snow Globe. It reuses the existing Apple login and supplies a native
160catalog for reconciliation. Original downloads temporarily consume guest and
161host disk space; the trial must establish a workable batch size and cache
162reclamation behavior.
163
164For a full Download Originals library, budget **about 1.85 TB free** in the shared
165Project/Documents container: 1.7 TB of photos plus an initial 150 GB allowance
166for databases, caches, and temporary files. That means freeing **about 1.60 TB
167more** against the pre-VM measurement above. A retained memo VM consumes space
168in this same pool, so add its actual allocation and remeasure before cleanup.
169The iCloud size is rounded and the allowance is a planning target. Export
170directly to Snow Globe to avoid a second full local media set.
171
172Any eventual cleanup must follow the recovery requirements in
173[AGENTS.md](../AGENTS.md). NAS space must then cover both relocated Clover
174data and the friend's export, including edited versions.
175
176## Dedicated VM route
177
178Apple supports iCloud in macOS VMs created using a macOS 15 or later restore
179image on a macOS 15 or later Apple silicon host. Upgrading a VM originally
180created with an older OS does not add this support. VM identities are tied
181to the host's Secure Enclave; moving a VM or running another clone can require
182reauthentication.
183[Apple's iCloud virtualization requirements](https://developer.apple.com/documentation/virtualization/using-icloud-with-macos-virtual-machines).
184
185This host runs macOS 15.7.5 and has 256 GiB RAM. Its existing Tart guest is
186the running `gui-agent`, exposed as `m3-devbox`. The installed
187Tart 2.41.1 supports fresh IPSW installation and a specified guest disk size.
188Apple's Sequoia license, section 2B(iii), permits two additional virtual macOS
189instances for the listed purposes, including personal noncommercial use.
190[macOS Sequoia license](https://www.apple.com/legal/sla/docs/macOSSequoia.pdf).
191
192The pilot has four vCPUs and 8 GiB RAM. Its initially 64 GB sparse disk has
193been enlarged to 128 GB for the macOS 26 compatibility test. It was created
194from Apple's macOS 15.6.1 restore image, build 24G90, with verified SHA-256
195`3d87686b691ac765eb6a6b3082b2334e2af9710096a00432dd519af89ff2ea78`.
196The Apple CDN URL and installation evidence are in its private `evidence`
197directory. The initial macOS 15.8.2 guest recognized only 161 of 348 as playable.
198The guest has now booted macOS 26.7.2 for the compatibility test, matching the
199phone’s iOS 26.6.1 generation.
200
201Reuse the installed signed Tart binary and the control approach documented
202in [the devbox runbook](mac-agent-vm.md), with a separate VM state directory
203under `/Volumes/Documents/VM/macOS Code Sandbox/icloud-export`. Its existing
204ancestor ACL protects this tree on Documents, where ownership is disabled.
205Keep its disk, credentials, and transfer directory there. Keep the devbox
206guest and its active work separate. A fresh install avoids copying devbox
207accounts, credentials, and workspace mounts, and establishes the required
208installation provenance; cloning is not categorically incompatible with iCloud.
209
210[Tart's fresh-install procedure](https://tart.run/quick-start/) begins with
211`tart create --from-ipsw IMAGE --disk-size 64 icloud-export`, followed by guest
212Setup Assistant. Use the isolated state directory for every command. Keep
213only a dedicated transfer folder available to the guest, then copy verified
214portable output to the NAS from the host.
215
216Console access, input, capture, guest storage, and the transfer path have been
217proved before Apple sign-in. The existing dashboard bridge selects `m3-devbox`
218specifically; it does not automatically control another guest. A dedicated
219guest console/control connection can run while Clover stays in the existing
220host login. This avoids creating or switching to another host account.
221The friend still provides Apple authentication and two-factor approval.
222Supported iCloud sign-in is a prerequisite; the pilot must also establish
223actual Voice Memos sync, complete audio downloads, and a verified export.
224
225### Resume this pilot
226
227The guest has been upgraded to macOS 26.7.2, matching the phone's iOS 26.6.1
228generation. It was cleanly shut down first. APFS clones of the stopped
229`disk.img`, `nvram.bin`, and `config.json` remain in `private/pre-tahoe`; all
23064 GB of the old disk passed byte comparison. Never run this backup
231concurrently or alter its VM identity.
232
233The guest disk is now 128 GB. Its Recovery partition was copied to the new
234end without changing partition identifiers; all 5,368,668,160 Recovery bytes
235passed hash verification, and both GPT copies pass CRC checks. Apple's
236`diskutil apfs resizeContainer` then expanded the guest system container,
237leaving about 97.8 GB free. No host partitions or devbox disks changed. A temporary Virtualization VNC
238server was closed before input because it listens on all host interfaces;
239normal runs remain headless with no host VNC listener.
240`evidence/pre-tahoe-backup.json` and `recovery-relocation.json` record these
241checks. Apple Software Update downloaded the 26.7.2 installer, and the in-place guest
242upgrade completed and the guest has booted macOS 26.7.2. The macOS 26 desktop is open. A one-boot automatic login bypassed unreliable
243Screen Sharing password entry, then its root cleanup job removed the setting,
244`/etc/kcpassword`, and its own plist. Automatic login is disabled; the guest now reports FileVault enabled. The local
245password, keychain, and Apple Account are unchanged. The one-boot setup helper
246was removed to prevent replay; evidence is `one-boot-login-validation.json`.
247`sysadminctl`'s prompted form returned error 22, so the temporary credential was
248written to its normal root-only file over stdin; no secret appeared in argv or
249logs. Executable signatures verify; the installer intentionally
250uses legacy resource omission rules rejected by a generic Gatekeeper app
251assessment. No security policy was changed. `startosinstall` uses the guest
252local volume-owner password over stdin, never command arguments. The completed login is verified by console UID 501, the login-done flag, and
253Finder/Dock processes; a GUI launchd domain alone does not establish login. The
254lock-state guard uses pinned guest SSH because the login-session Tart agent
255is unavailable before console login.
256
257All machine-specific state is under
258`/Volumes/Documents/VM/macOS Code Sandbox/icloud-export`. `state` is its
259isolated Tart home; the VM name is `icloud-export`. A symlink in
260`~/.tart/vms/icloud-export` names this same guest. The local guest account is
261`admin`; its generated password and separate SSH key remain in `private`.
262The friend's Apple credentials belong only in `private/apple-account.json`,
263with fields `apple_account` and `password`. The guest does not
264mount that directory.
265
266`private/run-vm.py` runs only this guest. Launch it detached through the
267existing localhost SSH route so it survives the controlling shell. It mounts
268`transfer/tooling` read-only and `transfer/output` read/write. The guest sees
269these as `/Volumes/My Shared Files/tooling` and `output`. Host paths outside
270these two directories are not shared with it.
271
272`private/vm-ui.py` takes JSON on stdin for guest screenshots and input.
273It uses vncdotool 1.3.0's Apple Screen Sharing authentication over SSH, with
274a guest-specific key and pinned host key. Its Unix viewer
275socket stays inside `private`. The physical Mac can remain locked. The
276temporary Tart installer VNC listener is stopped, its credentials are removed,
277and normal pilot starts omit `--vnc-experimental`. The temporary legacy VNC
278password was also disabled and removed; the console uses the local account.
279
280Native Screen Sharing maps `super` to Command. Its `clear` operation uses
281Command–A then Backspace; individual character events are paced. A public
282test containing upper/lowercase letters, underscore, hyphen, plus, and an
283email address was reproduced exactly. The sign-in field and guest clipboard
284were cleared after the check. Guest login and console reconnection passed
285after the OS update and the final VM restart. Automatic login remains disabled. The native console
286controller is now closed; the headless guest desktop remains active and SSH
287metadata access remains available. No host VNC listener is enabled.
288
289For guest commands, use the installed Tart binary with
290`TART_HOME="/Volumes/Documents/VM/macOS Code Sandbox/icloud-export/state"`
291and `exec icloud-export COMMAND`. The guest agent runs only its command RPC;
292clipboard integration is off. SSH offers a second command route and supplies
293the private screen-sharing tunnel. Never point the existing devbox installer
294or its lifecycle commands at this pilot.
295
296The synthetic transfer fixture was removed after passing its hash and mtime
297checks. Pilot evidence is in `evidence/pilot-nas-validation.json`.
298
299The iPhone passcode file was deleted after successful authorization. Never
300reset encrypted iCloud data. Read credentials only inside the input worker;
301never print or display their files. Authentication captures use restricted
302regions and mask inputs before writing images. Never capture a media window.
303
304`private/vm-ui.py --stream` maintains one guest console connection across JSON
305lines. Do not infer the guest lock state from the VNC connection status. Verify
306it through `ioreg`; automatic login remains disabled. Controller
307capture defaults to false. A menubar-only crop can verify the active app
308without exposing recording names. Start the controller PTY with `stty -icanon -echo`; macOS canonical input
309truncates long JSON lines. Counted Up/Down steps are supported, and counted
310Backspace requires the locked-guest guard. Explicit paced Shift combinations
311handle the US keyboard layout without exposing credentials.
312
313Voice Memos' migrated local store was preserved intact at guest
314`/Users/admin/iCloudExportRecovery/Recordings-before-refetch`. A fresh
315`Recordings` store has imported all 358 catalog rows. Its first stable pass
316retrieved 103 media paths, including 18 `.qta` recordings absent from the
317macOS 15 store. This changes only the local cache, not iCloud records.
318The fresh assets require native decoding/export for formats unsupported by the
319host FFmpeg; a decode failure alone does not establish source corruption.
320Launch GUI apps through the guest agent in the completed desktop session.
321OSLaunchdErrorDomain 125 was observed before console login completed.
322Keep all credential input on stdin. The optional post-upgrade assistant was
323closed without approving extra cloud services.
324
325`private/memo-export.py` reports aggregate status without arguments; `recovered`
326copies and validates stable files from the current native store. Selection uses
327the native playable predicate, including sub-0.1-second recordings. Existing
328validation is reused only for identical hashes, recording times, and catalog
329durations. Initial `pilot` and `available` inventories are historical evidence.
330`native-audio-check.py` validates every QTA audio track with the native decoder;
331PCM scratch is discarded after checking frame counts and duration.
332`build-portable.py` preserves original media and builds packet-identical stereo
333copies. `publish-macOS26.py` has completed its sealed native snapshot publication;
334do not replay it against the existing native destination. Future updates must
335preserve the flat public layout and private recovery material. Report 310 of
336348 complete recordings until the remaining 38 reconcile.
337
338### Photos without another Apple login
339
340The friend is unavailable for another verification code. The separate web-login
341worker has stopped, its pending request was denied in the VM, and its unused
342NAS password copy and cookies were removed. No additional code was requested.
343Use Photos in the already signed-in VM with Optimize Mac Storage, exporting
344requested originals and edits in batches through native Photos/PhotoKit or
345OSXPhotos.
346Photos is now enabled in that same Apple session and has begun importing its
347catalog. Optimize Mac Storage is selected and independently verified through
348its native settings control. Keep the library on the guest's local APFS disk
349and send exported files to Snow Globe. Apple advises against putting a Photos
350library on a network share.
351
352The catalog holds 38,142 assets: 37,804 visible (1,823,889,190,154 original
353bytes; 4,544 videos, 1,348 edited, 363 Live Photos), 303 hidden burst members,
354and 35 in Recently Deleted. No shared albums exist. OSXPhotos 0.78.0 runs in the
355guest from `~/.local/bin` (installed with a checksum-verified uv 0.9.7).
356
357PhotoKit needs Photos access from a responsible app with a usage string; CLI
358processes under SSH or Terminal are refused or time out before the prompt can be
359answered. `transfer/tooling/PhotoExport.app` is an ad-hoc-signed launcher
360(`net.paperclover.icloud-export.photos`) that spawns its arguments, and holds
361full Photos access in the guest. Terminal also received access while this was
362diagnosed. Prompts are `UserNotificationCenter` alerts; answer them with
363`run_steps` by button name, never with screenshots.
364
365Guest `~/PhotoExport` holds the private inventory, 110 date-ordered UUID batches
366(about 15 GB each; `b0000` is a 30-asset pilot), and per-batch logs, reports,
367and export databases. `driver.py` (from `private/photos-driver.py`), started by
368`open -n ~/Applications/PhotoExport.app --args /bin/sh ~/PhotoExport/run.command`,
369exports each batch with `--download-missing --use-photokit`, year directories,
370`{date}_{uuid}` names, XMP sidecars, and `--touch-file`, then repeats it once
371with `--update`. It keeps at most one unshipped batch and pauses below 25 GB
372free; `~/PhotoExport/STOP` halts it between batches. On the host,
373`private/photos-ship.py` streams each finished batch to Snow Globe, verifying
374every file's size, SHA-256, and mtime. Media goes to `Photos/<year>` in the
375delivery share; sidecars, reports, and export databases go to the private
376`Photos metadata/<batch>`. Only then is guest staging deleted. Logs:
377`evidence/photos-ship.log` and guest `driver.log`.
378
379The pilot exported 223 files (30 assets and their burst members) with no
380missing or failed items, and passed mtime, year-folder, and file-signature
381checks before shipping.
382
383Downloaded originals stay in the Photos cache until space pressure. In this
384guest the per-user `deleted` (CacheDelete) agent had a stale volume object
385("unable to validate volume"), so nothing was purgeable and Photos never evicted,
386even at 3 GB free. Killing that agent, then quitting and reopening Photos, made
387the cache purgeable; pressure at about 6 GB free then evicted 9.5 GB within
388seconds. Before each batch below 40 GB free, the driver preallocates a temporary
389balloon down to 6 GB, waits 30 s, and deletes it. If purges stop freeing space,
390check `NSURLVolumeAvailableCapacityForImportantUsageKey` and the `deleted` log first.
391
392A full local download of the 1.7 TB is not a prerequisite for this optimized
393route. Originals are temporarily downloaded by Photos as exports request them;
394Photos manages eviction. Cache reclamation is asynchronous, so verify bounded
395space use in a trial, monitor free space, and pause before exhausting the guest
396or host disk. Do not use OSXPhotos `--cleanup` to evict cloud originals: it
397removes files from the export destination.
398[Apple Photos and iCloud](https://support.apple.com/en-euro/guide/photos/phtf5e48489c/mac) ·
399[OSXPhotos maintainer on optimized storage](https://github.com/RhetTbull/osxphotos/discussions/1926) ·
400[Apple library-storage requirements](https://support.apple.com/en-us/108345).
401
402The alternative web downloader remains installed privately on Snow Globe:
403upstream `icloudpd` 1.33.0 at
404`/srv/clover/.icloud-export-private/2026-11-04/tools/icloudpd-1.33.0`, pinned to
405commit `3caeae584c3d8bc6fcc51a1c9f7e5413b1422fd3`. It requires its own Apple web
406session. Keep its cookies and credentials private; leave ADP enabled. It is
407not the current route while the friend cannot supply another code.
408
409### Remaining memo recovery checks
410
411The cached CloudKit records use LZFSE/LZVN compression. After decompression,
412all 38 exception assets' recorded sizes match the local files, including the
413five empty files. Their latest cloud modification predates this export:
41424 February 2026. Each has one audio-future generation in the cached index.
415No additional Voice Memos audio was found in its app/group stores. Native Core
416Audio decoding of all 33 nonempty exceptions recovers 19 short streams but
417none of the full catalog durations; the other 14 fail native decoding too.
418Some short files contain additional `mdat`/`moof` fragments, retained for repair
419experiments. The audio was never uploaded. The 38 cloud assets hold 1,221,488 bytes
420for 9.7 catalog hours, and each record's own local duration is 0–22 seconds,
421whereas healthy recordings match their catalog duration. No Mac-side or iCloud
422copy can supply the rest; only the iPhone (or a local backup of it) can, if its
423copy is complete. A normal app export has now been attempted for all 38: all 19 short files
424remain short, 18 supply no audio, and one empty item cannot be copied through
425the UI. Repair with a pinned build of `anthwlock/untrunc` produced decodable
426fragments for all 14 header-only files (16.831 seconds across their longest
427candidates), but no full catalog durations. The original 19 short files remain
428better than their reconstructed variants and are retained untouched.
429Evidence is `recovery-cloud-metadata-validation.json`, with private atom,
430packet, and native-decoder inventories. Never display media, names, or locations.
431
432## What each person does
433
434Once the export login is controllable, the agent can configure the export,
435oversee sync and retries, read the authorized memo database, restore names
436and dates, verify files, and manage the restricted NAS delivery.
437
438Guest installation and its local account can be prepared from Clover's
439existing host session. The separate host-account alternative requires Clover
440to authenticate account creation: the host shell runs as `clo` without
441passwordless sudo.
442
443The friend keeps the subscription active, enters Apple credentials and any
444encrypted-data authentication, approves two-factor requests on the iPhone,
445checks source counts, and tests Windows playback. Remote sign-in needs both
446credentials and two-factor approval; approving a notification is insufficient
447on its own.
448
449Current Cua Driver checks pass for Accessibility, Screen Recording permission,
450AX access, and the WindowServer connection in Clover's login. The locked host
451desktop refused GUI control; the pilot's independent guest console works.
452For the host-account alternative, first login and
453the GUI controller's access must be established in that login, including any
454per-user permissions. Keep the existing Knife session available in Clover's
455account. A host account switch must not log it out.
456These setup and authentication steps can be done through a trusted remote
457connection while the Mac is powered on and reachable; they do not inherently
458require a visit to the machine.
459
460Use the restricted credential file or a private remote-control session into
461the export VM for password entry. Device-passcode entry and two-factor
462approval may still require the friend. Credentials, recovery keys, and verification codes stay out of
463chat, shell arguments, repository files, and export logs. The temporary Mac
464becomes an authorized device with access to the account; the friend should
465understand that before signing in.
466
467## Before downloading
468
4691. Confirm the friend's iOS version and whether Advanced Data Protection is
470 enabled. The requested storage split is 9 GB of memos and about 1.7 TB of
471 Photos; other account categories are outside the export.
4722. Record Photos' photo/video counts with filters cleared. Inventory Hidden,
473 personal and Shared Photo Libraries, Shared Albums, bursts, RAW pairs,
474 edited media, and anything wanted from Recently Deleted. Do the same for
475 Voice Memos, including custom folders and Recently Deleted. Do not restore
476 deleted items without the friend's choice.
4773. Confirm both Photos and Voice Memos are enabled in iCloud on the iPhone
478 and have finished uploading. Phone-only items cannot be fetched from
479 iCloud. Resolve upload errors on the source first.
4804. Agree how future photos and recordings will be protected after migration.
481 A one-time export does not upload tomorrow's recordings.
482
483## Host account alternative
484
485Use a new standard user dedicated to this export, with its normal home under
486`/Users` on the startup disk. The 9 GB memo store does not justify a custom
487home directory or another APFS volume. Clover authenticates account creation
488through the Mac's administrator prompt; do not put passwords in command
489arguments. Test first login and the account's GUI control before sign-in.
490
491Sign into the friend's Apple Account in that user, leaving Clover's normal
492account signed in separately. Review the iCloud app switches during setup and
493enable Voice Memos only. Keep Photos and Drive sync off for this phase. Leave
494Find My, Passwords and Keychain, Messages, and purchase-account sign-in out
495of this setup. Decline merging unrelated local data.
496
497Only for the native Photos fallback, create the friend's Photos library in
498a private folder on the existing Project/Documents APFS disk after freeing
499space. Make it that user's System Photo Library, enable iCloud Photos, and
500select **Download Originals to this Mac**. Keep the Mac powered, awake, and
501connected during downloads. Do not put the live `.photoslibrary` on the NAS:
502Apple requires appropriate local Mac-formatted storage and warns against
503network storage for live libraries.
504[Apple's library storage requirements](https://support.apple.com/en-us/108345).
505
506A cable import would require the iCloud originals to be downloaded onto the
507iPhone first. It is therefore unsuitable for this 1.7 TB photo library; a cable
508copy does not inherently mean all embedded camera metadata is lost.
509[Apple's PC import requirements](https://support.apple.com/en-us/120267).
510
511Layered Voice Memos require at least macOS 15.2. The initial macOS 15 guest
512proved insufficient for this catalog; the phone runs iOS 26.6.1, and a macOS 26
513compatibility test is underway.
514[Apple's Voice Memos playback requirements](https://support.apple.com/en-euro/guide/voice-memos/vma2c8c0a040/mac).
515
516## Prove a small export first
517
518For the memo pilot, test ordinary, renamed, trimmed, long, and layered
519recordings where present. Include a recording close to New Year or a timezone
520boundary.
521
522For the subsequent Photos trial, test an old and a new photo, duplicate
523original filenames, an edited picture, a Live Photo, a video, a burst, a RAW
524pair if present, a hidden item, and a date corrected in Photos.
525
526The friend opens the pilot on Windows and checks appearance, sound, edits,
527duration, filenames, and dates. An original plus its edited rendition are two
528intentional versions. A Live Photo's image and video must both survive.
529Full-resolution originals and all companions must exist; thumbnails and a
530matching item count are insufficient.
531
532## Native Photos fallback
533
534Use [OSXPhotos](https://rhettbull.github.io/osxphotos/index.html) to export the
535downloaded library. Its year and UUID templates give ordinary directories
536without filename collisions. Keep original names and album membership in
537metadata rather than duplicating media into every album.
538[Template reference](https://rhettbull.github.io/osxphotos/template_help.html).
539
540Run the export as the operator with read access to the friend's downloaded
541library and write access to the destination. Using Clover's operator account
542for this phase keeps NAS credentials outside the temporary Apple login.
543Grant access only to the agreed source tree; macOS privacy permissions may
544also be required. Always pass the explicit library path. Do not let the tool
545fall back to Clover's own Photos library.
546
547Example paths below are placeholders. Confirm the NAS mount and its backing
548host before creating the destination. Keep export state on the Mac; only
549validated portable media belongs in the public share. Keep catalog metadata
550and native recovery files in restricted staging outside it.
551
552```sh
553uv tool install 'osxphotos==0.78.0'
554
555LIBRARY='/Volumes/Documents/iCloud Export/Photos Library.photoslibrary'
556WORK="$HOME/Library/Application Support/iCloud Export"
557mkdir -p "$WORK/photo-staging"
558chmod 700 "$WORK"
559
560osxphotos info --library "$LIBRARY" --json > "$WORK/photos-info.json"
561osxphotos query --library "$LIBRARY" --json > "$WORK/photos.json"
562
563osxphotos export "$WORK/photo-staging" \
564 --library "$LIBRARY" \
565 --directory '{created.year}' \
566 --filename '{created.strftime,%Y-%m-%d_%H%M%S}_{uuid}' \
567 --touch-file \
568 --sidecar xmp \
569 --keyword-template '{folder_album(>)}' \
570 --exportdb "$WORK/photos-export.db" \
571 --update \
572 --report "$WORK/photos-export.csv"
573```
574
575Capture dry-run and tool diagnostics privately; return only aggregate progress
576and integrity results. Never display private paths or media metadata. Add
577`--limit 30` for a pilot, then remove the limit for the full export. Retain the
578export database and options for retries and the final update. Keep dated
579copies of reports before subsequent runs replace them. Defaults export
580originals, edited versions, Live Photo videos, burst members, and associated
581RAW images; edited files receive an `_edited` suffix. `--touch-file` sets the
582media mtime from the Photos date. Do not enable overwrite or cleanup options.
583[Export command reference](https://rhettbull.github.io/osxphotos/cli).
584
585Wait for full hydration and inspect missing/error reports. For exceptions,
586OSXPhotos has `--download-missing`, but its documentation says this does not
587retrieve every burst member. Resolve every missing companion explicitly.
588For network exports, a local export database avoids putting its SQLite writes
589on SMB. XMP sidecars preserve catalog metadata without rewriting original
590media bytes; the JSON inventory keeps the broader catalog information. Retain
591these sidecars and inventories outside the public media-only share.
592[Export and metadata tutorial](https://rhettbull.github.io/osxphotos/tutorial.html).
593
594Apple's built-in fallback is **File → Export → Export Unmodified Original**
595with **Export IPTC as XMP**, plus a separate export of edited versions. A Live
596Photo exports as an image and a video. This fallback still needs inventory
597matching and timestamp correction for the proposed layout.
598[Apple's export procedure](https://support.apple.com/en-gb/guide/photos/pht6e157c5f/mac).
599
600## Export Voice Memos
601
602The database contains metadata; audio lives in separate files. Enable Voice
603Memos sync in the temporary user, open the app, and reconcile its recordings
604with the iPhone. A listed recording is not proof its audio is downloaded.
605[Apple's sync procedure](https://support.apple.com/en-ie/guide/voice-memos/vma6cc4d0571/mac).
606
607The current common store is:
608
609```text
610~/Library/Group Containers/group.com.apple.VoiceMemos.shared/Recordings/
611 CloudRecordings.db
612 CloudRecordings.db-wal when present
613 CloudRecordings.db-shm when present
614 media and supporting files
615```
616
617Preserve a private copy of the entire native store, including editing and
618supporting files. Use SQLite's backup API through a read-only source connection
619to obtain a consistent database snapshot including committed WAL data. Copying
620only the live `.db`, or opening it with `immutable=1`, can miss recent changes.
621Quiesce app edits during the copy and reconcile again afterward.
622[SQLite's backup API](https://www.sqlite.org/backup.html).
623
624Inspect the copied schema before implementing extraction. Published exporters
625use `ZCLOUDRECORDING`, `ZUNIQUEID`, `ZPATH`, `ZDATE`, and `ZDURATION`; one
626documents `ZEVICTIONDATE` for deleted state. These are reverse-engineered
627details, so verify their meaning against this account's app views. Titles can
628occupy different fields; preserve them mechanically in private metadata without
629displaying them. Native `.m4a` and `.qta` files both occur.
630[Exporter schema and SQLite handling](https://github.com/grapeot/voice-memos-skill/blob/master/docs/rfc.md).
631
632The extraction contract is:
633
634```text
635for each wanted logical recording in the database:
636 require a local, nonempty, decodable media file
637 reconcile duration, active/deleted state, and recording ID
638 preserve the original title privately without operator inspection
639 recording instant = ZDATE + 978307200 seconds since Unix epoch
640 copy native media without changing its bytes
641 export a rendered copy when edits/layers/effects need rendering
642 name portable output YYYY-MM-DD_HHMMSSZ_unique-id.ext
643 set mtime from the recording instant
644 record ID, full title, date, duration, folder, variant, and output path
645```
646
647The epoch conversion applies only after confirming `ZDATE` uses Apple's 2001
648epoch in this store. Use UTC for memo filenames and year folders if historical
649recording timezones are unavailable; preserve any known timezone separately.
650Opaque UUID filenames avoid Windows-invalid characters and private titles or
651locations in paths. Preserve full titles privately. Do not merge duplicate
652titles or infer dates from download times.
653
654An ordinary native M4A is suitable when its playback matches the app. Compare
655trimmed recordings too: raw file copying alone must not silently discard edits.
656For special recordings, export both the native editable source and a rendered
657M4A. Apple documents rendered sharing as mixing effects and layers into one
658file. The remote friend can select recordings on their iPhone, choose **Share
659→ Options → Rendered**, save a manageable batch locally, and upload it through
660a temporary folder-scoped HTTPS upload link. Export editable versions too when
661the source is not already preserved on the Mac. No physical proximity is
662needed; do not ask the phone to download the entire photo library.
663[Apple's multiple-recording sharing options](https://support.apple.com/en-mt/guide/iphone/iph3d6dc359/ios).
664
665Match these uploads to the database inventory before renaming or assigning
666dates; resolve duplicate titles explicitly. Export existing transcriptions as
667text when wanted, and retain their native source. Do not generate replacement
668transcripts as part of copying recordings.
669
670## Windows files and dates
671
672The agreed public layout contains media only. Photos is added after its trial:
673
674```text
675iCloud Export - 2026-11-04/
676 Voice Memos/
677 2025/
678 2025-06-12_212233Z_<id>.m4a
679 2026/
680 2026-06-12_212233Z_<id>.m4a
681 2026-06-12_212233Z_<id>.qta
682 Photos/
683 2024/
684 2024-06-12_142233_<uuid>.heic
685 2024-06-12_142233_<uuid>.mov
686 2024-06-12_142233_<uuid>_edited.jpeg
687```
688
689Native sources, catalog snapshots, names, album mappings, sidecars, validation
690manifests, and recovery tools stay outside the shared root. No Pilot, Available,
691Metadata, native-store, or tool folders belong in the public delivery.
692
693Preserve HEIC, JPEG, RAW, MOV, MP4, and native audio formats. HEIC/HEVC playback
694depends on the Windows apps and installed codecs; test the actual PC.
695[Microsoft's HEIF and HEVC guidance](https://support.microsoft.com/en-us/onedrive/upload-heif-and-hevc-photos-and-videos-to-onedrive).
696For audio and video, [VLC](https://www.videolan.org/vlc/features.html) is a
697useful pilot player. If needed, add separate JPEG or H.264 viewing copies with
698the same asset IDs and dates. Preserve the originals; JPEG, stereo mixes,
699and video transcodes do not retain every RAW, HDR, depth, or spatial property.
700Measure derivative space before converting a whole library.
701
702Use the Photos catalog's creation date and timezone for photo folders and
703mtime, including dates the friend corrected in Photos. Preserve the original
704embedded EXIF even when it disagrees; XMP and the inventory carry catalog
705corrections. Mtime is a UTC instant, while Windows displays it in the viewer's
706timezone. Windows Date created is a separate field and may reflect download
707time. Other documents retain their actual modification dates.
708
709In the restricted recovery store, create a manifest record for every delivered
710file with `relative_path`,
711`bytes`, `sha256`, and `mtime_utc` as ISO 8601 with an explicit UTC offset.
712Link media to logical IDs and variant roles in the inventories. Exclude the
713manifest itself from its own listing. Keep paths relative and check case
714collisions and Windows path limits. Generate and verify the manifest only
715after files and metadata are final.
716
717The public share’s TAR download preserves UTC recording timestamps to whole
718seconds in both archive metadata and a real extraction test. Its URL is the normal share URL
719plus `Voice%20Memos/?tar`; the archive contains `Voice Memos/<year>/...` only.
720Copyparty’s generated ZIP carries UTC DOS dates and a legacy Unix extra field
721that the tested extractor did not honor, shifting extracted mtimes. Use TAR
722for the handoff and have the friend verify the resulting Windows dates.
723Private evidence: `evidence/tar-time-validation.json`.
724
725Individual browser downloads may reset filesystem timestamps. Prefer the
726verified TAR route and check the friend's actual Windows extraction. Downloads
727should use manageable batches or individual-file range requests when necessary.
728
729## Snow Globe sharing
730
731Place portable output under `/srv/clover/Share/iCloud Export - 2026-11-04`, visible
732as `/Volumes/clover/Share/iCloud Export - 2026-11-04` on this Mac. Use the existing
733[Shares manager and Copyparty](copyparty.md), with a password and read-only
734permission scoped to that folder. Keep native app databases, credentials, and
735temporary writable uploads outside the read-only delivery link. Retain existing
736numeric owners; do not recursively change ownership of Clover's files.
737
738Create the delivery link after validation. Test its access from a signed-out
739client, including denial of parent and sibling folders, a large download,
740resume behavior, and the Windows timestamp procedure. For remote uploads of
741exceptional memos, use a separate restricted upload folder and revoke that
742access after reconciliation. This uses existing services and needs no new
743permanent iCloud container.
744
745## Direct Photos trial
746
747[The linked project](https://github.com/boredazfcuk/docker-icloudpd) wraps
748`icloudpd` in an Alpine container, adding scheduled downloads, credential and
749session storage, notifications, and optional conversions. Its scope is Photos
750and videos, including Live Photo components; it does not export Voice Memos
751or the whole account.
752
753Its README still says ADP must be disabled. In contrast, current upstream
754`icloudpd` documents `--adp-web-access`, introduced in 1.33.0, with approval
755from a trusted device and iCloud web access enabled. This is an upstream
756capability, not proof the wrapper image supports that option. Use a pinned
757version and prove the actual account flow; the plan does not require disabling
758ADP.
759[Upstream authentication documentation](https://icloud-photos-downloader.github.io/icloud_photos_downloader/authentication.html).
760
761For a direct-to-NAS pilot, run upstream in one-time copy mode with private
762authentication state. Retrieve original and adjusted assets, RAW alternatives,
763and Live Photo components from every wanted library; enumerate Shared Albums
764separately. Reconcile asset IDs and all variants. Shared Albums can contain
765reduced-resolution copies; obtain originals from their owner where needed.
766Regular album membership is not a reason to store the same original repeatedly.
767[Upstream features and modes](https://github.com/icloud-photos-downloader/icloud_photos_downloader).
768
769With upstream **1.33.0**, pass these trial options to the parser inside the
770private worker. Account fields come from the restricted file into process memory;
771keep credentials out of OS command arguments and keep all tool output private.
772The sample and sidecars stay outside the delivery share until verified.
773`--recent 30` is only for the trial; remove it for the full run and final
774reconciliation. Repeat for each library returned by `--list-libraries`, with a
775distinct destination per library.
776
777```sh
778--adp-web-access \
779 --cookie-directory '/srv/clover/.icloud-export-private/2026-11-04/photos-session' \
780 --directory '/srv/clover/.icloud-export-private/2026-11-04/Photos trial' \
781 --folder-structure '{:%Y}' \
782 --file-match-policy name-id7 \
783 --live-photo-mov-filename-policy original \
784 --size original --size adjusted --size alternative \
785 --live-photo-size original \
786 --xmp-sidecar \
787 --log-level error \
788 --recent 30
789```
790
791The repeated sizes request originals and available edits/RAW alternatives.
792The ID naming policy distinguishes repeated original filenames. Verify all
793IDs and variants, reported creation instants, mtimes, XMP contents, and year
794boundaries before accepting this route. Preserve the downloaded names unless
795the inventory supplies an unambiguous mapping to a new layout.
796[Asset sizes](https://icloud-photos-downloader.github.io/icloud_photos_downloader/size.html) ·
797[File naming](https://icloud-photos-downloader.github.io/icloud_photos_downloader/naming.html) ·
798[Command reference](https://icloud-photos-downloader.github.io/icloud_photos_downloader/reference.html).
799
800Keep cloud-deleting and local-mirroring features disabled: `auto_delete`,
801`delete_after_download`, `keep_icloud_recent_only`, and `delete_accompanying`;
802leave `keep_icloud_recent_days` unset. The wrapper configuration warns about
803filename collisions with flat or year-only download directories. Upstream
804documents automatic name de-duplication, so verify the pinned behavior with
805duplicate names instead of assuming either statement describes every build.
806Normalize to the final year/ID layout only after inventory reconciliation.
807Do not rewrite EXIF on archived originals.
808[Wrapper configuration](https://github.com/boredazfcuk/docker-icloudpd/blob/master/CONFIGURATION.md).
809
810## Finish before canceling
811
8121. Reconcile every wanted logical item and its expected files, including edits,
813 RAW pairs, bursts, Live Photos, hidden items, shared content, and special
814 recordings. Resolve all missing, empty, corrupt, and failed exports.
8152. Probe all audio/video files and decode-check images; inspect representative
816 playback on Windows, including long videos and trimmed or layered memos.
817 Have the friend compare their important items with the source app.
8183. Verify every delivered file's size and SHA-256 on Snow Globe and on the
819 friend's Windows copy. Check all media mtimes against the inventories,
820 allowing only the destination filesystem's timestamp precision.
8214. Take a final incremental pass after a short pause in new recordings and
822 edits, then reconcile new items and regenerate manifests. Keep the native
823 Mac stores until handoff succeeds.
8245. Establish independently verified copies on separate devices before cloud
825 deletion. With the native route, retain the Mac source, NAS delivery, and
826 friend's Windows copy through the deletion. ZFS redundancy is not another
827 copy, and this repository does not configure automatic snapshot retention.
828 Before destructive storage cleanup, satisfy [AGENTS.md](../AGENTS.md):
829 verify application/data recovery and any needed keys from two restricted
830 off-server copies stored separately. Apply this to relocated Clover data
831 too; do not count an unverified NAS move as permission to erase its source.
8326. The friend chooses when to stop syncing, remove cloud content, and downgrade.
833 Confirm remaining use fits the chosen plan and that future backups have a
834 destination. Never use the downloader to delete automatically during export.
835 The screenshot also shows 35.7 GB of backups; exporting the two media
836 categories does not by itself make the account fit the free 5 GB tier.
8377. Sign out inside the export VM or fallback host account. Remove the exact
838 export device from the friend's trusted devices and revoke downloader
839 sessions and temporary upload access. Remove the export VM or temporary
840 host account and native stores after the verified handoff and the recovery
841 requirements above are satisfied.
842
843**Deleting synced media can delete it on the friend's other devices.** Apple
844also warns that exceeding the downgraded capacity stops iCloud updates and
845backups. Keep cancellation and cloud cleanup after verification.
846[Synced deletion behavior](https://support.apple.com/en-us/108922) ·
847[Downgrade consequences](https://support.apple.com/en-us/108318).
848
849## Planning verification
850
851The Mac version, physical disks, APFS allocation, volume quotas/reserves,
852snapshot lists, and Snow Globe ZFS availability were read without opening
853personal content. OSXPhotos 0.78.0 was run in an isolated temporary tool
854environment for `export --help`, `query --help`, and `info --help`; the flags
855above exist in that build. Upstream icloudpd 1.33.0 was also checked with
856`--help` in isolation, including ADP, asset-size, naming, and XMP options.
857The current Cua Driver is 0.38.0; its status and read-only health checks pass
858in Clover's login. Passwordless host sudo is unavailable. The devbox was
859observed running with Tart 2.41.1; no lifecycle action was taken on it.
860Apple's iCloud requirements and license were checked for the VM route.
861The dedicated guest has passed command, screen capture/input, read-only
862tooling mount, writable transfer, AAC decode, and NAS hash/mtime checks.
863Apple sign-in, native sync, the friend's memo schema, and NAS byte/timestamp
864checks have passed. Full library reconciliation and the friend's Windows
865playback/date check remain required; Clover reports the Windows test has not
866yet been performed.
tools/router.py-55
......@@ -40,55 +40,6 @@ def proxy(upstreams, indent, uncompressed=False, host=None, keep_streams=False):
4040 return [*lines, f"{indent}}}"]
4141
4242
43def cliproxy_routes(host, upstreams, auth, keys, role):
44 native = [" request_header -Cookie", *proxy(upstreams, " ", keep_streams=True)]
45 if not role:
46 return [" @dashboard path / /management.html /sign-out /snow.oauth2/*",
47 " respond @dashboard 404", " request_header -Cookie", *proxy(upstreams, " ", keep_streams=True)]
48 if any(not re.fullmatch(r"[0-9a-f]{64}", keys.get(name, "")) for name in ("api_key", "management_key")):
49 raise ValueError("missing CLIProxyAPI gateway credentials")
50 return [
51 " route {",
52 " handle /snow.oauth2/* {", *proxy(auth, " "), " }",
53 " @credentials {",
54 " not path / /management.html /sign-out /snow.oauth2/*",
55 ' expression `{http.request.header.Authorization} != "" || {http.request.header.X-Management-Key} != "" || {http.request.header.X-Api-Key} != "" || {http.request.header.X-Goog-Api-Key} != ""`',
56 " }",
57 " handle @credentials {", *native, " }",
58 " @session path / /management.html /sign-out /v0/management/* /v8/management/* /v1/models",
59 " handle @session {",
60 f" reverse_proxy {auth} {{",
61 " method GET", " rewrite /snow.oauth2/auth",
62 " header_up X-Forwarded-Method {method}", " header_up X-Forwarded-Uri {uri}",
63 " @unauthorized status 401",
64 " handle_response @unauthorized {",
65 " @dashboard path / /management.html /sign-out",
66 " route {",
67 " redir @dashboard /snow.oauth2/sign_in?rd={scheme}://{host}{uri}",
68 " respond 401", " }", " }",
69 f" @allowed header X-Auth-Request-Groups *role:{role}*",
70 " handle_response @allowed {",
71 " method {method}", " rewrite {uri}",
72 " @foreign_origin {", " header Origin *", f" not header Origin https://{host}", " }",
73 " @unsafe_origin {", " method POST PUT PATCH DELETE", f" not header Origin https://{host}", " }",
74 " route {",
75 " respond @foreign_origin 403", " respond @unsafe_origin 403",
76 " redir / /management.html 302",
77 f" redir /sign-out /snow.oauth2/sign_out?rd=https%3A%2F%2Fsnowglobe.{os.environ['SNOWGLOBE_DOMAIN']}%2Fsign-in 303",
78 " @management path /v0/management/* /v8/management/*",
79 f' request_header @management Authorization "Bearer {keys["management_key"]}"',
80 " @models path /v1/models",
81 f' request_header @models Authorization "Bearer {keys["api_key"]}"',
82 " request_header -Cookie",
83 f" reverse_proxy {upstreams} {{",
84 " lb_try_duration 5s", " stream_close_delay 8h",
85 " header_down -Access-Control-Allow-Origin", ' header_down Cache-Control "no-store"',
86 " }", " }", " }",
87 " handle_response {", " respond 403", " }", " }", " }",
88 " handle {", *native, " }", " }",
89 ]
90
91
9243def shale_mcp_routes(port):
9344 return [" @shale_mcp_link path_regexp shale_mcp_link ^/-/snowglobe-mcp/([A-Za-z0-9_-]{43})$",
9445 " handle @shale_mcp_link {", " rewrite * /oauth/shale/link/{re.shale_mcp_link.1}",
......@@ -337,12 +288,6 @@ def render(token):
337288 service = next(iter(route["services"]))
338289 if not re.fullmatch(r"[a-z][a-z0-9-]*", service):
339290 raise ValueError(f"invalid service name: {service}")
340 if service == "cliproxy" or re.fullmatch(r"cliproxy-preview-[0-9a-f]{8}", service):
341 with open(os.path.join(ROUTE_DIR, service + ".json")) as source:
342 keys = json.load(source).get("cliproxy", {})
343 lines += [*cliproxy_routes(host, " ".join(sorted(route["upstreams"])),
344 " ".join(sorted(auth_upstreams)), keys, role), "}"]
345 continue
346291 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
347292 # Keep the Origin guard before every static/proxy handle;
348293 # otherwise Caddy sorts those handles ahead of respond.
tools/snowglobe.py-2
......@@ -626,8 +626,6 @@ def provision(data):
626626 write_private(route_dir / f"{data['id']}.json", json.dumps({
627627 "files": static, "dirs": directories, "identity": identity,
628628 "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths,
629 **({"cliproxy": {name: values[name] for name in ("api_key", "management_key")}}
630 if data["sourceId"] == "cliproxy" else {}),
631629 }))
632630
633631
tools/substation.md created+96
......@@ -0,0 +1,96 @@
1# Substation
2
3`https://substation.ai.paperclover.net` serves the Substation dashboard and its
4native APIs. Snow OAuth protects the dashboard and requires `infra-admin`.
5Substation's trusted-proxy mode accepts the identity header from the local
6OAuth proxy. Hub requests authenticate with Substation's own keys.
7
8## Set up accounts
9
101. Open the dashboard and sign in through Snow.
112. Open **Settings → Hub accounts** and add a Claude or Codex account.
123. Follow the provider's sign-in link. For Codex, enter the displayed device
13 code at the provider's page; for Claude, paste the returned code into
14 Substation. Repeat for each account.
154. Check that each account shows its quota and can take requests.
16
17The container includes both providers' login tools. Login scratch directories
18are disposable; the hub saves and refreshes the resulting account files in its
19own store. Production starts empty and imports no CPA accounts or history.
20
21## Connect clients
22
23The client key is saved on Sandwich at
24`~/.config/substation/client-key`, readable only by Clover. SSO mode deliberately
25does not reveal or rotate keys through Substation's dashboard. Nomad variable
26`nomad/jobs/substation` owns the initial `api_key` and `management_key`.
27
28For Claude Code:
29
30```sh
31export SUBSTATION_CLIENT_KEY="$(cat ~/.config/substation/client-key)"
32export ANTHROPIC_BASE_URL=https://substation.ai.paperclover.net
33export ANTHROPIC_AUTH_TOKEN="$SUBSTATION_CLIENT_KEY"
34claude
35```
36
37For Codex, use a separate `CODEX_HOME` to keep its normal sign-in and settings:
38
39```sh
40mkdir -p ~/.codex-substation
41```
42
43Save this as `~/.codex-substation/config.toml`:
44
45```toml
46model = "gpt-6.1-sol"
47model_provider = "substation"
48
49[model_providers.substation]
50name = "Substation"
51base_url = "https://substation.ai.paperclover.net/v1"
52env_key = "SUBSTATION_CLIENT_KEY"
53wire_api = "responses"
54```
55
56Then run:
57
58```sh
59export SUBSTATION_CLIENT_KEY="$(cat ~/.config/substation/client-key)"
60CODEX_HOME="$HOME/.codex-substation" codex
61```
62
63The [official OpenAI configuration documentation](https://learn.chatgpt.com/docs/config-file/config-advanced)
64describes custom providers. Choose a model the connected account supports.
65Substation serves Anthropic Messages and OpenAI Responses; it does not implement
66`/v1/chat/completions`.
67
68T3 Code's hub quota integration uses the same base address and the separate
69management key from `~/.config/substation/management-key` on Sandwich. The
70management API is `/v0/management`; it remains behind its own bearer key. The
71public reverse proxy reaches the hub over loopback, so the hub's remote
72management switch is unnecessary. A client key grants no management access.
73
74## Persistence and deployment
75
76Production state lives under `/srv/prod/substation/data`: `auth` holds provider
77accounts, `hub` holds routing state, `dashboard` holds SQLite history and
78settings, `home` holds the login tools' home, and `run` holds build identities.
79Nomad owns the generated keys and OAuth client credentials. Existing numeric
80service identities remain reserved when a service is retired.
81
82`service/substation/build-source.json` pins a local Jujutsu revision of
83`../substation`. That checkout's `Dockerfile.snowglobe` builds the program and
84includes pinned Claude and Codex login tools. The exporter includes its source
85manifest as build provenance. Snow Globe receives source and builds locally;
86it receives no GitHub SSH key or repository credentials.
87
88Stage with `python3 tools/deploy.py stage substation`. Previews start with empty
89storage, their own keys, and a separate Snow OAuth client. Updates preserve
90that preview's state. Use disposable accounts and prompts when testing.
91Production uses described, conflict-free main and `python3 tools/deploy.py prod`.
92The checkout-dependent updater is unavailable inside the immutable image.
93
94CPA's live dataset, preview, keys, and routing are retired after the replacement
95passes production checks. Restricted off-server recovery copies retain the old
96store; restoring them requires a separate explicit request.