diff --git a/service/cliproxy/Dockerfile b/service/cliproxy/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..59e5b21ffd7f7fc173f80fb279d642dfce011c25 --- /dev/null +++ b/service/cliproxy/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.io/eceasy/cli-proxy-api@sha256:c08d02fbd94927708721afd059bff9ff36a04d9cedb4d69f2c2671aba8239031 AS proxy +FROM docker.io/library/nginx@sha256:17ae566734b63632e543c907ba74757e0c1a25d812ab9f10a07a6bed98dd199c +COPY --from=proxy /CLIProxyAPI/CLIProxyAPI /CLIProxyAPI/CLIProxyAPI diff --git a/service/cliproxy/nginx.conf b/service/cliproxy/nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..6ac1195f0945c38e3dd54d33d6875a87a1e9265a --- /dev/null +++ b/service/cliproxy/nginx.conf @@ -0,0 +1,90 @@ +worker_processes 1; +pid /tmp/nginx.pid; +error_log /dev/stderr crit; +events { worker_connections 512; } +http { + access_log off; + server_tokens off; + absolute_redirect off; + map_hash_bucket_size 128; + server_names_hash_bucket_size 128; + client_body_temp_path /tmp/client; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + map $http_upgrade $connection_upgrade { default upgrade; '' close; } + map "$request_method:$uri" $client_route { + default 0; + ~^GET:/v1/models(?:/[^/]+)?$ 1; + ~^POST:/v1/(?:chat/completions|completions|messages(?:/count_tokens)?|responses(?:/compact)?|alpha/search)$ 1; + "GET:/v1/responses" 1; + ~^POST:/backend-api/codex/(?:responses(?:/compact)?|alpha/search)$ 1; + "GET:/backend-api/codex/responses" 1; + } + map "$request_method:$uri" $browser_oauth_callback { + default 0; + ~^(?:GET|HEAD):/(?:v0/management/oauth-callback|v8/management/oauth/callback|(?:anthropic|codex|antigravity|devin)/callback|callback)/?$ 1; + } + {{ with nomadVar "NOMAD_VARIABLE" }} + map $http_authorization $bearer_key { default 0; "Bearer {{ .api_key.Value }}" 1; } + map $http_x_api_key $claude_key { default 0; "{{ .api_key.Value }}" 1; } + map $client_key $upstream_authorization { default ""; 1 "Bearer {{ .api_key.Value }}"; } + {{ end }} + map "$bearer_key:$claude_key" $client_key { default 1; "0:0" 0; } + upstream cliproxy_backend { + server 127.0.0.1:8317; + } + server { + listen 8080 default_server; + server_name _; + location = /healthz { proxy_pass http://cliproxy_backend; } + location / { return 404; } + } + server { + listen 8080; + server_name ADMIN_HOST; + client_max_body_size 32m; + location = / { return 302 /management.html; } + location / { + if ($browser_oauth_callback = 1) { return 404; } + if ($arg_code != "") { return 404; } + proxy_pass http://cliproxy_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + proxy_read_timeout 3600s; + } + } + server { + listen 8080; + server_name API_HOST; + client_max_body_size 32m; + location = /healthz { proxy_pass http://cliproxy_backend; } + location = / { return 404; } + location / { + if ($client_route = 0) { return 404; } + if ($client_key = 0) { return 401; } + if ($arg_key != "") { return 400; } + if ($arg_auth_token != "") { return 400; } + proxy_pass http://cliproxy_backend$uri$is_args$args; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Cookie ""; + proxy_set_header Authorization $upstream_authorization; + proxy_set_header X-Api-Key ""; + proxy_set_header X-Goog-Api-Key ""; + proxy_set_header X-Management-Key ""; + proxy_set_header X-Forwarded-For ""; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Forwarded-Proto ""; + proxy_buffering off; + proxy_request_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + } +} diff --git a/service/cliproxy/prepare.py b/service/cliproxy/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..efb82d064a674f9e60ba72fc28ed14e377bae904 --- /dev/null +++ b/service/cliproxy/prepare.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import hashlib +import json +import os +from pathlib import Path +import stat +import sys +import tempfile +import urllib.request + + +PANEL_URL = "https://github.com/router-for-me/Cli-Proxy-API-Management-Center/releases/download/v1.26.5/management.html" +PANEL_SHA256 = "57558cd023475674634a114824b014d3e5ac161f6f3791e3d86b6adab962a700" +PANEL_SIZE = 3312180 + + +def prepare(data): + root = Path(data["hostRoot"]) / "data" + static = root / "static" + panel = static / "management.html" + if root.is_symlink() or static.is_symlink() or panel.is_symlink(): + raise ValueError("CLIProxyAPI panel paths must not be symbolic links") + static.mkdir(parents=True, exist_ok=True) + if panel.exists(): + metadata = panel.stat() + if not stat.S_ISREG(metadata.st_mode): + raise ValueError("CLIProxyAPI panel must be a regular file") + if metadata.st_size == PANEL_SIZE and hashlib.sha256(panel.read_bytes()).hexdigest() == PANEL_SHA256: + panel.chmod(0o444) + return + with urllib.request.urlopen(PANEL_URL, timeout=60) as response: + contents = response.read(PANEL_SIZE + 1) + if len(contents) != PANEL_SIZE or hashlib.sha256(contents).hexdigest() != PANEL_SHA256: + raise ValueError("CLIProxyAPI management panel failed checksum verification") + with tempfile.NamedTemporaryFile(dir=static, prefix=".panel-", delete=False) as file: + pending = Path(file.name) + try: + file.write(contents) + os.fchmod(file.fileno(), 0o444) + os.fchown(file.fileno(), data["uid"], data["uid"]) + file.flush() + os.fsync(file.fileno()) + pending.replace(panel) + finally: + pending.unlink(missing_ok=True) + os.chown(static, data["uid"], data["uid"]) + static.chmod(0o755) + + +if __name__ == "__main__": + prepare(json.load(sys.stdin)) diff --git a/service/cliproxy/service.pkl b/service/cliproxy/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a1af0a6d95cf36292347f6024b04c49e64c6ee2a --- /dev/null +++ b/service/cliproxy/service.pkl @@ -0,0 +1,59 @@ +amends "../../config/Service.pkl" + +import "../../config/site.pkl" as site + +name = "CLIProxyAPI" +tagline = "Codex and Claude API access" +access = "infra-admin" +stageIsolation = "fresh" +prepare = "prepare.py" + +secrets { + ["api_key"] {} + ["management_key"] {} +} + +container { + build = "." + entrypoint = "/bin/bash" + args { "/snowglobe/start.sh" } + readOnlyRoot = true + tmpfs { "/tmp:rw,nosuid,nodev,noexec,size=64m" } + http { + containerPort = 8080 + subdomain = "cliproxy" + authRole = "infra-admin" + checkPath = "/healthz" + plainHostnames { "cliproxy-api.\(site.domain)" } + } + volumes { + ["/data"] {} + ["/data/static"] { readOnly = true } + ["/snowglobe/start.sh"] { config = "start.sh" } + } + env { + ["HOME"] = "/data" + ["WRITABLE_PATH"] = "/data" + ["MANAGEMENT_STATIC_PATH"] = "/data/static" + ["TZ"] = "America/Los_Angeles" + } + secretFiles { + ["/run/secrets/cliproxy-seed.json"] = """ + {{ with nomadVar "nomad/jobs/\(module.id)" }} + { + "config-version": 8, + "server": {"host": "127.0.0.1", "port": 8317, "commercial-mode": true, "max-request-body-bytes": 33554432, "discovery": {"enabled": false}}, + "access": {"api-keys": [{{ .api_key.Value | toJSON }}]}, + "management": {"allow-remote": true, "secret-key": {{ .management_key.Value | toJSON }}, "disable-auto-update-panel": true}, + "oauth": {"auth-dir": "/data/auth"}, + "plugins": {"enabled": false, "dir": "/data/plugins"}, + "observability": {"logs": {"debug": false, "logging-to-file": false, "request-log": false}, "usage": {"usage-statistics-enabled": false}, "pprof": {"enable": false}} + } + {{ end }} + """ + ["/run/secrets/cliproxy-nginx.conf"] = read("nginx.conf").text + .replaceAll("NOMAD_VARIABLE", "nomad/jobs/\(module.id)") + .replaceAll("ADMIN_HOST", module.container.http.hostname!!) + .replaceAll("API_HOST", module.container.http.plainHostnames[0]) + } +} diff --git a/service/cliproxy/start.sh b/service/cliproxy/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..fa0afc605a0e6265129a1c98e14c455b7b57cb70 --- /dev/null +++ b/service/cliproxy/start.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -eu +umask 077 + +if [ -L /data/config.yaml ] || { [ -e /data/config.yaml ] && [ ! -f /data/config.yaml ]; }; then + echo 'CLIProxyAPI configuration must be a regular file' >&2 + exit 1 +fi +if [ ! -e /data/config.yaml ]; then + pending=$(mktemp /data/.config.XXXXXX) + trap 'rm -f "$pending"' EXIT HUP INT TERM + cat /run/secrets/cliproxy-seed.json > "$pending" + ln "$pending" /data/config.yaml || test -f /data/config.yaml + rm -f "$pending" + trap - EXIT HUP INT TERM +fi +test -s /data/config.yaml +chmod 600 /data/config.yaml +mkdir -p /data/auth /data/plugins +chmod 700 /data /data/auth /data/plugins +/CLIProxyAPI/CLIProxyAPI --config /data/config.yaml & +app=$! +/usr/sbin/nginx -c /run/secrets/cliproxy-nginx.conf -g 'daemon off;' & +gateway=$! +trap 'kill "$app" "$gateway" 2>/dev/null || :; wait "$app" "$gateway" 2>/dev/null || :' EXIT +trap 'exit 143' TERM +trap 'exit 130' INT +wait -n "$app" "$gateway" +exit 1 diff --git a/service/forward-auth/service.pkl b/service/forward-auth/service.pkl index be5a42fd94d7edd45a13eade8712ff8a0401d7ef..8a7d5f376d80747c44519580ba30b59451c89546 100644 --- a/service/forward-auth/service.pkl +++ b/service/forward-auth/service.pkl @@ -14,7 +14,7 @@ requirements { name = module.name redirectUris { when (isPreview) { "https://\(module.id).\(site.domain)/snow.oauth2/callback" } - when (!isPreview) { for (host in new Listing { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw" }) { + when (!isPreview) { for (host in new Listing { "music"; "seedbox"; "ddns"; "redis"; "pg"; "snr"; "rdr"; "logs"; "metrics"; "traces"; "jkt"; "openclaw"; "cliproxy" }) { "https://\(host).\(site.domain)/snow.oauth2/callback" } } } diff --git a/tools/cliproxy-test.py b/tools/cliproxy-test.py new file mode 100644 index 0000000000000000000000000000000000000000..d20c8cd31f6fc65eb960c8d11b26674de2155bbf --- /dev/null +++ b/tools/cliproxy-test.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +import argparse +import base64 +from contextlib import contextmanager +import hashlib +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +import snowglobe + + +SERVICE = Path(__file__).resolve().parents[1] / "service/cliproxy" +spec = importlib.util.spec_from_file_location("cliproxy_prepare", SERVICE / "prepare.py") +prepare = importlib.util.module_from_spec(spec) +spec.loader.exec_module(prepare) +NGINX = None +API_KEY = "a1" * 32 +MANAGEMENT_KEY = "b2" * 32 +ADMIN_HOST = "cliproxy-preview-12345678.paperclover.net" +API_HOST = "cliproxy-preview-12345678-cliproxy-api.paperclover.net" + + +class ServiceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + result = subprocess.run([os.environ.get("PKL", "pkl"), "eval", "-p", "uid=3150", "-p", "serviceId=cliproxy", "-p", "domain=paperclover.net", str(SERVICE / "service.pkl")], check=True, capture_output=True, text=True) + cls.data = json.loads(result.stdout) + cls.data.update(uid=3150, sourceId="cliproxy", inputs={}) + cls.data.pop("requirements") + cls.data['containers']['app']['image'] = 'localhost/cliproxy:test' + + def test_native_defaults_keep_credentials_private_and_requests_unlogged(self): + app = self.data["containers"]["app"] + seed = app["secretFiles"]["/run/secrets/cliproxy-seed.json"] + seed = re.sub(r"{{ with nomadVar [^}]+}}|{{ end }}", "", seed) + seed = seed.replace("{{ .api_key.Value | toJSON }}", json.dumps(API_KEY)) + seed = seed.replace("{{ .management_key.Value | toJSON }}", json.dumps(MANAGEMENT_KEY)) + config = json.loads(seed) + self.assertEqual(config["access"]["api-keys"], [API_KEY]) + self.assertNotEqual(config["access"]["api-keys"][0], config["management"]["secret-key"]) + self.assertTrue(config["server"]["commercial-mode"]) + self.assertFalse(config["observability"]["logs"]["request-log"]) + self.assertFalse(config["observability"]["pprof"]["enable"]) + self.assertFalse(config["plugins"]["enabled"]) + self.assertEqual(config["oauth"]["auth-dir"], "/data/auth") + self.assertTrue(config["management"]["disable-auto-update-panel"]) + self.assertEqual(app["http"]["authRole"], "infra-admin") + self.assertEqual(app["http"]["plainHostnames"], ['cliproxy-api.paperclover.net']) + self.assertEqual(self.data["stageIsolation"], "fresh") + + def test_rendered_job_has_only_nonroot_private_managed_storage(self): + rendered = snowglobe.render(self.data, {"cliproxy": self.data}) + self.assertEqual(rendered.count('user = "3150:3150"'), 1) + self.assertEqual(rendered.count('readonly_rootfs = true'), 1) + self.assertEqual(rendered.count('perms = "0400"'), 2) + self.assertIn('secrets/cliproxy-seed.json:/run/secrets/cliproxy-seed.json:ro', rendered) + self.assertIn('secrets/cliproxy-nginx.conf:/run/secrets/cliproxy-nginx.conf:ro', rendered) + self.assertIn('server 127.0.0.1:8317;', rendered) + self.assertIn('/data/static:/data/static:ro', rendered) + self.assertNotIn("NOMAD_SERVICE", rendered) + self.assertNotIn("NOMAD_VARIABLE", rendered) + self.assertNotIn("/srv/clover", rendered) + self.assertNotIn("host_port", rendered) + self.assertNotIn('env = true', rendered) + + def test_bootstrap_preserves_native_config_and_provider_credentials(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + data = root / "data" + data.mkdir() + seed = root / "seed.json" + seed.write_text(json.dumps({"access": {"api-keys": [API_KEY]}})) + binary = root / "binary" + binary.write_text('#!/usr/bin/env python3\nimport pathlib,signal,sys\nassert sys.argv[1] == "--config" and pathlib.Path(sys.argv[2]).stat().st_size\nsignal.pause()\n') + binary.chmod(0o700) + gateway = root / "gateway" + gateway.write_text('#!/bin/sh\nexit 0\n') + gateway.chmod(0o700) + script = (SERVICE / "start.sh").read_text().replace('/run/secrets/cliproxy-seed.json', str(seed)).replace('/CLIProxyAPI/CLIProxyAPI', str(binary)).replace('/usr/sbin/nginx', str(gateway)).replace('/data', str(data)) + self.assertEqual(subprocess.run(["bash", "-ec", script], timeout=5).returncode, 1) + config = data / "config.yaml" + self.assertEqual(json.loads(config.read_text())["access"]["api-keys"], [API_KEY]) + self.assertEqual(config.stat().st_mode & 0o777, 0o600) + self.assertEqual((data / "auth").stat().st_mode & 0o777, 0o700) + config.write_text('access:\n api-keys: [edited-key]\nprovider-setting: retained\n') + credential = data / "auth/fixture.json" + credential.write_text('{"synthetic":true}') + seed.write_text('different seed') + self.assertEqual(subprocess.run(["bash", "-ec", script], timeout=5).returncode, 1) + self.assertIn('provider-setting: retained', config.read_text()) + self.assertEqual(credential.read_text(), '{"synthetic":true}') + config.unlink() + config.symlink_to(seed) + result = subprocess.run(["bash", "-ec", script], capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(seed.read_text(), 'different seed') + + def test_panel_pin_rejects_bad_download_without_replacing_existing_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + static = root / "data/static" + static.mkdir(parents=True) + panel = static / "management.html" + panel.write_bytes(b"previous panel") + with patch.object(prepare.urllib.request, "urlopen", return_value=io.BytesIO(b"bad panel")): + with self.assertRaisesRegex(ValueError, "checksum"): + prepare.prepare({"hostRoot": str(root), "uid": os.getuid()}) + self.assertEqual(panel.read_bytes(), b"previous panel") + panel.unlink() + panel.symlink_to(root / "outside") + with self.assertRaisesRegex(ValueError, "symbolic"): + prepare.prepare({"hostRoot": str(root), "uid": os.getuid()}) + + def test_panel_verified_existing_asset_never_contacts_release_feed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + static = root / "data/static" + static.mkdir(parents=True) + panel = static / "management.html" + contents = b"pinned synthetic panel" + panel.write_bytes(contents) + with patch.object(prepare, "PANEL_SIZE", len(contents)), patch.object(prepare, "PANEL_SHA256", hashlib.sha256(contents).hexdigest()), patch.object(prepare.urllib.request, "urlopen", side_effect=AssertionError("unexpected download")): + prepare.prepare({"hostRoot": str(root), "uid": os.getuid()}) + self.assertEqual(panel.read_bytes(), contents) + + +@unittest.skipUnless(NGINX, "pass --nginx to test the real gateway binary") +class GatewayTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="cliproxy-gateway-") + self.addCleanup(self.temporary.cleanup) + root = Path(self.temporary.name) + self.observed = [] + self.secret_markers = [API_KEY] + observed = self.observed + + class Backend(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + self.respond() + + def do_POST(self): + self.respond() + + def respond(self): + observed.append((self.command, self.path, dict(self.headers), self.rfile.read(int(self.headers.get('Content-Length', '0'))))) + if self.headers.get("Upgrade", "").lower() == "websocket": + accept = base64.b64encode(hashlib.sha1((self.headers['Sec-WebSocket-Key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest()).decode() + self.send_response(101) + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", accept) + self.end_headers() + header = self.rfile.read(2) + mask = self.rfile.read(4) + body = self.rfile.read(2) + if header != b'\x81\x82' or bytes(value ^ mask[index % 4] for index, value in enumerate(body)) != b'hi': + raise AssertionError('incorrect client WebSocket frame') + self.wfile.write(b'\x81\x02hi') + self.wfile.flush() + self.close_connection = True + return + if self.headers.get("X-Synthetic-Stream"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(b'data: first\n\n') + self.wfile.flush() + time.sleep(.5) + self.wfile.write(b'data: last\n\n') + self.close_connection = True + return + body = json.dumps({"method": self.command, "path": self.path}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + backend = ThreadingHTTPServer(("127.0.0.1", 0), Backend) + threading.Thread(target=backend.serve_forever, daemon=True).start() + self.addCleanup(backend.server_close) + self.addCleanup(backend.shutdown) + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + self.port = reservation.getsockname()[1] + config = (SERVICE / "nginx.conf").read_text() + config = config.replace('server 127.0.0.1:8317;', 'server 127.0.0.1:' + str(backend.server_port) + ';') + config = re.sub(r'{{ with nomadVar .*?}}|{{ end }}', '', config) + config = config.replace('{{ .api_key.Value }}', API_KEY) + config = config.replace('/tmp/', str(root) + '/').replace('listen 8080', f'listen 127.0.0.1:{self.port}') + config = config.replace('ADMIN_HOST', ADMIN_HOST).replace('API_HOST', API_HOST) + self.assertNotIn('{{', config) + path = root / "nginx.conf" + path.write_text(config) + log = (root / "stderr").open('w+b') + self.addCleanup(log.close) + process = subprocess.Popen([NGINX, '-c', str(path), '-g', 'daemon off;'], stdout=log, stderr=log) + def stop(): + process.terminate() + process.wait(timeout=5) + log.seek(0) + contents = log.read() + for marker in self.secret_markers: + self.assertNotIn(marker.encode(), contents) + self.addCleanup(stop) + for _ in range(100): + if process.poll() is not None: + log.seek(0) + self.fail('nginx failed: ' + log.read().decode()) + try: + connection = socket.create_connection(('127.0.0.1', self.port), .1) + connection.close() + break + except OSError: + time.sleep(.02) + else: + self.fail('nginx did not listen') + + @contextmanager + def request(self, method, path, headers=None, body=None): + connection = http.client.HTTPConnection('127.0.0.1', self.port, timeout=3) + connection.request(method, path, body=body, headers={'Host': API_HOST, **(headers or {})}) + response = connection.getresponse() + try: + yield response + finally: + response.close() + connection.close() + + def test_management_callback_and_path_confusion_never_reach_backend(self): + for path in ['/', '/management.html', '/v8/management/config', '/v0/management/auth-files', '/v0/resource/plugins/x', '/codex/callback?code=synthetic', '/anthropic/callback', '/healthz/../v8/management/config', '/v1/models/../../v8/management/config', '/v1/models/%2e%2e/%2e%2e/v8/management/config', '/v1/models/%2f..%2f..%2fv8%2fmanagement%2fconfig', '/v1/responses/../management']: + with self.subTest(path=path), self.request('GET', path, {'Authorization': 'Bearer ' + API_KEY}) as response: + self.assertIn(response.status, [400, 404]) + response.read() + self.assertEqual(self.observed, []) + + def test_admin_hostname_has_native_panel_and_unknown_hostname_is_denied(self): + with self.request('GET', '/', {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 302) + self.assertEqual(response.getheader('Location'), '/management.html') + response.read() + with self.request('GET', '/management.html', {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 200) + response.read() + count = len(self.observed) + with self.request('GET', '/management.html', {'Host': 'other.example.test'}) as response: + self.assertEqual(response.status, 404) + response.read() + self.assertEqual(len(self.observed), count) + + def test_browser_oauth_callback_credentials_never_reach_upstream_or_logs(self): + code = 'synthetic-oauth-code-must-not-be-logged' + state = 'synthetic-oauth-state-must-not-be-logged' + self.secret_markers.extend([code, state]) + paths = ['/v0/management/oauth-callback', '/v8/management/oauth/callback', '/anthropic/callback', '/codex/callback', '/antigravity/callback', '/devin/callback', '/callback'] + for path in paths: + for method in ['GET', 'HEAD']: + for suffix in ['', '/']: + with self.subTest(path=path, method=method, suffix=suffix), self.request(method, path + suffix + '?code=' + code + '&state=' + state, {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 404) + response.read() + with self.request('GET', path + '?state=' + state, {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 404) + response.read() + with self.request('GET', '/future-provider/callback?code=' + code, {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 404) + response.read() + self.assertEqual(self.observed, []) + payload = json.dumps({'provider': 'codex', 'redirect_url': 'http://localhost:1455/auth/callback?code=' + code + '&state=' + state}).encode() + for path in paths[:2]: + with self.request('POST', path, {'Host': ADMIN_HOST, 'Content-Type': 'application/json'}, body=payload) as response: + self.assertEqual(response.status, 200) + response.read() + self.assertEqual(self.observed[-1][3], payload) + self.assertNotIn(code, self.observed[-1][1]) + self.assertNotIn(state, self.observed[-1][1]) + with self.request('GET', '/v8/management/oauth/status?state=' + state, {'Host': ADMIN_HOST}) as response: + self.assertEqual(response.status, 200) + response.read() + + def test_api_key_is_required_independently_of_upstream_auth(self): + for headers in [{}, {'Authorization': 'Bearer ' + MANAGEMENT_KEY}, {'Authorization': 'Bearer wrong'}, {'X-Api-Key': 'wrong'}]: + with self.request('GET', '/v1/models', headers) as response: + self.assertEqual(response.status, 401) + response.read() + self.assertEqual(self.observed, []) + for headers in [{'Authorization': 'Bearer ' + API_KEY}, {'X-Api-Key': API_KEY}]: + with self.request('GET', '/v1/models', headers) as response: + self.assertEqual(response.status, 200) + response.read() + self.assertEqual(self.observed[-1][2]['Authorization'], 'Bearer ' + API_KEY) + self.assertNotIn('X-Api-Key', self.observed[-1][2]) + for query in ['key=other', 'auth_token=other']: + with self.request('GET', '/v1/models?' + query, {'Authorization': 'Bearer ' + API_KEY}) as response: + self.assertEqual(response.status, 400) + response.read() + + def test_confirmed_codex_claude_methods_preserve_query_and_strip_admin_cookies(self): + for method, path in [('GET', '/v1/models'), ('POST', '/v1/chat/completions'), ('POST', '/v1/responses'), ('POST', '/v1/responses/compact'), ('POST', '/v1/messages'), ('POST', '/v1/messages/count_tokens'), ('POST', '/backend-api/codex/responses')]: + with self.subTest(path=path), self.request(method, path + '?fixture=yes', {'Authorization': 'Bearer ' + API_KEY, 'Cookie': 'snowglobe_session=synthetic', 'X-Management-Key': MANAGEMENT_KEY}) as response: + self.assertEqual(response.status, 200) + self.assertEqual(json.loads(response.read())['path'], path + '?fixture=yes') + self.assertNotIn('Cookie', self.observed[-1][2]) + self.assertNotIn('X-Management-Key', self.observed[-1][2]) + for method, path in [('DELETE', '/v1/models'), ('GET', '/v1/messages'), ('POST', '/v1/models'), ('OPTIONS', '/v1/responses')]: + with self.request(method, path, {'Authorization': 'Bearer ' + API_KEY}) as response: + self.assertEqual(response.status, 404) + response.read() + + def test_stream_is_delivered_before_upstream_finishes(self): + started = time.monotonic() + with self.request('POST', '/v1/responses', {'Authorization': 'Bearer ' + API_KEY, 'X-Synthetic-Stream': 'yes'}) as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.readline(), b'data: first\n') + self.assertLess(time.monotonic() - started, .4) + self.assertIn(b'data: last', response.read()) + + def test_websocket_upgrade_reaches_responses_route(self): + key = base64.b64encode(b'synthetic-16-key').decode() + with socket.create_connection(('127.0.0.1', self.port), timeout=3) as connection: + connection.sendall(('GET /v1/responses HTTP/1.1\r\nHost: ' + API_HOST + '\r\nAuthorization: Bearer ' + API_KEY + '\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ' + key + '\r\n\r\n').encode()) + headers = b'' + while not headers.endswith(b'\r\n\r\n'): + headers += connection.recv(1) + self.assertIn(b' 101 ', headers.split(b'\r\n')[0]) + self.assertIn(b'upgrade: websocket', headers.lower()) + connection.sendall(b'\x81\x82\x01\x02\x03\x04ik') + self.assertEqual(connection.recv(4), b'\x81\x02hi') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--nginx', type=Path) + options, remaining = parser.parse_known_args() + NGINX = str(options.nginx.resolve()) if options.nginx else None + GatewayTests.__unittest_skip__ = not NGINX + unittest.main(argv=[__file__, *remaining]) diff --git a/tools/cliproxy.md b/tools/cliproxy.md new file mode 100644 index 0000000000000000000000000000000000000000..0763db9d4687b2c88a60e5f14e4dc4915f1d5a44 --- /dev/null +++ b/tools/cliproxy.md @@ -0,0 +1,60 @@ +# CLIProxyAPI + +`cliproxy.paperclover.net` is the management panel, protected by Snow's +`infra-admin` role and CLIProxyAPI's independent management key. +`cliproxy-api.paperclover.net` is the client endpoint: use +`https://cliproxy-api.paperclover.net/v1` for OpenAI-compatible clients or +`https://cliproxy-api.paperclover.net` for Claude-compatible clients. + +Nomad variable `nomad/jobs/cliproxy` owns `api_key` and `management_key`. +Keep values in restricted private files or a password manager; do not paste them +into chat, command arguments, or tracked configuration. The native panel asks +for its management key once and offers **Remember password**. That stores the +key in this browser's local storage; use it only in a trusted browser profile. +API clients accept the separate client key through `Authorization: Bearer …` +or `X-Api-Key`. The gateway independently enforces this one infra-managed key, +even if the native panel removes its own API keys. Query-string keys are rejected. + +## Connect providers + +Open the management panel, sign in through Snow, then unlock the panel with the +management key. Its OAuth section supports **Codex** and **Claude**. Start the +chosen provider's login and complete consent in the provider's own browser page. +For remote OAuth flows that finish on an unreachable `localhost` URL, copy that +entire callback URL into the panel's callback form. Treat the callback URL as a +temporary credential. No callback ports or provider callbacks are exposed on +the client hostname. Browser GET/HEAD callback routes are blocked on the admin +hostname too: CLIProxyAPI logs URL queries, so callback credentials must be +submitted through the management POST form. API-key provider credentials can also be configured in +the native panel; their persistent home is the private service dataset. + +`/data/config.yaml` is seeded once from a read-only Nomad template. Native +configuration edits, hashed management key, OAuth accounts, and refreshed tokens +remain under `/srv/prod/cliproxy/data`; restarts and normal releases retain them. +Changing a generated Nomad key does not overwrite an existing native config: +coordinate key rotation in both Nomad and the native panel. Fresh staging uses +separate storage and generated keys and must use synthetic providers only. + +## Boundary and updates + +Only the confirmed Codex/Claude client routes are public: model listing, chat, +completions, messages/counting, Responses/compact/WebSocket, and Codex search +and direct aliases. Management, plugins, OAuth callbacks, unknown paths, and +unapproved methods return 404 on the API hostname. The app listens only on +container loopback; nginx publishes the sole port. Both processes exit together +if either fails. The host-side route requires Snow for management; nginx's +admin-host routing does not replace that authentication boundary. + +The service pins CLIProxyAPI v8.0.32 and the management panel v1.26.5. Panel +downloads are size- and SHA-256-checked during preparation; runtime auto-update +is disabled. Request-body logging is omitted by `commercial-mode: true` and +`request-log: false`. Plugins, discovery, usage aggregation, and profiling start +disabled. The read-only image has only `/data` and a private `/tmp` writable; +the pinned `/data/static` subdirectory is mounted read-only separately. + +Run `python3 tools/cliproxy-test.py --nginx /path/to/nginx` for bootstrap, +rendering, pinned-asset, and actual HTTP/streaming/WebSocket boundary checks. +The test uses temporary synthetic credentials and a loopback fixture upstream. +Stage with the normal deployment command, verify anonymous management is routed +to Snow, valid/invalid client-key behavior, provider persistence on restart, +and logs containing no fixture keys or prompts before production promotion. diff --git a/tools/router.py b/tools/router.py index a7a14d783d9126ae572ab607a7672d9b72c3d423..a77f740824e3eee31d28289885c9840763c45ec5 100644 --- a/tools/router.py +++ b/tools/router.py @@ -291,8 +291,10 @@ def render(token): if access and not any(tag.startswith("caddy-host=") for tag in tags): raise ValueError("route access role has no host") for tag in tags: - if tag.startswith("caddy-host="): + if tag.startswith(("caddy-host=", "caddy-plain-host=")): host = tag.split("=", 1)[1] + role = access[0] if access and tag.startswith("caddy-host=") else None + user_header = user_headers[0] if user_headers and role else None if not HOST.fullmatch(host): raise ValueError(f"invalid Caddy host: {host!r}") route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False, "secureCookies": False, "jellyfinAuth": False}) @@ -300,14 +302,14 @@ def render(token): if not (instance["ServiceName"] == "jellyfin" or re.fullmatch(r"jellyfin-preview-[0-9a-f]{8}", instance["ServiceName"])): raise ValueError("Jellyfin gateway tag belongs to another service") route["jellyfinAuth"] = True - if access and route["authRole"] not in (None, access[0]): + if role and route["authRole"] not in (None, role): raise ValueError("conflicting route access roles") - if user_headers and route["userHeader"] not in (None, user_headers[0]): + if user_header and route["userHeader"] not in (None, user_header): raise ValueError("conflicting authenticated user headers") - if access: - route["authRole"] = access[0] - if user_headers: - route["userHeader"] = user_headers[0] + if role: + route["authRole"] = role + if user_header: + route["userHeader"] = user_header if "caddy-internal=true" in tags: route["internal"] = True if "caddy-real-ip=true" in tags: diff --git a/tools/snowglobe-sso-test.py b/tools/snowglobe-sso-test.py index ea5bd0168b18167fea8d580939644e77af97c697..41bcbfd4fc21abf200e84b6b2c024648d384d9e3 100644 --- a/tools/snowglobe-sso-test.py +++ b/tools/snowglobe-sso-test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import os +import re from pathlib import Path import subprocess import tempfile @@ -8,6 +9,7 @@ import unittest from unittest.mock import patch import snowglobe +import router class SsoConnectivity(unittest.TestCase): @@ -17,7 +19,7 @@ class SsoConnectivity(unittest.TestCase): redirectUris { "https://fixture.example/api/callback" } }''' - def load(self, requirements=client, extra_hosts=()): + def load(self, requirements=client, extra_hosts=(), http=""): with tempfile.TemporaryDirectory(prefix="snowglobe-sso-") as temporary: fixture = Path(temporary) / "fixture.pkl" fixture.write_text(f'''amends {json.dumps((snowglobe.REPO / "config/Service.pkl").as_uri())} @@ -27,6 +29,7 @@ requirements {{ {requirements} }} containers {{ ["app"] {{ image = "localhost/fixture:pinned" + {http} extraHosts {{ {"; ".join(json.dumps(host) for host in extra_hosts)} }} }} ["worker"] {{ image = "localhost/fixture:pinned" }} @@ -66,6 +69,33 @@ containers {{ data = self.load(requirements=self.client + "\n" + other) self.assertEqual(self.render(data).count("snowglobe.fixture.example:host-gateway"), 2) + def test_plain_alias_omits_sso_role_and_identity_header(self): + data = self.load(requirements="", http='http { containerPort = 8080; subdomain = "fixture"; authRole = "infra-admin"; userHeader = "X-User"; plainHostnames { "api.fixture.example" } }') + tags = json.loads(re.search(r'tags = (\[.*\])', self.render(data))[1]) + def nomad(path, _): + if path == '/v1/services': + return [{'Namespace': 'default', 'Services': [{'ServiceName': name} for name in ('fixture', 'forward-auth')]}] + if path.startswith('/v1/service/'): + name = path.rsplit('/', 1)[1] + return [{'ServiceName': name, 'JobID': name, 'AllocID': name, 'Address': '127.0.0.1', 'Port': 28000, + 'Tags': tags if name == 'fixture' else []}] + return {'http': {'Status': 'success'}} + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + proof = root / 'proof'; proof.write_text('a' * 64) + environment = {'SNOWGLOBE_DOMAIN': 'fixture.example', 'SNOWGLOBE_DASHBOARD_PORT': '28001', + 'SNOWGLOBE_PROXY_TOKEN_FILE': str(proof), 'SNOWGLOBE_STATE': temporary} + with patch.dict(os.environ, environment, clear=True), patch.object(router, 'nomad', nomad), patch.object(router, 'ROUTE_DIR', str(root / 'routes')): + rendered = router.render('fixture') + protected = rendered.split('fixture.fixture.example {', 1)[1].split('\n}\n', 1)[0] + public = rendered.split('api.fixture.example {', 1)[1].split('\n}\n', 1)[0] + self.assertIn('/snow.oauth2/auth', protected) + self.assertIn('role:infra-admin', protected) + self.assertIn('X-User', protected) + self.assertNotIn('/snow.oauth2/', public) + self.assertNotIn('X-User', public) + self.assertIn('reverse_proxy 127.0.0.1:28000', public) + def test_services_without_sso_preserve_their_explicit_hosts(self): data = self.load(requirements="", extra_hosts=["other.example:host-gateway"]) rendered = self.render(data) diff --git a/tools/snowglobe.py b/tools/snowglobe.py index a5b5bca300ea0898895e5a830b2e88fa7d152710..2a7e21e1d1f8a0acae7239cc17ed956e9198bbc4 100644 --- a/tools/snowglobe.py +++ b/tools/snowglobe.py @@ -341,7 +341,8 @@ def render(data, definitions): if kind == "http" and endpoint.get("metricsPath"): tags.append("snowglobe-metrics-path=" + endpoint["metricsPath"]) if kind == "http" and endpoint.get("hostname"): - tags += ["caddy-host=" + host for host in [endpoint["hostname"], *endpoint["plainHostnames"]]] + tags.append("caddy-host=" + endpoint["hostname"]) + tags += ["caddy-plain-host=" + host for host in endpoint["plainHostnames"]] if endpoint.get("authRole"): tags.append("caddy-auth-role=" + endpoint["authRole"]) if endpoint.get("userHeader"): @@ -1053,13 +1054,15 @@ def main(): for port in task["tcp"]: port["hostPort"] = None if hostname: + hosts = {original_host: hostname, **{host: stage + "-" + host for host in http[0]["plainHostnames"]}} for task in data["containers"].values(): if task.get("http") and task["http"].get("hostname") == original_host: task["http"]["hostname"] = hostname - task["http"]["plainHostnames"] = [stage + "-" + host for host in task["http"]["plainHostnames"]] - task["env"] = {key: value.replace(original_host, hostname) for key, value in task["env"].items()} - if task.get("secretFiles"): - task["secretFiles"] = {key: value.replace(original_host, hostname) for key, value in task["secretFiles"].items()} + task["http"]["plainHostnames"] = [hosts[host] for host in task["http"]["plainHostnames"]] + for source, target in hosts.items(): + task["env"] = {key: value.replace(source, target) for key, value in task["env"].items()} + if task.get("secretFiles"): + task["secretFiles"] = {key: value.replace(source, target) for key, value in task["secretFiles"].items()} for request in data["inputs"].values(): if request["provider"] == "snowglobe": request["redirectUris"] = [uri.replace(original_host, hostname) for uri in request["redirectUris"]]