From 7caaa8c6b763db8a1e8934de235f956e13ff789d Mon Sep 17 00:00:00 2001 From: clover caruso Date: Fri, 16 Oct 2026 11:23:10 -0700 Subject: [PATCH] feat: connect devices and archive chats through a self-hosted server Clover wants one server address for web and desktop sign-in, access to connected machines, and archived conversations that remain readable while laptops sleep. T3 Connect remains optional, and workspaces, agent execution, and provider credentials stay on each machine by default. Use generic OIDC discovery with a confidential authorization-code client, S256 PKCE, and an exact HTTPS callback. Pin the configured issuer and immutable owner subject. Keep the client secret on the server; return desktop sign-in through a short-lived, single-use proof-bound handoff. Store rotating desktop credentials in OS-protected storage, renew access automatically, enforce interactive deadlines and issuer revalidation, and revoke active connections with their device. Keep local emergency recovery separate from normal login. Connect each desktop through authenticated outbound sockets so the server requires no inbound laptop address. Preserve environment identities and existing manually paired connections. Limit background connection and archive credentials to dedicated scopes; source grants retain their own operation permissions. Show devices, revocation, connection status, upload retries, and reclaimed storage in Settings. Persist the central inbox so settle, unsettle, snooze, wake, pin, and ordering work after a synced machine goes offline. Reconcile the latest intent only after its source command and snapshot are acknowledged, without replacing live source content or execution state. Publish content-addressed transcript and media closures to an independently available archive mount. Require the configured durable store identity, verify every object, and flush the catalog before evicting local payloads. Keep retry queues across restarts and retain local data after failed verification, concurrent changes, or active fork references. Read cold transcripts and signed media directly from the server while the source sleeps; restore full local bodies and attachments before unarchiving. Reclaim SQLite and attachment disk blocks while reporting retained native provider logs separately. Preserve shipped archive hashes, transcript ancestry, replay watermarks, media ranges, existing asset URLs, and managed account opt-in. Supply a generic configuration contract and optional read-only Compose secret mount without coupling Knife to a particular infrastructure repository. Assisted-by: gpt-6.1-sol --- apps/desktop/src/app/DesktopApp.ts | 2 + .../desktop/src/app/DesktopHostedAuth.test.ts | 271 ++++ apps/desktop/src/app/DesktopHostedAuth.ts | 363 +++++ .../desktop/src/app/hostedAuthHandoff.test.ts | 71 + apps/desktop/src/app/hostedAuthHandoff.ts | 58 + apps/desktop/src/ipc/DesktopIpcHandlers.ts | 12 + apps/desktop/src/ipc/channels.ts | 7 + apps/desktop/src/ipc/methods/hostedAuth.ts | 48 + apps/desktop/src/main.ts | 2 + apps/desktop/src/preload.ts | 6 + apps/mobile/src/App.tsx | 25 +- apps/mobile/src/connection/hub.ts | 5 + apps/mobile/src/connection/platform.ts | 6 +- apps/mobile/src/connection/storage.test.ts | 68 + .../features/cloud/CloudAuthProvider.test.ts | 1 + .../src/features/cloud/CloudAuthProvider.tsx | 28 +- .../cloud/ConnectOnboardingRouteScreen.tsx | 9 +- .../src/features/cloud/cloudAvailability.ts | 7 + .../features/cloud/connectionService.test.ts | 46 + .../src/features/cloud/connectionService.ts | 13 + .../connection/CloudEnvironmentRows.tsx | 6 +- .../connection/ConnectionsNewRouteScreen.tsx | 174 +- .../connection/ConnectionsRouteScreen.tsx | 2 + .../connection/LocalEnvironmentList.tsx | 14 +- .../connection/SelfHostedServerSection.tsx | 60 + .../src/features/connection/pairing.test.ts | 9 + .../mobile/src/features/connection/pairing.ts | 2 +- .../settings/SettingsAuthRouteScreen.tsx | 11 +- .../SettingsEnvironmentsRouteScreen.tsx | 2 + .../SettingsNotificationsRouteScreen.tsx | 7 +- .../features/settings/SettingsRouteScreen.tsx | 4 +- .../threads/ThreadArrangementSheet.tsx | 42 +- .../features/threads/thread-list-v2-items.tsx | 58 +- .../src/persistence/mobile-preferences.ts | 6 + apps/mobile/src/state/server.ts | 7 + apps/mobile/src/state/session.test.ts | 7 + apps/mobile/src/state/session.ts | 15 + .../state/thread-list-environments.test.ts | 41 +- .../src/state/thread-list-environments.ts | 27 +- apps/mobile/src/state/thread-order.test.ts | 7 +- apps/mobile/src/state/thread-order.ts | 18 +- .../src/archive/ArchiveAssetResponse.ts | 74 + apps/server/src/archive/ArchiveAssets.ts | 128 ++ apps/server/src/archive/ArchiveCatalog.ts | 356 ++++ apps/server/src/archive/ArchiveClient.ts | 216 +++ apps/server/src/archive/ArchiveConnection.ts | 44 + apps/server/src/archive/ArchiveFormat.ts | 40 + apps/server/src/archive/ArchiveHttp.ts | 157 ++ apps/server/src/archive/ArchiveObjects.ts | 127 ++ .../src/archive/ArchiveObjectsHttp.test.ts | 92 ++ apps/server/src/archive/ArchiveStorage.ts | 164 ++ .../src/archive/ArchivedProjectionStore.ts | 80 + .../src/archive/HostedArchiveConnection.ts | 33 + .../src/archive/TranscriptArchive.test.ts | 1438 +++++++++++++++++ apps/server/src/archive/TranscriptArchive.ts | 778 +++++++++ apps/server/src/archive/runtimeLayer.ts | 16 + apps/server/src/assets/AssetAccess.test.ts | 6 + apps/server/src/assets/AssetAccess.ts | 205 ++- apps/server/src/attachmentStore.ts | 38 +- apps/server/src/auth/EnvironmentAuth.test.ts | 35 + apps/server/src/auth/EnvironmentAuth.ts | 112 ++ apps/server/src/auth/HostedAuth.test.ts | 696 ++++++++ apps/server/src/auth/HostedAuth.ts | 739 +++++++++ apps/server/src/auth/HostedAuthConfig.ts | 79 + apps/server/src/auth/HostedAuthHttp.ts | 255 +++ apps/server/src/auth/SessionStore.test.ts | 103 +- apps/server/src/auth/SessionStore.ts | 175 +- apps/server/src/auth/http.test.ts | 55 +- apps/server/src/auth/http.ts | 22 + apps/server/src/cli/auth.ts | 33 +- apps/server/src/cli/project.ts | 9 + apps/server/src/cli/shared.ts | 116 +- apps/server/src/environment/Hub.test.ts | 326 ++++ apps/server/src/environment/Hub.ts | 417 +++++ apps/server/src/environment/HubHttp.ts | 89 + apps/server/src/environment/HubInbox.test.ts | 536 ++++++ .../src/environment/HubInbox.testkit.ts | 43 + apps/server/src/environment/HubInbox.ts | 661 ++++++++ .../src/environment/HubInboxHttp.test.ts | 126 ++ apps/server/src/environment/HubInboxHttp.ts | 128 ++ .../src/environment/HubProxy.auth.test.ts | 127 ++ .../src/environment/HubProxy.stream.test.ts | 116 ++ apps/server/src/environment/HubProxy.test.ts | 324 ++++ apps/server/src/environment/HubProxy.ts | 232 +++ .../src/environment/OutboundHub.test.ts | 242 +++ apps/server/src/environment/OutboundHub.ts | 295 ++++ .../src/environment/OutboundSocket.test.ts | 114 ++ apps/server/src/environment/OutboundSocket.ts | 83 + apps/server/src/environment/OutboundSource.ts | 169 ++ apps/server/src/environment/SharedServer.ts | 46 + .../src/environment/SourceConnection.test.ts | 132 ++ .../src/environment/SourceConnection.ts | 115 ++ .../src/environment/SourceConnectionHttp.ts | 95 ++ apps/server/src/http.ts | 151 +- .../orchestration-v2/ProjectionMaintenance.ts | 15 +- .../src/orchestration-v2/ProjectionStore.ts | 7 + .../src/orchestration-v2/runtimeLayer.ts | 28 +- apps/server/src/persistence/AuthSessions.ts | 34 +- apps/server/src/persistence/Migrations.ts | 10 + .../Migrations/063_TranscriptArchives.ts | 26 + .../Migrations/064_AuthSessionDelegation.ts | 7 + .../persistence/Migrations/065_HubInbox.ts | 23 + .../Migrations/066_HostedAuthDevices.ts | 26 + .../Migrations/067_ArchiveCatalogAndSync.ts | 10 + .../persistence/OrchestrationEventStore.ts | 7 +- .../src/provider/SharedAccountClient.ts | 40 +- apps/server/src/server.ts | 167 +- apps/server/src/storageCleanup.test.ts | 88 +- apps/server/src/storageCleanup.ts | 10 + apps/web/src/AppRoot.tsx | 2 + apps/web/src/cloud/CloudAuthBoundary.test.tsx | 105 ++ apps/web/src/cloud/CloudAuthBoundary.tsx | 29 + apps/web/src/cloud/CloudAuthentication.tsx | 40 + apps/web/src/cloud/activation.ts | 55 + apps/web/src/cloud/managedAuth.tsx | 3 + apps/web/src/components/ChatView.tsx | 39 +- apps/web/src/components/Sidebar.tsx | 120 +- .../components/auth/PairingRouteSurface.tsx | 59 +- .../clerk/BrowserManagedAuthShell.tsx | 9 +- .../clerk/ElectronManagedAuthShell.tsx | 10 +- .../clerk/T3ConnectSidebarSignIn.tsx | 25 +- .../cloud/ConnectCliAuthSurface.tsx | 19 +- .../cloud/ConnectOnboardingDialog.tsx | 14 +- .../onboarding/WelcomeWizard.import.test.tsx | 1 + .../WelcomeWizard.terminal.test.tsx | 1 + .../onboarding/WelcomeWizard.test.tsx | 1 + .../components/onboarding/WelcomeWizard.tsx | 86 +- .../settings/ConnectionsSettings.tsx | 109 +- .../settings/SelfHostedServer.test.tsx | 88 + .../components/settings/SelfHostedServer.tsx | 568 +++++++ .../components/settings/pairingUrls.test.ts | 6 + .../src/components/settings/pairingUrls.ts | 4 +- .../src/components/settings/settingsSearch.ts | 4 +- .../components/threadActionMenu.logic.test.ts | 19 +- .../src/components/threadActionMenu.logic.ts | 34 +- apps/web/src/connection/hostedAuth.test.ts | 201 +++ apps/web/src/connection/hostedAuth.ts | 112 ++ apps/web/src/connection/hub.ts | 5 + apps/web/src/connection/platform.ts | 19 +- apps/web/src/connection/storage.test.ts | 63 + .../environments/primary/bootstrap.test.ts | 25 + apps/web/src/environments/primary/target.ts | 28 +- apps/web/src/hooks/useThreadActionMenu.ts | 8 +- apps/web/src/main.tsx | 39 +- apps/web/src/routes/__root.tsx | 2 + apps/web/src/routes/_chat.index.tsx | 6 +- apps/web/src/state/entities.ts | 32 +- apps/web/src/state/server.ts | 6 + apps/web/src/state/session.ts | 15 + .../state/use-orchestration-command.test.ts | 19 +- .../src/state/use-orchestration-command.ts | 12 +- apps/web/test/environmentHttpTest.ts | 1 + apps/web/vite.config.ts | 4 +- compose.sso.yaml | 12 + docs/operations/development.md | 82 +- docs/user/shared-server.md | 30 +- packages/client-runtime/package.json | 4 + .../src/authorization/remote.test.ts | 57 + .../src/authorization/remote.ts | 12 +- .../client-runtime/src/connection/catalog.ts | 22 +- .../client-runtime/src/connection/hub.test.ts | 575 +++++++ packages/client-runtime/src/connection/hub.ts | 580 +++++++ .../client-runtime/src/connection/index.ts | 1 + .../client-runtime/src/connection/layer.ts | 15 + .../src/connection/onboarding.test.ts | 36 +- .../src/connection/onboarding.ts | 3 +- .../src/connection/presentation.test.ts | 16 + .../src/connection/presentation.ts | 8 +- .../src/connection/registry.test.ts | 17 +- .../client-runtime/src/connection/registry.ts | 21 +- .../src/connection/resolver.test.ts | 30 + .../client-runtime/src/connection/resolver.ts | 72 +- .../src/connection/routes.test.ts | 13 + .../client-runtime/src/connection/routes.ts | 21 +- .../src/environment/endpoint.test.ts | 29 +- .../src/environment/endpoint.ts | 8 - .../client-runtime/src/operations/commands.ts | 25 +- .../src/platform/capabilities.ts | 9 + .../src/platform/persistence.ts | 5 +- packages/client-runtime/src/rpc/http.ts | 15 +- .../src/state/archiveAssetHttp.ts | 41 + .../src/state/archiveThreadHttp.ts | 38 + .../client-runtime/src/state/assets.test.ts | 129 ++ packages/client-runtime/src/state/assets.ts | 93 +- .../client-runtime/src/state/attachments.ts | 7 + .../src/state/environmentHttpAuth.test.ts | 26 + .../src/state/environmentHttpAuth.ts | 8 +- .../src/state/hub-capabilities.test.ts | 125 ++ packages/client-runtime/src/state/hub.ts | 173 ++ .../src/state/hubSignedUrl.test.ts | 234 +++ .../client-runtime/src/state/hubSignedUrl.ts | 50 + .../client-runtime/src/state/orchestration.ts | 38 +- packages/client-runtime/src/state/runtime.ts | 12 + packages/client-runtime/src/state/shell.ts | 22 +- .../src/state/threadCommands.test.ts | 42 + .../src/state/threads-sync.test.ts | 73 +- packages/client-runtime/src/state/threads.ts | 53 +- .../client-runtime/src/webhookAddress.test.ts | 6 + packages/client-runtime/src/webhookAddress.ts | 3 +- packages/contracts/src/auth.ts | 17 + packages/contracts/src/desktopHostedAuth.ts | 15 + packages/contracts/src/environmentHttp.ts | 13 + packages/contracts/src/hostedAuth.ts | 70 + packages/contracts/src/hub.ts | 116 ++ packages/contracts/src/index.ts | 6 + packages/contracts/src/ipc.ts | 6 + packages/contracts/src/outbound.ts | 32 + packages/contracts/src/transcriptArchive.ts | 76 + packages/shared/package.json | 4 + packages/shared/src/advertisedEndpoint.ts | 16 +- packages/shared/src/devProxy.ts | 10 +- packages/shared/src/hostedServerUrl.ts | 14 + packages/shared/src/remote.test.ts | 19 + packages/shared/src/remote.ts | 43 +- 214 files changed, 17689 insertions(+), 759 deletions(-) create mode 100644 apps/desktop/src/app/DesktopHostedAuth.test.ts create mode 100644 apps/desktop/src/app/DesktopHostedAuth.ts create mode 100644 apps/desktop/src/app/hostedAuthHandoff.test.ts create mode 100644 apps/desktop/src/app/hostedAuthHandoff.ts create mode 100644 apps/desktop/src/ipc/methods/hostedAuth.ts create mode 100644 apps/mobile/src/connection/hub.ts create mode 100644 apps/mobile/src/features/cloud/cloudAvailability.ts create mode 100644 apps/mobile/src/features/cloud/connectionService.test.ts create mode 100644 apps/mobile/src/features/cloud/connectionService.ts create mode 100644 apps/mobile/src/features/connection/SelfHostedServerSection.tsx create mode 100644 apps/server/src/archive/ArchiveAssetResponse.ts create mode 100644 apps/server/src/archive/ArchiveAssets.ts create mode 100644 apps/server/src/archive/ArchiveCatalog.ts create mode 100644 apps/server/src/archive/ArchiveClient.ts create mode 100644 apps/server/src/archive/ArchiveConnection.ts create mode 100644 apps/server/src/archive/ArchiveFormat.ts create mode 100644 apps/server/src/archive/ArchiveHttp.ts create mode 100644 apps/server/src/archive/ArchiveObjects.ts create mode 100644 apps/server/src/archive/ArchiveObjectsHttp.test.ts create mode 100644 apps/server/src/archive/ArchiveStorage.ts create mode 100644 apps/server/src/archive/ArchivedProjectionStore.ts create mode 100644 apps/server/src/archive/HostedArchiveConnection.ts create mode 100644 apps/server/src/archive/TranscriptArchive.test.ts create mode 100644 apps/server/src/archive/TranscriptArchive.ts create mode 100644 apps/server/src/archive/runtimeLayer.ts create mode 100644 apps/server/src/auth/HostedAuth.test.ts create mode 100644 apps/server/src/auth/HostedAuth.ts create mode 100644 apps/server/src/auth/HostedAuthConfig.ts create mode 100644 apps/server/src/auth/HostedAuthHttp.ts create mode 100644 apps/server/src/environment/Hub.test.ts create mode 100644 apps/server/src/environment/Hub.ts create mode 100644 apps/server/src/environment/HubHttp.ts create mode 100644 apps/server/src/environment/HubInbox.test.ts create mode 100644 apps/server/src/environment/HubInbox.testkit.ts create mode 100644 apps/server/src/environment/HubInbox.ts create mode 100644 apps/server/src/environment/HubInboxHttp.test.ts create mode 100644 apps/server/src/environment/HubInboxHttp.ts create mode 100644 apps/server/src/environment/HubProxy.auth.test.ts create mode 100644 apps/server/src/environment/HubProxy.stream.test.ts create mode 100644 apps/server/src/environment/HubProxy.test.ts create mode 100644 apps/server/src/environment/HubProxy.ts create mode 100644 apps/server/src/environment/OutboundHub.test.ts create mode 100644 apps/server/src/environment/OutboundHub.ts create mode 100644 apps/server/src/environment/OutboundSocket.test.ts create mode 100644 apps/server/src/environment/OutboundSocket.ts create mode 100644 apps/server/src/environment/OutboundSource.ts create mode 100644 apps/server/src/environment/SharedServer.ts create mode 100644 apps/server/src/environment/SourceConnection.test.ts create mode 100644 apps/server/src/environment/SourceConnection.ts create mode 100644 apps/server/src/environment/SourceConnectionHttp.ts create mode 100644 apps/server/src/persistence/Migrations/063_TranscriptArchives.ts create mode 100644 apps/server/src/persistence/Migrations/064_AuthSessionDelegation.ts create mode 100644 apps/server/src/persistence/Migrations/065_HubInbox.ts create mode 100644 apps/server/src/persistence/Migrations/066_HostedAuthDevices.ts create mode 100644 apps/server/src/persistence/Migrations/067_ArchiveCatalogAndSync.ts create mode 100644 apps/web/src/cloud/CloudAuthBoundary.test.tsx create mode 100644 apps/web/src/cloud/CloudAuthBoundary.tsx create mode 100644 apps/web/src/cloud/CloudAuthentication.tsx create mode 100644 apps/web/src/cloud/activation.ts create mode 100644 apps/web/src/components/settings/SelfHostedServer.test.tsx create mode 100644 apps/web/src/components/settings/SelfHostedServer.tsx create mode 100644 apps/web/src/connection/hostedAuth.test.ts create mode 100644 apps/web/src/connection/hostedAuth.ts create mode 100644 apps/web/src/connection/hub.ts create mode 100644 compose.sso.yaml create mode 100644 packages/client-runtime/src/connection/hub.test.ts create mode 100644 packages/client-runtime/src/connection/hub.ts create mode 100644 packages/client-runtime/src/state/archiveAssetHttp.ts create mode 100644 packages/client-runtime/src/state/archiveThreadHttp.ts create mode 100644 packages/client-runtime/src/state/hub-capabilities.test.ts create mode 100644 packages/client-runtime/src/state/hub.ts create mode 100644 packages/client-runtime/src/state/hubSignedUrl.test.ts create mode 100644 packages/client-runtime/src/state/hubSignedUrl.ts create mode 100644 packages/contracts/src/desktopHostedAuth.ts create mode 100644 packages/contracts/src/hostedAuth.ts create mode 100644 packages/contracts/src/hub.ts create mode 100644 packages/contracts/src/outbound.ts create mode 100644 packages/contracts/src/transcriptArchive.ts create mode 100644 packages/shared/src/hostedServerUrl.ts diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.ts index 938989eece546a80052bc4fb156c2826d3cc40e1..bfb987f8ddd7c64e4099311b994ed3b03c285707 100644 --- a/apps/desktop/src/app/DesktopApp.ts +++ b/apps/desktop/src/app/DesktopApp.ts @@ -14,6 +14,7 @@ import * as ElectronSafeStorage from "../electron/ElectronSafeStorage.ts"; import { installDesktopIpcHandlers } from "../ipc/DesktopIpcHandlers.ts"; import * as DesktopAppActivation from "./DesktopAppActivation.ts"; import * as DesktopAppIdentity from "./DesktopAppIdentity.ts"; +import * as DesktopHostedAuth from "./DesktopHostedAuth.ts"; import * as DesktopClerk from "./DesktopClerk.ts"; import * as DesktopApplicationMenu from "../window/DesktopApplicationMenu.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; @@ -320,6 +321,7 @@ const startup = Effect.gen(function* () { yield* appIdentity.configure; yield* lifecycle.register; + yield* (yield* DesktopHostedAuth.DesktopHostedAuth).configure; yield* clerk.configure; yield* electronApp.whenReady.pipe( diff --git a/apps/desktop/src/app/DesktopHostedAuth.test.ts b/apps/desktop/src/app/DesktopHostedAuth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..9a79620c0dcffad48d360ded172fd74bd95e8e94 --- /dev/null +++ b/apps/desktop/src/app/DesktopHostedAuth.test.ts @@ -0,0 +1,271 @@ +// @effect-diagnostics preferSchemaOverJson:off - The fake HTTPS peer inspects and emits JSON wire payloads. +import { describe, expect, it } from "@effect/vitest"; +import * as NodePath from "@effect/platform-node/NodePath"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { layerRemoteHttpClient } from "@t3tools/client-runtime/rpc"; +import * as DesktopHostedAuth from "./DesktopHostedAuth.ts"; +import * as DesktopEnvironment from "./DesktopEnvironment.ts"; +import * as ElectronApp from "../electron/ElectronApp.ts"; +import * as ElectronSafeStorage from "../electron/ElectronSafeStorage.ts"; +import * as ElectronShell from "../electron/ElectronShell.ts"; +import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; +import { PRIMARY_LOCAL_ENVIRONMENT_ID } from "@t3tools/contracts"; +import type { DesktopBackendInstance } from "../backend/DesktopBackendPool.ts"; +import * as DesktopLocalEnvironmentAuth from "../backend/DesktopLocalEnvironmentAuth.ts"; + +const server = "https://knife.example.test"; +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function harness() { + const files = new Map(); + const requests: Array<{ + path: string; + body: Record; + authorization: string | null; + redirect: RequestRedirect | undefined; + }> = []; + const opened: string[] = []; + const control = { + available: true, + exchangeStatus: 200, + backend: false, + deleteStatus: 200, + redirectRefresh: false, + }; + const grant = { + deviceId: "device", + accessToken: "access-in-memory", + refreshToken: "private-refresh-v1", + expiresAt: "2035-01-01T00:00:00.000Z", + }; + const layer = Layer.mergeAll( + NodePath.layerPosix, + Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + stateDir: "/isolated", + } as DesktopEnvironment.DesktopEnvironment["Service"]), + Layer.mock(ElectronApp.ElectronApp)({}), + Layer.mock(HostProcess.Arguments)([]), + Layer.mock(DesktopBackendPool.DesktopBackendPool)({ + list: Effect.sync(() => + control.backend + ? [ + { + id: PRIMARY_LOCAL_ENVIRONMENT_ID, + currentConfig: Effect.succeedSome({ + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }), + } as DesktopBackendInstance, + ] + : [], + ), + }), + Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth)({ + getBearerToken: Effect.succeed("local-admin"), + }), + Layer.succeed(ElectronShell.ElectronShell, { + openExternal: (url) => + Effect.sync(() => { + opened.push(String(url)); + return true; + }), + copyText: () => Effect.void, + openSystemSettings: () => Effect.succeed(false), + }), + Layer.succeed(ElectronSafeStorage.ElectronSafeStorage, { + isEncryptionAvailable: Effect.sync(() => control.available), + encryptString: (value) => Effect.succeed(encoder.encode(`encrypted:${value}`)), + decryptString: (value) => Effect.succeed(decoder.decode(value).slice("encrypted:".length)), + selectedStorageBackend: Effect.succeedNone, + }), + FileSystem.layerNoop({ + exists: (path) => Effect.sync(() => files.has(path)), + readFile: (path) => Effect.sync(() => files.get(path)!), + writeFile: (path, data) => + Effect.sync(() => { + files.set(path, data); + }), + rename: (from, to) => + Effect.sync(() => { + files.set(to, files.get(from)!); + files.delete(from); + }), + remove: (path) => + Effect.sync(() => { + files.delete(path); + }), + }), + layerRemoteHttpClient(async (input, init) => { + const request = new Request(input, init); + const path = new URL(request.url).pathname; + const body = + request.method === "DELETE" ? {} : ((await request.json()) as Record); + requests.push({ + path, + body, + authorization: request.headers.get("authorization"), + redirect: init?.redirect, + }); + if (path.endsWith("/refresh") && control.redirectRefresh) { + if (init?.redirect === "error") throw new TypeError("Redirect blocked"); + return Response.json({ ...grant, accessToken: "redirected-credential" }); + } + if (path === "/api/auth/devices/token") + return Response.json({ accessToken: "source-narrow", expiresAt: "2035-01-01T00:00:00Z" }); + if (path === "/api/self-hosted/connection") + return Response.json( + { connected: true }, + { status: request.method === "DELETE" ? control.deleteStatus : 200 }, + ); + if (path.endsWith("/start")) + return Response.json({ authorizationUrl: `${server}/api/auth/oidc/start?request=one-use` }); + if (path.endsWith("/exchange")) + return Response.json(grant, { status: control.exchangeStatus }); + if (path.endsWith("/refresh")) + return Response.json({ ...grant, refreshToken: "private-refresh-v2" }); + return new Response(null, { status: 404 }); + }), + ); + const callback = () => { + const request = requests.findLast((value) => value.path.endsWith("/start"))!; + return `t3code://app/self-hosted/callback?${new URLSearchParams({ server, state: String(request.body.state), code: "x".repeat(43) })}`; + }; + return { files, requests, opened, control, layer, callback }; +} + +describe("desktop hosted account custody", () => { + it.effect("rejects refresh redirects without replacing saved credentials", () => + Effect.gen(function* () { + const test = harness(); + const first = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* first.begin(server); + yield* first.receive(test.callback()); + const before = test.files.get("/isolated/hosted-account.bin"); + test.control.redirectRefresh = true; + const restarted = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + const result = yield* restarted.accessToken(server).pipe(Effect.result); + expect(result._tag).toBe("Failure"); + expect(test.requests.findLast((request) => request.path.endsWith("/refresh"))?.redirect).toBe( + "error", + ); + expect(test.files.get("/isolated/hosted-account.bin")).toEqual(before); + }), + ); + + it.effect("keeps credentials when the local source rejects disconnect", () => + Effect.gen(function* () { + const test = harness(); + test.control.backend = true; + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* auth.begin(server); + yield* auth.receive(test.callback()); + test.control.deleteStatus = 503; + expect((yield* auth.disconnect.pipe(Effect.result))._tag).toBe("Failure"); + expect(test.files.has("/isolated/hosted-account.bin")).toBe(true); + expect(test.requests.every((request) => request.redirect === "error")).toBe(true); + test.control.deleteStatus = 200; + yield* auth.disconnect; + expect(test.files.size).toBe(0); + }), + ); + + it.effect("gives the local source only a dedicated archive/connect grant", () => + Effect.gen(function* () { + const test = harness(); + test.control.backend = true; + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* auth.begin(server); + yield* auth.receive(test.callback()); + const narrow = test.requests.find((request) => request.path === "/api/auth/devices/token"); + expect(narrow?.authorization).toBe("Bearer access-in-memory"); + expect(narrow?.body).toEqual({ + scopes: ["archive:read", "archive:write", "environment:connect"], + }); + const bridge = test.requests.find( + (request) => request.path === "/api/self-hosted/connection", + ); + expect(bridge?.authorization).toBe("Bearer local-admin"); + expect(bridge?.body).toEqual({ + serverUrl: server, + authorization: "Bearer source-narrow", + expiresAt: "2035-01-01T00:00:00.000Z", + }); + expect(JSON.stringify(bridge)).not.toContain("access-in-memory"); + expect(JSON.stringify(bridge)).not.toContain("refresh"); + }), + ); + + it.effect( + "keeps refresh credentials main-only, consumes handoff once, and renews after restart", + () => + Effect.gen(function* () { + const test = harness(); + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* auth.begin(server); + expect(test.opened).toEqual([`${server}/api/auth/oidc/start?request=one-use`]); + yield* auth.receive(test.callback()); + yield* auth.receive(test.callback()); + expect(test.requests.filter((value) => value.path.endsWith("/exchange"))).toHaveLength(1); + expect(yield* auth.accessToken(server)).toBe("access-in-memory"); + expect(yield* auth.state).toEqual({ serverUrl: server, phase: "connected", error: null }); + const saved = decoder.decode(test.files.get("/isolated/hosted-account.bin")!); + expect(saved).toContain("private-refresh-v1"); + expect(saved).not.toContain("access-in-memory"); + const restarted = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + expect(yield* restarted.accessToken(server)).toBe("access-in-memory"); + expect(test.requests.findLast((value) => value.path.endsWith("/refresh"))!.body).toEqual({ + refreshToken: "private-refresh-v1", + }); + expect(decoder.decode(test.files.get("/isolated/hosted-account.bin")!)).toContain( + "private-refresh-v2", + ); + }), + ); + + it.effect("preserves an existing credential when a new sign-in fails or is canceled", () => + Effect.gen(function* () { + const test = harness(); + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* auth.begin(server); + yield* auth.receive(test.callback()); + const saved = test.files.get("/isolated/hosted-account.bin"); + yield* auth.begin(server); + yield* auth.cancel; + yield* auth.receive(test.callback()); + expect(test.files.get("/isolated/hosted-account.bin")).toEqual(saved); + test.control.exchangeStatus = 401; + yield* auth.begin(server); + yield* auth.receive(test.callback()); + expect((yield* auth.state).phase).toBe("error"); + expect(test.files.get("/isolated/hosted-account.bin")).toEqual(saved); + }), + ); + + it.effect("rejects callbacks for another state or server without exchanging proof", () => + Effect.gen(function* () { + const test = harness(); + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + yield* auth.begin(server); + const wrong = new URL(test.callback()); + wrong.searchParams.set("server", "https://other.example.test"); + yield* auth.receive(wrong.href); + expect(test.requests.filter((value) => value.path.endsWith("/exchange"))).toEqual([]); + expect(test.files.size).toBe(0); + }), + ); + + it.effect("does not start sign-in when protected credential storage is unavailable", () => + Effect.gen(function* () { + const test = harness(); + test.control.available = false; + const auth = yield* DesktopHostedAuth.make.pipe(Effect.provide(test.layer)); + expect(Option.isSome(yield* Effect.option(auth.begin(server)))).toBe(false); + expect(test.requests).toEqual([]); + expect(test.opened).toEqual([]); + }), + ); +}); diff --git a/apps/desktop/src/app/DesktopHostedAuth.ts b/apps/desktop/src/app/DesktopHostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..24a442227efced3833aa9e323aff623019d387d2 --- /dev/null +++ b/apps/desktop/src/app/DesktopHostedAuth.ts @@ -0,0 +1,363 @@ +import { normalizeHostedServer as hostedServerUrl } from "@t3tools/shared/hostedServerUrl"; +import { + DesktopHostedAuthState, + AuthArchiveReadScope, + AuthArchiveWriteScope, + AuthEnvironmentConnectScope, + HostedAuthDeviceStartResult, + HostedAuthGrant, + PRIMARY_LOCAL_ENVIRONMENT_ID, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import * as ElectronApp from "../electron/ElectronApp.ts"; +import * as ElectronSafeStorage from "../electron/ElectronSafeStorage.ts"; +import * as ElectronShell from "../electron/ElectronShell.ts"; +import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; +import * as DesktopLocalEnvironmentAuth from "../backend/DesktopLocalEnvironmentAuth.ts"; +import * as DesktopEnvironment from "./DesktopEnvironment.ts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { + createHostedAuthAttempt, + isHostedAuthCallback, + readHostedAuthCallback, + type HostedAuthAttempt, +} from "./hostedAuthHandoff.ts"; + +const SavedCredential = Schema.Struct({ + serverUrl: Schema.String, + deviceId: Schema.String, + refreshToken: Schema.String, +}); + +const encodeSavedCredential = Schema.encodeEffect(Schema.fromJsonString(SavedCredential)); + +export class DesktopHostedAuthError extends Schema.TaggedError()( + "DesktopHostedAuthError", + { message: Schema.String }, +) {} + +export class DesktopHostedAuth extends Context.Service< + DesktopHostedAuth, + { + readonly state: Effect.Effect; + readonly begin: (server: string) => Effect.Effect; + readonly accessToken: (server: string) => Effect.Effect; + readonly disconnect: Effect.Effect; + readonly cancel: Effect.Effect; + readonly receive: (url: string) => Effect.Effect; + readonly configure: Effect.Effect; + } +>()("@t3tools/desktop/app/DesktopHostedAuth") {} + +export const make = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const safe = yield* ElectronSafeStorage.ElectronSafeStorage; + const shell = yield* ElectronShell.ElectronShell; + const app = yield* ElectronApp.ElectronApp; + const http = yield* HttpClient.HttpClient; + const args = yield* HostProcess.Arguments; + const lock = yield* Semaphore.make(1); + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const localAuth = yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth; + const credentialPath = path.join(environment.stateDir, "hosted-account.bin"); + let pending: HostedAuthAttempt | null = null; + let access: { serverUrl: string; token: string; expiresAt: number } | null = null; + let source: { serverUrl: string; token: string; expiresAt: number } | null = null; + let state: DesktopHostedAuthState = { serverUrl: null, phase: "idle", error: null }; + + const fail = () => + new DesktopHostedAuthError({ message: "Sign in to your server again to reconnect." }); + const read = Effect.gen(function* () { + if (!(yield* fs.exists(credentialPath))) return null; + const bytes = yield* fs.readFile(credentialPath); + return yield* safe + .decryptString(bytes) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SavedCredential)))); + }).pipe(Effect.mapError(fail)); + + const protectedStorageAvailable = Effect.gen(function* () { + return ( + (yield* safe.isEncryptionAvailable) && + Option.getOrNull(yield* safe.selectedStorageBackend) !== "basic_text" + ); + }); + + const save = (serverUrl: string, grant: HostedAuthGrant) => + Effect.gen(function* () { + if (!(yield* protectedStorageAvailable)) return yield* fail(); + const json = yield* encodeSavedCredential({ + serverUrl, + deviceId: grant.deviceId, + refreshToken: grant.refreshToken, + }); + const bytes = yield* safe.encryptString(json); + yield* fs.writeFile(`${credentialPath}.tmp`, bytes, { mode: 0o600 }); + yield* fs.rename(`${credentialPath}.tmp`, credentialPath); + access = { + serverUrl, + token: grant.accessToken, + expiresAt: DateTime.toEpochMillis(grant.expiresAt), + }; + }).pipe(Effect.mapError(fail)); + + const post = ( + server: string, + endpoint: string, + body: unknown, + schema: Schema.Codec, + authorization?: string, + ) => + Effect.scoped( + Effect.gen(function* () { + const request = HttpClientRequest.post(`${server}${endpoint}`).pipe( + HttpClientRequest.bodyJsonUnsafe(body), + HttpClientRequest.setHeaders(authorization === undefined ? {} : { authorization }), + ); + const response = yield* HttpClient.withScope(http).execute(request); + if (response.status !== 200) return yield* fail(); + return yield* response.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(schema))); + }), + ).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.mapError(fail), + Effect.timeout("30 seconds"), + Effect.mapError(fail), + ); + + const deliver = (serverUrl: string, token: string) => + Effect.scoped( + Effect.gen(function* () { + const primary = (yield* pool.list).find((item) => item.id === PRIMARY_LOCAL_ENVIRONMENT_ID); + const config = primary === undefined ? Option.none() : yield* primary.currentConfig; + if (Option.isNone(config)) return; + if ( + source?.serverUrl !== serverUrl || + source.expiresAt <= (yield* Clock.currentTimeMillis) + 60_000 + ) { + const narrow = yield* post( + serverUrl, + "/api/auth/devices/token", + { + scopes: [AuthArchiveReadScope, AuthArchiveWriteScope, AuthEnvironmentConnectScope], + }, + Schema.toCodecJson( + Schema.Struct({ accessToken: Schema.String, expiresAt: Schema.DateTimeUtc }), + ), + `Bearer ${token}`, + ); + source = { + serverUrl, + token: narrow.accessToken, + expiresAt: DateTime.toEpochMillis(narrow.expiresAt), + }; + } + const localToken = yield* localAuth.getBearerToken; + const response = yield* HttpClient.withScope(http).execute( + HttpClientRequest.post( + new URL("/api/self-hosted/connection", config.value.httpBaseUrl).href, + ).pipe( + HttpClientRequest.setHeader("authorization", `Bearer ${localToken}`), + HttpClientRequest.bodyJsonUnsafe({ + serverUrl, + authorization: `Bearer ${source.token}`, + expiresAt: DateTime.formatIso(DateTime.makeUnsafe(source.expiresAt)), + }), + ), + ); + if (response.status < 200 || response.status >= 300) return yield* fail(); + }), + ).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.mapError(fail), + ); + + const resolveAccessToken = (server: string) => + Effect.gen(function* () { + const serverUrl = yield* Effect.try({ try: () => hostedServerUrl(server), catch: fail }); + const saved = yield* read; + if (saved === null || saved.serverUrl !== serverUrl) return yield* fail(); + if ( + access?.serverUrl === serverUrl && + access.expiresAt > (yield* Clock.currentTimeMillis) + 60_000 + ) + return access.token; + const grant = yield* post( + serverUrl, + "/api/auth/devices/refresh", + { refreshToken: saved.refreshToken }, + Schema.toCodecJson(HostedAuthGrant), + ); + yield* save(serverUrl, grant); + yield* deliver(serverUrl, grant.accessToken).pipe(Effect.ignore); + return grant.accessToken; + }); + + const accessToken = (server: string) => lock.withPermits(1)(resolveAccessToken(server)); + + const receive = (url: string) => + lock + .withPermits(1)( + Effect.gen(function* () { + if (pending === null || !isHostedAuthCallback(url)) return; + const attempt = pending; + const now = yield* Clock.currentTimeMillis; + const proof = yield* Effect.try({ + try: () => readHostedAuthCallback(url, attempt, now), + catch: fail, + }); + pending = null; + const grant = yield* post( + attempt.server, + "/api/auth/devices/exchange", + proof, + Schema.toCodecJson(HostedAuthGrant), + ); + yield* save(attempt.server, grant); + source = null; + yield* deliver(attempt.server, grant.accessToken).pipe(Effect.ignore); + state = { serverUrl: attempt.server, phase: "connected", error: null }; + }), + ) + .pipe( + Effect.catch(() => + Effect.sync(() => { + state = { ...state, phase: "error", error: "Sign-in did not finish. Sign in again." }; + }), + ), + ); + + return DesktopHostedAuth.of({ + state: Effect.gen(function* () { + if (pending !== null && (yield* Clock.currentTimeMillis) >= pending.expiresAt) { + pending = null; + state = { ...state, phase: "error", error: "Sign-in expired. Sign in again." }; + } + if (state.phase !== "idle") return state; + const saved = yield* read; + return saved === null + ? state + : { serverUrl: saved.serverUrl, phase: "connected", error: null }; + }), + begin: (server) => + lock.withPermits(1)( + Effect.gen(function* () { + if (!(yield* protectedStorageAvailable)) return yield* fail(); + const now = yield* Clock.currentTimeMillis; + const attempt = yield* Effect.try({ + try: () => createHostedAuthAttempt(server, now), + catch: fail, + }); + const result = yield* post( + attempt.server, + "/api/auth/devices/start", + { + state: attempt.state, + codeChallenge: attempt.codeChallenge, + client: { label: "The Knife Desktop", deviceType: "desktop" }, + }, + HostedAuthDeviceStartResult, + ); + const url = new URL(result.authorizationUrl); + if ( + url.origin !== new URL(attempt.server).origin || + url.protocol !== "https:" || + url.username || + url.password + ) + return yield* fail(); + pending = attempt; + state = { serverUrl: attempt.server, phase: "waiting", error: null }; + if (!(yield* shell.openExternal(url.href))) { + pending = null; + state = { + ...state, + phase: "error", + error: "Couldn't open your browser. Sign in again.", + }; + return yield* fail(); + } + }).pipe(Effect.mapError(fail)), + ), + accessToken, + cancel: lock.withPermits(1)( + Effect.sync(() => { + if (pending === null) return; + pending = null; + state = { serverUrl: null, phase: "idle", error: null }; + }), + ), + disconnect: lock.withPermits(1)( + Effect.gen(function* () { + const primary = (yield* pool.list).find((item) => item.id === PRIMARY_LOCAL_ENVIRONMENT_ID); + const config = primary === undefined ? Option.none() : yield* primary.currentConfig; + if (Option.isSome(config)) { + const localToken = yield* localAuth.getBearerToken; + const response = yield* Effect.scoped( + HttpClient.withScope(http).execute( + HttpClientRequest.make("DELETE")( + new URL("/api/self-hosted/connection", config.value.httpBaseUrl).href, + ).pipe(HttpClientRequest.setHeader("authorization", `Bearer ${localToken}`)), + ), + ).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.mapError(fail), + ); + if (response.status < 200 || response.status >= 300) return yield* fail(); + } + yield* fs.remove(credentialPath, { force: true }).pipe(Effect.mapError(fail)); + access = null; + source = null; + pending = null; + state = { serverUrl: null, phase: "idle", error: null }; + }).pipe(Effect.mapError(fail)), + ), + receive, + configure: Effect.gen(function* () { + const run = Effect.runPromiseWith(yield* Effect.context()); + yield* app.on("open-url", (event: { preventDefault: () => void }, url: string) => { + if (!isHostedAuthCallback(url)) return; + event.preventDefault(); + void run(receive(url)); + }); + yield* app.on("second-instance", (_event: unknown, argv: readonly string[]) => { + const url = argv.find(isHostedAuthCallback); + if (url) void run(receive(url)); + }); + yield* Effect.forkScoped( + app.whenReady.pipe( + Effect.andThen( + Effect.forever( + lock + .withPermits(1)( + Effect.gen(function* () { + const saved = yield* read; + if (saved !== null) { + const token = yield* resolveAccessToken(saved.serverUrl); + yield* deliver(saved.serverUrl, token); + } + }), + ) + .pipe(Effect.ignore, Effect.andThen(Effect.sleep("1 minute"))), + ), + ), + ), + ); + const initial = args.find(isHostedAuthCallback); + if (initial) yield* receive(initial); + }), + }); +}); + +export const layer = Layer.effect(DesktopHostedAuth, make); diff --git a/apps/desktop/src/app/hostedAuthHandoff.test.ts b/apps/desktop/src/app/hostedAuthHandoff.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..6c740c42cfe813d195c8a5289903cb1d7b69aaea --- /dev/null +++ b/apps/desktop/src/app/hostedAuthHandoff.test.ts @@ -0,0 +1,71 @@ +import { normalizeHostedServer as hostedServerUrl } from "@t3tools/shared/hostedServerUrl"; +// @effect-diagnostics nodeBuiltinImport:off - Verify the desktop-only PKCE bytes independently. +import * as NodeCrypto from "node:crypto"; +import { describe, expect, it } from "@effect/vitest"; +import { + createHostedAuthAttempt, + isHostedAuthCallback, + readHostedAuthCallback, +} from "./hostedAuthHandoff.ts"; + +const server = "https://knife.example.test"; +const callback = (state: string, origin = server) => + `t3code://app/self-hosted/callback?${new URLSearchParams({ code: "one-time-proof", state, server: origin })}`; + +describe("desktop hosted sign-in handoff", () => { + it("creates independent state and S256 proof material for each login", () => { + const first = createHostedAuthAttempt(server, 0); + const second = createHostedAuthAttempt(server, 0); + expect(first.verifier).toHaveLength(43); + expect(first.codeChallenge).toBe( + NodeCrypto.createHash("sha256").update(first.verifier).digest("base64url"), + ); + expect(first.state).not.toBe(second.state); + expect(first.verifier).not.toBe(second.verifier); + expect(readHostedAuthCallback(callback(first.state), first, 1)).toEqual({ + code: "one-time-proof", + codeVerifier: first.verifier, + }); + }); + + it.each([ + "http://knife.example.test", + "https://user:password@knife.example.test", + "https://knife.example.test?target=other", + "https://knife.example.test#token", + ])("rejects unsafe server addresses: %s", (address) => { + expect(() => hostedServerUrl(address)).toThrow(); + }); + + it("rejects a mounted path and canonicalizes root HTTPS origins", () => { + expect(() => hostedServerUrl("https://knife.example.test/knife/")).toThrow(); + expect(hostedServerUrl(`${server}/`)).toBe(server); + }); + + it.each([ + "https://app/self-hosted/callback", + "t3code://other/self-hosted/callback", + "t3code://user@app/self-hosted/callback", + "t3code://app:123/self-hosted/callback", + "t3code://app/self-hosted/callback/extra", + ])("does not intercept unrelated links: %s", (url) => { + expect(isHostedAuthCallback(url)).toBe(false); + }); + + it("rejects callbacks from another server, login, or expired attempt", () => { + const pending = createHostedAuthAttempt(server, 100); + expect(() => + readHostedAuthCallback(callback(pending.state, "https://other.example.test"), pending, 101), + ).toThrow(); + expect(() => readHostedAuthCallback(callback("wrong-state"), pending, 101)).toThrow(); + expect(() => + readHostedAuthCallback(callback(pending.state), pending, pending.expiresAt), + ).toThrow(); + expect(() => + readHostedAuthCallback(callback(pending.state) + "&code=other", pending, 101), + ).toThrow(); + expect(() => + readHostedAuthCallback(callback(pending.state) + "#proof", pending, 101), + ).toThrow(); + }); +}); diff --git a/apps/desktop/src/app/hostedAuthHandoff.ts b/apps/desktop/src/app/hostedAuthHandoff.ts new file mode 100644 index 0000000000000000000000000000000000000000..06984fca70030bced3ae4088ace8dae8ddd67ebc --- /dev/null +++ b/apps/desktop/src/app/hostedAuthHandoff.ts @@ -0,0 +1,58 @@ +// @effect-diagnostics nodeBuiltinImport:off - Desktop login proofs never cross into the renderer. +import * as NodeCrypto from "node:crypto"; + +import { normalizeHostedServer as hostedServerUrl } from "@t3tools/shared/hostedServerUrl"; + +export function createHostedAuthAttempt(server: string, now: number) { + const verifier = NodeCrypto.randomBytes(32).toString("base64url"); + return { + server: hostedServerUrl(server), + state: NodeCrypto.randomBytes(32).toString("base64url"), + verifier, + codeChallenge: NodeCrypto.createHash("sha256").update(verifier).digest("base64url"), + expiresAt: now + 10 * 60_000, + }; +} + +export type HostedAuthAttempt = ReturnType; + +export function isHostedAuthCallback(value: string): boolean { + try { + const url = new URL(value); + return ( + url.protocol === "t3code:" && + url.host === "app" && + !url.username && + !url.password && + url.pathname === "/self-hosted/callback" + ); + } catch { + return false; + } +} + +export function readHostedAuthCallback(value: string, pending: HostedAuthAttempt, now: number) { + if (!isHostedAuthCallback(value)) throw new Error("This sign-in link is invalid."); + const url = new URL(value); + const code = url.searchParams.get("code"); + const state = url.searchParams.get("state"); + const server = url.searchParams.get("server"); + if ( + url.hash || + url.searchParams.size !== 3 || + !code || + code.length > 512 || + !state || + !server || + now >= pending.expiresAt || + hostedServerUrl(server) !== pending.server + ) { + throw new Error("This sign-in link has expired or belongs to another sign-in. Sign in again."); + } + const actual = Buffer.from(state); + const expected = Buffer.from(pending.state); + if (actual.length !== expected.length || !NodeCrypto.timingSafeEqual(actual, expected)) { + throw new Error("This sign-in link belongs to another sign-in. Sign in again."); + } + return { code, codeVerifier: pending.verifier }; +} diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 585bbb3864b0fbe8ddbca903baefdc6740976ccf..22e1cc6afc5922f058bbbec8c2b50e4feab78325 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -1,3 +1,10 @@ +import { + cancelHostedAuth, + beginHostedAuth, + getHostedAuthState, + getHostedAccessToken, + disconnectHostedAuth, +} from "./methods/hostedAuth.ts"; import { openVideo } from "./methods/media.ts"; import * as Effect from "effect/Effect"; @@ -104,6 +111,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(getClientSettings); yield* ipc.handle(setClientSettings); + yield* ipc.handle(cancelHostedAuth); + yield* ipc.handle(beginHostedAuth); + yield* ipc.handle(getHostedAuthState); + yield* ipc.handle(getHostedAccessToken); + yield* ipc.handle(disconnectHostedAuth); yield* ipc.handle(getConnectionCatalog); yield* ipc.handle(getSnapShotState); yield* ipc.handle(setupSnapShot); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 9a8e05d20ee25dda9a209ddcb9f574354a6117b4..0613cf05091803e1366712046d287c0a1ee0b622 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -126,3 +126,10 @@ export const COMPLETE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:complete-legacy-lo export const DESKTOP_TERMINAL_CHANNEL = "desktop:terminal"; export const OPEN_VIDEO_CHANNEL = "desktop:open-video"; + +export const BEGIN_HOSTED_AUTH_CHANNEL = "desktop:hosted-auth:begin"; +export const GET_HOSTED_AUTH_STATE_CHANNEL = "desktop:hosted-auth:state"; +export const GET_HOSTED_ACCESS_TOKEN_CHANNEL = "desktop:hosted-auth:access-token"; +export const DISCONNECT_HOSTED_AUTH_CHANNEL = "desktop:hosted-auth:disconnect"; + +export const CANCEL_HOSTED_AUTH_CHANNEL = "desktop:hosted-auth:cancel"; diff --git a/apps/desktop/src/ipc/methods/hostedAuth.ts b/apps/desktop/src/ipc/methods/hostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..add74ed7168c22accaa1603f5cc5c858b4cb5f64 --- /dev/null +++ b/apps/desktop/src/ipc/methods/hostedAuth.ts @@ -0,0 +1,48 @@ +import { DesktopHostedAuthState } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as DesktopHostedAuth from "../../app/DesktopHostedAuth.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +export const beginHostedAuth = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.BEGIN_HOSTED_AUTH_CHANNEL, + payload: Schema.String, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.hostedAuth.begin")(function* (server) { + return yield* (yield* DesktopHostedAuth.DesktopHostedAuth).begin(server); + }), +}); +export const getHostedAuthState = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_HOSTED_AUTH_STATE_CHANNEL, + payload: Schema.Void, + result: DesktopHostedAuthState, + handler: Effect.fn("desktop.ipc.hostedAuth.state")(function* () { + return yield* (yield* DesktopHostedAuth.DesktopHostedAuth).state; + }), +}); +export const getHostedAccessToken = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_HOSTED_ACCESS_TOKEN_CHANNEL, + payload: Schema.String, + result: Schema.String, + handler: Effect.fn("desktop.ipc.hostedAuth.accessToken")(function* (server) { + return yield* (yield* DesktopHostedAuth.DesktopHostedAuth).accessToken(server); + }), +}); +export const disconnectHostedAuth = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.DISCONNECT_HOSTED_AUTH_CHANNEL, + payload: Schema.Void, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.hostedAuth.disconnect")(function* () { + return yield* (yield* DesktopHostedAuth.DesktopHostedAuth).disconnect; + }), +}); + +export const cancelHostedAuth = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.CANCEL_HOSTED_AUTH_CHANNEL, + payload: Schema.Void, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.hostedAuth.cancel")(function* () { + return yield* (yield* DesktopHostedAuth.DesktopHostedAuth).cancel; + }), +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 85f987a3cda4fcfa4314b2fbda70910e4ac832ab..d5f1e5e88238c3bf379ffed0b24c31e66900bee9 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -39,6 +39,7 @@ import * as DesktopAppActivation from "./app/DesktopAppActivation.ts"; import * as DesktopWebLinks from "./app/DesktopWebLinks.ts"; import * as DesktopAppIdentity from "./app/DesktopAppIdentity.ts"; import * as DesktopConnectionCatalogStore from "./app/DesktopConnectionCatalogStore.ts"; +import * as DesktopHostedAuth from "./app/DesktopHostedAuth.ts"; import * as DesktopClerk from "./app/DesktopClerk.ts"; import * as DesktopCliCommand from "./app/DesktopCliCommand.ts"; import * as DesktopApplicationMenu from "./window/DesktopApplicationMenu.ts"; @@ -222,6 +223,7 @@ const layerDesktopLocalEnvironmentAuth = DesktopLocalEnvironmentAuth.layer.pipe( ); const layerDesktopApplication = Layer.mergeAll( + DesktopHostedAuth.layer, DesktopLifecycle.layer, DesktopVideo.layer, layerDesktopAppActivation, diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 0278a55944a6b20aa5a99b88aa34787a891f6c50..5bdfbace672e9308e972463f0072edf8719140e6 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -161,6 +161,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { dismissSnapShotAnimation: (id) => ipcRenderer.invoke(IpcChannels.DISMISS_SNAP_SHOT_ANIMATION_CHANNEL, id), acknowledgeSnapShot: (id) => ipcRenderer.invoke(IpcChannels.ACKNOWLEDGE_SNAP_SHOT_CHANNEL, id), + beginHostedAuth: (server) => ipcRenderer.invoke(IpcChannels.BEGIN_HOSTED_AUTH_CHANNEL, server), + getHostedAuthState: () => ipcRenderer.invoke(IpcChannels.GET_HOSTED_AUTH_STATE_CHANNEL), + getHostedAccessToken: (server) => + ipcRenderer.invoke(IpcChannels.GET_HOSTED_ACCESS_TOKEN_CHANNEL, server), + cancelHostedAuth: () => ipcRenderer.invoke(IpcChannels.CANCEL_HOSTED_AUTH_CHANNEL), + disconnectHostedAuth: () => ipcRenderer.invoke(IpcChannels.DISCONNECT_HOSTED_AUTH_CHANNEL), getConnectionCatalog: () => ipcRenderer.invoke(IpcChannels.GET_CONNECTION_CATALOG_CHANNEL), setConnectionCatalog: (catalog) => ipcRenderer.invoke(IpcChannels.SET_CONNECTION_CATALOG_CHANNEL, catalog), diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index b1b286b32ac4a9f436918bedca6c40c3cd42b169..81fa8840d1b5ffdaa90a72cee0fb77cd6f38b786 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -2,12 +2,12 @@ import { UiSettingsSyncProvider } from "./state/uiSettingsSync"; import { PermissionUpdateNotice } from "./components/PermissionUpdateNotice"; import * as Linking from "expo-linking"; import * as SplashScreen from "expo-splash-screen"; -import { useEffect } from "react"; +import { useCallback, useEffect, useRef, type RefObject } from "react"; import { StatusBar } from "react-native"; import { GestureHandlerRootView } from "react-native-gesture-handler"; import { KeyboardProvider } from "react-native-keyboard-controller"; import { SafeAreaProvider } from "react-native-safe-area-context"; -import { createStaticNavigation } from "@react-navigation/native"; +import { createStaticNavigation, type NavigationState } from "@react-navigation/native"; import { RegistryContext } from "@effect/atom-react"; import { ThreadArrangementHost } from "./features/threads/ThreadArrangementSheet"; @@ -61,13 +61,20 @@ function SplashScreenCoordinator() { export default function App() { useUiRuntimeMemoryWarningGc(); + const navigationState = useRef(undefined); + const saveNavigationState = useCallback((state: NavigationState | undefined) => { + navigationState.current = state; + }, []); return ( - + @@ -75,7 +82,10 @@ export default function App() { ); } -function AppContent() { +function AppContent(props: { + readonly navigationState: RefObject; + readonly saveNavigationState: (state: NavigationState | undefined) => void; +}) { const { themeAppearance } = useAppearancePreferences(); const navigationTheme = useMobileNavigationTheme(); @@ -97,7 +107,12 @@ function AppContent() { - + diff --git a/apps/mobile/src/connection/hub.ts b/apps/mobile/src/connection/hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..20adbe9997c59050029d7597e10ceb82041853b6 --- /dev/null +++ b/apps/mobile/src/connection/hub.ts @@ -0,0 +1,5 @@ +import { createHubConnectionAtoms } from "@t3tools/client-runtime/state/hub"; + +import { connectionAtomRuntime } from "./runtime"; + +export const hubConnection = createHubConnectionAtoms(connectionAtomRuntime); diff --git a/apps/mobile/src/connection/platform.ts b/apps/mobile/src/connection/platform.ts index e8817de69eeb2a6fcf0b32d775fd4b339349430c..fdc3f3e781794661ba2026bd23f5c4418f5bb567 100644 --- a/apps/mobile/src/connection/platform.ts +++ b/apps/mobile/src/connection/platform.ts @@ -254,11 +254,13 @@ const layerProvidedCapabilities = layerCapabilities.pipe(Layer.provide(Runtime.l const layerEnvironmentOwnedDataCleanup = Layer.succeed( Persistence.EnvironmentOwnedDataCleanup, Persistence.EnvironmentOwnedDataCleanup.of({ - clear: (environmentId) => + clear: (environmentId, options) => Effect.all( [ Effect.promise(() => clearThreadOutboxEnvironment(environmentId)), - Effect.promise(() => clearComposerDraftsEnvironment(environmentId)), + ...(options?.retainDrafts + ? [] + : [Effect.promise(() => clearComposerDraftsEnvironment(environmentId))]), Effect.sync(() => clearThreadComposerErrorsForEnvironment(environmentId)), ], { concurrency: "unbounded", discard: true }, diff --git a/apps/mobile/src/connection/storage.test.ts b/apps/mobile/src/connection/storage.test.ts index 7de8ded26639f8b55972565d2fe7985e713f86f3..f764c218d822cfa255b1ba3dc20a0364dab665c7 100644 --- a/apps/mobile/src/connection/storage.test.ts +++ b/apps/mobile/src/connection/storage.test.ts @@ -1,3 +1,14 @@ +import { + BearerConnectionCredential, + BearerConnectionProfile, + BearerConnectionRegistration, + BearerConnectionTarget, + CredentialStore, + HubConnectionCredential, +} from "@t3tools/client-runtime/connection"; +import { EnvironmentId } from "@t3tools/contracts"; +import { Persistence } from "@t3tools/client-runtime/platform"; +import * as Option from "effect/Option"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import { vi } from "vite-plus/test"; @@ -12,6 +23,7 @@ vi.mock("expo-secure-store", () => ({ setItemAsync: vi.fn(), })); +import * as ConnectionStorage from "./storage"; import { CONNECTION_CATALOG_KEY, LEGACY_CONNECTIONS_KEY, make } from "./catalog-store"; import * as MobileSecureStorage from "../persistence/mobile-secure-storage"; @@ -34,6 +46,62 @@ function makeStorage(initial: Readonly>) { } describe("mobile connection catalog storage", () => { + it.effect("restores hub route pointers and the single bearer after reopening storage", () => + Effect.gen(function* () { + const memory = makeStorage({}); + const connectionId = "hub:hub:source"; + const hubCredentialId = "hub-session:hub"; + const credential = new HubConnectionCredential({ connectionId: hubCredentialId }); + const bearer = new BearerConnectionCredential({ token: "single-hub-secret" }); + const registration = new BearerConnectionRegistration({ + target: new BearerConnectionTarget({ + connectionId, + environmentId: EnvironmentId.make("source"), + label: "Source", + }), + profile: new BearerConnectionProfile({ + connectionId, + environmentId: EnvironmentId.make("source"), + label: "Source", + httpBaseUrl: "https://hub.example/environments/source/", + wsBaseUrl: "wss://hub.example/environments/source/", + }), + credential, + }); + yield* Effect.gen(function* () { + const credentials = yield* CredentialStore.ConnectionCredentialStore; + const registrations = yield* Persistence.ConnectionRegistrationStore; + yield* credentials.put(hubCredentialId, bearer); + yield* registrations.register(registration, [registration.target]); + expect(yield* credentials.get(connectionId)).toEqual(Option.some(credential)); + }).pipe( + Effect.provide(ConnectionStorage.layer), + Effect.provideService(MobileSecureStorage.MobileSecureStorage, memory.storage), + Effect.scoped, + ); + + yield* Effect.gen(function* () { + const credentials = yield* CredentialStore.ConnectionCredentialStore; + expect(yield* credentials.get(connectionId)).toEqual(Option.some(credential)); + expect(yield* credentials.get(hubCredentialId)).toEqual(Option.some(bearer)); + }).pipe( + Effect.provide(ConnectionStorage.layer), + Effect.provideService(MobileSecureStorage.MobileSecureStorage, memory.storage), + Effect.scoped, + ); + expect(JSON.parse(memory.values.get(CONNECTION_CATALOG_KEY)!).credentials).toEqual([ + { + connectionId: hubCredentialId, + credential: { _tag: "BearerConnectionCredential", token: bearer.token }, + }, + { + connectionId, + credential: { _tag: "HubConnectionCredential", connectionId: hubCredentialId }, + }, + ]); + }), + ); + it.effect("recovers from a corrupt current catalog", () => Effect.gen(function* () { const memory = makeStorage({ diff --git a/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts b/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts index 5fe74f6731418ae9b10b3c21de2d85b3860a1c79..bc1e05d4d466968d12986ae7b9cea63d567b2ccf 100644 --- a/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts +++ b/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts @@ -27,6 +27,7 @@ vi.mock("../../connection/catalog", () => ({ })); vi.mock("./cloud-drafts", () => ({ removeCloudEnvironments: {} })); +vi.mock("../../state/preferences", () => ({ mobilePreferencesAtom: {} })); vi.mock("../../state/use-composer-drafts", () => ({ getComposerCloudAccountId: vi.fn(async () => null), restoreCloudComposerDrafts: vi.fn(async () => undefined), diff --git a/apps/mobile/src/features/cloud/CloudAuthProvider.tsx b/apps/mobile/src/features/cloud/CloudAuthProvider.tsx index fffdd2343044f753e14acd6286d7aa93bcb3fc27..39e382af866ddb1bdbb81dfc9c4740642b0decee 100644 --- a/apps/mobile/src/features/cloud/CloudAuthProvider.tsx +++ b/apps/mobile/src/features/cloud/CloudAuthProvider.tsx @@ -1,5 +1,6 @@ import { ClerkProvider, useAuth } from "@clerk/expo"; import { tokenCache } from "@clerk/expo/token-cache"; +import { useAtomValue } from "@effect/atom-react"; import { ManagedRelay, setManagedRelaySession } from "@t3tools/client-runtime/relay"; import { reportAtomCommandResult, @@ -9,6 +10,7 @@ import { } from "@t3tools/client-runtime/state/runtime"; import * as Effect from "effect/Effect"; import { type ReactNode, useEffect, useRef } from "react"; +import { AsyncResult } from "effect/reactivity"; import { runtime } from "../../lib/runtime"; import { appAtomRegistry } from "../../state/atom-registry"; @@ -25,6 +27,10 @@ import { import { clearConnectOnboardingRequest, requestConnectOnboarding } from "./connectOnboarding"; import { resolveCloudPublicConfig, resolveRelayClerkTokenOptions } from "./publicConfig"; import { removeCloudEnvironments } from "./cloud-drafts"; +import { environmentCatalog } from "../../connection/catalog"; +import { mobilePreferencesAtom } from "../../state/preferences"; +import { CloudAuthAvailableContext } from "./cloudAvailability"; +import { shouldEnableCloudAuth } from "./connectionService"; function resetManagedRelayTokenCache() { return settleAsyncResult(() => @@ -196,23 +202,37 @@ function CloudAuthBridge(props: { readonly children: ReactNode }) { } export function CloudAuthProvider(props: { readonly children: ReactNode }) { + const preferences = useAtomValue(mobilePreferencesAtom); + const catalog = useAtomValue(environmentCatalog.catalogValueAtom); const config = resolveCloudPublicConfig(); const publishableKey = config.clerk.publishableKey; const relayUrl = config.relay.url; + const enabled = + Boolean(publishableKey && relayUrl) && + shouldEnableCloudAuth( + AsyncResult.isSuccess(preferences) ? preferences.value.connectionService : undefined, + catalog.entries.values(), + ); useEffect(() => { - if (!publishableKey || !relayUrl) { + if (!enabled) { deactivateCloudRelayAccount(); } - }, [publishableKey, relayUrl]); + }, [enabled]); - if (!publishableKey || !relayUrl) { + if (!catalog.isReady || AsyncResult.isInitial(preferences)) { + return null; + } + + if (!enabled || !publishableKey) { return props.children; } return ( - {props.children} + + {props.children} + ); } diff --git a/apps/mobile/src/features/cloud/ConnectOnboardingRouteScreen.tsx b/apps/mobile/src/features/cloud/ConnectOnboardingRouteScreen.tsx index 7987e8ff6d66fd3e504503dab4bc9e3ee6ceb5a0..1f59e388859f5c663eb08690de7601e2b2fe1452 100644 --- a/apps/mobile/src/features/cloud/ConnectOnboardingRouteScreen.tsx +++ b/apps/mobile/src/features/cloud/ConnectOnboardingRouteScreen.tsx @@ -14,7 +14,7 @@ import { CloudEnvironmentRows } from "../connection/CloudEnvironmentRows"; import { splitEnvironmentSections } from "../connection/environmentSections"; import { useConnectionController } from "../connection/useConnectionController"; import { optOutOfConnectOnboarding } from "./connectOnboardingOptOut"; -import { hasCloudPublicConfig } from "./publicConfig"; +import { useCloudAuthAvailable } from "./cloudAvailability"; /** * Post-sign-in onboarding sheet for T3 Connect. Mobile never publishes @@ -25,11 +25,12 @@ import { hasCloudPublicConfig } from "./publicConfig"; */ export function ConnectOnboardingRouteScreen() { const navigation = useNavigation(); + const cloudAvailable = useCloudAuthAvailable(); // The route is deep-linkable; without cloud config the sheet would present // empty with no chrome to dismiss it, so bail back out instead. useEffect(() => { - if (hasCloudPublicConfig()) { + if (cloudAvailable) { return; } if (navigation.canGoBack()) { @@ -37,9 +38,9 @@ export function ConnectOnboardingRouteScreen() { } else { navigation.dispatch(StackActions.replace("Home")); } - }, [navigation]); + }, [cloudAvailable, navigation]); - return hasCloudPublicConfig() ? : null; + return cloudAvailable ? : null; } function ConfiguredConnectOnboardingRouteScreen() { diff --git a/apps/mobile/src/features/cloud/cloudAvailability.ts b/apps/mobile/src/features/cloud/cloudAvailability.ts new file mode 100644 index 0000000000000000000000000000000000000000..73fd7360d0c956fdecbe14e3621ec7ff39a268dc --- /dev/null +++ b/apps/mobile/src/features/cloud/cloudAvailability.ts @@ -0,0 +1,7 @@ +import { createContext, useContext } from "react"; + +export const CloudAuthAvailableContext = createContext(false); + +export function useCloudAuthAvailable(): boolean { + return useContext(CloudAuthAvailableContext); +} diff --git a/apps/mobile/src/features/cloud/connectionService.test.ts b/apps/mobile/src/features/cloud/connectionService.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..047e3c6308a15d67c915b30e3d5bf49157f71162 --- /dev/null +++ b/apps/mobile/src/features/cloud/connectionService.test.ts @@ -0,0 +1,46 @@ +import { + BearerConnectionTarget, + RelayConnectionTarget, + type ConnectionCatalogEntry, +} from "@t3tools/client-runtime/connection"; +import { EnvironmentId } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import { describe, expect, it } from "vite-plus/test"; + +import { shouldEnableCloudAuth } from "./connectionService"; + +const direct: ConnectionCatalogEntry = { + target: new BearerConnectionTarget({ + environmentId: EnvironmentId.make("machine"), + connectionId: "direct", + label: "Laptop", + }), + profile: Option.none(), + enabled: true, +}; + +describe("optional mobile T3 Connect", () => { + it("keeps fresh, direct, and self-hosted sessions independent of cloud sign-in", () => { + expect(shouldEnableCloudAuth(undefined, [])).toBe(false); + expect(shouldEnableCloudAuth(undefined, [direct])).toBe(false); + expect(shouldEnableCloudAuth("self-hosted", [direct])).toBe(false); + expect(shouldEnableCloudAuth("t3-connect", [])).toBe(true); + }); + + it("restores existing cloud routes even when a direct route is preferred", () => { + const entry: ConnectionCatalogEntry = { + ...direct, + alternateRoutes: [ + { + target: new RelayConnectionTarget({ + environmentId: direct.target.environmentId, + label: "Laptop", + }), + profile: Option.none(), + }, + ], + }; + expect(shouldEnableCloudAuth(undefined, [entry])).toBe(true); + expect(shouldEnableCloudAuth("self-hosted", [entry])).toBe(true); + }); +}); diff --git a/apps/mobile/src/features/cloud/connectionService.ts b/apps/mobile/src/features/cloud/connectionService.ts new file mode 100644 index 0000000000000000000000000000000000000000..4fa5cad6a0a9a68f4e033eff1032d88685bb1545 --- /dev/null +++ b/apps/mobile/src/features/cloud/connectionService.ts @@ -0,0 +1,13 @@ +import { connectionRoutes, type ConnectionCatalogEntry } from "@t3tools/client-runtime/connection"; + +export function shouldEnableCloudAuth( + service: "t3-connect" | "self-hosted" | undefined, + entries: Iterable, +): boolean { + return ( + service === "t3-connect" || + [...entries].some((entry) => + connectionRoutes(entry).some((route) => route.target._tag === "RelayConnectionTarget"), + ) + ); +} diff --git a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx index 4d4dbbc424da4ef886ecb01d5e7baf527a682d42..056b02186ecb41656c59e0fe3a2b3322df601104 100644 --- a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx +++ b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx @@ -21,7 +21,7 @@ import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; import type { ConnectedEnvironmentSummary } from "../../state/remote-runtime-types"; import { serverEnvironment } from "../../state/server"; import { availableCloudEnvironmentPresentation } from "../cloud/cloudEnvironmentPresentation"; -import { hasCloudPublicConfig } from "../cloud/publicConfig"; +import { useCloudAuthAvailable } from "../cloud/cloudAvailability"; import { ConnectionStatusDot } from "./ConnectionStatusDot"; import { environmentStatusText } from "./environmentStatusText"; import { type RelayEnvironmentView, useConnectionController } from "./useConnectionController"; @@ -52,14 +52,14 @@ interface CloudEnvironmentRowsProps { * its errors) requires a signed-in session. */ export function CloudEnvironmentRows(props: CloudEnvironmentRowsProps) { + const cloudAvailable = useCloudAuthAvailable(); // Showcase captures run without a Clerk publishable key, so `ClerkProvider` // is never mounted and any `useAuth` call throws — the fixture states whether // the rows are signed in instead of asking Clerk. if (props.showcaseSignedIn !== undefined) { return props.showcaseSignedIn ? : null; } - // No cloud config means no `ClerkProvider` either, so `useAuth` would throw. - if (!hasCloudPublicConfig()) { + if (!cloudAvailable) { return ; } return ; diff --git a/apps/mobile/src/features/connection/ConnectionsNewRouteScreen.tsx b/apps/mobile/src/features/connection/ConnectionsNewRouteScreen.tsx index 89580939d4c550b51948672e0015ca30818748b2..0973d3a5530757d84b93c53f3aa98e3e78242b19 100644 --- a/apps/mobile/src/features/connection/ConnectionsNewRouteScreen.tsx +++ b/apps/mobile/src/features/connection/ConnectionsNewRouteScreen.tsx @@ -1,5 +1,6 @@ import { ScreenScrollView as ScrollView } from "../../components/ScreenScrollView"; import { useAuth } from "@clerk/expo"; +import { useAtomSet, useAtomValue } from "@effect/atom-react"; import { CameraView, useCameraPermissions } from "expo-camera"; import * as Haptics from "expo-haptics"; import { @@ -30,6 +31,11 @@ import { SymbolView } from "../../components/AppSymbol"; import { hasCloudPublicConfig } from "../cloud/publicConfig"; import { CloudEnvironmentRows } from "./CloudEnvironmentRows"; import { splitEnvironmentSections } from "./environmentSections"; +import { MaterialButton } from "../../components/MaterialButton"; +import { useCloudAuthAvailable } from "../cloud/cloudAvailability"; +import { hubConnection } from "../../connection/hub"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; type ConnectionsNewRouteParams = { readonly mode?: string; @@ -51,9 +57,23 @@ export function ConnectionsNewRouteScreen({ const navigation = useNavigation(); const routeName = useRoute().name; const params = route.params ?? {}; + const t3ConnectAvailable = hasCloudPublicConfig(); + const cloudAvailable = useCloudAuthAvailable(); + const connectHub = useAtomCommand(hubConnection.connect, { reportFailure: false }); + const hub = useAtomValue(hubConnection.stateValueAtom); + const preferences = useAtomValue(mobilePreferencesAtom); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); // Deep-link prefill exists for development automation only. A production // link must not arrive with attacker-chosen host and token already filled. const routePairingUrl = __DEV__ ? (params.pairingUrl?.trim() ?? "") : ""; + const [step, setStep] = useState<"choose" | "self-hosted" | "direct" | "cloud">( + params.routeFor || routePairingUrl ? "direct" : "choose", + ); + const [method, setMethod] = useState<"t3-connect" | "self-hosted">( + AsyncResult.isSuccess(preferences) && t3ConnectAvailable + ? (preferences.value.connectionService ?? "self-hosted") + : "self-hosted", + ); const shouldAutoConnect = __DEV__ && routePairingUrl.length > 0 && @@ -161,13 +181,20 @@ export function ConnectionsNewRouteScreen({ setIsSubmitting(true); onChangeConnectionPairingUrl(pairingUrl); try { - const result = await onConnectPress(pairingUrl, params.routeFor); + const result = + step === "self-hosted" + ? await connectHub({ pairingUrl }) + : await onConnectPress(pairingUrl, params.routeFor); void Haptics.notificationAsync( - AsyncResult.isSuccess(result) + result._tag === "Success" ? Haptics.NotificationFeedbackType.Success : Haptics.NotificationFeedbackType.Error, ); - if (AsyncResult.isSuccess(result)) { + if (result._tag === "Success") { + if (step === "self-hosted") { + savePreferences({ connectionService: "self-hosted" }); + onChangeConnectionPairingUrl(""); + } if (replaceWithHome || !navigation.canGoBack()) { navigation.dispatch(StackActions.replace("Home")); } else { @@ -178,7 +205,15 @@ export function ConnectionsNewRouteScreen({ setIsSubmitting(false); } }, - [navigation, onChangeConnectionPairingUrl, onConnectPress, params.routeFor], + [ + connectHub, + navigation, + onChangeConnectionPairingUrl, + onConnectPress, + params.routeFor, + savePreferences, + step, + ], ); const handleSubmit = useCallback(async () => { @@ -194,13 +229,106 @@ export function ConnectionsNewRouteScreen({ void connectAndClose(routePairingUrl, true); }, [connectAndClose, routePairingUrl, shouldAutoConnect]); + if (step === "choose") { + return ( + + + + Choose how to reach your computers. + + + {( + [ + ["t3-connect", "T3 Connect", "Sign in with your T3 account."], + ["self-hosted", "Self-hosted server", "Connect through your own server."], + ] as const + ).map(([value, label, description]) => ( + setMethod(value)} + className="min-h-20 flex-row items-center gap-4 px-4 py-3 active:opacity-70 disabled:opacity-45" + > + + + {label} + + {value === "t3-connect" && !t3ConnectAvailable + ? "Unavailable in this app build." + : description} + + + + ))} + + { + if (method === "t3-connect") { + if (cloudAvailable) setStep("cloud"); + else navigation.navigate("SettingsSheet", { screen: "SettingsAuth" }); + } else setStep("self-hosted"); + }} + /> + setStep("direct")} + /> + + + ); + } + + if (step === "cloud") { + return ( + + + + setStep("choose")} + /> + + + ); + } + + const pairingError = step === "self-hosted" ? hub.error : pairingConnectionError; + return ( - {pairingConnectionError ?? - "For machines on your local network or tailnet. The machine keeps its own provider credentials."} + {pairingError ?? + (step === "self-hosted" + ? "Create a pairing code on your server. Its machines will appear here." + : "For machines on your local network or tailnet. The machine keeps its own provider credentials.")} - {hasCloudPublicConfig() ? : null} + {!params.routeFor ? ( + setStep("choose")} + /> + ) : null} @@ -393,9 +532,6 @@ function T3ConnectSection() { return ( - - Or use the managed relay - {isSignedIn ? ( + {label} diff --git a/apps/mobile/src/features/connection/ConnectionsRouteScreen.tsx b/apps/mobile/src/features/connection/ConnectionsRouteScreen.tsx index 1e7f128ed43b929ee399953f78eacfc62b4d398c..3890822742d224536d11e523ff5f38bea5412ffa 100644 --- a/apps/mobile/src/features/connection/ConnectionsRouteScreen.tsx +++ b/apps/mobile/src/features/connection/ConnectionsRouteScreen.tsx @@ -10,6 +10,7 @@ import { AndroidScreenHeader } from "../../components/AndroidScreenHeader"; import { useRemoteConnections } from "../../state/use-remote-environment-registry"; import { LocalEnvironmentList } from "./LocalEnvironmentList"; import { GitHubRoutingSettings } from "./GitHubRoutingSettings"; +import { SelfHostedServerSection } from "./SelfHostedServerSection"; export function ConnectionsRouteScreen() { const { @@ -59,6 +60,7 @@ export function ConnectionsRouteScreen() { paddingTop: 16, }} > + ; @@ -22,6 +24,8 @@ export function LocalEnvironmentList({ readonly expandedId: EnvironmentId | null; readonly onToggle: (environmentId: EnvironmentId) => void; }) { + const hub = useAtomValue(hubConnection.stateValueAtom); + if (environments.length === 0) { return ( - No environments connected yet.{"\n"}Tap{" "} - + to add one. + {hub.connection ? ( + <>No machines connected.{"\n"}Join this server from a computer to add one. + ) : ( + <> + No environments connected yet.{"\n"}Tap{" "} + + to add one. + + )} ); diff --git a/apps/mobile/src/features/connection/SelfHostedServerSection.tsx b/apps/mobile/src/features/connection/SelfHostedServerSection.tsx new file mode 100644 index 0000000000000000000000000000000000000000..af5af80a1099f8ee39f1db39ff86e78b58dfa2f3 --- /dev/null +++ b/apps/mobile/src/features/connection/SelfHostedServerSection.tsx @@ -0,0 +1,60 @@ +import { useAtomValue } from "@effect/atom-react"; +import { Alert, View } from "react-native"; + +import { AppText as Text } from "../../components/AppText"; +import { ListRowLayout } from "../../components/ListRowLayout"; +import { hubConnection } from "../../connection/hub"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { SettingsRow } from "../settings/components/SettingsRow"; +import { SettingsSection } from "../settings/components/SettingsSection"; + +export function SelfHostedServerSection() { + const hub = useAtomValue(hubConnection.stateValueAtom); + const refresh = useAtomCommand(hubConnection.refresh, { reportFailure: false }); + const disconnect = useAtomCommand(hubConnection.disconnect, { reportFailure: false }); + + if (hub.connection === null) return null; + + const connection = hub.connection; + + return ( + + + + void refresh(undefined)} + /> + + Alert.alert( + `Disconnect ${connection.label}?`, + "Remove this phone’s server access. Direct connections and running work stay in place.", + [ + { text: "Cancel", style: "cancel" }, + { + text: "Disconnect", + style: "destructive", + onPress: () => void disconnect(undefined), + }, + ], + ) + } + /> + + {hub.error ? ( + + {hub.error} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/connection/pairing.test.ts b/apps/mobile/src/features/connection/pairing.test.ts index 1939276847949fbe439db9ee3b6442d655f90fce..c29d6a82187a3bf224ddf82bb7d385a114298818 100644 --- a/apps/mobile/src/features/connection/pairing.test.ts +++ b/apps/mobile/src/features/connection/pairing.test.ts @@ -51,6 +51,15 @@ describe("extractPairingUrlFromQrPayload", () => { }); describe("parsePairingUrl", () => { + it.each(["", "pair", "pair/"])("preserves a proxied environment address before %s", (route) => { + expect( + parsePairingUrl(`https://hub.example/prefix/environments/one/${route}?old=1#token=code`), + ).toEqual({ + host: "https://hub.example/prefix/environments/one", + code: "code", + }); + }); + it("reads hosted pairing links into backend host fields", () => { expect( parsePairingUrl( diff --git a/apps/mobile/src/features/connection/pairing.ts b/apps/mobile/src/features/connection/pairing.ts index ee9e4b9ec8f3cc63b67ef384250be1709271da0a..4a31f6efc4c99589303e9684f28a4b6a273bc0fa 100644 --- a/apps/mobile/src/features/connection/pairing.ts +++ b/apps/mobile/src/features/connection/pairing.ts @@ -63,7 +63,7 @@ export function parsePairingUrl(url: string): { host: string; code: string } { parsed.hash = ""; parsed.search = ""; - parsed.pathname = "/"; + parsed.pathname = parsed.pathname.replace(/\/pair\/?$/, "/"); return { host: parsed.toString().replace(/\/$/, ""), code }; } catch { return { host: trimmed, code: "" }; diff --git a/apps/mobile/src/features/settings/SettingsAuthRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsAuthRouteScreen.tsx index e6e23fd78be92daaa733a825d10bc6281c179f5d..17c555579256e0fb7d1a36243556db185017c073 100644 --- a/apps/mobile/src/features/settings/SettingsAuthRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsAuthRouteScreen.tsx @@ -3,9 +3,12 @@ import { AuthView, type UserProfileCustomPage, UserProfileView } from "@clerk/ex import { StackActions, useNavigation } from "@react-navigation/native"; import { useCallback, useEffect, useLayoutEffect, useRef } from "react"; import { View } from "react-native"; +import { useAtomSet } from "@effect/atom-react"; import { hasCloudPublicConfig } from "../cloud/publicConfig"; import { T3ConnectProfilePage } from "../cloud/T3ConnectProfilePage"; +import { useCloudAuthAvailable } from "../cloud/cloudAvailability"; +import { updateMobilePreferencesAtom } from "../../state/preferences"; // Custom rows in Clerk's native profile. Mirrors the web UserButton pages. const USER_PROFILE_CUSTOM_PAGES = [ @@ -19,14 +22,18 @@ const USER_PROFILE_CUSTOM_PAGES = [ export function SettingsAuthRouteScreen() { const navigation = useNavigation(); + const cloudAvailable = useCloudAuthAvailable(); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); useLayoutEffect(() => { if (!hasCloudPublicConfig()) { navigation.dispatch(StackActions.replace("SettingsContent")); + } else if (!cloudAvailable) { + savePreferences({ connectionService: "t3-connect" }); } - }, [navigation]); + }, [cloudAvailable, navigation, savePreferences]); - return hasCloudPublicConfig() ? : null; + return cloudAvailable ? : null; } function ConfiguredSettingsAuthRouteScreen() { diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx index 003178a03aca465bde2e6f8971cd922772a3d54f..b436a2af1c1dd4e730fbbab02664965c2a9456d3 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx @@ -13,6 +13,7 @@ import { AndroidHeaderIconButton } from "../../components/AndroidScreenHeader"; import { CloudEnvironmentRows } from "../connection/CloudEnvironmentRows"; import { LocalEnvironmentList } from "../connection/LocalEnvironmentList"; import { GitHubRoutingSettings } from "../connection/GitHubRoutingSettings"; +import { SelfHostedServerSection } from "../connection/SelfHostedServerSection"; import { splitEnvironmentSections } from "../connection/environmentSections"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { useRemoteConnections } from "../../state/use-remote-environment-registry"; @@ -162,6 +163,7 @@ export function SettingsEnvironmentsRouteScreen() { ) : undefined } > + - Notifications require T3 Connect in this app build. + Notifications aren’t available through self-hosted servers. T3 Connect is required. diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 1dc1aae57ec6f4d6488f35a3d38548bd6bcd1766..2620a51684629d07588ae58d6ea290b2bbe0aff3 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -5,7 +5,7 @@ import { Platform, View } from "react-native"; import { deriveProjectGroupLabel } from "@t3tools/client-runtime/state/project-grouping"; import { useSafeAreaInsets } from "react-native-safe-area-context"; -import { hasCloudPublicConfig } from "../cloud/publicConfig"; +import { useCloudAuthAvailable } from "../cloud/cloudAvailability"; import { useAdaptiveWorkspaceLayout } from "../layout/AdaptiveWorkspaceLayout"; import { NativeHeaderToolbar, NativeStackScreenOptions } from "../../native/StackHeader"; import { useSavedRemoteConnections } from "../../state/use-remote-environment-registry"; @@ -21,7 +21,7 @@ import { useSettingsEnvironmentFilter } from "./settings-environment-filter"; export function SettingsRouteScreen() { const navigation = useNavigation(); const { layout } = useAdaptiveWorkspaceLayout(); - const content = hasCloudPublicConfig() ? ( + const content = useCloudAuthAvailable() ? ( ) : ( diff --git a/apps/mobile/src/features/threads/ThreadArrangementSheet.tsx b/apps/mobile/src/features/threads/ThreadArrangementSheet.tsx index 97dc6cf85eca70599ba0415a9434bc8f01c7b181..c2bd5a286208360e73c305a6638decfe753da1a0 100644 --- a/apps/mobile/src/features/threads/ThreadArrangementSheet.tsx +++ b/apps/mobile/src/features/threads/ThreadArrangementSheet.tsx @@ -13,7 +13,7 @@ import { useSafeAreaInsets } from "react-native-safe-area-context"; import { AppText as Text } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; import { scopedThreadKey } from "../../lib/scopedEntities"; -import { environmentServerConfigsAtom } from "../../state/server"; +import { environmentCapabilitiesAtom } from "../../state/server"; import { environmentThreadShells } from "../../state/threads"; import { pendingThreadOrderAtom, @@ -151,7 +151,7 @@ function DragHandle(props: { export function ThreadArrangementSheet(props: { onClose: () => void }) { const insets = useSafeAreaInsets(); const threads = useAtomValue(environmentThreadShells.navigationThreadShellsAtom); - const configs = useAtomValue(environmentServerConfigsAtom); + const capabilitiesByEnvironment = useAtomValue(environmentCapabilitiesAtom); const queuedThreadKeys = useAtomValue(queuedThreadKeysAtom); const pendingOrder = useAtomValue(pendingThreadOrderAtom); const dropBusy = useAtomValue(threadDropBusyAtom); @@ -180,13 +180,13 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { queuedThreadKeys, pendingOrder, settlementEnvironmentIds: new Set( - [...configs].flatMap(([id, config]) => - config.environment.capabilities.threadSettlement ? [id] : [], + [...capabilitiesByEnvironment].flatMap(([id, supported]) => + supported.threadSettlement ? [id] : [], ), ), snoozeEnvironmentIds: new Set( - [...configs].flatMap(([id, config]) => - config.environment.capabilities.threadSnooze ? [id] : [], + [...capabilitiesByEnvironment].flatMap(([id, supported]) => + supported.threadSnooze ? [id] : [], ), ), }; @@ -205,7 +205,14 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { snoozed: parked.filter((thread) => effectiveSnoozed(thread, { now })), settled: parked.filter((thread) => !effectiveSnoozed(thread, { now })), }; - }, [threads, configs, now, queuedThreadKeys, pendingOrder, workingShelfEnabled]); + }, [ + threads, + capabilitiesByEnvironment, + now, + queuedThreadKeys, + pendingOrder, + workingShelfEnabled, + ]); const planners = useMemo(() => { const planner = (section: "pinned" | "active") => createThreadMovePlanner({ @@ -215,11 +222,11 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { // A time-ordered inbox has no slots, so Active takes no drops while // the Working beta is on. The saved arrangement stays untouched. reorderableEnvironmentIds: new Set( - [...configs].flatMap(([id, config]) => + [...capabilitiesByEnvironment].flatMap(([id, supported]) => ( section === "pinned" - ? config.environment.capabilities.threadPinReorder - : !workingShelfEnabled && config.environment.capabilities.threadActiveReorder + ? supported.threadPinReorder + : !workingShelfEnabled && supported.threadActiveReorder ) ? [id] : [], @@ -227,7 +234,7 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { ), }); return { pinned: planner("pinned"), active: planner("active") }; - }, [sections, threads, configs, workingShelfEnabled]); + }, [sections, threads, capabilitiesByEnvironment, workingShelfEnabled]); const rows = useMemo(() => { const result: Row[] = []; let offset = 0; @@ -298,7 +305,7 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { if (target.section === "settled") { if ( current.sourceSection !== "settled" && - configs.get(current.thread.environmentId)?.environment.capabilities.threadSettlement + capabilitiesByEnvironment.get(current.thread.environmentId)?.threadSettlement ) destination = { section: "settled", targetId: null, placement: "before" }; } else if (latest.current.planners[target.section](keyOf(current.thread), candidate) !== null) @@ -421,8 +428,7 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { item.section === "pinned" || item.section === "active" ? planners[item.section] : null; - const capabilities = - thread && configs.get(thread.environmentId)?.environment.capabilities; + const capabilities = thread && capabilitiesByEnvironment.get(thread.environmentId); const sectionActions = thread ? (["pinned", "active", "settled"] as const).flatMap<{ name: "pinned" | "active" | "settled"; @@ -480,10 +486,10 @@ export function ThreadArrangementSheet(props: { onClose: () => void }) { dropBusy || pendingOrder !== null || !( - configs.get(thread.environmentId)?.environment.capabilities - .threadPinReorder || - configs.get(thread.environmentId)?.environment.capabilities - .threadActiveReorder + capabilitiesByEnvironment.get(thread.environmentId) + ?.threadPinReorder || + capabilitiesByEnvironment.get(thread.environmentId) + ?.threadActiveReorder ) } sectionActions={sectionActions} diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 9c1cbe34bb2cd5f9ecdf1fbb0551b31230caedb4..cf8ca49623cb2682ad49d632f2c423fc946dac14 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -34,7 +34,7 @@ import { ProviderIcon, ProviderInstanceIcon } from "../../components/ProviderIco import { cn } from "../../lib/cn"; import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; -import { useEnvironmentScope } from "../../state/session"; +import { useEnvironmentScope, useThreadOrganizationPermission } from "../../state/session"; import type { PendingNewTask } from "../../state/use-pending-new-tasks"; import { useThreadPr } from "../../state/use-thread-pr"; import { useSwipeRowDormant } from "../home/swipe-row-activation"; @@ -645,6 +645,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const handleDelete = useCallback(() => onDeleteThread(thread), [onDeleteThread, thread]); const handleRename = useCallback(() => onRenameThread(thread), [onRenameThread, thread]); const canOperateThread = useEnvironmentScope(thread.environmentId, AuthOrchestrationOperateScope); + const canOrganizeThread = useThreadOrganizationPermission(thread.environmentId); const handleRegenerateTitle = useCallback( () => onRegenerateThreadTitle(thread), [onRegenerateThreadTitle, thread], @@ -962,7 +963,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { : null, [handleMenuAction, snoozePresetActions, swipeActions.secondary, thread.title], ); - const swipeAccessibilityHint = !canOperateThread + const swipeAccessibilityHint = !canOrganizeThread ? "Opens the thread" : secondaryAction === null ? `Opens the thread. Swipe left to ${primaryAction.label.toLowerCase()}.` @@ -1301,7 +1302,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { ); - if (!canOperateThread) return rowContent(() => {}); + if (!canOrganizeThread) return rowContent(() => {}); return ( @@ -1330,27 +1331,36 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { > {(close) => ( + !canOperateThread && + ["new-thread-on-branch", "rename", "regenerate-title", "archive", "delete"].includes( + action.id ?? "", + ) + ? { ...action, attributes: { ...action.attributes, disabled: true } } + : action, + )} onPressAction={handleMenuAction} shouldOpenOnLongPress > diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index c587869c10c5567d40aa3cfc303f7d351fe8d518..0e85f59c2b10a25e24cc573e9d936bbb66d344b6 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -55,6 +55,7 @@ const isThreadListFilters = Schema.is(ThreadListFilters); const isPullRequestListPreferences = Schema.is(PullRequestListPreferences); export interface Preferences { + readonly connectionService?: "t3-connect" | "self-hosted"; readonly uiSettingsSync?: UiSettingsSyncCache | null; readonly pullRequestHideTestFiles?: boolean; readonly pullRequestMergeMethod?: PullRequestMergeMethod; @@ -139,6 +140,7 @@ export class MobilePreferencesStore extends Context.Service< function sanitizePreferences(parsed: Preferences): Preferences { const preferences: { + connectionService?: "t3-connect" | "self-hosted"; uiSettingsSync?: UiSettingsSyncCache | null; pullRequestHideTestFiles?: boolean; pullRequestMergeMethod?: PullRequestMergeMethod; @@ -169,6 +171,10 @@ function sanitizePreferences(parsed: Preferences): Preferences { pullRequestListFilters?: PullRequestListPreferences; } = {}; + if (parsed.connectionService === "t3-connect" || parsed.connectionService === "self-hosted") { + preferences.connectionService = parsed.connectionService; + } + if (typeof parsed.pullRequestHideTestFiles === "boolean") { preferences.pullRequestHideTestFiles = parsed.pullRequestHideTestFiles; } diff --git a/apps/mobile/src/state/server.ts b/apps/mobile/src/state/server.ts index 820646c6d8980876cee35b7a641c52062808dbea..3e076ef6b283966a17992872c68fd5ceae39c723 100644 --- a/apps/mobile/src/state/server.ts +++ b/apps/mobile/src/state/server.ts @@ -1,3 +1,5 @@ +import { createHubEnvironmentCapabilitiesAtom } from "@t3tools/client-runtime/state/hub"; +import { hubConnection } from "../connection/hub"; import { createServerEnvironmentAtoms } from "@t3tools/client-runtime/state/server"; import { createEnvironmentServerConfigsAtom } from "@t3tools/client-runtime/state/shell"; @@ -15,7 +17,12 @@ export const environmentServerConfigsAtom = createEnvironmentServerConfigsAtom({ catalogValueAtom: environmentCatalog.catalogValueAtom, serverConfigValueAtom: serverEnvironment.configValueAtom, }); +export const environmentCapabilitiesAtom = createHubEnvironmentCapabilitiesAtom( + environmentServerConfigsAtom, + hubConnection, +); export const threadListEnvironmentsAtom = createThreadListEnvironmentsAtom( environmentServerConfigsAtom, + environmentCapabilitiesAtom, ); diff --git a/apps/mobile/src/state/session.test.ts b/apps/mobile/src/state/session.test.ts index aeb8f684e0f7bfe84b4d8da5c3ba0f6c0e8b721c..ba947189b2e620b926e026b0a037e8d0dbd16f6e 100644 --- a/apps/mobile/src/state/session.test.ts +++ b/apps/mobile/src/state/session.test.ts @@ -18,6 +18,13 @@ vi.mock("@effect/atom-react", async () => { }, }; }); +vi.mock("../connection/hub", () => ({ + hubConnection: { + organizationPermissionAtom: Atom.family((_id: EnvironmentId | null) => + Atom.make(false).pipe(Atom.keepAlive), + ), + }, +})); vi.mock("../connection/runtime", () => ({ connectionAtomRuntime: {} })); vi.mock("@t3tools/client-runtime/state/session", () => ({ createEnvironmentSessionAtoms: () => ({ diff --git a/apps/mobile/src/state/session.ts b/apps/mobile/src/state/session.ts index 8d3fa134869c58ef6e75df73c2dd18b59d92ce09..a0686fc4ef3877b9aff908cbeb1bfa2c9292a812 100644 --- a/apps/mobile/src/state/session.ts +++ b/apps/mobile/src/state/session.ts @@ -1,6 +1,8 @@ +import { hubConnection } from "../connection/hub"; import { useAtomValue } from "@effect/atom-react"; import { createEnvironmentSessionAtoms } from "@t3tools/client-runtime/state/session"; import { + AuthOrchestrationOperateScope, type AuthEnvironmentScope, type AuthSessionState, type EnvironmentId, @@ -78,3 +80,16 @@ export function usePreparedConnection(environmentId: EnvironmentId | null) { : environmentSession.preparedConnectionValueAtom(environmentId), ); } + +export function useThreadOrganizationPermission(environmentId: EnvironmentId | null): boolean { + const source = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const central = useAtomValue(hubConnection.organizationPermissionAtom(environmentId)); + return source || central; +} + +export function readThreadOrganizationPermission(environmentId: EnvironmentId): boolean { + return ( + readEnvironmentScope(environmentId, AuthOrchestrationOperateScope) || + appAtomRegistry.get(hubConnection.organizationPermissionAtom(environmentId)) + ); +} diff --git a/apps/mobile/src/state/thread-list-environments.test.ts b/apps/mobile/src/state/thread-list-environments.test.ts index 8c99457a96466d049b6ea1e4a22a2e25bf444871..afe4a4c5f79ff52f1fc06772aa97f25e2353bfc9 100644 --- a/apps/mobile/src/state/thread-list-environments.test.ts +++ b/apps/mobile/src/state/thread-list-environments.test.ts @@ -38,7 +38,11 @@ function harness() { [OTHER_ID, config], ]), ); - const list = createThreadListEnvironmentsAtom(configs); + const capabilities = Atom.make( + (get) => + new Map([...get(configs)].map(([id, config]) => [id, config.environment.capabilities])), + ); + const list = createThreadListEnvironmentsAtom(configs, capabilities); let sourceNotifications = 0; let listNotifications = 0; registry.subscribe(configs, () => sourceNotifications++); @@ -220,3 +224,38 @@ describe("thread list environment projection", () => { } }); }); + +it("shows organization affordances on a new client without a source server configuration", () => { + const registry = AtomRegistry.make(); + try { + const configs = Atom.make>(new Map()); + const capabilities = Atom.make( + new Map([ + [ + ID, + { + ...config.environment.capabilities, + threadSettlement: true, + threadSnooze: true, + threadPinning: true, + threadPinReorder: true, + threadActiveReorder: true, + threadAutoSettleOptOut: true, + }, + ], + ]), + ); + const list = createThreadListEnvironmentsAtom(configs, capabilities); + const result = registry.get(list); + expect([...result.settlementEnvironmentIds]).toEqual([ID]); + expect([...result.snoozeEnvironmentIds]).toEqual([ID]); + expect([...result.pinningEnvironmentIds]).toEqual([ID]); + expect([...result.pinReorderEnvironmentIds]).toEqual([ID]); + expect([...result.activeReorderEnvironmentIds]).toEqual([ID]); + expect(result.titleRegenerationEnvironmentIds.size).toBe(0); + expect(result.providersByEnvironmentId.get(ID)).toEqual([]); + expect(registry.get(configs).size).toBe(0); + } finally { + registry.dispose(); + } +}); diff --git a/apps/mobile/src/state/thread-list-environments.ts b/apps/mobile/src/state/thread-list-environments.ts index 9df8a12ac502369108795a8375828b13ee089f8b..99f5bd6d2298a102ee5ba11ac0c877b3fe167684 100644 --- a/apps/mobile/src/state/thread-list-environments.ts +++ b/apps/mobile/src/state/thread-list-environments.ts @@ -1,6 +1,7 @@ import { resolveEnvironmentMachineKind, type EnvironmentId, + type ExecutionEnvironmentCapabilities, type EnvironmentMachineKind, type ServerConfig, type ServerProvider, @@ -22,19 +23,23 @@ const capabilityKeys = [ "threadTitleRegeneration", ] as const; -function selectEnvironment(config: ServerConfig) { +const EMPTY_PROVIDERS: ThreadListProvider[] = []; + +function selectEnvironment( + config: ServerConfig | undefined, + capabilities: ExecutionEnvironmentCapabilities, +) { return { - providers: config.providers.map( - ({ instanceId, driver, displayName, accentColor, iconUrl }) => ({ + providers: + config?.providers.map(({ instanceId, driver, displayName, accentColor, iconUrl }) => ({ instanceId, driver, displayName, accentColor, iconUrl, - }), - ), - machineKind: resolveEnvironmentMachineKind(config), - capabilities: config.environment.capabilities, + })) ?? EMPTY_PROVIDERS, + machineKind: resolveEnvironmentMachineKind(config ?? null), + capabilities, }; } @@ -96,15 +101,17 @@ function collectEnvironments(environments: ReadonlyMap>, + capabilitiesAtom: Atom.Atom>, ) { let previous = new Map(); let result = collectEnvironments(previous); return Atom.make((get) => { + const capabilities = get(capabilitiesAtom); const configs = get(configsAtom); const next = new Map(); - let changed = configs.size !== previous.size; - for (const [id, config] of configs) { - const selected = selectEnvironment(config); + let changed = capabilities.size !== previous.size; + for (const [id, supported] of capabilities) { + const selected = selectEnvironment(configs.get(id), supported); const prior = previous.get(id); if (prior && sameProviders(prior.providers, selected.providers)) { selected.providers = prior.providers; diff --git a/apps/mobile/src/state/thread-order.test.ts b/apps/mobile/src/state/thread-order.test.ts index d39120a5000595dd8a9f03a08c1467b07d7901ba..4b9c7bb35ecce495ff2a981829a6bd68f45eac7a 100644 --- a/apps/mobile/src/state/thread-order.test.ts +++ b/apps/mobile/src/state/thread-order.test.ts @@ -23,7 +23,12 @@ vi.mock("./threads", async () => { }); vi.mock("./server", async () => { const { Atom } = await import("effect/reactivity"); - return { environmentServerConfigsAtom: Atom.make(new Map()).pipe(Atom.keepAlive) }; + return { + threadListEnvironmentsAtom: Atom.make({ + settlementEnvironmentIds: new Set(), + snoozeEnvironmentIds: new Set(), + }).pipe(Atom.keepAlive), + }; }); vi.mock("./use-thread-outbox", async () => { const { Atom } = await import("effect/reactivity"); diff --git a/apps/mobile/src/state/thread-order.ts b/apps/mobile/src/state/thread-order.ts index 7667d47d159d283d00eda2f565d8f3e054d9d61b..2b3a7b3d7dd73fcfab5814ab26757781e01d07cb 100644 --- a/apps/mobile/src/state/thread-order.ts +++ b/apps/mobile/src/state/thread-order.ts @@ -8,7 +8,7 @@ import { } from "../features/threads/threadOrder"; import { getThreadListV2OrderedSection } from "../features/threads/threadListV2"; import { appAtomRegistry } from "./atom-registry"; -import { environmentServerConfigsAtom } from "./server"; +import { threadListEnvironmentsAtom } from "./server"; import { environmentThreadShells } from "./threads"; import { queuedThreadKeysAtom } from "./use-thread-outbox"; @@ -51,22 +51,14 @@ export function beginPendingThreadOrder(pending: PendingThreadOrder) { if (refreshPendingOrder !== refresh) return; const current = appAtomRegistry.get(pendingThreadOrderAtom); if (current === null) return; - const configs = appAtomRegistry.get(environmentServerConfigsAtom); + const capabilities = appAtomRegistry.get(threadListEnvironmentsAtom); const ordered = getThreadListV2OrderedSection({ threads: appAtomRegistry.get(environmentThreadShells.threadShellsAtom), section: current.section, now: new Date().toISOString(), queuedThreadKeys: appAtomRegistry.get(queuedThreadKeysAtom), - settlementEnvironmentIds: new Set( - [...configs].flatMap(([id, config]) => - config.environment.capabilities.threadSettlement === true ? [id] : [], - ), - ), - snoozeEnvironmentIds: new Set( - [...configs].flatMap(([id, config]) => - config.environment.capabilities.threadSnooze === true ? [id] : [], - ), - ), + settlementEnvironmentIds: capabilities.settlementEnvironmentIds, + snoozeEnvironmentIds: capabilities.snoozeEnvironmentIds, }); const next = reconcilePendingThreadOrder(current, ordered); if (next === null) cancel(); @@ -76,7 +68,7 @@ export function beginPendingThreadOrder(pending: PendingThreadOrder) { appAtomRegistry.set(pendingThreadOrderAtom, pending); unsubscribers.push( appAtomRegistry.subscribe(environmentThreadShells.threadShellsAtom, refresh), - appAtomRegistry.subscribe(environmentServerConfigsAtom, refresh), + appAtomRegistry.subscribe(threadListEnvironmentsAtom, refresh), appAtomRegistry.subscribe(queuedThreadKeysAtom, refresh), ); return { diff --git a/apps/server/src/archive/ArchiveAssetResponse.ts b/apps/server/src/archive/ArchiveAssetResponse.ts new file mode 100644 index 0000000000000000000000000000000000000000..0f751798723d718e9a97c0f73f9e28a454d928c7 --- /dev/null +++ b/apps/server/src/archive/ArchiveAssetResponse.ts @@ -0,0 +1,74 @@ +import { ArchiveError } from "@t3tools/contracts"; +import * as ArchiveClient from "./ArchiveClient.ts"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import { HttpServerResponse } from "effect/http"; +import { + injectMcpAppCookieBootstrap, + injectMcpAppThemeBootstrap, + MCP_APP_MAX_HTML_BYTES, +} from "@t3tools/shared/mcpApp"; + +export const remoteResponse = Effect.fn("ArchiveAssetResponse.remoteResponse")(function* ( + asset: { + readonly location: string; + readonly hash: string; + readonly mimeType: string; + readonly download: boolean; + readonly fileName: string; + readonly renderIntent?: "mcp-app"; + }, + range: string | undefined, +) { + const client = yield* ArchiveClient.ArchiveClient; + const result = yield* client.open(asset.location, asset.hash, range); + if (result.kind === "file") return result; + const response = result.response; + const headers: Record = { + "content-type": asset.mimeType, + "cache-control": "private, no-store", + "x-content-type-options": "nosniff", + }; + for (const name of ["content-length", "content-range", "accept-ranges"]) { + const value = response.headers[name]; + if (value !== undefined) headers[name] = value; + } + if (asset.download) + headers["content-disposition"] = + `attachment; filename*=UTF-8''${encodeURIComponent(asset.fileName)}`; + if (asset.mimeType === "image/svg+xml") + headers["content-security-policy"] = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; + if ( + asset.renderIntent === "mcp-app" && + !asset.download && + asset.mimeType.startsWith("text/html") + ) { + const parts: Array = []; + let length = 0; + yield* response.stream.pipe( + Stream.runForEach((bytes) => + Effect.gen(function* () { + length += bytes.length; + if (length > MCP_APP_MAX_HTML_BYTES) + return yield* new ArchiveError({ reason: "integrity" }); + parts.push(bytes); + }), + ), + ); + const bytes = new Uint8Array(length); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.length; + } + const body = injectMcpAppCookieBootstrap( + injectMcpAppThemeBootstrap(new TextDecoder().decode(bytes)), + ); + delete headers["content-length"]; + return { kind: "response" as const, response: HttpServerResponse.text(body, { headers }) }; + } + return { + kind: "response" as const, + response: HttpServerResponse.stream(response.stream, { status: response.status, headers }), + }; +}); diff --git a/apps/server/src/archive/ArchiveAssets.ts b/apps/server/src/archive/ArchiveAssets.ts new file mode 100644 index 0000000000000000000000000000000000000000..f3c3d2885e54ee599dd9c0c922c8217bde880812 --- /dev/null +++ b/apps/server/src/archive/ArchiveAssets.ts @@ -0,0 +1,128 @@ +import { + ArchiveError, + ArchiveObjectHash, + AuthSessionId, + EnvironmentId, + ThreadId, + type AssetCreateUrlResult, + type AssetResource, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as ArchiveCatalog from "./ArchiveCatalog.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import { + base64UrlDecodeUtf8, + base64UrlEncode, + signPayload, + timingSafeEqualBase64Url, +} from "../auth/utils.ts"; + +const Claims = Schema.Struct({ + version: Schema.Literal(1), + environmentId: EnvironmentId, + threadId: ThreadId, + manifestHash: ArchiveObjectHash, + hash: ArchiveObjectHash, + sessionId: AuthSessionId, + expiresAt: Schema.Number, + fileName: Schema.String, + mimeType: Schema.String, + download: Schema.Boolean, + renderIntent: Schema.optionalKey(Schema.Literal("mcp-app")), +}); +const encodeClaims = Schema.encodeSync(Schema.fromJsonString(Claims)); +const decodeClaims = Schema.decodeUnknownOption(Schema.fromJsonString(Claims)); +const storageError = (cause: unknown) => + new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); + +export class ArchiveAssets extends Context.Service< + ArchiveAssets, + { + readonly createUrl: ( + environmentId: EnvironmentId, + resource: AssetResource, + sessionId: AuthSessionId, + ) => Effect.Effect; + readonly resolve: (token: string) => Effect.Effect< + Option.Option<{ + readonly path: string; + readonly fileName: string; + readonly mimeType: string; + readonly download: boolean; + readonly renderIntent?: "mcp-app"; + }>, + ArchiveError + >; + } +>()("t3/archive/ArchiveAssets") {} + +export const layer = Layer.effect( + ArchiveAssets, + Effect.gen(function* () { + const catalog = yield* ArchiveCatalog.ArchiveCatalog; + const sessions = yield* SessionStore.SessionStore; + const secret = yield* (yield* ServerSecretStore.ServerSecretStore) + .getOrCreateRandom("archive-asset-signing-key", 32) + .pipe(Effect.mapError(storageError)); + return ArchiveAssets.of({ + createUrl: (environmentId, resource, sessionId) => + Effect.gen(function* () { + const asset = yield* catalog.asset(environmentId, resource); + const expiresAt = (yield* Clock.currentTimeMillis) + 5 * 60_000; + const payload = base64UrlEncode( + encodeClaims({ version: 1, environmentId, ...asset, sessionId, expiresAt }), + ); + const token = `${payload}.${signPayload(payload, secret)}`; + return { + relativeUrl: `/api/archive/assets/${token}/${encodeURIComponent(asset.fileName)}`, + expiresAt, + }; + }), + resolve: (token) => + Effect.gen(function* () { + const separator = token.indexOf("."); + if (separator <= 0) return Option.none(); + const payload = token.slice(0, separator); + if (!timingSafeEqualBase64Url(token.slice(separator + 1), signPayload(payload, secret))) + return Option.none(); + const decoded = yield* Effect.try(() => decodeClaims(base64UrlDecodeUtf8(payload))).pipe( + Effect.orElseSucceed(() => Option.none()), + ); + if (Option.isNone(decoded)) return Option.none(); + const claims = decoded.value; + if (claims.expiresAt <= (yield* Clock.currentTimeMillis)) return Option.none(); + const active = yield* sessions.verifyActive(claims.sessionId).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (!active) return Option.none(); + const file = yield* catalog + .openAsset(claims.environmentId, claims.threadId, claims.manifestHash, claims.hash) + .pipe( + Effect.asSome, + Effect.catchTags({ + ArchiveError: (error) => + error.reason === "unavailable" || error.reason === "changed" + ? Effect.succeedNone + : Effect.fail(error), + }), + ); + if (Option.isNone(file)) return Option.none(); + return Option.some({ + path: file.value, + fileName: claims.fileName, + mimeType: claims.mimeType, + download: claims.download, + ...(claims.renderIntent ? { renderIntent: claims.renderIntent } : {}), + }); + }), + }); + }), +); diff --git a/apps/server/src/archive/ArchiveCatalog.ts b/apps/server/src/archive/ArchiveCatalog.ts new file mode 100644 index 0000000000000000000000000000000000000000..4fd8599e4c55eb47fe5f9f79df14d0539ddda8ee --- /dev/null +++ b/apps/server/src/archive/ArchiveCatalog.ts @@ -0,0 +1,356 @@ +import { + ArchiveCatalogEntry, + type AssetResource, + ArchiveError, + ArchivePublication, + EnvironmentId, + ThreadId, + type OrchestrationV2ThreadDetailSnapshot, +} from "@t3tools/contracts"; +// Streaming verification keeps archive media memory bounded. +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as NodePath from "@effect/platform-node/NodePath"; +import { isWindowsAbsolutePath } from "@t3tools/shared/path"; +import { hostPreviewMimeTypeFromExtension } from "@t3tools/shared/filePreview"; +import { parseAttachmentIdFromRelativePath } from "../attachmentStore.ts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ArchiveObjects from "./ArchiveObjects.ts"; +import { Bundle, Row } from "./ArchiveFormat.ts"; + +const decodeBundle = Schema.decodeEffect(Schema.fromJsonString(Bundle)); +const decodeRows = Schema.decodeEffect(Schema.fromJsonString(Schema.Array(Row))); +const decodeEntry = Schema.decodeEffect( + Schema.fromJsonString(Schema.toCodecJson(ArchiveCatalogEntry)), +); +const encodeEntry = Schema.encodeEffect( + Schema.fromJsonString(Schema.toCodecJson(ArchiveCatalogEntry)), +); +const isArchiveError = Schema.is(ArchiveError); +const storageError = (cause: unknown) => + isArchiveError(cause) + ? cause + : new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); + +export class ArchiveCatalog extends Context.Service< + ArchiveCatalog, + { + readonly publish: ( + input: ArchivePublication, + ) => Effect.Effect; + readonly list: Effect.Effect< + { readonly entries: ReadonlyArray }, + ArchiveError + >; + readonly read: ( + environmentId: EnvironmentId, + threadId: ThreadId, + ) => Effect.Effect; + readonly asset: ( + environmentId: EnvironmentId, + resource: AssetResource, + ) => Effect.Effect< + { + readonly threadId: ThreadId; + readonly manifestHash: string; + readonly hash: string; + readonly fileName: string; + readonly mimeType: string; + readonly download: boolean; + readonly renderIntent?: "mcp-app"; + }, + ArchiveError + >; + readonly openAsset: ( + environmentId: EnvironmentId, + threadId: ThreadId, + manifestHash: string, + hash: string, + ) => Effect.Effect; + readonly withdraw: ( + environmentId: EnvironmentId, + threadId: ThreadId, + manifestHash: string, + ) => Effect.Effect; + } +>()("t3/archive/ArchiveCatalog") {} + +export const layer = Layer.effect( + ArchiveCatalog, + Effect.gen(function* () { + const objects = yield* ArchiveObjects.ArchiveObjects; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const windowsPath = yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)); + const lock = yield* Semaphore.make(1); + const directory = Effect.gen(function* () { + if (Option.isNone(objects.root)) return yield* new ArchiveError({ reason: "unavailable" }); + // Object lookup verifies the configured archive storage before index access. + yield* objects.directory; + const directory = path.join(objects.root.value, "catalog"); + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + return directory; + }); + const entryPath = (directory: string, environmentId: EnvironmentId, threadId: ThreadId) => + path.join( + directory, + NodeCrypto.createHash("sha256") + .update(JSON.stringify([environmentId, threadId])) + .digest("hex") + ".json", + ); + const verifiedFile = Effect.fn("ArchiveCatalog.verifiedFile")(function* (hash: string) { + const file = yield* objects.file(hash); + const digest = NodeCrypto.createHash("sha256"); + yield* fs.stream(file).pipe( + Stream.runForEach((bytes) => + Effect.sync(() => { + digest.update(bytes); + }), + ), + ); + if (digest.digest("hex") !== hash) return yield* new ArchiveError({ reason: "integrity" }); + return file; + }); + const readManifest = (hash: string) => + verifiedFile(hash).pipe(Effect.flatMap(fs.readFileString), Effect.flatMap(decodeBundle)); + const load = Effect.fn("ArchiveCatalog.load")(function* (hash: string) { + const manifest = yield* readManifest(hash); + const hashes = new Set([ + manifest.events, + ...manifest.segments.map((segment) => segment.hash), + ...manifest.context.map((segment) => segment.hash), + ...manifest.assets.map((asset) => asset.hash), + ]); + for (const hash of hashes) yield* verifiedFile(hash); + return manifest; + }); + const get = Effect.fn("ArchiveCatalog.get")(function* ( + environmentId: EnvironmentId, + threadId: ThreadId, + ) { + const file = entryPath(yield* directory, environmentId, threadId); + if (!(yield* fs.exists(file))) return yield* new ArchiveError({ reason: "unavailable" }); + const entry = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeEntry)); + if (entry.environmentId !== environmentId || entry.thread.id !== threadId) + return yield* new ArchiveError({ reason: "integrity" }); + return entry; + }); + const materialize = (manifest: typeof Bundle.Type) => + Effect.gen(function* () { + const segments = yield* Effect.forEach( + [ + ...manifest.context, + ...manifest.segments.map((segment) => ({ + ...segment, + table: `orchestration_v2_projection_${segment.table}`, + })), + ], + (segment) => + Effect.gen(function* () { + const rows = yield* fs + .readFileString(yield* objects.file(segment.hash)) + .pipe(Effect.flatMap(decodeRows)); + return { table: segment.table, rows }; + }), + ); + const events = yield* fs + .readFileString(yield* objects.file(manifest.events)) + .pipe(Effect.flatMap(decodeRows)); + return yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const segment of segments) + for (const row of segment.rows) + yield* sql`INSERT OR REPLACE INTO ${sql(segment.table)} ${sql.insert(row)}`; + for (const row of events) + yield* sql`INSERT OR REPLACE INTO orchestration_events ${sql.insert(row)}`; + return yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadSnapshot( + manifest.threadId, + ); + }).pipe( + Effect.provide( + Layer.fresh( + ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)), + ), + ), + ); + }); + const list = Effect.gen(function* () { + if (Option.isNone(objects.root)) return { entries: [] }; + const dir = yield* directory; + const names = (yield* fs.readDirectory(dir)).filter((name) => + /^[a-f0-9]{64}\.json$/.test(name), + ); + const entries = yield* Effect.forEach( + names, + (name) => fs.readFileString(path.join(dir, name)).pipe(Effect.flatMap(decodeEntry)), + { concurrency: 4 }, + ); + return { + entries: entries.sort( + (a, b) => b.thread.updatedAt.epochMilliseconds - a.thread.updatedAt.epochMilliseconds, + ), + }; + }).pipe(Effect.mapError(storageError)); + return ArchiveCatalog.of({ + publish: (input) => + lock + .withPermit( + Effect.gen(function* () { + yield* objects.verifyDurableStore; + const manifest = yield* load(input.manifestHash); + const snapshot = yield* materialize(manifest); + const thread = ProjectionStore.threadShellFromProjection(snapshot.projection); + if ( + thread.id !== manifest.threadId || + !thread.archivedAt || + thread.deletedAt || + thread.projectId !== input.project.id + ) + return yield* new ArchiveError({ reason: "integrity" }); + const entry = ArchiveCatalogEntry.make({ + environmentId: input.environmentId, + manifestHash: input.manifestHash, + throughSequence: manifest.throughSequence, + thread, + project: input.project, + }); + const dir = yield* directory; + const file = entryPath(dir, input.environmentId, thread.id); + if (yield* fs.exists(file)) { + const previous = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeEntry)); + if ( + previous.throughSequence > entry.throughSequence || + (previous.throughSequence === entry.throughSequence && + previous.manifestHash !== entry.manifestHash) + ) + return yield* new ArchiveError({ reason: "changed" }); + } + yield* Effect.scoped( + Effect.gen(function* () { + const temporary = yield* fs.makeTempFileScoped({ + directory: dir, + prefix: ".publish-", + }); + yield* fs.writeFileString(temporary, yield* encodeEntry(entry), { mode: 0o600 }); + yield* fs + .open(temporary, { flag: "r+" }) + .pipe(Effect.flatMap((file) => file.sync)); + yield* fs.rename(temporary, file); + if (!(yield* HostProcess.isWindows)) + yield* fs.open(dir).pipe(Effect.flatMap((file) => file.sync)); + }), + ); + return entry; + }), + ) + .pipe(Effect.mapError(storageError)), + list, + read: (environmentId, threadId) => + Effect.gen(function* () { + const entry = yield* get(environmentId, threadId); + const manifest = yield* load(entry.manifestHash); + if (manifest.threadId !== threadId || manifest.throughSequence !== entry.throughSequence) + return yield* new ArchiveError({ reason: "integrity" }); + return yield* materialize(manifest); + }).pipe(Effect.mapError(storageError)), + asset: (environmentId, resource) => + Effect.gen(function* () { + if ( + resource._tag !== "attachment" && + resource._tag !== "media-file" && + resource._tag !== "workspace-file" + ) + return yield* new ArchiveError({ reason: "unavailable" }); + const entries = (yield* list).entries.filter( + (entry) => + entry.environmentId === environmentId && + (resource._tag === "attachment" || entry.thread.id === resource.threadId), + ); + for (const entry of entries) { + const manifest = yield* readManifest(entry.manifestHash); + const sourcePath = isWindowsAbsolutePath(entry.project.workspaceRoot) + ? windowsPath + : path; + const requested = + resource._tag === "attachment" + ? undefined + : sourcePath.resolve( + entry.thread.worktreePath ?? entry.project.workspaceRoot, + resource.path, + ); + const found = manifest.assets.find((asset) => + resource._tag === "attachment" + ? parseAttachmentIdFromRelativePath(sourcePath.basename(asset.path)) === + resource.attachmentId + : sourcePath.normalize(asset.path) === requested, + ); + if (!found) continue; + yield* verifiedFile(found.hash); + const fileName = + resource._tag === "attachment" + ? (resource.fileName ?? sourcePath.basename(found.path)) + : sourcePath.basename(found.path); + const mimeType = + hostPreviewMimeTypeFromExtension(sourcePath.extname(found.path)) ?? + hostPreviewMimeTypeFromExtension(sourcePath.extname(fileName)) ?? + "application/octet-stream"; + return { + threadId: entry.thread.id, + manifestHash: entry.manifestHash, + hash: found.hash, + fileName, + mimeType, + download: + resource._tag === "attachment" && + resource.disposition !== "inline" && + !/^(image|video)\//u.test(mimeType), + ...(resource._tag === "attachment" && resource.renderIntent === "mcp-app" + ? { renderIntent: "mcp-app" as const } + : {}), + }; + } + return yield* new ArchiveError({ reason: "unavailable" }); + }).pipe(Effect.mapError(storageError)), + openAsset: (environmentId, threadId, manifestHash, hash) => + Effect.gen(function* () { + const entry = yield* get(environmentId, threadId); + if (entry.manifestHash !== manifestHash) + return yield* new ArchiveError({ reason: "changed" }); + const manifest = yield* readManifest(manifestHash); + if (!manifest.assets.some((asset) => asset.hash === hash)) + return yield* new ArchiveError({ reason: "unavailable" }); + return yield* objects.file(hash); + }).pipe(Effect.mapError(storageError)), + withdraw: (environmentId, threadId, manifestHash) => + Effect.scoped( + lock + .withPermit( + Effect.gen(function* () { + const dir = yield* directory; + const file = entryPath(dir, environmentId, threadId); + if (!(yield* fs.exists(file))) return; + const current = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeEntry)); + if (current.manifestHash !== manifestHash) return; + yield* fs.remove(entryPath(dir, environmentId, threadId), { force: true }); + if (!(yield* HostProcess.isWindows)) + yield* fs.open(dir).pipe(Effect.flatMap((file) => file.sync)); + }), + ) + .pipe(Effect.mapError(storageError)), + ), + }); + }), +); diff --git a/apps/server/src/archive/ArchiveClient.ts b/apps/server/src/archive/ArchiveClient.ts new file mode 100644 index 0000000000000000000000000000000000000000..a3b9ad1e732eecf3cacdcfed97e40a0aa19ff83e --- /dev/null +++ b/apps/server/src/archive/ArchiveClient.ts @@ -0,0 +1,216 @@ +import { ArchiveError, ArchiveObjectHash, ArchiveObjectReceipt } from "@t3tools/contracts"; +// Incremental hashes keep media uploads bounded; Effect Crypto hashes whole buffers. +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import type * as Scope from "effect/Scope"; +import { HttpClient, HttpClientRequest, HttpClientResponse, FetchHttpClient } from "effect/http"; +import * as ArchiveConnection from "./ArchiveConnection.ts"; +import * as ArchiveCatalog from "./ArchiveCatalog.ts"; +import { + ArchiveCatalogEntry, + type ArchivePublication, + type EnvironmentId, + type ThreadId, +} from "@t3tools/contracts"; +import * as ArchiveObjects from "./ArchiveObjects.ts"; + +const decodeHash = Schema.decodeEffect(ArchiveObjectHash); +const storageError = (cause: unknown) => + new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); +const connectionError = (cause: unknown) => + new ArchiveError({ reason: "unavailable", cause: Redacted.make(cause) }); + +export class ArchiveClient extends Context.Service< + ArchiveClient, + { + readonly enabled: Effect.Effect; + readonly publish: ( + input: ArchivePublication, + ) => Effect.Effect; + readonly withdraw: ( + environmentId: EnvironmentId, + threadId: ThreadId, + manifestHash: string, + ) => Effect.Effect; + readonly location: Effect.Effect; + readonly upload: ( + file: string, + ) => Effect.Effect; + readonly open: ( + origin: string, + hash: string, + range?: string, + ) => Effect.Effect< + | { kind: "file"; path: string } + | { kind: "response"; response: HttpClientResponse.HttpClientResponse }, + ArchiveError, + Scope.Scope + >; + readonly download: ( + location: string, + hash: string, + destination: string, + ) => Effect.Effect; + } +>()("t3/archive/ArchiveClient") {} + +export const layer = Layer.effect( + ArchiveClient, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const http = yield* HttpClient.HttpClient; + const connection = yield* ArchiveConnection.ArchiveConnection; + const catalog = yield* ArchiveCatalog.ArchiveCatalog; + const local = yield* ArchiveObjects.ArchiveObjects; + const enabled = connection.enabled; + const target = connection.target; + const location = target.pipe( + Effect.flatMap((connection) => + Option.isSome(connection) + ? Effect.succeed(connection.value.url) + : Option.isSome(local.root) + ? Effect.succeed(`file:${local.root.value}`) + : Effect.fail(new ArchiveError({ reason: "unavailable" })), + ), + ); + const upload = Effect.fn("ArchiveClient.upload")(function* (file: string) { + const digest = NodeCrypto.createHash("sha256"); + yield* fs.stream(file).pipe( + Stream.runForEach((bytes) => + Effect.sync(() => { + digest.update(bytes); + }), + ), + Effect.mapError(storageError), + ); + const hash = digest.digest("hex"); + const connection = yield* target; + if (Option.isNone(connection)) + return yield* local.put(hash, fs.stream(file)).pipe(Effect.mapError(storageError)); + const receipt = yield* HttpClientRequest.put( + `${connection.value.url}/api/archive/objects/${hash}`, + ).pipe( + HttpClientRequest.setHeader("authorization", connection.value.authorization), + HttpClientRequest.bodyStream(fs.stream(file), { contentType: "application/octet-stream" }), + http.execute, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(ArchiveObjectReceipt)), + Effect.mapError(connectionError), + Effect.timeout("5 minutes"), + Effect.mapError(connectionError), + ); + const stat = yield* fs.stat(file).pipe(Effect.mapError(storageError)); + if (receipt.hash !== hash || BigInt(receipt.size) !== stat.size) + return yield* new ArchiveError({ reason: "integrity" }); + return receipt; + }); + const open = Effect.fn("ArchiveClient.open")(function* ( + origin: string, + input: string, + range?: string, + ) { + const hash = yield* decodeHash(input).pipe(Effect.mapError(storageError)); + if ((yield* location) !== origin) return yield* new ArchiveError({ reason: "unavailable" }); + const connection = yield* target; + if (Option.isNone(connection)) + return { kind: "file" as const, path: yield* local.file(hash) }; + const response = yield* HttpClient.withScope(http) + .execute( + HttpClientRequest.get(`${connection.value.url}/api/archive/objects/${hash}`).pipe( + HttpClientRequest.setHeaders({ + authorization: connection.value.authorization, + ...(range ? { range } : {}), + "accept-encoding": "identity", + }), + ), + ) + .pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.timeout("10 seconds"), + Effect.mapError(connectionError), + ); + if (![200, 206, 416].includes(response.status)) + return yield* new ArchiveError({ reason: "unavailable" }); + return { kind: "response" as const, response }; + }); + const download = Effect.fn("ArchiveClient.download")(function* ( + origin: string, + input: string, + destination: string, + ) { + const hash = yield* decodeHash(input).pipe(Effect.mapError(storageError)); + if ((yield* location) !== origin) return yield* new ArchiveError({ reason: "unavailable" }); + const connection = yield* target; + const body: Stream.Stream = Option.isNone(connection) + ? fs.stream(yield* local.file(hash)) + : (yield* HttpClientRequest.get(`${connection.value.url}/api/archive/objects/${hash}`).pipe( + HttpClientRequest.setHeader("authorization", connection.value.authorization), + http.execute, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.timeout("10 seconds"), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.mapError(connectionError), + )).stream; + const digest = NodeCrypto.createHash("sha256"); + yield* body.pipe( + Stream.timeoutOrElse({ + duration: "10 seconds", + orElse: () => Stream.fail(new ArchiveError({ reason: "unavailable" })), + }), + Stream.tap((bytes) => + Effect.sync(() => { + digest.update(bytes); + }), + ), + Stream.run(fs.sink(destination, { flag: "wx", mode: 0o600 })), + Effect.mapError(connectionError), + ); + if (digest.digest("hex") !== hash) return yield* new ArchiveError({ reason: "integrity" }); + }); + const publish = (input: ArchivePublication) => + Effect.gen(function* () { + const connection = yield* target; + if (Option.isNone(connection)) return yield* catalog.publish(input); + return yield* HttpClientRequest.post(`${connection.value.url}/api/archive/catalog`).pipe( + HttpClientRequest.setHeader("authorization", connection.value.authorization), + HttpClientRequest.bodyJson(input), + Effect.flatMap(http.execute), + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap( + HttpClientResponse.schemaBodyJson(Schema.toCodecJson(ArchiveCatalogEntry)), + ), + Effect.timeout("5 minutes"), + Effect.mapError(connectionError), + ); + }); + const withdraw = (environmentId: EnvironmentId, threadId: ThreadId, manifestHash: string) => + Effect.gen(function* () { + const connection = yield* target; + if (Option.isNone(connection)) + return yield* catalog.withdraw(environmentId, threadId, manifestHash); + yield* HttpClientRequest.delete( + `${connection.value.url}/api/archive/catalog/${encodeURIComponent(environmentId)}/${encodeURIComponent(threadId)}`, + ).pipe( + HttpClientRequest.setHeader("authorization", connection.value.authorization), + HttpClientRequest.bodyJson({ manifestHash }), + Effect.flatMap(http.execute), + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.mapError(connectionError), + Effect.timeout("10 seconds"), + Effect.mapError(connectionError), + ); + }); + return ArchiveClient.of({ enabled, location, upload, download, open, publish, withdraw }); + }), +); diff --git a/apps/server/src/archive/ArchiveConnection.ts b/apps/server/src/archive/ArchiveConnection.ts new file mode 100644 index 0000000000000000000000000000000000000000..3a3b519c032e2fd5dd95825ca1b5e513ade81480 --- /dev/null +++ b/apps/server/src/archive/ArchiveConnection.ts @@ -0,0 +1,44 @@ +import { ArchiveError } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as SharedServer from "../environment/SharedServer.ts"; + +export class ArchiveConnection extends Context.Service< + ArchiveConnection, + { + readonly enabled: Effect.Effect; + readonly target: Effect.Effect< + Option.Option<{ readonly url: string; readonly authorization: string }>, + ArchiveError + >; + } +>()("t3/archive/ArchiveConnection") {} + +export const layerLegacy = Layer.effect( + ArchiveConnection, + Effect.gen(function* () { + const shared = yield* SharedServer.SharedServer; + const enabled = yield* Config.Boolean("T3CODE_ARCHIVE_OFFLOAD").pipe( + Config.withDefault(false), + Effect.orDie, + ); + return ArchiveConnection.of({ + enabled: Effect.succeed(enabled), + target: shared.connection.pipe( + Effect.map( + Option.map((connection) => ({ + url: connection.url, + authorization: `Bearer ${connection.token}`, + })), + ), + Effect.mapError( + (cause) => new ArchiveError({ reason: "unavailable", cause: Redacted.make(cause) }), + ), + ), + }); + }), +); diff --git a/apps/server/src/archive/ArchiveFormat.ts b/apps/server/src/archive/ArchiveFormat.ts new file mode 100644 index 0000000000000000000000000000000000000000..8daab973206f52230c6edbe92d4df900d84bc87c --- /dev/null +++ b/apps/server/src/archive/ArchiveFormat.ts @@ -0,0 +1,40 @@ +import { ArchiveObjectHash, ThreadId } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +export const payloadTables = ["messages", "turn_items", "subagents"] as const; +export const Table = Schema.Literals(payloadTables); +const Cell = Schema.Union([Schema.String, Schema.Number, Schema.Null]); +export const Row = Schema.Record(Schema.String, Cell); +export const Segment = Schema.Struct({ table: Table, rows: Schema.Array(Row) }); +export const Metadata = Schema.Array(Segment); +export const canonicalTables = [ + "threads", + "runs", + "run_attempts", + "nodes", + "subagents", + "provider_sessions", + "provider_session_bindings", + "provider_threads", + "provider_turns", + "runtime_requests", + "messages", + "plans", + "turn_items", + "checkpoint_scopes", + "checkpoints", + "context_handoffs", + "context_transfers", +] as const; +export const CanonicalTable = Schema.Literals( + canonicalTables.map((name) => `orchestration_v2_projection_${name}`), +); +export const Bundle = Schema.Struct({ + version: Schema.Literal(1), + threadId: ThreadId, + throughSequence: Schema.Number, + segments: Schema.Array(Schema.Struct({ table: Table, hash: ArchiveObjectHash })), + events: ArchiveObjectHash, + context: Schema.Array(Schema.Struct({ table: CanonicalTable, hash: ArchiveObjectHash })), + assets: Schema.Array(Schema.Struct({ path: Schema.String, hash: ArchiveObjectHash })), +}); diff --git a/apps/server/src/archive/ArchiveHttp.ts b/apps/server/src/archive/ArchiveHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..809ea4b4970ba0b2b100e0b83161ad9f77545896 --- /dev/null +++ b/apps/server/src/archive/ArchiveHttp.ts @@ -0,0 +1,157 @@ +import { + ArchiveCatalogEntry, + ArchiveError, + ArchiveAssetCreateUrlInput, + AssetCreateUrlResult, + ArchiveObjectHash, + ArchiveCatalogResult, + ArchivePublication, + OrchestrationV2ThreadDetailSnapshot, + AuthArchiveReadScope, + AuthArchiveWriteScope, + EnvironmentId, + HostedAuthError, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; +import * as HostedAuth from "../auth/HostedAuth.ts"; +import { archiveRouteErrors, authenticateRawRouteWithScope, assetFileResponse } from "../http.ts"; +import * as ArchiveAssets from "./ArchiveAssets.ts"; +import * as Option from "effect/Option"; +import * as ArchiveCatalog from "./ArchiveCatalog.ts"; + +const json = (schema: Schema.Codec) => { + const encode = Schema.encodeEffect(Schema.toCodecJson(schema)); + return (value: A) => + encode(value).pipe( + Effect.flatMap((body) => + HttpServerResponse.json(body, { headers: { "cache-control": "no-store" } }), + ), + ); +}; +const decodeParams = Schema.decodeUnknownEffect( + Schema.Struct({ environmentId: EnvironmentId, threadId: ThreadId }), +); +const params = HttpRouter.params.pipe(Effect.flatMap(decodeParams)); +const catalogJson = json(ArchiveCatalogResult); +const entryJson = json(ArchiveCatalogEntry); +const assetJson = json(AssetCreateUrlResult); +const snapshotJson = json(OrchestrationV2ThreadDetailSnapshot); +const isHostedAuthError = Schema.is(HostedAuthError); +const withdrawal = HttpServerRequest.schemaBodyJson( + Schema.Struct({ manifestHash: ArchiveObjectHash }), +); +const errors = (effect: Effect.Effect) => + archiveRouteErrors(effect).pipe( + Effect.catch((error) => { + if (isHostedAuthError(error)) + return Effect.succeed(HttpServerResponse.empty({ status: 403 })); + if (Schema.isSchemaError(error)) + return Effect.succeed(HttpServerResponse.empty({ status: 400 })); + return Effect.fail(error); + }), + ); + +export const layer = Layer.mergeAll( + HttpRouter.add( + "POST", + "/api/archive/assets", + Effect.gen(function* () { + const session = yield* authenticateRawRouteWithScope(AuthArchiveReadScope); + const input = yield* HttpServerRequest.schemaBodyJson(ArchiveAssetCreateUrlInput); + return yield* (yield* ArchiveAssets.ArchiveAssets) + .createUrl(input.environmentId, input.resource, session.sessionId) + .pipe(Effect.flatMap(assetJson)); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/archive/assets/*", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) return HttpServerResponse.empty({ status: 404 }); + const suffix = url.value.pathname.slice("/api/archive/assets/".length); + const separator = suffix.indexOf("/"); + if (separator <= 0) return HttpServerResponse.empty({ status: 404 }); + const asset = yield* (yield* ArchiveAssets.ArchiveAssets).resolve(suffix.slice(0, separator)); + if (Option.isNone(asset)) return HttpServerResponse.empty({ status: 404 }); + return yield* assetFileResponse( + asset.value, + request.headers.range, + request.headers["if-range"], + request.method === "HEAD" ? "HEAD" : "GET", + ).pipe( + Effect.mapError( + (cause) => new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }), + ), + Effect.map((response) => + HttpServerResponse.setHeaders(response, { + "cache-control": "private, no-store", + "content-type": + asset.value.mimeType === "text/html" + ? "text/html; charset=utf-8" + : asset.value.mimeType, + ...(asset.value.mimeType === "image/svg+xml" + ? { + "content-security-policy": + "default-src 'none'; style-src 'unsafe-inline'; sandbox", + } + : {}), + }), + ), + ); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/archive/catalog", + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthArchiveReadScope); + return yield* (yield* ArchiveCatalog.ArchiveCatalog).list.pipe(Effect.flatMap(catalogJson)); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/archive/catalog", + Effect.gen(function* () { + const session = yield* authenticateRawRouteWithScope(AuthArchiveWriteScope); + const input = yield* HttpServerRequest.schemaBodyJson(ArchivePublication); + yield* (yield* HostedAuth.HostedAuth).authorizeEnvironment(session, input.environmentId); + return yield* (yield* ArchiveCatalog.ArchiveCatalog) + .publish(input) + .pipe(Effect.flatMap(entryJson)); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/archive/catalog/:environmentId/:threadId", + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthArchiveReadScope); + const input = yield* params; + return yield* (yield* ArchiveCatalog.ArchiveCatalog) + .read(input.environmentId, input.threadId) + .pipe(Effect.flatMap(snapshotJson)); + }).pipe(errors), + ), + HttpRouter.add( + "DELETE", + "/api/archive/catalog/:environmentId/:threadId", + Effect.gen(function* () { + const session = yield* authenticateRawRouteWithScope(AuthArchiveWriteScope); + const input = yield* params; + yield* (yield* HostedAuth.HostedAuth).authorizeEnvironment(session, input.environmentId); + const { manifestHash } = yield* withdrawal; + yield* (yield* ArchiveCatalog.ArchiveCatalog).withdraw( + input.environmentId, + input.threadId, + manifestHash, + ); + return HttpServerResponse.empty({ status: 204 }); + }).pipe(errors), + ), +); diff --git a/apps/server/src/archive/ArchiveObjects.ts b/apps/server/src/archive/ArchiveObjects.ts new file mode 100644 index 0000000000000000000000000000000000000000..e3365e14df7ceb96c57182a3e8d7cc7df822361e --- /dev/null +++ b/apps/server/src/archive/ArchiveObjects.ts @@ -0,0 +1,127 @@ +import { ArchiveError, ArchiveObjectHash, ArchiveObjectReceipt } from "@t3tools/contracts"; +// Incremental hashes keep media uploads bounded; Effect Crypto hashes whole buffers. +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +const decodeHash = Schema.decodeEffect(ArchiveObjectHash); +const isArchiveError = Schema.is(ArchiveError); +const storageError = (cause: unknown) => + new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); + +export class ArchiveObjects extends Context.Service< + ArchiveObjects, + { + readonly verifyDurableStore: Effect.Effect; + readonly directory: Effect.Effect; + readonly root: Option.Option; + readonly file: (hash: string) => Effect.Effect; + readonly put: ( + hash: string, + body: Stream.Stream, + ) => Effect.Effect; + } +>()("t3/archive/ArchiveObjects") {} + +export const layer = Layer.effect( + ArchiveObjects, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* Config.option(Config.String("T3CODE_ARCHIVE_DIR")).pipe(Effect.orDie); + const expectedStoreId = yield* Config.option(Config.String("T3CODE_ARCHIVE_STORE_ID")).pipe( + Effect.orDie, + ); + const directory = Effect.gen(function* () { + if (Option.isNone(root)) return yield* new ArchiveError({ reason: "unavailable" }); + if (Option.isSome(expectedStoreId)) { + const actual = yield* fs + .readFileString(path.join(root.value, ".knife-archive-store-id")) + .pipe(Effect.mapError(storageError)); + if (actual.trim() !== expectedStoreId.value) + return yield* new ArchiveError({ reason: "storage" }); + } + const stat = yield* fs.stat(root.value).pipe(Effect.mapError(storageError)); + if (stat.type !== "Directory") return yield* new ArchiveError({ reason: "storage" }); + const dir = path.join(root.value, "objects"); + yield* fs + .makeDirectory(dir, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError(storageError)); + return dir; + }); + const file = Effect.fn("ArchiveObjects.file")(function* (input: string) { + const hash = yield* decodeHash(input).pipe(Effect.mapError(storageError)); + const name = path.join(yield* directory, hash); + const stat = yield* fs.stat(name).pipe(Effect.mapError(storageError)); + if (stat.type !== "File") return yield* new ArchiveError({ reason: "storage" }); + return name; + }); + const put = (input: string, body: Stream.Stream) => + Effect.scoped( + Effect.gen(function* () { + const hash = yield* decodeHash(input).pipe(Effect.mapError(storageError)); + const dir = yield* directory; + const temporary = yield* fs + .makeTempDirectoryScoped({ directory: dir, prefix: ".upload-" }) + .pipe(Effect.mapError(storageError)); + const staged = path.join(temporary, "object"); + const digest = NodeCrypto.createHash("sha256"); + let size = 0; + yield* body.pipe( + Stream.tap((bytes) => + Effect.sync(() => { + digest.update(bytes); + size += bytes.length; + }), + ), + Stream.run(fs.sink(staged, { flag: "wx", mode: 0o600 })), + Effect.mapError((cause) => (isArchiveError(cause) ? cause : storageError(cause))), + ); + if (digest.digest("hex") !== hash) + return yield* new ArchiveError({ reason: "integrity" }); + const verified = NodeCrypto.createHash("sha256"); + yield* fs.stream(staged).pipe( + Stream.runForEach((bytes) => + Effect.sync(() => { + verified.update(bytes); + }), + ), + Effect.mapError(storageError), + ); + if (verified.digest("hex") !== hash) + return yield* new ArchiveError({ reason: "integrity" }); + yield* fs.open(staged, { flag: "r+" }).pipe( + Effect.flatMap((file) => file.sync), + Effect.mapError(storageError), + ); + yield* fs.rename(staged, path.join(dir, hash)).pipe(Effect.mapError(storageError)); + if (!(yield* HostProcess.isWindows)) + yield* fs.open(dir).pipe( + Effect.flatMap((file) => file.sync), + Effect.mapError(storageError), + ); + return { hash, size }; + }), + ); + return ArchiveObjects.of({ + root, + directory, + file, + put, + verifyDurableStore: Effect.gen(function* () { + if (Option.isNone(expectedStoreId)) return yield* new ArchiveError({ reason: "storage" }); + yield* directory; + }), + }); + }), +); diff --git a/apps/server/src/archive/ArchiveObjectsHttp.test.ts b/apps/server/src/archive/ArchiveObjectsHttp.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..9fa4db431ec1f6b8b5e40a5f8756bb4cca435c26 --- /dev/null +++ b/apps/server/src/archive/ArchiveObjectsHttp.test.ts @@ -0,0 +1,92 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + AuthArchiveReadScope, + AuthArchiveWriteScope, + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, +} from "@t3tools/contracts"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { HttpRouter } from "effect/http"; +import * as ServerConfig from "../config.ts"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import { layerArchiveUploadRoute, layerArchiveDownloadRoute } from "../http.ts"; +import * as ArchiveObjects from "./ArchiveObjects.ts"; + +it.live( + "requires archive scopes, acknowledges verified bytes, and serves authenticated byte ranges", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "knife-archive-http-" }); + const layers = Layer.mergeAll(ArchiveObjects.layer, EnvironmentAuth.layerRuntime).pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "knife-archive-http-state-" }), + ), + Layer.provide( + ConfigProvider.layer(ConfigProvider.fromUnknown({ T3CODE_ARCHIVE_DIR: root })), + ), + ); + yield* Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const read = yield* auth.issueSession({ scopes: [AuthArchiveReadScope] }); + const legacy = yield* auth.issueSession({ + scopes: [AuthOrchestrationReadScope, AuthOrchestrationOperateScope], + }); + const write = yield* auth.issueSession({ scopes: [AuthArchiveWriteScope] }); + const context = yield* Effect.context< + EnvironmentAuth.EnvironmentAuth | ArchiveObjects.ArchiveObjects | FileSystem.FileSystem + >(); + const app = yield* Effect.acquireRelease( + Effect.sync(() => + HttpRouter.toWebHandler( + Layer.merge(layerArchiveUploadRoute, layerArchiveDownloadRoute), + { disableLogger: true }, + ), + ), + (app) => Effect.promise(() => app.dispose()), + ); + const bytes = new TextEncoder().encode("0123456789"); + const hash = Buffer.from(yield* (yield* Crypto.Crypto).digest("SHA-256", bytes)).toString( + "hex", + ); + const request = (method: string, token?: string, range?: string, objectHash = hash) => + Effect.promise(() => + app.handler( + new Request(`http://127.0.0.1/api/archive/objects/${objectHash}`, { + method, + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(range ? { range } : {}), + }, + ...(method === "PUT" ? { body: bytes } : {}), + }), + context, + ), + ); + assert.strictEqual((yield* request("PUT")).status, 401); + assert.strictEqual((yield* request("PUT", read.token)).status, 403); + assert.strictEqual((yield* request("PUT", legacy.token)).status, 200); + assert.strictEqual((yield* request("GET", legacy.token)).status, 200); + const uploaded = yield* request("PUT", write.token); + assert.strictEqual(uploaded.status, 200); + assert.deepStrictEqual(yield* Effect.promise(() => uploaded.json()), { hash, size: 10 }); + assert.strictEqual( + (yield* request("PUT", write.token, undefined, "0".repeat(64))).status, + 400, + ); + assert.strictEqual((yield* request("GET", write.token)).status, 403); + const partial = yield* request("GET", read.token, "bytes=3-6"); + assert.strictEqual(partial.status, 206); + assert.strictEqual(partial.headers.get("content-range"), "bytes 3-6/10"); + assert.strictEqual(yield* Effect.promise(() => partial.text()), "3456"); + assert.strictEqual((yield* request("GET", read.token, "bytes=20-")).status, 416); + const full = yield* request("GET", read.token); + assert.strictEqual(yield* Effect.promise(() => full.text()), "0123456789"); + }).pipe(Effect.provide(layers)); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/archive/ArchiveStorage.ts b/apps/server/src/archive/ArchiveStorage.ts new file mode 100644 index 0000000000000000000000000000000000000000..67b1c9598f263133d8666d21f09db30db58944d3 --- /dev/null +++ b/apps/server/src/archive/ArchiveStorage.ts @@ -0,0 +1,164 @@ +import { ArchiveError, type ArchiveStatus } from "@t3tools/contracts"; +// Allocated filesystem blocks are unavailable through Effect FileSystem. +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Config from "effect/Config"; +import * as FileSystem from "effect/FileSystem"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +const storageError = (cause: unknown) => + new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); +const isMissingFile = Schema.is(Schema.Struct({ code: Schema.Literal("ENOENT") })); +const Home = Schema.Struct({ + homePath: Schema.optionalKey(Schema.String), + shadowHomePath: Schema.optionalKey(Schema.String), +}); +const decodeHome = Schema.decodeOption(Home); + +export class ArchiveStorage extends Context.Service< + ArchiveStorage, + { + readonly inspect: Effect.Effect; + readonly reclaim: Effect.Effect; + } +>()("t3/archive/ArchiveStorage") {} + +export const layer = Layer.effect( + ArchiveStorage, + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const settings = yield* ServerSettings.ServerSettingsService; + const home = yield* HostProcess.HomeDirectory; + const path = yield* Path.Path; + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const inheritedCodexHome = yield* Config.option(Config.String("CODEX_HOME")).pipe(Effect.orDie); + const inheritedClaudeHome = yield* Config.option(Config.String("CLAUDE_CONFIG_DIR")).pipe( + Effect.orDie, + ); + const measure = (roots: ReadonlyArray) => + Effect.tryPromise({ + try: async () => { + let bytes = 0; + let allocated = 0; + const visited = new Set(); + const pending = [...roots]; + while (pending.length) { + const file = pending.pop()!; + let stat; + try { + stat = await NodeFSP.lstat(file); + } catch (error) { + if (isMissingFile(error)) continue; + throw error; + } + if (stat.isSymbolicLink()) continue; + const identity = `${stat.dev}:${stat.ino}`; + if (visited.has(identity)) continue; + visited.add(identity); + if (stat.isDirectory()) + for (const name of await NodeFSP.readdir(file)) pending.push(path.join(file, name)); + else if (stat.isFile()) { + bytes += stat.size; + allocated += stat.blocks * 512; + } + } + return { bytes, allocated }; + }, + catch: storageError, + }); + const databaseFiles = Effect.gen(function* () { + const databases = yield* sql<{ name: string; file: string }>`PRAGMA database_list`; + const file = databases.find((database) => database.name === "main")?.file; + return file ? [file, `${file}-wal`, `${file}-shm`] : []; + }); + const inspect = Effect.gen(function* () { + const configured = yield* settings.getSettings; + const roots = [config.providerLogsDir]; + const homes = new Map>([ + [ + "codex", + new Set([ + path.join(home, ".codex"), + path.join(config.stateDir, "providers/codex/shared"), + ]), + ], + [ + "claudeAgent", + new Set([ + path.join(home, ".claude"), + path.join(config.stateDir, "providers/claudeAgent/shared"), + ]), + ], + ]); + if (inheritedCodexHome._tag === "Some") + homes.get("codex")!.add(path.resolve(inheritedCodexHome.value)); + if (inheritedClaudeHome._tag === "Some") + homes.get("claudeAgent")!.add(path.resolve(inheritedClaudeHome.value)); + for (const [id, instance] of Object.entries(configured.providerInstances)) { + if (instance.driver !== "codex" && instance.driver !== "claudeAgent") continue; + const decoded = decodeHome(instance.config ?? {}); + const values = + decoded._tag === "Some" ? [decoded.value.homePath, decoded.value.shadowHomePath] : []; + for (const variable of instance.environment ?? []) + if (variable.name === (instance.driver === "codex" ? "CODEX_HOME" : "CLAUDE_CONFIG_DIR")) + values.push(variable.value); + if (instance.driver === "codex" && id !== "codex") + values.push(path.join(config.stateDir, "providers", "codex", id, "shadow")); + for (const value of values) { + const raw = value?.trim(); + if (!raw) continue; + const resolved = raw.startsWith("~/") ? path.join(home, raw.slice(2)) : path.resolve(raw); + homes.get(instance.driver)!.add(resolved); + } + } + for (const root of homes.get("codex")!) { + for (const directory of ["sessions", "archived_sessions", "sqlite", "logs", "log"]) + roots.push(path.join(root, directory)); + if (!(yield* fs.exists(root))) continue; + for (const name of yield* fs.readDirectory(root)) + if ( + /^(?:(?:state|logs)_\d+\.sqlite(?:-wal|-shm)?|history\.jsonl|session_index\.jsonl)$/.test( + name, + ) + ) + roots.push(path.join(root, name)); + } + for (const root of homes.get("claudeAgent")!) roots.push(path.join(root, "projects")); + const database = yield* measure(yield* databaseFiles); + const attachments = yield* measure([config.attachmentsDir]); + const native = yield* measure(roots); + return { + allocatedDatabaseBytes: database.allocated, + attachmentsBytes: attachments.bytes, + nativeProviderLogBytes: native.bytes, + }; + }).pipe(Effect.mapError(storageError)); + const reclaim = Effect.gen(function* () { + const pending = yield* sql<{ + pending: number; + }>`SELECT pending FROM transcript_archive_reclamation WHERE singleton = 1`; + if (!pending[0]?.pending) return; + const files = yield* databaseFiles; + const before = yield* measure(files); + // VACUUM runs on the existing SQLite writer, after the verified offload transaction commits. + yield* sql`VACUUM`; + const checkpoint = yield* sql<{ busy: number }>`PRAGMA wal_checkpoint(TRUNCATE)`; + if (checkpoint[0]?.busy) return yield* new ArchiveError({ reason: "storage" }); + const after = yield* measure(files); + const now = yield* Clock.currentTimeMillis; + yield* sql`UPDATE transcript_archive_reclamation SET pending = 0, completed_at = ${now}, before_allocated_bytes = ${before.allocated}, after_allocated_bytes = ${after.allocated} WHERE singleton = 1`; + }).pipe(Effect.mapError(storageError)); + return ArchiveStorage.of({ inspect, reclaim }); + }), +); diff --git a/apps/server/src/archive/ArchivedProjectionStore.ts b/apps/server/src/archive/ArchivedProjectionStore.ts new file mode 100644 index 0000000000000000000000000000000000000000..e540cea668628f8687b630222af40e09258b5cbc --- /dev/null +++ b/apps/server/src/archive/ArchivedProjectionStore.ts @@ -0,0 +1,80 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as TranscriptArchive from "./TranscriptArchive.ts"; +import type { ThreadId } from "@t3tools/contracts"; + +export const layer = Layer.effect( + ProjectionStore.ProjectionStoreV2, + Effect.gen(function* () { + const base = yield* ProjectionStore.ProjectionStoreV2; + const archive = yield* TranscriptArchive.TranscriptArchive; + const read = ( + id: ThreadId, + effect: Effect.Effect, + includeHistory = false, + ) => + archive + .read(id, effect, includeHistory) + .pipe( + Effect.mapError( + (cause) => new ProjectionStore.ProjectionStoreReadError({ threadId: id, cause }), + ), + ); + return ProjectionStore.ProjectionStoreV2.of({ + ...base, + apply: (event) => { + const restore = + event.type === "thread.unarchived" || + ["message.updated", "turn-item.updated", "subagent.updated"].includes(event.type); + return restore + ? archive.restore(event.threadId).pipe( + Effect.mapError( + (cause) => + new ProjectionStore.ProjectionStoreApplyEventError({ + eventType: event.type, + cause, + }), + ), + Effect.andThen(base.apply(event)), + ) + : base.apply(event); + }, + getThreadProjection: (id) => read(id, base.getThreadProjection(id), true), + getThreadRecords: (id, fields, filter) => + fields.some((field) => ["messages", "turnItems", "subagents"].includes(field)) + ? read(id, base.getThreadRecords(id, fields, filter)) + : base.getThreadRecords(id, fields, filter), + getThreadSnapshot: (id) => read(id, base.getThreadSnapshot(id), true), + getThreadSnapshotWindow: (id, options) => + read(id, base.getThreadSnapshotWindow(id, options), true), + getThreadHistoryPage: (id, cursor, through, conversation) => + read(id, base.getThreadHistoryPage(id, cursor, through, conversation), true), + getTimelinePage: (id, options) => read(id, base.getTimelinePage(id, options), true), + getTurnItem: (input) => read(input.threadId, base.getTurnItem(input)), + getThreadAttachmentIds: (id) => read(id, base.getThreadAttachmentIds(id)), + getTurnStartContext: (id, run) => read(id, base.getTurnStartContext(id, run)), + getTurnStartHistory: (id, runs) => read(id, base.getTurnStartHistory(id, runs)), + getRuntimeRecoveryProjection: (id) => read(id, base.getRuntimeRecoveryProjection(id)), + getPendingNativeUserInputs: (id, turn) => read(id, base.getPendingNativeUserInputs(id, turn)), + + getProviderControlContext: (id, target) => + read(id, base.getProviderControlContext(id, target)), + + getRuntimeResponseContext: (id, request) => + read(id, base.getRuntimeResponseContext(id, request)), + getRunMessage: (id, run) => read(id, base.getRunMessage(id, run)), + searchThread: (input) => read(input.threadId, base.searchThread(input), true), + searchThreadStream: (input) => + archive + .readStream(input.threadId, base.searchThreadStream(input)) + .pipe( + Stream.mapError( + (cause) => + new ProjectionStore.ProjectionStoreReadError({ threadId: input.threadId, cause }), + ), + ), + }); + }), +).pipe(Layer.provide(ProjectionStore.layer)); diff --git a/apps/server/src/archive/HostedArchiveConnection.ts b/apps/server/src/archive/HostedArchiveConnection.ts new file mode 100644 index 0000000000000000000000000000000000000000..840f8f8229a20dfac4db2df2b84db653b7f98767 --- /dev/null +++ b/apps/server/src/archive/HostedArchiveConnection.ts @@ -0,0 +1,33 @@ +import { ArchiveError } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as SourceConnection from "../environment/SourceConnection.ts"; +import * as ArchiveConnection from "./ArchiveConnection.ts"; + +export const layer = Layer.effect( + ArchiveConnection.ArchiveConnection, + Effect.gen(function* () { + const legacy = yield* ArchiveConnection.ArchiveConnection; + const source = yield* SourceConnection.SourceConnection; + return ArchiveConnection.ArchiveConnection.of({ + enabled: source.current.pipe( + Effect.flatMap((current) => + Option.isSome(current) ? Effect.succeed(true) : legacy.enabled, + ), + ), + target: Effect.gen(function* () { + const current = yield* source.current; + if (Option.isNone(current)) return yield* legacy.target; + const now = yield* DateTime.now; + if (current.value.expiresAt.epochMilliseconds <= now.epochMilliseconds) + return yield* new ArchiveError({ reason: "unavailable" }); + return Option.some({ + url: current.value.serverUrl, + authorization: current.value.authorization, + }); + }), + }); + }), +).pipe(Layer.provide(ArchiveConnection.layerLegacy)); diff --git a/apps/server/src/archive/TranscriptArchive.test.ts b/apps/server/src/archive/TranscriptArchive.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ae702b861661efff130403f27b18e65636bf26b6 --- /dev/null +++ b/apps/server/src/archive/TranscriptArchive.test.ts @@ -0,0 +1,1438 @@ +import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + AssetCreateUrlResult, + ArchiveCatalogResult, + AuthAdministrativeScopes, + AuthSessionId, + HubError, + ORCHESTRATION_PROTOCOL_VERSION, + OrchestrationV2ThreadDetailSnapshot, + AuthArchiveReadScope, + AuthArchiveWriteScope, + AuthOrchestrationReadScope, + ArchiveError, + EnvironmentId, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + TurnItemId, + ChatAttachmentId, + RunId, + NodeId, + ProviderDriverKind, + type OrchestrationV2AppThread, +} from "@t3tools/contracts"; +import * as TestClock from "effect/testing/TestClock"; +import * as ArchiveCatalog from "./ArchiveCatalog.ts"; +import * as Hub from "../environment/Hub.ts"; +import * as HubInbox from "../environment/HubInbox.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as HostedAuth from "../auth/HostedAuth.ts"; +import * as HostedAuthConfig from "../auth/HostedAuthConfig.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as ArchiveAssets from "./ArchiveAssets.ts"; +import * as ArchiveHttp from "./ArchiveHttp.ts"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as ArchiveConnection from "./ArchiveConnection.ts"; +import * as SharedServer from "../environment/SharedServer.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as ArchiveClient from "./ArchiveClient.ts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Crypto from "effect/Crypto"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpRouter } from "effect/http"; +import * as ServerConfig from "../config.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts"; +import * as EventStore from "../orchestration-v2/EventStore.ts"; +import * as EventSink from "../orchestration-v2/EventSink.ts"; +import * as ApplicationEvents from "../persistence/OrchestrationEventStore.ts"; +import * as ArchivedProjectionStore from "./ArchivedProjectionStore.ts"; +import * as ArchiveObjects from "./ArchiveObjects.ts"; +import * as ArchiveRuntime from "./runtimeLayer.ts"; +import * as TranscriptArchive from "./TranscriptArchive.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import { createDeterministicAttachmentId } from "../attachmentStore.ts"; +import { ASSET_ROUTE_PREFIX, issueAssetUrl, resolveAsset } from "../assets/AssetAccess.ts"; +import * as NativeAppIconResolver from "../assets/NativeAppIconResolver.ts"; +import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; + +const decodeAssetUrl = Schema.decodeUnknownEffect(AssetCreateUrlResult); +const decodeCatalog = Schema.decodeUnknownEffect(Schema.toCodecJson(ArchiveCatalogResult)); +const decodeSnapshot = Schema.decodeUnknownEffect( + Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot), +); +const threadId = ThreadId.make("archive-review"); +const fixture = (physical = false) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const temporary = yield* fs.makeTempDirectoryScoped({ prefix: "knife-archive-test-" }); + const root = path.join(temporary, "nas"); + yield* fs.makeDirectory(root); + yield* fs.writeFileString(path.join(root, ".knife-archive-store-id"), "archive-test-store"); + const config = yield* ServerConfig.ServerConfig.pipe( + Effect.provide(ServerConfig.layerTest(temporary, { prefix: "knife-archive-test-" })), + ); + const database = physical + ? SqlitePersistence.layerFromPath(path.join(temporary, "source.sqlite")) + : SqlitePersistence.layerMemory; + const runtime = ArchiveRuntime.layer.pipe( + Layer.provide(Layer.succeed(HostProcess.HomeDirectory, temporary)), + Layer.provide(ArchiveConnection.layerLegacy.pipe(Layer.provide(SharedServer.layer))), + Layer.provideMerge(ServerSettings.layerTest()), + Layer.provideMerge(ServerSecretStore.layer), + Layer.provideMerge(Layer.succeed(ServerConfig.ServerConfig, config)), + Layer.provideMerge(database), + Layer.provide( + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("Unexpected HTTP request")), + ), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + T3CODE_ARCHIVE_DIR: root, + T3CODE_ARCHIVE_STORE_ID: "archive-test-store", + T3CODE_ARCHIVE_OFFLOAD: "true", + }), + ), + ), + ); + const stores = Layer.mergeAll( + ArchivedProjectionStore.layer, + EventStore.layerFromOrchestrationEventStore.pipe(Layer.provideMerge(ApplicationEvents.layer)), + ).pipe(Layer.provideMerge(runtime)); + const layer = Layer.mergeAll( + stores, + EventSink.layer.pipe(Layer.provide(stores)), + ProjectionMaintenance.layer.pipe(Layer.provide(ProjectionStore.layer), Layer.provide(stores)), + ); + return { layer, root, path, temporary }; + }); + +const seed = Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const instanceId = ProviderInstanceId.make("codex"); + yield* (yield* SqlClient.SqlClient)`INSERT OR IGNORE INTO projection_projects(project_id, title, workspace_root, scripts_json, created_at, updated_at) VALUES(${ProjectId.make("archive-project")}, 'Archive workspace', '/archive/workspace', '[]', ${DateTime.formatIso(now)}, ${DateTime.formatIso(now)})`; + const thread: OrchestrationV2AppThread = { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId: ProjectId.make("archive-project"), + title: "Searchable archive title", + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "model" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }; + yield* sink.write({ + events: [ + { + id: EventId.make("archive-created"), + type: "thread.created", + threadId, + occurredAt: now, + payload: thread, + }, + { + id: EventId.make("archive-message"), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: MessageId.make("archive-message"), + threadId, + runId: null, + nodeId: null, + role: "assistant", + createdBy: "agent", + creationSource: "provider", + text: "private archived message body", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("archive-item"), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make("archive-item"), + threadId, + runId: null, + nodeId: null, + type: "assistant_message", + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + title: null, + startedAt: now, + completedAt: now, + streaming: false, + status: "completed", + ordinal: 1, + messageId: MessageId.make("archive-message"), + text: "private archived message body", + updatedAt: now, + }, + }, + { + id: EventId.make("archive-archived"), + type: "thread.archived", + threadId, + occurredAt: now, + payload: { ...thread, archivedAt: now }, + }, + ], + }); + return thread; +}); + +it.layer(NodeServices.layer)("transcript archive", (it) => { + it.effect("reads the durable catalog after the source database is closed", () => + Effect.gen(function* () { + const f = yield* fixture(); + const environmentId = yield* Effect.gen(function* () { + yield* seed; + yield* (yield* TranscriptArchive.TranscriptArchive).offload(threadId); + return yield* (yield* ServerEnvironment.ServerEnvironmentIdentity).getEnvironmentId; + }).pipe(Effect.provide(f.layer)); + const central = ArchiveCatalog.layer.pipe( + Layer.provideMerge(ArchiveObjects.layer), + Layer.provide( + ConfigProvider.layer(ConfigProvider.fromUnknown({ T3CODE_ARCHIVE_DIR: f.root })), + ), + ); + yield* Effect.gen(function* () { + const catalog = yield* ArchiveCatalog.ArchiveCatalog; + const entries = (yield* catalog.list).entries; + assert.strictEqual(entries.length, 1); + assert.strictEqual(entries[0]?.environmentId, environmentId); + const snapshot = yield* catalog.read(environmentId, threadId); + assert.strictEqual(snapshot.projection.messages[0]?.text, "private archived message body"); + assert.strictEqual( + snapshot.projection.turnItems.find((item) => item.type === "assistant_message")?.text, + "private archived message body", + ); + assert.isNotNull(snapshot.projection.thread.archivedAt); + const descriptor = { + environmentId, + label: "Sleeping laptop", + platform: { os: "linux" as const, arch: "x64" as const }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: false }, + }; + const hub = Hub.Hub.of({ + directory: () => + Effect.succeed({ + hubEnvironmentId: EnvironmentId.make("central-nas"), + environments: [ + { + environmentId, + label: descriptor.label, + httpBaseUrl: "https://archive.example.test/environments/source", + scopes: AuthAdministrativeScopes, + descriptor, + }, + ], + }), + authorize: () => Effect.succeed(AuthAdministrativeScopes), + connect: () => Effect.fail(new HubError({ reason: "upstream" })), + descriptor: () => Effect.succeed(descriptor), + manage: () => Effect.die("Unexpected management"), + }); + const discovered = yield* Effect.gen(function* () { + const inbox = yield* HubInbox.HubInbox; + const sql = yield* SqlClient.SqlClient; + assert.strictEqual((yield* sql`SELECT 1 FROM hub_inbox`).length, 0); + return yield* inbox.read({ + sessionId: AuthSessionId.make("fresh-desktop"), + subject: "owner", + method: "bearer-access-token", + scopes: AuthAdministrativeScopes, + }); + }).pipe( + Effect.provide( + HubInbox.layer.pipe( + Layer.provide(Layer.succeed(Hub.Hub, hub)), + Layer.provide(Layer.succeed(ArchiveCatalog.ArchiveCatalog, catalog)), + Layer.provide( + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("Source must not be contacted")), + ), + ), + Layer.provideMerge(Layer.fresh(SqlitePersistence.layerMemory)), + ), + ), + ); + assert.strictEqual(discovered.environments[0]?.environmentId, environmentId); + assert.strictEqual(discovered.environments[0]?.snapshot.archivedThreads[0]?.id, threadId); + assert.strictEqual( + discovered.environments[0]?.snapshot.projects[0]?.id, + ProjectId.make("archive-project"), + ); + assert.strictEqual( + discovered.environments[0]?.snapshot.projects[0]?.workspaceRoot, + "/archive/workspace", + ); + assert.strictEqual( + (yield* catalog.read(EnvironmentId.make("other-source"), threadId).pipe(Effect.flip)) + .reason, + "unavailable", + ); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString( + f.path.join(f.root, "objects", entries[0]!.manifestHash), + "corrupt", + ); + assert.strictEqual( + (yield* catalog.read(environmentId, threadId).pipe(Effect.flip)).reason, + "integrity", + ); + }).pipe(Effect.provide(central)); + }), + ); + + it.effect( + "keeps source payloads when publication cannot verify the complete object closure", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + yield* seed; + const client = yield* ArchiveClient.ArchiveClient; + const fs = yield* FileSystem.FileSystem; + const archive = yield* TranscriptArchive.TranscriptArchive.pipe( + Effect.provide(Layer.fresh(TranscriptArchive.layer)), + Effect.provideService(ArchiveClient.ArchiveClient, { + ...client, + publish: (input) => + Effect.gen(function* () { + const bundle = JSON.parse( + yield* fs.readFileString(f.path.join(f.root, "objects", input.manifestHash)), + ); + yield* fs.remove(f.path.join(f.root, "objects", bundle.events)); + return yield* client.publish(input); + }), + }), + ); + assert.strictEqual( + (yield* archive.offload(threadId).pipe(Effect.flip)).reason, + "storage", + ); + assert.strictEqual( + (yield* (yield* SqlClient.SqlClient)`SELECT 1 FROM transcript_archives`).length, + 0, + ); + assert.strictEqual( + (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection(threadId)) + .messages[0]?.text, + "private archived message body", + ); + assert.strictEqual( + (yield* (yield* ArchiveCatalog.ArchiveCatalog).list).entries.length, + 0, + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("persists failed uploads across restart and retries after their backoff", () => + Effect.gen(function* () { + const f = yield* fixture(true); + const expectedEnvironmentId = yield* Effect.gen(function* () { + yield* seed; + const client = yield* ArchiveClient.ArchiveClient; + const archive = yield* TranscriptArchive.TranscriptArchive.pipe( + Effect.provide(Layer.fresh(TranscriptArchive.layer)), + Effect.provideService(ArchiveClient.ArchiveClient, { + ...client, + upload: () => Effect.fail(new ArchiveError({ reason: "unavailable" })), + }), + ); + yield* archive.sweep; + const status = yield* archive.manage({ type: "archive.status" }); + assert.strictEqual(status.failures[0]?.attempts, 1); + assert.strictEqual(status.offloaded, 0); + return yield* (yield* ServerEnvironment.ServerEnvironmentIdentity).getEnvironmentId; + }).pipe(Effect.provide(f.layer)); + yield* Effect.gen(function* () { + const archive = yield* TranscriptArchive.TranscriptArchive; + assert.strictEqual( + yield* (yield* ServerEnvironment.ServerEnvironmentIdentity).getEnvironmentId, + expectedEnvironmentId, + ); + yield* archive.sweep; + assert.strictEqual( + (yield* archive.manage({ type: "archive.status" })).failures[0]?.attempts, + 1, + ); + yield* TestClock.adjust("1 minute"); + yield* archive.sweep; + assert.strictEqual((yield* archive.manage({ type: "archive.status" })).offloaded, 1); + }).pipe(Effect.provide(Layer.fresh(f.layer))); + }), + ); + + it.effect( + "reclaims allocated SQLite payload bytes and retains measured native provider logs", + () => + Effect.gen(function* () { + const f = yield* fixture(true); + const body = "large archived body ".repeat(100_000); + yield* Effect.gen(function* () { + yield* seed; + const archive = yield* TranscriptArchive.TranscriptArchive; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const message = (yield* projections.getThreadProjection(threadId)).messages[0]!; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("large-body"), + type: "message.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...message, text: body }, + }, + ], + }); + yield* fs.makeDirectory(config.providerLogsDir, { recursive: true }); + const logFile = f.path.join(config.providerLogsDir, "provider.jsonl"); + yield* fs.writeFileString(logFile, "native provider log stays local"); + yield* sql`PRAGMA wal_checkpoint(TRUNCATE)`; + const before = yield* archive.manage({ type: "archive.status" }); + yield* archive.sweep; + const after = yield* archive.manage({ type: "archive.status" }); + assert.isAbove( + before.storage.allocatedDatabaseBytes, + after.storage.allocatedDatabaseBytes, + ); + assert.isAbove( + after.lastReclamation!.beforeAllocatedBytes, + after.lastReclamation!.afterAllocatedBytes, + ); + assert.strictEqual( + after.storage.nativeProviderLogBytes, + Buffer.byteLength("native provider log stays local"), + ); + assert.strictEqual(yield* fs.readFileString(logFile), "native provider log stays local"); + assert.strictEqual( + (yield* projections.getThreadProjection(threadId)).messages[0]?.text, + body, + ); + yield* (yield* ProjectionMaintenance.ProjectionMaintenanceV2).rebuild; + assert.strictEqual( + (yield* projections.getThreadProjection(threadId)).messages[0]?.text, + body, + ); + }).pipe(Effect.provide(f.layer)); + yield* Effect.gen(function* () { + const archive = yield* TranscriptArchive.TranscriptArchive; + const projections = yield* ProjectionStore.ProjectionStoreV2; + assert.strictEqual( + (yield* projections.getThreadProjection(threadId)).messages[0]?.text, + body, + ); + yield* archive.restore(threadId); + const thread = (yield* projections.getThreadProjection(threadId)).thread; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("restart-unarchive"), + type: "thread.unarchived", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...thread, archivedAt: null }, + }, + ], + }); + const restored = yield* projections.getThreadProjection(threadId); + assert.isNull(restored.thread.archivedAt); + assert.strictEqual(restored.messages[0]?.text, body); + }).pipe(Effect.provide(Layer.fresh(f.layer))); + }), + ); + + it.effect( + "enforces narrow HTTP archive scopes and the authenticated device environment binding", + () => + Effect.gen(function* () { + const f = yield* fixture(); + const settings = { + publicUrl: "https://archive.example.test", + issuer: "https://identity.example.test", + clientId: "archive-test", + clientSecret: Redacted.make("test-only"), + ownerSub: "test-owner", + maxInteractiveAgeSeconds: 3600, + issuerRevalidationSeconds: 3600, + }; + const layers = ArchiveAssets.layer.pipe( + Layer.provideMerge(NodeHttpPlatform.layer), + Layer.provideMerge( + HostedAuth.layer.pipe( + Layer.provideMerge(EnvironmentAuth.layer), + Layer.provide( + Layer.succeed(HostedAuthConfig.HostedAuthConfig, Option.some(settings)), + ), + Layer.provide( + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("Unexpected identity request")), + ), + ), + Layer.provideMerge(f.layer), + ), + ), + ); + yield* Effect.gen(function* () { + yield* seed; + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const imageId = ChatAttachmentId.make( + createDeterministicAttachmentId(threadId, "nas-image")!, + ); + const movieId = ChatAttachmentId.make( + `${createDeterministicAttachmentId(threadId, "nas-movie")!}-mp4`, + ); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + const imageFile = f.path.join(config.attachmentsDir, `${imageId}.png`); + const movieFile = f.path.join(config.attachmentsDir, `${movieId}.mp4`); + yield* fs.writeFile(imageFile, new Uint8Array([137, 80, 78, 71, 1, 2, 3])); + yield* fs.writeFileString(movieFile, "0123456789"); + const message = (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection( + threadId, + )).messages[0]!; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("nas-attachments"), + type: "message.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { + ...message, + attachments: [ + { + type: "image", + id: imageId, + name: "picture.png", + mimeType: "image/png", + sizeBytes: 7, + }, + { + type: "file", + id: movieId, + name: "movie.mp4", + mimeType: "video/mp4", + sizeBytes: 10, + }, + ], + }, + }, + ], + }); + yield* (yield* TranscriptArchive.TranscriptArchive).sweep; + assert.isFalse(yield* fs.exists(imageFile)); + assert.isFalse(yield* fs.exists(movieFile)); + const environmentId = yield* (yield* ServerEnvironment.ServerEnvironmentIdentity) + .getEnvironmentId; + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const read = yield* auth.issueSession({ scopes: [AuthArchiveReadScope] }); + const write = yield* auth.issueSession({ scopes: [AuthArchiveWriteScope] }); + const legacy = yield* auth.issueSession({ scopes: [AuthOrchestrationReadScope] }); + const sql = yield* SqlClient.SqlClient; + const accountHash = Buffer.from( + yield* (yield* Crypto.Crypto).digest( + "SHA-256", + new TextEncoder().encode(`${settings.issuer}\0${settings.ownerSub}`), + ), + ).toString("base64url"); + const now = yield* DateTime.now; + yield* sql`INSERT INTO auth_devices(device_id, account_id, environment_id, client_json, created_at, expires_at, validated_at) VALUES('archive-device', ${accountHash}, ${environmentId}, '{}', ${now.epochMilliseconds}, ${now.epochMilliseconds + 3600000}, ${now.epochMilliseconds})`; + const bound = yield* (yield* SessionStore.SessionStore).issue({ + deviceId: "archive-device", + method: "bearer-access-token", + scopes: [AuthArchiveReadScope, AuthArchiveWriteScope], + }); + const context = yield* Effect.context< + Layer.Success | FileSystem.FileSystem + >(); + const app = yield* Effect.acquireRelease( + Effect.sync(() => HttpRouter.toWebHandler(ArchiveHttp.layer, { disableLogger: true })), + (app) => Effect.promise(() => app.dispose()), + ); + const request = ( + method: string, + suffix: string, + token?: string, + body?: unknown, + range?: string, + ) => + Effect.promise(() => + app.handler( + new Request( + `https://archive.example.test${suffix.startsWith("/api/archive/") ? suffix : `/api/archive/catalog${suffix}`}`, + { + method, + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(range ? { range } : {}), + ...(body ? { "content-type": "application/json" } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }, + ), + context, + ), + ); + assert.strictEqual((yield* request("GET", "")).status, 401); + assert.strictEqual((yield* request("GET", "", legacy.token)).status, 403); + assert.strictEqual((yield* request("GET", "", write.token)).status, 403); + const response = yield* request("GET", "", read.token); + assert.strictEqual(response.status, 200); + const catalog = yield* decodeCatalog(yield* Effect.promise(() => response.json())); + const entry = catalog.entries[0]!; + const publication = { + environmentId, + manifestHash: entry.manifestHash, + project: entry.project, + }; + assert.strictEqual((yield* request("POST", "", read.token, publication)).status, 403); + assert.strictEqual((yield* request("POST", "", write.token, publication)).status, 403); + assert.strictEqual( + (yield* request("POST", "", bound.token, { + ...publication, + environmentId: "other-environment", + })).status, + 403, + ); + assert.strictEqual((yield* request("POST", "", bound.token, publication)).status, 200); + const suffix = `/${encodeURIComponent(environmentId)}/${threadId}`; + const detail = yield* request("GET", suffix, read.token); + assert.strictEqual(detail.status, 200); + const snapshot = yield* decodeSnapshot(yield* Effect.promise(() => detail.json())); + assert.strictEqual( + snapshot.projection.messages[0]?.text, + "private archived message body", + ); + const assetInput = { + environmentId, + resource: { _tag: "attachment", attachmentId: imageId, fileName: "picture.png" }, + }; + assert.strictEqual( + (yield* request("POST", "/api/archive/assets", write.token, assetInput)).status, + 403, + ); + assert.strictEqual( + (yield* request("POST", "/api/archive/assets", read.token, { + ...assetInput, + environmentId: "other-environment", + })).status, + 503, + ); + assert.strictEqual( + (yield* request("POST", "/api/archive/assets", read.token, { + environmentId, + resource: { _tag: "media-file", threadId, path: "/etc/passwd" }, + })).status, + 503, + ); + const issued = yield* request("POST", "/api/archive/assets", bound.token, assetInput); + assert.strictEqual(issued.status, 200); + const imageUrl = (yield* decodeAssetUrl(yield* Effect.promise(() => issued.json()))) + .relativeUrl; + const image = yield* request("GET", imageUrl); + assert.strictEqual(image.status, 200); + assert.strictEqual(image.headers.get("content-type"), "image/png"); + assert.strictEqual(image.headers.get("cache-control"), "private, no-store"); + assert.deepEqual( + new Uint8Array(yield* Effect.promise(() => image.arrayBuffer())), + new Uint8Array([137, 80, 78, 71, 1, 2, 3]), + ); + const movieIssued = yield* request("POST", "/api/archive/assets", bound.token, { + environmentId, + resource: { _tag: "attachment", attachmentId: movieId, fileName: "movie.mp4" }, + }); + assert.strictEqual(movieIssued.status, 200); + const movieUrl = (yield* decodeAssetUrl(yield* Effect.promise(() => movieIssued.json()))) + .relativeUrl; + const movie = yield* request("GET", movieUrl, undefined, undefined, "bytes=3-6"); + assert.strictEqual(movie.status, 206); + assert.strictEqual(movie.headers.get("content-type"), "video/mp4"); + assert.strictEqual(movie.headers.get("content-range"), "bytes 3-6/10"); + assert.strictEqual(yield* Effect.promise(() => movie.text()), "3456"); + assert.strictEqual( + (yield* request("GET", movieUrl, undefined, undefined, "bytes=20-")).status, + 416, + ); + yield* auth.revokeSession(bound.sessionId); + assert.strictEqual((yield* request("GET", imageUrl)).status, 404); + const renewed = yield* (yield* SessionStore.SessionStore).issue({ + deviceId: "archive-device", + method: "bearer-access-token", + scopes: [AuthArchiveReadScope, AuthArchiveWriteScope], + }); + const renewedAsset = yield* request( + "POST", + "/api/archive/assets", + renewed.token, + assetInput, + ); + const renewedUrl = (yield* decodeAssetUrl( + yield* Effect.promise(() => renewedAsset.json()), + )).relativeUrl; + yield* TestClock.adjust("5 minutes"); + assert.strictEqual((yield* request("GET", renewedUrl)).status, 404); + assert.strictEqual( + (yield* request("DELETE", suffix, read.token, { manifestHash: entry.manifestHash })) + .status, + 403, + ); + assert.strictEqual( + (yield* request("DELETE", suffix, renewed.token, { manifestHash: "0".repeat(64) })) + .status, + 204, + ); + assert.strictEqual( + (yield* (yield* ArchiveCatalog.ArchiveCatalog).list).entries.length, + 1, + ); + assert.strictEqual( + (yield* request("DELETE", suffix, renewed.token, { manifestHash: entry.manifestHash })) + .status, + 204, + ); + assert.strictEqual( + (yield* (yield* ArchiveCatalog.ArchiveCatalog).list).entries.length, + 0, + ); + yield* sql`UPDATE auth_devices SET revoked_at = ${now.epochMilliseconds} WHERE device_id = 'archive-device'`; + assert.strictEqual((yield* request("POST", "", renewed.token, publication)).status, 401); + }).pipe(Effect.provide(layers)); + }), + ); + + it.effect( + "rejects an older publication and keeps a newer archive when an old withdrawal arrives", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + yield* seed; + const archive = yield* TranscriptArchive.TranscriptArchive; + const catalog = yield* ArchiveCatalog.ArchiveCatalog; + const client = yield* ArchiveClient.ArchiveClient; + yield* archive.offload(threadId); + const old = (yield* catalog.list).entries[0]!; + yield* archive.restore(threadId); + const message = (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection( + threadId, + )).messages[0]!; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("newer-archive"), + type: "message.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...message, text: "newer archived body" }, + }, + ], + }); + yield* archive.offload(threadId); + const current = (yield* catalog.list).entries[0]!; + assert.isAbove(current.throughSequence, old.throughSequence); + assert.strictEqual( + (yield* client + .publish({ + environmentId: old.environmentId, + manifestHash: old.manifestHash, + project: old.project, + }) + .pipe(Effect.flip)).reason, + "changed", + ); + yield* catalog.withdraw(old.environmentId, threadId, old.manifestHash); + assert.strictEqual((yield* catalog.list).entries[0]?.manifestHash, current.manifestHash); + assert.strictEqual( + (yield* catalog.read(current.environmentId, threadId)).projection.messages[0]?.text, + "newer archived body", + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect( + "rejects an absent or wrong provisioned storage identity before creating archive directories", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "knife-archive-identity-" }); + const layer = ArchiveObjects.layer.pipe( + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + T3CODE_ARCHIVE_DIR: root, + T3CODE_ARCHIVE_STORE_ID: "test-store", + }), + ), + ), + ); + yield* Effect.gen(function* () { + const objects = yield* ArchiveObjects.ArchiveObjects; + assert.strictEqual((yield* objects.directory.pipe(Effect.flip)).reason, "storage"); + assert.isFalse(yield* fs.exists(path.join(root, "objects"))); + yield* fs.writeFileString(path.join(root, ".knife-archive-store-id"), "different-store"); + assert.strictEqual((yield* objects.directory.pipe(Effect.flip)).reason, "storage"); + assert.isFalse(yield* fs.exists(path.join(root, "objects"))); + yield* fs.writeFileString(path.join(root, ".knife-archive-store-id"), "test-store\n"); + assert.strictEqual(yield* objects.directory, path.join(root, "objects")); + }).pipe(Effect.provide(layer)); + }), + ); + + it.effect("retains archived payloads while a run is queued, running, or awaiting input", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const archive = yield* TranscriptArchive.TranscriptArchive; + for (const status of ["preparing", "queued", "starting", "running", "waiting"]) { + yield* sql`INSERT INTO orchestration_v2_projection_runs(run_id, thread_id, ordinal, provider, status, requested_at, payload_json) VALUES('busy-run', ${threadId}, 1, 'codex', ${status}, '2026-01-01T00:00:00Z', '{}')`; + yield* archive.offload(threadId); + yield* archive.sweep; + assert.strictEqual((yield* sql`SELECT 1 FROM transcript_archives`).length, 0); + yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = 'busy-run'`; + } + assert.strictEqual( + (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection(threadId)) + .messages[0]?.text, + "private archived message body", + ); + yield* archive.sweep; + assert.strictEqual((yield* sql`SELECT 1 FROM transcript_archives`).length, 1); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("rejects a corrupt archive without unarchiving or replacing local indexes", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const thread = yield* seed; + const archive = yield* TranscriptArchive.TranscriptArchive; + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const projections = yield* ProjectionStore.ProjectionStoreV2; + yield* archive.offload(threadId); + const entry = (yield* sql<{ + object_hash: string; + }>`SELECT object_hash FROM transcript_archives WHERE thread_id = ${threadId}`)[0]!; + yield* fs.writeFileString(f.path.join(f.root, "objects", entry.object_hash), "corrupt"); + assert.isTrue( + (yield* projections.getThreadProjection(threadId).pipe(Effect.exit))._tag === "Failure", + ); + const result = yield* (yield* EventSink.EventSinkV2) + .write({ + events: [ + { + id: EventId.make("corrupt-unarchive"), + type: "thread.unarchived", + threadId, + occurredAt: yield* DateTime.now, + payload: thread, + }, + ], + }) + .pipe(Effect.exit); + assert.strictEqual(result._tag, "Failure"); + assert.isNotNull((yield* projections.getThreadShell(threadId))?.archivedAt); + assert.strictEqual( + (yield* sql`SELECT 1 FROM transcript_archives WHERE thread_id = ${threadId}`).length, + 1, + ); + const message = (yield* sql<{ + payload_json: string; + }>`SELECT payload_json FROM orchestration_v2_projection_messages WHERE thread_id = ${threadId}`)[0]!; + assert.isFalse(message.payload_json.includes("private archived")); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("does not acknowledge an object when its durable file flush fails", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const failure = PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "sync", + }); + const objects = yield* ArchiveObjects.ArchiveObjects.pipe( + Effect.provide( + Layer.fresh(ArchiveObjects.layer).pipe( + Layer.provide( + ConfigProvider.layer(ConfigProvider.fromUnknown({ T3CODE_ARCHIVE_DIR: f.root })), + ), + ), + ), + Effect.provideService(FileSystem.FileSystem, { + ...fs, + open: (file, options) => + fs + .open(file, options) + .pipe(Effect.map((handle) => ({ ...handle, sync: Effect.fail(failure) }))), + }), + ); + const bytes = new TextEncoder().encode("not durable"); + const hash = Buffer.from(yield* (yield* Crypto.Crypto).digest("SHA-256", bytes)).toString( + "hex", + ); + assert.strictEqual( + (yield* objects.put(hash, Stream.succeed(bytes)).pipe(Effect.flip)).reason, + "storage", + ); + assert.isFalse(yield* fs.exists(f.path.join(f.root, "objects", hash))); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("retains source payloads when the archive store identity is absent or mismatched", () => + Effect.gen(function* () { + for (const mode of ["unconfigured", "missing", "mismatch"] as const) { + const f = yield* fixture(); + yield* Effect.gen(function* () { + yield* seed; + const fs = yield* FileSystem.FileSystem; + const client = yield* ArchiveClient.ArchiveClient; + let archive = yield* TranscriptArchive.TranscriptArchive; + if (mode === "unconfigured") { + const catalog = yield* ArchiveCatalog.ArchiveCatalog.pipe( + Effect.provide( + Layer.fresh( + ArchiveCatalog.layer.pipe( + Layer.provide(ArchiveObjects.layer), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ T3CODE_ARCHIVE_DIR: f.root }), + ), + ), + ), + ), + ), + ); + archive = yield* TranscriptArchive.TranscriptArchive.pipe( + Effect.provide(Layer.fresh(TranscriptArchive.layer)), + Effect.provideService(ArchiveClient.ArchiveClient, { + ...client, + publish: catalog.publish, + }), + ); + } else if (mode === "missing") + yield* fs.remove(f.path.join(f.root, ".knife-archive-store-id")); + else + yield* fs.writeFileString( + f.path.join(f.root, ".knife-archive-store-id"), + "wrong-store", + ); + assert.strictEqual( + (yield* archive.offload(threadId).pipe(Effect.flip)).reason, + "storage", + ); + assert.strictEqual( + (yield* (yield* SqlClient.SqlClient)`SELECT 1 FROM transcript_archives`).length, + 0, + ); + assert.strictEqual( + (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection(threadId)) + .messages[0]?.text, + "private archived message body", + ); + }).pipe(Effect.provide(f.layer)); + } + }), + ); + + it.effect( + "keeps existing attachment URLs playable after eviction and restores bytes on unarchive", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const thread = yield* seed; + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const archive = yield* TranscriptArchive.TranscriptArchive; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const id = ChatAttachmentId.make(createDeterministicAttachmentId(threadId, "recording")!); + const file = f.path.join(config.attachmentsDir, `${id}.png`); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + const bytes = new Uint8Array([137, 80, 78, 71, 1, 2, 3]); + yield* fs.writeFile(file, bytes); + const message = (yield* projections.getThreadProjection(threadId)).messages[0]!; + yield* sink.write({ + events: [ + { + id: EventId.make("attachment-added"), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + ...message, + attachments: [ + { + type: "image", + id, + name: "recording.png", + mimeType: "image/png", + sizeBytes: bytes.length, + }, + ], + }, + }, + ], + }); + const issue = issueAssetUrl({ + resource: { + _tag: "attachment", + attachmentId: id, + fileName: "recording.png", + mimeType: "image/png", + }, + }); + const old = yield* issue; + const resolve = (url: string) => { + const suffix = url.slice(`${ASSET_ROUTE_PREFIX}/`.length); + const separator = suffix.indexOf("/"); + return resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1)); + }; + assert.strictEqual((yield* resolve(old.relativeUrl))?.kind, "file"); + yield* archive.sweep; + assert.isFalse(yield* fs.exists(file)); + for (const url of [old.relativeUrl, (yield* issue).relativeUrl]) { + const asset = yield* resolve(url); + assert.strictEqual(asset?.kind, "archived-file"); + if (asset?.kind !== "archived-file") + return yield* Effect.die("Expected archived asset"); + const opened = yield* (yield* ArchiveClient.ArchiveClient).open( + asset.location, + asset.hash, + ); + assert.strictEqual(opened.kind, "file"); + if (opened.kind === "file") + assert.deepStrictEqual(yield* fs.readFile(opened.path), bytes); + } + yield* sink.write({ + events: [ + { + id: EventId.make("attachment-unarchive"), + type: "thread.unarchived", + threadId, + occurredAt: now, + payload: thread, + }, + ], + }); + assert.deepStrictEqual(yield* fs.readFile(file), bytes); + assert.strictEqual((yield* resolve(old.relativeUrl))?.kind, "file"); + }).pipe( + Effect.provide( + Layer.mergeAll( + f.layer, + WorkspacePaths.layer, + Layer.mock(NativeAppIconResolver.NativeAppIconResolver)({}), + Layer.mock(ProjectFaviconResolver.ProjectFaviconResolver)({}), + Layer.mock(Orchestrator.OrchestratorV2)({}), + ), + ), + ); + }), + ); + + it.effect( + "keeps fork history readable and its source attachment warm through offline rebuild", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const thread = yield* seed; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sink = yield* EventSink.EventSinkV2; + const archive = yield* TranscriptArchive.TranscriptArchive; + const fs = yield* FileSystem.FileSystem; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const forkId = ThreadId.make("archive-fork"); + const runId = RunId.make("archive-source-run"); + const nodeId = NodeId.make("archive-source-node"); + const message = (yield* projections.getThreadProjection(threadId)).messages[0]!; + const attachmentId = ChatAttachmentId.make( + createDeterministicAttachmentId(threadId, "source")!, + ); + const config = yield* ServerConfig.ServerConfig; + const attachment = f.path.join(config.attachmentsDir, `${attachmentId}.png`); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + yield* fs.writeFileString(attachment, "source attachment"); + const fork = { + ...thread, + id: forkId, + forkedFrom: { type: "run" as const, threadId, runId }, + lineage: { + parentThreadId: threadId, + relationshipToParent: "fork" as const, + rootThreadId: threadId, + }, + }; + const item = (yield* projections.getThreadProjection(threadId)).turnItems[0]!; + yield* sink.write({ + events: [ + { + id: EventId.make("source-run"), + type: "run.created", + threadId, + runId, + nodeId, + driver: ProviderDriverKind.make("codex"), + occurredAt: now, + payload: { + id: runId, + threadId, + ordinal: 1, + providerInstanceId: thread.providerInstanceId, + modelSelection: thread.modelSelection, + providerThreadId: null, + userMessageId: message.id, + rootNodeId: nodeId, + activeAttemptId: null, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }, + }, + { + id: EventId.make("source-run-message"), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + ...message, + runId, + attachments: [ + { + type: "image", + id: attachmentId, + name: "source.png", + mimeType: "image/png", + sizeBytes: 17, + }, + ], + }, + }, + { + id: EventId.make("source-run-item"), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { ...item, runId }, + }, + { + id: EventId.make("fork-created"), + type: "thread.created", + threadId: forkId, + occurredAt: now, + payload: fork, + }, + ], + }); + const before = yield* projections.getThreadProjection(forkId); + assert.isTrue( + before.visibleTurnItems.some( + (item) => + item.item.type === "assistant_message" && + item.item.text === "private archived message body", + ), + ); + yield* archive.sweep; + assert.isTrue(yield* fs.exists(attachment)); + assert.deepStrictEqual(yield* projections.getThreadProjection(forkId), before); + const objects = f.path.join(f.root, "objects"); + yield* fs.rename(objects, `${objects}-offline`); + assert.isTrue( + (yield* (yield* ProjectionMaintenance.ProjectionMaintenanceV2).rebuild).valid, + ); + yield* fs.rename(`${objects}-offline`, objects); + assert.deepStrictEqual(yield* projections.getThreadProjection(forkId), before); + yield* sink.write({ + events: [ + { + id: EventId.make("fork-archived"), + type: "thread.archived", + threadId: forkId, + occurredAt: now, + payload: { ...fork, archivedAt: now }, + }, + ], + }); + yield* archive.sweep; + assert.isFalse(yield* fs.exists(attachment)); + const sourceManifest = (yield* sql<{ + object_hash: string; + }>`SELECT object_hash FROM transcript_archives WHERE thread_id = ${threadId}`)[0]!; + const forkManifest = (yield* sql<{ + object_hash: string; + }>`SELECT object_hash FROM transcript_archives WHERE thread_id = ${forkId}`)[0]!; + const source = JSON.parse( + yield* fs.readFileString(f.path.join(objects, sourceManifest.object_hash)), + ); + const savedFork = JSON.parse( + yield* fs.readFileString(f.path.join(objects, forkManifest.object_hash)), + ); + assert.strictEqual( + savedFork.context.find( + (s: { table: string }) => s.table === "orchestration_v2_projection_messages", + ).hash, + source.segments.find((s: { table: string }) => s.table === "messages").hash, + ); + assert.deepStrictEqual(savedFork.assets, source.assets); + assert.deepStrictEqual( + (yield* projections.getThreadProjection(forkId)).visibleTurnItems, + before.visibleTurnItems, + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect( + "offloads bodies only after verification, reads without retaining them, rebuilds offline, and restores before unarchive", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const thread = yield* seed; + const archive = yield* TranscriptArchive.TranscriptArchive; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const events = yield* ApplicationEvents.OrchestrationEventStore; + const before = yield* projections.getThreadProjection(threadId); + const highWater = yield* events.latestAgentSequence(threadId); + yield* archive.offload(threadId); + const raw = yield* sql<{ + payload_json: string; + }>`SELECT payload_json FROM orchestration_v2_projection_messages WHERE thread_id = ${threadId}`; + assert.isFalse(raw[0]?.payload_json.includes("private archived")); + const shell = yield* projections.getThreadShell(threadId); + assert.strictEqual(shell?.title, "Searchable archive title"); + assert.strictEqual(shell?.visibleItemCount, before.visibleTurnItems.length); + assert.strictEqual(yield* events.latestAgentSequence(threadId), highWater); + assert.isFalse( + (yield* events.getAgentReplayStats({ + threadId, + afterSequence: 0, + throughSequence: highWater, + maxEvents: 100, + })).hasCreateEvent, + ); + assert.deepStrictEqual(yield* projections.getThreadProjection(threadId), before); + assert.strictEqual( + (yield* sql`SELECT 1 FROM transcript_archives WHERE thread_id = ${threadId}`).length, + 1, + ); + assert.isFalse( + (yield* sql<{ + payload_json: string; + }>`SELECT payload_json FROM orchestration_v2_projection_messages WHERE thread_id = ${threadId}`)[0]?.payload_json.includes( + "private archived", + ), + ); + const fs = yield* FileSystem.FileSystem; + const objects = f.path.join(f.root, "objects"); + yield* fs.rename(objects, `${objects}-offline`); + assert.isTrue( + (yield* (yield* ProjectionMaintenance.ProjectionMaintenanceV2).rebuild).valid, + ); + assert.isTrue( + (yield* projections.getThreadProjection(threadId).pipe(Effect.exit))._tag === "Failure", + ); + assert.strictEqual( + (yield* projections.getThreadShell(threadId))?.title, + "Searchable archive title", + ); + yield* fs.rename(`${objects}-offline`, objects); + const now = yield* DateTime.now; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("archive-unarchived"), + type: "thread.unarchived", + threadId, + occurredAt: now, + payload: thread, + }, + ], + }); + assert.strictEqual( + (yield* sql`SELECT 1 FROM transcript_archives WHERE thread_id = ${threadId}`).length, + 0, + ); + assert.strictEqual( + (yield* projections.getThreadProjection(threadId)).messages[0]?.text, + "private archived message body", + ); + assert.isTrue( + (yield* (yield* ProjectionMaintenance.ProjectionMaintenanceV2).rebuild).valid, + ); + assert.strictEqual( + (yield* projections.getThreadProjection(threadId)).messages[0]?.text, + "private archived message body", + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("never evicts local bodies when the destination is missing", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + yield* seed; + const fs = yield* FileSystem.FileSystem; + yield* fs.remove(f.root, { recursive: true }); + assert.strictEqual( + (yield* (yield* TranscriptArchive.TranscriptArchive).offload(threadId).pipe(Effect.flip)) + .reason, + "storage", + ); + assert.strictEqual( + (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection(threadId)) + .messages[0]?.text, + "private archived message body", + ); + assert.strictEqual( + (yield* (yield* SqlClient.SqlClient)`SELECT 1 FROM transcript_archives`).length, + 0, + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("rejects a wrong object hash without publishing partial bytes", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const store = yield* ArchiveObjects.ArchiveObjects; + const hash = "0".repeat(64); + const error = yield* store + .put(hash, Stream.make(new TextEncoder().encode("wrong bytes"))) + .pipe(Effect.flip); + assert.strictEqual(error.reason, "integrity"); + assert.isFalse( + yield* (yield* FileSystem.FileSystem).exists(f.path.join(f.root, "objects", hash)), + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + it.effect("an unarchive committed during upload wins over offloading", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.gen(function* () { + const thread = yield* seed; + const uploaded = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const client = yield* ArchiveClient.ArchiveClient; + const archive = yield* TranscriptArchive.TranscriptArchive.pipe( + Effect.provide(Layer.fresh(TranscriptArchive.layer)), + Effect.provideService(ArchiveClient.ArchiveClient, { + ...client, + upload: (file) => + client.upload(file).pipe( + Effect.tap(() => Deferred.succeed(uploaded, undefined)), + Effect.tap(() => Deferred.await(resume)), + ), + }), + ); + const offload = yield* archive.offload(threadId).pipe(Effect.flip, Effect.forkScoped); + yield* Effect.raceFirst( + Deferred.await(uploaded), + Fiber.await(offload).pipe(Effect.flatMap((exit) => Effect.die(exit))), + ); + const now = yield* DateTime.now; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("race-unarchive"), + type: "thread.unarchived", + threadId, + occurredAt: now, + payload: thread, + }, + ], + }); + yield* Deferred.succeed(resume, undefined); + assert.strictEqual((yield* Fiber.join(offload)).reason, "changed"); + assert.strictEqual( + (yield* (yield* SqlClient.SqlClient)`SELECT 1 FROM transcript_archives`).length, + 0, + ); + assert.strictEqual( + (yield* (yield* ProjectionStore.ProjectionStoreV2).getThreadProjection(threadId)) + .messages[0]?.text, + "private archived message body", + ); + yield* archive.sweep; + assert.strictEqual((yield* (yield* ArchiveCatalog.ArchiveCatalog).list).entries.length, 0); + }).pipe(Effect.provide(f.layer)); + }), + ); +}); diff --git a/apps/server/src/archive/TranscriptArchive.ts b/apps/server/src/archive/TranscriptArchive.ts new file mode 100644 index 0000000000000000000000000000000000000000..444ba538e69258fca627a0e6fa5068a0c584f078 --- /dev/null +++ b/apps/server/src/archive/TranscriptArchive.ts @@ -0,0 +1,778 @@ +import { ArchiveError, ProjectId } from "@t3tools/contracts"; +import * as ArchiveClient from "./ArchiveClient.ts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import { type ArchiveManagementRequest, type ArchiveStatus, ThreadId } from "@t3tools/contracts"; +import { + extractMarkdownLinkHrefs, + resolveMarkdownFileLinkTarget, +} from "@t3tools/shared/markdownLinks"; +import { + isWorkspaceImagePreviewPath, + audioMimeTypeFromExtension, + isWorkspaceVideoPreviewPath, +} from "@t3tools/shared/filePreview"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; +import { + resolveAttachmentPathById, + attachmentPathsById, + threadHtmlRenderAttachmentIds, + parseAttachmentIdFromRelativePath, + parseThreadSegmentFromAttachmentId, + toSafeThreadAttachmentSegment, +} from "../attachmentStore.ts"; +import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ArchiveStorage from "./ArchiveStorage.ts"; +import * as ServerConfig from "../config.ts"; + +import { + payloadTables, + Table, + Segment, + Row, + Metadata, + canonicalTables, + CanonicalTable, + Bundle, +} from "./ArchiveFormat.ts"; + +type LoadedBundle = Omit & { + segments: typeof Metadata.Type; + events: ReadonlyArray; + context: ReadonlyArray<{ + table: typeof CanonicalTable.Type; + rows: ReadonlyArray; + }>; +}; +const decodeMetadata = Schema.decodeEffect(Schema.fromJsonString(Metadata)); +const decodeBundle = Schema.decodeEffect(Schema.fromJsonString(Bundle)); +const encodeBundle = Schema.encodeEffect(Schema.fromJsonString(Bundle)); +const encodeRows = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(Row))); +const decodeRows = Schema.decodeEffect(Schema.fromJsonString(Schema.Array(Row))); +const RecordJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)); +const decodeRecord = Schema.decodeSync(RecordJson); +const encodeRecord = Schema.encodeSync(RecordJson); +const encodeMetadata = Schema.encodeSync(Schema.fromJsonString(Metadata)); +const ids = { + messages: "message_id", + turn_items: "turn_item_id", + subagents: "subagent_id", +} as const; +const isArchiveError = Schema.is(ArchiveError); +const archiveError = (cause: unknown) => + isArchiveError(cause) + ? cause + : new ArchiveError({ reason: "storage", cause: Redacted.make(cause) }); + +/** Local columns keep timeline ordering and shell state; missing bodies never decode as transcripts. */ +function metadataPayload(table: typeof Table.Type, json: string) { + const payload = decodeRecord(json); + if (table === "turn_items" && (payload.type === "error" || payload.type === "secret_request")) + return json; + const keys = + table === "turn_items" + ? [ + "id", + "threadId", + "runId", + "nodeId", + "type", + "status", + "ordinal", + "messageId", + "inputIntent", + "createdBy", + "subagentId", + "requestId", + "createdAt", + "updatedAt", + ] + : table === "messages" + ? ["id", "threadId", "role", "createdBy", "createdAt", "updatedAt", "attachments"] + : [ + "id", + "threadId", + "status", + "childThreadId", + "origin", + "providerThreadId", + "createdAt", + "updatedAt", + ]; + return encodeRecord({ + archivedPayload: true, + ...Object.fromEntries(keys.filter((key) => key in payload).map((key) => [key, payload[key]])), + }); +} + +export class TranscriptArchive extends Context.Service< + TranscriptArchive, + { + readonly enabled: Effect.Effect; + readonly manage: ( + request: ArchiveManagementRequest, + ) => Effect.Effect; + readonly offload: (threadId: ThreadId) => Effect.Effect; + readonly restore: (threadId: ThreadId) => Effect.Effect; + readonly read: ( + threadId: ThreadId, + read: Effect.Effect, + includeHistory?: boolean, + ) => Effect.Effect; + readonly readStream: ( + threadId: ThreadId, + stream: Stream.Stream, + ) => Stream.Stream; + readonly sweep: Effect.Effect; + readonly asset: ( + paths: ReadonlyArray, + ) => Effect.Effect< + Option.Option<{ path: string; location: string; hash: string }>, + ArchiveError + >; + } +>()("t3/archive/TranscriptArchive") {} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const client = yield* ArchiveClient.ArchiveClient; + const identity = yield* ServerEnvironment.ServerEnvironmentIdentity; + const projects = yield* ProjectStore.ProjectStoreV2; + const storage = yield* ArchiveStorage.ArchiveStorage; + const activeThreads = sql`SELECT thread_id FROM orchestration_v2_projection_runs WHERE status IN ('preparing', 'queued', 'starting', 'running', 'waiting') UNION SELECT thread_id FROM orchestration_v2_effect_outbox WHERE status IN ('pending', 'running') UNION SELECT thread_id FROM orchestration_v2_projection_turn_items WHERE type IN ('command_execution', 'dynamic_tool', 'subagent') AND status IN ('pending', 'running', 'waiting') UNION SELECT thread_id FROM orchestration_v2_projection_runtime_requests WHERE status = 'pending'`; + const catalog = (threadId: ThreadId) => + sql<{ + thread_id: ThreadId; + location: string; + object_hash: string; + through_sequence: number; + }>`SELECT thread_id, location, object_hash, through_sequence FROM transcript_archives WHERE thread_id = ${threadId}`; + const ancestry = (threadId: ThreadId) => sql`WITH RECURSIVE ancestry(thread_id) AS ( + SELECT ${threadId} UNION SELECT json_extract(t.payload_json, '$.forkedFrom.threadId') + FROM orchestration_v2_projection_threads t JOIN ancestry a ON t.thread_id = a.thread_id + WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run' + )`; + const dependencies = (threadId: ThreadId) => + sql<{ thread_id: string }>`${ancestry(threadId)} SELECT thread_id FROM ancestry`; + const entries = (threadId: ThreadId, includeHistory = true) => + includeHistory + ? sql<{ + thread_id: ThreadId; + location: string; + object_hash: string; + through_sequence: number; + }>`${ancestry(threadId)} SELECT a.thread_id, a.location, a.object_hash, a.through_sequence FROM transcript_archives a JOIN ancestry s ON s.thread_id = a.thread_id ORDER BY a.thread_id = ${threadId} DESC` + : catalog(threadId); + + const load = ( + threadId: ThreadId, + entry: { location: string; object_hash: string; through_sequence: number }, + ) => + Effect.scoped( + Effect.gen(function* () { + const temporary = yield* fs.makeTempDirectoryScoped({ prefix: "knife-archive-read-" }); + const file = path.join(temporary, "transcript"); + yield* client.download(entry.location, entry.object_hash, file); + const manifest = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeBundle)); + const hashes = new Set([ + manifest.events, + ...manifest.segments.map((row) => row.hash), + ...manifest.context.map((row) => row.hash), + ]); + const loaded = yield* Effect.forEach( + [...hashes], + (hash) => + Effect.gen(function* () { + const file = path.join(temporary, hash); + yield* client.download(entry.location, hash, file); + return [ + hash, + yield* fs.readFileString(file).pipe(Effect.flatMap(decodeRows)), + ] as const; + }), + { concurrency: 4 }, + ); + const chunks = new Map(loaded); + const segments: Array = []; + for (const segment of manifest.segments) { + const rows = chunks.get(segment.hash); + if (!rows) return yield* new ArchiveError({ reason: "integrity" }); + segments.push({ table: segment.table, rows }); + } + const context: Array = []; + for (const segment of manifest.context) { + const rows = chunks.get(segment.hash); + if (!rows) return yield* new ArchiveError({ reason: "integrity" }); + context.push({ table: segment.table, rows }); + } + const events = chunks.get(manifest.events); + if (!events) return yield* new ArchiveError({ reason: "integrity" }); + const bundle = { ...manifest, segments, context, events }; + if (bundle.threadId !== threadId || bundle.throughSequence !== entry.through_sequence) + return yield* new ArchiveError({ reason: "integrity" }); + for (const segment of bundle.segments) + for (const row of segment.rows) { + if ( + row.thread_id !== threadId || + typeof row[ids[segment.table]] !== "string" || + typeof row.payload_json !== "string" + ) + return yield* new ArchiveError({ reason: "integrity" }); + } + for (const event of bundle.events) { + if ( + typeof event.sequence !== "number" || + typeof event.payload_json !== "string" || + event.stream_id !== threadId + ) + return yield* new ArchiveError({ reason: "integrity" }); + } + return bundle; + }), + ).pipe(Effect.mapError(archiveError)); + const install = (bundle: LoadedBundle) => + Effect.gen(function* () { + for (const segment of bundle.segments) + for (const row of segment.rows) { + yield* sql`UPDATE ${sql(`orchestration_v2_projection_${segment.table}`)} SET payload_json = ${row.payload_json} WHERE thread_id = ${bundle.threadId} AND ${sql(ids[segment.table])} = ${row[ids[segment.table]]}`; + } + for (const event of bundle.events) + yield* sql`UPDATE orchestration_events SET payload_json = ${event.payload_json} WHERE sequence = ${event.sequence} AND stream_id = ${bundle.threadId} AND payload_json = 'null'`; + yield* sql`DELETE FROM transcript_archives WHERE thread_id = ${bundle.threadId}`; + }); + const restore = Effect.fn("TranscriptArchive.restore")(function* (threadId: ThreadId) { + const entry = (yield* catalog(threadId))[0]; + if (!entry) return; + const bundle = yield* load(threadId, entry); + yield* sql.withTransaction( + Effect.gen(function* () { + const current = (yield* catalog(threadId))[0]; + if (!current) return; + if (current.object_hash !== entry.object_hash) + return yield* new ArchiveError({ reason: "changed" }); + for (const asset of bundle.assets) { + const relative = path.relative(config.attachmentsDir, asset.path); + if ( + relative.startsWith("..") || + path.isAbsolute(relative) || + (yield* fs.exists(asset.path)) + ) + continue; + yield* Effect.scoped( + Effect.gen(function* () { + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true, mode: 0o700 }); + const temp = yield* fs.makeTempDirectoryScoped({ + directory: config.attachmentsDir, + prefix: ".restore-", + }); + const file = path.join(temp, "attachment"); + yield* client.download(entry.location, asset.hash, file); + yield* fs.makeDirectory(path.dirname(asset.path), { recursive: true, mode: 0o700 }); + yield* fs.open(file, { flag: "r+" }).pipe(Effect.flatMap((handle) => handle.sync)); + yield* fs.rename(file, asset.path); + if (!(yield* HostProcess.isWindows)) + yield* fs.open(config.attachmentsDir).pipe(Effect.flatMap((handle) => handle.sync)); + }), + ); + } + yield* install(bundle); + }), + ); + }, Effect.mapError(archiveError)); + const read = ( + threadId: ThreadId, + operation: Effect.Effect, + includeHistory = true, + ) => + Effect.gen(function* () { + const ancestors = yield* entries(threadId, includeHistory).pipe( + Effect.mapError(archiveError), + ); + const bundles: Array<{ payload: LoadedBundle; hash: string }> = []; + const covered = new Set(); + for (const ancestor of ancestors) { + if (covered.has(ancestor.thread_id)) continue; + const bundle = yield* load(ancestor.thread_id, ancestor); + bundles.push({ payload: bundle, hash: ancestor.object_hash }); + covered.add(ancestor.thread_id); + for (const segment of bundle.context) + for (const row of segment.rows) { + if ( + segment.table === "orchestration_v2_projection_threads" && + typeof row.thread_id === "string" + ) + covered.add(row.thread_id); + } + } + if (bundles.length === 0) return yield* operation; + const rollback = Symbol(); + let result: Exit.Exit | undefined; + yield* sql + .withTransaction( + Effect.gen(function* () { + for (const { payload: bundle, hash } of bundles) { + const current = (yield* catalog(bundle.threadId))[0]; + if (!current) continue; + if ( + current.object_hash !== hash || + current.through_sequence !== bundle.throughSequence + ) + return yield* new ArchiveError({ reason: "changed" }); + yield* install(bundle); + for (const segment of bundle.context) + for (const row of segment.rows) { + if (row.thread_id === bundle.threadId) continue; + yield* sql`INSERT OR REPLACE INTO ${sql(segment.table)} ${sql.insert(row)}`; + } + } + result = yield* Effect.exit(operation); + return yield* Effect.fail(rollback); + }), + ) + .pipe( + Effect.catchIf( + (error) => error === rollback, + () => Effect.void, + ), + Effect.mapError(archiveError), + ); + if (!result) return yield* Effect.die("An archived read must produce a result"); + return yield* result; + }); + const publish = Effect.fn("TranscriptArchive.publish")(function* ( + threadId: ThreadId, + manifestHash: string, + ) { + const row = (yield* sql<{ + project_id: string; + }>`SELECT project_id FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId}`)[0]; + if (!row) return yield* new ArchiveError({ reason: "changed" }); + const project = yield* projects.getShell(ProjectId.make(row.project_id)); + if (Option.isNone(project)) return yield* new ArchiveError({ reason: "changed" }); + return yield* client.publish({ + environmentId: yield* identity.getEnvironmentId, + manifestHash, + project: project.value, + }); + }, Effect.mapError(archiveError)); + const offload = Effect.fn("TranscriptArchive.offload")(function* (threadId: ThreadId) { + if (!(yield* client.enabled) || (yield* catalog(threadId)).length > 0) return; + const location = yield* client.location; + const snapshot = yield* read( + threadId, + sql + .withTransaction( + Effect.gen(function* () { + const thread = (yield* sql<{ + archived_at: string | null; + deleted_at: string | null; + }>`SELECT archived_at, deleted_at FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId}`)[0]; + if (!thread?.archived_at || thread.deleted_at) return undefined; + const busy = + yield* sql`SELECT 1 FROM (${activeThreads}) WHERE thread_id = ${threadId} LIMIT 1`; + if (busy.length) return undefined; + const bounds = yield* sql<{ + sequence: number; + }>`SELECT COALESCE(MAX(sequence), 0) AS sequence FROM orchestration_events WHERE stream_id = ${threadId} AND aggregate_kind = 'thread' AND application_event_version = 2`; + const segments = []; + for (const table of payloadTables) { + const rows = yield* sql< + typeof Row.Type + >`SELECT * FROM ${sql(`orchestration_v2_projection_${table}`)} WHERE thread_id = ${threadId} ORDER BY rowid`; + segments.push({ table, rows }); + } + const events = yield* sql`SELECT *, + CASE WHEN event_type IN ('subagent.updated', 'turn-item.updated') AND json_type(payload_json, '$.promptSequence') = 'integer' + THEN json_set(json_remove(payload_json, '$.promptSequence'), '$.prompt', (SELECT json_extract(a.payload_json, '$.prompt') FROM orchestration_events a WHERE a.sequence = json_extract(orchestration_events.payload_json, '$.promptSequence'))) + ELSE payload_json END AS payload_json + FROM orchestration_events WHERE stream_id = ${threadId} AND aggregate_kind = 'thread' AND application_event_version = 2 ORDER BY sequence`; + const ancestors = yield* dependencies(threadId); + const sourceIds = ancestors.map((row) => row.thread_id); + const context: Array<{ table: string; rows: Array }> = []; + for (const name of canonicalTables) { + const predicate = + name === "context_transfers" + ? sql`source_thread_id IN ${sql.in(sourceIds)} OR target_thread_id IN ${sql.in(sourceIds)}` + : name === "provider_sessions" + ? sql`thread_id IN ${sql.in(sourceIds)} OR provider_session_id IN (SELECT provider_session_id FROM orchestration_v2_projection_provider_session_bindings WHERE thread_id IN ${sql.in(sourceIds)})` + : name === "provider_threads" + ? sql`thread_id IN ${sql.in(sourceIds)} OR owner_node_id IN (SELECT node_id FROM orchestration_v2_projection_nodes WHERE thread_id IN ${sql.in(sourceIds)}) OR provider_thread_id IN (SELECT provider_thread_id FROM orchestration_v2_projection_subagents WHERE thread_id IN ${sql.in(sourceIds)})` + : sql`thread_id IN ${sql.in(sourceIds)}`; + context.push({ + table: `orchestration_v2_projection_${name}`, + rows: [ + ...(yield* sql< + typeof Row.Type + >`SELECT * FROM ${sql(`orchestration_v2_projection_${name}`)} WHERE (${predicate}) ${payloadTables.some((table) => table === name) ? sql`AND thread_id <> ${threadId}` : sql``} ORDER BY rowid`), + ], + }); + } + const revisions = yield* sql<{ + thread_id: string; + sequence: number; + }>`SELECT stream_id AS thread_id, MAX(sequence) AS sequence FROM orchestration_events WHERE aggregate_kind = 'thread' AND application_event_version = 2 AND stream_id IN ${sql.in(sourceIds)} GROUP BY stream_id`; + return { + segments, + events, + context, + revisions, + throughSequence: bounds[0]?.sequence ?? 0, + }; + }), + ) + .pipe(NodeSqliteClient.readOnly), + ); + if (!snapshot) return; + const files = new Set(); + const addAttachment = (attachmentId: string) => + Effect.gen(function* () { + const input = { attachmentsDir: config.attachmentsDir, attachmentId }; + const local = resolveAttachmentPathById(input); + if (local) { + files.add(local); + return; + } + const candidates = attachmentPathsById(input); + const archived = (yield* sql<{ + path: string; + }>`SELECT path FROM transcript_archive_assets WHERE path IN ${sql.in(candidates)} AND location = ${location} LIMIT 1`)[0]; + if (!archived) return yield* new ArchiveError({ reason: "unavailable" }); + files.add(archived.path); + }); + const workspaces = yield* sql<{ + thread_id: string; + cwd: string | null; + }>`SELECT t.thread_id, COALESCE(json_extract(t.payload_json, '$.worktreePath'), p.workspace_root) AS cwd FROM orchestration_v2_projection_threads t LEFT JOIN projection_projects p ON p.project_id = t.project_id WHERE t.thread_id IN ${sql.in(snapshot.revisions.map((row) => row.thread_id))}`; + const roots = new Map(workspaces.map((row) => [row.thread_id, row.cwd ?? undefined])); + const assetSegments = [ + ...snapshot.segments.map((segment) => ({ + ...segment, + table: `orchestration_v2_projection_${segment.table}`, + })), + ...snapshot.context.filter((segment) => + ["orchestration_v2_projection_messages", "orchestration_v2_projection_turn_items"].includes( + segment.table, + ), + ), + ]; + for (const segment of assetSegments) + for (const row of segment.rows) { + const owner = ThreadId.make(String(row.thread_id)); + const cwd = roots.get(owner); + const payload = decodeRecord(String(row.payload_json)); + if ( + segment.table === "orchestration_v2_projection_messages" && + Array.isArray(payload.attachments) + ) + for (const attachment of payload.attachments) { + if ( + attachment && + typeof attachment === "object" && + "id" in attachment && + typeof attachment.id === "string" + ) { + yield* addAttachment(attachment.id); + } + } + if (segment.table === "orchestration_v2_projection_turn_items") + for (const id of threadHtmlRenderAttachmentIds(owner, [ + { + toolName: typeof payload.toolName === "string" ? payload.toolName : null, + output: payload.output, + }, + ])) { + yield* addAttachment(id); + } + for (const text of Object.values(payload)) + if (typeof text === "string") + for (const href of extractMarkdownLinkHrefs(text)) { + const target = resolveMarkdownFileLinkTarget(href, cwd); + if ( + target && + (isWorkspaceImagePreviewPath(target) || + audioMimeTypeFromExtension(path.extname(target)) || + isWorkspaceVideoPreviewPath(target)) + ) + files.add(target); + } + } + const assets = yield* Effect.forEach([...files], (file) => + Effect.gen(function* () { + if (yield* fs.exists(file)) return { path: file, hash: (yield* client.upload(file)).hash }; + const previous = (yield* sql<{ + location: string; + object_hash: string; + }>`SELECT location, object_hash FROM transcript_archive_assets WHERE path = ${file} LIMIT 1`)[0]; + if (!previous || previous.location !== location) + return yield* new ArchiveError({ reason: "unavailable" }); + return { path: file, hash: previous.object_hash }; + }), + ); + const bundle = { + version: 1 as const, + threadId, + throughSequence: snapshot.throughSequence, + segments: snapshot.segments, + events: snapshot.events, + context: snapshot.context, + assets, + }; + const receipt = yield* Effect.scoped( + Effect.gen(function* () { + const temp = yield* fs.makeTempDirectoryScoped({ prefix: "knife-archive-write-" }); + let ordinal = 0; + const storeRows = (rows: ReadonlyArray) => + Effect.gen(function* () { + const file = path.join(temp, `rows-${ordinal++}`); + yield* fs.writeFileString(file, yield* encodeRows(rows), { mode: 0o600 }); + return (yield* client.upload(file)).hash; + }); + const segments = []; + for (const segment of bundle.segments) + if (segment.rows.length) + segments.push({ table: segment.table, hash: yield* storeRows(segment.rows) }); + const context = []; + for (const segment of bundle.context) { + for (const rows of Map.groupBy( + segment.rows, + (row) => row.thread_id ?? row.source_thread_id ?? "shared", + ).values()) { + context.push({ table: segment.table, hash: yield* storeRows(rows) }); + } + } + const manifest = { ...bundle, segments, context, events: yield* storeRows(bundle.events) }; + const file = path.join(temp, "transcript"); + yield* fs.writeFileString(file, yield* encodeBundle(manifest), { mode: 0o600 }); + const uploaded = yield* client.upload(file); + yield* client.download(location, uploaded.hash, path.join(temp, "verified")); + return uploaded; + }), + ); + const environmentId = yield* identity.getEnvironmentId; + const published = yield* publish(threadId, receipt.hash); + if ( + published.environmentId !== environmentId || + published.thread.id !== threadId || + published.manifestHash !== receipt.hash || + published.throughSequence !== snapshot.throughSequence + ) + return yield* new ArchiveError({ reason: "integrity" }); + yield* sql`INSERT INTO transcript_archive_publications(thread_id, location, object_hash) VALUES(${threadId}, ${location}, ${receipt.hash}) ON CONFLICT(thread_id) DO UPDATE SET location = excluded.location, object_hash = excluded.object_hash`; + yield* sql.withTransaction( + Effect.gen(function* () { + const rows = + yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId} AND archived_at IS NOT NULL AND deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM transcript_archives WHERE thread_id = ${threadId}) AND NOT EXISTS (SELECT 1 FROM (${activeThreads}) WHERE thread_id = ${threadId}) AND ${snapshot.throughSequence} = (SELECT MAX(sequence) FROM orchestration_events WHERE stream_id = ${threadId} AND aggregate_kind = 'thread' AND application_event_version = 2)`; + if (!rows.length) return yield* new ArchiveError({ reason: "changed" }); + for (const revision of snapshot.revisions) { + const changed = + yield* sql`SELECT 1 WHERE ${revision.sequence} <> (SELECT MAX(sequence) FROM orchestration_events WHERE aggregate_kind = 'thread' AND application_event_version = 2 AND stream_id = ${revision.thread_id})`; + if (changed.length) return yield* new ArchiveError({ reason: "changed" }); + } + const metadata: typeof Metadata.Type = snapshot.segments.map((segment) => ({ + ...segment, + rows: segment.rows.map((row) => ({ + ...row, + payload_json: metadataPayload(segment.table, String(row.payload_json)), + })), + })); + yield* sql`INSERT INTO transcript_archives(thread_id, location, object_hash, through_sequence, metadata_json) VALUES(${threadId}, ${location}, ${receipt.hash}, ${snapshot.throughSequence}, ${encodeMetadata(metadata)})`; + for (const segment of metadata) + for (const row of segment.rows) + yield* sql`UPDATE ${sql(`orchestration_v2_projection_${segment.table}`)} SET payload_json = ${row.payload_json} WHERE ${sql(ids[segment.table])} = ${row[ids[segment.table]]} AND thread_id = ${threadId}`; + yield* sql`UPDATE orchestration_events SET payload_json = json_set(json_remove(payload_json, '$.promptSequence'), '$.prompt', (SELECT json_extract(a.payload_json, '$.prompt') FROM orchestration_events a WHERE a.sequence = json_extract(orchestration_events.payload_json, '$.promptSequence'))) WHERE json_type(payload_json, '$.promptSequence') = 'integer' AND json_extract(payload_json, '$.promptSequence') IN (SELECT sequence FROM orchestration_events WHERE stream_id = ${threadId})`; + yield* sql`UPDATE orchestration_events SET payload_json = 'null' WHERE stream_id = ${threadId} AND application_event_version = 2 AND aggregate_kind = 'thread' AND sequence <= ${snapshot.throughSequence} AND event_type IN ('message.updated', 'turn-item.updated', 'subagent.updated')`; + for (const asset of assets) + yield* sql`INSERT INTO transcript_archive_assets(thread_id, path, location, object_hash) VALUES(${threadId}, ${asset.path}, ${location}, ${asset.hash}) ON CONFLICT(thread_id, path) DO UPDATE SET location = excluded.location, object_hash = excluded.object_hash`; + yield* sql`DELETE FROM transcript_archive_failures WHERE thread_id = ${threadId}`; + yield* sql`UPDATE transcript_archive_reclamation SET pending = 1 WHERE singleton = 1`; + }), + ); + }, Effect.mapError(archiveError)); + const evictAttachments = sql.withTransaction( + Effect.gen(function* () { + const assets = yield* sql<{ + thread_id: ThreadId; + path: string; + }>`SELECT a.thread_id, a.path FROM transcript_archive_assets a JOIN transcript_archives c ON c.thread_id = a.thread_id`; + const references = yield* sql<{ + id: string; + }>`SELECT DISTINCT json_extract(a.value, '$.id') AS id FROM orchestration_v2_projection_messages m JOIN orchestration_v2_projection_threads t ON t.thread_id = m.thread_id, json_each(m.payload_json, '$.attachments') a WHERE t.archived_at IS NULL AND t.deleted_at IS NULL`; + const warmSources = yield* sql<{ thread_id: string }>`WITH RECURSIVE sources(thread_id) AS ( + SELECT thread_id FROM orchestration_v2_projection_threads WHERE archived_at IS NULL AND deleted_at IS NULL + UNION SELECT json_extract(t.payload_json, '$.forkedFrom.threadId') FROM orchestration_v2_projection_threads t JOIN sources s ON s.thread_id = t.thread_id WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run' + ) SELECT thread_id FROM sources`; + const referenced = new Set(references.map((row) => row.id)); + const warm = new Set(warmSources.map((row) => row.thread_id)); + for (const asset of assets) { + const relative = path.relative(config.attachmentsDir, asset.path); + if (relative.startsWith("..") || path.isAbsolute(relative) || warm.has(asset.thread_id)) + continue; + const id = parseAttachmentIdFromRelativePath(relative); + if ( + !id || + referenced.has(id) || + parseThreadSegmentFromAttachmentId(id) !== toSafeThreadAttachmentSegment(asset.thread_id) + ) + continue; + yield* fs.remove(asset.path, { force: true }); + } + }), + ); + const sweep = Effect.gen(function* () { + if (!(yield* client.enabled)) return; + const environmentId = yield* identity.getEnvironmentId; + const now = yield* Clock.currentTimeMillis; + const attempt = (threadId: ThreadId, operation: Effect.Effect) => + operation.pipe( + Effect.andThen(sql`DELETE FROM transcript_archive_failures WHERE thread_id = ${threadId}`), + Effect.catchTags({ + ArchiveError: (error) => + error.reason === "changed" + ? Effect.void + : sql`INSERT INTO transcript_archive_failures(thread_id, attempts, reason, next_attempt_at) VALUES(${threadId}, 1, ${error.reason}, ${now + 60_000}) ON CONFLICT(thread_id) DO UPDATE SET attempts = attempts + 1, reason = excluded.reason, next_attempt_at = ${now} + MIN(3600000, 60000 * (1 << MIN(attempts, 6)))`.pipe( + Effect.andThen( + Effect.logWarning("Transcript archive sync will retry", { + threadId, + reason: error.reason, + }), + ), + ), + }), + ); + const unpublished = yield* sql<{ + thread_id: ThreadId; + object_hash: string; + location: string; + }>`SELECT a.thread_id, a.object_hash, a.location FROM transcript_archives a LEFT JOIN transcript_archive_publications p ON p.thread_id = a.thread_id LEFT JOIN transcript_archive_failures f ON f.thread_id = a.thread_id WHERE p.thread_id IS NULL AND COALESCE(f.next_attempt_at, 0) <= ${now} LIMIT 10`; + for (const entry of unpublished) { + if (entry.location !== (yield* client.location)) continue; + yield* attempt( + entry.thread_id, + Effect.gen(function* () { + yield* publish(entry.thread_id, entry.object_hash); + yield* sql`INSERT INTO transcript_archive_publications(thread_id, location, object_hash) VALUES(${entry.thread_id}, ${entry.location}, ${entry.object_hash}) ON CONFLICT(thread_id) DO UPDATE SET location = excluded.location, object_hash = excluded.object_hash`; + }).pipe(Effect.mapError(archiveError)), + ); + } + const warm = yield* sql<{ + thread_id: ThreadId; + location: string; + object_hash: string; + }>`SELECT p.thread_id, p.location, p.object_hash FROM transcript_archive_publications p JOIN orchestration_v2_projection_threads t ON t.thread_id = p.thread_id LEFT JOIN transcript_archive_failures f ON f.thread_id = p.thread_id WHERE (t.archived_at IS NULL OR t.deleted_at IS NOT NULL) AND COALESCE(f.next_attempt_at, 0) <= ${now}`; + for (const entry of warm) { + if (entry.location !== (yield* client.location)) continue; + yield* attempt( + entry.thread_id, + Effect.gen(function* () { + yield* client.withdraw(environmentId, entry.thread_id, entry.object_hash); + yield* sql`DELETE FROM transcript_archive_publications WHERE thread_id = ${entry.thread_id} AND object_hash = ${entry.object_hash}`; + }).pipe(Effect.mapError(archiveError)), + ); + } + const candidates = yield* sql<{ + thread_id: ThreadId; + }>`SELECT t.thread_id FROM orchestration_v2_projection_threads t LEFT JOIN transcript_archive_failures f ON f.thread_id = t.thread_id WHERE t.archived_at IS NOT NULL AND t.deleted_at IS NULL AND COALESCE(f.next_attempt_at, 0) <= ${now} AND NOT EXISTS (SELECT 1 FROM transcript_archives a WHERE a.thread_id = t.thread_id) AND NOT EXISTS (SELECT 1 FROM (${activeThreads}) active WHERE active.thread_id = t.thread_id) ORDER BY t.archived_at LIMIT 10`; + for (const row of candidates) yield* attempt(row.thread_id, offload(row.thread_id)); + yield* evictAttachments; + yield* storage.reclaim; + }).pipe(Effect.mapError(archiveError)); + const queuedThreads = sql`SELECT t.thread_id FROM orchestration_v2_projection_threads t LEFT JOIN transcript_archives a ON a.thread_id = t.thread_id LEFT JOIN transcript_archive_publications p ON p.thread_id = t.thread_id WHERE (t.archived_at IS NOT NULL AND t.deleted_at IS NULL AND (a.thread_id IS NULL OR p.thread_id IS NULL)) OR ((t.archived_at IS NULL OR t.deleted_at IS NOT NULL) AND p.thread_id IS NOT NULL)`; + const manage = Effect.fn("TranscriptArchive.manage")(function* ( + request: ArchiveManagementRequest, + ) { + if (request.type === "archive.retry") + yield* sql`DELETE FROM transcript_archive_failures ${request.threadId ? sql`WHERE thread_id = ${request.threadId}` : sql``}`; + const counts = yield* sql<{ + offloaded: number; + pending: number; + }>`SELECT (SELECT COUNT(*) FROM transcript_archives) AS offloaded, (SELECT COUNT(*) FROM (${queuedThreads})) AS pending`; + const failures = yield* sql<{ + threadId: ThreadId; + attempts: number; + reason: string; + }>`SELECT thread_id AS threadId, attempts, reason FROM transcript_archive_failures ORDER BY thread_id`; + return { + enabled: yield* client.enabled, + queue: yield* sql<{ + threadId: ThreadId; + nextAttemptAt: number; + }>`SELECT t.thread_id AS threadId, COALESCE(f.next_attempt_at, 0) AS nextAttemptAt FROM (${queuedThreads}) t LEFT JOIN transcript_archive_failures f ON f.thread_id = t.thread_id ORDER BY nextAttemptAt, t.thread_id`, + storage: yield* storage.inspect, + lastReclamation: + (yield* sql<{ + beforeAllocatedBytes: number; + afterAllocatedBytes: number; + }>`SELECT before_allocated_bytes AS beforeAllocatedBytes, after_allocated_bytes AS afterAllocatedBytes FROM transcript_archive_reclamation WHERE singleton = 1 AND completed_at IS NOT NULL`)[0] ?? + null, + offloaded: counts[0]?.offloaded ?? 0, + pending: counts[0]?.pending ?? 0, + failures, + }; + }, Effect.mapError(archiveError)); + return TranscriptArchive.of({ + enabled: client.enabled, + manage, + offload, + restore, + read, + readStream: (id, stream) => + Stream.unwrap( + entries(id).pipe( + Effect.mapError(archiveError), + Effect.flatMap((found) => + found.length === 0 + ? Effect.succeed(stream) + : read(id, Stream.runCollect(stream)).pipe(Effect.map(Stream.fromIterable)), + ), + ), + ), + sweep, + asset: (paths) => + sql<{ + path: string; + location: string; + object_hash: string; + }>`SELECT path, location, object_hash FROM transcript_archive_assets WHERE path IN ${sql.in(paths)} ORDER BY rowid DESC LIMIT 1`.pipe( + Effect.map((rows) => + Option.fromUndefinedOr(rows[0]).pipe( + Option.map((row) => ({ + path: row.path, + location: row.location, + hash: row.object_hash, + })), + ), + ), + Effect.mapError(archiveError), + ), + }); +}); + +export const layer = Layer.effect(TranscriptArchive, make).pipe(Layer.provide(ProjectStore.layer)); + +/** Rebuild cold timeline indexes locally; no NAS request or transcript body is needed. */ +export const rebuildMetadata = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ + metadata_json: string; + }>`SELECT metadata_json FROM transcript_archives ORDER BY through_sequence`; + for (const entry of rows) + for (const segment of yield* decodeMetadata(entry.metadata_json)) { + for (const row of segment.rows) + yield* sql`INSERT INTO ${sql(`orchestration_v2_projection_${segment.table}`)} ${sql.insert(row)} ON CONFLICT DO NOTHING`; + } +}); diff --git a/apps/server/src/archive/runtimeLayer.ts b/apps/server/src/archive/runtimeLayer.ts new file mode 100644 index 0000000000000000000000000000000000000000..7f93c9ef491fb4cc3f768f125a7402be65cbf15e --- /dev/null +++ b/apps/server/src/archive/runtimeLayer.ts @@ -0,0 +1,16 @@ +import * as Layer from "effect/Layer"; +import * as SharedServer from "../environment/SharedServer.ts"; +import * as ArchiveObjects from "./ArchiveObjects.ts"; +import * as ArchiveClient from "./ArchiveClient.ts"; +import * as ArchiveCatalog from "./ArchiveCatalog.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ArchiveStorage from "./ArchiveStorage.ts"; +import * as TranscriptArchive from "./TranscriptArchive.ts"; + +export const layer = TranscriptArchive.layer.pipe( + Layer.provideMerge(ArchiveClient.layer.pipe(Layer.provideMerge(ArchiveCatalog.layer))), + Layer.provideMerge(ArchiveObjects.layer), + Layer.provideMerge(ArchiveStorage.layer), + Layer.provideMerge(SharedServer.layer), + Layer.provideMerge(ServerEnvironment.layerIdentity), +); diff --git a/apps/server/src/assets/AssetAccess.test.ts b/apps/server/src/assets/AssetAccess.test.ts index 9f4f16722ea8e84e30c8418fd7589a7ec4f8d7d4..306eb98daf24acf6313d0235b1f3125669800537 100644 --- a/apps/server/src/assets/AssetAccess.test.ts +++ b/apps/server/src/assets/AssetAccess.test.ts @@ -26,6 +26,8 @@ import { vi } from "vite-plus/test"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; +import * as TranscriptArchive from "../archive/TranscriptArchive.ts"; +import * as Option from "effect/Option"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts"; @@ -106,6 +108,10 @@ const oversizedScreenshotItem = { }; const layerTest = Layer.mergeAll( + Layer.mock(TranscriptArchive.TranscriptArchive)({ + enabled: Effect.succeed(false), + asset: () => Effect.succeed(Option.none()), + }), NodeHttpPlatform.layer, Layer.mock(Orchestrator.OrchestratorV2)({ getTurnItem: ({ itemId }) => diff --git a/apps/server/src/assets/AssetAccess.ts b/apps/server/src/assets/AssetAccess.ts index 54ee6d605749ccd7b7e61869d0beb87cee7ce70b..ebd183f860836fe10e029181db7e25a5685d8973 100644 --- a/apps/server/src/assets/AssetAccess.ts +++ b/apps/server/src/assets/AssetAccess.ts @@ -55,8 +55,13 @@ import { timingSafeEqualBase64Url, } from "../auth/utils.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts"; +import { + attachmentPathsById, + parseAttachmentFileExtension, + resolveAttachmentPathById, +} from "../attachmentStore.ts"; import * as ServerConfig from "../config.ts"; +import * as TranscriptArchive from "../archive/TranscriptArchive.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; @@ -96,7 +101,20 @@ const PREVIEW_ASSET_EXTENSIONS = new Set([ ".woff2", ]); +const ArchivedFileClaims = Schema.Struct({ + version: Schema.Literal(1), + kind: Schema.Literal("archived-file"), + path: Schema.String, + location: Schema.String, + hash: Schema.String, + fileName: Schema.String, + mimeType: Schema.String, + download: Schema.Boolean, + renderIntent: Schema.optionalKey(Schema.Literal("mcp-app")), + expiresAt: Schema.Number, +}); const AssetClaimsSchema = Schema.Union([ + ArchivedFileClaims, Schema.Struct({ version: Schema.Literal(1), kind: Schema.Literal("workspace-file"), @@ -176,6 +194,7 @@ const decodeAssetClaims = Schema.decodeUnknownOption(AssetClaimsJson); const encodeAssetClaims = Schema.encodeSync(AssetClaimsJson); export type ResolvedAsset = + | ({ readonly kind: "archived-file" } & Omit) | { readonly kind: "file"; readonly path: string; @@ -511,6 +530,72 @@ const finalizeWorkspaceFileAsset = Effect.fn("AssetAccess.finalizeWorkspaceFileA }, ); +const archivedFile = Effect.fn("AssetAccess.archivedFile")(function* ( + paths: ReadonlyArray, + options: { + readonly expiresAt: number; + readonly fileName?: string | undefined; + readonly mimeType?: string | undefined; + readonly download?: boolean | undefined; + readonly renderIntent?: "mcp-app" | undefined; + }, +) { + const archive = yield* TranscriptArchive.TranscriptArchive; + const path = yield* Path.Path; + const found = yield* archive.asset(paths); + if (Option.isNone(found)) return null; + const fileName = options.fileName ?? path.basename(found.value.path); + return { + ...found.value, + version: 1 as const, + kind: "archived-file" as const, + fileName, + mimeType: + options.mimeType ?? + hostPreviewMimeTypeFromExtension(path.extname(fileName)) ?? + "application/octet-stream", + download: options.download ?? false, + ...(options.renderIntent ? { renderIntent: options.renderIntent } : {}), + expiresAt: options.expiresAt, + }; +}); + +const issueClaimUrl = Effect.fn("AssetAccess.issueClaimUrl")(function* ( + resource: AssetResource, + initialClaims: AssetClaims, + fileName: string, + sourcePath: string | undefined, + imageDimensions: ImageDimensions | null, +) { + let claims = initialClaims; + let expiresAt = claims.expiresAt; + const secretStore = yield* ServerSecretStore.ServerSecretStore; + const signingSecret = yield* secretStore.getOrCreateRandom(SIGNING_SECRET_NAME, 32).pipe( + Effect.mapError( + (cause) => + new AssetSigningKeyLoadError({ + resource, + cause, + }), + ), + ); + if (claims.kind === "project-favicon" || claims.kind === "project-favicon-external") { + const issuedAt = yield* Clock.currentTimeMillis; + expiresAt = + (Math.floor(issuedAt / PROJECT_FAVICON_TOKEN_BUCKET_MS) + 2) * + PROJECT_FAVICON_TOKEN_BUCKET_MS; + claims = { ...claims, expiresAt }; + } + const encodedPayload = base64UrlEncode(encodeAssetClaims(claims)); + const token = `${encodedPayload}.${signPayload(encodedPayload, signingSecret)}`; + return { + relativeUrl: `${ASSET_ROUTE_PREFIX}/${token}/${encodeURIComponent(fileName)}`, + expiresAt, + ...(sourcePath !== undefined ? { sourcePath } : {}), + ...(imageDimensions !== null ? { imageDimensions } : {}), + }; +}); + export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (input: { readonly resource: AssetResource; readonly workspaceRoot?: string; @@ -527,6 +612,68 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i let sourcePath: string | undefined; let imageDimensions: ImageDimensions | null = null; + const resource = input.resource; + let archivePaths: ReadonlyArray = []; + if (resource._tag === "attachment") { + const input = { + attachmentsDir: (yield* ServerConfig.ServerConfig).attachmentsDir, + attachmentId: resource.attachmentId, + }; + if (!resolveAttachmentPathById(input)) archivePaths = attachmentPathsById(input); + } else if (resource._tag === "media-file" || resource._tag === "workspace-file") { + const expanded = expandHomePath(resource.path, yield* HostProcess.HomeDirectory); + if (path.isAbsolute(expanded)) archivePaths = [expanded]; + else if (input.workspaceRoot) archivePaths = [path.resolve(input.workspaceRoot, expanded)]; + } + const present = + resource._tag === "attachment" + ? [] + : yield* Effect.forEach(archivePaths, (file) => + fileSystem + .exists(file) + .pipe( + Effect.mapError( + (cause) => new AssetWorkspaceAssetInspectionError({ resource, cause }), + ), + ), + ); + const archived = + archivePaths.length > 0 && !present.some(Boolean) + ? yield* (yield* TranscriptArchive.TranscriptArchive) + .asset(archivePaths) + .pipe( + Effect.mapError((cause) => new AssetWorkspaceAssetInspectionError({ resource, cause })), + ) + : Option.none(); + if (Option.isSome(archived)) { + const archivedPath = archived.value.path; + fileName = + resource._tag === "attachment" + ? (resource.fileName ?? path.basename(archivedPath)) + : path.basename(archivedPath); + const mimeType = + (resource._tag === "attachment" ? resource.mimeType : undefined) ?? + hostPreviewMimeTypeFromExtension(path.extname(fileName)) ?? + "application/octet-stream"; + claims = { + version: 1, + kind: "archived-file", + path: archivedPath, + location: archived.value.location, + hash: archived.value.hash, + fileName, + mimeType, + download: + resource._tag === "attachment" && + resource.disposition !== "inline" && + !/^(image|video)\//u.test(mimeType), + ...(resource._tag === "attachment" && resource.renderIntent === "mcp-app" + ? { renderIntent: "mcp-app" as const } + : {}), + expiresAt, + }; + return yield* issueClaimUrl(resource, claims, fileName, archivedPath, null); + } switch (input.resource._tag) { case "media-file": { let requestedPath = expandHomePath(input.resource.path, yield* HostProcess.HomeDirectory); @@ -819,31 +966,7 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i } } - const secretStore = yield* ServerSecretStore.ServerSecretStore; - const signingSecret = yield* secretStore.getOrCreateRandom(SIGNING_SECRET_NAME, 32).pipe( - Effect.mapError( - (cause) => - new AssetSigningKeyLoadError({ - resource: input.resource, - cause, - }), - ), - ); - if (claims.kind === "project-favicon" || claims.kind === "project-favicon-external") { - const issuedAt = yield* Clock.currentTimeMillis; - expiresAt = - (Math.floor(issuedAt / PROJECT_FAVICON_TOKEN_BUCKET_MS) + 2) * - PROJECT_FAVICON_TOKEN_BUCKET_MS; - claims = { ...claims, expiresAt }; - } - const encodedPayload = base64UrlEncode(encodeAssetClaims(claims)); - const token = `${encodedPayload}.${signPayload(encodedPayload, signingSecret)}`; - return { - relativeUrl: `${ASSET_ROUTE_PREFIX}/${token}/${encodeURIComponent(fileName)}`, - expiresAt, - ...(sourcePath !== undefined ? { sourcePath } : {}), - ...(imageDimensions !== null ? { imageDimensions } : {}), - }; + return yield* issueClaimUrl(input.resource, claims, fileName, sourcePath, imageDimensions); }); export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( @@ -864,13 +987,25 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( const claims = decodeClaims(encodedPayload); if (!claims || claims.expiresAt <= (yield* Clock.currentTimeMillis)) return null; + if (claims.kind === "archived-file") { + if (decodeRelativePath(relativePath) !== claims.fileName) return null; + return { ...claims, kind: "archived-file" } satisfies ResolvedAsset; + } + if (claims.kind === "attachment") { const config = yield* ServerConfig.ServerConfig; const attachmentPath = resolveAttachmentPathById({ attachmentsDir: config.attachmentsDir, attachmentId: claims.attachmentId, }); - if (!attachmentPath) return null; + if (!attachmentPath) + return yield* archivedFile( + attachmentPathsById({ + attachmentsDir: config.attachmentsDir, + attachmentId: claims.attachmentId, + }), + claims, + ).pipe(Effect.orElseSucceed(() => null)); const fileSystem = yield* FileSystem.FileSystem; const info = yield* optionOnNotFound(fileSystem.stat(attachmentPath)).pipe( Effect.tapError((cause) => @@ -959,6 +1094,10 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( ), Effect.orElseSucceed(() => null), ); + if (canonicalFile === null) + return yield* archivedFile([claims.filePath], { expiresAt: claims.expiresAt }).pipe( + Effect.orElseSucceed(() => null), + ); if (canonicalFile !== claims.filePath) return null; const isIcns = canonicalFile.toLowerCase().endsWith(".icns"); const mimeType = isIcns @@ -982,7 +1121,11 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( workspaceRoot: claims.workspaceRoot, relativePath: claims.relativePath, }); - return exactWorkspaceFile ? yield* imagePreviewAsset(exactWorkspaceFile) : null; + return exactWorkspaceFile + ? yield* imagePreviewAsset(exactWorkspaceFile) + : yield* archivedFile([path.resolve(claims.workspaceRoot, claims.relativePath)], { + expiresAt: claims.expiresAt, + }).pipe(Effect.orElseSucceed(() => null)); } const segments = decodedPath.split(/[\\/]/); if ( @@ -999,5 +1142,9 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( workspaceRoot: claims.workspaceRoot, relativePath: joinedRelativePath, }); - return workspaceFile ? ({ kind: "file", path: workspaceFile } satisfies ResolvedAsset) : null; + return workspaceFile + ? ({ kind: "file", path: workspaceFile } satisfies ResolvedAsset) + : yield* archivedFile([path.resolve(claims.workspaceRoot, joinedRelativePath)], { + expiresAt: claims.expiresAt, + }).pipe(Effect.orElseSucceed(() => null)); }); diff --git a/apps/server/src/attachmentStore.ts b/apps/server/src/attachmentStore.ts index 29275dc55d76b2d6ffb2bffd5b3ace35e5afab8e..38bd81d5106c09a77f517320b32aaa4b670e7626 100644 --- a/apps/server/src/attachmentStore.ts +++ b/apps/server/src/attachmentStore.ts @@ -182,32 +182,30 @@ export function resolveAttachmentPath(input: { }); } -export function resolveAttachmentPathById(input: { +export function attachmentPathsById(input: { readonly attachmentsDir: string; readonly attachmentId: string; -}): string | null { +}): ReadonlyArray { const normalizedId = normalizeAttachmentRelativePath(input.attachmentId); - if (!normalizedId || normalizedId.includes("/") || normalizedId.includes(".")) { - return null; - } + if (!normalizedId || normalizedId.includes("/") || normalizedId.includes(".")) return []; const fileExtension = parseAttachmentFileExtension(normalizedId); - if (fileExtension) { - const filePath = resolveAttachmentRelativePath({ - attachmentsDir: input.attachmentsDir, - relativePath: `${normalizedId}.${fileExtension.toLowerCase()}`, - }); - return filePath && NodeFS.existsSync(filePath) ? filePath : null; - } - for (const extension of ATTACHMENT_FILENAME_EXTENSIONS) { - const maybePath = resolveAttachmentRelativePath({ + const extensions = fileExtension + ? [`.${fileExtension.toLowerCase()}`] + : ATTACHMENT_FILENAME_EXTENSIONS; + return extensions.flatMap((extension) => { + const file = resolveAttachmentRelativePath({ attachmentsDir: input.attachmentsDir, relativePath: `${normalizedId}${extension}`, }); - if (maybePath && NodeFS.existsSync(maybePath)) { - return maybePath; - } - } - return null; + return file ? [file] : []; + }); +} + +export function resolveAttachmentPathById(input: { + readonly attachmentsDir: string; + readonly attachmentId: string; +}): string | null { + return attachmentPathsById(input).find((file) => NodeFS.existsSync(file)) ?? null; } export type AttachmentClaimPlan = @@ -310,7 +308,7 @@ export function sweepStalePendingAttachments(input: { return { deleted }; } -function parseAttachmentIdFromRelativePath(relativePath: string): string | null { +export function parseAttachmentIdFromRelativePath(relativePath: string): string | null { const normalized = normalizeAttachmentRelativePath(relativePath); if (!normalized || normalized.includes("/")) { return null; diff --git a/apps/server/src/auth/EnvironmentAuth.test.ts b/apps/server/src/auth/EnvironmentAuth.test.ts index c25b35a61cc09e4b307b3d608ca101cf735a51cb..476e5443e9253b0122fd87625a6eafa22b9c6047 100644 --- a/apps/server/src/auth/EnvironmentAuth.test.ts +++ b/apps/server/src/auth/EnvironmentAuth.test.ts @@ -708,4 +708,39 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { expect(verified.sessionId).toBe(bearer.sessionId); }).pipe(Effect.provide(layerEnvironmentAuth({ mode: "web", host: "192.168.1.50" }))), ); + it.effect("delegates only the parent grant and rejects access after its revocation", () => + Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const parent = yield* auth.issueSession(); + const issued = yield* auth.issueDelegatedSession(parent, { + scopes: ["orchestration:read", "access:write"], + }); + const child = yield* auth.authenticateHttpRequest(makeBearerRequest(issued.access_token)); + expect(child.parentSessionId).toBe(parent.sessionId); + expect(issued.token_type).toBe("Bearer"); + expect(issued.expires_in).toBe(3600); + expect( + (yield* auth + .issueDelegatedSession(child, { scopes: ["orchestration:read"] }) + .pipe(Effect.flip))._tag, + ).toBe("ServerAuthScopeNotGrantedError"); + const limited = yield* auth.issueSession({ scopes: ["access:write", "orchestration:read"] }); + expect( + (yield* auth + .issueDelegatedSession(limited, { scopes: ["orchestration:operate"] }) + .pipe(Effect.flip))._tag, + ).toBe("ServerAuthScopeNotGrantedError"); + expect( + (yield* auth + .issueDelegatedSession(parent, { scopes: ["orchestration:read", "orchestration:read"] }) + .pipe(Effect.flip))._tag, + ).toBe("ServerAuthInvalidScopeError"); + yield* auth.revokeSession(parent.sessionId); + expect( + (yield* auth + .authenticateHttpRequest(makeBearerRequest(issued.access_token)) + .pipe(Effect.flip))._tag, + ).toBe("ServerAuthInvalidCredentialError"); + }).pipe(Effect.provide(layerEnvironmentAuth())), + ); }); diff --git a/apps/server/src/auth/EnvironmentAuth.ts b/apps/server/src/auth/EnvironmentAuth.ts index 8a59f22fca9f9bada6443dbd0c57ba1a845f7d7e..c944202d30488ce73df364e05cabd88216bf23c6 100644 --- a/apps/server/src/auth/EnvironmentAuth.ts +++ b/apps/server/src/auth/EnvironmentAuth.ts @@ -12,6 +12,7 @@ import { type AuthClientMetadata, type AuthClientSession, type AuthCreatePairingCredentialInput, + type AuthCreateDelegatedSessionInput, type AuthEnvironmentScope, type AuthMcpClientAccess, type AuthPairingLink, @@ -46,6 +47,8 @@ import { REUSABLE_DEV_SESSION_EXPIRES_AT, resolveReusableDevAuth } from "./Reusa import { verifyRequestDpopProof } from "./dpop.ts"; import { DESKTOP_RENDERER_ORIGINS } from "../httpCors.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as HostedAuthConfig from "./HostedAuthConfig.ts"; +import { signPayload, timingSafeEqualBase64Url } from "./utils.ts"; const DEFAULT_SESSION_SUBJECT = "cli-issued-session"; export const INTERNAL_ADMINISTRATIVE_BOOTSTRAP_SUBJECT = "administrative-bootstrap"; @@ -106,6 +109,8 @@ export interface McpClientSession { } export interface AuthenticatedSession { + readonly parentSessionId?: AuthSessionId; + readonly deviceId?: string; readonly sessionId: AuthSessionId; readonly subject: string; readonly method: ServerAuthSessionMethod; @@ -501,6 +506,13 @@ export class EnvironmentAuth extends Context.Service< AuthAccessTokenResult, ServerAuthInvalidCredentialError | ServerAuthInvalidRequestError | ServerAuthInternalError >; + readonly issueDelegatedSession: ( + parent: AuthenticatedSession, + input: AuthCreateDelegatedSessionInput, + ) => Effect.Effect< + AuthAccessTokenResult, + ServerAuthInvalidRequestError | ServerAuthInternalError + >; readonly createPairingLink: (input?: { readonly ttl?: Duration.Duration; readonly label?: string; @@ -738,6 +750,9 @@ export const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const descriptor = yield* policy.getDescriptor(); const config = yield* ServerConfig.ServerConfig; + const hostedConfiguration = Option.flatten( + yield* Effect.serviceOption(HostedAuthConfig.HostedAuthConfig), + ); const devAuth = resolveReusableDevAuth(config); const authenticateToken = ( @@ -760,6 +775,10 @@ export const make = Effect.gen(function* () { Effect.flatMap(rejectMcpClientAudience), Effect.map((session) => ({ sessionId: session.sessionId, + ...(session.parentSessionId === undefined + ? {} + : { parentSessionId: session.parentSessionId }), + ...(session.deviceId === undefined ? {} : { deviceId: session.deviceId }), subject: session.subject, method: session.method, scopes: session.scopes, @@ -769,6 +788,14 @@ export const make = Effect.gen(function* () { ); const resolveRequestCredential = (request: HttpServerRequest.HttpServerRequest) => { + const hostedCookie = request.cookies[HostedAuthConfig.HOSTED_SESSION_COOKIE]; + if ( + Option.isSome(hostedConfiguration) && + hostedCookie !== undefined && + request.headers.authorization === undefined + ) { + return { token: hostedCookie, source: "cookie" as const }; + } const selectedCredential = selectRequestCredential( request, sessions.cookieName, @@ -788,6 +815,7 @@ export const make = Effect.gen(function* () { // site (another localhost port, a LAN IP's other ports) still get the cookie, // so cookie sessions must not authenticate their requests or sockets. const trustedBrowserOrigins = new Set([ + ...(Option.isSome(hostedConfiguration) ? [hostedConfiguration.value.publicUrl] : []), ...(config.devUrl ? [config.devUrl.origin] : []), ...DESKTOP_RENDERER_ORIGINS, ...config.devAllowedOrigins, @@ -812,6 +840,48 @@ export const make = Effect.gen(function* () { ); } return authenticateToken(credential.token).pipe( + Effect.tap( + Effect.fnUntraced(function* (session) { + if (!session.deviceId || !isBrowserCookieCredential(credential.source)) return; + const origin = Option.isSome(hostedConfiguration) + ? hostedConfiguration.value.publicUrl + : undefined; + const unsafe = !["GET", "HEAD", "OPTIONS"].includes(request.method); + const websocket = request.headers.upgrade?.toLowerCase() === "websocket"; + if ( + !origin || + ((unsafe || websocket || request.headers.origin !== undefined) && + request.headers.origin !== origin) + ) { + return yield* new ServerAuthInvalidCredentialError({ + diagnostic: "Hosted browser sessions require the configured origin.", + }); + } + if (unsafe) { + const secret = yield* secretStore + .getOrCreateRandom(HostedAuthConfig.HOSTED_CSRF_SECRET, 32) + .pipe( + Effect.mapError( + (cause) => + new ServerAuthInvalidCredentialError({ + diagnostic: "Browser request verification is unavailable.", + cause, + }), + ), + ); + if ( + !timingSafeEqualBase64Url( + request.headers["x-csrf-token"] ?? "", + signPayload(session.deviceId, secret), + ) + ) { + return yield* new ServerAuthInvalidCredentialError({ + diagnostic: "Browser request verification failed.", + }); + } + } + }), + ), Effect.flatMap((session) => { if (session.proofKeyThumbprint) { if (!dpopToken || dpopToken !== credential.token) { @@ -1032,6 +1102,43 @@ export const make = Effect.gen(function* () { ); }; + const issueDelegatedSession: EnvironmentAuth["Service"]["issueDelegatedSession"] = Effect.fn( + "EnvironmentAuth.issueDelegatedSession", + )(function* (parent, input) { + if (parent.parentSessionId !== undefined || !parent.scopes.includes(AuthAccessWriteScope)) { + return yield* new ServerAuthScopeNotGrantedError({}); + } + if (input.scopes.length === 0 || new Set(input.scopes).size !== input.scopes.length) { + return yield* new ServerAuthInvalidScopeError({}); + } + if (input.scopes.some((scope) => !parent.scopes.includes(scope))) { + return yield* new ServerAuthScopeNotGrantedError({}); + } + const issued = yield* sessions + .issue({ + parentSessionId: parent.sessionId, + subject: "hub-delegated-session", + method: "bearer-access-token", + ttl: Duration.hours(1), + scopes: input.scopes, + client: { deviceType: "bot", ...(input.label ? { label: input.label } : {}) }, + }) + .pipe( + Effect.mapError((cause) => new ServerAuthAuthenticatedAccessTokenIssueError({ cause })), + ); + const now = yield* DateTime.now; + return { + access_token: issued.token, + issued_token_type: AuthAccessTokenType, + token_type: "Bearer", + expires_in: Math.max( + 0, + Math.floor((issued.expiresAt.epochMilliseconds - now.epochMilliseconds) / 1000), + ), + scope: encodeOAuthScope(issued.scopes), + } satisfies AuthAccessTokenResult; + }); + const issuePairingCredentialForSubject = (input: { readonly scopes: ReadonlyArray; readonly subject: string; @@ -1259,6 +1366,10 @@ export const make = Effect.gen(function* () { Effect.flatMap(rejectMcpClientAudience), Effect.map((session) => ({ sessionId: session.sessionId, + ...(session.parentSessionId === undefined + ? {} + : { parentSessionId: session.parentSessionId }), + ...(session.deviceId === undefined ? {} : { deviceId: session.deviceId }), subject: session.subject, method: session.method, scopes: session.scopes, @@ -1355,6 +1466,7 @@ export const make = Effect.gen(function* () { getSessionState, createBrowserSession, exchangeBootstrapCredentialForAccessToken, + issueDelegatedSession, createPairingLink, issuePairingCredential, issueStartupPairingCredential, diff --git a/apps/server/src/auth/HostedAuth.test.ts b/apps/server/src/auth/HostedAuth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..5865f34a3cb9f250df441acc4b0ab3fa1116faeb --- /dev/null +++ b/apps/server/src/auth/HostedAuth.test.ts @@ -0,0 +1,696 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { EnvironmentId, HostedAuthAccount } from "@t3tools/contracts"; +import { exportJWK, generateKeyPair, SignJWT, type JWTPayload } from "jose"; +import * as Crypto from "effect/Crypto"; +import * as Base64Url from "effect/encoding/Base64Url"; +import * as DateTime from "effect/DateTime"; +import * as FileSystem from "effect/FileSystem"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpClient, HttpClientResponse, HttpRouter, HttpServerRequest } from "effect/http"; +import * as ServerConfig from "../config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as EnvironmentAuth from "./EnvironmentAuth.ts"; +import * as HostedAuth from "./HostedAuth.ts"; +import * as HostedAuthConfig from "./HostedAuthConfig.ts"; +import * as HostedAuthHttp from "./HostedAuthHttp.ts"; +import * as ServerSecretStore from "./ServerSecretStore.ts"; +import * as SessionStore from "./SessionStore.ts"; + +const settings = { + publicUrl: "https://knife.example", + issuer: "https://issuer.example", + clientId: "knife-test", + clientSecret: Redacted.make("test-client-secret"), + ownerSub: "configured-owner", + maxInteractiveAgeSeconds: 86_400, + issuerRevalidationSeconds: 60, +}; +const client = { label: "Test computer", deviceType: "desktop" } as const; +const verifier = "v".repeat(43); +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +const fixture = Effect.gen(function* () { + const key = yield* Effect.promise(() => generateKeyPair("RS256")); + const jwk = yield* Effect.promise(() => exportJWK(key.publicKey)); + let nonce = ""; + let claims: JWTPayload = {}; + let refreshDenied = false; + let refreshCalls = 0; + let refreshGate: + | { entered: Deferred.Deferred; release: Deferred.Deferred } + | undefined; + let lastTokenBody: URLSearchParams | undefined; + const http = HttpClient.make((request) => + Effect.gen(function* () { + const url = new URL(request.url); + if (url.pathname.endsWith("openid-configuration")) + return HttpClientResponse.fromWeb( + request, + Response.json({ + issuer: settings.issuer, + authorization_endpoint: `${settings.issuer}/authorize`, + token_endpoint: `${settings.issuer}/token`, + jwks_uri: `${settings.issuer}/jwks`, + code_challenge_methods_supported: ["S256"], + response_types_supported: ["code"], + token_endpoint_auth_methods_supported: ["client_secret_basic"], + id_token_signing_alg_values_supported: ["RS256"], + }), + ); + if (url.pathname === "/jwks") + return HttpClientResponse.fromWeb( + request, + Response.json({ keys: [{ ...jwk, kid: "test" }] }), + ); + if (request.body._tag !== "Uint8Array") return yield* Effect.die("Expected token form"); + lastTokenBody = new URLSearchParams(new TextDecoder().decode(request.body.body)); + if (lastTokenBody.get("grant_type") === "refresh_token") { + refreshCalls++; + if (refreshGate) { + yield* Deferred.succeed(refreshGate.entered, undefined); + yield* Deferred.await(refreshGate.release); + } + if (refreshDenied) + return HttpClientResponse.fromWeb( + request, + Response.json({ error: "invalid_grant" }, { status: 400 }), + ); + } + const now = Math.floor((yield* DateTime.now).epochMilliseconds / 1_000); + const token = yield* Effect.promise(() => + new SignJWT({ + nonce, + sub: settings.ownerSub, + iss: settings.issuer, + aud: settings.clientId, + iat: now, + exp: now + 3_600, + ...claims, + }) + .setProtectedHeader({ alg: "RS256", kid: "test" }) + .sign(key.privateKey), + ); + return HttpClientResponse.fromWeb( + request, + Response.json({ + access_token: "issuer-access-never-delivered", + refresh_token: "issuer-refresh-never-delivered", + id_token: token, + token_type: "Bearer", + }), + ); + }), + ); + const layer = HostedAuth.layer.pipe( + Layer.provideMerge(EnvironmentAuth.layer), + Layer.provideMerge(SqlitePersistence.layerMemory), + Layer.provideMerge(ServerSecretStore.layer), + Layer.provide( + Layer.succeed(ServerEnvironment.ServerEnvironmentIdentity, { + getEnvironmentId: Effect.succeed(EnvironmentId.make("test-hub")), + }), + ), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "knife-hosted-auth-" })), + Layer.provideMerge(Layer.succeed(HostedAuthConfig.HostedAuthConfig, Option.some(settings))), + Layer.provide(Layer.succeed(HttpClient.HttpClient, http)), + ); + return { + layer, + setNonce: (value: string) => { + nonce = value; + }, + setClaims: (value: JWTPayload) => { + claims = value; + }, + denyRefresh: () => { + refreshDenied = true; + }, + pauseRefresh: (entered: Deferred.Deferred, release: Deferred.Deferred) => { + refreshGate = { entered, release }; + }, + refreshCalls: () => refreshCalls, + lastTokenBody: () => lastTokenBody, + }; +}); + +const beginLogin = (f: Effect.Success, native = true) => + Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + let request: string | undefined; + if (native) { + const challenge = Base64Url.encode( + yield* (yield* Crypto.Crypto).digest("SHA-256", new TextEncoder().encode(verifier)), + ); + const start = yield* auth.startDevice({ + codeChallenge: challenge, + state: "s".repeat(43), + client, + }); + request = new URL(start.authorizationUrl).searchParams.get("request") ?? undefined; + } + const begun = yield* auth.begin(request ? { request } : { returnTo: "/projects?selected=one" }); + const url = new URL(begun.url); + f.setNonce(url.searchParams.get("nonce") ?? ""); + return { + ...begun, + state: url.searchParams.get("state") ?? "", + challenge: url.searchParams.get("code_challenge") ?? "", + }; + }); +const nativeLogin = (f: Effect.Success) => + Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const begun = yield* beginLogin(f); + const callback = yield* auth.callback({ ...begun, code: "issuer-code", client }); + const url = new URL(callback.url); + expect(url.protocol).toBe("t3code:"); + expect(url.host + url.pathname).toBe("app/self-hosted/callback"); + expect(url.searchParams.get("state")).toBe("s".repeat(43)); + expect(url.searchParams.get("server")).toBe(settings.publicUrl); + const code = url.searchParams.get("code") ?? ""; + return { grant: yield* auth.exchange({ code, codeVerifier: verifier }), code }; + }); + +it.layer(NodeServices.layer)("HostedAuth", (it) => { + it.effect( + "keeps issuer credentials server-side and binds a single-use handoff to the native proof", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const begun = yield* beginLogin(f); + const error = yield* auth + .callback({ ...begun, browserCookie: "another-browser", code: "code", client }) + .pipe(Effect.flip); + expect(error.reason).toBe("transaction"); + const second = yield* beginLogin(f); + const callback = yield* auth.callback({ ...second, code: "code", client }); + const code = new URL(callback.url).searchParams.get("code") ?? ""; + expect( + (yield* auth.exchange({ code, codeVerifier: "x".repeat(43) }).pipe(Effect.flip)).reason, + ).toBe("credential"); + const grant = yield* auth.exchange({ code, codeVerifier: verifier }); + expect(grant.refreshToken).not.toContain("issuer-"); + expect(grant.accessToken).not.toContain("issuer-"); + const submitted = f.lastTokenBody()?.get("code_verifier") ?? ""; + expect( + Base64Url.encode( + yield* (yield* Crypto.Crypto).digest("SHA-256", new TextEncoder().encode(submitted)), + ), + ).toBe(second.challenge); + expect(f.lastTokenBody()?.get("redirect_uri")).toBe( + `${settings.publicUrl}/api/auth/oidc/callback`, + ); + expect( + (yield* auth.exchange({ code, codeVerifier: verifier }).pipe(Effect.flip)).reason, + ).toBe("credential"); + const session = yield* (yield* SessionStore.SessionStore).verify(grant.accessToken); + expect(session.deviceId).toBe(grant.deviceId); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect.each([ + { sub: "another-user" }, + { iss: "https://another-issuer.example" }, + { aud: "another-client" }, + { azp: "another-client" }, + { nonce: "another-transaction" }, + { aud: [settings.clientId, "another-client"], azp: undefined }, + ])("rejects an untrusted ID token %#", (claims) => + Effect.gen(function* () { + const f = yield* fixture; + f.setClaims(claims); + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const begun = yield* beginLogin(f); + expect( + (yield* auth.callback({ ...begun, code: "code", client }).pipe(Effect.flip)).reason, + ).toBe("identity"); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("rotates refresh tokens and revokes the device family on authentic reuse", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const { grant } = yield* nativeLogin(f); + const renewed = yield* auth.refresh(grant.refreshToken); + expect(renewed.deviceId).toBe(grant.deviceId); + expect(renewed.refreshToken).not.toBe(grant.refreshToken); + expect((yield* auth.refresh(grant.refreshToken).pipe(Effect.flip)).reason).toBe( + "credential", + ); + expect((yield* sessions.verify(renewed.accessToken).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect((yield* auth.refresh(renewed.refreshToken).pipe(Effect.result))._tag).toBe( + "Failure", + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect( + "device revocation invalidates HTTP, websocket tickets and live streams while retaining manual sessions", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const { grant } = yield* nativeLogin(f); + const session = yield* sessions.verify(grant.accessToken); + const manual = yield* sessions.issue({ subject: "existing-paired-client" }); + const child = yield* sessions.issue({ + parentSessionId: session.sessionId, + scopes: ["orchestration:read"], + }); + expect((yield* sessions.verify(child.token)).deviceId).toBe(grant.deviceId); + const ticket = yield* sessions.issueWebSocketToken(session.sessionId); + const stopped = yield* sessions + .awaitInvalidation(session.sessionId) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* sessions.verifyActive(session.sessionId); + yield* sessions.verifyActive((yield* sessions.verify(child.token)).sessionId); + yield* auth.revoke(session, grant.deviceId); + expect((yield* sessions.verifyActive(session.sessionId).pipe(Effect.result))._tag).toBe( + "Failure", + ); + yield* Fiber.join(stopped); + for (const token of [grant.accessToken, child.token]) + expect((yield* sessions.verify(token).pipe(Effect.result))._tag).toBe("Failure"); + expect( + (yield* sessions.verifyWebSocketToken(ticket.token).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect((yield* sessions.verify(manual.token)).subject).toBe("existing-paired-client"); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("revalidates issuer renewal and stops a disabled issuer grant", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const { grant } = yield* nativeLogin(f); + yield* TestClock.adjust(Duration.seconds(61)); + f.denyRefresh(); + expect((yield* auth.refresh(grant.refreshToken).pipe(Effect.flip)).reason).toBe( + "reauthentication", + ); + expect(f.refreshCalls()).toBe(1); + expect( + (yield* (yield* SessionStore.SessionStore).verify(grant.accessToken).pipe(Effect.result)) + ._tag, + ).toBe("Failure"); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("attenuated tokens cannot renew their caller's access lifetime", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const { grant } = yield* nativeLogin(f); + const parent = yield* sessions.verify(grant.accessToken); + yield* TestClock.adjust(Duration.minutes(5)); + const first = yield* auth.token(parent, ["archive:read"]); + expect(first.expiresAt.epochMilliseconds).toBe(grant.expiresAt.epochMilliseconds); + yield* TestClock.adjust(Duration.minutes(5)); + const narrowed = yield* sessions.verify(first.token); + const second = yield* auth.token(narrowed, ["archive:read"]); + expect(second.expiresAt.epochMilliseconds).toBe(grant.expiresAt.epochMilliseconds); + yield* TestClock.adjust(Duration.minutes(6)); + expect((yield* sessions.verifyActive(second.sessionId).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect((yield* auth.token(narrowed, ["archive:read"]).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect(f.refreshCalls()).toBe(0); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect( + "binds only one active device to a source and prevents narrow grants from escalating", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const first = (yield* nativeLogin(f)).grant; + const second = (yield* nativeLogin(f)).grant; + const source = EnvironmentId.make("existing-source-id"); + const a = yield* sessions.verify(first.accessToken); + const b = yield* sessions.verify(second.accessToken); + yield* auth.bindEnvironment(a, source); + yield* auth.bindEnvironment(a, source); + const narrow = yield* auth.token(a, ["archive:read", "archive:write"]); + const narrowed = yield* sessions.verify(narrow.token); + yield* auth.authorizeEnvironment(narrowed, source); + expect( + (yield* auth + .authorizeEnvironment(narrowed, EnvironmentId.make("another-source")) + .pipe(Effect.flip)).reason, + ).toBe("forbidden"); + expect((yield* auth.token(narrowed, ["providers:manage"]).pipe(Effect.flip)).reason).toBe( + "forbidden", + ); + expect((yield* auth.revoke(narrowed, b.deviceId ?? "").pipe(Effect.flip)).reason).toBe( + "forbidden", + ); + yield* auth.bindEnvironment(b, source); + yield* auth.authorizeEnvironment(b, source); + expect((yield* sessions.verify(first.accessToken).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect((yield* auth.refresh(first.refreshToken).pipe(Effect.result))._tag).toBe( + "Failure", + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect( + "enforces canonical origin and CSRF on hosted cookies without trusting forwarded hosts", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const hosted = yield* HostedAuth.HostedAuth; + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; + const begun = yield* beginLogin(f, false); + const result = yield* hosted.callback({ ...begun, code: "code", client }); + if (!result.session) return yield* Effect.die("Expected web session"); + const session = yield* sessions.verify(result.session.token); + const account = yield* hosted.account(session); + const request = (origin?: string, csrf?: string, extra = {}) => + HttpServerRequest.fromWeb( + new Request(`${settings.publicUrl}/api/hub/manage`, { + method: "POST", + headers: { + cookie: `${HostedAuthConfig.HOSTED_SESSION_COOKIE}=${result.session?.token}`, + ...(origin ? { origin } : {}), + ...(csrf ? { "x-csrf-token": csrf } : {}), + ...extra, + }, + }), + ); + for (const invalid of [ + request(), + request(settings.publicUrl), + request("https://evil.example", account.csrfToken, { + "x-forwarded-host": "evil.example", + "sec-fetch-site": "same-origin", + }), + ]) + expect((yield* auth.authenticateHttpRequest(invalid).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect( + (yield* auth.authenticateHttpRequest(request(settings.publicUrl, account.csrfToken))) + .deviceId, + ).toBe(session.deviceId); + }).pipe(Effect.provide(f.layer)); + }), + ); + it.effect( + "expires a handoff after sixty seconds and requires interactive enrollment after the configured maximum age", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const begun = yield* beginLogin(f); + const callback = yield* auth.callback({ ...begun, code: "code", client }); + const code = new URL(callback.url).searchParams.get("code") ?? ""; + yield* TestClock.adjust(Duration.seconds(60)); + expect( + (yield* auth.exchange({ code, codeVerifier: verifier }).pipe(Effect.flip)).reason, + ).toBe("credential"); + const { grant } = yield* nativeLogin(f); + yield* TestClock.adjust(Duration.seconds(settings.maxInteractiveAgeSeconds)); + expect((yield* auth.refresh(grant.refreshToken).pipe(Effect.flip)).reason).toBe( + "reauthentication", + ); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("cannot mint access when a device is revoked during issuer renewal", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const { grant } = yield* nativeLogin(f); + const current = yield* sessions.verify(grant.accessToken); + yield* TestClock.adjust(Duration.seconds(61)); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + f.pauseRefresh(entered, release); + const refreshing = yield* auth + .refresh(grant.refreshToken) + .pipe(Effect.result, Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(entered); + yield* auth.revoke(current, grant.deviceId); + yield* Deferred.succeed(release, undefined); + expect((yield* Fiber.join(refreshing))._tag).toBe("Failure"); + expect( + Option.isNone( + yield* (yield* ServerSecretStore.ServerSecretStore).get( + `hosted-oidc-${grant.deviceId}`, + ), + ), + ).toBe(true); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("delivers secure cookies and JSON dates through the actual HTTP routes", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const context = yield* Effect.context< + | HostedAuth.HostedAuth + | EnvironmentAuth.EnvironmentAuth + | HostedAuthConfig.HostedAuthConfig + >(); + const app = yield* Effect.acquireRelease( + Effect.sync(() => HttpRouter.toWebHandler(HostedAuthHttp.layer, { disableLogger: true })), + (app) => Effect.promise(() => app.dispose()), + ); + const request = (path: string, headers: Record = {}, method = "GET") => + Effect.promise(() => + app.handler(new Request(`${settings.publicUrl}${path}`, { method, headers }), context), + ); + const failed = yield* request("/api/auth/oidc/callback?error=access_denied"); + expect(failed.status).toBe(303); + expect(failed.headers.get("location")).toBe(`${settings.publicUrl}/pair?signInError=1`); + expect(failed.headers.getSetCookie()[0]).toContain( + HostedAuthConfig.HOSTED_TRANSACTION_COOKIE, + ); + const start = yield* request("/api/auth/oidc/start?returnTo=%2Fprojects"); + expect(start.status).toBe(303); + expect(start.headers.getSetCookie()[0]).toContain("Max-Age=300"); + const issuerUrl = new URL(start.headers.get("location") ?? ""); + f.setNonce(issuerUrl.searchParams.get("nonce") ?? ""); + const transactionCookie = start.headers.getSetCookie()[0]?.split(";")[0] ?? ""; + const callback = yield* request( + `/api/auth/oidc/callback?code=issuer-code&state=${issuerUrl.searchParams.get("state")}`, + { cookie: transactionCookie }, + ); + expect(callback.status).toBe(303); + expect(callback.headers.get("location")).toBe(`${settings.publicUrl}/projects`); + const rawCookie = + callback.headers + .getSetCookie() + .find((value) => value.startsWith(HostedAuthConfig.HOSTED_SESSION_COOKIE + "=")) ?? ""; + for (const attribute of ["Secure", "HttpOnly", "SameSite=Lax", "Path=/"]) + expect(rawCookie).toContain(attribute); + expect(rawCookie).not.toContain("Domain="); + const cookie = rawCookie.split(";")[0] ?? ""; + const accountResponse = yield* request("/api/auth/account", { cookie }); + expect(accountResponse.status).toBe(200); + const account = yield* Effect.promise(() => accountResponse.json()).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.toCodecJson(HostedAuthAccount))), + ); + expect( + (yield* request( + "/api/auth/session/refresh", + { cookie, origin: settings.publicUrl }, + "POST", + )).status, + ).toBe(401); + const renewed = yield* request( + "/api/auth/session/refresh", + { cookie, origin: settings.publicUrl, "x-csrf-token": account.csrfToken }, + "POST", + ); + expect(renewed.status).toBe(200); + expect( + renewed.headers + .getSetCookie() + .some((value) => value.startsWith(HostedAuthConfig.HOSTED_SESSION_COOKIE + "=")), + ).toBe(true); + expect((yield* request("/api/auth/account", { cookie })).status).toBe(401); + const rotatedCookie = + renewed.headers + .getSetCookie() + .find((value) => value.startsWith(HostedAuthConfig.HOSTED_SESSION_COOKIE + "=")) + ?.split(";")[0] ?? ""; + const otherTab = yield* request( + "/api/auth/session/refresh", + { + cookie: rotatedCookie, + origin: settings.publicUrl, + "x-csrf-token": account.csrfToken, + }, + "POST", + ); + expect(otherTab.status).toBe(200); + const currentCookie = + otherTab.headers + .getSetCookie() + .find((value) => value.startsWith(HostedAuthConfig.HOSTED_SESSION_COOKIE + "=")) + ?.split(";")[0] ?? ""; + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const another = yield* nativeLogin(f); + const anotherAccount = yield* auth.account( + yield* sessions.verify(another.grant.accessToken), + ); + expect( + (yield* request( + "/api/auth/session/refresh", + { + cookie: currentCookie, + origin: settings.publicUrl, + "x-csrf-token": anotherAccount.csrfToken, + }, + "POST", + )).status, + ).toBe(401); + const currentToken = decodeURIComponent( + currentCookie.slice(currentCookie.indexOf("=") + 1), + ); + const currentSession = yield* sessions.verify(currentToken); + yield* auth.revoke(currentSession, account.deviceId); + expect((yield* request("/api/auth/account", { cookie: currentCookie })).status).toBe(401); + expect( + (yield* request( + "/api/auth/session/refresh", + { + cookie: currentCookie, + origin: settings.publicUrl, + "x-csrf-token": account.csrfToken, + }, + "POST", + )).status, + ).toBe(401); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("keeps local emergency recovery independent of issuer availability", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const a = (yield* nativeLogin(f)).grant; + const b = (yield* nativeLogin(f)).grant; + const manual = yield* sessions.issue({ subject: "local-recovery" }); + f.denyRefresh(); + expect(yield* auth.emergencyRevokeAll).toBe(2); + for (const grant of [a, b]) + expect((yield* sessions.verify(grant.accessToken).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect((yield* sessions.verify(manual.token)).subject).toBe("local-recovery"); + expect(yield* auth.emergencyRevokeAll).toBe(0); + }).pipe(Effect.provide(f.layer)); + }), + ); + + it.effect("rejects configured SSO without an owner pin and disables SSO when unconfigured", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "knife-hosted-config-" }); + const path = `${directory}/sso.json`; + yield* fs.writeFileString( + path, + encodeJson({ + publicUrl: settings.publicUrl, + issuer: settings.issuer, + clientId: settings.clientId, + clientSecret: "test", + }), + ); + const read = Effect.gen(function* () { + return yield* HostedAuthConfig.HostedAuthConfig; + }); + const bad = yield* read.pipe( + Effect.provide( + HostedAuthConfig.layer.pipe( + Layer.provide( + ConfigProvider.layer(ConfigProvider.fromUnknown({ T3CODE_SSO_CONFIG: path })), + ), + ), + ), + Effect.flip, + ); + expect(bad._tag).toBe("HostedAuthError"); + const disabled = yield* read.pipe( + Effect.provide( + HostedAuthConfig.layer.pipe( + Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({}))), + ), + ), + ); + expect(Option.isNone(disabled)).toBe(true); + }), + ); + it.effect("re-enrolls an expired device without changing its existing environment identity", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* Effect.gen(function* () { + const auth = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const original = (yield* nativeLogin(f)).grant; + const id = EnvironmentId.make("stable-environment"); + yield* auth.bindEnvironment(yield* sessions.verify(original.accessToken), id); + yield* TestClock.adjust(Duration.seconds(settings.maxInteractiveAgeSeconds)); + const replacement = (yield* nativeLogin(f)).grant; + const session = yield* sessions.verify(replacement.accessToken); + yield* auth.bindEnvironment(session, id); + yield* auth.authorizeEnvironment(session, id); + expect( + (yield* auth.devices(session)).find((device) => device.deviceId === original.deviceId) + ?.revokedAt, + ).not.toBe(null); + }).pipe(Effect.provide(f.layer)); + }), + ); +}); diff --git a/apps/server/src/auth/HostedAuth.ts b/apps/server/src/auth/HostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..f6504c1a2e4be3aa5debe87651ede4a070a727ff --- /dev/null +++ b/apps/server/src/auth/HostedAuth.ts @@ -0,0 +1,739 @@ +import { + AuthAdministrativeScopes, + AuthAccessReadScope, + AuthAccessWriteScope, + AuthEnvironmentConnectScope, + AuthClientPresentationMetadata, + AuthSessionId, + EnvironmentId, + HostedAuthError, + type AuthEnvironmentScope, + type HostedAuthAccount, + type HostedAuthDevice, + type HostedAuthDeviceExchange, + type HostedAuthDeviceStart, + type HostedAuthDeviceStartResult, + type HostedAuthGrant, +} from "@t3tools/contracts"; +import { createLocalJWKSet, jwtVerify } from "jose"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Base64Url from "effect/encoding/Base64Url"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import * as KeyedLock from "@t3tools/shared/KeyedLock"; +import type { AuthenticatedSession } from "./EnvironmentAuth.ts"; +import * as HostedAuthConfig from "./HostedAuthConfig.ts"; +import * as ServerSecretStore from "./ServerSecretStore.ts"; +import * as SessionStore from "./SessionStore.ts"; +import { signPayload } from "./utils.ts"; + +const isHostedAuthError = Schema.is(HostedAuthError); +const encodeClient = Schema.encodeEffect(Schema.fromJsonString(AuthClientPresentationMetadata)); + +const Discovery = Schema.Struct({ + issuer: Schema.String, + authorization_endpoint: Schema.String, + token_endpoint: Schema.String, + jwks_uri: Schema.String, + code_challenge_methods_supported: Schema.Array(Schema.String), + response_types_supported: Schema.Array(Schema.String), + token_endpoint_auth_methods_supported: Schema.Array(Schema.String), + id_token_signing_alg_values_supported: Schema.Array(Schema.String), +}); +const Tokens = Schema.Struct({ + access_token: Schema.NonEmptyString, + token_type: Schema.NonEmptyString, + refresh_token: Schema.optionalKey(Schema.NonEmptyString), + id_token: Schema.optionalKey(Schema.NonEmptyString), +}); +const Device = Schema.Struct({ + deviceId: Schema.String, + accountId: Schema.String, + environmentId: Schema.NullOr(EnvironmentId), + client: Schema.fromJsonString(AuthClientPresentationMetadata), + createdAt: Schema.Number, + expiresAt: Schema.Number, + validatedAt: Schema.Number, + revokedAt: Schema.NullOr(Schema.Number), +}); +type Device = typeof Device.Type; +type NativeRequest = { readonly input: HostedAuthDeviceStart; readonly expiresAt: number }; +type Login = { + readonly browserHash: string; + readonly nonce: string; + readonly verifier: string; + readonly returnTo: string; + readonly expiresAt: number; + readonly native?: HostedAuthDeviceStart; +}; +type Handoff = { + readonly input: HostedAuthDeviceStart; + readonly refreshToken: string; + readonly expiresAt: number; +}; + +export class HostedAuth extends Context.Service< + HostedAuth, + { + readonly enabled: boolean; + readonly startDevice: ( + input: HostedAuthDeviceStart, + ) => Effect.Effect; + readonly begin: (input: { + request?: string; + returnTo?: string; + }) => Effect.Effect<{ url: string; browserCookie: string }, HostedAuthError>; + readonly callback: (input: { + state: string; + code: string; + browserCookie: string; + client: AuthClientPresentationMetadata; + }) => Effect.Effect<{ url: string; session?: SessionStore.IssuedSession }, HostedAuthError>; + readonly exchange: ( + input: HostedAuthDeviceExchange, + ) => Effect.Effect; + readonly refresh: (refreshToken: string) => Effect.Effect; + readonly renewBrowser: ( + session: AuthenticatedSession, + ) => Effect.Effect; + readonly account: ( + session: AuthenticatedSession, + ) => Effect.Effect; + readonly devices: ( + session: AuthenticatedSession, + ) => Effect.Effect, HostedAuthError>; + readonly revoke: ( + session: AuthenticatedSession, + deviceId: string, + ) => Effect.Effect; + readonly token: ( + session: AuthenticatedSession, + scopes: ReadonlyArray, + ) => Effect.Effect; + readonly bindEnvironment: ( + session: AuthenticatedSession, + environmentId: EnvironmentId, + ) => Effect.Effect; + readonly authorizeEnvironment: ( + session: AuthenticatedSession, + environmentId: EnvironmentId, + ) => Effect.Effect; + readonly emergencyRevokeAll: Effect.Effect; + } +>()("t3/auth/HostedAuth") {} + +const make = Effect.gen(function* () { + const configuration = yield* HostedAuthConfig.HostedAuthConfig; + const http = yield* HttpClient.HttpClient; + const sql = yield* SqlClient.SqlClient; + const crypto = yield* Crypto.Crypto; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const sessions = yield* SessionStore.SessionStore; + const locks = yield* KeyedLock.make(); + const requests = new Map(); + const logins = new Map(); + const handoffs = new Map(); + const config = Effect.fromOption(configuration).pipe( + Effect.mapError(() => new HostedAuthError({ reason: "disabled" })), + ); + const random = crypto.randomBytes(32).pipe( + Effect.map(Base64Url.encode), + Effect.mapError( + (cause) => new HostedAuthError({ reason: "storage", cause: Redacted.make(cause) }), + ), + ); + const hash = (text: string) => + crypto.digest("SHA-256", new TextEncoder().encode(text)).pipe( + Effect.map(Base64Url.encode), + Effect.mapError( + (cause) => new HostedAuthError({ reason: "storage", cause: Redacted.make(cause) }), + ), + ); + const storage = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + isHostedAuthError(cause) + ? cause + : new HostedAuthError({ reason: "storage", cause: Redacted.make(cause) }), + ), + ); + const accountId = config.pipe( + Effect.flatMap((value) => hash(`${value.issuer}\0${value.ownerSub}`)), + ); + const prune = (now: number) => { + for (const entries of [requests, logins, handoffs]) + for (const [key, value] of entries) if (value.expiresAt <= now) entries.delete(key); + if (requests.size + logins.size + handoffs.size >= 1_000) + return Effect.fail(new HostedAuthError({ reason: "transaction" })); + return Effect.void; + }; + const requestJson = ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Decoder, + ) => + Effect.gen(function* () { + const response = yield* http + .execute(request) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" })); + if (response.status !== 200) + return yield* new HostedAuthError({ + reason: + response.status === 400 || response.status === 401 ? "reauthentication" : "upstream", + }); + return yield* response.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(schema))); + }).pipe( + Effect.timeout("15 seconds"), + Effect.mapError((cause) => + isHostedAuthError(cause) + ? cause + : new HostedAuthError({ reason: "upstream", cause: Redacted.make(cause) }), + ), + Effect.scoped, + ); + const discovery = Effect.gen(function* () { + const settings = yield* config; + const result = yield* requestJson( + HttpClientRequest.get( + `${settings.issuer.replace(/\/$/u, "")}/.well-known/openid-configuration`, + ), + Discovery, + ); + if ( + result.issuer !== settings.issuer || + !result.code_challenge_methods_supported.includes("S256") || + !result.response_types_supported.includes("code") || + !result.token_endpoint_auth_methods_supported.includes("client_secret_basic") || + !result.id_token_signing_alg_values_supported.includes("RS256") + ) + return yield* new HostedAuthError({ reason: "configuration" }); + for (const endpoint of [ + result.authorization_endpoint, + result.token_endpoint, + result.jwks_uri, + ]) { + const valid = yield* Effect.try({ + try: () => { + const url = new URL(endpoint); + return url.protocol === "https:" && !url.username && !url.password && !url.hash; + }, + catch: () => new HostedAuthError({ reason: "configuration" }), + }); + if (!valid) return yield* new HostedAuthError({ reason: "configuration" }); + } + return result; + }); + const exchangeTokens = (metadata: typeof Discovery.Type, body: Record) => + Effect.gen(function* () { + const settings = yield* config; + const basic = Buffer.from( + `${encodeURIComponent(settings.clientId)}:${encodeURIComponent(Redacted.value(settings.clientSecret))}`, + ).toString("base64"); + return yield* requestJson( + HttpClientRequest.post(metadata.token_endpoint).pipe( + HttpClientRequest.setHeader("authorization", `Basic ${basic}`), + HttpClientRequest.bodyUrlParams(body), + ), + Tokens, + ); + }); + const verifyIdentity = (metadata: typeof Discovery.Type, token: string, nonce?: string) => + Effect.gen(function* () { + const settings = yield* config; + const jwks = yield* requestJson( + HttpClientRequest.get(metadata.jwks_uri), + Schema.Struct({ + keys: Schema.Array( + Schema.Struct({ + kty: Schema.String, + n: Schema.optionalKey(Schema.String), + e: Schema.optionalKey(Schema.String), + kid: Schema.optionalKey(Schema.String), + alg: Schema.optionalKey(Schema.String), + use: Schema.optionalKey(Schema.String), + }), + ), + }), + ); + const now = yield* DateTime.now; + const claims = yield* Effect.tryPromise({ + try: () => + jwtVerify(token, createLocalJWKSet({ keys: [...jwks.keys] }), { + algorithms: ["RS256"], + issuer: settings.issuer, + audience: settings.clientId, + currentDate: DateTime.toDate(now), + clockTolerance: 5, + requiredClaims: ["sub", "iss", "aud", "iat", "exp"], + }), + catch: () => new HostedAuthError({ reason: "identity" }), + }); + if ( + claims.payload.sub !== settings.ownerSub || + (nonce !== undefined && claims.payload.nonce !== nonce) || + (claims.payload.azp !== undefined && claims.payload.azp !== settings.clientId) || + (Array.isArray(claims.payload.aud) && + claims.payload.aud.length > 1 && + claims.payload.azp !== settings.clientId) + ) + return yield* new HostedAuthError({ reason: "identity" }); + }); + const loadDevice = (id: string) => + Effect.gen(function* () { + const rows = yield* storage( + sql`SELECT device_id AS "deviceId", account_id AS "accountId", environment_id AS "environmentId", client_json AS "client", created_at AS "createdAt", expires_at AS "expiresAt", validated_at AS "validatedAt", revoked_at AS "revokedAt" FROM auth_devices WHERE device_id = ${id}`.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Device))), + ), + ); + const device = rows[0]; + const now = yield* DateTime.now; + if ( + !device || + device.accountId !== (yield* accountId) || + device.revokedAt !== null || + device.expiresAt <= now.epochMilliseconds + ) + return yield* new HostedAuthError({ reason: "reauthentication" }); + return device; + }); + const sessionDevice = (session: AuthenticatedSession) => + session.deviceId === undefined + ? Effect.fail(new HostedAuthError({ reason: "forbidden" })) + : loadDevice(session.deviceId); + const revokeDevice = (deviceId: string) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const rows = yield* storage( + sql.withTransaction( + Effect.gen(function* () { + yield* sql`UPDATE auth_devices SET revoked_at = COALESCE(revoked_at, ${now.epochMilliseconds}) WHERE device_id = ${deviceId}`; + yield* sql`DELETE FROM auth_device_refresh WHERE device_id = ${deviceId}`; + return yield* sql`SELECT session_id AS "sessionId" FROM auth_sessions WHERE device_id = ${deviceId}`.pipe( + Effect.flatMap( + Schema.decodeUnknownEffect( + Schema.Array(Schema.Struct({ sessionId: AuthSessionId })), + ), + ), + ); + }), + ), + ); + for (const row of rows) yield* storage(sessions.revoke(row.sessionId)); + yield* storage(secrets.remove(`hosted-oidc-${deviceId}`)); + }); + const issueAccess = ( + device: Device, + method: "browser-session-cookie" | "bearer-access-token", + scopes: ReadonlyArray = AuthAdministrativeScopes, + options?: { readonly replaceSessionId?: AuthSessionId; readonly expiresAt?: DateTime.DateTime }, + ) => + Effect.gen(function* () { + const now = yield* DateTime.now; + return yield* storage( + sessions.issue({ + deviceId: device.deviceId, + subject: device.accountId, + method, + scopes, + ttl: Duration.millis( + Math.min( + 900_000, + device.expiresAt - now.epochMilliseconds, + options?.expiresAt === undefined + ? Infinity + : options.expiresAt.epochMilliseconds - now.epochMilliseconds, + ), + ), + client: { deviceType: device.client.deviceType ?? "unknown", ...device.client }, + ...(options?.replaceSessionId === undefined + ? {} + : { replaceSessionId: options.replaceSessionId }), + }), + ); + }); + const createDevice = (client: typeof AuthClientPresentationMetadata.Type, refreshToken: string) => + Effect.gen(function* () { + const settings = yield* config; + const now = yield* DateTime.now; + const id = yield* storage(crypto.randomUUIDv4()); + const device: Device = { + deviceId: id, + accountId: yield* accountId, + environmentId: null, + client, + createdAt: now.epochMilliseconds, + expiresAt: now.epochMilliseconds + settings.maxInteractiveAgeSeconds * 1_000, + validatedAt: now.epochMilliseconds, + revokedAt: null, + }; + yield* storage(secrets.set(`hosted-oidc-${id}`, new TextEncoder().encode(refreshToken))); + const encodedClient = yield* storage(encodeClient(client)); + yield* storage( + sql`INSERT INTO auth_devices(device_id, account_id, client_json, created_at, expires_at, validated_at) VALUES (${id}, ${device.accountId}, ${encodedClient}, ${device.createdAt}, ${device.expiresAt}, ${device.validatedAt})`, + ); + return device; + }); + const renewIssuer = (device: Device) => + Effect.gen(function* () { + const settings = yield* config; + const now = yield* DateTime.now; + if (device.validatedAt + settings.issuerRevalidationSeconds * 1_000 > now.epochMilliseconds) + return; + const refresh = yield* storage(secrets.get(`hosted-oidc-${device.deviceId}`)); + if (Option.isNone(refresh)) return yield* new HostedAuthError({ reason: "reauthentication" }); + const metadata = yield* discovery; + const tokens = yield* exchangeTokens(metadata, { + grant_type: "refresh_token", + refresh_token: new TextDecoder().decode(refresh.value), + }).pipe( + Effect.catchTags({ + HostedAuthError: (error) => + error.reason === "reauthentication" + ? revokeDevice(device.deviceId).pipe(Effect.andThen(Effect.fail(error))) + : Effect.fail(error), + }), + ); + if (tokens.id_token) + yield* verifyIdentity(metadata, tokens.id_token).pipe( + Effect.catchTags({ + HostedAuthError: (error) => + revokeDevice(device.deviceId).pipe(Effect.andThen(Effect.fail(error))), + }), + ); + if (tokens.token_type.toLowerCase() !== "bearer") + return yield* new HostedAuthError({ reason: "identity" }); + if (tokens.refresh_token) + yield* storage( + secrets.set( + `hosted-oidc-${device.deviceId}`, + new TextEncoder().encode(tokens.refresh_token), + ), + ); + const updated = yield* storage( + sql`UPDATE auth_devices SET validated_at = ${now.epochMilliseconds} WHERE device_id = ${device.deviceId} AND revoked_at IS NULL RETURNING device_id`, + ); + if (updated.length === 0) { + yield* storage(secrets.remove(`hosted-oidc-${device.deviceId}`)); + return yield* new HostedAuthError({ reason: "reauthentication" }); + } + }); + const makeGrant = (device: Device) => + Effect.gen(function* () { + const refreshToken = yield* random; + const tokenHash = yield* hash(refreshToken); + yield* storage( + sql`INSERT INTO auth_device_refresh(token_hash, device_id, expires_at) VALUES (${tokenHash}, ${device.deviceId}, ${device.expiresAt})`, + ); + const issued = yield* issueAccess(device, "bearer-access-token"); + return { + deviceId: device.deviceId, + accessToken: issued.token, + refreshToken, + expiresAt: DateTime.toUtc(issued.expiresAt), + } satisfies HostedAuthGrant; + }); + + return HostedAuth.of({ + enabled: Option.isSome(configuration), + startDevice: (input) => + Effect.gen(function* () { + const settings = yield* config; + const now = yield* DateTime.now; + yield* prune(now.epochMilliseconds); + const encoded = yield* storage(encodeClient(input.client)); + if (encoded.length > 2_048) return yield* new HostedAuthError({ reason: "transaction" }); + const id = yield* random; + requests.set(id, { input, expiresAt: now.epochMilliseconds + 300_000 }); + const url = new URL("/api/auth/oidc/start", settings.publicUrl); + url.searchParams.set("request", id); + return { authorizationUrl: url.href }; + }), + begin: (input) => + Effect.gen(function* () { + const settings = yield* config; + const now = yield* DateTime.now; + yield* prune(now.epochMilliseconds); + const native = input.request ? requests.get(input.request) : undefined; + if (input.request && !native) return yield* new HostedAuthError({ reason: "transaction" }); + if (input.request) requests.delete(input.request); + const returnTo = yield* Effect.try({ + try: () => { + const url = new URL(input.returnTo ?? "/", settings.publicUrl); + if (url.origin !== settings.publicUrl) throw new Error(); + return `${url.pathname}${url.search}${url.hash}`; + }, + catch: () => new HostedAuthError({ reason: "transaction" }), + }); + const metadata = yield* discovery; + const state = yield* random; + const browserCookie = yield* random; + const nonce = yield* random; + const verifier = yield* random; + logins.set(state, { + browserHash: yield* hash(browserCookie), + nonce, + verifier, + returnTo, + expiresAt: now.epochMilliseconds + 300_000, + ...(native ? { native: native.input } : {}), + }); + const url = new URL(metadata.authorization_endpoint); + for (const [key, value] of Object.entries({ + client_id: settings.clientId, + response_type: "code", + scope: "openid offline_access", + redirect_uri: `${settings.publicUrl}/api/auth/oidc/callback`, + state, + nonce, + code_challenge: yield* hash(verifier), + code_challenge_method: "S256", + })) + url.searchParams.set(key, value); + return { url: url.href, browserCookie }; + }), + callback: (input) => + Effect.gen(function* () { + const settings = yield* config; + const now = yield* DateTime.now; + yield* prune(now.epochMilliseconds); + const login = logins.get(input.state); + logins.delete(input.state); + if (!login || login.browserHash !== (yield* hash(input.browserCookie))) + return yield* new HostedAuthError({ reason: "transaction" }); + const metadata = yield* discovery; + const tokens = yield* exchangeTokens(metadata, { + grant_type: "authorization_code", + code: input.code, + code_verifier: login.verifier, + redirect_uri: `${settings.publicUrl}/api/auth/oidc/callback`, + }); + if ( + !tokens.id_token || + !tokens.refresh_token || + tokens.token_type.toLowerCase() !== "bearer" + ) + return yield* new HostedAuthError({ reason: "identity" }); + yield* verifyIdentity(metadata, tokens.id_token, login.nonce); + if (login.native) { + const code = yield* random; + handoffs.set(yield* hash(code), { + input: login.native, + refreshToken: tokens.refresh_token, + expiresAt: now.epochMilliseconds + 60_000, + }); + const url = new URL("t3code://app/self-hosted/callback"); + for (const [key, value] of Object.entries({ + code, + state: login.native.state, + server: settings.publicUrl, + })) + url.searchParams.set(key, value); + return { url: url.href }; + } + const device = yield* createDevice(input.client, tokens.refresh_token); + const session = yield* issueAccess(device, "browser-session-cookie"); + return { url: new URL(login.returnTo, settings.publicUrl).href, session }; + }), + exchange: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + yield* prune(now.epochMilliseconds); + const key = yield* hash(input.code); + const handoff = handoffs.get(key); + if (!handoff || handoff.input.codeChallenge !== (yield* hash(input.codeVerifier))) + return yield* new HostedAuthError({ reason: "credential" }); + handoffs.delete(key); + const device = yield* createDevice(handoff.input.client, handoff.refreshToken); + return yield* storage(sql.withTransaction(makeGrant(device))); + }), + refresh: (refreshToken) => + Effect.gen(function* () { + const tokenHash = yield* hash(refreshToken); + const lookup = + sql`SELECT device_id AS "deviceId", consumed_at AS "consumedAt" FROM auth_device_refresh WHERE token_hash = ${tokenHash}`.pipe( + Effect.flatMap( + Schema.decodeUnknownEffect( + Schema.Array( + Schema.Struct({ + deviceId: Schema.String, + consumedAt: Schema.NullOr(Schema.Number), + }), + ), + ), + ), + ); + const rows = yield* storage(lookup); + const found = rows[0]; + if (!found) return yield* new HostedAuthError({ reason: "credential" }); + return yield* locks.withLock( + found.deviceId, + Effect.gen(function* () { + const current = (yield* storage(lookup))[0]; + if (!current) return yield* new HostedAuthError({ reason: "credential" }); + if (current.consumedAt !== null) { + yield* revokeDevice(current.deviceId); + return yield* new HostedAuthError({ reason: "credential" }); + } + const device = yield* loadDevice(current.deviceId); + yield* renewIssuer(device); + return yield* storage( + sql.withTransaction( + Effect.gen(function* () { + yield* loadDevice(current.deviceId); + const now = yield* DateTime.now; + const changed = + yield* sql`UPDATE auth_device_refresh SET consumed_at = ${now.epochMilliseconds} WHERE token_hash = ${tokenHash} AND consumed_at IS NULL RETURNING token_hash`; + if (changed.length !== 1) + return yield* new HostedAuthError({ reason: "credential" }); + return yield* makeGrant(device); + }), + ), + ); + }), + ); + }), + renewBrowser: (session) => + Effect.gen(function* () { + const device = yield* sessionDevice(session); + return yield* locks.withLock( + device.deviceId, + Effect.gen(function* () { + yield* renewIssuer(device); + return yield* storage( + sql.withTransaction( + Effect.gen(function* () { + const current = yield* loadDevice(device.deviceId); + return yield* issueAccess(current, "browser-session-cookie", session.scopes, { + replaceSessionId: session.sessionId, + }); + }), + ), + ); + }), + ); + }), + account: (session) => + Effect.gen(function* () { + const device = yield* sessionDevice(session); + const secret = yield* storage( + secrets.getOrCreateRandom(HostedAuthConfig.HOSTED_CSRF_SECRET, 32), + ); + return { + accountId: device.accountId, + deviceId: device.deviceId, + csrfToken: signPayload(device.deviceId, secret), + expiresAt: DateTime.toUtc(session.expiresAt ?? DateTime.makeUnsafe(device.expiresAt)), + }; + }), + devices: (session) => + Effect.gen(function* () { + if (!session.scopes.includes(AuthAccessReadScope)) + return yield* new HostedAuthError({ reason: "forbidden" }); + const device = yield* sessionDevice(session); + const rows = yield* storage( + sql`SELECT device_id AS "deviceId", account_id AS "accountId", environment_id AS "environmentId", client_json AS "client", created_at AS "createdAt", expires_at AS "expiresAt", validated_at AS "validatedAt", revoked_at AS "revokedAt" FROM auth_devices WHERE account_id = ${device.accountId} ORDER BY created_at DESC`.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Device))), + ), + ); + return rows.map((row) => ({ + deviceId: row.deviceId, + client: row.client, + createdAt: DateTime.makeUnsafe(row.createdAt), + expiresAt: DateTime.makeUnsafe(row.expiresAt), + revokedAt: row.revokedAt === null ? null : DateTime.makeUnsafe(row.revokedAt), + current: row.deviceId === device.deviceId, + ...(row.environmentId === null ? {} : { environmentId: row.environmentId }), + })); + }), + revoke: (session, deviceId) => + Effect.gen(function* () { + if (session.deviceId !== deviceId && !session.scopes.includes(AuthAccessWriteScope)) + return yield* new HostedAuthError({ reason: "forbidden" }); + const current = yield* sessionDevice(session); + const matches = yield* storage( + sql`SELECT 1 FROM auth_devices WHERE device_id = ${deviceId} AND account_id = ${current.accountId}`, + ); + if (matches.length !== 1) return yield* new HostedAuthError({ reason: "forbidden" }); + yield* revokeDevice(deviceId); + }), + token: (session, scopes) => + Effect.gen(function* () { + const device = yield* sessionDevice(session); + if ( + !session.expiresAt || + scopes.length === 0 || + scopes.some((scope) => !session.scopes.includes(scope)) + ) + return yield* new HostedAuthError({ reason: "forbidden" }); + const expiresAt = session.expiresAt; + return yield* storage( + sql.withTransaction( + Effect.gen(function* () { + yield* sessions.verifyActive(session.sessionId); + yield* loadDevice(device.deviceId); + return yield* issueAccess(device, "bearer-access-token", scopes, { expiresAt }); + }), + ), + ); + }), + bindEnvironment: (session, environmentId) => + Effect.gen(function* () { + if (!session.scopes.includes(AuthEnvironmentConnectScope)) + return yield* new HostedAuthError({ reason: "forbidden" }); + const device = yield* sessionDevice(session); + if (device.environmentId !== null && device.environmentId !== environmentId) + return yield* new HostedAuthError({ reason: "forbidden" }); + const replaced = yield* storage( + sql.withTransaction( + Effect.gen(function* () { + yield* loadDevice(device.deviceId); + const conflicts = + yield* sql`SELECT device_id AS "deviceId", account_id AS "accountId" FROM auth_devices WHERE environment_id = ${environmentId} AND device_id != ${device.deviceId} AND revoked_at IS NULL`.pipe( + Effect.flatMap( + Schema.decodeUnknownEffect( + Schema.Array( + Schema.Struct({ deviceId: Schema.String, accountId: Schema.String }), + ), + ), + ), + ); + if (conflicts.some((previous) => previous.accountId !== device.accountId)) + return yield* new HostedAuthError({ reason: "forbidden" }); + const now = yield* DateTime.now; + for (const previous of conflicts) + yield* sql`UPDATE auth_devices SET revoked_at = ${now.epochMilliseconds} WHERE device_id = ${previous.deviceId}`; + yield* sql`UPDATE auth_devices SET environment_id = ${environmentId} WHERE device_id = ${device.deviceId} AND revoked_at IS NULL`; + return conflicts; + }), + ), + ); + for (const previous of replaced) yield* revokeDevice(previous.deviceId); + }), + authorizeEnvironment: (session, environmentId) => + Effect.gen(function* () { + const device = yield* sessionDevice(session); + if (device.environmentId !== environmentId) + return yield* new HostedAuthError({ reason: "forbidden" }); + }), + emergencyRevokeAll: Effect.gen(function* () { + const rows = yield* storage( + sql`SELECT device_id AS "deviceId" FROM auth_devices WHERE revoked_at IS NULL`.pipe( + Effect.flatMap( + Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ deviceId: Schema.String }))), + ), + ), + ); + for (const row of rows) yield* revokeDevice(row.deviceId); + return rows.length; + }), + }); +}); + +export const layer = Layer.effect(HostedAuth, make); diff --git a/apps/server/src/auth/HostedAuthConfig.ts b/apps/server/src/auth/HostedAuthConfig.ts new file mode 100644 index 0000000000000000000000000000000000000000..19bf31dc7a840245c5869bdf52e8b72fd5720d66 --- /dev/null +++ b/apps/server/src/auth/HostedAuthConfig.ts @@ -0,0 +1,79 @@ +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { HostedAuthError } from "@t3tools/contracts"; + +export const HOSTED_SESSION_COOKIE = "__Host-knife-session"; +export const HOSTED_TRANSACTION_COOKIE = "__Host-knife-oidc"; +export const HOSTED_CSRF_SECRET = "hosted-auth-csrf"; + +const Settings = Schema.Struct({ + publicUrl: Schema.NonEmptyString, + issuer: Schema.NonEmptyString, + clientId: Schema.NonEmptyString, + clientSecret: Schema.NonEmptyString, + ownerSub: Schema.NonEmptyString, + maxInteractiveAgeSeconds: Schema.optionalKey( + Schema.Int.check(Schema.isBetween({ minimum: 900, maximum: 31_536_000 })), + ), + issuerRevalidationSeconds: Schema.optionalKey( + Schema.Int.check(Schema.isBetween({ minimum: 60, maximum: 86_400 })), + ), +}); + +export class HostedAuthConfig extends Context.Service< + HostedAuthConfig, + Option.Option<{ + readonly publicUrl: string; + readonly issuer: string; + readonly clientId: string; + readonly clientSecret: Redacted.Redacted; + readonly ownerSub: string; + readonly maxInteractiveAgeSeconds: number; + readonly issuerRevalidationSeconds: number; + }> +>()("t3/auth/HostedAuthConfig") {} + +const make = Effect.gen(function* () { + const path = yield* Config.option(Config.String("T3CODE_SSO_CONFIG")); + if (Option.isNone(path)) return Option.none(); + const fs = yield* FileSystem.FileSystem; + const settings = yield* fs.readFileString(path.value).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(Settings))), + Effect.mapError(() => new HostedAuthError({ reason: "configuration" })), + ); + const valid = yield* Effect.try({ + try: () => { + const origin = new URL(settings.publicUrl); + const issuer = new URL(settings.issuer); + if ( + origin.protocol !== "https:" || + origin.origin !== settings.publicUrl || + issuer.protocol !== "https:" || + issuer.username || + issuer.password || + issuer.search || + issuer.hash || + origin.username || + origin.password + ) + throw new Error(); + return origin.origin; + }, + catch: () => new HostedAuthError({ reason: "configuration" }), + }); + return Option.some({ + ...settings, + publicUrl: valid, + clientSecret: Redacted.make(settings.clientSecret), + maxInteractiveAgeSeconds: settings.maxInteractiveAgeSeconds ?? 2_592_000, + issuerRevalidationSeconds: settings.issuerRevalidationSeconds ?? 3_600, + }); +}); + +export const layer = Layer.effect(HostedAuthConfig, make); diff --git a/apps/server/src/auth/HostedAuthHttp.ts b/apps/server/src/auth/HostedAuthHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..7d8ba75fc27c52c948bb507a78b9a551c3fa0027 --- /dev/null +++ b/apps/server/src/auth/HostedAuthHttp.ts @@ -0,0 +1,255 @@ +import { + HostedAuthAccount, + HostedAuthDevice, + HostedAuthDeviceExchange, + HostedAuthDeviceRefresh, + HostedAuthDeviceStart, + HostedAuthDeviceStartResult, + HostedAuthError, + HostedAuthGrant, + HostedAuthTokenRequest, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; +import * as EnvironmentAuth from "./EnvironmentAuth.ts"; +import * as HostedAuth from "./HostedAuth.ts"; +import { + HostedAuthConfig, + HOSTED_SESSION_COOKIE, + HOSTED_TRANSACTION_COOKIE, +} from "./HostedAuthConfig.ts"; +import { deriveAuthClientMetadata } from "./utils.ts"; + +const isHostedAuthError = Schema.is(HostedAuthError); + +const headers = { + "cache-control": "no-store", + pragma: "no-cache", + "referrer-policy": "no-referrer", +}; +const cookieOptions = { secure: true, httpOnly: true, sameSite: "lax", path: "/" } as const; +const errors = (effect: Effect.Effect) => + effect.pipe( + Effect.catch((error) => { + if (isHostedAuthError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { _tag: error._tag, reason: error.reason }, + { + headers, + status: + error.reason === "disabled" + ? 404 + : error.reason === "forbidden" + ? 403 + : error.reason === "credential" || error.reason === "reauthentication" + ? 401 + : error.reason === "transaction" || error.reason === "identity" + ? 400 + : 503, + }, + ), + ); + if (EnvironmentAuth.isServerAuthCredentialError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe({ code: "unauthorized" }, { status: 401, headers }), + ); + if (Schema.isSchemaError(error)) + return Effect.succeed(HttpServerResponse.empty({ status: 400, headers })); + return Effect.fail(error); + }), + ); +const authenticate = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + return yield* (yield* EnvironmentAuth.EnvironmentAuth).authenticateHttpRequest(request); +}); +const browser = authenticate.pipe( + Effect.flatMap((session) => + session.method === "browser-session-cookie" + ? Effect.succeed(session) + : Effect.fail(new HostedAuthError({ reason: "forbidden" })), + ), +); +const query = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = new URL(request.url, "https://request.invalid"); + return url.searchParams; +}); + +export const layer = Layer.mergeAll( + HttpRouter.add( + "GET", + "/api/auth/oidc/config", + Effect.gen(function* () { + return HttpServerResponse.jsonUnsafe( + { enabled: (yield* HostedAuth.HostedAuth).enabled }, + { headers }, + ); + }), + ), + HttpRouter.add( + "POST", + "/api/auth/devices/start", + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(HostedAuthDeviceStart); + const result = yield* (yield* HostedAuth.HostedAuth).startDevice(input); + return yield* HttpServerResponse.schemaJson(HostedAuthDeviceStartResult)(result, { headers }); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/auth/oidc/start", + Effect.gen(function* () { + const params = yield* query; + const result = yield* (yield* HostedAuth.HostedAuth).begin({ + ...(params.has("request") ? { request: params.get("request") ?? "" } : {}), + ...(params.has("returnTo") ? { returnTo: params.get("returnTo") ?? "" } : {}), + }); + return HttpServerResponse.redirect(result.url, { status: 303, headers }).pipe( + HttpServerResponse.setCookieUnsafe(HOSTED_TRANSACTION_COOKIE, result.browserCookie, { + ...cookieOptions, + maxAge: "5 minutes", + }), + ); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/auth/oidc/callback", + Effect.gen(function* () { + const params = yield* query; + const request = yield* HttpServerRequest.HttpServerRequest; + const result = yield* (yield* HostedAuth.HostedAuth).callback({ + state: params.get("state") ?? "", + code: params.get("code") ?? "", + browserCookie: request.cookies[HOSTED_TRANSACTION_COOKIE] ?? "", + client: deriveAuthClientMetadata({ request }), + }); + let response = HttpServerResponse.redirect(result.url, { status: 303, headers }).pipe( + HttpServerResponse.expireCookieUnsafe(HOSTED_TRANSACTION_COOKIE, cookieOptions), + ); + if (result.session) + response = HttpServerResponse.setCookieUnsafe( + response, + HOSTED_SESSION_COOKIE, + result.session.token, + { ...cookieOptions, expires: DateTime.toDate(result.session.expiresAt) }, + ); + return response; + }).pipe( + Effect.catchIf(isHostedAuthError, (error) => + Effect.gen(function* () { + const config = yield* HostedAuthConfig; + if (Option.isNone(config)) return yield* error; + return HttpServerResponse.redirect( + new URL("/pair?signInError=1", config.value.publicUrl).href, + { status: 303, headers }, + ).pipe(HttpServerResponse.expireCookieUnsafe(HOSTED_TRANSACTION_COOKIE, cookieOptions)); + }), + ), + errors, + ), + ), + HttpRouter.add( + "POST", + "/api/auth/devices/exchange", + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(HostedAuthDeviceExchange); + const result = yield* (yield* HostedAuth.HostedAuth).exchange(input); + return yield* HttpServerResponse.schemaJson(Schema.toCodecJson(HostedAuthGrant))(result, { + headers, + }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/auth/devices/refresh", + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(HostedAuthDeviceRefresh); + const result = yield* (yield* HostedAuth.HostedAuth).refresh(input.refreshToken); + return yield* HttpServerResponse.schemaJson(Schema.toCodecJson(HostedAuthGrant))(result, { + headers, + }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/auth/session/refresh", + Effect.gen(function* () { + const current = yield* browser; + const auth = yield* HostedAuth.HostedAuth; + const session = yield* auth.renewBrowser(current); + const account = yield* auth.account({ + ...current, + sessionId: session.sessionId, + expiresAt: session.expiresAt, + }); + return (yield* HttpServerResponse.schemaJson(Schema.toCodecJson(HostedAuthAccount))(account, { + headers, + })).pipe( + HttpServerResponse.setCookieUnsafe(HOSTED_SESSION_COOKIE, session.token, { + ...cookieOptions, + expires: DateTime.toDate(session.expiresAt), + }), + ); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/auth/account", + Effect.gen(function* () { + const account = yield* (yield* HostedAuth.HostedAuth).account(yield* authenticate); + return yield* HttpServerResponse.schemaJson(Schema.toCodecJson(HostedAuthAccount))(account, { + headers, + }); + }).pipe(errors), + ), + HttpRouter.add( + "GET", + "/api/auth/devices", + Effect.gen(function* () { + const devices = yield* (yield* HostedAuth.HostedAuth).devices(yield* authenticate); + return yield* HttpServerResponse.schemaJson( + Schema.toCodecJson(Schema.Array(HostedAuthDevice)), + )(devices, { headers }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/auth/devices/:deviceId/revoke", + Effect.gen(function* () { + const params = yield* HttpRouter.params; + yield* (yield* HostedAuth.HostedAuth).revoke(yield* authenticate, params.deviceId ?? ""); + return HttpServerResponse.empty({ headers }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/auth/devices/token", + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(HostedAuthTokenRequest); + const issued = yield* (yield* HostedAuth.HostedAuth).token(yield* authenticate, input.scopes); + return yield* HttpServerResponse.schemaJson( + Schema.toCodecJson( + Schema.Struct({ accessToken: Schema.String, expiresAt: Schema.DateTimeUtc }), + ), + )({ accessToken: issued.token, expiresAt: DateTime.toUtc(issued.expiresAt) }, { headers }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/auth/logout", + Effect.gen(function* () { + const session = yield* authenticate; + if (!session.deviceId) return yield* new HostedAuthError({ reason: "forbidden" }); + yield* (yield* HostedAuth.HostedAuth).revoke(session, session.deviceId); + return HttpServerResponse.empty({ headers }).pipe( + HttpServerResponse.expireCookieUnsafe(HOSTED_SESSION_COOKIE, cookieOptions), + ); + }).pipe(errors), + ), +); diff --git a/apps/server/src/auth/SessionStore.test.ts b/apps/server/src/auth/SessionStore.test.ts index 05da2ac755def2f36613bf74627787b6bb3a97f1..27c25bef391303e3d5de48c5c5f3572814779f2d 100644 --- a/apps/server/src/auth/SessionStore.test.ts +++ b/apps/server/src/auth/SessionStore.test.ts @@ -1,7 +1,12 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; -import { AuthStandardClientScopes, EnvironmentId } from "@t3tools/contracts"; +import { + AuthAdministrativeScopes, + AuthStandardClientScopes, + EnvironmentId, +} from "@t3tools/contracts"; import { expect, it } from "@effect/vitest"; import * as Duration from "effect/Duration"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; @@ -853,4 +858,100 @@ it.layer(NodeServices.layer)("SessionStore.layer", (it) => { expect((yield* readRow)[0]).toEqual({ surface: "mobile", appVersion: "1.3.0" }); }).pipe(Effect.provide(Layer.mergeAll(layerSessionStore(), SqlitePersistence.layerMemory))), ); + it.effect("binds delegated credentials and active sockets to the machine grant", () => + Effect.gen(function* () { + const sessions = yield* SessionStore.SessionStore; + const parent = yield* sessions.issue({ + scopes: AuthAdministrativeScopes, + method: "bearer-access-token", + }); + const child = yield* sessions.issue({ + parentSessionId: parent.sessionId, + scopes: ["orchestration:read"], + method: "bearer-access-token", + }); + const verified = yield* sessions.verify(child.token); + expect(verified.parentSessionId).toBe(parent.sessionId); + const ticket = yield* sessions.issueWebSocketToken(child.sessionId); + expect((yield* sessions.verifyWebSocketToken(ticket.token)).parentSessionId).toBe( + parent.sessionId, + ); + const invalidation = yield* sessions + .awaitInvalidation(child.sessionId) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* sessions.revoke(parent.sessionId); + yield* Fiber.join(invalidation); + expect((yield* sessions.verify(child.token).pipe(Effect.flip))._tag).toBe( + "SessionTokenRevokedError", + ); + expect((yield* sessions.verifyWebSocketToken(ticket.token).pipe(Effect.flip))._tag).toBe( + "WebSocketSessionRevokedError", + ); + expect(yield* sessions.listActive()).toEqual([]); + yield* sessions.awaitInvalidation(child.sessionId); + }).pipe(Effect.scoped, Effect.provide(layerSessionStore())), + ); + + it.effect("caps delegated session expiry and rejects nested or expanded grants", () => + Effect.gen(function* () { + const sessions = yield* SessionStore.SessionStore; + const parent = yield* sessions.issue({ + scopes: ["access:write", "orchestration:read"], + ttl: Duration.seconds(1), + }); + const child = yield* sessions.issue({ + parentSessionId: parent.sessionId, + scopes: ["orchestration:read"], + ttl: Duration.hours(1), + }); + expect(child.expiresAt.epochMilliseconds).toBe(parent.expiresAt.epochMilliseconds); + expect( + (yield* sessions + .issue({ parentSessionId: child.sessionId, scopes: ["orchestration:read"] }) + .pipe(Effect.flip))._tag, + ).toBe("SessionCredentialIssueError"); + expect( + (yield* sessions + .issue({ parentSessionId: parent.sessionId, scopes: ["orchestration:operate"] }) + .pipe(Effect.flip))._tag, + ).toBe("SessionCredentialIssueError"); + const invalidation = yield* sessions + .awaitInvalidation(child.sessionId) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(Duration.seconds(1)); + yield* Fiber.join(invalidation); + expect((yield* sessions.verify(child.token).pipe(Effect.flip))._tag).toBe( + "SessionTokenExpiredError", + ); + expect(yield* sessions.listActive()).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(layerSessionStore())), + ); + it.effect("closes a delegated socket when the CLI revokes its parent in the database", () => + Effect.gen(function* () { + const sessions = yield* SessionStore.SessionStore; + const repository = yield* AuthSessions.AuthSessionRepository; + const parent = yield* sessions.issue({ scopes: AuthAdministrativeScopes }); + const child = yield* sessions.issue({ + parentSessionId: parent.sessionId, + scopes: ["orchestration:read"], + }); + const invalidated = yield* Deferred.make(); + yield* sessions + .awaitInvalidation(child.sessionId) + .pipe( + Effect.andThen(Deferred.succeed(invalidated, undefined)), + Effect.forkScoped({ startImmediately: true }), + ); + expect(yield* Deferred.isDone(invalidated)).toBe(false); + yield* repository.revoke({ sessionId: parent.sessionId, revokedAt: yield* DateTime.now }); + yield* TestClock.adjust(Duration.seconds(5)); + yield* Deferred.await(invalidated); + expect((yield* sessions.verify(child.token).pipe(Effect.flip))._tag).toBe( + "SessionTokenRevokedError", + ); + }).pipe( + Effect.scoped, + Effect.provide(layerSessionStore().pipe(Layer.provideMerge(SqlitePersistence.layerMemory))), + ), + ); }); diff --git a/apps/server/src/auth/SessionStore.ts b/apps/server/src/auth/SessionStore.ts index 169ce5597f1496420d0a5fb73070fa5c4cd70152..97a3b6a33f21993cabadd58aabed712d33172e77 100644 --- a/apps/server/src/auth/SessionStore.ts +++ b/apps/server/src/auth/SessionStore.ts @@ -1,6 +1,7 @@ import { AuthSessionId, AuthAdministrativeScopes, + AuthAccessWriteScope, AuthStandardClientScopes, AuthEnvironmentScopes, type AuthClientMetadata, @@ -51,6 +52,8 @@ export interface IssuedSession { } export interface VerifiedSession { + readonly parentSessionId?: AuthSessionId; + readonly deviceId?: string; readonly sessionId: AuthSessionId; readonly token: string; readonly method: ServerAuthSessionMethod; @@ -371,6 +374,8 @@ export class SessionStore extends Context.Service< readonly cookieName: string; readonly legacyCookieName: string | undefined; readonly issue: (input?: { + readonly parentSessionId?: AuthSessionId; + readonly deviceId?: string; readonly ttl?: Duration.Duration; readonly subject?: string; readonly method?: ServerAuthSessionMethod; @@ -387,6 +392,9 @@ export class SessionStore extends Context.Service< readonly replaceSessionId?: AuthSessionId; }) => Effect.Effect; readonly verify: (token: string) => Effect.Effect; + readonly verifyActive: ( + sessionId: AuthSessionId, + ) => Effect.Effect; readonly issueWebSocketToken: ( sessionId: AuthSessionId, input?: { @@ -546,6 +554,56 @@ export const make = Effect.gen(function* () { sessionId, }).pipe(Effect.asVoid); + const verifyDelegationParent = Effect.fn("SessionStore.verifyDelegationParent")(function* ( + parentSessionId: AuthSessionId, + observedAt: DateTime.Utc, + ) { + const parent = yield* authSessions + .getById({ sessionId: parentSessionId }) + .pipe( + Effect.mapError( + (cause) => new SessionCredentialVerificationError({ sessionId: parentSessionId, cause }), + ), + ); + if (Option.isNone(parent) || parent.value.parentSessionId !== undefined) { + return yield* new UnknownSessionTokenError({ sessionId: parentSessionId }); + } + if (parent.value.revokedAt !== null) { + return yield* new SessionTokenRevokedError({ + sessionId: parentSessionId, + revokedAt: parent.value.revokedAt, + }); + } + if (parent.value.expiresAt.epochMilliseconds <= observedAt.epochMilliseconds) { + return yield* new SessionTokenExpiredError({ + sessionId: parentSessionId, + expiresAt: parent.value.expiresAt, + observedAt, + }); + } + return parent.value; + }); + + const verifyActive = Effect.fn("SessionStore.verifyActive")(function* (sessionId: AuthSessionId) { + const observedAt = yield* DateTime.now; + const row = yield* authSessions + .getById({ sessionId }) + .pipe( + Effect.mapError((cause) => new SessionCredentialVerificationError({ sessionId, cause })), + ); + if (Option.isNone(row)) return yield* new UnknownSessionTokenError({ sessionId }); + if (row.value.revokedAt !== null) + return yield* new SessionTokenRevokedError({ sessionId, revokedAt: row.value.revokedAt }); + if (row.value.expiresAt.epochMilliseconds <= observedAt.epochMilliseconds) + return yield* new SessionTokenExpiredError({ + sessionId, + expiresAt: row.value.expiresAt, + observedAt, + }); + if (row.value.parentSessionId !== undefined) + yield* verifyDelegationParent(row.value.parentSessionId, observedAt); + }); + const loadActiveSession = (sessionId: AuthSessionId) => Effect.gen(function* () { const row = yield* authSessions.getById({ sessionId }); @@ -559,6 +617,12 @@ export const make = Effect.gen(function* () { if (!connected && row.value.expiresAt.epochMilliseconds <= now.epochMilliseconds) { return Option.none(); } + if (row.value.parentSessionId !== undefined) { + const parent = yield* verifyDelegationParent(row.value.parentSessionId, now).pipe( + Effect.catchIf(isSessionCredentialInvalidError, () => Effect.succeed(undefined)), + ); + if (parent === undefined) return Option.none(); + } return Option.some( toAuthClientSession({ sessionId: row.value.sessionId, @@ -665,9 +729,27 @@ export const make = Effect.gen(function* () { .pipe(Effect.mapError((cause) => new SessionCredentialIssueError({ cause }))), ); const issuedAt = yield* DateTime.now; - const expiresAt = DateTime.add(issuedAt, { + const requestedExpiresAt = DateTime.add(issuedAt, { milliseconds: Duration.toMillis(input?.ttl ?? DEFAULT_SESSION_TTL), }); + const parent = + input?.parentSessionId === undefined + ? undefined + : yield* verifyDelegationParent(input.parentSessionId, issuedAt).pipe( + Effect.flatMap((parent) => + parent.scopes.includes(AuthAccessWriteScope) && + (input.scopes ?? AuthStandardClientScopes).every((scope) => + parent.scopes.includes(scope), + ) + ? Effect.succeed(parent) + : Effect.fail(new UnknownSessionTokenError({ sessionId: parent.sessionId })), + ), + Effect.mapError((cause) => new SessionCredentialIssueError({ sessionId, cause })), + ); + const expiresAt = + parent && parent.expiresAt.epochMilliseconds < requestedExpiresAt.epochMilliseconds + ? parent.expiresAt + : requestedExpiresAt; const claims: SessionClaims = { v: 1, kind: "session", @@ -697,8 +779,11 @@ export const make = Effect.gen(function* () { ); const signature = signPayload(encodedPayload, signingSecret); const client = input?.client ?? createDefaultClientMetadata(); + const deviceId = input?.deviceId ?? parent?.deviceId; const sessionRecord = { sessionId, + ...(input?.parentSessionId === undefined ? {} : { parentSessionId: input.parentSessionId }), + ...(deviceId === undefined ? {} : { deviceId }), subject: claims.sub, scopes: claims.scopes, method: claims.method, @@ -848,8 +933,15 @@ export const make = Effect.gen(function* () { }); } + if (row.value.parentSessionId !== undefined) { + yield* verifyDelegationParent(row.value.parentSessionId, observedAt); + } return { sessionId: claims.sid, + ...(row.value.parentSessionId === undefined + ? {} + : { parentSessionId: row.value.parentSessionId }), + ...(row.value.deviceId === undefined ? {} : { deviceId: row.value.deviceId }), token, method: claims.method, client: toClientMetadata(row.value.client), @@ -958,8 +1050,15 @@ export const make = Effect.gen(function* () { }); } + if (row.value.parentSessionId !== undefined) { + yield* verifyDelegationParent(row.value.parentSessionId, observedAt); + } return { sessionId: row.value.sessionId, + ...(row.value.parentSessionId === undefined + ? {} + : { parentSessionId: row.value.parentSessionId }), + ...(row.value.deviceId === undefined ? {} : { deviceId: row.value.deviceId }), token, method: row.value.method, client: toClientMetadata(row.value.client), @@ -998,21 +1097,24 @@ export const make = Effect.gen(function* () { const revoke: SessionStore["Service"]["revoke"] = Effect.fn("SessionStore.revoke")( function* (sessionId) { const revokedAt = yield* DateTime.now; - const revoked = yield* authSessions + const revokedSessionIds = yield* authSessions .revoke({ sessionId, revokedAt, }) .pipe(Effect.mapError((cause) => new SessionRevocationError({ sessionId, cause }))); - if (revoked) { + if (revokedSessionIds.length > 0) { yield* Ref.update(connectedSessionsRef, (current) => { const next = new Map(current); - next.delete(sessionId); + for (const revokedSessionId of revokedSessionIds) next.delete(revokedSessionId); return next; }); - yield* emitRemoved(sessionId); + yield* Effect.forEach(revokedSessionIds, emitRemoved, { + concurrency: "unbounded", + discard: true, + }); } - return revoked; + return revokedSessionIds.length > 0; }, ); @@ -1055,26 +1157,52 @@ export const make = Effect.gen(function* () { Effect.gen(function* () { // Subscribe before reading: revocation can race with the WebSocket upgrade. const subscription = yield* PubSub.subscribe(changesPubSub); - const row = yield* authSessions - .getById({ sessionId }) - .pipe( - Effect.mapError( - (cause) => new SessionCredentialVerificationError({ sessionId, cause }), - ), - ); - if (Option.isNone(row) || row.value.revokedAt !== null) return; - const now = yield* DateTime.now; - const remaining = row.value.expiresAt.epochMilliseconds - now.epochMilliseconds; - if (remaining <= 0) return; - yield* Effect.raceFirst( - Stream.fromSubscription(subscription).pipe( + while (true) { + const row = yield* authSessions + .getById({ sessionId }) + .pipe( + Effect.mapError( + (cause) => new SessionCredentialVerificationError({ sessionId, cause }), + ), + ); + if (Option.isNone(row) || row.value.revokedAt !== null) return; + const now = yield* DateTime.now; + const parent = + row.value.parentSessionId === undefined + ? undefined + : yield* authSessions + .getById({ sessionId: row.value.parentSessionId }) + .pipe( + Effect.mapError( + (cause) => new SessionCredentialVerificationError({ sessionId, cause }), + ), + ); + if (parent !== undefined && (Option.isNone(parent) || parent.value.revokedAt !== null)) + return; + const expiresAt = + parent !== undefined && Option.isSome(parent) + ? Math.min( + row.value.expiresAt.epochMilliseconds, + parent.value.expiresAt.epochMilliseconds, + ) + : row.value.expiresAt.epochMilliseconds; + const remaining = expiresAt - now.epochMilliseconds; + if (remaining <= 0) return; + const removed = Stream.fromSubscription(subscription).pipe( Stream.filter( - (change) => change.type === "clientRemoved" && change.sessionId === sessionId, + (change) => + change.type === "clientRemoved" && + (change.sessionId === sessionId || change.sessionId === row.value.parentSessionId), ), Stream.runHead, - ), - Effect.sleep(Duration.millis(remaining)), - ); + Effect.as(true), + ); + // CLI revocation writes the same database from another process. + const recheck = Effect.sleep(Duration.millis(Math.min(remaining, 5_000))).pipe( + Effect.as(false), + ); + if (yield* Effect.raceFirst(removed, recheck)) return; + } }), ); }); @@ -1084,6 +1212,7 @@ export const make = Effect.gen(function* () { legacyCookieName, issue, verify, + verifyActive, issueWebSocketToken, verifyWebSocketToken, listActive, diff --git a/apps/server/src/auth/http.test.ts b/apps/server/src/auth/http.test.ts index 064578f727a01c28d65c5e1b383c1f0a4abc75fb..c5a60e5416fdc84a723d0ede23dc3bbb67faea08 100644 --- a/apps/server/src/auth/http.test.ts +++ b/apps/server/src/auth/http.test.ts @@ -4,6 +4,7 @@ import { AuthTokenExchangeGrantType, AuthEnvironmentBootstrapTokenType, AuthAccessTokenType, + AuthAccessTokenResult, EnvironmentAuthenticatedAuth, EnvironmentHttpApi, } from "@t3tools/contracts"; @@ -31,6 +32,14 @@ import * as EnvironmentAuth from "./EnvironmentAuth.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; import * as AuthHttp from "./http.ts"; +const unusedSecretStore = ServerSecretStore.ServerSecretStore.of({ + get: () => Effect.succeedNone, + set: () => Effect.void, + create: () => Effect.void, + getOrCreateRandom: () => Effect.die("Not used by these routes."), + remove: () => Effect.void, +}); + const DEV_TOKEN = "reusable-dev-auth-token-that-is-long-enough"; class AuthTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.auth) {} @@ -68,6 +77,7 @@ const layerRoutes = HttpApiBuilder.layer(AuthTestApi).pipe( ); const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeAccessToken = Schema.decodeUnknownSync(AuthAccessTokenResult); const postJson = (path: string, body: unknown, headers?: Readonly>) => new Request(`http://127.0.0.1${path}`, { method: "POST", @@ -78,13 +88,6 @@ const postJson = (path: string, body: unknown, headers?: Readonly Effect.gen(function* () { const crypto = yield* Crypto.Crypto; - const unusedSecretStore = ServerSecretStore.ServerSecretStore.of({ - get: () => Effect.succeedNone, - set: () => Effect.void, - create: () => Effect.void, - getOrCreateRandom: () => Effect.die("Not used by these routes."), - remove: () => Effect.void, - }); const requestContext = Context.make(Crypto.Crypto, crypto).pipe( Context.add(ServerSecretStore.ServerSecretStore, unusedSecretStore), ); @@ -237,3 +240,41 @@ it.effect("exports only verified T3 Connect requests", () => expect(localSpans).toEqual(["EnvironmentAuth.authenticateHttpRequest", "environment.handler"]); }).pipe(Effect.scoped), ); + +it.effect("delegated credentials cannot mint an independent grant", () => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const requestContext = Context.make(Crypto.Crypto, crypto).pipe( + Context.add(ServerSecretStore.ServerSecretStore, unusedSecretStore), + ); + return yield* Effect.acquireUseRelease( + Effect.sync(() => HttpRouter.toWebHandler(layerRoutes, { disableLogger: true })), + (server) => + Effect.promise(async () => { + const response = await server.handler( + postJson( + "/api/auth/delegated-session", + { + scopes: ["orchestration:read", "access:write"], + }, + { authorization: `Bearer ${DEV_TOKEN}` }, + ), + requestContext, + ); + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toContain("no-store"); + const issued = decodeAccessToken(await response.json()); + const authorization = `Bearer ${issued.access_token}`; + for (const path of ["/api/auth/pairing-token", "/api/auth/delegated-session"]) { + const rejected = await server.handler( + postJson(path, { scopes: ["orchestration:read"] }, { authorization }), + requestContext, + ); + expect(rejected.status).toBe(400); + expect(await rejected.json()).toMatchObject({ reason: "scope_not_granted" }); + } + }), + (server) => Effect.promise(() => server.dispose()), + ); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/auth/http.ts b/apps/server/src/auth/http.ts index 40dee5b6bcfd5e09e28d12726cb0d0cce03474da..1c6814d0af32f9e694b1b7f79d34c69219873bcf 100644 --- a/apps/server/src/auth/http.ts +++ b/apps/server/src/auth/http.ts @@ -432,6 +432,8 @@ export const layer = HttpApiBuilder.group( function* (args) { yield* annotateEnvironmentRequest(args.endpoint.name); const session = yield* requireEnvironmentScope(AuthAccessWriteScope); + if (session.parentSessionId !== undefined) + return yield* failEnvironmentInvalidRequest("scope_not_granted"); const delegatedScopes = args.payload.scopes ?? AuthStandardClientScopes; if ( delegatedScopes.length === 0 || @@ -451,6 +453,26 @@ export const layer = HttpApiBuilder.group( ), ), ) + .handle( + "delegatedSession", + Effect.fn("environment.auth.delegatedSession")( + function* (args) { + yield* annotateEnvironmentRequest(args.endpoint.name); + const session = yield* requireEnvironmentScope(AuthAccessWriteScope); + yield* appendCredentialResponseHeaders; + return yield* serverAuth.issueDelegatedSession( + { ...session, scopes: [...session.scopes] }, + args.payload, + ); + }, + Effect.catchIf(EnvironmentAuth.isServerAuthInvalidRequestError, (error) => + failEnvironmentInvalidRequest(EnvironmentAuth.serverAuthInvalidRequestReason(error)), + ), + Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => + failEnvironmentInternal("access_token_issuance_failed", error), + ), + ), + ) .handle( "pairingLinks", Effect.fn("environment.auth.pairingLinks")( diff --git a/apps/server/src/cli/auth.ts b/apps/server/src/cli/auth.ts index 32934a97ca46b63db76032b4e5f8673897aa0b99..816aeff1ae81219c161b128287f6bd99d31d6898 100644 --- a/apps/server/src/cli/auth.ts +++ b/apps/server/src/cli/auth.ts @@ -8,9 +8,12 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as References from "effect/References"; +import { FetchHttpClient } from "effect/http"; import { Argument, Command, Flag, GlobalFlag } from "effect/cli"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as HostedAuth from "../auth/HostedAuth.ts"; +import * as HostedAuthConfig from "../auth/HostedAuthConfig.ts"; import { formatIssuedPairingCredential, @@ -243,7 +246,35 @@ const sessionCommand = Command.make("session").pipe( Command.withSubcommands([sessionIssueCommand, sessionListCommand, sessionRevokeCommand]), ); +const devicesCommand = Command.make("devices").pipe( + Command.withSubcommands([ + Command.make("revoke-all", authLocationFlags).pipe( + Command.withDescription("Revoke all SSO devices and sessions using local server access."), + Command.withHandler((flags) => + Effect.gen(function* () { + const config = yield* resolveCliAuthConfig(flags, yield* GlobalFlag.LogLevel); + const count = yield* Effect.gen(function* () { + return yield* (yield* HostedAuth.HostedAuth).emergencyRevokeAll; + }).pipe( + Effect.provide( + HostedAuth.layer.pipe( + Layer.provide(EnvironmentAuth.layerRuntime), + Layer.provide(ServerConfig.layer(config)), + Layer.provide(Layer.succeed(HostedAuthConfig.HostedAuthConfig, Option.none())), + Layer.provide(FetchHttpClient.layer), + ), + ), + ); + yield* Console.log( + `Revoked ${count} SSO devices. Existing pairing credentials remain valid.`, + ); + }), + ), + ), + ]), +); + export const authCommand = Command.make("auth").pipe( Command.withDescription("Manage the local auth control plane for headless deployments."), - Command.withSubcommands([pairingCommand, sessionCommand]), + Command.withSubcommands([pairingCommand, sessionCommand, devicesCommand]), ); diff --git a/apps/server/src/cli/project.ts b/apps/server/src/cli/project.ts index 43253040a942a1b7d3661e9b0b6fcd98220ce617..cad8767acab73161ea0069a67027d7c874d24eff 100644 --- a/apps/server/src/cli/project.ts +++ b/apps/server/src/cli/project.ts @@ -29,6 +29,9 @@ import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ServerOwnership from "../serverOwnership.ts"; import * as RuntimeLayer from "../orchestration-v2/runtimeLayer.ts"; +import * as ArchiveRuntime from "../archive/runtimeLayer.ts"; +import * as ArchiveConnection from "../archive/ArchiveConnection.ts"; +import * as SharedServer from "../environment/SharedServer.ts"; import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts"; import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; @@ -201,6 +204,12 @@ const projectCommandUuid = Crypto.Crypto.pipe( ); const layerProjectCliRuntime = RuntimeLayer.layerProjectService.pipe( + Layer.provideMerge( + ArchiveRuntime.layer.pipe( + Layer.provide(ArchiveConnection.layerLegacy.pipe(Layer.provide(SharedServer.layer))), + Layer.provide(ServerSecretStore.layer), + ), + ), Layer.provideMerge(ProjectEnrichmentService.layer), Layer.provideMerge(RepositoryIdentityResolver.layer), Layer.provideMerge( diff --git a/apps/server/src/cli/shared.ts b/apps/server/src/cli/shared.ts index 7a585335c7be58e3c7047c659e3455b82d34c83f..62b16a24bf6c1745d55d902b78ba53ac4862e437 100644 --- a/apps/server/src/cli/shared.ts +++ b/apps/server/src/cli/shared.ts @@ -1,5 +1,16 @@ import { AuthProvidersManageScope, + AuthAccessWriteScope, + EnvironmentId, + HubDirectory, + HubError, + type HubManagementRequest, + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + ArchiveStatus, + ArchiveError, + type ArchiveManagementRequest, + ThreadId, ProviderInstanceId, SharedAccountError, SharedAccountId, @@ -19,9 +30,14 @@ import * as ServerConfig from "../config.ts"; import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; import { authLocationFlags, resolveCliAuthConfig, type CliAuthLocationFlags } from "./config.ts"; -const encodeStatus = Schema.encodeEffect(Schema.fromJsonString(SharedProfilesStatus)); +const encodeStatus = Schema.encodeEffect( + Schema.fromJsonString(Schema.Union([SharedProfilesStatus, ArchiveStatus, HubDirectory])), +); -const run = (flags: CliAuthLocationFlags, request: SharedProfilesRequest) => +const run = ( + flags: CliAuthLocationFlags, + request: SharedProfilesRequest | ArchiveManagementRequest | HubManagementRequest, +) => Effect.gen(function* () { const config = yield* resolveCliAuthConfig(flags, yield* GlobalFlag.LogLevel); const state = yield* readPersistedServerRuntimeState(config.serverRuntimeStatePath); @@ -29,19 +45,32 @@ const run = (flags: CliAuthLocationFlags, request: SharedProfilesRequest) => return yield* Effect.gen(function* () { const auth = yield* EnvironmentAuth.EnvironmentAuth; const http = yield* HttpClient.HttpClient; + const [endpoint, scopes] = request.type.startsWith("hub.") + ? (["/api/hub/manage", [AuthAccessWriteScope]] as const) + : request.type.startsWith("archive.") + ? ([ + "/api/archive/manage", + [AuthOrchestrationReadScope, AuthOrchestrationOperateScope], + ] as const) + : (["/api/shared/profiles", [AuthProvidersManageScope]] as const); const result = yield* Effect.acquireUseRelease( - auth.issueSession({ scopes: [AuthProvidersManageScope], label: "t3 shared cli" }), + auth.issueSession({ + scopes, + label: "t3 shared cli", + }), (session) => - HttpClientRequest.post(`${state.value.origin}/api/shared/profiles`).pipe( + HttpClientRequest.post(`${state.value.origin}${endpoint}`).pipe( HttpClientRequest.setHeader("authorization", `Bearer ${session.token}`), HttpClientRequest.bodyJson(request), Effect.flatMap(http.execute), Effect.flatMap((response) => response.status === 200 - ? HttpClientResponse.schemaBodyJson(SharedProfilesStatus)(response) - : HttpClientResponse.schemaBodyJson(SharedAccountError)(response).pipe( - Effect.flatMap(Effect.fail), - ), + ? HttpClientResponse.schemaBodyJson( + Schema.Union([SharedProfilesStatus, ArchiveStatus, HubDirectory]), + )(response) + : HttpClientResponse.schemaBodyJson( + Schema.Union([SharedAccountError, ArchiveError, HubError]), + )(response).pipe(Effect.flatMap(Effect.fail)), ), Effect.timeout("30 seconds"), ), @@ -51,15 +80,35 @@ const run = (flags: CliAuthLocationFlags, request: SharedProfilesRequest) => }).pipe( Effect.scoped, Effect.provide( - EnvironmentAuth.layerRuntime.pipe( - Layer.provide(ServerConfig.layer(config)), - Layer.provide(Layer.succeed(References.MinimumLogLevel, "Error")), + Layer.mergeAll( + EnvironmentAuth.layerRuntime.pipe( + Layer.provide(ServerConfig.layer(config)), + Layer.provide(Layer.succeed(References.MinimumLogLevel, "Error")), + ), + FetchHttpClient.layer, ), ), - Effect.provide(FetchHttpClient.layer), ); }); +const archiveStatus = Command.make("status", authLocationFlags).pipe( + Command.withHandler((flags) => run(flags, { type: "archive.status" })), +); +const archiveRetry = Command.make("retry", { + ...authLocationFlags, + threadId: Argument.String("thread-id").pipe(Argument.withSchema(ThreadId), Argument.optional), +}).pipe( + Command.withDescription("Retry transcript uploads that stopped after failures."), + Command.withHandler((flags) => + run(flags, { + type: "archive.retry", + ...(Option.isSome(flags.threadId) ? { threadId: flags.threadId.value } : {}), + }), + ), +); +const archive = Command.make("archive").pipe( + Command.withSubcommands([archiveStatus, archiveRetry]), +); const status = Command.make("status", authLocationFlags).pipe( Command.withDescription("List shared accounts on this host."), Command.withHandler((flags) => run(flags, { type: "status" })), @@ -115,7 +164,48 @@ const remove = Command.make("remove", { Command.withHandler((flags) => run(flags, { type: "remove", id: flags.id })), ); +const hub = Command.make("hub").pipe( + Command.withDescription("Manage machines reached through this server."), + Command.withSubcommands([ + Command.make("configure", { + ...authLocationFlags, + publicUrl: Argument.String("public-url"), + }).pipe( + Command.withHandler((flags) => + run(flags, { type: "hub.configure", publicUrl: flags.publicUrl }), + ), + ), + Command.make("status", authLocationFlags).pipe( + Command.withHandler((flags) => run(flags, { type: "hub.status" })), + ), + Command.make("enroll", { + ...authLocationFlags, + url: Argument.String("private-machine-url"), + environmentId: Argument.String("environment-id").pipe(Argument.withSchema(EnvironmentId)), + tokenFile: Flag.String("token-file"), + }).pipe( + Command.withDescription("Enroll a machine using its private administrative bearer grant."), + Command.withHandler((flags) => + run(flags, { + type: "hub.enroll", + url: flags.url, + environmentId: flags.environmentId, + tokenFile: flags.tokenFile, + }), + ), + ), + Command.make("remove", { + ...authLocationFlags, + environmentId: Argument.String("environment-id").pipe(Argument.withSchema(EnvironmentId)), + }).pipe( + Command.withHandler((flags) => + run(flags, { type: "hub.remove", environmentId: flags.environmentId }), + ), + ), + ]), +); + export const sharedCommand = Command.make("shared").pipe( Command.withDescription("Manage shared provider accounts."), - Command.withSubcommands([status, connect, disconnect, importAccount, sync, remove]), + Command.withSubcommands([status, connect, disconnect, importAccount, sync, remove, archive, hub]), ); diff --git a/apps/server/src/environment/Hub.test.ts b/apps/server/src/environment/Hub.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c05b48ad59f55c4f1766c2fbfa3374c9f78ee5a8 --- /dev/null +++ b/apps/server/src/environment/Hub.test.ts @@ -0,0 +1,326 @@ +import { assert, it } from "@effect/vitest"; +import { + AuthAccessReadScope, + AuthAccessWriteScope, + AuthAdministrativeScopes, + AuthFilesystemReadScope, + AuthPreviewOperateScope, + authScopeResponse, + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthSessionId, + EnvironmentId, + ORCHESTRATION_PROTOCOL_VERSION, + type AuthEnvironmentScope, + type ExecutionEnvironmentDescriptor, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; +import * as Hub from "./Hub.ts"; +import * as OutboundHub from "./OutboundHub.ts"; +import * as HostedAuthConfig from "../auth/HostedAuthConfig.ts"; + +const leafId = EnvironmentId.make("leaf-a"); +const descriptor: ExecutionEnvironmentDescriptor = { + environmentId: leafId, + label: "Laptop", + platform: { os: "linux", arch: "x64" }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: true }, +}; +const session = (scopes: ReadonlyArray = AuthAdministrativeScopes) => ({ + sessionId: AuthSessionId.make("phone"), + subject: "phone", + method: "bearer-access-token" as const, + scopes, +}); + +const fixture = ( + grant: ReadonlyArray = AuthAdministrativeScopes, + permissions: "modern" | "legacy" | "empty" = "modern", +) => + Effect.gen(function* () { + const data = new Map(); + const requests: Array<{ url: string; authorization?: string; body: string }> = []; + let identity = leafId; + let serverVersion = "test"; + let unavailable = false; + let malformed = false; + const client = HttpClient.make((request) => + Effect.sync(() => { + requests.push({ + url: request.url, + ...(request.headers.authorization + ? { authorization: request.headers.authorization } + : {}), + body: + request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "", + }); + if (unavailable) + return HttpClientResponse.fromWeb(request, new Response(null, { status: 503 })); + let result: unknown; + if (request.url.endsWith("/.well-known/t3/environment")) + result = malformed ? {} : { ...descriptor, environmentId: identity, serverVersion }; + else if (request.url.endsWith("/api/auth/session")) + result = { + authenticated: true, + auth: { + policy: "remote-reachable", + bootstrapMethods: ["one-time-token"], + sessionMethods: ["bearer-access-token"], + sessionCookieName: "session", + }, + scopes: authScopeResponse(grant).scopes, + ...(permissions === "legacy" + ? {} + : { permissions: permissions === "empty" ? [] : grant }), + }; + else if (request.url.endsWith("/api/auth/delegated-session")) + result = { + access_token: "private-leaf-session", + issued_token_type: "urn:ietf:params:oauth:token-type:access_token", + token_type: "Bearer", + expires_in: 3600, + scope: JSON.parse(requests.at(-1)?.body ?? "{}").scopes?.join(" "), + }; + else throw new Error(`Unexpected request ${request.url}`); + return HttpClientResponse.fromWeb(request, Response.json(result)); + }), + ); + const store = ServerSecretStore.ServerSecretStore.of({ + get: (key) => Effect.sync(() => Option.fromUndefinedOr(data.get(key))), + set: (key, value) => + Effect.sync(() => { + data.set(key, value); + }), + remove: (key) => + Effect.sync(() => { + data.delete(key); + }), + create: () => Effect.die("unused"), + getOrCreateRandom: () => Effect.die("unused"), + }); + const layer = Hub.layer.pipe( + Layer.provide(Layer.succeed(HostedAuthConfig.HostedAuthConfig, Option.none())), + Layer.provide( + Layer.succeed(OutboundHub.OutboundHub, { + enrollments: Effect.succeed([]), + connect: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + accept: () => Effect.die("unused"), + channel: () => Effect.die("unused"), + }), + ), + Layer.provide(Layer.succeed(ServerSecretStore.ServerSecretStore, store)), + Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), + Layer.provide( + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed(EnvironmentId.make("hub")), + getDescriptor: Effect.succeed({ + ...descriptor, + environmentId: EnvironmentId.make("hub"), + }), + }), + ), + Layer.provide( + Layer.succeed( + FileSystem.FileSystem, + FileSystem.makeNoop({ readFileString: () => Effect.succeed("leaf-admin-token\n") }), + ), + ), + ); + const service = yield* Hub.Hub.pipe(Effect.provide(layer)); + const enroll = service.manage({ + type: "hub.enroll", + url: "http://leaf.private/base/", + environmentId: leafId, + tokenFile: "/private/grant", + }); + return { + service, + requests, + data, + enroll, + restart: Hub.Hub.pipe(Effect.provide(layer)), + changeVersion: (value: string) => { + serverVersion = value; + }, + setUnavailable: () => { + unavailable = true; + }, + setMalformed: () => { + malformed = true; + }, + changeIdentity: (id: EnvironmentId) => { + identity = id; + }, + }; + }); + +it.effect( + "requires an explicit canonical public URL and persists an identity-checked private enrollment", + () => + Effect.gen(function* () { + const f = yield* fixture(); + assert.strictEqual((yield* f.enroll.pipe(Effect.flip)).reason, "configuration"); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/prefix" }); + yield* f.enroll; + const directory = yield* (yield* f.restart).directory(session()); + assert.strictEqual( + directory.environments[0]?.httpBaseUrl, + "https://hub.example/prefix/environments/leaf-a/", + ); + assert.strictEqual(directory.hubEnvironmentId, "hub"); + assert.strictEqual(JSON.stringify(directory).includes("leaf-admin-token"), false); + assert.strictEqual(JSON.stringify(directory).includes("leaf.private"), false); + assert.strictEqual(f.requests[0]?.authorization, undefined); + }), +); + +it.effect("does not send a private grant to a different machine identity", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + f.changeIdentity(EnvironmentId.make("impostor")); + assert.strictEqual((yield* f.enroll.pipe(Effect.flip)).reason, "identity"); + assert.strictEqual(f.requests.length, 1); + assert.strictEqual(f.requests[0]?.authorization, undefined); + }), +); + +it.effect( + "bounds delegated sessions by both client and enrolled machine grants and keeps leaf credentials private", + () => + Effect.gen(function* () { + const f = yield* fixture([ + AuthAccessReadScope, + AuthAccessWriteScope, + AuthOrchestrationReadScope, + ]); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + const connected = yield* f.service.connect( + leafId, + session([AuthOrchestrationReadScope, AuthOrchestrationOperateScope, AuthAccessWriteScope]), + ); + assert.strictEqual(connected.token, "private-leaf-session"); + const pairing = f.requests.find((r) => r.url.endsWith("delegated-session")); + assert.deepStrictEqual(JSON.parse(pairing?.body ?? "{}").scopes, [ + AuthOrchestrationReadScope, + ]); + assert.deepStrictEqual((yield* f.service.directory(session())).environments[0]?.scopes, [ + AuthOrchestrationReadScope, + ]); + const count = f.requests.length; + yield* f.service.connect( + leafId, + session([AuthOrchestrationReadScope, AuthOrchestrationOperateScope, AuthAccessWriteScope]), + ); + assert.strictEqual(f.requests.length, count); + }), +); + +it.effect( + "removing or replacing a machine invalidates its active proxy and rejects subsequent connections", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + const connected = yield* f.service.connect(leafId, session()); + yield* f.service.manage({ type: "hub.remove", environmentId: leafId }); + yield* connected.invalidated; + assert.strictEqual( + (yield* f.service.connect(leafId, session()).pipe(Effect.flip)).reason, + "not-found", + ); + assert.deepStrictEqual((yield* f.service.directory(session())).environments, []); + }), +); + +it.effect("checks machine identity again before releasing its saved delegation grant", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + f.changeIdentity(EnvironmentId.make("replacement")); + const count = f.requests.length; + assert.strictEqual( + (yield* f.service.connect(leafId, session()).pipe(Effect.flip)).reason, + "identity", + ); + assert.strictEqual(f.requests.length, count + 1); + assert.strictEqual(f.requests.at(-1)?.authorization, undefined); + }), +); + +it.effect( + "refreshes changed machine descriptors, persists them for offline discovery, and never hides an identity mismatch", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + f.changeVersion("updated"); + assert.strictEqual((yield* f.service.descriptor(leafId)).serverVersion, "updated"); + assert.strictEqual( + (yield* f.service.directory(session())).environments[0]?.descriptor.serverVersion, + "updated", + ); + f.setUnavailable(); + const restarted = yield* f.restart; + assert.strictEqual((yield* restarted.descriptor(leafId)).serverVersion, "updated"); + }), +); + +it.effect( + "rejects malformed or changed discovery identities instead of treating them as offline", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + f.changeIdentity(EnvironmentId.make("replacement")); + assert.strictEqual( + (yield* f.service.directory(session()).pipe(Effect.flip)).reason, + "identity", + ); + f.setMalformed(); + assert.strictEqual( + (yield* f.service.descriptor(leafId).pipe(Effect.flip)).reason, + "identity", + ); + }), +); + +it.effect.each(["modern", "legacy"] as const)( + "delegates granular permissions from a %s machine's auth response", + (permissions) => + Effect.gen(function* () { + const f = yield* fixture(AuthAdministrativeScopes, permissions); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + yield* f.enroll; + yield* f.service.connect(leafId, session()); + const pairing = f.requests.find((request) => request.url.endsWith("delegated-session")); + const scopes = JSON.parse(pairing?.body ?? "{}").scopes; + assert.include(scopes, AuthFilesystemReadScope); + assert.include(scopes, AuthPreviewOperateScope); + const directory = yield* f.service.directory(session()); + assert.strictEqual(directory.environments[0]?.scopes.includes(AuthFilesystemReadScope), true); + assert.strictEqual(directory.environments[0]?.scopes.includes(AuthPreviewOperateScope), true); + }), +); + +it.effect("does not expand an explicitly empty machine permission grant", () => + Effect.gen(function* () { + const f = yield* fixture(AuthAdministrativeScopes, "empty"); + yield* f.service.manage({ type: "hub.configure", publicUrl: "https://hub.example/" }); + assert.strictEqual((yield* f.enroll.pipe(Effect.flip)).reason, "credential"); + }), +); diff --git a/apps/server/src/environment/Hub.ts b/apps/server/src/environment/Hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..42d402d4ddab81919a68d28f7e16dc7bcc118a5f --- /dev/null +++ b/apps/server/src/environment/Hub.ts @@ -0,0 +1,417 @@ +import { + AuthAccessWriteScope, + AuthAccessReadScope, + AuthAdministrativeScopes, + AuthAccessTokenResult, + AuthEnvironmentScopes, + AuthSessionState, + AuthStandardClientScopes, + sessionGrantsScope, + EnvironmentId, + ExecutionEnvironmentDescriptor, + HubError, + type AuthEnvironmentScope, + type HubDirectory, + type HubManagementRequest, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Clock from "effect/Clock"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Redacted from "effect/Redacted"; +import * as Semaphore from "effect/Semaphore"; +import { + FetchHttpClient, + HttpBody, + HttpClient, + HttpClientRequest, + HttpClientResponse, +} from "effect/http"; + +import type { AuthenticatedSession } from "../auth/EnvironmentAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; +import * as OutboundHub from "./OutboundHub.ts"; +import * as HostedAuthConfig from "../auth/HostedAuthConfig.ts"; + +const isHubError = Schema.is(HubError); +const descriptorsEqual = Schema.toEquivalence(ExecutionEnvironmentDescriptor); + +const Machine = Schema.Struct({ + url: Schema.String, + token: Schema.NonEmptyString, + descriptor: ExecutionEnvironmentDescriptor, + scopes: AuthEnvironmentScopes, +}); +type Machine = typeof Machine.Type; +const Configuration = Schema.Struct({ + publicUrl: Schema.String, + machines: Schema.Array(Machine), +}); +const jsonConfiguration = Schema.fromJsonString(Configuration); + +export class Hub extends Context.Service< + Hub, + { + readonly directory: ( + session: AuthenticatedSession, + options?: { readonly refresh?: boolean }, + ) => Effect.Effect; + readonly manage: (input: HubManagementRequest) => Effect.Effect; + readonly descriptor: ( + id: EnvironmentId, + ) => Effect.Effect; + readonly authorize: ( + id: EnvironmentId, + session: AuthenticatedSession, + ) => Effect.Effect, HubError>; + readonly connect: ( + id: EnvironmentId, + session: AuthenticatedSession, + ) => Effect.Effect< + { + readonly url: string; + readonly token: string; + readonly invalidated: Effect.Effect; + }, + HubError + >; + } +>()("t3/environment/Hub") {} + +const normalizeUrl = (value: string) => + Effect.try({ + try: () => { + const url = new URL(value); + if ( + !["http:", "https:"].includes(url.protocol) || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error("Expected an HTTP base URL without credentials, query, or fragment."); + } + url.pathname = `${url.pathname.replace(/\/+$/, "")}/`; + return url.href; + }, + catch: (cause) => new HubError({ reason: "address", cause: Redacted.make(cause) }), + }); + +const grantScopes = (machine: Machine, session: AuthenticatedSession) => + AuthStandardClientScopes.filter( + (scope) => machine.scopes.includes(scope) && session.scopes.includes(scope), + ); + +/** @public Service construction is part of the canonical Effect module API. */ +export const make = Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const files = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const environment = yield* ServerEnvironment.ServerEnvironment; + const outbound = yield* OutboundHub.OutboundHub; + const hosted = yield* HostedAuthConfig.HostedAuthConfig; + const lock = yield* Semaphore.make(1); + const stored = yield* secrets + .get("hub") + .pipe( + Effect.mapError((cause) => new HubError({ reason: "storage", cause: Redacted.make(cause) })), + ); + let configuration = Option.isSome(stored) + ? yield* Schema.decodeEffect(jsonConfiguration)(new TextDecoder().decode(stored.value)).pipe( + Effect.mapError( + (cause) => new HubError({ reason: "storage", cause: Redacted.make(cause) }), + ), + ) + : ({ publicUrl: "", machines: [] } satisfies typeof Configuration.Type); + const removals = new Map>(); + for (const machine of configuration.machines) { + removals.set(machine.descriptor.environmentId, yield* Deferred.make()); + } + const credentials = new Map(); + const publicUrl = () => + Option.isSome(hosted) ? `${hosted.value.publicUrl}/` : configuration.publicUrl; + + const request = ( + machine: Pick, + path: string, + schema: Schema.Decoder, + body?: HttpBody.HttpBody, + ) => + Effect.gen(function* () { + let outgoing = HttpClientRequest.make(body ? "POST" : "GET")(new URL(path, machine.url).href); + if (machine.token) outgoing = HttpClientRequest.bearerToken(outgoing, machine.token); + if (body) outgoing = HttpClientRequest.setBody(outgoing, body); + const response = yield* client + .execute(outgoing) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })); + if (response.status !== 200) + return yield* new HubError({ + reason: response.status === 401 || response.status === 403 ? "credential" : "upstream", + }); + return yield* HttpClientResponse.schemaBodyJson(schema)(response).pipe( + Effect.mapError( + (cause) => + new HubError({ + reason: path === ".well-known/t3/environment" ? "identity" : "upstream", + cause: Redacted.make(cause), + }), + ), + ); + }).pipe( + Effect.timeout("10 seconds"), + Effect.mapError((cause) => + isHubError(cause) + ? cause + : new HubError({ reason: "upstream", cause: Redacted.make(cause) }), + ), + Effect.scoped, + ); + + const find = (id: EnvironmentId): Effect.Effect => { + const machine = configuration.machines.find((value) => value.descriptor.environmentId === id); + return machine ? Effect.succeed(machine) : Effect.fail(new HubError({ reason: "not-found" })); + }; + const persist = (next: typeof Configuration.Type) => + Schema.encodeEffect(jsonConfiguration)(next).pipe( + Effect.flatMap((json) => secrets.set("hub", new TextEncoder().encode(json))), + Effect.mapError((cause) => new HubError({ reason: "storage", cause: Redacted.make(cause) })), + Effect.tap(() => + Effect.sync(() => { + configuration = next; + }), + ), + ); + + const refreshDescriptor = (id: EnvironmentId) => + Effect.gen(function* () { + const remote = (yield* outbound.enrollments).find( + (value) => value.descriptor.environmentId === id, + ); + if (remote) return remote.descriptor; + const machine = yield* find(id); + const descriptor = yield* request( + { ...machine, token: "" }, + ".well-known/t3/environment", + ExecutionEnvironmentDescriptor, + ).pipe( + Effect.catchTags({ + HubError: (error) => + error.reason === "upstream" ? Effect.succeed(machine.descriptor) : Effect.fail(error), + }), + ); + if (descriptor.environmentId !== id) return yield* new HubError({ reason: "identity" }); + if (!descriptorsEqual(descriptor, machine.descriptor)) { + yield* lock.withPermit( + Effect.gen(function* () { + const current = yield* find(id); + if (current.url !== machine.url) return; + yield* persist({ + ...configuration, + machines: configuration.machines.map((value) => + value.descriptor.environmentId === id ? { ...value, descriptor } : value, + ), + }); + }), + ); + } + return descriptor; + }); + + return Hub.of({ + directory: (session, options) => + Effect.gen(function* () { + const baseUrl = publicUrl(); + if (!baseUrl) return yield* new HubError({ reason: "configuration" }); + const registered = yield* outbound.enrollments; + return { + hubEnvironmentId: yield* environment.getEnvironmentId, + environments: yield* Effect.forEach( + [ + ...configuration.machines.filter( + (machine) => + !registered.some( + (value) => value.descriptor.environmentId === machine.descriptor.environmentId, + ), + ), + ...registered.map((value) => ({ + descriptor: value.descriptor, + scopes: AuthStandardClientScopes, + })), + ], + (machine) => + Effect.gen(function* () { + const descriptor = + options?.refresh === false + ? machine.descriptor + : yield* refreshDescriptor(machine.descriptor.environmentId); + return { + environmentId: machine.descriptor.environmentId, + label: descriptor.label, + descriptor, + httpBaseUrl: new URL( + `environments/${encodeURIComponent(machine.descriptor.environmentId)}/`, + baseUrl, + ).href, + scopes: AuthStandardClientScopes.filter( + (scope) => machine.scopes.includes(scope) && session.scopes.includes(scope), + ), + }; + }), + { concurrency: "unbounded" }, + ), + }; + }), + descriptor: refreshDescriptor, + authorize: (id, session) => + Effect.gen(function* () { + if ((yield* outbound.enrollments).some((value) => value.descriptor.environmentId === id)) { + const scopes = AuthStandardClientScopes.filter((scope) => session.scopes.includes(scope)); + if (scopes.length === 0) return yield* new HubError({ reason: "credential" }); + return scopes; + } + return yield* find(id).pipe( + Effect.flatMap((machine) => { + const scopes = grantScopes(machine, session); + return scopes.length > 0 + ? Effect.succeed(scopes) + : Effect.fail(new HubError({ reason: "credential" })); + }), + ); + }), + manage: (input) => + lock.withPermit( + Effect.gen(function* () { + if (input.type === "hub.status") return; + if (input.type === "hub.configure") { + const publicUrl = yield* normalizeUrl(input.publicUrl); + yield* persist({ ...configuration, publicUrl }); + return; + } + if (!publicUrl()) return yield* new HubError({ reason: "configuration" }); + if (input.type === "hub.remove") { + if ( + (yield* outbound.enrollments).some( + (value) => value.descriptor.environmentId === input.environmentId, + ) + ) { + yield* outbound.remove(input.environmentId); + return; + } + yield* find(input.environmentId); + yield* persist({ + ...configuration, + machines: configuration.machines.filter( + (machine) => machine.descriptor.environmentId !== input.environmentId, + ), + }); + const removal = removals.get(input.environmentId); + if (removal) yield* Deferred.succeed(removal, undefined); + removals.delete(input.environmentId); + for (const key of credentials.keys()) + if (key.startsWith(`${input.environmentId}:`)) credentials.delete(key); + return; + } + const url = yield* normalizeUrl(input.url); + const token = (yield* files + .readFileString(input.tokenFile) + .pipe( + Effect.mapError( + (cause) => new HubError({ reason: "storage", cause: Redacted.make(cause) }), + ), + )).trim(); + const descriptor = yield* request( + { url, token: "" }, + ".well-known/t3/environment", + ExecutionEnvironmentDescriptor, + ); + if ( + descriptor.environmentId !== input.environmentId || + descriptor.environmentId === (yield* environment.getEnvironmentId) + ) { + return yield* new HubError({ reason: "identity" }); + } + const state = yield* request({ url, token }, "api/auth/session", AuthSessionState); + if ( + !sessionGrantsScope(state, AuthAccessReadScope) || + !sessionGrantsScope(state, AuthAccessWriteScope) + ) { + return yield* new HubError({ reason: "credential" }); + } + yield* persist({ + ...configuration, + machines: [ + ...configuration.machines.filter( + (machine) => machine.descriptor.environmentId !== input.environmentId, + ), + { + url, + token, + descriptor, + scopes: AuthAdministrativeScopes.filter((scope) => + sessionGrantsScope(state, scope), + ), + }, + ], + }); + const removal = removals.get(input.environmentId); + if (removal) yield* Deferred.succeed(removal, undefined); + removals.set(input.environmentId, yield* Deferred.make()); + for (const key of credentials.keys()) + if (key.startsWith(`${input.environmentId}:`)) credentials.delete(key); + }), + ), + connect: (id, session) => + lock.withPermit( + Effect.gen(function* () { + if ( + (yield* outbound.enrollments).some((value) => value.descriptor.environmentId === id) + ) { + return yield* outbound.connect(id, session); + } + const machine = yield* find(id); + const scopes = grantScopes(machine, session); + if (scopes.length === 0) return yield* new HubError({ reason: "credential" }); + const key = `${id}:${session.sessionId}:${scopes.toSorted().join(" ")}`; + const now = yield* Clock.currentTimeMillis; + for (const [key, credential] of credentials) { + if (credential.expiresAt <= now + 5_000) credentials.delete(key); + } + const cached = credentials.get(key); + let token = cached && cached.expiresAt > now + 5_000 ? cached.token : undefined; + if (!token) { + // A changed endpoint must prove the enrolled identity before receiving its private grant. + const descriptor = yield* request( + { ...machine, token: "" }, + ".well-known/t3/environment", + ExecutionEnvironmentDescriptor, + ); + if (descriptor.environmentId !== id) return yield* new HubError({ reason: "identity" }); + const issued = yield* request( + machine, + "api/auth/delegated-session", + AuthAccessTokenResult, + HttpBody.jsonUnsafe({ scopes, label: "Self-hosted server client" }), + ); + if ( + issued.token_type !== "Bearer" || + issued.scope.split(" ").some((scope) => !scopes.some((granted) => granted === scope)) + ) { + return yield* new HubError({ reason: "credential" }); + } + token = issued.access_token; + credentials.set(key, { token, expiresAt: now + issued.expires_in * 1_000 }); + } + const removal = removals.get(id); + if (!removal) return yield* new HubError({ reason: "not-found" }); + return { url: machine.url, token, invalidated: Deferred.await(removal) }; + }), + ), + }); +}); + +export const layer = Layer.effect(Hub, make); diff --git a/apps/server/src/environment/HubHttp.ts b/apps/server/src/environment/HubHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..23cfed77ca1c412642525b418ffc281357c7d0c4 --- /dev/null +++ b/apps/server/src/environment/HubHttp.ts @@ -0,0 +1,89 @@ +import { + AuthAccessWriteScope, + EnvironmentId, + HubError, + HubManagementRequest, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as HubProxy from "./HubProxy.ts"; +import * as Schema from "effect/Schema"; +import * as Hub from "./Hub.ts"; + +const isHubError = Schema.is(HubError); + +const errors = (effect: Effect.Effect) => + effect.pipe( + Effect.catch((error) => { + if (isHubError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { _tag: error._tag, reason: error.reason, message: error.message }, + { + status: + error.reason === "not-found" + ? 404 + : error.reason === "credential" + ? 403 + : error.reason === "address" + ? 400 + : 502, + }, + ), + ); + if (EnvironmentAuth.isServerAuthCredentialError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe({ code: "unauthorized" }, { status: 401 }), + ); + return Effect.succeed( + HttpServerResponse.jsonUnsafe({ code: "internal_error" }, { status: 500 }), + ); + }), + ); + +const authenticate = Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const request = yield* HttpServerRequest.HttpServerRequest; + return yield* auth.authenticateWebSocketUpgrade(request); +}); + +const directory = Effect.gen(function* () { + const session = yield* authenticate; + return HttpServerResponse.jsonUnsafe( + yield* (yield* Hub.Hub).directory(session, { refresh: false }), + { + headers: { "cache-control": "no-store" }, + }, + ); +}); + +const manage = Effect.gen(function* () { + const session = yield* authenticate; + if (!session.scopes.includes(AuthAccessWriteScope)) + return HttpServerResponse.empty({ status: 403 }); + const input = yield* HttpServerRequest.schemaBodyJson(HubManagementRequest); + const hub = yield* Hub.Hub; + yield* hub.manage(input); + return HttpServerResponse.jsonUnsafe(yield* hub.directory(session, { refresh: false }), { + headers: { "cache-control": "no-store" }, + }); +}); + +const proxy = Effect.gen(function* () { + const params = yield* HttpRouter.params; + const request = yield* HttpServerRequest.HttpServerRequest; + const proxy = yield* HubProxy.HubProxy; + return yield* proxy.forward( + request, + EnvironmentId.make(params.environmentId ?? ""), + `/${params["*"] ?? ""}`, + ); +}); + +export const layer = Layer.mergeAll( + HttpRouter.add("GET", "/api/hub/environments", directory.pipe(errors)), + HttpRouter.add("POST", "/api/hub/manage", manage.pipe(errors)), + HttpRouter.add("*", "/environments/:environmentId/*", proxy.pipe(errors)), +); diff --git a/apps/server/src/environment/HubInbox.test.ts b/apps/server/src/environment/HubInbox.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..98c8355dee2e7e7376e9653e9ac3247ca1f2015a --- /dev/null +++ b/apps/server/src/environment/HubInbox.test.ts @@ -0,0 +1,536 @@ +import { assert, it } from "@effect/vitest"; +import { + AuthAdministrativeScopes, + AuthOrchestrationOperateScope, + AuthOrchestrationReadScope, + AuthSessionId, + CommandId, + EnvironmentId, + HubError, + HubOrganizationCommand, + ORCHESTRATION_PROTOCOL_VERSION, + OrchestrationV2ShellSnapshot, + OrchestrationV2ThreadShell, + ThreadId, + type AuthEnvironmentScope, + type ExecutionEnvironmentDescriptor, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as ArchiveCatalog from "../archive/ArchiveCatalog.ts"; +import * as ArchiveObjects from "../archive/ArchiveObjects.ts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as Hub from "./Hub.ts"; +import * as HubInbox from "./HubInbox.ts"; +import { hubInboxThread as thread } from "./HubInbox.testkit.ts"; + +const encodeSnapshot = Schema.encodeEffect(Schema.toCodecJson(OrchestrationV2ShellSnapshot)); +const decodeCommand = Schema.decodeEffect( + Schema.fromJsonString(Schema.toCodecJson(HubOrganizationCommand)), +); + +const environmentId = EnvironmentId.make("laptop"); +const threadId = ThreadId.make("chat"); +const session = { + sessionId: AuthSessionId.make("phone"), + subject: "phone", + method: "bearer-access-token" as const, + scopes: AuthAdministrativeScopes, +}; + +const fixture = Effect.gen(function* () { + let snapshot: OrchestrationV2ShellSnapshot = { + schemaVersion: 2, + snapshotSequence: 0, + projects: [], + threads: [thread], + archivedThreads: [], + }; + let online = true; + let scopes: ReadonlyArray = AuthAdministrativeScopes; + let rejection = false; + let beforeDispatch: Effect.Effect = Effect.void; + let afterDispatch: Effect.Effect = Effect.void; + const commands: Array = []; + const client = HttpClient.make((request) => + Effect.gen(function* () { + if (!online) return HttpClientResponse.fromWeb(request, new Response(null, { status: 503 })); + assert.equal(request.headers.authorization, "Bearer delegated-token"); + if (request.method === "GET") + return HttpClientResponse.fromWeb(request, Response.json(yield* encodeSnapshot(snapshot))); + if (rejection) + return HttpClientResponse.fromWeb(request, new Response(null, { status: 409 })); + assert.equal(request.url, "http://source.private/api/hub/organization"); + if (request.body._tag !== "Uint8Array") return yield* Effect.die("Missing command body"); + const command = yield* decodeCommand(new TextDecoder().decode(request.body.body)); + commands.push(command); + yield* beforeDispatch; + const now = yield* DateTime.now; + snapshot = { + ...snapshot, + snapshotSequence: snapshot.snapshotSequence + 1, + threads: snapshot.threads.map((value) => { + switch (command.type) { + case "thread.settle": + return { + ...value, + settledOverride: "settled" as const, + settledAt: command.settledAt ?? now, + pinnedAt: null, + pinOrderKey: null, + activeOrderKey: null, + }; + case "thread.unsettle": + return { + ...value, + settledOverride: "active" as const, + settledAt: null, + unsettledAt: now, + }; + case "thread.pin": + return { + ...value, + pinnedAt: now, + pinOrderKey: command.orderKey ?? null, + snoozedUntil: null, + snoozedAt: null, + }; + case "thread.unpin": + return { ...value, pinnedAt: null, pinOrderKey: null }; + case "thread.snooze": + return { + ...value, + snoozedUntil: DateTime.makeUnsafe(command.snoozedUntil), + snoozedAt: now, + }; + case "thread.unsnooze": + return { ...value, snoozedUntil: null, snoozedAt: null }; + case "thread.pin.reorder": + return { ...value, pinOrderKey: command.orderKey }; + case "thread.active.reorder": + return { ...value, activeOrderKey: command.orderKey }; + case "thread.auto-settle.set": + return { ...value, autoSettleDisabledAt: command.enabled ? null : now }; + } + }), + }; + const sequence = snapshot.snapshotSequence; + yield* afterDispatch; + return HttpClientResponse.fromWeb(request, Response.json({ sequence })); + }).pipe(Effect.orDie), + ); + const descriptor: ExecutionEnvironmentDescriptor = { + environmentId, + label: "Laptop", + platform: { os: "linux", arch: "x64" }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: true }, + }; + const hub = Hub.Hub.of({ + directory: (current, options) => { + if (current.subject !== "hub-inbox") assert.equal(options?.refresh, false); + return Effect.succeed({ + hubEnvironmentId: EnvironmentId.make("hub"), + environments: [ + { + environmentId, + label: "Laptop", + httpBaseUrl: "http://hub/environments/laptop/", + descriptor, + scopes: scopes.filter((scope) => current.scopes.includes(scope)), + }, + ], + }); + }, + authorize: (id, current) => + id === environmentId + ? Effect.succeed(scopes.filter((scope) => current.scopes.includes(scope))) + : Effect.fail(new HubError({ reason: "not-found" })), + connect: () => + Effect.succeed({ + url: "http://source.private/", + token: "delegated-token", + invalidated: Effect.never, + }), + descriptor: () => Effect.succeed(descriptor), + manage: () => Effect.die("Unused"), + }); + const layer = HubInbox.layer.pipe( + Layer.provide( + ArchiveCatalog.layer.pipe( + Layer.provide(ArchiveObjects.layer), + Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({}))), + Layer.provide(NodeServices.layer), + ), + ), + Layer.provide(Layer.succeed(Hub.Hub, hub)), + Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), + ); + const restart = Layer.build(layer).pipe( + Effect.map((context) => Context.get(context, HubInbox.HubInbox)), + ); + const inbox = yield* restart; + yield* inbox.refresh; + yield* inbox.refresh; + let sequence = 0; + type CommandInput = { + [T in HubOrganizationCommand["type"]]: Omit< + Extract, + "commandId" | "threadId" + >; + }[HubOrganizationCommand["type"]]; + const organize = (command: CommandInput) => + inbox.organize( + { + environmentId, + command: { commandId: CommandId.make(`command-${++sequence}`), threadId, ...command }, + }, + session, + ); + const read = inbox.read(session).pipe(Effect.map((value) => value.environments[0]!)); + return { + inbox, + restart, + organize, + read, + commands, + online: (value: boolean) => { + online = value; + }, + reject: (value: boolean) => { + rejection = value; + }, + scopes: (value: ReadonlyArray) => { + scopes = value; + }, + source: (update: Partial) => { + snapshot = { + ...snapshot, + snapshotSequence: snapshot.snapshotSequence + 1, + threads: [{ ...snapshot.threads[0]!, ...update }], + }; + }, + beforeDispatch: (effect: Effect.Effect) => { + beforeDispatch = effect; + }, + afterDispatch: (effect: Effect.Effect) => { + afterDispatch = effect; + }, + removeThread: () => { + snapshot = { ...snapshot, threads: [] }; + }, + restoreThread: () => { + snapshot = { ...snapshot, threads: [thread] }; + }, + }; +}); + +it.effect("discovers a newly created source chat when organized before the next catalog poll", () => + Effect.gen(function* () { + const f = yield* fixture; + f.removeThread(); + yield* f.inbox.refresh; + assert.deepEqual((yield* f.read).snapshot.threads, []); + f.restoreThread(); + const entry = yield* f.organize({ type: "thread.settle" }); + assert.equal(entry.snapshot.threads[0]?.settledOverride, "settled"); + assert.deepEqual(entry.pendingThreadIds, [threadId]); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "supports the organization commands and reversals without changing execution metadata", + () => + Effect.gen(function* () { + const f = yield* fixture; + const now = yield* DateTime.now; + f.online(false); + yield* f.organize({ + type: "thread.snooze", + snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }); + const settled = yield* f.organize({ type: "thread.settle" }); + assert.isNull(settled.snapshot.threads[0]?.snoozedUntil); + assert.isNull(settled.snapshot.threads[0]?.snoozedAt); + yield* f.organize({ type: "thread.unsettle", reason: "user" }); + yield* f.organize({ + type: "thread.snooze", + snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }); + yield* f.organize({ type: "thread.unsnooze", reason: "user" }); + yield* f.organize({ type: "thread.pin", orderKey: "a0" }); + const reordered = yield* f.organize({ type: "thread.pin.reorder", orderKey: "a1" }); + assert.equal(reordered.snapshot.threads[0]?.pinOrderKey, "a1"); + yield* f.organize({ type: "thread.unpin" }); + yield* f.organize({ type: "thread.active.reorder", orderKey: "a2" }); + yield* f.organize({ type: "thread.auto-settle.set", enabled: false }); + const enabled = yield* f.organize({ type: "thread.auto-settle.set", enabled: true }); + assert.isNull(enabled.snapshot.threads[0]?.autoSettleDisabledAt); + assert.isNull(enabled.snapshot.threads[0]?.pinnedAt); + assert.equal(enabled.snapshot.threads[0]?.activeOrderKey, "a2"); + assert.equal(enabled.snapshot.threads[0]?.status, "idle"); + f.online(true); + yield* f.inbox.refresh; + assert.deepEqual((yield* f.read).pendingThreadIds, []); + assert.equal((yield* f.read).snapshot.threads[0]?.activeOrderKey, "a2"); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "removes pending organization when a fresh source shell no longer contains the chat", + () => + Effect.gen(function* () { + const f = yield* fixture; + f.online(false); + yield* f.organize({ type: "thread.settle" }); + f.removeThread(); + f.online(true); + yield* f.inbox.refresh; + const entry = yield* f.read; + assert.deepEqual(entry.pendingThreadIds, []); + assert.deepEqual(entry.snapshot.threads, []); + assert.deepEqual(f.commands, []); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("accepts a newer direct source change after the central command committed", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.organize({ type: "thread.settle" }); + f.afterDispatch(Effect.sync(() => f.source({ settledOverride: "active", settledAt: null }))); + yield* f.inbox.refresh; + const entry = yield* f.read; + assert.deepEqual(entry.pendingThreadIds, []); + assert.equal(entry.snapshot.threads[0]?.settledOverride, "active"); + assert.equal(f.commands.filter((value) => value.type === "thread.settle").length, 1); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("recognizes new source-authored work without comparing clocks across machines", () => + Effect.gen(function* () { + const f = yield* fixture; + f.source({ latestUserAuthoredMessageAt: DateTime.makeUnsafe("2020-01-01T00:00:00Z") }); + yield* f.inbox.refresh; + f.online(false); + yield* f.organize({ type: "thread.settle" }); + f.source({ + latestUserAuthoredMessageAt: DateTime.makeUnsafe("2020-01-01T00:00:01Z"), + status: "running", + }); + f.online(true); + yield* f.inbox.refresh; + assert.equal((yield* f.read).snapshot.threads[0]?.settledOverride, "active"); + assert.isFalse(f.commands.some((value) => value.type === "thread.settle")); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "persists offline organization, shares it with another client, and restores it after restart", + () => + Effect.gen(function* () { + const f = yield* fixture; + f.online(false); + yield* f.inbox.refresh; + const settled = yield* f.organize({ type: "thread.settle" }); + assert.equal(settled.snapshot.threads[0]?.settledOverride, "settled"); + assert.deepEqual(settled.pendingThreadIds, [threadId]); + const second = { ...session, sessionId: AuthSessionId.make("second-phone") }; + assert.equal( + (yield* f.inbox.read(second)).environments[0]?.snapshot.threads[0]?.settledOverride, + "settled", + ); + const restarted = yield* f.restart; + assert.equal( + (yield* restarted.read(session)).environments[0]?.snapshot.threads[0]?.settledOverride, + "settled", + ); + assert.deepEqual(f.commands, []); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "wakes a settle when a new source message arrives during dispatch and reconciles it active", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.organize({ type: "thread.settle" }); + const now = yield* DateTime.now; + f.beforeDispatch( + Effect.sync(() => f.source({ latestUserMessageAt: DateTime.add(now, { seconds: 1 }) })), + ); + yield* f.inbox.refresh; + const raced = yield* f.read; + assert.equal(raced.snapshot.threads[0]?.settledOverride, "active"); + assert.deepEqual(raced.pendingThreadIds, [threadId]); + assert.equal(f.commands.filter((value) => value.type === "thread.settle").length, 1); + f.beforeDispatch(Effect.void); + yield* f.inbox.refresh; + const confirmed = yield* f.read; + assert.equal(confirmed.snapshot.threads[0]?.settledOverride, "active"); + assert.deepEqual(confirmed.pendingThreadIds, []); + assert.equal(f.commands.filter((value) => value.type === "thread.unsettle").length, 1); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("reconciles current intent after reconnect without replaying an undone settle", () => + Effect.gen(function* () { + const f = yield* fixture; + f.online(false); + yield* f.organize({ type: "thread.settle" }); + yield* f.organize({ type: "thread.unsettle", reason: "user" }); + yield* f.organize({ type: "thread.pin", orderKey: "a0" }); + f.online(true); + yield* f.inbox.refresh; + assert.isFalse(f.commands.some((value) => value.type === "thread.settle")); + const result = yield* f.read; + assert.deepEqual(result.pendingThreadIds, []); + assert.isNotNull(result.snapshot.threads[0]?.pinnedAt); + assert.equal(result.snapshot.threads[0]?.settledOverride, "active"); + f.source({ pinnedAt: null, pinOrderKey: null }); + yield* f.inbox.refresh; + assert.isNull((yield* f.read).snapshot.threads[0]?.pinnedAt); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("keeps execution state and pending intent when source settlement is blocked", () => + Effect.gen(function* () { + const f = yield* fixture; + f.source({ status: "running" }); + yield* f.inbox.refresh; + f.reject(true); + yield* f.organize({ type: "thread.settle" }); + yield* f.inbox.refresh; + const blocked = yield* f.read; + assert.equal(blocked.snapshot.threads[0]?.status, "running"); + assert.equal(blocked.snapshot.threads[0]?.settledOverride, "settled"); + assert.deepEqual(blocked.pendingThreadIds, [threadId]); + f.source({ status: "completed" }); + f.reject(false); + yield* f.inbox.refresh; + assert.deepEqual((yield* f.read).pendingThreadIds, []); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("expires snooze while the source is offline and never resends an elapsed snooze", () => + Effect.gen(function* () { + const f = yield* fixture; + const now = yield* DateTime.now; + f.online(false); + yield* f.organize({ + type: "thread.snooze", + snoozedUntil: DateTime.formatIso(DateTime.add(now, { minutes: 1 })), + }); + yield* TestClock.adjust("2 minutes"); + assert.isNull((yield* f.read).snapshot.threads[0]?.snoozedUntil); + f.online(true); + yield* f.inbox.refresh; + assert.isFalse(f.commands.some((value) => value.type === "thread.snooze")); + assert.deepEqual((yield* f.read).pendingThreadIds, []); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("checks current hub and enrollment grants before accepting offline changes", () => + Effect.gen(function* () { + const f = yield* fixture; + f.online(false); + f.scopes([AuthOrchestrationReadScope]); + assert.equal((yield* f.organize({ type: "thread.settle" }).pipe(Effect.flip))._tag, "HubError"); + f.scopes(AuthAdministrativeScopes); + assert.equal( + (yield* f.inbox + .organize( + { + environmentId, + command: { type: "thread.settle", commandId: CommandId.make("denied"), threadId }, + }, + { ...session, scopes: [AuthOrchestrationReadScope] }, + ) + .pipe(Effect.flip))._tag, + "HubError", + ); + assert.deepEqual( + (yield* f.inbox.read({ ...session, scopes: [AuthOrchestrationOperateScope] })).environments, + [], + ); + assert.isNull((yield* f.read).snapshot.threads[0]?.settledOverride); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect("does not acknowledge a newer command while an older source request is in flight", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.organize({ type: "thread.settle" }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + f.beforeDispatch( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(release))), + ); + const refresh = yield* f.inbox.refresh.pipe(Effect.forkScoped); + yield* Deferred.await(started); + yield* f.organize({ type: "thread.unsettle", reason: "user" }); + const releaseReversal = yield* Deferred.make(); + f.beforeDispatch(Deferred.await(releaseReversal)); + yield* Deferred.succeed(release, undefined); + const pending = yield* f.read; + assert.equal(pending.snapshot.threads[0]?.settledOverride, "active"); + assert.deepEqual(pending.pendingThreadIds, [threadId]); + yield* Deferred.succeed(releaseReversal, undefined); + yield* Fiber.join(refresh); + yield* f.inbox.refresh; + assert.deepEqual((yield* f.read).pendingThreadIds, []); + assert.equal((yield* f.read).snapshot.threads[0]?.settledOverride, "active"); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "deduplicates retried commands after later organization changes and acknowledgement", + () => + Effect.gen(function* () { + const f = yield* fixture; + const command = { + type: "thread.settle" as const, + threadId, + commandId: CommandId.make("retry-settle"), + }; + yield* f.inbox.organize({ environmentId, command }, session); + yield* f.inbox.refresh; + yield* f.organize({ type: "thread.unsettle", reason: "user" }); + yield* f.inbox.refresh; + const retry = yield* f.inbox.organize({ environmentId, command }, session); + assert.equal(retry.snapshot.threads[0]?.settledOverride, "active"); + assert.deepEqual(retry.pendingThreadIds, []); + assert.equal(f.commands.filter((value) => value.type === "thread.settle").length, 1); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); + +it.effect( + "wakes a pending snooze on fresh source completion and preserves its execution status", + () => + Effect.gen(function* () { + const f = yield* fixture; + const now = yield* DateTime.now; + f.online(false); + yield* f.organize({ + type: "thread.snooze", + snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }); + f.source({ status: "completed", latestRunCompletedAt: DateTime.add(now, { minutes: 1 }) }); + f.online(true); + yield* f.inbox.refresh; + const entry = yield* f.read; + assert.isNull(entry.snapshot.threads[0]?.snoozedUntil); + assert.equal(entry.snapshot.threads[0]?.status, "completed"); + assert.deepEqual(entry.pendingThreadIds, []); + assert.isFalse(f.commands.some((value) => value.type === "thread.snooze")); + }).pipe(Effect.provide(SqlitePersistence.layerMemory), Effect.scoped), +); diff --git a/apps/server/src/environment/HubInbox.testkit.ts b/apps/server/src/environment/HubInbox.testkit.ts new file mode 100644 index 0000000000000000000000000000000000000000..f739232da0c19c8c23f2a4c1d5adaa8ccbaec450 --- /dev/null +++ b/apps/server/src/environment/HubInbox.testkit.ts @@ -0,0 +1,43 @@ +import { OrchestrationV2ThreadShell, ThreadId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Schema from "effect/Schema"; + +const threadId = ThreadId.make("chat"); +export const hubInboxThread = Schema.decodeSync(OrchestrationV2ThreadShell)({ + id: threadId, + projectId: "project", + title: "Chat on a sleeping computer", + providerInstanceId: "codex", + modelSelection: { instanceId: "codex", model: "gpt-6" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + createdAt: DateTime.makeUnsafe(0), + updatedAt: DateTime.makeUnsafe(0), + archivedAt: null, + settledOverride: null, + settledAt: null, + snoozedUntil: null, + snoozedAt: null, + pinnedAt: null, + pinOrderKey: null, + activeOrderKey: null, + autoSettleDisabledAt: null, + lastVisitedAt: null, + deletedAt: null, + latestRunId: null, + activeRunId: null, + status: "idle", + pendingRuntimeRequest: null, + latestVisibleMessage: null, + latestUserMessageAt: null, + hasActionableProposedPlan: false, + itemCount: 0, + visibleItemCount: 0, +}); diff --git a/apps/server/src/environment/HubInbox.ts b/apps/server/src/environment/HubInbox.ts new file mode 100644 index 0000000000000000000000000000000000000000..34d608a50c5606dd218a285a3ec386d2e09d5749 --- /dev/null +++ b/apps/server/src/environment/HubInbox.ts @@ -0,0 +1,661 @@ +import { + AuthAdministrativeScopes, + AuthArchiveReadScope, + AuthOrchestrationOperateScope, + AuthOrchestrationReadScope, + AuthSessionId, + CommandId, + HubError, + HubInboxEnvironment, + HubOrganizationCommand, + HUB_THREAD_ORGANIZATION_FIELDS, + ORCHESTRATION_PROTOCOL_HEADER, + ORCHESTRATION_PROTOCOL_VERSION_TEXT, + OrchestrationV2ShellSnapshot, + OrchestrationV2ThreadShell, + type EnvironmentId, + type HubInbox as HubInboxResult, + type HubInboxCommand, + type ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Struct from "effect/Struct"; +import { + FetchHttpClient, + HttpBody, + HttpClient, + HttpClientRequest, + HttpClientResponse, +} from "effect/http"; +import type { AuthenticatedSession } from "../auth/EnvironmentAuth.ts"; +import * as Hub from "./Hub.ts"; +import * as ArchiveCatalog from "../archive/ArchiveCatalog.ts"; +import { ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION } from "../orchestration-v2/ProjectionStore.ts"; + +const Organization = Schema.Struct( + Struct.pick(OrchestrationV2ThreadShell.fields, HUB_THREAD_ORGANIZATION_FIELDS), +); +type Organization = typeof Organization.Type; +const jsonOrganization = Schema.fromJsonString(Schema.toCodecJson(Organization)); +const jsonSnapshot = Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ShellSnapshot)); +const SourceActivity = Schema.Struct({ + userMessageAt: Schema.NullOr(Schema.Number), + completedAt: Schema.NullOr(Schema.Number), + pendingRequestId: Schema.NullOr(Schema.String), +}); +const jsonSourceActivity = Schema.fromJsonString(SourceActivity); +const encodeCommand = Schema.encodeEffect(Schema.toCodecJson(HubOrganizationCommand)); +const isHubError = Schema.is(HubError); +const sourceActivity = (thread: OrchestrationV2ThreadShell): typeof SourceActivity.Type => { + const authoredAt = thread.latestUserAuthoredMessageAt ?? thread.latestUserMessageAt; + return { + userMessageAt: authoredAt == null ? null : DateTime.toEpochMillis(authoredAt), + completedAt: + thread.latestRunCompletedAt == null + ? null + : DateTime.toEpochMillis(thread.latestRunCompletedAt), + pendingRequestId: thread.pendingRuntimeRequest?.id ?? null, + }; +}; + +const internalSession: AuthenticatedSession = { + sessionId: AuthSessionId.make("hub-inbox"), + subject: "hub-inbox", + method: "bearer-access-token", + scopes: AuthAdministrativeScopes, +}; + +export class HubInboxError extends Schema.TaggedError()("HubInboxError", { + reason: Schema.Literals(["thread-missing", "thread-state", "storage"]), + cause: Schema.optional(Schema.Redacted(Schema.Defect())), +}) { + override get message(): string { + switch (this.reason) { + case "thread-missing": + return "This chat is unavailable. Reconnect its computer to refresh the chat list."; + case "thread-state": + return "The chat changed. Refresh its status and try again."; + case "storage": + return "The server could not save the chat. Check its storage and try again."; + } + } +} + +export class HubInbox extends Context.Service< + HubInbox, + { + readonly read: ( + session: AuthenticatedSession, + ) => Effect.Effect; + readonly organize: ( + input: HubInboxCommand, + session: AuthenticatedSession, + ) => Effect.Effect; + readonly refresh: Effect.Effect; + } +>()("t3/environment/HubInbox") {} + +const applyCommand = ( + value: Organization, + command: HubOrganizationCommand, + now: DateTime.Utc, +): Organization => { + switch (command.type) { + case "thread.settle": + return { + ...value, + settledOverride: "settled", + settledAt: + value.settledOverride === "settled" ? value.settledAt : (command.settledAt ?? now), + unsettledAt: null, + pinnedAt: null, + pinOrderKey: null, + activeOrderKey: null, + snoozedUntil: null, + snoozedAt: null, + }; + case "thread.unsettle": + return { + ...value, + settledOverride: "active", + settledAt: null, + unsettledAt: value.settledOverride === "active" ? value.unsettledAt : now, + }; + case "thread.snooze": + return { ...value, snoozedUntil: DateTime.makeUnsafe(command.snoozedUntil), snoozedAt: now }; + case "thread.unsnooze": + return { ...value, snoozedUntil: null, snoozedAt: null }; + case "thread.pin": + return { + ...value, + pinnedAt: value.pinnedAt ?? now, + pinOrderKey: + value.pinnedAt == null ? (command.orderKey ?? value.pinOrderKey) : value.pinOrderKey, + settledOverride: value.settledOverride === "settled" ? "active" : value.settledOverride, + settledAt: value.settledOverride === "settled" ? null : value.settledAt, + snoozedUntil: null, + snoozedAt: null, + }; + case "thread.unpin": + return { ...value, pinnedAt: null, pinOrderKey: null }; + case "thread.pin.reorder": + return { ...value, pinOrderKey: command.orderKey }; + case "thread.active.reorder": + return { ...value, activeOrderKey: command.orderKey }; + case "thread.auto-settle.set": + return { + ...value, + autoSettleDisabledAt: command.enabled ? null : (value.autoSettleDisabledAt ?? now), + }; + } +}; + +const wakeForSourceActivity = ( + desired: Organization, + baseline: typeof SourceActivity.Type, + thread: OrchestrationV2ThreadShell, + now: DateTime.Utc, +): Organization => { + const current = sourceActivity(thread); + const wakeSnooze = + desired.snoozedUntil != null && + (DateTime.toEpochMillis(desired.snoozedUntil) <= DateTime.toEpochMillis(now) || + (current.pendingRequestId !== null && + current.pendingRequestId !== baseline.pendingRequestId) || + (current.completedAt !== null && + current.completedAt !== baseline.completedAt && + (thread.status === "completed" || thread.status === "failed"))); + const wakeSettlement = + desired.settledOverride === "settled" && + current.userMessageAt !== null && + current.userMessageAt !== baseline.userMessageAt; + if (!wakeSnooze && !wakeSettlement) return desired; + return { + ...desired, + ...(wakeSnooze ? { snoozedUntil: null, snoozedAt: null } : {}), + ...(wakeSettlement + ? ({ settledOverride: "active", settledAt: null, unsettledAt: now } as const) + : {}), + }; +}; + +const sourceCommands = ( + thread: OrchestrationV2ThreadShell, + desired: Organization, + revision: string, +): ReadonlyArray => { + const commands: Array = []; + const base = { threadId: thread.id, commandId: CommandId.make(`${revision}:settle`) }; + if (desired.settledOverride !== thread.settledOverride) { + if (desired.settledOverride === "settled") + commands.push({ + ...base, + type: "thread.settle", + ...(desired.settledAt == null ? {} : { settledAt: desired.settledAt }), + }); + if (desired.settledOverride === "active") + commands.push({ ...base, type: "thread.unsettle", reason: "user" }); + } + if ((desired.pinnedAt != null) !== (thread.pinnedAt != null)) + commands.push( + desired.pinnedAt != null + ? { + ...base, + commandId: CommandId.make(`${revision}:pin`), + type: "thread.pin", + ...(desired.pinOrderKey == null ? {} : { orderKey: desired.pinOrderKey }), + } + : { ...base, commandId: CommandId.make(`${revision}:pin`), type: "thread.unpin" }, + ); + if ( + desired.pinnedAt != null && + thread.pinnedAt != null && + desired.pinOrderKey != null && + desired.pinOrderKey !== thread.pinOrderKey + ) + commands.push({ + ...base, + commandId: CommandId.make(`${revision}:pin-order`), + type: "thread.pin.reorder", + orderKey: desired.pinOrderKey, + }); + if ( + desired.pinnedAt == null && + desired.settledOverride !== "settled" && + desired.activeOrderKey != null && + desired.activeOrderKey !== thread.activeOrderKey + ) + commands.push({ + ...base, + commandId: CommandId.make(`${revision}:active-order`), + type: "thread.active.reorder", + orderKey: desired.activeOrderKey, + }); + if ((desired.autoSettleDisabledAt != null) !== (thread.autoSettleDisabledAt != null)) + commands.push({ + ...base, + commandId: CommandId.make(`${revision}:auto-settle`), + type: "thread.auto-settle.set", + enabled: desired.autoSettleDisabledAt == null, + }); + if ( + (desired.snoozedUntil == null ? null : DateTime.toEpochMillis(desired.snoozedUntil)) !== + (thread.snoozedUntil == null ? null : DateTime.toEpochMillis(thread.snoozedUntil)) + ) + commands.push( + desired.snoozedUntil == null + ? { + ...base, + commandId: CommandId.make(`${revision}:unsnooze`), + type: "thread.unsnooze", + reason: "user", + } + : { + ...base, + commandId: CommandId.make(`${revision}:snooze`), + type: "thread.snooze", + snoozedUntil: DateTime.formatIso(desired.snoozedUntil), + }, + ); + return commands; +}; + +const storageError = (cause: unknown) => + new HubInboxError({ reason: "storage", cause: Redacted.make(cause) }); + +const make = Effect.gen(function* () { + const hub = yield* Hub.Hub; + const archive = yield* ArchiveCatalog.ArchiveCatalog; + const sql = yield* SqlClient.SqlClient; + const client = yield* HttpClient.HttpClient; + const lock = yield* Semaphore.make(1); + const refreshLock = yield* Semaphore.make(1); + + const expireSnoozes = Effect.fnUntraced(function* (id: EnvironmentId) { + const now = yield* Clock.currentTimeMillis; + const rows = yield* sql<{ + thread_id: string; + command_id: string; + organization_json: string; + }>`SELECT thread_id, command_id, organization_json FROM hub_inbox_pending WHERE environment_id = ${id}`.pipe( + Effect.mapError(storageError), + ); + for (const row of rows) { + const value = yield* Schema.decodeEffect(jsonOrganization)(row.organization_json).pipe( + Effect.mapError(storageError), + ); + if (value.snoozedUntil == null || DateTime.toEpochMillis(value.snoozedUntil) > now) continue; + const json = yield* Schema.encodeEffect(jsonOrganization)({ + ...value, + snoozedUntil: null, + snoozedAt: null, + }).pipe(Effect.mapError(storageError)); + yield* sql`UPDATE hub_inbox_pending SET organization_json = ${json}, command_id = ${`${row.command_id}:expired`} WHERE environment_id = ${id} AND thread_id = ${row.thread_id} AND command_id = ${row.command_id}`.pipe( + Effect.mapError(storageError), + ); + } + }); + + const readEnvironment = Effect.fnUntraced(function* (id: EnvironmentId) { + yield* expireSnoozes(id); + const { rows, pending } = yield* sql + .withTransaction( + Effect.all({ + rows: sql<{ + snapshot_json: string; + }>`SELECT snapshot_json FROM hub_inbox WHERE environment_id = ${id}`, + pending: sql<{ + thread_id: ThreadId; + organization_json: string; + }>`SELECT thread_id, organization_json FROM hub_inbox_pending WHERE environment_id = ${id}`, + }), + ) + .pipe(Effect.mapError(storageError)); + const row = rows[0]; + if (!row) return undefined; + const snapshot = yield* Schema.decodeEffect(jsonSnapshot)(row.snapshot_json).pipe( + Effect.mapError(storageError), + ); + const overrides = new Map( + yield* Effect.forEach(pending, (value) => + Schema.decodeEffect(jsonOrganization)(value.organization_json).pipe( + Effect.map((state) => [value.thread_id, state] as const), + Effect.mapError(storageError), + ), + ), + ); + return HubInboxEnvironment.make({ + environmentId: id, + snapshot: { + ...snapshot, + threads: snapshot.threads.map((thread) => ({ ...thread, ...overrides.get(thread.id) })), + }, + pendingThreadIds: pending.map((value) => value.thread_id), + }); + }); + + const request = Effect.fnUntraced(function* ( + id: EnvironmentId, + path: string, + decoder: Schema.Decoder, + command?: HubOrganizationCommand, + ) { + const upstream = yield* hub.connect(id, internalSession); + let outgoing = HttpClientRequest.make(command ? "POST" : "GET")( + new URL(path, upstream.url).href, + ).pipe( + HttpClientRequest.bearerToken(upstream.token), + HttpClientRequest.setHeader( + ORCHESTRATION_PROTOCOL_HEADER, + ORCHESTRATION_PROTOCOL_VERSION_TEXT, + ), + ); + if (command) + outgoing = HttpClientRequest.setBody( + outgoing, + HttpBody.jsonUnsafe(yield* encodeCommand(command).pipe(Effect.mapError(storageError))), + ); + return yield* Effect.raceFirst( + client.execute(outgoing).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" }), + Effect.flatMap( + Effect.fnUntraced(function* (response) { + if (response.status !== 200) + return yield* new HubError({ + reason: + response.status === 401 || response.status === 403 ? "credential" : "upstream", + }); + return yield* HttpClientResponse.schemaBodyJson(decoder)(response); + }), + ), + Effect.timeout("10 seconds"), + Effect.mapError((cause) => + isHubError(cause) + ? cause + : new HubError({ reason: "upstream", cause: Redacted.make(cause) }), + ), + ), + upstream.invalidated.pipe( + Effect.andThen(Effect.fail(new HubError({ reason: "credential" }))), + ), + ).pipe(Effect.scoped); + }); + + const refreshEnvironment = Effect.fnUntraced(function* (id: EnvironmentId) { + yield* expireSnoozes(id); + let snapshot = yield* request(id, "api/orchestration/shell", OrchestrationV2ShellSnapshot); + const pending = yield* sql<{ + thread_id: ThreadId; + command_id: string; + organization_json: string; + source_activity_json: string; + }>`SELECT thread_id, command_id, organization_json, source_activity_json FROM hub_inbox_pending WHERE environment_id = ${id}`.pipe( + Effect.mapError(storageError), + ); + const confirmedRevisions: Array<{ threadId: ThreadId; commandId: string; sequence: number }> = + []; + for (const row of pending) { + const thread = snapshot.threads.find((value) => value.id === row.thread_id); + if (!thread || thread.archivedAt !== null || thread.deletedAt !== null) { + confirmedRevisions.push({ + threadId: row.thread_id, + commandId: row.command_id, + sequence: snapshot.snapshotSequence, + }); + continue; + } + let desired = yield* Schema.decodeEffect(jsonOrganization)(row.organization_json).pipe( + Effect.mapError(storageError), + ); + const baseline = yield* Schema.decodeEffect(jsonSourceActivity)( + row.source_activity_json, + ).pipe(Effect.mapError(storageError)); + const updated = wakeForSourceActivity(desired, baseline, thread, yield* DateTime.now); + if (updated !== desired) { + desired = updated; + const json = yield* Schema.encodeEffect(jsonOrganization)(desired).pipe( + Effect.mapError(storageError), + ); + const revision = `${row.command_id}:activity`; + yield* sql`UPDATE hub_inbox_pending SET organization_json = ${json}, command_id = ${revision} WHERE environment_id = ${id} AND thread_id = ${row.thread_id} AND command_id = ${row.command_id}`.pipe( + Effect.mapError(storageError), + ); + row.command_id = revision; + row.organization_json = json; + } + const commands = sourceCommands(thread, desired, row.command_id); + let dispatched = true; + let sequence = snapshot.snapshotSequence; + for (const command of commands) { + const current = yield* sql<{ + command_id: string; + }>`SELECT command_id FROM hub_inbox_pending WHERE environment_id = ${id} AND thread_id = ${row.thread_id}`.pipe( + Effect.mapError(storageError), + ); + if (current[0]?.command_id !== row.command_id) { + dispatched = false; + break; + } + const result = yield* request( + id, + "api/hub/organization", + Schema.Struct({ sequence: Schema.Number }), + command, + ).pipe(Effect.result); + if (result._tag === "Failure") { + dispatched = false; + break; + } + sequence = Math.max(sequence, result.success.sequence); + } + if (!dispatched) continue; + confirmedRevisions.push({ threadId: row.thread_id, commandId: row.command_id, sequence }); + } + if (confirmedRevisions.some((value) => value.sequence > snapshot.snapshotSequence)) + snapshot = yield* request(id, "api/orchestration/shell", OrchestrationV2ShellSnapshot); + const json = yield* Schema.encodeEffect(jsonSnapshot)(snapshot).pipe( + Effect.mapError(storageError), + ); + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql`INSERT INTO hub_inbox(environment_id, snapshot_json) VALUES (${id}, ${json}) ON CONFLICT(environment_id) DO UPDATE SET snapshot_json = excluded.snapshot_json`; + for (const revision of confirmedRevisions.filter( + (value) => value.sequence <= snapshot.snapshotSequence, + )) { + const row = pending.find( + (value) => + value.thread_id === revision.threadId && value.command_id === revision.commandId, + ); + const thread = snapshot.threads.find((value) => value.id === revision.threadId); + if (row && thread && thread.archivedAt === null && thread.deletedAt === null) { + const desired = yield* Schema.decodeEffect(jsonOrganization)( + row.organization_json, + ).pipe(Effect.mapError(storageError)); + const baseline = yield* Schema.decodeEffect(jsonSourceActivity)( + row.source_activity_json, + ).pipe(Effect.mapError(storageError)); + const updated = wakeForSourceActivity(desired, baseline, thread, yield* DateTime.now); + if (updated !== desired) { + const json = yield* Schema.encodeEffect(jsonOrganization)(updated).pipe( + Effect.mapError(storageError), + ); + yield* sql`UPDATE hub_inbox_pending SET organization_json = ${json}, command_id = ${`${revision.commandId}:activity`} WHERE environment_id = ${id} AND thread_id = ${revision.threadId} AND command_id = ${revision.commandId}`; + continue; + } + } + yield* sql`DELETE FROM hub_inbox_pending WHERE environment_id = ${id} AND thread_id = ${revision.threadId} AND command_id = ${revision.commandId}`; + } + }), + ) + .pipe(Effect.mapError(storageError)); + }); + + const refresh = refreshLock.withPermit( + Effect.gen(function* () { + const directory = yield* hub.directory(internalSession); + yield* Effect.forEach( + directory.environments.filter((value) => value.scopes.includes(AuthOrchestrationReadScope)), + (value) => + refreshEnvironment(value.environmentId).pipe( + Effect.catchTags({ + HubError: () => Effect.void, + }), + ), + { concurrency: 4, discard: true }, + ); + }), + ); + + yield* refresh.pipe( + Effect.catchTags({ + HubError: () => Effect.void, + HubInboxError: () => Effect.logWarning("Hub inbox storage is unavailable"), + }), + Effect.repeat(Schedule.spaced("15 seconds")), + Effect.forkScoped, + ); + + const authorize = Effect.fnUntraced(function* (id: EnvironmentId, session: AuthenticatedSession) { + const scopes = yield* hub.authorize(id, session); + if ( + !scopes.includes(AuthOrchestrationReadScope) || + !scopes.includes(AuthOrchestrationOperateScope) + ) + return yield* new HubError({ reason: "credential" }); + }); + + return HubInbox.of({ + refresh, + read: (session) => + Effect.gen(function* () { + const directory = yield* hub.directory(session, { refresh: false }); + const archived = session.scopes.includes(AuthArchiveReadScope) + ? (yield* archive.list.pipe(Effect.orElseSucceed(() => ({ entries: [] })))).entries + : []; + const values = yield* Effect.forEach( + directory.environments.filter((value) => + value.scopes.includes(AuthOrchestrationReadScope), + ), + (value) => + Effect.gen(function* () { + const cached = yield* readEnvironment(value.environmentId); + const entries = archived.filter( + (entry) => entry.environmentId === value.environmentId, + ); + if (!entries.length) return cached; + const snapshot = cached?.snapshot ?? { + schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION, + snapshotSequence: 0, + projects: [], + threads: [], + archivedThreads: [], + }; + const fresh = entries.filter( + (entry) => + entry.throughSequence > snapshot.snapshotSequence || + ![...snapshot.threads, ...snapshot.archivedThreads].some( + (thread) => thread.id === entry.thread.id, + ), + ); + const replaced = new Set(fresh.map((entry) => entry.thread.id)); + const projects = new Map(snapshot.projects.map((project) => [project.id, project])); + for (const entry of fresh) + if (!projects.has(entry.project.id)) projects.set(entry.project.id, entry.project); + return HubInboxEnvironment.make({ + environmentId: value.environmentId, + pendingThreadIds: cached?.pendingThreadIds ?? [], + snapshot: { + ...snapshot, + snapshotSequence: Math.max( + snapshot.snapshotSequence, + ...fresh.map((entry) => entry.throughSequence), + ), + projects: [...projects.values()], + threads: snapshot.threads.filter((thread) => !replaced.has(thread.id)), + archivedThreads: [ + ...snapshot.archivedThreads.filter((thread) => !replaced.has(thread.id)), + ...fresh.map((entry) => entry.thread), + ], + }, + }); + }), + ); + return { environments: values.filter((value) => value !== undefined) }; + }), + organize: (input, session) => + Effect.gen(function* () { + yield* authorize(input.environmentId, session); + const cached = yield* readEnvironment(input.environmentId); + if (!cached?.snapshot.threads.some((value) => value.id === input.command.threadId)) + yield* refreshLock.withPermit(refreshEnvironment(input.environmentId)); + return yield* lock.withPermit( + Effect.gen(function* () { + yield* authorize(input.environmentId, session); + const entry = yield* readEnvironment(input.environmentId); + const receipt = + yield* sql`SELECT 1 FROM hub_inbox_commands WHERE environment_id = ${input.environmentId} AND command_id = ${input.command.commandId}`.pipe( + Effect.mapError(storageError), + ); + if (entry && receipt.length > 0) return entry; + const thread = entry?.snapshot.threads.find( + (value) => value.id === input.command.threadId, + ); + if (!entry || !thread || thread.deletedAt !== null) + return yield* new HubInboxError({ reason: "thread-missing" }); + const command = input.command; + const now = yield* DateTime.now; + if ( + thread.archivedAt !== null || + (command.type === "thread.pin.reorder" && thread.pinnedAt == null) || + (command.type === "thread.active.reorder" && + (thread.pinnedAt != null || thread.settledOverride === "settled")) || + (command.type === "thread.snooze" && + Date.parse(command.snoozedUntil) <= DateTime.toEpochMillis(now)) + ) + return yield* new HubInboxError({ reason: "thread-state" }); + const desired = applyCommand( + Struct.pick(thread, HUB_THREAD_ORGANIZATION_FIELDS), + command, + now, + ); + const json = yield* Schema.encodeEffect(jsonOrganization)(desired).pipe( + Effect.mapError(storageError), + ); + const baseline = yield* Schema.encodeEffect(jsonSourceActivity)( + sourceActivity(thread), + ).pipe(Effect.mapError(storageError)); + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* sql`INSERT INTO hub_inbox_pending(environment_id, thread_id, command_id, organization_json, source_activity_json) VALUES (${input.environmentId}, ${command.threadId}, ${command.commandId}, ${json}, ${baseline}) ON CONFLICT(environment_id, thread_id) DO UPDATE SET command_id = excluded.command_id, organization_json = excluded.organization_json, source_activity_json = excluded.source_activity_json`; + yield* sql`INSERT INTO hub_inbox_commands(environment_id, command_id) VALUES (${input.environmentId}, ${command.commandId})`; + }), + ) + .pipe(Effect.mapError(storageError)); + return { + ...entry, + pendingThreadIds: [...new Set([...entry.pendingThreadIds, thread.id])], + snapshot: { + ...entry.snapshot, + threads: entry.snapshot.threads.map((value) => + value.id === thread.id ? { ...value, ...desired } : value, + ), + }, + }; + }), + ); + }), + }); +}); + +export const layer = Layer.effect(HubInbox, make); diff --git a/apps/server/src/environment/HubInboxHttp.test.ts b/apps/server/src/environment/HubInboxHttp.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..cf90d9123ae251d91c6e88fda48d51ee48578b35 --- /dev/null +++ b/apps/server/src/environment/HubInboxHttp.test.ts @@ -0,0 +1,126 @@ +import { assert, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + AuthOrchestrationOperateScope, + AuthOrchestrationReadScope, + AuthSessionId, + type HubOrganizationCommand, + HubInbox as HubInboxSchema, + HubInboxEnvironment, + EnvironmentId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as DateTime from "effect/DateTime"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import { HttpRouter } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as HubInbox from "./HubInbox.ts"; +import * as HubInboxHttp from "./HubInboxHttp.ts"; +import { hubInboxThread } from "./HubInbox.testkit.ts"; + +const decodeInbox = Schema.decodeUnknownEffect(Schema.toCodecJson(HubInboxSchema)); + +it.effect( + "authenticates inbox requests and restricts source reconciliation to organization commands", + () => + Effect.gen(function* () { + const dispatched: Array = []; + let entry: HubInboxEnvironment = { + environmentId: EnvironmentId.make("laptop"), + snapshot: { + schemaVersion: 2, + snapshotSequence: 1, + projects: [], + threads: [ + { + ...hubInboxThread, + snoozedUntil: DateTime.makeUnsafe("2030-01-01T00:00:00Z"), + snoozedAt: DateTime.makeUnsafe(0), + }, + ], + archivedThreads: [], + }, + pendingThreadIds: [hubInboxThread.id], + }; + const auth = { + authenticateWebSocketUpgrade: ( + request: import("effect/http/HttpServerRequest").HttpServerRequest, + ) => + request.headers.authorization + ? Effect.succeed({ + sessionId: AuthSessionId.make("phone"), + subject: "phone", + method: "bearer-access-token" as const, + scopes: + request.headers.authorization === "Bearer operate" + ? [AuthOrchestrationReadScope, AuthOrchestrationOperateScope] + : [AuthOrchestrationReadScope], + }) + : Effect.fail(new EnvironmentAuth.ServerAuthMissingCredentialError({})), + } as unknown as EnvironmentAuth.EnvironmentAuth["Service"]; + const orchestrator = { + dispatch: (command: HubOrganizationCommand) => + Effect.sync(() => { + dispatched.push(command.type); + return { sequence: 1 }; + }), + } as unknown as Orchestrator.OrchestratorV2["Service"]; + const { handler, dispose } = HttpRouter.toWebHandler( + HubInboxHttp.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + Layer.succeed(EnvironmentAuth.EnvironmentAuth, auth), + Layer.succeed(Orchestrator.OrchestratorV2, orchestrator), + Layer.succeed(HubInbox.HubInbox, { + read: () => Effect.succeed({ environments: [entry] }), + organize: () => Effect.succeed(entry), + refresh: Effect.void, + }), + ), + ), + Layer.provideMerge(NodeServices.layer), + ), + ); + yield* Effect.addFinalizer(() => Effect.promise(dispose)); + const send = (path: string, token?: string, type?: string, etag?: string) => + Effect.promise(() => + handler( + new Request(`http://hub${path}`, { + method: type ? "POST" : "GET", + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + "content-type": "application/json", + ...(etag ? { "if-none-match": etag } : {}), + }, + ...(type + ? { body: JSON.stringify({ type, threadId: "chat", commandId: "command" }) } + : {}), + }), + ), + ); + assert.equal((yield* send("/api/hub/inbox")).status, 401); + const first = yield* send("/api/hub/inbox", "read"); + assert.equal(first.status, 200); + const decoded = yield* decodeInbox(yield* Effect.promise(() => first.json())); + assert.equal( + DateTime.formatIso(decoded.environments[0]!.snapshot.threads[0]!.snoozedUntil!), + "2030-01-01T00:00:00.000Z", + ); + const etag = first.headers.get("etag")!; + for (const validator of [etag, `W/${etag}`, `"other", W/${etag}`, "*"]) + assert.equal((yield* send("/api/hub/inbox", "read", undefined, validator)).status, 304); + assert.equal( + (yield* send("/api/hub/inbox", "read", undefined, '"other", W/"different"')).status, + 200, + ); + entry = { ...entry, pendingThreadIds: [] }; + assert.equal((yield* send("/api/hub/inbox", "read", undefined, etag)).status, 200); + assert.equal((yield* send("/api/hub/organization", "read", "thread.settle")).status, 403); + assert.equal((yield* send("/api/hub/organization", "operate", "thread.stop")).status, 400); + assert.deepEqual(dispatched, []); + assert.equal((yield* send("/api/hub/organization", "operate", "thread.settle")).status, 200); + assert.deepEqual(dispatched, ["thread.settle"]); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/environment/HubInboxHttp.ts b/apps/server/src/environment/HubInboxHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..712795c96836e125e94ae233feb3dbf0c924dbf4 --- /dev/null +++ b/apps/server/src/environment/HubInboxHttp.ts @@ -0,0 +1,128 @@ +import { + AuthOrchestrationOperateScope, + HubError, + HubInboxCommand, + HubOrganizationCommand, + HubInbox as HubInboxSchema, + HubInboxEnvironment, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Crypto from "effect/Crypto"; +import * as Hex from "effect/encoding/Hex"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as HubInbox from "./HubInbox.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; + +const isHubError = Schema.is(HubError); +const isInboxError = Schema.is(HubInbox.HubInboxError); +const isDispatchError = Schema.is(Orchestrator.OrchestratorDispatchError); +const encodeInbox = Schema.encodeEffect(Schema.fromJsonString(Schema.toCodecJson(HubInboxSchema))); +const encodeEnvironment = Schema.encodeEffect(Schema.toCodecJson(HubInboxEnvironment)); + +const authenticate = Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + return yield* auth.authenticateWebSocketUpgrade(yield* HttpServerRequest.HttpServerRequest); +}); + +const errors = (effect: Effect.Effect) => + effect.pipe( + Effect.catch((error) => { + if (EnvironmentAuth.isServerAuthCredentialError(error)) + return Effect.succeed(HttpServerResponse.empty({ status: 401 })); + if (isHubError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { message: error.message, reason: error.reason }, + { + status: + error.reason === "credential" + ? 403 + : error.reason === "configuration" || error.reason === "not-found" + ? 404 + : 502, + }, + ), + ); + if (isInboxError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { message: error.message, reason: error.reason }, + { + status: + error.reason === "thread-missing" + ? 404 + : error.reason === "thread-state" + ? 409 + : 500, + }, + ), + ); + if (Schema.isSchemaError(error)) + return Effect.succeed(HttpServerResponse.empty({ status: 400 })); + if (isDispatchError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { message: new HubInbox.HubInboxError({ reason: "thread-state" }).message }, + { status: 409 }, + ), + ); + return Effect.succeed(HttpServerResponse.empty({ status: 500 })); + }), + ); + +export const layer = Layer.mergeAll( + HttpRouter.add( + "GET", + "/api/hub/inbox", + Effect.gen(function* () { + const value = yield* (yield* HubInbox.HubInbox).read(yield* authenticate); + const body = yield* encodeInbox(value); + const digest = yield* (yield* Crypto.Crypto).digest( + "SHA-256", + new TextEncoder().encode(body), + ); + const etag = `"${Hex.encode(digest)}"`; + const request = yield* HttpServerRequest.HttpServerRequest; + const headers = { etag, "cache-control": "no-store" }; + const unchanged = request.headers["if-none-match"]?.split(",").some((value) => { + const candidate = value.trim(); + return candidate === "*" || candidate.replace(/^W\//, "") === etag; + }); + return unchanged + ? HttpServerResponse.empty({ status: 304, headers }) + : HttpServerResponse.text(body, { + headers: { ...headers, "content-type": "application/json" }, + }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/hub/inbox", + Effect.gen(function* () { + const session = yield* authenticate; + const input = yield* HttpServerRequest.schemaBodyJson(HubInboxCommand); + const value = yield* (yield* HubInbox.HubInbox).organize(input, session); + return HttpServerResponse.jsonUnsafe(yield* encodeEnvironment(value), { + headers: { "cache-control": "no-store" }, + }); + }).pipe(errors), + ), + HttpRouter.add( + "POST", + "/api/hub/organization", + Effect.gen(function* () { + const session = yield* authenticate; + if (!session.scopes.includes(AuthOrchestrationOperateScope)) + return yield* new HubError({ reason: "credential" }); + const command = yield* HttpServerRequest.schemaBodyJson(HubOrganizationCommand); + const result = yield* (yield* Orchestrator.OrchestratorV2).dispatch(command); + return HttpServerResponse.jsonUnsafe( + { sequence: result.sequence }, + { headers: { "cache-control": "no-store" } }, + ); + }).pipe(errors), + ), +); diff --git a/apps/server/src/environment/HubProxy.auth.test.ts b/apps/server/src/environment/HubProxy.auth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f881ca75f9f16508d2208752277a214276a868a6 --- /dev/null +++ b/apps/server/src/environment/HubProxy.auth.test.ts @@ -0,0 +1,127 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { AuthOrchestrationReadScope, EnvironmentId } from "@t3tools/contracts"; +import { computeDpopAccessTokenHash, computeDpopJwkThumbprint } from "@t3tools/shared/dpop"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { HttpClient, HttpClientResponse, HttpRouter } from "effect/http"; +import { exportJWK, generateKeyPair, SignJWT } from "jose"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as ServerConfig from "../config.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; +import * as Hub from "./Hub.ts"; +import * as HubHttp from "./HubHttp.ts"; +import * as HubProxy from "./HubProxy.ts"; + +it.live( + "verifies DPoP against the prefixed hub URL, rejects replay and revoked clients, and never sends their proof to a leaf", + () => + Effect.gen(function* () { + const config = ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-auth-test-" }); + const services = yield* Layer.build( + EnvironmentAuth.layer.pipe( + Layer.provideMerge(SqlitePersistence.layerMemory), + Layer.provide(ServerSecretStore.layer), + Layer.provide(ServerEnvironment.layerIdentity), + Layer.provide(config), + ), + ); + const auth = Context.get(services, EnvironmentAuth.EnvironmentAuth); + const sessions = Context.get(services, SessionStore.SessionStore); + const keys = yield* Effect.promise(() => generateKeyPair("ES256")); + const exported = yield* Effect.promise(() => exportJWK(keys.publicKey)); + if (!exported.x || !exported.y) return yield* Effect.die("Missing public key coordinates"); + const jwk = { kty: "EC" as const, crv: "P-256" as const, x: exported.x, y: exported.y }; + const token = yield* sessions.issue({ + method: "dpop-access-token", + subject: "phone", + proofKeyThumbprint: computeDpopJwkThumbprint(jwk), + scopes: [AuthOrchestrationReadScope], + }); + const requests: Array<{ url: string; authorization?: string; proof?: string }> = []; + const hub = Hub.Hub.of({ + authorize: () => Effect.succeed([AuthOrchestrationReadScope]), + descriptor: () => Effect.die("unused"), + directory: () => + Effect.succeed({ hubEnvironmentId: EnvironmentId.make("hub"), environments: [] }), + manage: () => Effect.die("unused"), + connect: () => + Effect.succeed({ + url: "http://leaf.private/", + token: "leaf-private-token", + invalidated: Effect.never, + }), + }); + const client = HttpClient.make((request) => + Effect.sync(() => { + requests.push({ + url: request.url, + ...(request.headers.authorization + ? { authorization: request.headers.authorization } + : {}), + ...(request.headers.dpop ? { proof: request.headers.dpop } : {}), + }); + return HttpClientResponse.fromWeb(request, new Response("leaf response")); + }), + ); + const dependencies = Layer.mergeAll( + Layer.succeedContext(services), + Layer.succeed(Hub.Hub, hub), + Layer.succeed(HttpClient.HttpClient, client), + ); + const { handler, dispose } = HttpRouter.toWebHandler( + HubHttp.layer.pipe( + Layer.provideMerge(HubProxy.layer.pipe(Layer.provide(dependencies))), + Layer.provideMerge(dependencies), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(dispose)); + const publicUrl = "https://hub.example/environments/leaf/api/example"; + const iat = Math.floor((yield* Clock.currentTimeMillis) / 1_000); + const proof = (url: string, id: string) => + Effect.promise(() => + new SignJWT({ + htm: "GET", + htu: url, + jti: id, + iat, + ath: computeDpopAccessTokenHash(token.token), + }) + .setProtectedHeader({ typ: "dpop+jwt", alg: "ES256", jwk }) + .sign(keys.privateKey), + ); + const request = (dpop: string) => + Effect.promise(() => + handler( + new Request(publicUrl, { + headers: { + authorization: `DPoP ${token.token}`, + host: "hub.example", + "x-forwarded-proto": "https", + dpop, + }, + }), + ), + ); + const valid = yield* proof(publicUrl, "first"); + const response = yield* request(valid); + expect(response.status).toBe(200); + expect(yield* Effect.promise(() => response.text())).toBe("leaf response"); + expect(requests).toEqual([ + { url: "http://leaf.private/api/example", authorization: "Bearer leaf-private-token" }, + ]); + expect((yield* request(valid)).status).toBe(401); + expect( + (yield* request(yield* proof("https://hub.example/api/example", "wrong-prefix"))).status, + ).toBe(401); + yield* auth.revokeSession(token.sessionId); + expect((yield* request(yield* proof(publicUrl, "revoked"))).status).toBe(401); + expect(requests).toHaveLength(1); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/environment/HubProxy.stream.test.ts b/apps/server/src/environment/HubProxy.stream.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..2548f2bfd7b85379ff761954856e9f887fe676b2 --- /dev/null +++ b/apps/server/src/environment/HubProxy.stream.test.ts @@ -0,0 +1,116 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { NodeWS } from "@effect/platform-node/NodeSocket"; +import { AuthOrchestrationReadScope, AuthSessionId, EnvironmentId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { FetchHttpClient, HttpRouter, HttpServer } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as Hub from "./Hub.ts"; +import * as HubHttp from "./HubHttp.ts"; +import * as HubProxy from "./HubProxy.ts"; + +it.effect.each(["client", "machine"] as const)( + "streams opaque binary frames with a private leaf grant and closes both sockets on %s revocation", + (invalidation) => + Effect.gen(function* () { + const upstream = yield* Effect.acquireRelease( + Effect.sync(() => new NodeWS.WebSocketServer({ port: 0 })), + (server) => + Effect.promise( + () => + new Promise((resolve) => { + for (const client of server.clients) client.terminate(); + server.close(() => resolve()); + }), + ), + ); + if (!upstream.address()) + yield* Effect.promise( + () => new Promise((resolve) => upstream.once("listening", resolve)), + ); + const address = upstream.address(); + if (!address || typeof address !== "object") throw new Error("Missing test port"); + const closedUpstream = Promise.withResolvers(); + const requests: Array<{ path?: string; authorization?: string }> = []; + upstream.on("connection", (socket, request) => { + requests.push({ + ...(request.url ? { path: request.url } : {}), + ...(request.headers.authorization + ? { authorization: request.headers.authorization } + : {}), + }); + socket.on("message", (data, binary) => socket.send(data, { binary })); + socket.on("close", () => closedUpstream.resolve()); + socket.send(Uint8Array.of(0x83)); + }); + const revokedClient = yield* Deferred.make(); + const revokedMachine = yield* Deferred.make(); + const dependencies = Layer.mergeAll( + Layer.succeed(Hub.Hub, { + connect: () => + Effect.succeed({ + url: `http://127.0.0.1:${address.port}/base/`, + token: "leaf-private", + invalidated: Deferred.await(revokedMachine), + }), + } as unknown as Hub.Hub["Service"]), + Layer.succeed(EnvironmentAuth.EnvironmentAuth, { + authenticateWebSocketUpgrade: () => + Effect.succeed({ + sessionId: AuthSessionId.make("phone"), + subject: "phone", + method: "bearer-access-token", + scopes: [AuthOrchestrationReadScope], + }), + } as unknown as EnvironmentAuth.EnvironmentAuth["Service"]), + Layer.succeed(SessionStore.SessionStore, { + awaitInvalidation: () => Deferred.await(revokedClient), + } as unknown as SessionStore.SessionStore["Service"]), + FetchHttpClient.layer, + ); + const services = yield* Layer.build( + HttpRouter.serve(HubHttp.layer, { disableListenLog: true }).pipe( + Layer.provideMerge(HubProxy.layer.pipe(Layer.provide(dependencies))), + Layer.provideMerge(dependencies), + Layer.provideMerge(NodeHttpServer.layerTest), + ), + ); + const origin = HttpServer.formatAddress( + Context.get(services, HttpServer.HttpServer).address, + ).replace(/^http/, "ws"); + const client = yield* Effect.acquireRelease( + Effect.sync( + () => + new NodeWS.WebSocket( + `${origin}/environments/${EnvironmentId.make("leaf")}/ws?wsTicket=hub-private&protocolVersion=1`, + ), + ), + (socket) => Effect.sync(() => socket.close()), + ); + client.binaryType = "arraybuffer"; + const received = Promise.withResolvers(); + const closedClient = Promise.withResolvers(); + const frames: number[][] = []; + client.addEventListener("error", (event) => received.reject(event.error)); + client.addEventListener("close", () => closedClient.resolve()); + client.addEventListener("message", ({ data }) => { + frames.push(Array.from(new Uint8Array(data as ArrayBuffer))); + if (frames.length === 1) client.send(Uint8Array.of(0x03, 1)); + else received.resolve(); + }); + yield* Effect.promise(() => received.promise); + expect(frames).toEqual([[0x83], [0x03, 1]]); + expect(requests).toEqual([ + { path: "/base/ws?protocolVersion=1", authorization: "Bearer leaf-private" }, + ]); + yield* Deferred.succeed( + invalidation === "client" ? revokedClient : revokedMachine, + undefined, + ); + yield* Effect.promise(() => Promise.all([closedClient.promise, closedUpstream.promise])); + }), +); diff --git a/apps/server/src/environment/HubProxy.test.ts b/apps/server/src/environment/HubProxy.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..965423fc40654a84835ee8a986eb1a60cd2b9809 --- /dev/null +++ b/apps/server/src/environment/HubProxy.test.ts @@ -0,0 +1,324 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthSessionId, + EnvironmentId, + ORCHESTRATION_PROTOCOL_VERSION, + type ExecutionEnvironmentDescriptor, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse, HttpRouter } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as Hub from "./Hub.ts"; +import * as HubHttp from "./HubHttp.ts"; +import * as HubProxy from "./HubProxy.ts"; + +const descriptor: ExecutionEnvironmentDescriptor = { + environmentId: EnvironmentId.make("leaf"), + label: "Laptop", + platform: { os: "linux", arch: "x64" }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: true }, +}; +const authDescriptor = { + policy: "remote-reachable" as const, + bootstrapMethods: ["one-time-token" as const], + sessionMethods: ["bearer-access-token" as const], + sessionCookieName: "hub-session", +}; +const session = { + sessionId: AuthSessionId.make("phone"), + subject: "phone", + method: "bearer-access-token" as const, + scopes: [AuthOrchestrationReadScope, AuthOrchestrationOperateScope], +}; + +const fixture = ( + responseFactory: ( + request: import("effect/http/HttpClientRequest").HttpClientRequest, + ) => Response = () => new Response("leaf"), +) => + Effect.gen(function* () { + const removed = yield* Deferred.make(); + const revoked = yield* Deferred.make(); + const requests: Array<{ + url: string; + headers: Readonly>; + body: string; + }> = []; + let finalized = 0; + const client = HttpClient.make((request, _url, signal) => + Effect.gen(function* () { + signal.addEventListener("abort", () => { + finalized++; + }); + const body = + request.body._tag === "Stream" + ? yield* request.body.stream.pipe( + Stream.runCollect, + Effect.orDie, + Effect.map((chunks) => + chunks.map((chunk) => new TextDecoder().decode(chunk)).join(""), + ), + ) + : ""; + requests.push({ url: request.url, headers: request.headers, body }); + return HttpClientResponse.fromWeb(request, responseFactory(request)); + }), + ); + const hub = Hub.Hub.of({ + authorize: () => Effect.succeed([AuthOrchestrationReadScope]), + descriptor: () => Effect.succeed(descriptor), + directory: () => + Effect.succeed({ + hubEnvironmentId: EnvironmentId.make("hub"), + environments: [ + { + ...descriptor, + descriptor, + httpBaseUrl: "https://hub.test/environments/leaf/", + scopes: [AuthOrchestrationReadScope], + }, + ], + }), + manage: () => Effect.die("unused"), + connect: () => + Effect.succeed({ + url: "http://leaf.private/base/", + token: "private-leaf-token", + invalidated: Deferred.await(removed), + }), + }); + const auth = { + authenticateWebSocketUpgrade: ( + request: import("effect/http/HttpServerRequest").HttpServerRequest, + ) => + request.headers.authorization || request.url.includes("wsTicket=hub-ticket") + ? Effect.succeed(session) + : Effect.fail(new EnvironmentAuth.ServerAuthMissingCredentialError({})), + issueWebSocketTicket: () => + Effect.succeed({ ticket: "hub-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }), + getDescriptor: () => Effect.succeed(authDescriptor), + } as unknown as EnvironmentAuth.EnvironmentAuth["Service"]; + const dependencies = Layer.mergeAll( + Layer.succeed(Hub.Hub, hub), + Layer.succeed(EnvironmentAuth.EnvironmentAuth, auth), + Layer.succeed(SessionStore.SessionStore, { + awaitInvalidation: () => Deferred.await(revoked), + } as unknown as SessionStore.SessionStore["Service"]), + Layer.succeed(HttpClient.HttpClient, client), + ); + const { handler, dispose } = HttpRouter.toWebHandler( + HubHttp.layer.pipe( + Layer.provideMerge(HubProxy.layer.pipe(Layer.provide(dependencies))), + Layer.provideMerge(dependencies), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(dispose)); + const request = (path: string, init: RequestInit = {}) => + handler( + new Request(`http://hub.test${path}`, { + ...init, + headers: { authorization: "Bearer hub-token", ...init.headers }, + }), + ); + return { + handler, + request, + requests, + revoke: Deferred.succeed(revoked, undefined), + remove: Deferred.succeed(removed, undefined), + finalized: () => finalized, + }; + }); + +describe("self-hosted server proxy", () => { + it.effect("does not advertise a compressed upstream size for a decoded response", () => + Effect.gen(function* () { + const f = yield* fixture( + () => + new Response("expanded", { + headers: { "content-encoding": "gzip", "content-length": "1" }, + }), + ); + const response = yield* Effect.promise(() => f.request("/environments/leaf/api/example")); + expect(response.headers.get("content-encoding")).toBeNull(); + expect(response.headers.get("content-length")).toBeNull(); + expect(yield* Effect.promise(() => response.text())).toBe("expanded"); + }), + ); + + it.effect("requires hub authentication and management permission", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.promise(async () => { + expect((await f.handler(new Request("http://hub.test/api/hub/environments"))).status).toBe( + 401, + ); + expect((await f.request("/api/hub/manage", { method: "POST", body: "{}" })).status).toBe( + 403, + ); + expect((await f.request("/api/hub/environments")).status).toBe(200); + }); + }), + ); + + it.effect( + "replaces hub credentials, preserves byte ranges, streams bodies, and releases upstream resources", + () => + Effect.gen(function* () { + const f = yield* fixture( + () => + new Response("bytes", { + status: 206, + headers: { + "content-range": "bytes 1-5/10", + "set-cookie": "leaf=private", + "x-private-secret": "private", + location: "/base/api/next", + }, + }), + ); + yield* Effect.promise(async () => { + const response = await f.request( + "/environments/leaf/api/assets?wsTicket=hub-ticket&asset=one", + { + method: "PUT", + headers: { + dpop: "hub-proof", + cookie: "hub-session=secret", + range: "bytes=1-5", + "x-forwarded-host": "evil.test", + }, + body: "upload", + }, + ); + expect(response.status).toBe(206); + expect(await response.text()).toBe("bytes"); + expect(response.headers.get("content-range")).toBe("bytes 1-5/10"); + expect(response.headers.get("set-cookie")).toBeNull(); + expect(response.headers.get("x-private-secret")).toBeNull(); + expect(response.headers.get("location")).toBe("/environments/leaf/api/next"); + expect(f.requests).toHaveLength(1); + expect(f.requests[0]).toMatchObject({ + url: "http://leaf.private/base/api/assets?asset=one", + body: "upload", + headers: { authorization: "Bearer private-leaf-token", range: "bytes=1-5" }, + }); + expect(f.requests[0]?.headers).not.toHaveProperty("dpop"); + expect(f.requests[0]?.headers).not.toHaveProperty("cookie"); + expect(f.requests[0]?.headers).not.toHaveProperty("x-forwarded-host"); + expect(f.finalized()).toBe(1); + }); + }), + ); + + it.effect.each([ + "/api/auth/pairing-token", + "/api/auth/delegated-session", + "/api/auth/%64elegated-session", + "/api/auth/browser-session", + "/api/auth/clients", + "/oauth/token", + "/.well-known/oauth-authorization-server", + "/%61pi/auth/pairing-token", + "/api%2fauth/pairing-token", + "/%6fauth/token", + ])("does not expose leaf credentials through %s", (path) => + Effect.gen(function* () { + const f = yield* fixture(); + const response = yield* Effect.promise(() => + f.request(`/environments/leaf${path}`, { method: "POST" }), + ); + expect(response.status).toBe(404); + expect(f.requests).toHaveLength(0); + }), + ); + + it.effect.each(["/%61pi/auth/websocket-ticket", "/api%2fauth/websocket-ticket"])( + "keeps encoded %s ticket requests on the hub", + (path) => + Effect.gen(function* () { + const f = yield* fixture(); + const response = yield* Effect.promise(() => + f.request(`/environments/leaf${path}`, { method: "POST" }), + ); + expect(response.status).toBe(200); + expect(yield* Effect.promise(() => response.json())).toMatchObject({ + ticket: "hub-ticket", + }); + expect(f.requests).toHaveLength(0); + }), + ); + + it.effect( + "serves bounded hub session and ticket responses locally and preserves public leaf identity", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.promise(async () => { + const state = await (await f.request("/environments/leaf/api/auth/session")).json(); + expect(state).toMatchObject({ + authenticated: true, + auth: authDescriptor, + scopes: [AuthOrchestrationReadScope], + }); + expect( + await ( + await f.request("/environments/leaf/api/auth/websocket-ticket", { method: "POST" }) + ).json(), + ).toMatchObject({ ticket: "hub-ticket" }); + expect( + await ( + await f.handler( + new Request("http://hub.test/environments/leaf/.well-known/t3/environment"), + ) + ).json(), + ).toMatchObject({ environmentId: "leaf" }); + expect(f.requests).toHaveLength(0); + }); + }), + ); + + it.effect("never turns a wildcard path into an arbitrary upstream origin", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.promise(async () => { + await (await f.request("/environments/leaf/https://evil.test/steal")).text(); + }); + expect(new URL(f.requests[0]?.url ?? "").origin).toBe("http://leaf.private"); + }), + ); + + it.effect.each(["revoke", "remove"] as const)( + "interrupts an active HTTP stream on %s", + (action) => + Effect.gen(function* () { + const f = yield* fixture( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("first")); + }, + }), + ), + ); + const response = yield* Effect.promise(() => f.request("/environments/leaf/api/stream")); + if (!response.body) return yield* Effect.die("Missing response stream"); + const reader = response.body.getReader(); + const first = yield* Effect.promise(() => reader.read()); + expect(new TextDecoder().decode(first.value)).toBe("first"); + yield* f[action]; + expect((yield* Effect.promise(() => reader.read())).done).toBe(true); + expect(f.finalized()).toBe(1); + }), + ); +}); diff --git a/apps/server/src/environment/HubProxy.ts b/apps/server/src/environment/HubProxy.ts new file mode 100644 index 0000000000000000000000000000000000000000..7a7a594e5b0c5a690522d025076a21cadb0b7352 --- /dev/null +++ b/apps/server/src/environment/HubProxy.ts @@ -0,0 +1,232 @@ +import { EnvironmentId, HubError, authScopeResponse } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Redacted from "effect/Redacted"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import * as Socket from "effect/socket/Socket"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, + HttpServerRequest, + HttpServerResponse, +} from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as Hub from "./Hub.ts"; + +const isHubError = Schema.is(HubError); + +export class HubProxy extends Context.Service< + HubProxy, + { + readonly forward: ( + request: HttpServerRequest.HttpServerRequest, + id: EnvironmentId, + path: string, + ) => Effect.Effect< + HttpServerResponse.HttpServerResponse, + | HubError + | EnvironmentAuth.ServerAuthCredentialError + | EnvironmentAuth.ServerAuthInternalError, + Scope.Scope + >; + } +>()("t3/environment/HubProxy") {} + +const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, token: string) => { + const headers: Record = { + authorization: `Bearer ${token}`, + "accept-encoding": "identity", + }; + const dropped = new Set([ + "authorization", + "cookie", + "dpop", + "host", + "origin", + "connection", + "upgrade", + "sec-websocket-key", + "sec-websocket-version", + "sec-websocket-extensions", + "sec-websocket-protocol", + "content-length", + "accept-encoding", + "proxy-authorization", + "proxy-authenticate", + "te", + "trailer", + "keep-alive", + "forwarded", + "x-forwarded-host", + "x-forwarded-proto", + "x-forwarded-prefix", + ]); + for (const [name, value] of Object.entries(request.headers)) { + if (value !== undefined && !dropped.has(name)) headers[name] = value; + } + return headers; +}; + +const pump = (source: Socket.Socket, sink: Socket.Writer) => + Effect.gen(function* () { + const { pull } = yield* source.reader; + while (true) yield* sink.writeAll(yield* pull); + }); + +const make = Effect.gen(function* () { + const hub = yield* Hub.Hub; + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; + const client = HttpClient.withScope(yield* HttpClient.HttpClient); + return HubProxy.of({ + forward: (request, id, path) => + Effect.gen(function* () { + const parsed = HttpServerRequest.toURL(request); + if (Option.isNone(parsed)) return HttpServerResponse.empty({ status: 400 }); + if (request.method === "GET" && path === "/.well-known/t3/environment") { + return HttpServerResponse.jsonUnsafe(yield* hub.descriptor(id)); + } + const session = yield* auth.authenticateWebSocketUpgrade(request); + const decodedPath = yield* Effect.try({ + try: () => decodeURIComponent(path), + catch: () => new HubError({ reason: "not-found" }), + }); + if ( + decodedPath !== path && + /^\/(api\/auth(?:\/|$)|oauth(?:\/|$)|\.well-known\/oauth)/i.test(decodedPath) + ) { + return HttpServerResponse.empty({ status: 404 }); + } + if (request.method === "GET" && path === "/api/auth/session") { + const scopes = yield* hub.authorize(id, session); + return HttpServerResponse.jsonUnsafe( + { + authenticated: true, + auth: yield* auth.getDescriptor(), + ...authScopeResponse(scopes), + sessionMethod: session.method, + ...(session.expiresAt ? { expiresAt: session.expiresAt } : {}), + }, + { headers: { "cache-control": "no-store" } }, + ); + } + if (request.method === "POST" && path === "/api/auth/websocket-ticket") { + yield* hub.authorize(id, session); + return HttpServerResponse.jsonUnsafe(yield* auth.issueWebSocketTicket(session), { + headers: { "cache-control": "no-store" }, + }); + } + // These responses can reveal or mint private leaf credentials and must stay on the hub. + if ( + path.startsWith("/api/auth/") || + path === "/oauth" || + path.startsWith("/oauth/") || + path.startsWith("/.well-known/oauth") + ) { + return HttpServerResponse.empty({ status: 404 }); + } + const upstream = yield* hub.connect(id, session); + const invalidated = Effect.raceFirst( + upstream.invalidated, + sessions.awaitInvalidation(session.sessionId), + ); + const url = new URL(upstream.url); + url.pathname += path.slice(1); + url.search = parsed.value.search; + url.searchParams.delete("wsTicket"); + const headers = forwardHeaders(request, upstream.token); + if (request.headers.upgrade?.toLowerCase() === "websocket") { + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return yield* Effect.scoped( + Effect.gen(function* () { + const socket = yield* Socket.makeWebSocket(url.href, { + headers, + openTimeout: "10 seconds", + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructorWS)); + const client = yield* request.upgrade; + const clientWriter = yield* client.writer; + const upstreamWriter = yield* socket.writer; + yield* Effect.raceFirst( + Effect.raceFirst(pump(socket, clientWriter), pump(client, upstreamWriter)), + invalidated, + ).pipe(Effect.ignoreCause); + return HttpServerResponse.empty(); + }), + ); + } + const outgoing = HttpClientRequest.make(request.method)(url.href).pipe( + HttpClientRequest.setHeaders(headers), + request.method === "GET" || request.method === "HEAD" + ? (self) => self + : HttpClientRequest.bodyStream(request.stream), + ); + const response = yield* Effect.raceFirst( + client + .execute(outgoing) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })), + invalidated.pipe(Effect.andThen(Effect.fail(new HubError({ reason: "credential" })))), + ); + const responseHeaders: Record = {}; + for (const [name, value] of Object.entries(response.headers)) { + if ( + value === undefined || + ![ + "content-type", + "content-length", + "content-range", + "accept-ranges", + "content-disposition", + "etag", + "last-modified", + "vary", + "content-security-policy", + "x-content-type-options", + "retry-after", + ].includes(name) + ) + continue; + responseHeaders[name] = value; + } + if (response.headers["content-encoding"]) delete responseHeaders["content-length"]; + if (response.headers.location) { + const destination = new URL(response.headers.location, url); + const privateBase = new URL(upstream.url); + if ( + destination.origin === privateBase.origin && + destination.pathname.startsWith(privateBase.pathname) + ) { + const prefix = parsed.value.pathname.slice( + 0, + parsed.value.pathname.length - path.length + 1, + ); + responseHeaders.location = `${prefix}${destination.pathname.slice(privateBase.pathname.length)}${destination.search}${destination.hash}`; + } + } + responseHeaders["cache-control"] = "no-store, no-transform"; + return HttpServerResponse.stream(response.stream.pipe(Stream.interruptWhen(invalidated)), { + status: response.status, + headers: responseHeaders, + ...(responseHeaders["content-type"] + ? { contentType: responseHeaders["content-type"] } + : {}), + }); + }).pipe( + Effect.mapError((cause) => + isHubError(cause) || + EnvironmentAuth.isServerAuthCredentialError(cause) || + EnvironmentAuth.isServerAuthInternalError(cause) + ? cause + : new HubError({ reason: "upstream", cause: Redacted.make(cause) }), + ), + ), + }); +}); + +export const layer = Layer.effect(HubProxy, make); diff --git a/apps/server/src/environment/OutboundHub.test.ts b/apps/server/src/environment/OutboundHub.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f8a917b7dd21cf8c8dfcb97d0c05417d8391fb68 --- /dev/null +++ b/apps/server/src/environment/OutboundHub.test.ts @@ -0,0 +1,242 @@ +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import { expect, it } from "@effect/vitest"; +import { + AuthEnvironmentConnectScope, + AuthOrchestrationReadScope, + AuthAccessWriteScope, + AuthSessionId, + EnvironmentId, + ORCHESTRATION_PROTOCOL_VERSION, + OutboundHubMessage, + type OutboundSourceMessage, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import * as Schema from "effect/Schema"; +import * as Socket from "effect/socket/Socket"; +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServer, + HttpServerRequest, + HttpServerResponse, +} from "effect/http"; +import * as HostedAuth from "../auth/HostedAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as OutboundHub from "./OutboundHub.ts"; +import { bridge } from "./OutboundSocket.ts"; + +const decodeHubMessage = Schema.decodeEffect(Schema.fromJsonString(OutboundHubMessage)); + +const socketPair = Effect.gen(function* () { + const inbound = yield* Queue.unbounded(); + const outbound = yield* Queue.unbounded(); + const endpoint = (read: typeof inbound, write: typeof outbound) => { + const send = (frame: string | Uint8Array | Socket.CloseEvent) => + Socket.isCloseEvent(frame) ? Effect.void : Queue.offer(write, frame).pipe(Effect.asVoid); + return Socket.make({ + reader: Effect.succeed({ + pull: Queue.take(read).pipe(Effect.map((frame) => [frame] as const)), + upgrade: () => Effect.void, + }), + writer: Effect.succeed({ + write: send, + writeAll: (frames) => Effect.forEach(frames, send, { discard: true }), + }), + }); + }; + return { left: endpoint(inbound, outbound), right: endpoint(outbound, inbound) }; +}); +const session = { + sessionId: AuthSessionId.make("device-session"), + deviceId: "device-a", + subject: "owner", + method: "bearer-access-token" as const, + scopes: [AuthEnvironmentConnectScope], +}; +const descriptor = { + environmentId: EnvironmentId.make("existing-laptop-id"), + label: "Laptop", + platform: { os: "linux" as const, arch: "x64" as const }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: true }, +}; + +const fixture = Effect.gen(function* () { + const data = new Map(); + const revoked = yield* Deferred.make(); + const serviceLayer = OutboundHub.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerSecretStore.ServerSecretStore, { + get: (key: string) => Effect.sync(() => Option.fromUndefinedOr(data.get(key))), + set: (key: string, value: Uint8Array) => + Effect.sync(() => { + data.set(key, value); + }), + } as unknown as ServerSecretStore.ServerSecretStore["Service"]), + Layer.succeed(HostedAuth.HostedAuth, { + bindEnvironment: () => Effect.void, + } as unknown as HostedAuth.HostedAuth["Service"]), + Layer.succeed(SessionStore.SessionStore, { + awaitInvalidation: () => Deferred.await(revoked), + } as unknown as SessionStore.SessionStore["Service"]), + ), + ), + ); + const hub = yield* OutboundHub.OutboundHub.pipe(Effect.provide(serviceLayer)); + const ready = yield* Deferred.make(); + const requests: Array = []; + const control = yield* socketPair; + const accepted = yield* hub.accept(session, control.left).pipe(Effect.forkScoped); + const reader = yield* control.right.reader; + const writer = yield* control.right.writer; + const send = (message: OutboundSourceMessage) => writer.write(JSON.stringify(message)); + const source = (port?: number) => + Effect.gen(function* () { + yield* send({ type: "register", descriptor }); + return yield* reader.pull.pipe( + Effect.flatMap((frames) => + Effect.forEach( + frames, + (frame) => + Effect.gen(function* () { + const message = yield* decodeHubMessage( + typeof frame === "string" ? frame : new TextDecoder().decode(frame), + ); + requests.push(message); + if (message.type === "ping") { + yield* Deferred.succeed(ready, undefined); + return yield* send({ type: "pong" }); + } + if (message.type === "grant") + return yield* send({ type: "grant", id: message.id, token: "scoped-leaf-token" }); + if (port === undefined) return; + const channel = yield* socketPair; + yield* hub.channel(session, message.id, channel.left).pipe(Effect.forkScoped); + yield* Effect.scoped( + Effect.gen(function* () { + const local = yield* NodeSocket.makeNet({ host: "127.0.0.1", port }); + return yield* bridge(local, channel.right); + }), + ).pipe(Effect.ignore, Effect.forkScoped); + }), + { discard: true }, + ), + ), + Effect.forever, + ); + }); + return { + hub, + data, + accepted, + requests, + source, + ready, + send, + revoke: Deferred.succeed(revoked, undefined), + restart: OutboundHub.OutboundHub.pipe(Effect.provide(serviceLayer)), + }; +}); + +it.live( + "keeps enrolled IDs across restart and removes interactive authority on device revocation", + () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.source().pipe(Effect.forkScoped); + yield* Deferred.await(f.ready); + const browser = { + ...session, + sessionId: AuthSessionId.make("browser"), + scopes: [AuthOrchestrationReadScope, AuthAccessWriteScope], + }; + const grant = yield* f.hub.connect(descriptor.environmentId, browser); + expect(grant.token).toBe("scoped-leaf-token"); + expect(f.requests.find((message) => message.type === "grant")).toMatchObject({ + scopes: [AuthOrchestrationReadScope], + }); + yield* f.hub.connect(descriptor.environmentId, browser); + expect(f.requests.filter((message) => message.type === "grant")).toHaveLength(1); + expect((yield* (yield* f.restart).enrollments)[0]?.descriptor.environmentId).toBe( + descriptor.environmentId, + ); + yield* f.revoke; + yield* Fiber.join(f.accepted); + yield* grant.invalidated; + expect( + (yield* f.hub.connect(descriptor.environmentId, browser).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect((yield* f.hub.enrollments)[0]?.descriptor.environmentId).toBe( + descriptor.environmentId, + ); + }).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("streams HTTP bytes through an outbound channel without a routable laptop URL", () => + Effect.gen(function* () { + const routes = HttpRouter.add( + "POST", + "/echo", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + expect(request.headers.authorization).toBe("Bearer scoped-leaf-token"); + return HttpServerResponse.uint8Array(new Uint8Array(yield* request.arrayBuffer)); + }), + ); + const server = yield* Layer.build( + HttpRouter.serve(routes, { disableListenLog: true }).pipe( + Layer.provideMerge(NodeHttpServer.layerTest), + ), + ); + const address = Context.get(server, HttpServer.HttpServer).address; + if (address._tag !== "InetAddressV4" && address._tag !== "InetAddressV6") + return yield* Effect.die("Missing fixture port"); + const f = yield* fixture; + yield* f.source(address.port).pipe(Effect.forkScoped); + yield* Deferred.await(f.ready); + const grant = yield* f.hub.connect(descriptor.environmentId, { + ...session, + scopes: [AuthOrchestrationReadScope], + }); + const bytes = new Uint8Array(524_288).map((_, index) => index % 251); + const response = yield* HttpClient.execute( + HttpClientRequest.post(`${grant.url}echo`).pipe( + HttpClientRequest.bodyUint8Array(bytes), + HttpClientRequest.bearerToken(grant.token), + ), + ); + expect(new Uint8Array(yield* response.arrayBuffer)).toEqual(bytes); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, FetchHttpClient.layer))), +); + +it.live( + "requires a hosted device, a connect grant, and the original session for one-use channels", + () => + Effect.gen(function* () { + const f = yield* fixture; + const socket = yield* socketPair; + const { deviceId: _deviceId, ...unbound } = session; + expect((yield* f.hub.accept(unbound, socket.left).pipe(Effect.result))._tag).toBe("Failure"); + expect( + (yield* f.hub.accept({ ...session, scopes: [] }, socket.left).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect( + (yield* f.hub + .channel({ ...session, deviceId: "device-b" }, "unissued", socket.left) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/environment/OutboundHub.ts b/apps/server/src/environment/OutboundHub.ts new file mode 100644 index 0000000000000000000000000000000000000000..24b816407217ee0b8845686d897d515f52b6c305 --- /dev/null +++ b/apps/server/src/environment/OutboundHub.ts @@ -0,0 +1,295 @@ +import * as NodeSocketServer from "@effect/platform-node/NodeSocketServer"; +import { + AuthEnvironmentConnectScope, + AuthStandardClientScopes, + EnvironmentId, + ExecutionEnvironmentDescriptor, + HubError, + OutboundSourceMessage, + OutboundHubMessage, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Clock from "effect/Clock"; +import * as Crypto from "effect/Crypto"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Socket from "effect/socket/Socket"; +import type { AuthenticatedSession } from "../auth/EnvironmentAuth.ts"; +import * as HostedAuth from "../auth/HostedAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as SessionStore from "../auth/SessionStore.ts"; +import { bridge } from "./OutboundSocket.ts"; + +const encodeHub = Schema.encodeSync(Schema.fromJsonString(OutboundHubMessage)); +const decodeSource = Schema.decodeEffect(Schema.fromJsonString(OutboundSourceMessage)); + +const Enrollment = Schema.Struct({ + deviceId: Schema.String, + descriptor: ExecutionEnvironmentDescriptor, +}); +const Stored = Schema.fromJsonString(Schema.Array(Enrollment)); +const decodeStored = Schema.decodeEffect(Stored); +const encodeStored = Schema.encodeEffect(Stored); +type Enrollment = typeof Enrollment.Type; +type Peer = { + readonly session: AuthenticatedSession; + readonly url: string; + readonly send: (message: OutboundHubMessage) => Effect.Effect; + readonly closed: Deferred.Deferred; + readonly grants: Map>; + readonly credentials: Map; +}; + +export class OutboundHub extends Context.Service< + OutboundHub, + { + readonly enrollments: Effect.Effect>; + readonly connect: ( + id: EnvironmentId, + session: AuthenticatedSession, + ) => Effect.Effect< + { + readonly url: string; + readonly token: string; + readonly invalidated: Effect.Effect; + }, + HubError + >; + readonly remove: (id: EnvironmentId) => Effect.Effect; + readonly accept: ( + session: AuthenticatedSession, + socket: Socket.Socket, + ) => Effect.Effect; + readonly channel: ( + session: AuthenticatedSession, + id: string, + socket: Socket.Socket, + ) => Effect.Effect; + } +>()("t3/environment/OutboundHub") {} + +const isHubError = Schema.is(HubError); +const failure = (cause: unknown) => + isHubError(cause) ? cause : new HubError({ reason: "upstream", cause: Redacted.make(cause) }); + +const make = Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const hosted = yield* HostedAuth.HostedAuth; + const sessions = yield* SessionStore.SessionStore; + const crypto = yield* Crypto.Crypto; + const lock = yield* Semaphore.make(1); + const stored = yield* secrets.get("outbound-environments").pipe(Effect.mapError(failure)); + let enrollments = Option.isSome(stored) + ? yield* decodeStored(new TextDecoder().decode(stored.value)).pipe(Effect.mapError(failure)) + : []; + const peers = new Map(); + const channels = new Map< + string, + { + peer: Peer; + socket: Socket.Socket; + done: Deferred.Deferred; + attached: Deferred.Deferred; + claimed: boolean; + } + >(); + const persist = (next: ReadonlyArray) => + encodeStored(next).pipe( + Effect.flatMap((json) => + secrets.set("outbound-environments", new TextEncoder().encode(json)), + ), + Effect.tap(() => + Effect.sync(() => { + enrollments = next; + }), + ), + Effect.mapError(failure), + ); + + const accept: OutboundHub["Service"]["accept"] = (session, socket) => + Effect.gen(function* () { + if (!session.deviceId || !session.scopes.includes(AuthEnvironmentConnectScope)) { + return yield* new HubError({ reason: "credential" }); + } + const deviceId = session.deviceId; + const reader = yield* socket.reader; + const writer = yield* socket.writer; + const writeLock = yield* Semaphore.make(1); + const send = (message: OutboundHubMessage) => + writeLock.withPermit(writer.write(encodeHub(message))); + const read = reader.pull.pipe( + Effect.flatMap((frames) => + Effect.forEach(frames, (frame) => { + if ((typeof frame === "string" ? Buffer.byteLength(frame) : frame.byteLength) > 65_536) + return Effect.fail(new HubError({ reason: "identity" })); + const text = typeof frame === "string" ? frame : new TextDecoder().decode(frame); + return decodeSource(text).pipe(Effect.mapError(failure)); + }), + ), + Effect.timeout("60 seconds"), + ); + const initial = yield* read.pipe(Effect.timeout("10 seconds")); + if (initial.length !== 1 || initial[0]?.type !== "register") + return yield* new HubError({ reason: "identity" }); + const descriptor = initial[0].descriptor; + const id = descriptor.environmentId; + yield* hosted.bindEnvironment(session, id); + yield* lock.withPermit( + Effect.gen(function* () { + yield* persist([ + ...enrollments.filter((value) => value.descriptor.environmentId !== id), + { deviceId, descriptor }, + ]); + }), + ); + const server = yield* NodeSocketServer.make({ host: "127.0.0.1", port: 0 }); + if (server.address._tag !== "InetAddressV4") + return yield* new HubError({ reason: "upstream" }); + const closed = yield* Deferred.make(); + const peer: Peer = { + session, + url: `http://127.0.0.1:${server.address.port}/`, + send, + closed, + grants: new Map(), + credentials: new Map(), + }; + const previous = peers.get(id); + if (previous) yield* Deferred.succeed(previous.closed, undefined); + peers.set(id, peer); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + if (peers.get(id) === peer) peers.delete(id); + yield* Deferred.succeed(closed, undefined); + for (const [key, channel] of channels) + if (channel.peer === peer) { + channels.delete(key); + yield* Deferred.succeed(channel.done, undefined); + } + }), + ); + yield* server + .run((local) => + Effect.scoped( + Effect.gen(function* () { + if (channels.size >= 512) return; + const channelId = yield* crypto.randomUUIDv4(); + const done = yield* Deferred.make(); + const attached = yield* Deferred.make(); + channels.set(channelId, { peer, socket: local, done, attached, claimed: false }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + channels.delete(channelId); + }), + ); + yield* send({ type: "connect", id: channelId }); + yield* Deferred.await(attached).pipe(Effect.timeout("10 seconds")); + yield* Deferred.await(done); + }), + ).pipe(Effect.ignore), + ) + .pipe(Effect.forkScoped); + yield* Effect.forever( + send({ type: "ping" }).pipe(Effect.andThen(Effect.sleep("20 seconds"))), + ).pipe(Effect.forkScoped); + const messages = Effect.forever( + read.pipe( + Effect.flatMap((messages) => + Effect.forEach( + messages, + (message) => { + if (message.type !== "grant") + return message.type === "pong" + ? Effect.void + : Effect.fail(new HubError({ reason: "identity" })); + const pending = peer.grants.get(message.id); + return pending + ? Deferred.succeed(pending, message.token).pipe(Effect.asVoid) + : Effect.void; + }, + { discard: true }, + ), + ), + ), + ); + yield* Effect.raceFirst( + messages, + Effect.raceFirst(Deferred.await(closed), sessions.awaitInvalidation(session.sessionId)), + ); + }).pipe(Effect.scoped, Effect.mapError(failure)); + + return OutboundHub.of({ + enrollments: Effect.sync(() => enrollments), + accept, + remove: (id) => + lock.withPermit( + Effect.gen(function* () { + yield* persist(enrollments.filter((value) => value.descriptor.environmentId !== id)); + const peer = peers.get(id); + if (peer) yield* Deferred.succeed(peer.closed, undefined); + }), + ), + connect: (id, session) => + Effect.gen(function* () { + const peer = peers.get(id); + if (!peer) return yield* new HubError({ reason: "upstream" }); + const scopes = AuthStandardClientScopes.filter((scope) => session.scopes.includes(scope)); + if (scopes.length === 0) return yield* new HubError({ reason: "credential" }); + const now = yield* Clock.currentTimeMillis; + for (const [key, credential] of peer.credentials) + if (credential.expiresAt <= now) peer.credentials.delete(key); + const key = `${session.sessionId}:${scopes.join(" ")}`; + const cached = peer.credentials.get(key); + if (cached) + return { url: peer.url, token: cached.token, invalidated: Deferred.await(peer.closed) }; + if (peer.grants.size >= 128 || peer.credentials.size >= 512) + return yield* new HubError({ reason: "upstream" }); + const grantId = yield* crypto.randomUUIDv4(); + const grant = yield* Deferred.make(); + peer.grants.set(grantId, grant); + const token = yield* Effect.gen(function* () { + yield* peer.send({ type: "grant", id: grantId, scopes }); + return yield* Effect.raceFirst( + Deferred.await(grant), + Deferred.await(peer.closed).pipe( + Effect.andThen(Effect.fail(new HubError({ reason: "upstream" }))), + ), + ); + }).pipe( + Effect.timeout("10 seconds"), + Effect.ensuring( + Effect.sync(() => { + peer.grants.delete(grantId); + }), + ), + ); + peer.credentials.set(key, { token, expiresAt: now + 60_000 }); + return { url: peer.url, token, invalidated: Deferred.await(peer.closed) }; + }).pipe(Effect.mapError(failure)), + channel: (session, id, socket) => + Effect.gen(function* () { + const channel = channels.get(id); + if ( + !channel || + channel.claimed || + session.deviceId !== channel.peer.session.deviceId || + session.sessionId !== channel.peer.session.sessionId + ) + return yield* new HubError({ reason: "credential" }); + channel.claimed = true; + yield* Deferred.succeed(channel.attached, undefined); + yield* Effect.raceFirst( + bridge(channel.socket, socket), + Deferred.await(channel.peer.closed), + ).pipe(Effect.ensuring(Deferred.succeed(channel.done, undefined)), Effect.ignore); + }), + }); +}); + +export const layer = Layer.effect(OutboundHub, make); diff --git a/apps/server/src/environment/OutboundSocket.test.ts b/apps/server/src/environment/OutboundSocket.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c754e54bcef4f3c88639e58501c9243d3f7b3f5 --- /dev/null +++ b/apps/server/src/environment/OutboundSocket.test.ts @@ -0,0 +1,114 @@ +import { expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Socket from "effect/socket/Socket"; +import { bridge } from "./OutboundSocket.ts"; + +type Frame = string | Uint8Array; +const endpoint = (incoming: Queue.Queue, write: (frame: Frame) => Effect.Effect) => { + const send = (frame: Frame | Socket.CloseEvent) => + Socket.isCloseEvent(frame) ? Effect.void : write(frame); + return Socket.make({ + reader: Effect.succeed({ + pull: Queue.take(incoming).pipe(Effect.map((frame) => [frame] as const)), + upgrade: () => Effect.void, + }), + writer: Effect.succeed({ + write: send, + writeAll: (frames) => Effect.forEach(frames, send, { discard: true }), + }), + }); +}; +const tunnelPair = Effect.gen(function* () { + const left = yield* Queue.unbounded(); + const right = yield* Queue.unbounded(); + const sent: Uint8Array[] = []; + const full = yield* Deferred.make(); + return { + left: endpoint(left, (frame) => + Effect.gen(function* () { + if (typeof frame !== "string" && frame[0] === 0) { + sent.push(frame); + if (sent.length === 16) yield* Deferred.succeed(full, undefined); + } + yield* Queue.offer(right, frame); + }), + ), + right: endpoint(right, (frame) => Queue.offer(left, frame).pipe(Effect.asVoid)), + injectLeft: (frame: Frame) => Queue.offer(left, frame), + sent, + full, + }; +}); +const tcp = (expectedBytes: number, stalled?: Deferred.Deferred) => + Effect.gen(function* () { + const outgoing = yield* Queue.unbounded(); + const received: Uint8Array[] = []; + const done = yield* Deferred.make(); + const socket = endpoint(outgoing, (frame) => + Effect.gen(function* () { + if (stalled) yield* Deferred.await(stalled); + received.push(typeof frame === "string" ? new TextEncoder().encode(frame) : frame); + if (received.reduce((size, chunk) => size + chunk.byteLength, 0) >= expectedBytes) + yield* Deferred.succeed(done, undefined); + }), + ); + return { socket, received, done, send: (frame: Frame) => Queue.offer(outgoing, frame) }; + }); + +it.effect( + "bounds a stalled tunnel to sixteen 64KiB payloads and releases a sender waiting for credit on cancellation", + () => + Effect.gen(function* () { + const pair = yield* tunnelPair; + const left = yield* tcp(0); + const gate = yield* Deferred.make(); + const right = yield* tcp(1, gate); + const finalized = yield* Deferred.make(); + const sender = yield* bridge(left.socket, pair.left).pipe( + Effect.ensuring(Deferred.succeed(finalized, undefined)), + Effect.forkScoped({ startImmediately: true }), + ); + yield* bridge(right.socket, pair.right).pipe(Effect.forkScoped({ startImmediately: true })); + yield* left.send(new Uint8Array(64 * 1024 * 40)); + yield* Deferred.await(pair.full); + expect(pair.sent).toHaveLength(16); + expect(pair.sent.reduce((size, frame) => size + frame.byteLength - 5, 0)).toBe(1024 * 1024); + expect(pair.sent.every((frame) => frame.byteLength <= 64 * 1024 + 5)).toBe(true); + yield* Fiber.interrupt(sender); + expect(yield* Deferred.isDone(finalized)).toBe(true); + }), +); + +it.effect("streams bursts in both directions without credit deadlock or byte loss", () => + Effect.gen(function* () { + const pair = yield* tunnelPair; + const bytes = new Uint8Array(64 * 1024 * 40).map((_, index) => index % 251); + const reverse = bytes.toReversed(); + const left = yield* tcp(bytes.length); + const right = yield* tcp(bytes.length); + yield* bridge(left.socket, pair.left).pipe(Effect.forkScoped({ startImmediately: true })); + yield* bridge(right.socket, pair.right).pipe(Effect.forkScoped({ startImmediately: true })); + yield* left.send(bytes); + yield* right.send(reverse); + yield* Deferred.await(left.done); + yield* Deferred.await(right.done); + expect(Buffer.concat(left.received)).toEqual(Buffer.from(reverse)); + expect(Buffer.concat(right.received)).toEqual(Buffer.from(bytes)); + }), +); + +it.effect("rejects unsolicited acknowledgments instead of manufacturing send credit", () => + Effect.gen(function* () { + const pair = yield* tunnelPair; + const local = yield* tcp(0); + const running = yield* bridge(local.socket, pair.left).pipe( + Effect.result, + Effect.forkScoped({ startImmediately: true }), + ); + yield* pair.injectLeft(new Uint8Array([1, 0, 0, 0, 0])); + expect((yield* Fiber.join(running))._tag).toBe("Failure"); + }), +); diff --git a/apps/server/src/environment/OutboundSocket.ts b/apps/server/src/environment/OutboundSocket.ts new file mode 100644 index 0000000000000000000000000000000000000000..91aab1b1869d969fc322e3943d543d5f80bc41c3 --- /dev/null +++ b/apps/server/src/environment/OutboundSocket.ts @@ -0,0 +1,83 @@ +import { HubError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Semaphore from "effect/Semaphore"; +import * as Socket from "effect/socket/Socket"; + +const CHUNK_BYTES = 64 * 1024; +const WINDOW_CHUNKS = 16; + +/** ACKs follow TCP drain, bounding WebSocket's otherwise unbounded send buffer. */ +export const bridge = (tcp: Socket.Socket, tunnel: Socket.Socket) => + Effect.gen(function* () { + const tcpReader = yield* tcp.reader; + const tunnelReader = yield* tunnel.reader; + const tcpWriter = yield* tcp.writer; + const tunnelWriter = yield* tunnel.writer; + const credits = yield* Semaphore.make(WINDOW_CHUNKS); + const writes = yield* Semaphore.make(1); + const pending = new Set(); + let sequence = 0; + let expected = 0; + + const send = tcpReader.pull.pipe( + Effect.flatMap((chunks) => + Effect.forEach( + chunks, + (chunk) => + Effect.gen(function* () { + const bytes = typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk; + for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_BYTES) { + yield* credits.take(1); + if (sequence > 0xffff_ffff) return yield* new HubError({ reason: "upstream" }); + const id = sequence++; + const payload = bytes.subarray(offset, offset + CHUNK_BYTES); + const frame = new Uint8Array(5 + payload.byteLength); + new DataView(frame.buffer).setUint32(1, id); + frame.set(payload, 5); + pending.add(id); + yield* writes.withPermit(tunnelWriter.write(frame)); + } + }), + { discard: true }, + ), + ), + Effect.forever, + ); + const receive = tunnelReader.pull.pipe( + Effect.flatMap((frames) => + Effect.forEach( + frames, + (frame) => + Effect.gen(function* () { + if ( + typeof frame === "string" || + frame.byteLength < 5 || + frame.byteLength > CHUNK_BYTES + 5 + ) { + return yield* new HubError({ reason: "identity" }); + } + const id = new DataView(frame.buffer, frame.byteOffset, frame.byteLength).getUint32( + 1, + ); + if (frame[0] === 1) { + if (frame.byteLength !== 5 || !pending.delete(id)) + return yield* new HubError({ reason: "identity" }); + yield* credits.release(1); + return; + } + if (frame[0] !== 0 || frame.byteLength === 5 || id !== expected++) { + return yield* new HubError({ reason: "identity" }); + } + yield* tcpWriter.write(frame.subarray(5)); + const ack = new Uint8Array(5); + ack[0] = 1; + new DataView(ack.buffer).setUint32(1, id); + yield* writes.withPermit(tunnelWriter.write(ack)); + }), + { discard: true }, + ), + ), + Effect.forever, + ); + return yield* Effect.raceFirst(send, receive); + }); diff --git a/apps/server/src/environment/OutboundSource.ts b/apps/server/src/environment/OutboundSource.ts new file mode 100644 index 0000000000000000000000000000000000000000..d8b3fe0c021f9cede0323d700d109ca05b8aedc4 --- /dev/null +++ b/apps/server/src/environment/OutboundSource.ts @@ -0,0 +1,169 @@ +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import { + AuthStandardClientScopes, + HubError, + OutboundHubMessage, + type AuthSessionId, + type HostedSourceConnection, + OutboundSourceMessage, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpServer } from "effect/http"; +import * as Socket from "effect/socket/Socket"; +import * as SessionStore from "../auth/SessionStore.ts"; +import * as ServerActivation from "../serverActivation.ts"; +import { bridge } from "./OutboundSocket.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; +import * as SourceConnection from "./SourceConnection.ts"; + +const encodeSource = Schema.encodeSync(Schema.fromJsonString(OutboundSourceMessage)); +const decodeHub = Schema.decodeEffect(Schema.fromJsonString(OutboundHubMessage)); + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const connection = yield* SourceConnection.SourceConnection; + const sessions = yield* SessionStore.SessionStore; + const environment = yield* ServerEnvironment.ServerEnvironment; + const server = yield* HttpServer.HttpServer; + const address = server.address; + if (address._tag !== "InetAddressV4" && address._tag !== "InetAddressV6") return; + + const connect = (target: HostedSourceConnection) => + Effect.scoped( + Effect.gen(function* () { + const now = yield* DateTime.now; + const remaining = target.expiresAt.epochMilliseconds - now.epochMilliseconds; + if (remaining <= 0) return yield* Effect.never; + const url = new URL("api/hub/outbound", target.serverUrl); + url.protocol = "wss:"; + const socket = yield* Socket.makeWebSocket(url.href, { + headers: { authorization: target.authorization }, + openTimeout: "10 seconds", + highWaterMark: 1024 * 1024, + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructorWS)); + const reader = yield* socket.reader; + const writer = yield* socket.writer; + const lock = yield* Semaphore.make(1); + const send = (message: OutboundSourceMessage) => + lock.withPermit(writer.write(encodeSource(message))); + const grants = new Map(); + const channels = yield* Semaphore.make(512); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* connection.setConnected(false); + yield* Effect.forEach( + grants.keys(), + (id) => sessions.revoke(id).pipe(Effect.ignore), + { concurrency: 8, discard: true }, + ); + }), + ); + yield* send({ type: "register", descriptor: yield* environment.getDescriptor }); + const run = reader.pull.pipe( + Effect.timeout("60 seconds"), + Effect.flatMap((frames) => + Effect.forEach( + frames, + (frame) => + Effect.gen(function* () { + if ( + (typeof frame === "string" ? Buffer.byteLength(frame) : frame.byteLength) > + 65_536 + ) + return yield* new HubError({ reason: "identity" }); + const text = + typeof frame === "string" ? frame : new TextDecoder().decode(frame); + const message = yield* decodeHub(text); + yield* connection.setConnected(true); + if (message.type === "ping") return yield* send({ type: "pong" }); + if (message.type === "grant") { + if ( + message.scopes.length === 0 || + message.scopes.some( + (scope) => !AuthStandardClientScopes.some((allowed) => allowed === scope), + ) + ) { + return yield* new HubError({ reason: "credential" }); + } + const now = yield* DateTime.now; + for (const [id, expiresAt] of grants) + if (expiresAt <= now.epochMilliseconds) grants.delete(id); + if (grants.size >= 1_024) return yield* new HubError({ reason: "upstream" }); + const issued = yield* sessions.issue({ + subject: "self-hosted-outbound", + method: "bearer-access-token", + scopes: [...new Set(message.scopes)], + ttl: Duration.millis( + Math.min( + 120_000, + target.expiresAt.epochMilliseconds - now.epochMilliseconds, + ), + ), + }); + grants.set(issued.sessionId, issued.expiresAt.epochMilliseconds); + return yield* send({ type: "grant", id: message.id, token: issued.token }); + } + yield* Effect.scoped( + Effect.gen(function* () { + const channelUrl = new URL( + `api/hub/outbound/channel/${encodeURIComponent(message.id)}`, + target.serverUrl, + ); + channelUrl.protocol = "wss:"; + const channel = yield* Socket.makeWebSocket(channelUrl.href, { + headers: { authorization: target.authorization }, + openTimeout: "10 seconds", + highWaterMark: 1024 * 1024, + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructorWS)); + const local = yield* NodeSocket.makeNet({ + host: "127.0.0.1", + port: address.port, + openTimeout: "10 seconds", + }); + return yield* bridge(local, channel); + }), + ).pipe( + channels.withPermitsIfAvailable(1), + Effect.ignore, + Effect.forkScoped({ startImmediately: true }), + ); + }), + { discard: true }, + ), + ), + Effect.forever, + ); + yield* Effect.raceFirst(run, Effect.sleep(Duration.millis(remaining))); + }), + ); + + yield* ServerActivation.forkParked( + connection.changes.pipe( + Stream.switchMap((target) => + Option.isNone(target) + ? Stream.never + : Stream.fromEffect( + connect(target.value).pipe( + Effect.retry( + Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ), + ), + ), + ), + ), + Stream.runDrain, + ), + ); + }), +); diff --git a/apps/server/src/environment/SharedServer.ts b/apps/server/src/environment/SharedServer.ts new file mode 100644 index 0000000000000000000000000000000000000000..6070c22274edac284f20aa7d060a29e4b112c754 --- /dev/null +++ b/apps/server/src/environment/SharedServer.ts @@ -0,0 +1,46 @@ +import { SharedAccountError } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; + +const Connection = Schema.Struct({ url: Schema.String, token: Schema.NonEmptyString }); +const decode = Schema.decodeEffect(Schema.fromJsonString(Connection)); +const encode = Schema.encodeEffect(Schema.fromJsonString(Connection)); + +export class SharedServer extends Context.Service< + SharedServer, + { + readonly connection: Effect.Effect, SharedAccountError>; + readonly save: (connection: typeof Connection.Type) => Effect.Effect; + readonly disconnect: Effect.Effect; + } +>()("t3/environment/SharedServer") {} + +export const layer = Layer.effect( + SharedServer, + Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const storageError = (cause: unknown) => + new SharedAccountError({ reason: "storage", cause: Redacted.make(cause) }); + return SharedServer.of({ + connection: secrets.get("shared-server").pipe( + Effect.flatMap((bytes) => + Option.isNone(bytes) + ? Effect.succeedNone + : decode(new TextDecoder().decode(bytes.value)).pipe(Effect.asSome), + ), + Effect.mapError(storageError), + ), + save: (value) => + encode(value).pipe( + Effect.flatMap((json) => secrets.set("shared-server", new TextEncoder().encode(json))), + Effect.mapError(storageError), + ), + disconnect: secrets.remove("shared-server").pipe(Effect.mapError(storageError)), + }); + }), +); diff --git a/apps/server/src/environment/SourceConnection.test.ts b/apps/server/src/environment/SourceConnection.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..1cd4c309f5884fa2df6541bc7a1232fff740d077 --- /dev/null +++ b/apps/server/src/environment/SourceConnection.test.ts @@ -0,0 +1,132 @@ +import { expect, it } from "@effect/vitest"; +import { HostedSourceConnection } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import * as SourceConnection from "./SourceConnection.ts"; + +const sourceCodec = Schema.toCodecJson(HostedSourceConnection); +const encodeSource = Schema.encodeEffect(sourceCodec); +const decodeSource = Schema.decodeEffect(sourceCodec); +const scopes = ["archive:read", "archive:write", "environment:connect"]; +const fixture = Effect.gen(function* () { + const now = yield* DateTime.now; + const expiresAt = DateTime.add(now, { minutes: 15 }); + const input = { + serverUrl: "https://host.example", + authorization: "Bearer narrow-token", + expiresAt, + }; + let requests = 0; + let permissions = scopes; + let authenticated = true; + let status = 200; + let remoteExpiry = DateTime.formatIso(expiresAt); + const http = HttpClient.make((request) => + Effect.sync(() => { + requests++; + expect(request.url).toBe("https://host.example/api/auth/session"); + expect(request.headers.authorization).toBe("Bearer narrow-token"); + return HttpClientResponse.fromWeb( + request, + Response.json( + { + authenticated, + permissions, + expiresAt: remoteExpiry, + sessionMethod: "bearer-access-token", + }, + { status }, + ), + ); + }), + ); + const source = yield* SourceConnection.SourceConnection.pipe( + Effect.provide( + SourceConnection.layer.pipe(Layer.provide(Layer.succeed(HttpClient.HttpClient, http))), + ), + ); + return { + source, + input, + requests: () => requests, + setPermissions: (value: string[]) => { + permissions = value; + }, + deny: () => { + status = 401; + }, + setAuthenticated: (value: boolean) => { + authenticated = value; + }, + setExpiry: (value: string) => { + remoteExpiry = value; + }, + }; +}); + +it.effect( + "accepts only verified narrow source grants, normalizes URL and deduplicates replay", + () => + Effect.gen(function* () { + const f = yield* fixture; + const wire = yield* encodeSource(f.input); + const input = yield* decodeSource(wire); + yield* f.source.update(input); + yield* f.source.setConnected(true); + yield* f.source.update({ ...input, serverUrl: "https://host.example/" }); + expect(f.requests()).toBe(1); + expect(yield* f.source.connected).toBe(true); + expect(Option.getOrThrow(yield* f.source.current).serverUrl).toBe("https://host.example/"); + yield* f.source.update(undefined); + expect(Option.isNone(yield* f.source.current)).toBe(true); + }), +); + +it.effect.each([ + { permissions: scopes.slice(0, 2) }, + { permissions: [...scopes, "providers:manage"] }, + { permissions: [...scopes, "access:write"] }, + { permissions: ["archive:read", "archive:read", "environment:connect"] }, +])("rejects non-source permission set $permissions", ({ permissions }) => + Effect.gen(function* () { + const f = yield* fixture; + f.setPermissions(permissions); + expect((yield* f.source.update(f.input).pipe(Effect.flip)).reason).toBe("credential"); + expect(Option.isNone(yield* f.source.current)).toBe(true); + }), +); + +it.effect("rejects denied, anonymous and shorter-lived remote credentials", () => + Effect.gen(function* () { + for (const change of [ + (f: Effect.Success) => f.deny(), + (f: Effect.Success) => f.setAuthenticated(false), + (f: Effect.Success) => f.setExpiry("1970-01-01T00:00:00.000Z"), + ]) { + const f = yield* fixture; + change(f); + expect((yield* f.source.update(f.input).pipe(Effect.flip)).reason).toBe("credential"); + expect(Option.isNone(yield* f.source.current)).toBe(true); + } + }), +); + +it.effect("rejects unsafe origins and invalid grant lifetimes before sending the token", () => + Effect.gen(function* () { + const f = yield* fixture; + const now = yield* DateTime.now; + for (const input of [ + { ...f.input, serverUrl: "http://host.example" }, + { ...f.input, serverUrl: "https://host.example/another-path" }, + { ...f.input, serverUrl: "https://user:password@host.example" }, + { ...f.input, expiresAt: now }, + { ...f.input, expiresAt: DateTime.add(now, { hours: 2 }) }, + ]) + expect((yield* f.source.update(input).pipe(Effect.result))._tag).toBe("Failure"); + expect(f.requests()).toBe(0); + }), +); diff --git a/apps/server/src/environment/SourceConnection.ts b/apps/server/src/environment/SourceConnection.ts new file mode 100644 index 0000000000000000000000000000000000000000..3c892b08c90e9140a4a0245e62177bf2616f0307 --- /dev/null +++ b/apps/server/src/environment/SourceConnection.ts @@ -0,0 +1,115 @@ +import { + AuthArchiveReadScope, + AuthArchiveWriteScope, + AuthEnvironmentConnectScope, + HostedSourceConnection, + HubError, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; + +const SourceSession = Schema.toCodecJson( + Schema.Struct({ + authenticated: Schema.Literal(true), + permissions: Schema.Array(Schema.String), + expiresAt: Schema.DateTimeUtc, + sessionMethod: Schema.Literal("bearer-access-token"), + }), +); +const sourceScopes = [AuthArchiveReadScope, AuthArchiveWriteScope, AuthEnvironmentConnectScope]; + +export class SourceConnection extends Context.Service< + SourceConnection, + { + readonly current: Effect.Effect>; + readonly connected: Effect.Effect; + readonly setConnected: (connected: boolean) => Effect.Effect; + readonly update: (input: HostedSourceConnection | undefined) => Effect.Effect; + readonly changes: Stream.Stream>; + } +>()("t3/environment/SourceConnection") {} + +const make = Effect.gen(function* () { + const current = yield* SubscriptionRef.make(Option.none()); + const http = yield* HttpClient.HttpClient; + const lock = yield* Semaphore.make(1); + let connected = false; + return SourceConnection.of({ + current: SubscriptionRef.get(current), + connected: Effect.sync(() => connected), + setConnected: (value) => + Effect.sync(() => { + connected = value; + }), + changes: SubscriptionRef.changes(current), + update: (input) => + lock.withPermit( + Effect.gen(function* () { + if (input) { + const url = yield* Effect.try({ + try: () => new URL(input.serverUrl), + catch: () => new HubError({ reason: "address" }), + }); + const now = yield* DateTime.now; + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/" || + !/^Bearer [A-Za-z0-9._~-]+$/u.test(input.authorization) || + input.expiresAt.epochMilliseconds <= now.epochMilliseconds || + input.expiresAt.epochMilliseconds > now.epochMilliseconds + 3_600_000 + ) { + return yield* new HubError({ reason: "credential" }); + } + const previous = yield* SubscriptionRef.get(current); + const serverUrl = `${url.origin}/`; + if ( + Option.isSome(previous) && + previous.value.serverUrl === serverUrl && + previous.value.authorization === input.authorization && + previous.value.expiresAt.epochMilliseconds === input.expiresAt.epochMilliseconds + ) + return; + const verified = yield* Effect.gen(function* () { + const response = yield* http + .execute( + HttpClientRequest.get(new URL("api/auth/session", serverUrl).href).pipe( + HttpClientRequest.setHeader("authorization", input.authorization), + ), + ) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" })); + if (response.status !== 200) return yield* new HubError({ reason: "credential" }); + return yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(SourceSession)), + ); + }).pipe( + Effect.timeout("10 seconds"), + Effect.scoped, + Effect.mapError(() => new HubError({ reason: "credential" })), + ); + if ( + verified.expiresAt.epochMilliseconds < input.expiresAt.epochMilliseconds || + verified.permissions.length !== sourceScopes.length || + !sourceScopes.every((scope) => verified.permissions.includes(scope)) + ) { + return yield* new HubError({ reason: "credential" }); + } + yield* SubscriptionRef.set(current, Option.some({ ...input, serverUrl })); + } else yield* SubscriptionRef.set(current, Option.none()); + }), + ), + }); +}); + +export const layer = Layer.effect(SourceConnection, make); diff --git a/apps/server/src/environment/SourceConnectionHttp.ts b/apps/server/src/environment/SourceConnectionHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..b3d4365617b3e180c7c32a870e735257873d82c4 --- /dev/null +++ b/apps/server/src/environment/SourceConnectionHttp.ts @@ -0,0 +1,95 @@ +import { + AuthAccessReadScope, + AuthAccessWriteScope, + AuthEnvironmentConnectScope, + HostedSourceConnection, + HubError, + SelfHostedConnectionStatus, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http"; +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as OutboundHub from "./OutboundHub.ts"; +import * as SourceConnection from "./SourceConnection.ts"; + +const isHubError = Schema.is(HubError); +const statusResponse = HttpServerResponse.schemaJson(SelfHostedConnectionStatus); + +const errors = (effect: Effect.Effect) => + effect.pipe( + Effect.catch((error) => + Effect.succeed( + HttpServerResponse.empty({ + status: EnvironmentAuth.isServerAuthCredentialError(error) + ? 401 + : Schema.isSchemaError(error) + ? 400 + : isHubError(error) && error.reason === "credential" + ? 403 + : isHubError(error) && error.reason === "address" + ? 400 + : 503, + }), + ), + ), + ); + +const connection = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* auth.authenticateHttpRequest(request); + const source = yield* SourceConnection.SourceConnection; + if (request.method === "GET") { + if (!session.scopes.includes(AuthAccessReadScope)) + return HttpServerResponse.empty({ status: 403 }); + const current = yield* source.current; + return yield* statusResponse( + { + configured: Option.isSome(current), + connected: yield* source.connected, + expiresAt: Option.isSome(current) ? DateTime.formatIso(current.value.expiresAt) : null, + }, + { headers: { "cache-control": "no-store" } }, + ); + } + if ( + session.subject !== "desktop-bootstrap" || + session.parentSessionId !== undefined || + !session.scopes.includes(AuthAccessWriteScope) + ) { + return HttpServerResponse.empty({ status: 403 }); + } + yield* source.update( + request.method === "DELETE" + ? undefined + : yield* HttpServerRequest.schemaBodyJson(Schema.toCodecJson(HostedSourceConnection)), + ); + return HttpServerResponse.empty({ status: 204 }); +}); + +const outbound = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const session = yield* (yield* EnvironmentAuth.EnvironmentAuth).authenticateWebSocketUpgrade( + request, + ); + const hub = yield* OutboundHub.OutboundHub; + const params = yield* HttpRouter.params; + if (!session.deviceId || !session.scopes.includes(AuthEnvironmentConnectScope)) + return HttpServerResponse.empty({ status: 403 }); + const socket = yield* request.upgrade; + if (params.channelId) yield* hub.channel(session, params.channelId, socket); + else yield* hub.accept(session, socket); + return HttpServerResponse.empty(); +}); + +export const layer = Layer.mergeAll( + HttpRouter.add("GET", "/api/self-hosted/connection", connection.pipe(errors)), + HttpRouter.add("POST", "/api/self-hosted/connection", connection.pipe(errors)), + HttpRouter.add("DELETE", "/api/self-hosted/connection", connection.pipe(errors)), + HttpRouter.add("GET", "/api/hub/outbound", outbound.pipe(errors)), + HttpRouter.add("GET", "/api/hub/outbound/channel/:channelId", outbound.pipe(errors)), +); diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 0700736c8a691b6e67553222cf2ff3bed42045fe..7512de5b500e7ee83391615d7adc1fd157136f14 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -1,8 +1,15 @@ import * as Mime from "effect/http/Mime"; import { AuthOrchestrationOperateScope, + AuthArchiveReadScope, + AuthArchiveWriteScope, AuthOrchestrationReadScope, AuthProvidersManageScope, + ArchiveManagementRequest, + ArchiveError, + EnvironmentAuthInvalidError, + EnvironmentInternalError, + EnvironmentScopeRequiredError, SharedAccountRequest, SharedProfilesRequest, SharedAccountError, @@ -35,6 +42,9 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import { OtlpTracer, OtlpSerialization } from "effect/observability"; import * as ServerConfig from "./config.ts"; +import * as ArchiveObjects from "./archive/ArchiveObjects.ts"; +import * as ArchiveAssetResponse from "./archive/ArchiveAssetResponse.ts"; +import * as TranscriptArchive from "./archive/TranscriptArchive.ts"; import { ASSET_ROUTE_PREFIX, readMcpAppHtml, resolveAsset } from "./assets/AssetAccess.ts"; import { githubMediaResponse } from "./assets/GitHubMediaFetch.ts"; import { statMediaFile, streamMediaFile, type OpenMediaFile } from "./assets/MediaFile.ts"; @@ -307,7 +317,10 @@ export function resolveDevRedirectUrl(devUrl: URL, requestUrl: URL): string { return redirectUrl.toString(); } -const authenticateRawRouteWithScope = (scope: AuthEnvironmentScope) => +export const authenticateRawRouteWithScope = ( + scope: AuthEnvironmentScope, + legacyScope?: AuthEnvironmentScope, +) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; @@ -322,11 +335,126 @@ const authenticateRawRouteWithScope = (scope: AuthEnvironmentScope) => failEnvironmentInternal("internal_error", error), ), ); - if (!session.scopes.includes(scope)) { + if (!session.scopes.includes(scope) && !(legacyScope && session.scopes.includes(legacyScope))) { return yield* failEnvironmentScopeRequired(scope); } + return session; }); +const isArchiveError = Schema.is(ArchiveError); +const isArchiveAuthError = Schema.is( + Schema.Union([ + EnvironmentAuthInvalidError, + EnvironmentInternalError, + EnvironmentScopeRequiredError, + ]), +); +export const archiveRouteErrors = (effect: Effect.Effect) => + effect.pipe( + Effect.catch((error): Effect.Effect => { + if (isArchiveError(error)) + return Effect.succeed( + HttpServerResponse.jsonUnsafe( + { _tag: error._tag, reason: error.reason }, + { + status: error.reason === "integrity" ? 400 : 503, + headers: { "cache-control": "no-store" }, + }, + ), + ); + if (isArchiveAuthError(error)) return HttpServerRespondable.toResponse(error); + return Effect.fail(error); + }), + ); + +export const layerArchiveManagementRoute = HttpRouter.add( + "POST", + "/api/archive/manage", + Effect.gen(function* () { + const input = yield* HttpServerRequest.schemaBodyJson(ArchiveManagementRequest).pipe( + Effect.mapError( + (cause) => new ArchiveError({ reason: "integrity", cause: Redacted.make(cause) }), + ), + ); + yield* authenticateRawRouteWithScope( + input.type === "archive.status" ? AuthArchiveReadScope : AuthArchiveWriteScope, + input.type === "archive.status" ? AuthOrchestrationReadScope : AuthOrchestrationOperateScope, + ); + const archive = yield* TranscriptArchive.TranscriptArchive; + return yield* archive + .manage(input) + .pipe( + Effect.flatMap((status) => + HttpServerResponse.json(status, { headers: { "cache-control": "no-store" } }), + ), + ); + }).pipe(archiveRouteErrors), +); + +export const layerArchiveUploadRoute = HttpRouter.add( + "PUT", + "/api/archive/objects/:hash", + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthArchiveWriteScope, AuthOrchestrationOperateScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const params = yield* HttpRouter.params; + const objects = yield* ArchiveObjects.ArchiveObjects; + return yield* objects + .put(params.hash ?? "", request.stream) + .pipe( + Effect.flatMap((receipt) => + HttpServerResponse.json(receipt, { headers: { "cache-control": "no-store" } }), + ), + ); + }).pipe(archiveRouteErrors, Effect.withTracerEnabled(false)), +); + +export const layerArchiveDownloadRoute = HttpRouter.add( + "GET", + "/api/archive/objects/:hash", + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthArchiveReadScope, AuthOrchestrationReadScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const params = yield* HttpRouter.params; + const objects = yield* ArchiveObjects.ArchiveObjects; + const fs = yield* FileSystem.FileSystem; + return yield* Effect.gen(function* () { + const file = yield* objects.file(params.hash ?? ""); + const stat = yield* fs.stat(file); + const range = + request.headers.range && !request.headers["if-range"] + ? assetByteRange(request.headers.range, stat.size) + : null; + if (range?._tag === "Unsatisfiable") + return HttpServerResponse.empty({ + status: 416, + headers: { "content-range": `bytes */${stat.size}` }, + }); + const bounded = range?._tag === "Range" ? range : undefined; + return HttpServerResponse.stream( + fs.stream(file, { + offset: bounded?.offset ?? 0n, + ...(bounded ? { bytesToRead: bounded.bytesToRead } : {}), + }), + { + status: bounded ? 206 : 200, + headers: { + "content-type": "application/octet-stream", + "content-length": String(bounded?.bytesToRead ?? stat.size), + "cache-control": "private, no-store", + "accept-ranges": "bytes", + ...(bounded ? { "content-range": bounded.contentRange } : {}), + }, + }, + ); + }).pipe( + Effect.orElseSucceed(() => + HttpServerResponse.empty({ status: 503, headers: { "cache-control": "no-store" } }), + ), + ); + }).pipe(archiveRouteErrors, Effect.withTracerEnabled(false)), +); + export const layerSharedAccountRoute = HttpRouter.add( "POST", "/api/shared/accounts", @@ -519,6 +647,25 @@ export const layerAssetRoute = HttpRouter.add( headers: { "cache-control": "private, max-age=3600", "x-content-type-options": "nosniff" }, }); } + if (asset.kind === "archived-file") { + return yield* ArchiveAssetResponse.remoteResponse( + asset, + request.headers["if-range"] ? undefined : request.headers.range, + ).pipe( + Effect.flatMap((result) => + result.kind === "response" + ? Effect.succeed(result.response) + : assetFileResponse( + { ...asset, path: result.path }, + request.headers.range, + request.headers["if-range"], + ), + ), + Effect.orElseSucceed(() => + HttpServerResponse.empty({ status: 503, headers: { "cache-control": "no-store" } }), + ), + ); + } if (asset.kind === "github-media") { return yield* githubMediaResponse(asset, request.headers).pipe( Effect.tapError((cause) => diff --git a/apps/server/src/orchestration-v2/ProjectionMaintenance.ts b/apps/server/src/orchestration-v2/ProjectionMaintenance.ts index ccd7e383593310cca3c05df9e0be7f031798c2ab..cbaf83d867b3e6b6204b5fef118d195378743be8 100644 --- a/apps/server/src/orchestration-v2/ProjectionMaintenance.ts +++ b/apps/server/src/orchestration-v2/ProjectionMaintenance.ts @@ -8,6 +8,7 @@ import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as EventStore from "./EventStore.ts"; +import * as TranscriptArchive from "../archive/TranscriptArchive.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; export interface ProjectionVerificationV2 { @@ -171,6 +172,13 @@ export const layer: Layer.Layer< if (page.length < pageSize) break; yield* Effect.yieldNow; } + yield* TranscriptArchive.rebuildMetadata.pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + yield* sql`INSERT INTO orchestration_v2_turn_item_positions(thread_id, turn_item_id, ordinal) + SELECT item.thread_id, item.turn_item_id, item.ordinal FROM orchestration_v2_projection_turn_items item + WHERE EXISTS (SELECT 1 FROM transcript_archives a WHERE a.thread_id = item.thread_id) + ON CONFLICT(thread_id, turn_item_id) DO UPDATE SET ordinal = excluded.ordinal`; const now = DateTime.formatIso(yield* DateTime.now); yield* sql` INSERT INTO orchestration_v2_projection_metadata ( @@ -182,7 +190,7 @@ export const layer: Layer.Layer< VALUES ( 'thread-projections', ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, - ${lastSequence}, + ${throughSequence}, ${now} ) ON CONFLICT(projection_name) DO UPDATE SET @@ -302,7 +310,10 @@ export const layer: Layer.Layer< ) { if (retainedThreadIds.has(row.stream_id)) obsolete.push(row.sequence); else retainedThreadIds.add(row.stream_id); - } else if (row.event_type === "message.updated" || row.event_type === "node.updated") { + } else if ( + row.entity_id !== null && + (row.event_type === "message.updated" || row.event_type === "node.updated") + ) { const key = encodeEntityKey([row.event_type, row.stream_id, row.entity_id]); if (retainedEntityKeys.has(key)) obsolete.push(row.sequence); else retainedEntityKeys.add(key); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ad92e25c8516a51d154706f4f6d97e7f2cbe05e7..1162daf5c6fec7d324e640bfa9e69c1db5a82588 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -3938,6 +3938,13 @@ export const layer: Layer.Layer = payload_json FROM ${sql(`orchestration_v2_projection_${table.name}`)} WHERE rowid > ${afterRowId} + ${ + ["messages", "turn_items", "subagents"].includes(table.name) + ? sql`AND NOT ( + json_extract(payload_json, '$.archivedPayload') = 1 + AND EXISTS (SELECT 1 FROM transcript_archives a WHERE a.thread_id = ${sql(`orchestration_v2_projection_${table.name}`)}.thread_id))` + : sql`` + } ORDER BY rowid ASC LIMIT ${pageSize} `; for (const row of rows) { diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 18ad606d9767864335b98b5a2e0f7f1e79a5dfdd..2089d9ed5cdbe631a377782bf6a8a4b88a02d979 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -27,6 +27,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ArchivedProjectionStore from "../archive/ArchivedProjectionStore.ts"; import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; @@ -72,7 +73,7 @@ const layerCommandReceiptStoreProvided = CommandReceiptStore.layerFromApplicatio const layerStores = Layer.mergeAll( layerEventInfrastructure, layerEventStoreProvided, - ProjectionStore.layer, + ArchivedProjectionStore.layer, ProjectStore.layer, layerCommandReceiptStoreProvided, EffectOutbox.layer, @@ -82,6 +83,7 @@ const layerStores = Layer.mergeAll( export const layerEventSink = EventSink.layerFromStores.pipe(Layer.provide(layerStores)); const layerEventSinkProvided = layerEventSink; const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe( + Layer.provide(ProjectionStore.layer), Layer.provide(layerStores), ); const layerLegacyV1ThreadImporterProvided = LegacyV1ThreadImporter.layer.pipe( @@ -92,7 +94,7 @@ export const layerProjectService = ProjectService.layer.pipe( Layer.provide( Layer.mergeAll( ProjectStore.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, layerEventSinkProvided, IdAllocator.layer, layerLegacyV1ThreadImporterProvided, @@ -105,7 +107,7 @@ const layerProviderEventIngestorProvided = ProviderEventIngestor.layer.pipe( Layer.mergeAll( layerEventSinkProvided, IdAllocator.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, ThreadCommandExecutor.layer, ), ), @@ -131,13 +133,13 @@ const layerProviderSessionManagerProvided = ProviderSessionManager.layer.pipe( layerEventSinkProvided, IdAllocator.layer, layerProviderEventIngestorProvided, - ProjectionStore.layer, + ArchivedProjectionStore.layer, ), ), ); const layerProviderAuthServiceProvided = ProviderAuthService.layer.pipe( - Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)), + Layer.provide(Layer.merge(ArchivedProjectionStore.layer, layerProviderSessionManagerProvided)), ); const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe( @@ -158,7 +160,7 @@ const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pip layerContextHandoffServiceProvided, layerEventSinkProvided, IdAllocator.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, layerProviderSessionManagerProvided, layerProviderAuthServiceProvided, layerRunExecutionServiceProvided, @@ -168,10 +170,10 @@ const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pip ); const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe( - Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)), + Layer.provide(Layer.merge(ArchivedProjectionStore.layer, layerProviderSessionManagerProvided)), ); const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe( - Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)), + Layer.provide(Layer.merge(ArchivedProjectionStore.layer, layerProviderSessionManagerProvided)), ); const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe( Layer.provide( @@ -180,7 +182,7 @@ const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.p layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, layerProviderSessionManagerProvided, ThreadCommandExecutor.layer, layerRuntimePolicyProvided, @@ -193,12 +195,12 @@ const layerCheckpointCaptureServiceProvided = CheckpointCaptureService.layer.pip layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, ), ), ); const layerRunFinalizationServiceProvided = RunFinalizationService.layer.pipe( - Layer.provide(Layer.merge(layerCheckpointCaptureServiceProvided, ProjectionStore.layer)), + Layer.provide(Layer.merge(layerCheckpointCaptureServiceProvided, ArchivedProjectionStore.layer)), ); const layerOrchestratorProvided = Orchestrator.layer.pipe( @@ -320,7 +322,7 @@ const layerProviderRuntimeRecoveryProvided = ProviderRuntimeRecoveryService.laye layerStores, layerEventSinkProvided, IdAllocator.layer, - ProjectionStore.layer, + ArchivedProjectionStore.layer, ), ), ); @@ -357,7 +359,7 @@ export const layerProduction = Layer.mergeAll( layerScheduledTaskProvided, layerSecretRequestsProvided, UsageLimitRecoveryWorker.layer.pipe( - Layer.provide(Layer.mergeAll(ProjectionStore.layer, layerThreadManagementProvided)), + Layer.provide(Layer.mergeAll(ArchivedProjectionStore.layer, layerThreadManagementProvided)), ), layerProviderContinuationWorkerProvided, layerAgentSessionImporterProvided, diff --git a/apps/server/src/persistence/AuthSessions.ts b/apps/server/src/persistence/AuthSessions.ts index ac65c446dcd8d86ddc384352630a83e925b8593f..d31916218c1f2e6370565bf8dbbddd15a25951a7 100644 --- a/apps/server/src/persistence/AuthSessions.ts +++ b/apps/server/src/persistence/AuthSessions.ts @@ -33,6 +33,8 @@ export type AuthSessionClientMetadataRecord = typeof AuthSessionClientMetadataRe export const AuthSessionRecord = Schema.Struct({ sessionId: AuthSessionId, + parentSessionId: Schema.optionalKey(AuthSessionId), + deviceId: Schema.optionalKey(Schema.String), subject: Schema.String, scopes: AuthEnvironmentScopes, method: ServerAuthSessionMethod, @@ -46,6 +48,8 @@ export type AuthSessionRecord = typeof AuthSessionRecord.Type; export const CreateAuthSessionInput = Schema.Struct({ sessionId: AuthSessionId, + parentSessionId: Schema.optionalKey(AuthSessionId), + deviceId: Schema.optionalKey(Schema.String), subject: Schema.String, scopes: AuthEnvironmentScopes, method: ServerAuthSessionMethod, @@ -119,7 +123,7 @@ export class AuthSessionRepository extends Context.Service< ) => Effect.Effect, AuthSessionRepositoryError>; readonly revoke: ( input: RevokeAuthSessionInput, - ) => Effect.Effect; + ) => Effect.Effect, AuthSessionRepositoryError>; readonly revokeAllExcept: ( input: RevokeOtherAuthSessionsInput, ) => Effect.Effect, AuthSessionRepositoryError>; @@ -134,6 +138,8 @@ export class AuthSessionRepository extends Context.Service< const AuthSessionDbRow = Schema.Struct({ sessionId: AuthSessionId, + parentSessionId: Schema.NullOr(AuthSessionId), + deviceId: Schema.NullOr(Schema.String), subject: Schema.String, scopes: Schema.fromJsonString(AuthEnvironmentScopes), method: ServerAuthSessionMethod, @@ -151,6 +157,8 @@ const AuthSessionDbRow = Schema.Struct({ const AuthSessionRawDbRow = Schema.Struct({ sessionId: Schema.String, + parentSessionId: Schema.Unknown, + deviceId: Schema.Unknown, subject: Schema.Unknown, scopes: Schema.Unknown, method: Schema.Unknown, @@ -171,6 +179,8 @@ const decodeAuthSessionDbRow = Schema.decodeUnknownEffect(AuthSessionDbRow); function toAuthSessionRecord(row: typeof AuthSessionDbRow.Type): AuthSessionRecord { return { sessionId: row.sessionId, + ...(row.parentSessionId === null ? {} : { parentSessionId: row.parentSessionId }), + ...(row.deviceId === null ? {} : { deviceId: row.deviceId }), subject: row.subject, scopes: row.scopes, method: row.method, @@ -215,6 +225,8 @@ export const make = Effect.gen(function* () { sql` INSERT INTO auth_sessions ( session_id, + parent_session_id, + device_id, subject, scopes, method, @@ -230,6 +242,8 @@ export const make = Effect.gen(function* () { ) VALUES ( ${input.sessionId}, + ${input.parentSessionId ?? null}, + ${input.deviceId ?? null}, ${input.subject}, ${JSON.stringify(input.scopes)}, ${input.method}, @@ -256,6 +270,8 @@ export const make = Effect.gen(function* () { sql` SELECT session_id AS "sessionId", + parent_session_id AS "parentSessionId", + device_id AS "deviceId", subject AS "subject", scopes AS "scopes", method AS "method", @@ -271,6 +287,10 @@ export const make = Effect.gen(function* () { revoked_at AS "revokedAt" FROM auth_sessions WHERE session_id = ${sessionId} + AND (device_id IS NULL OR EXISTS ( + SELECT 1 FROM auth_devices AS device + WHERE device.device_id = auth_sessions.device_id AND device.revoked_at IS NULL + )) `, }); @@ -300,6 +320,8 @@ export const make = Effect.gen(function* () { sql` SELECT session_id AS "sessionId", + parent_session_id AS "parentSessionId", + device_id AS "deviceId", subject AS "subject", scopes AS "scopes", method AS "method", @@ -315,6 +337,12 @@ export const make = Effect.gen(function* () { revoked_at AS "revokedAt" FROM auth_sessions WHERE revoked_at IS NULL + AND (parent_session_id IS NULL OR EXISTS ( + SELECT 1 FROM auth_sessions AS parent + WHERE parent.session_id = auth_sessions.parent_session_id + AND parent.revoked_at IS NULL + AND parent.expires_at > ${now} + )) AND (expires_at > ${now} OR ${sql.in("session_id", connectedSessionIds)}) ORDER BY issued_at DESC, session_id DESC `, @@ -352,7 +380,7 @@ export const make = Effect.gen(function* () { sql` UPDATE auth_sessions SET revoked_at = ${revokedAt} - WHERE session_id = ${sessionId} + WHERE (session_id = ${sessionId} OR parent_session_id = ${sessionId}) AND revoked_at IS NULL RETURNING session_id AS "sessionId" `, @@ -476,7 +504,7 @@ export const make = Effect.gen(function* () { { sessionId: input.sessionId }, ), ), - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows.map((row) => row.sessionId)), ); const revokeAllExcept: AuthSessionRepository["Service"]["revokeAllExcept"] = (input) => diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 1792046effbfc968022b7ec3c0b8cadd5dbfeb37..309a8a4eda5ce2941f390f5ba48321186001f7af 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -76,6 +76,11 @@ import Migration0059 from "./Migrations/059_McpAppModelContext.ts"; import Migration0060 from "./Migrations/060_ThreadSnapshotWindowIndexes.ts"; import Migration0061 from "./Migrations/061_PullRequestReviewGuides.ts"; import Migration0062 from "./Migrations/062_ThreadFailureLookupIndex.ts"; +import Migration0063 from "./Migrations/063_TranscriptArchives.ts"; +import Migration0064 from "./Migrations/064_AuthSessionDelegation.ts"; +import Migration0065 from "./Migrations/065_HubInbox.ts"; +import Migration0066 from "./Migrations/066_HostedAuthDevices.ts"; +import Migration0067 from "./Migrations/067_ArchiveCatalogAndSync.ts"; /** * Migration loader with all migrations defined inline. @@ -152,6 +157,11 @@ export const migrationEntries = [ [60, "ThreadSnapshotWindowIndexes", Migration0060], [61, "PullRequestReviewGuides", Migration0061], [62, "ThreadFailureLookupIndex", Migration0062], + [63, "TranscriptArchives", Migration0063], + [64, "AuthSessionDelegation", Migration0064], + [65, "HubInbox", Migration0065], + [66, "HostedAuthDevices", Migration0066], + [67, "ArchiveCatalogAndSync", Migration0067], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/063_TranscriptArchives.ts b/apps/server/src/persistence/Migrations/063_TranscriptArchives.ts new file mode 100644 index 0000000000000000000000000000000000000000..f90785ab3d64a59c62f632805cac3986e0908713 --- /dev/null +++ b/apps/server/src/persistence/Migrations/063_TranscriptArchives.ts @@ -0,0 +1,26 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE transcript_archives ( + thread_id TEXT PRIMARY KEY, + location TEXT NOT NULL, + object_hash TEXT NOT NULL, + through_sequence INTEGER NOT NULL, + metadata_json TEXT NOT NULL + )`; + yield* sql`CREATE TABLE transcript_archive_assets ( + thread_id TEXT NOT NULL, + path TEXT NOT NULL, + location TEXT NOT NULL, + object_hash TEXT NOT NULL, + PRIMARY KEY(thread_id, path) + )`; + yield* sql`CREATE INDEX transcript_archive_assets_path_idx ON transcript_archive_assets(path)`; + yield* sql`CREATE TABLE transcript_archive_failures ( + thread_id TEXT PRIMARY KEY, + attempts INTEGER NOT NULL, + reason TEXT NOT NULL + )`; +}); diff --git a/apps/server/src/persistence/Migrations/064_AuthSessionDelegation.ts b/apps/server/src/persistence/Migrations/064_AuthSessionDelegation.ts new file mode 100644 index 0000000000000000000000000000000000000000..0dce19dc09193dcde994fc535ba1299883f26ac8 --- /dev/null +++ b/apps/server/src/persistence/Migrations/064_AuthSessionDelegation.ts @@ -0,0 +1,7 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE auth_sessions ADD COLUMN parent_session_id TEXT`; +}); diff --git a/apps/server/src/persistence/Migrations/065_HubInbox.ts b/apps/server/src/persistence/Migrations/065_HubInbox.ts new file mode 100644 index 0000000000000000000000000000000000000000..0e09a9a911d153493e1fd7ecd66e0180b231f562 --- /dev/null +++ b/apps/server/src/persistence/Migrations/065_HubInbox.ts @@ -0,0 +1,23 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE hub_inbox ( + environment_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL + )`; + yield* sql`CREATE TABLE hub_inbox_pending ( + environment_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + command_id TEXT NOT NULL, + organization_json TEXT NOT NULL, + source_activity_json TEXT NOT NULL, + PRIMARY KEY(environment_id, thread_id) + )`; + yield* sql`CREATE TABLE hub_inbox_commands ( + environment_id TEXT NOT NULL, + command_id TEXT NOT NULL, + PRIMARY KEY(environment_id, command_id) + )`; +}); diff --git a/apps/server/src/persistence/Migrations/066_HostedAuthDevices.ts b/apps/server/src/persistence/Migrations/066_HostedAuthDevices.ts new file mode 100644 index 0000000000000000000000000000000000000000..bc770a5c277dde4dba02411330b4eda78d61dde4 --- /dev/null +++ b/apps/server/src/persistence/Migrations/066_HostedAuthDevices.ts @@ -0,0 +1,26 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE auth_devices ( + device_id TEXT PRIMARY KEY, + account_id TEXT NOT NULL, + environment_id TEXT, + client_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + validated_at INTEGER NOT NULL, + revoked_at INTEGER + )`; + yield* sql`CREATE UNIQUE INDEX auth_devices_environment ON auth_devices(environment_id) WHERE environment_id IS NOT NULL AND revoked_at IS NULL`; + yield* sql`CREATE TABLE auth_device_refresh ( + token_hash TEXT PRIMARY KEY, + device_id TEXT NOT NULL REFERENCES auth_devices(device_id), + expires_at INTEGER NOT NULL, + consumed_at INTEGER + )`; + yield* sql`CREATE INDEX auth_device_refresh_device ON auth_device_refresh(device_id)`; + yield* sql`ALTER TABLE auth_sessions ADD COLUMN device_id TEXT REFERENCES auth_devices(device_id)`; + yield* sql`CREATE INDEX auth_sessions_device ON auth_sessions(device_id)`; +}); diff --git a/apps/server/src/persistence/Migrations/067_ArchiveCatalogAndSync.ts b/apps/server/src/persistence/Migrations/067_ArchiveCatalogAndSync.ts new file mode 100644 index 0000000000000000000000000000000000000000..9f4bd740702e4b2d7995e3e5fc8c74fa4bc2a418 --- /dev/null +++ b/apps/server/src/persistence/Migrations/067_ArchiveCatalogAndSync.ts @@ -0,0 +1,10 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE transcript_archive_failures ADD COLUMN next_attempt_at INTEGER NOT NULL DEFAULT 0`; + yield* sql`CREATE TABLE transcript_archive_publications (thread_id TEXT PRIMARY KEY, location TEXT NOT NULL, object_hash TEXT NOT NULL)`; + yield* sql`CREATE TABLE transcript_archive_reclamation (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), pending INTEGER NOT NULL, completed_at INTEGER, before_allocated_bytes INTEGER, after_allocated_bytes INTEGER)`; + yield* sql`INSERT INTO transcript_archive_reclamation(singleton, pending) VALUES(1, 0)`; +}); diff --git a/apps/server/src/persistence/OrchestrationEventStore.ts b/apps/server/src/persistence/OrchestrationEventStore.ts index 45b44808b4f0d2f6dfc39398d14f3d5ccd10c0e2..a11e9648b5b3abd68bc1a6813453d78a700848ae 100644 --- a/apps/server/src/persistence/OrchestrationEventStore.ts +++ b/apps/server/src/persistence/OrchestrationEventStore.ts @@ -459,6 +459,9 @@ const makeEventStore = Effect.gen(function* () { : sql`INDEXED BY idx_orchestration_events_application_high_water` } WHERE sequence > ${input.afterSequence} + AND NOT (payload_json = 'null' AND EXISTS ( + SELECT 1 FROM transcript_archives a WHERE a.thread_id = orchestration_events.stream_id AND orchestration_events.sequence <= a.through_sequence + )) AND sequence <= ${input.throughSequence ?? Number.MAX_SAFE_INTEGER} AND ( ${ @@ -640,7 +643,7 @@ const makeEventStore = Effect.gen(function* () { SELECT COUNT(*) AS "eventCount", COALESCE(SUM(octet_length(payload_json)), 0) AS "rawPayloadBytes", - COALESCE(MAX(event_type = 'thread.created'), 0) AS "hasCreateEvent" + COALESCE(MAX(event_type = 'thread.created'), 0) * (1 - COALESCE(MAX(payload_json = 'null'), 0)) AS "hasCreateEvent" FROM ( SELECT ${restoredPayloadJson} AS payload_json, event_type FROM orchestration_events @@ -665,7 +668,7 @@ const makeEventStore = Effect.gen(function* () { sql<{ readonly eventCount: number; readonly rawPayloadBytes: number }>` SELECT COUNT(*) AS "eventCount", - COALESCE(SUM(octet_length(${restoredPayloadJson})), 0) AS "rawPayloadBytes" + CASE WHEN MAX(payload_json = 'null') THEN 9007199254740991 ELSE COALESCE(SUM(octet_length(${restoredPayloadJson})), 0) END AS "rawPayloadBytes" FROM orchestration_events INDEXED BY idx_orchestration_events_application_high_water WHERE sequence > ${input.afterSequence} AND sequence <= ${input.throughSequence} diff --git a/apps/server/src/provider/SharedAccountClient.ts b/apps/server/src/provider/SharedAccountClient.ts index e7660723d55f2064bda0a8ca11c2d51f10e2b504..52bd47ab1205cdcb5370cd33a4974860e967ac0e 100644 --- a/apps/server/src/provider/SharedAccountClient.ts +++ b/apps/server/src/provider/SharedAccountClient.ts @@ -19,6 +19,7 @@ import * as Schema from "effect/Schema"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as SharedAccounts from "./SharedAccounts.ts"; +import * as SharedServer from "../environment/SharedServer.ts"; const Connection = Schema.Struct({ url: Schema.String, token: Schema.NonEmptyString }); const CachedAccess = Schema.Struct({ revision: Schema.Int, credentials: SharedAccountAccess }); @@ -28,8 +29,6 @@ const Tokens = Schema.Struct({ id_token: Schema.optionalKey(Schema.NonEmptyString), }); -const decodeConnection = Schema.decodeEffect(Schema.fromJsonString(Connection)); -const encodeConnection = Schema.encodeEffect(Schema.fromJsonString(Connection)); const decodeAccess = Schema.decodeEffect(Schema.fromJsonString(CachedAccess)); const encodeAccess = Schema.encodeEffect(Schema.fromJsonString(CachedAccess)); const decodeCommit = Schema.decodeEffect(Schema.fromJsonString(SharedAccountCommit)); @@ -60,16 +59,8 @@ const make = Effect.gen(function* () { const local = yield* SharedAccounts.SharedAccounts; const locks = yield* KeyedLock.make(); - const connection = secrets.get("shared-server").pipe( - Effect.flatMap((bytes) => - Option.isNone(bytes) - ? Effect.succeedNone - : decodeConnection(new TextDecoder().decode(bytes.value)).pipe(Effect.map(Option.some)), - ), - Effect.mapError( - (cause) => new SharedAccountError({ reason: "storage", cause: Redacted.make(cause) }), - ), - ); + const sharedServer = yield* SharedServer.SharedServer; + const connection = sharedServer.connection; const requestRemote = (target: typeof Connection.Type, input: SharedAccountRequest) => HttpClientRequest.post(`${target.url}/api/shared/accounts`).pipe( @@ -121,18 +112,7 @@ const make = Effect.gen(function* () { }); const target = { ...input, url }; yield* requestRemote(target, { type: "list" }); - const json = yield* encodeConnection(target).pipe( - Effect.mapError( - (cause) => new SharedAccountError({ reason: "storage", cause: Redacted.make(cause) }), - ), - ); - yield* secrets - .set("shared-server", new TextEncoder().encode(json)) - .pipe( - Effect.mapError( - (cause) => new SharedAccountError({ reason: "storage", cause: Redacted.make(cause) }), - ), - ); + yield* sharedServer.save(target); }); const access = Effect.fn("SharedAccountClient.access")(function* ( @@ -309,14 +289,10 @@ const make = Effect.gen(function* () { configure, access: (id, force) => locks.withLock(id, access(id, force)), configured: connection.pipe(Effect.map(Option.isSome)), - disconnect: secrets - .remove("shared-server") - .pipe( - Effect.mapError( - (cause) => new SharedAccountError({ reason: "storage", cause: Redacted.make(cause) }), - ), - ), + disconnect: sharedServer.disconnect, }); }); -export const layer = Layer.effect(SharedAccountClient, make); +export const layer = Layer.effect(SharedAccountClient, make).pipe( + Layer.provide(SharedServer.layer), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 6eb2926b367606badd6025f92017810d0b030069..47782e1e5a6c0977caadf5f31972de4548626397 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,3 +1,21 @@ +import * as TranscriptArchive from "./archive/TranscriptArchive.ts"; +import * as Hub from "./environment/Hub.ts"; +import * as HubProxy from "./environment/HubProxy.ts"; +import * as HubHttp from "./environment/HubHttp.ts"; +import * as HubInbox from "./environment/HubInbox.ts"; +import * as HubInboxHttp from "./environment/HubInboxHttp.ts"; +import * as ArchiveRuntime from "./archive/runtimeLayer.ts"; +import * as ArchiveHttp from "./archive/ArchiveHttp.ts"; +import * as ArchiveAssets from "./archive/ArchiveAssets.ts"; +import * as HostedArchiveConnection from "./archive/HostedArchiveConnection.ts"; +import * as HostedAuth from "./auth/HostedAuth.ts"; +import * as HostedAuthConfig from "./auth/HostedAuthConfig.ts"; +import * as HostedAuthHttp from "./auth/HostedAuthHttp.ts"; +import * as OutboundHub from "./environment/OutboundHub.ts"; +import * as OutboundSource from "./environment/OutboundSource.ts"; +import * as SourceConnection from "./environment/SourceConnection.ts"; +import * as SourceConnectionHttp from "./environment/SourceConnectionHttp.ts"; +import * as SharedServer from "./environment/SharedServer.ts"; import * as SharedAccounts from "./provider/SharedAccounts.ts"; import * as SharedAccountClient from "./provider/SharedAccountClient.ts"; import * as ManagedProviderHome from "./provider/ManagedProviderHome.ts"; @@ -450,6 +468,7 @@ const layerProjectFaviconResolver = ProjectFaviconResolver.layer.pipe( const layerServerEnvironment = ServerEnvironment.layer.pipe(Layer.provide(ServerSecretStore.layer)); const layerAuth = EnvironmentAuth.layer.pipe( + Layer.provideMerge(HostedAuthConfig.layer), Layer.provideMerge(layerPersistence), Layer.provide(layerServerEnvironment), Layer.provide(ServerSecretStore.layer), @@ -553,6 +572,19 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( ); const layerRuntimeCoreDependenciesBase = Layer.mergeAll( + Layer.effectDiscard( + Effect.gen(function* () { + const archive = yield* TranscriptArchive.TranscriptArchive; + yield* ServerActivation.forkParked( + Effect.forever( + archive.sweep.pipe( + Effect.ignore({ log: true }), + Effect.andThen(Effect.sleep("1 minute")), + ), + ), + ); + }), + ), Layer.effectDiscard( Effect.gen(function* () { const profiles = yield* SharedProfiles.SharedProfiles; @@ -647,63 +679,87 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( ), ); -const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe( - Layer.provideMerge(SharedProfiles.layer), - Layer.provideMerge(ManagedProviderHome.layer), - Layer.provideMerge(SharedAccountClient.layer), - Layer.provideMerge(SharedAccounts.layer), - Layer.provideMerge(layerPtyAdapter), - // Search, prepare, status inspection, and turn launch share one registry - // cache so every client and provider instance sees the same prepared agents. - Layer.provideMerge(AcpRegistrySupport.layerFromHost.pipe(Layer.provide(ProviderHostLive.layer))), - // Provider event loggers are shared by every driver instance and the - // orchestration runtime, so they are provided once here. - // `ModelManifest.layer` is the model manifest, refreshed from the repo's - // `model-manifest.json` on `main`. Drivers read their entry through the - // `ModelCatalog` port it provides. - Layer.provideMerge( - Layer.mergeAll( - layerProviderEventLoggers, - ModelManifest.layerModelCatalog.pipe(Layer.provideMerge(ModelManifest.layer)), - ResetCreditCoordinator.layer, - ProviderLatestVersions.layer, - McpProviderSessions.layer, +const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase + .pipe( + Layer.provideMerge( + Layer.mergeAll(HubInbox.layer.pipe(Layer.provide(layerPersistence)), HubProxy.layer), ), - ), - // `OpenCodeDriver.create()` yields `OpenCodeRuntime`; previously the old - // `ProviderRegistry.layer` pulled `OpenCodeRuntimeLive` in for itself, but - // the rewritten registry reads snapshots off the instance registry and - // no longer transitively provides it. Exposing it at the runtime level - // keeps a single Live for all opencode consumers. - Layer.provideMerge( - OpenCodeRuntime.layer.pipe( - Layer.provide( - Layer.unwrap( - Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - return OpenCodeServerLedger.layer({ stateDir: config.stateDir }); - }), + Layer.provideMerge(Hub.layer), + Layer.provideMerge(OutboundHub.layer), + Layer.provideMerge(HostedAuth.layer.pipe(Layer.provide(layerPersistence))), + Layer.provideMerge(HostedAuthConfig.layer), + Layer.provideMerge(SharedProfiles.layer.pipe(Layer.provide(layerServerSettings))), + Layer.provideMerge(ManagedProviderHome.layer), + Layer.provideMerge(SharedAccountClient.layer), + Layer.provideMerge(SharedAccounts.layer), + Layer.provideMerge(ArchiveAssets.layer), + Layer.provideMerge( + ArchiveRuntime.layer.pipe( + Layer.provide(layerPersistence), + Layer.provide(HostedArchiveConnection.layer), + Layer.provide(layerServerSettings), + ), + ), + ) + .pipe( + Layer.provideMerge(SourceConnection.layer), + Layer.provideMerge(SharedServer.layer), + Layer.provideMerge(layerPtyAdapter), + // Search, prepare, status inspection, and turn launch share one registry + // cache so every client and provider instance sees the same prepared agents. + Layer.provideMerge( + AcpRegistrySupport.layerFromHost.pipe(Layer.provide(ProviderHostLive.layer)), + ), + // Provider event loggers are shared by every driver instance and the + // orchestration runtime, so they are provided once here. + // `ModelManifest.layer` is the model manifest, refreshed from the repo's + // `model-manifest.json` on `main`. Drivers read their entry through the + // `ModelCatalog` port it provides. + Layer.provideMerge( + Layer.mergeAll( + layerProviderEventLoggers, + ModelManifest.layerModelCatalog.pipe(Layer.provideMerge(ModelManifest.layer)), + ResetCreditCoordinator.layer, + ProviderLatestVersions.layer, + McpProviderSessions.layer, + ), + ), + // `OpenCodeDriver.create()` yields `OpenCodeRuntime`; previously the old + // `ProviderRegistry.layer` pulled `OpenCodeRuntimeLive` in for itself, but + // the rewritten registry reads snapshots off the instance registry and + // no longer transitively provides it. Exposing it at the runtime level + // keeps a single Live for all opencode consumers. + Layer.provideMerge( + OpenCodeRuntime.layer.pipe( + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return OpenCodeServerLedger.layer({ stateDir: config.stateDir }); + }), + ), ), ), ), - ), - Layer.provideMerge(layerWorkspace), - Layer.provideMerge(ProjectEnrichmentService.layer), - Layer.provideMerge(Layer.mergeAll(NativeAppIconResolver.layer, layerProjectFaviconResolver)), - Layer.provideMerge(layerRepositoryIdentityResolver), - Layer.provideMerge(layerServerEnvironment), - Layer.provideMerge(layerAuth), - Layer.provideMerge(ServerSecretStore.layer), - Layer.provideMerge( - Layer.mergeAll( - CloudCliTokenManager.layer.pipe( - Layer.provide(ServerSecretStore.layer), - Layer.provide(ExternalLauncher.layer), + Layer.provideMerge(layerWorkspace), + Layer.provideMerge(ProjectEnrichmentService.layer), + Layer.provideMerge(Layer.mergeAll(NativeAppIconResolver.layer, layerProjectFaviconResolver)), + Layer.provideMerge(layerRepositoryIdentityResolver), + Layer.provideMerge(layerServerEnvironment), + Layer.provideMerge(layerAuth), + Layer.provideMerge(ServerSecretStore.layer), + ) + .pipe( + Layer.provideMerge( + Layer.mergeAll( + CloudCliTokenManager.layer.pipe( + Layer.provide(ServerSecretStore.layer), + Layer.provide(ExternalLauncher.layer), + ), + layerCloudManagedEndpointRuntime, ), - layerCloudManagedEndpointRuntime, ), - ), -); + ); const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( // Misc. @@ -745,9 +801,17 @@ const layerMakeRoutes = Layer.mergeAll( ServerHttp.layerOtlpTracesProxyRoute, ServerHttp.layerAssetRoute, ServerHttp.layerAttachmentUploadRoute, + ServerHttp.layerArchiveManagementRoute, + ServerHttp.layerArchiveUploadRoute, + ServerHttp.layerArchiveDownloadRoute, ServerHttp.layerSharedAccountRoute, ServerHttp.layerSharedProfilesRoute, DeviceHubProxy.layer, + HubHttp.layer, + HubInboxHttp.layer, + HostedAuthHttp.layer, + SourceConnectionHttp.layer, + ArchiveHttp.layer, ServerBrowserStream.routeLayer, ServerHttp.layerStaticAndDevRoute, Ws.layer, @@ -1129,6 +1193,7 @@ const layerMakeServer = Layer.unwrap( layerTailscaleServe, layerCloudDesiredLinkReconcile, HeapSnapshot.layer, + OutboundSource.layer, ); return layerServerApplication.pipe( diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index a29e5d2a051ecec68380841ddbc5cc082a089692..f59257a59b8dde32d154cdd762ad0ea86f30070d 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -1,6 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Context from "effect/Context"; +import * as ConfigProvider from "effect/ConfigProvider"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; @@ -241,14 +242,26 @@ const cleanupFixture = ( worktreeAfterDays: 8, worktreeOnDelete: true, }, + archiveOffload = false, ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const config = yield* ServerConfig.ServerConfig; + const storedConfig = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(storedConfig.browserArtifactsDir, { recursive: true }); + yield* fs.makeDirectory(storedConfig.logsDir, { recursive: true }); + const config = { + ...storedConfig, + browserArtifactsDir: yield* fs.realPath(storedConfig.browserArtifactsDir), + logsDir: yield* fs.realPath(storedConfig.logsDir), + }; const git = yield* GitVcsDriver.GitVcsDriver; const repo = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-repo-" }); const command = (cwd: string, args: string[]) => - git.execute({ operation: "cleanup-test", cwd, args }); + git.execute({ + operation: "cleanup-test", + cwd, + args: ["-c", "commit.gpgSign=false", "-c", "tag.gpgSign=false", ...args], + }); yield* command(repo, ["init", "-b", "main"]); yield* command(repo, ["config", "user.email", "cleanup@example.test"]); yield* command(repo, ["config", "user.name", "Cleanup test"]); @@ -271,8 +284,18 @@ const cleanupFixture = ( let fullStatusCalls = 0; let snapshotReads = 0; let events: Stream.Stream = Stream.empty; + let pendingArchive = false; const activated = yield* Deferred.make(); - const context = yield* Layer.build(StorageCleanup.layer).pipe( + const context = yield* Layer.build( + StorageCleanup.layer.pipe( + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ T3CODE_ARCHIVE_OFFLOAD: archiveOffload }), + ), + ), + ), + ).pipe( + Effect.provideService(ServerConfig.ServerConfig, config), Effect.provideService(GitVcsDriver.GitVcsDriver, { ...git, execute: (input) => @@ -319,24 +342,26 @@ const cleanupFixture = ( } as unknown as Orchestrator.OrchestratorV2["Service"]), Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) => Effect.sync(() => - openSessionCwd !== null && strings.join("").includes("projection_provider_sessions") - ? [ - { - payload_json: JSON.stringify({ - id: "session-1", - driver: "codex", - providerInstanceId: "codex", - status: "ready", - cwd: openSessionCwd, - model: null, - capabilities: CodexProviderCapabilitiesV2, - createdAt: "2026-06-01T12:00:00.000Z", - updatedAt: "2026-06-01T12:00:00.000Z", - lastError: null, - }), - }, - ] - : [], + pendingArchive && strings.join("").includes("transcript_archives") + ? [{}] + : openSessionCwd !== null && strings.join("").includes("projection_provider_sessions") + ? [ + { + payload_json: JSON.stringify({ + id: "session-1", + driver: "codex", + providerInstanceId: "codex", + status: "ready", + cwd: openSessionCwd, + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: "2026-06-01T12:00:00.000Z", + updatedAt: "2026-06-01T12:00:00.000Z", + lastError: null, + }), + }, + ] + : [], )) as unknown as SqlClient.SqlClient), Effect.provideService(GitManager.GitManager, { invalidateStatus: () => Effect.void, @@ -394,6 +419,9 @@ const cleanupFixture = ( setOpenSessionCwd: (cwd: string | null) => { openSessionCwd = cwd; }, + setPendingArchive: (pending: boolean) => { + pendingArchive = pending; + }, fullStatusCalls: () => fullStatusCalls, snapshotReads: () => snapshotReads, /** Starts the hourly sweep and the subscription to `domainEvents`, which wait until now. */ @@ -464,6 +492,24 @@ const runCleanupTest = ( ) => effect.pipe(Effect.provide(cleanupTestLayer), Effect.scoped); describe("storage cleanup reports and local file policies", () => { + it.live.each([false, true])( + "keeps unverified transcript media before worktree cleanup only when offloading is enabled (%s)", + (enabled) => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(undefined, enabled); + fixture.setPendingArchive(true); + const report = yield* fixture.service.runNow; + expect(report.entries[0]?.outcome).toBe(enabled ? "kept" : "removed"); + if (enabled) { + expect(report.entries[0]?.reason).toContain("Transcript archive is not verified"); + fixture.setPendingArchive(false); + expect((yield* fixture.service.runNow).entries[0]?.outcome).toBe("removed"); + } + }), + ), + ); + it.live("measures nested regular files without following symlinks", () => runCleanupTest( Effect.gen(function* () { diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 627bc410c52bb18b301abce39131689da00c1149..94ccc8cae21363f4e37f53bdc0630e85b41b820f 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -18,6 +18,7 @@ import type { import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Context from "effect/Context"; +import * as Config from "effect/Config"; import * as Deferred from "effect/Deferred"; import * as Layer from "effect/Layer"; import * as Clock from "effect/Clock"; @@ -179,6 +180,10 @@ const make = Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const sql = yield* SqlClient.SqlClient; + const archiveOffload = yield* Config.Boolean("T3CODE_ARCHIVE_OFFLOAD").pipe( + Config.withDefault(false), + Effect.orDie, + ); const git = yield* GitVcsDriver.GitVcsDriver; const gitManager = yield* GitManager.GitManager; const terminals = yield* TerminalManager.TerminalManager; @@ -578,6 +583,11 @@ const make = Effect.gen(function* () { // the removal. Read only this path and this thread: a full snapshot per // worktree takes seconds on large databases. const blockedSinceCheck = Effect.fnUntraced(function* () { + if (archiveOffload) { + const pendingArchive = + yield* sql`SELECT 1 FROM orchestration_v2_projection_threads t WHERE t.archived_at IS NOT NULL AND t.deleted_at IS NULL AND json_extract(t.payload_json, '$.worktreePath') = ${worktreePath} AND NOT EXISTS (SELECT 1 FROM transcript_archives a WHERE a.thread_id = t.thread_id) LIMIT 1`; + if (pendingArchive.length) return "transcript archive is not verified"; + } const latestProjects = yield* projectStore.listShells(); if (yield* containsProjectRoot(worktreePath, [project, ...latestProjects])) return "contains a project checkout"; diff --git a/apps/web/src/AppRoot.tsx b/apps/web/src/AppRoot.tsx index 1ec12cfdd7129ad45e875b3753bcb8a3ec45b921..61be30d5c9ea0a5284fc2a7cd3f4277e175c3da3 100644 --- a/apps/web/src/AppRoot.tsx +++ b/apps/web/src/AppRoot.tsx @@ -2,6 +2,7 @@ import { RouterProvider } from "@tanstack/react-router"; import { BrowserProfileReporter } from "./browser/BrowserProfileReporter"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; +import { CloudAuthentication } from "./cloud/CloudAuthentication"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; import type { AppRouter } from "./router"; @@ -15,6 +16,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { return ( + diff --git a/apps/web/src/cloud/CloudAuthBoundary.test.tsx b/apps/web/src/cloud/CloudAuthBoundary.test.tsx new file mode 100644 index 0000000000000000000000000000000000000000..fd0943b7724f7135405288c5a9de87e776f6bbc4 --- /dev/null +++ b/apps/web/src/cloud/CloudAuthBoundary.test.tsx @@ -0,0 +1,105 @@ +// @vitest-environment jsdom +import { act, useEffect, type ReactNode } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ loads: 0, mounts: 0 })); +vi.mock("../env", () => ({ isElectron: false })); +vi.mock("./publicConfig", () => ({ + hasCloudPublicConfig: () => true, + resolveCloudPublicConfig: () => ({ clerkPublishableKey: "test-key" }), +})); +vi.mock("../components/clerk/BrowserManagedAuthShell", () => { + mocks.loads += 1; + return { + default: ({ children }: { children: ReactNode }) => { + useEffect(() => { + mocks.mounts += 1; + }, []); + return children; + }, + }; +}); + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + localStorage.clear(); + vi.resetModules(); + mocks.loads = 0; + mocks.mounts = 0; + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +it("leaves Clerk dormant for local and self-hosted use and preserves the editor on activation", async () => { + const { CloudAuthBoundary } = await import("./CloudAuthBoundary"); + const { activateCloudAuthentication } = await import("./activation"); + await act(async () => + root.render( + <> + + Use T3 Connect}> +

Cloud account

+ + , + ), + ); + const editor = container.querySelector("input")!; + editor.value = "Unsent draft"; + editor.focus(); + expect(mocks.loads).toBe(0); + expect(mocks.mounts).toBe(0); + expect(container.textContent).toContain("Use T3 Connect"); + await act(async () => { + activateCloudAuthentication(); + await vi.dynamicImportSettled(); + }); + expect(mocks.loads).toBe(1); + expect(mocks.mounts).toBe(1); + expect(container.textContent).toContain("Cloud account"); + expect(container.querySelector("input")).toBe(editor); + expect(editor.value).toBe("Unsent draft"); + expect(document.activeElement).toBe(editor); +}); + +it("restores an explicit Connect opt-in without a new sign-in request", async () => { + localStorage.setItem("knife:t3-connect-enabled", "true"); + const { useCloudAuthentication } = await import("./activation"); + function State() { + const state = useCloudAuthentication(); + return

{JSON.stringify(state)}

; + } + await act(async () => root.render()); + expect(container.textContent).toBe('{"enabled":true,"signInRequested":false}'); +}); + +it("forgetting Connect opt-in returns cloud fragments to fallback without touching the editor", async () => { + const { CloudAuthBoundary } = await import("./CloudAuthBoundary"); + const { activateCloudAuthentication, deactivateCloudAuthentication } = + await import("./activation"); + await act(async () => { + activateCloudAuthentication(); + root.render( + <> + + Sign in

}> +

Account

+ + , + ); + await vi.dynamicImportSettled(); + }); + const editor = container.querySelector("input")!; + await act(async () => deactivateCloudAuthentication()); + expect(localStorage.getItem("knife:t3-connect-enabled")).toBeNull(); + expect(container.textContent).toBe("Sign in"); + expect(container.querySelector("input")).toBe(editor); + expect(editor.value).toBe("unsent"); +}); diff --git a/apps/web/src/cloud/CloudAuthBoundary.tsx b/apps/web/src/cloud/CloudAuthBoundary.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6c441ff7b86ddcf6e1c984f66c287785c6088a99 --- /dev/null +++ b/apps/web/src/cloud/CloudAuthBoundary.tsx @@ -0,0 +1,29 @@ +import { lazy, Suspense, type ReactNode } from "react"; + +import { isElectron } from "../env"; +import { useCloudAuthentication } from "./activation"; +import { hasCloudPublicConfig, resolveCloudPublicConfig } from "./publicConfig"; + +const ManagedAuthShell = lazy(() => + isElectron + ? import("../components/clerk/ElectronManagedAuthShell") + : import("../components/clerk/BrowserManagedAuthShell"), +); + +/** Only cloud fragments gain a provider, so activation preserves the surrounding app. */ +export function CloudAuthBoundary({ + children, + fallback = null, +}: { + readonly children: ReactNode; + readonly fallback?: ReactNode; +}) { + const { enabled } = useCloudAuthentication(); + const { clerkPublishableKey } = resolveCloudPublicConfig(); + if (!enabled || !clerkPublishableKey || !hasCloudPublicConfig()) return fallback; + return ( + + {children} + + ); +} diff --git a/apps/web/src/cloud/CloudAuthentication.tsx b/apps/web/src/cloud/CloudAuthentication.tsx new file mode 100644 index 0000000000000000000000000000000000000000..9e122c0a2adc7ab247f7d4528ba9a873f0d389b6 --- /dev/null +++ b/apps/web/src/cloud/CloudAuthentication.tsx @@ -0,0 +1,40 @@ +import { useAuth, useClerk } from "@clerk/react"; +import { useEffect } from "react"; + +import { isElectron } from "../env"; +import { useEnvironments } from "../state/environments"; +import { resolveClerkSignInProps } from "../components/clerk/authRedirect"; +import { + activateCloudAuthentication, + finishCloudSignInRequest, + useCloudAuthentication, +} from "./activation"; +import { CloudAuthBoundary } from "./CloudAuthBoundary"; +import { ManagedRelayAuthProvider } from "./managedAuth"; + +export function CloudAuthentication() { + const { environments } = useEnvironments(); + const savedRelay = environments.some((environment) => environment.relayManaged); + useEffect(() => { + if (savedRelay) activateCloudAuthentication(); + }, [savedRelay]); + return ( + + + + + + ); +} + +function SignInRequest() { + const { isLoaded, isSignedIn } = useAuth(); + const clerk = useClerk(); + const { signInRequested } = useCloudAuthentication(); + useEffect(() => { + if (!isLoaded || !signInRequested) return; + finishCloudSignInRequest(); + if (!isSignedIn) clerk.openSignIn(resolveClerkSignInProps(window.location.href, isElectron)); + }, [clerk, isLoaded, isSignedIn, signInRequested]); + return null; +} diff --git a/apps/web/src/cloud/activation.ts b/apps/web/src/cloud/activation.ts new file mode 100644 index 0000000000000000000000000000000000000000..f1d14eabcf0c4ca7292dbcee8edc6540c2221a57 --- /dev/null +++ b/apps/web/src/cloud/activation.ts @@ -0,0 +1,55 @@ +import { useSyncExternalStore } from "react"; + +import { hasCloudPublicConfig } from "./publicConfig"; + +const STORAGE_KEY = "knife:t3-connect-enabled"; +const listeners = new Set<() => void>(); + +function restoredOptIn(): boolean { + try { + return globalThis.localStorage?.getItem(STORAGE_KEY) === "true"; + } catch { + return false; + } +} + +let state = { enabled: restoredOptIn(), signInRequested: false }; + +export function activateCloudAuthentication(signInRequested = false): void { + if (!hasCloudPublicConfig()) return; + const requested = state.signInRequested || signInRequested; + if (state.enabled && state.signInRequested === requested) return; + state = { enabled: true, signInRequested: requested }; + try { + localStorage.setItem(STORAGE_KEY, "true"); + } catch { + // The current session can still use Connect when browser storage is blocked. + } + for (const listener of listeners) listener(); +} + +export function deactivateCloudAuthentication(): void { + state = { enabled: false, signInRequested: false }; + try { + localStorage.removeItem(STORAGE_KEY); + } catch { + // Blocked storage does not prevent signing out in this window. + } + for (const listener of listeners) listener(); +} + +export function finishCloudSignInRequest(): void { + if (!state.signInRequested) return; + state = { ...state, signInRequested: false }; + for (const listener of listeners) listener(); +} + +export function useCloudAuthentication() { + return useSyncExternalStore( + (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + () => state, + ); +} diff --git a/apps/web/src/cloud/managedAuth.tsx b/apps/web/src/cloud/managedAuth.tsx index f9bf1c966f40b84321a0aa118d4e81d5437f8882..afa4187d9f24e9a3b06641030db2c94d130e1658 100644 --- a/apps/web/src/cloud/managedAuth.tsx +++ b/apps/web/src/cloud/managedAuth.tsx @@ -12,6 +12,7 @@ import { environmentCatalog } from "../connection/catalog"; import { runtime } from "../lib/runtime"; import { appAtomRegistry } from "../rpc/atomRegistry"; import { useAtomCommand } from "../state/use-atom-command"; +import { deactivateCloudAuthentication } from "./activation"; import { resolveRelayClerkTokenOptions } from "./publicConfig"; export function deactivateManagedRelayAuthentication(): void { @@ -71,6 +72,8 @@ export function ManagedRelayAuthProvider({ children }: { readonly children: Reac if (!isSignedIn || !userId) { deactivateManagedRelayAuthentication(); + if (previousAccount !== undefined && previousAccount !== null) + deactivateCloudAuthentication(); if (previousAccount !== null) { void queueAccountCleanup(); } diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 76ec21e6040b535fa942091af4b635213592ae68..8138e9e58074d2759e0c8673fc825d4a773354f7 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -413,9 +413,10 @@ import { useEnvironmentDisconnectDelay } from "../hooks/useEnvironmentDisconnect import { selectThreadTerminalUiState, useTerminalUiStateStore } from "../terminalUiStateStore"; import { useKnownTerminalSessions, useThreadRunningTerminalIds } from "../state/terminalSessions"; import { useEnvironmentQuery } from "../state/query"; -import { useEnvironmentScope } from "~/state/session"; +import { useEnvironmentScope, useThreadOrganizationPermission } from "~/state/session"; import { environmentServerConfigsAtom, + environmentCapabilitiesAtom, primaryServerAvailableEditorsAtom, primaryServerKeybindingsAtom, serverEnvironment, @@ -583,7 +584,11 @@ import { clampFileAttachmentUploadBytes } from "@t3tools/client-runtime/state/at import { appAtomRegistry } from "../rpc/atomRegistry"; import { fileAttachmentCapabilityBlockReason } from "./chat/composerAttachmentFiles"; import { assetEnvironment } from "../state/assets"; -import { readEnvironmentScope, readPreparedConnection } from "../state/session"; +import { + readEnvironmentScope, + readPreparedConnection, + readThreadOrganizationPermission, +} from "../state/session"; import { useAtomCommand } from "../state/use-atom-command"; import { useOrchestrationCommand } from "../state/use-orchestration-command"; import { useAtomQueryRunner } from "../state/use-atom-query-runner"; @@ -1591,6 +1596,8 @@ export default function ChatView(props: ChatViewProps) { forceExpandedMobileComposer = false, } = props; const canOperateThread = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const canOrganizeThread = useThreadOrganizationPermission(environmentId); + const organizationCapabilities = useAtomValue(environmentCapabilitiesAtom).get(environmentId); const canOperateTerminal = useEnvironmentScope(environmentId, AuthTerminalOperateScope); const hasTerminalWriteAccess = useCallback( () => readEnvironmentScope(environmentId, AuthTerminalOperateScope), @@ -7205,9 +7212,9 @@ export default function ChatView(props: ChatViewProps) { useRightPanelStore.getState().openPullRequest(activeThreadRef, pullRequestPanelTarget); }, [activeThreadRef, pullRequestPanelTarget, supportsPullRequests]); const pullRequestSurfaceAvailable = supportsPullRequests && pullRequestPanelTarget !== null; - const supportsSettlement = serverConfig?.environment.capabilities.threadSettlement === true; - const supportsSnooze = serverConfig?.environment.capabilities.threadSnooze === true; - const supportsPinning = serverConfig?.environment.capabilities.threadPinning === true; + const supportsSettlement = organizationCapabilities?.threadSettlement === true; + const supportsSnooze = organizationCapabilities?.threadSnooze === true; + const supportsPinning = organizationCapabilities?.threadPinning === true; const activeThreadPinned = supportsPinning && activeThreadShell?.pinnedAt != null; const activeThreadSnoozed = activeThreadShell !== null && @@ -7271,6 +7278,7 @@ export default function ChatView(props: ChatViewProps) { const activeThreadSettled = supportsSettlement && activeThreadShell?.settledOverride === "settled"; const unsettleThreadMutation = useOrchestrationCommand(threadEnvironment.unsettle, { + organization: true, reportFailure: false, }); // Keyed by thread, not a boolean: the pending state must follow the thread @@ -7279,10 +7287,7 @@ export default function ChatView(props: ChatViewProps) { const [unsettlingThreadKey, setUnsettlingThreadKey] = useState(null); const isUnsettling = unsettlingThreadKey !== null && unsettlingThreadKey === activeThreadKey; const handleUnsettleActiveThread = useCallback(async () => { - if ( - !activeThreadRef || - !readEnvironmentScope(activeThreadRef.environmentId, AuthOrchestrationOperateScope) - ) + if (!activeThreadRef || !readThreadOrganizationPermission(activeThreadRef.environmentId)) return; const threadKey = scopedThreadKey(activeThreadRef); setUnsettlingThreadKey(threadKey); @@ -7306,15 +7311,13 @@ export default function ChatView(props: ChatViewProps) { } }, [activeThreadRef, unsettleThreadMutation]); const unsnoozeThreadMutation = useOrchestrationCommand(threadEnvironment.unsnooze, { + organization: true, reportFailure: false, }); const [unsnoozingThreadKey, setUnsnoozingThreadKey] = useState(null); const isUnsnoozing = unsnoozingThreadKey !== null && unsnoozingThreadKey === activeThreadKey; const handleUnsnoozeActiveThread = useCallback(async () => { - if ( - !activeThreadRef || - !readEnvironmentScope(activeThreadRef.environmentId, AuthOrchestrationOperateScope) - ) + if (!activeThreadRef || !readThreadOrganizationPermission(activeThreadRef.environmentId)) return; const threadKey = scopedThreadKey(activeThreadRef); setUnsnoozingThreadKey(threadKey); @@ -7747,7 +7750,7 @@ export default function ChatView(props: ChatViewProps) { : "Snoozed" } actionLabel={isUnsnoozing ? "Waking..." : "Wake now"} - actionDisabled={!canOperateThread || isUnsnoozing} + actionDisabled={!canOrganizeThread || isUnsnoozing} onAction={() => void handleUnsnoozeActiveThread()} /> ) : activeThreadSettled ? ( @@ -7759,7 +7762,7 @@ export default function ChatView(props: ChatViewProps) { : "Settled" } actionLabel={isUnsettling ? "Un-settling..." : "Un-settle"} - actionDisabled={!canOperateThread || isUnsettling} + actionDisabled={!canOrganizeThread || isUnsettling} onAction={() => void handleUnsettleActiveThread()} /> ) : activeThreadWokeVisible ? ( @@ -7777,7 +7780,7 @@ export default function ChatView(props: ChatViewProps) { activeThreadShell?.snoozedUntil, activeThreadSnoozed, activeThreadWokeVisible, - canOperateThread, + canOrganizeThread, handleUnsettleActiveThread, handleUnsnoozeActiveThread, isUnsettling, @@ -8149,7 +8152,7 @@ export default function ChatView(props: ChatViewProps) { } if (command === "thread.settle") { - if (!readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; + if (!readThreadOrganizationPermission(environmentId)) return; event.preventDefault(); event.stopPropagation(); if (!isServerThread || !activeThreadRef || !supportsSettlement) return; @@ -8175,7 +8178,7 @@ export default function ChatView(props: ChatViewProps) { } if (command === "thread.pin") { - if (!readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; + if (!readThreadOrganizationPermission(environmentId)) return; event.preventDefault(); event.stopPropagation(); if (!isServerThread || !activeThreadRef || !supportsPinning) return; diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 246a075bf93f509c91cebae5f3f66ea47846e0d6..50f4213691bccec74221ae62b3360b89e76e603d 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -164,14 +164,23 @@ import { useProjects, useThreadShells, } from "../state/entities"; -import { environmentServerConfigsAtom, primaryServerKeybindingsAtom } from "../state/server"; +import { + environmentCapabilitiesAtom, + environmentServerConfigsAtom, + primaryServerKeybindingsAtom, +} from "../state/server"; import { environmentProjects } from "../state/projects"; import { vcsEnvironment } from "../state/vcs"; import { threadEnvironment } from "../state/threads"; import { useEnvironmentQuery } from "../state/query"; import { useThreadSearch } from "../state/queries"; import { useOrchestrationCommand } from "../state/use-orchestration-command"; -import { readEnvironmentScope, useEnvironmentScope } from "../state/session"; +import { + readEnvironmentScope, + useEnvironmentScope, + readThreadOrganizationPermission, + useThreadOrganizationPermission, +} from "../state/session"; import { buildThreadRouteParams, resolveActiveThreadRouteRef, @@ -188,6 +197,7 @@ import { buildDraftActionMenuItems, buildThreadActionMenuItems, threadActionRequiresOperate, + threadActionIsOrganization, } from "./threadActionMenu.logic"; import { animateSidebarLayoutChanges, @@ -315,16 +325,20 @@ const inboxReturns = createInboxReturnTracker(); function canOperateThreads( threads: ReadonlyArray>, + organization = false, ): boolean { return threads.every((thread) => - readEnvironmentScope(thread.environmentId, AuthOrchestrationOperateScope), + organization + ? readThreadOrganizationPermission(thread.environmentId) + : readEnvironmentScope(thread.environmentId, AuthOrchestrationOperateScope), ); } function checkThreadOperations( threads: ReadonlyArray>, + organization = false, ): boolean { - if (canOperateThreads(threads)) return true; + if (canOperateThreads(threads, organization)) return true; toastManager.add({ type: "error", title: "Thread action unavailable", @@ -659,10 +673,10 @@ function SortableThreadRow(props: { environmentId: EnvironmentId; children: (bag: SortableThreadRowBag) => ReactNode; }) { - const canOperateThread = useEnvironmentScope(props.environmentId, AuthOrchestrationOperateScope); + const canOrganizeThread = useThreadOrganizationPermission(props.environmentId); const { listeners, setNodeRef, transform, transition, isDragging } = useSortable({ id: props.id, - disabled: { draggable: props.disabled || !canOperateThread }, + disabled: { draggable: props.disabled || !canOrganizeThread }, animateLayoutChanges: animateSidebarLayoutChanges, }); // dnd-kit memoizes each field but not the bag, so the memoized row would @@ -1214,6 +1228,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ); const threadKey = scopedThreadKey(threadRef); const canOperateThread = useEnvironmentScope(thread.environmentId, AuthOrchestrationOperateScope); + const canOrganizeThread = useThreadOrganizationPermission(thread.environmentId); useEffect(() => { if (!canOperateThread && isRenaming) onCancelRename(); }, [canOperateThread, isRenaming, onCancelRename]); @@ -1549,11 +1564,11 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // Snooze is offered only where it can succeed: capability-gated and never // on blocked-on-you work or queued turns (the server rejects both). const showSnoozeButton = - canOperateThread && + canOrganizeThread && props.snoozeSupported && canSnooze(thread, { now: new Date().toISOString() }); const showHoverActions = - (canOperateThread && props.settlementSupported) || showSnoozeButton || hasUnsentDraft; + (canOrganizeThread && props.settlementSupported) || showSnoozeButton || hasUnsentDraft; // If the thread becomes blocked while the popover is open, the button // unmounts without firing onOpenChange(false). Deriving the flag keeps a // stale true from permanently hiding the status label / pinning the @@ -1757,7 +1772,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { const showPin = props.isPinned && (!sortable?.isDragging || (props.dragOverPinned && props.dropVerb === null)); const pinIndicator = showPin ? ( - props.pinningSupported && canOperateThread && !sortable?.isDragging ? ( + props.pinningSupported && canOrganizeThread && !sortable?.isDragging ? ( s.clearSelection); const setSelectionAnchor = useThreadSelectionStore((s) => s.setAnchor); @@ -2833,7 +2849,7 @@ export default function Sidebar() { const draggable = new Set(); const activeReorderable = new Set(); for (const thread of visible) { - const capabilities = serverConfigs.get(thread.environmentId)?.environment.capabilities; + const capabilities = environmentCapabilities.get(thread.environmentId); // Threads on servers without the settlement capability (old server, // or descriptor not loaded yet) never classify as settled: the user // could neither un-settle nor pin them, so auto-settling them would @@ -2923,7 +2939,7 @@ export default function Sidebar() { nowMinute, optimisticDrop, scopedProjectKeys, - serverConfigs, + environmentCapabilities, snoozeWakeTick, threads, workingShelfEnabled, @@ -3608,8 +3624,7 @@ export default function Sidebar() { const originSection = sectionByThreadKeyRef.current.get(originKey); const canApply = (key: string) => { const thread = threadByKeyRef.current.get(key); - const capabilities = - thread && serverConfigs.get(thread.environmentId)?.environment.capabilities; + const capabilities = thread && environmentCapabilities.get(thread.environmentId); return ( thread !== undefined && sectionByThreadKeyRef.current.get(key) === originSection && @@ -3661,7 +3676,7 @@ export default function Sidebar() { onAbort: () => {}, }); }, - [attemptUnsettle, attemptUnsnooze, serverConfigs, settleThreads], + [attemptUnsettle, attemptUnsnooze, environmentCapabilities, settleThreads], ); const pinnedKeys = useMemo( () => @@ -4004,8 +4019,7 @@ export default function Sidebar() { activePinned: source.pinnedAt != null, activeSettled: source.settledOverride === "settled", supportsSettlement: - serverConfigs.get(source.environmentId)?.environment.capabilities.threadSettlement === - true, + environmentCapabilities.get(source.environmentId)?.threadSettlement === true, target, pinnedOrder: pinnedKeys, pinnedKeysById, @@ -4025,7 +4039,7 @@ export default function Sidebar() { }, [ activeKeysById, pinnedKeysById, - serverConfigs, + environmentCapabilities, activeKeys, activeReorderableThreadKeys, draggedThreadKey, @@ -4054,8 +4068,7 @@ export default function Sidebar() { activePinned: activeThread.pinnedAt != null, activeSettled: activeThread.settledOverride === "settled", supportsSettlement: - serverConfigs.get(activeThread.environmentId)?.environment.capabilities - .threadSettlement === true, + environmentCapabilities.get(activeThread.environmentId)?.threadSettlement === true, target, pinnedOrder: pinnedKeys, pinnedKeysById, @@ -4084,6 +4097,7 @@ export default function Sidebar() { const thread = threadByKey.get(key); return thread ? [thread] : []; }), + true, ) ) return; @@ -4192,7 +4206,7 @@ export default function Sidebar() { [ activeKeysById, pinnedKeysById, - serverConfigs, + environmentCapabilities, activeKeys, activeReorderableThreadKeys, draggableThreadKeys, @@ -4306,13 +4320,12 @@ export default function Sidebar() { ); const canSnoozeSelection = selectedThreads.every( (thread) => - serverConfigs.get(thread.environmentId)?.environment.capabilities.threadSnooze === true && + environmentCapabilities.get(thread.environmentId)?.threadSnooze === true && canSnooze(thread, { now: selectionNow.toISOString() }), ); const titleRegenerationThreads = selectedThreads.filter( (thread) => - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadTitleRegeneration === true, + environmentCapabilities.get(thread.environmentId)?.threadTitleRegeneration === true, ); const regeneratableTitleThreads = titleRegenerationThreads.filter( (thread) => thread.titleRegeneration == null, @@ -4326,8 +4339,8 @@ export default function Sidebar() { // is omitted entirely when nothing selected is pinned. const pinnedSelectedThreads = selectedThreads.filter( (thread) => - serverConfigs.get(thread.environmentId)?.environment.capabilities.threadPinning === - true && thread.pinnedAt != null, + environmentCapabilities.get(thread.environmentId)?.threadPinning === true && + thread.pinnedAt != null, ); const unpinMenuItem = buildBulkUnpinContextMenuItem({ pinnedCount: pinnedSelectedThreads.length, @@ -4337,30 +4350,30 @@ export default function Sidebar() { api.contextMenu.show( [ ...(unpinMenuItem - ? [{ ...unpinMenuItem, disabled: !canOperateThreads(pinnedSelectedThreads) }] + ? [{ ...unpinMenuItem, disabled: !canOperateThreads(pinnedSelectedThreads, true) }] : []), { id: "settle", label: `Settle (${count})`, - disabled: settlingThreads.length === 0 || !canOperateThreads(settlingThreads), + disabled: settlingThreads.length === 0 || !canOperateThreads(settlingThreads, true), }, ...(canSnoozeSelection ? [ { id: "snooze", label: `Snooze (${count})`, - disabled: !canOperateThreads(selectedThreads), + disabled: !canOperateThreads(selectedThreads, true), children: [ ...snoozePresets.map((preset) => ({ id: `snooze:${preset.id}`, label: `${preset.label} (${preset.whenLabel})`, - disabled: !canOperateThreads(selectedThreads), + disabled: !canOperateThreads(selectedThreads, true), })), { id: "snooze:custom", label: "Custom…", separatorBefore: true, - disabled: !canOperateThreads(selectedThreads), + disabled: !canOperateThreads(selectedThreads, true), }, ], }, @@ -4402,7 +4415,16 @@ export default function Sidebar() { }) : selectedThreads; if (clicked.value === "settle" && actionTargets.length === 0) return; - if (clicked.value !== "mark-unread" && !checkThreadOperations(actionTargets)) return; + if ( + clicked.value !== "mark-unread" && + !checkThreadOperations( + actionTargets, + clicked.value === "settle" || + clicked.value === "unpin" || + clicked.value.startsWith("snooze:"), + ) + ) + return; if (clicked.value?.startsWith("snooze:")) { const preset = clicked.value === "snooze:custom" @@ -4548,7 +4570,7 @@ export default function Sidebar() { markThreadUnread, performSnooze, removeFromSelection, - serverConfigs, + environmentCapabilities, settleThreads, updateThreadMetadata, timestampFormat, @@ -4629,18 +4651,15 @@ export default function Sidebar() { // Environments without // the settlement capability get no lifecycle items at all. const supportsSettlement = - serverConfigs.get(thread.environmentId)?.environment.capabilities.threadSettlement === - true; + environmentCapabilities.get(thread.environmentId)?.threadSettlement === true; const supportsSnooze = - serverConfigs.get(thread.environmentId)?.environment.capabilities.threadSnooze === true; + environmentCapabilities.get(thread.environmentId)?.threadSnooze === true; const supportsPinning = - serverConfigs.get(thread.environmentId)?.environment.capabilities.threadPinning === true; + environmentCapabilities.get(thread.environmentId)?.threadPinning === true; const supportsAutoSettleOptOut = - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadAutoSettleOptOut === true; + environmentCapabilities.get(thread.environmentId)?.threadAutoSettleOptOut === true; const supportsTitleRegeneration = - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadTitleRegeneration === true; + environmentCapabilities.get(thread.environmentId)?.threadTitleRegeneration === true; const isRegeneratingTitle = thread.titleRegeneration != null; const isSettled = settledThreadKeysRef.current.has(threadKey); const isSnoozed = snoozedThreadKeysRef.current.has(threadKey); @@ -4658,6 +4677,7 @@ export default function Sidebar() { const clicked = await settlePromise(() => api.contextMenu.show( buildThreadActionMenuItems({ + canOrganize: readThreadOrganizationPermission(threadRef.environmentId), canOperate: readEnvironmentScope( threadRef.environmentId, AuthOrchestrationOperateScope, @@ -4689,7 +4709,11 @@ export default function Sidebar() { ), ); if (clicked._tag === "Failure" || clicked.value === null) return; - if (threadActionRequiresOperate(clicked.value) && !checkThreadOperations([thread])) return; + if ( + threadActionRequiresOperate(clicked.value) && + !checkThreadOperations([thread], threadActionIsOrganization(clicked.value)) + ) + return; if (clicked.value?.startsWith("snooze:")) { const preset = clicked.value === "snooze:custom" @@ -4897,7 +4921,7 @@ export default function Sidebar() { openProjectSettings, projectScopeKey, projectByKey, - serverConfigs, + environmentCapabilities, setProjectScopeKey, setThreadAutoSettle, startThreadRename, @@ -5329,16 +5353,16 @@ export default function Sidebar() { : "settle" } settlementSupported={ - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadSettlement === true + environmentCapabilities.get(thread.environmentId) + ?.threadSettlement === true } snoozeSupported={ - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadSnooze === true + environmentCapabilities.get(thread.environmentId)?.threadSnooze === + true } pinningSupported={ - serverConfigs.get(thread.environmentId)?.environment.capabilities - .threadPinning === true + environmentCapabilities.get(thread.environmentId)?.threadPinning === + true } isPinned={thread.pinnedAt != null} sortable={sortable} diff --git a/apps/web/src/components/auth/PairingRouteSurface.tsx b/apps/web/src/components/auth/PairingRouteSurface.tsx index 4eac3c2b70a107b9179d48ab552d9d4cb270387e..97262d225417d08373b0e0d7019750396f419f2b 100644 --- a/apps/web/src/components/auth/PairingRouteSurface.tsx +++ b/apps/web/src/components/auth/PairingRouteSurface.tsx @@ -1,3 +1,4 @@ +import { beginHostedSignIn } from "../../connection/hostedAuth"; import { Alert, AlertDescription } from "../ui/alert"; import type { AuthSessionState } from "@t3tools/contracts"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; @@ -28,7 +29,63 @@ export function PairingPendingSurface() { ); } -export function PairingRouteSurface({ +export function PairingRouteSurface(props: Parameters[0]) { + const [enabled, setEnabled] = useState(null); + const [error, setError] = useState(() => + new URLSearchParams(window.location.search).has("signInError") + ? "Sign-in did not finish. Try again." + : null, + ); + const attemptedSignIn = useRef(false); + useEffect(() => { + let stopped = false; + void fetch("/api/auth/oidc/config", { credentials: "same-origin", redirect: "error" }) + .then(async (response) => response.ok && (await response.json()).enabled === true) + .then((value) => { + if (!stopped) setEnabled(value); + }) + .catch(() => { + if (!stopped) setEnabled(false); + }); + return () => { + stopped = true; + }; + }, []); + useEffect(() => { + if (!enabled || error || attemptedSignIn.current || peekPairingTokenFromUrl()) return; + attemptedSignIn.current = true; + void beginHostedSignIn(window.location.origin).catch(() => + setError("Couldn't start sign-in. Try again."), + ); + }, [enabled, error]); + if (enabled === false || peekPairingTokenFromUrl()) + return ; + return ( + + + {enabled === null ? ( +

Connecting…

+ ) : ( + + )} + {error ?

{error}

: null} + + ); +} + +function LegacyPairingRouteSurface({ auth, initialErrorMessage, onAuthenticated, diff --git a/apps/web/src/components/clerk/BrowserManagedAuthShell.tsx b/apps/web/src/components/clerk/BrowserManagedAuthShell.tsx index 285e9f7c760b25ef0803dd482074508f4bdc8c28..6534037072bfb4d4b00fbe6febb07c0aebc6b9af 100644 --- a/apps/web/src/components/clerk/BrowserManagedAuthShell.tsx +++ b/apps/web/src/components/clerk/BrowserManagedAuthShell.tsx @@ -1,15 +1,8 @@ import { ClerkProvider } from "@clerk/react"; import type { ReactNode } from "react"; -import { ManagedRelayAuthProvider } from "../../cloud/managedAuth"; import { clerkAppearance } from "./clerkAppearance"; -/** - * Browser half of the managed-auth boundary, loaded lazily from the entry so - * cloudless local mode never downloads a Clerk runtime. The browser provider - * stays small on its own: it hotloads clerk-js at runtime instead of bundling - * it. - */ export default function BrowserManagedAuthShell({ publishableKey, children, @@ -19,7 +12,7 @@ export default function BrowserManagedAuthShell({ }) { return ( - {children} + {children} ); } diff --git a/apps/web/src/components/clerk/ElectronManagedAuthShell.tsx b/apps/web/src/components/clerk/ElectronManagedAuthShell.tsx index 3030c2a395589cb1fc692397dc29e756cc25c643..71e69401ea1e8de4be932ecf3eb3f6aeb299b90d 100644 --- a/apps/web/src/components/clerk/ElectronManagedAuthShell.tsx +++ b/apps/web/src/components/clerk/ElectronManagedAuthShell.tsx @@ -2,15 +2,9 @@ import { passkeys } from "@clerk/electron/passkeys"; import { ClerkProvider } from "@clerk/electron/react"; import type { ReactNode } from "react"; -import { ManagedRelayAuthProvider } from "../../cloud/managedAuth"; import { clerkAppearance } from "./clerkAppearance"; -/** - * Electron half of the managed-auth boundary. The Electron provider statically - * bundles the full clerk-js runtime, so this module must only ever load - * lazily, and only inside the desktop shell — importing it eagerly would put - * clerk-js back into every client's startup graph. - */ +/** Loaded after Connect activation to keep Clerk’s bundled runtime out of startup. */ export default function ElectronManagedAuthShell({ publishableKey, children, @@ -20,7 +14,7 @@ export default function ElectronManagedAuthShell({ }) { return ( - {children} + {children} ); } diff --git a/apps/web/src/components/clerk/T3ConnectSidebarSignIn.tsx b/apps/web/src/components/clerk/T3ConnectSidebarSignIn.tsx index e75ce33ceaf327b32e7e1db3610766def459feb5..2df3c37c152811030d7ffa62a5c245a4c6470c14 100644 --- a/apps/web/src/components/clerk/T3ConnectSidebarSignIn.tsx +++ b/apps/web/src/components/clerk/T3ConnectSidebarSignIn.tsx @@ -1,3 +1,5 @@ +import { CloudAuthBoundary } from "../../cloud/CloudAuthBoundary"; +import { activateCloudAuthentication } from "../../cloud/activation"; import { UserButton, useAuth } from "@clerk/react"; import { LogInIcon } from "lucide-react"; @@ -9,13 +11,32 @@ import { useT3ConnectAuthPrompt } from "./useT3ConnectAuthPrompt"; export function T3ConnectSidebarSignIn() { if (!hasCloudPublicConfig()) return null; - return ; + return ( + + + activateCloudAuthentication(true)}> + + Sign in to T3 Connect + + + + } + > + + + ); } export function T3ConnectSidebarAvatar() { if (!hasCloudPublicConfig()) return null; - return ; + return ( + + + + ); } function ConfiguredT3ConnectSidebarAvatar() { diff --git a/apps/web/src/components/cloud/ConnectCliAuthSurface.tsx b/apps/web/src/components/cloud/ConnectCliAuthSurface.tsx index 4bae618a81e08852c8d8b5e0ce82d0f7b3339bdc..052d7fdd5ea5e58a9a2ecfa58c445e79a5272c33 100644 --- a/apps/web/src/components/cloud/ConnectCliAuthSurface.tsx +++ b/apps/web/src/components/cloud/ConnectCliAuthSurface.tsx @@ -1,3 +1,5 @@ +import { CloudAuthBoundary } from "../../cloud/CloudAuthBoundary"; +import { activateCloudAuthentication } from "../../cloud/activation"; import { useAuth, useClerk } from "@clerk/react"; import { readConnectAuthorizeRequest } from "@t3tools/shared/connectAuth"; import { useCallback, useEffect, useRef, useState } from "react"; @@ -44,7 +46,7 @@ const invalidLinkMessage = { * with the loopback redirect URI so the code returns straight to the waiting * CLI. Headless hosts use Clerk's device authorization page instead. */ -export function ConnectCliAuthorizeSurface() { +function ConfiguredConnectCliAuthorizeSurface() { const [request] = useState(() => readConnectAuthorizeRequest(new URL(window.location.href))); const clerk = useClerk(); const { isLoaded, isSignedIn } = useAuth(); @@ -111,3 +113,18 @@ export function ConnectCliAuthorizeSurface() { ); } + +export function ConnectCliAuthorizeSurface() { + useEffect(() => activateCloudAuthentication(), []); + return ( + +

Loading sign-in…

+ + } + > + + + ); +} diff --git a/apps/web/src/components/cloud/ConnectOnboardingDialog.tsx b/apps/web/src/components/cloud/ConnectOnboardingDialog.tsx index f793a127f43bc6197c65c00c2f626bf9429bbb8f..93ba8ba4322e229740bd18cc75527c56b94b598a 100644 --- a/apps/web/src/components/cloud/ConnectOnboardingDialog.tsx +++ b/apps/web/src/components/cloud/ConnectOnboardingDialog.tsx @@ -1,11 +1,7 @@ +import { CloudAuthBoundary } from "../../cloud/CloudAuthBoundary"; import { useAuth } from "@clerk/react"; import { useAtomValue } from "@effect/atom-react"; -import { - AuthAdministrativeScopes, - AuthRelayReadScope, - AuthRelayWriteScope, - type AuthSessionState, -} from "@t3tools/contracts"; +import { AuthRelayReadScope, AuthRelayWriteScope, type AuthSessionState } from "@t3tools/contracts"; import { AsyncResult, Atom } from "effect/reactivity"; import { useEffect, useRef, useState } from "react"; @@ -43,7 +39,11 @@ import { WizardSteps, WizardPopup, WizardHeader, WizardPanel, WizardFooter } fro export function ConnectOnboardingDialog() { if (!hasCloudPublicConfig()) return null; - return ; + return ( + + + + ); } type OnboardingStep = "publish" | "devices"; diff --git a/apps/web/src/components/onboarding/WelcomeWizard.import.test.tsx b/apps/web/src/components/onboarding/WelcomeWizard.import.test.tsx index 7c4497e63eea321cff5973b9f52f9ed81978cb0b..1e0a9a2395a0d4913b6bd09d742c2c5baac33e48 100644 --- a/apps/web/src/components/onboarding/WelcomeWizard.import.test.tsx +++ b/apps/web/src/components/onboarding/WelcomeWizard.import.test.tsx @@ -36,6 +36,7 @@ const state = vi.hoisted(() => ({ refreshScan: vi.fn(), })); +vi.mock("../settings/SelfHostedServer", () => ({ RemoteConnectionFlow: () => null })); vi.mock("../../connection/runtime", () => ({ connectionAtomRuntime: undefined })); vi.mock("@t3tools/client-runtime/state/session", () => ({ createEnvironmentSessionAtoms: () => ({ diff --git a/apps/web/src/components/onboarding/WelcomeWizard.terminal.test.tsx b/apps/web/src/components/onboarding/WelcomeWizard.terminal.test.tsx index 56cd702411a09ab0fb989e7ee12bb90e0deaaeef..da35565bdd1a4cedce05033c8563582edde07ca8 100644 --- a/apps/web/src/components/onboarding/WelcomeWizard.terminal.test.tsx +++ b/apps/web/src/components/onboarding/WelcomeWizard.terminal.test.tsx @@ -49,6 +49,7 @@ const state = vi.hoisted(() => ({ done: vi.fn(), })); +vi.mock("../settings/SelfHostedServer", () => ({ RemoteConnectionFlow: () => null })); vi.mock("../../connection/runtime", () => ({ connectionAtomRuntime: undefined })); vi.mock("@t3tools/client-runtime/state/session", () => ({ createEnvironmentSessionAtoms: () => ({ diff --git a/apps/web/src/components/onboarding/WelcomeWizard.test.tsx b/apps/web/src/components/onboarding/WelcomeWizard.test.tsx index 439a2cc579de769fd063f1a2a81cb613c00f5fd7..ca96138c1de89994de7c2c3355808874805dd197 100644 --- a/apps/web/src/components/onboarding/WelcomeWizard.test.tsx +++ b/apps/web/src/components/onboarding/WelcomeWizard.test.tsx @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ toast: vi.fn(), projects: [] as Array<{ id: string; environmentId: string; workspaceRoot: string }>, })); +vi.mock("../settings/SelfHostedServer", () => ({ RemoteConnectionFlow: () => null })); vi.mock("../../state/session", () => ({ useEnvironmentScope: () => true, useEnvironmentsWithScope: (environments: Array<{ environmentId: string }>) => diff --git a/apps/web/src/components/onboarding/WelcomeWizard.tsx b/apps/web/src/components/onboarding/WelcomeWizard.tsx index aadc33b63ecabcaafd47553ebb32e95184374296..419a2598ec444bbe3cb103be5b489904d3113cc3 100644 --- a/apps/web/src/components/onboarding/WelcomeWizard.tsx +++ b/apps/web/src/components/onboarding/WelcomeWizard.tsx @@ -1,3 +1,5 @@ +import { RemoteConnectionFlow } from "../settings/SelfHostedServer"; +import { CloudAuthBoundary } from "../../cloud/CloudAuthBoundary"; import { useAuth } from "@clerk/react"; import { useAtomValue } from "@effect/atom-react"; import type { @@ -28,7 +30,6 @@ import { CheckIcon, ChevronRightIcon, CloudIcon, - LinkIcon, MonitorIcon, TerminalIcon, } from "lucide-react"; @@ -257,7 +258,6 @@ export function WelcomeWizard({ {step === "connection" ? ( ; - readonly expandPairingInitially: boolean; readonly selectedIds: ReadonlySet; readonly onSelectionChange: (ids: ReadonlySet) => void; readonly onToggleEnvironment: (environmentId: EnvironmentId, checked: boolean) => void; @@ -317,7 +315,7 @@ function ConnectionStep({ const directEnvironments = environments.filter( (environment) => !cloudEnabled || !isOnboardingRelayEnvironment(environment), ); - const [pairingOpen, setPairingOpen] = useState(expandPairingInitially); + const [pairingOpen, setPairingOpen] = useState(false); const [isPairing, setIsPairing] = useState(false); const { ready, skippedIds } = resolveOnboardingSetup(environments, selectedIds); const skippedLabels = environments @@ -382,47 +380,47 @@ function ConnectionStep({ ) : null}
- {cloudEnabled ? ( - setPairingOpen(true)} + cloudContent={ + + Loading sign-in… +

+ } + > + + + } /> - ) : null} -
- - +
+ Back + + { + setPairingOpen(false); + onPaired(environmentId); + requestAnimationFrame(() => continueRef.current?.focus()); + }} + /> +
+ )} {skippedLabels.length > 0 ? (

@@ -447,7 +445,7 @@ function ConnectionStep({ + + } + > + + + ); } -function EmptyRemoteEnvironments({ cloudEnabled = true }: { readonly cloudEnabled?: boolean }) { +function EmptyRemoteEnvironments() { return ( @@ -2014,11 +2029,7 @@ function EmptyRemoteEnvironments({ cloudEnabled = true }: { readonly cloudEnable No saved remote environments - - {cloudEnabled - ? "Click “Add environment” to pair another environment, or connect one from T3 Connect." - : "Click “Add environment” to pair another environment."} - + Add an environment to connect your computers. ); @@ -2038,12 +2049,13 @@ function CloudRemoteEnvironmentRows({ empty={} /> ) : savedEnvironments.length === 0 ? ( - + ) : null; } export function ConnectionsSettings() { const desktopBridge = window.desktopBridge; + const [directPairing, setDirectPairing] = useState(false); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const { environments } = useEnvironments(); const primaryEnvironment = usePrimaryEnvironment(); @@ -4075,6 +4087,7 @@ export function ConnectionsSettings() { return ( {primarySettings} + - {routeTarget ? `Add a route to ${routeTarget.label}` : "Add Environment"} + {routeTarget ? `Add a route to ${routeTarget.label}` : "Connect"} {routeTarget ? "Pair this machine again over another address, such as its Tailscale name. It joins the existing routes instead of adding a second machine." - : "Pair another environment to this client."} + : "Connect to your computers through an account or your own server."} -

-
- {renderConnectionModeCard({ - mode: "remote", - title: "Remote link", - description: "Enter a backend host and pairing code.", - icon: , - })} - {desktopBridge - ? renderConnectionModeCard({ - mode: "ssh", - title: "SSH", - description: - "Use local SSH config, agent, and tunnels for the backend.", - icon: , - }) - : null} + {!routeTarget && !directPairing ? ( + setDirectPairing(true)} + onConnected={() => setAddBackendDialogOpen(false)} + onCloudSelected={() => setAddBackendDialogOpen(false)} + /> + ) : ( +
+ {!routeTarget ? ( + + ) : null} +
+ {renderConnectionModeCard({ + mode: "remote", + title: "Remote link", + description: "Enter a backend host and pairing code.", + icon: , + })} + {desktopBridge + ? renderConnectionModeCard({ + mode: "ssh", + title: "SSH", + description: + "Use local SSH config, agent, and tunnels for the backend.", + icon: , + }) + : null} +
+ + {savedBackendMode === "ssh" ? renderSshFields() : renderRemoteModeBody()} +
- - {savedBackendMode === "ssh" ? renderSshFields() : renderRemoteModeBody()} - -
+ )} @@ -4169,15 +4196,17 @@ export function ConnectionsSettings() { /> {hasCloudPublicConfig() ? ( - setPendingT3ConnectRemoval(null)} - onConfirm={() => { - if (!pendingT3ConnectRemoval) return; - setPendingT3ConnectRemoval(null); - void removeSavedBackend(pendingT3ConnectRemoval); - }} - /> + + setPendingT3ConnectRemoval(null)} + onConfirm={() => { + if (!pendingT3ConnectRemoval) return; + setPendingT3ConnectRemoval(null); + void removeSavedBackend(pendingT3ConnectRemoval); + }} + /> + ) : null} diff --git a/apps/web/src/components/settings/SelfHostedServer.test.tsx b/apps/web/src/components/settings/SelfHostedServer.test.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6c0b5bdbeba831996f268a7f69373764008bdac5 --- /dev/null +++ b/apps/web/src/components/settings/SelfHostedServer.test.tsx @@ -0,0 +1,88 @@ +import { StrictMode, act } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test"; + +const state = vi.hoisted(() => ({ + connect: vi.fn(), + connection: null, +})); +vi.mock("@effect/atom-react", () => ({ useAtomValue: () => ({ connection: state.connection }) })); +vi.mock("../../connection/hub", () => ({ + hubConnection: { stateValueAtom: {}, connectAuthenticated: state.connect }, +})); +vi.mock("../../state/use-atom-command", () => ({ useAtomCommand: () => state.connect })); +vi.mock("../../cloud/activation", () => ({ activateCloudAuthentication: vi.fn() })); +vi.mock("../../cloud/publicConfig", () => ({ + hasCloudPublicConfig: () => false, + resolveCloudPublicConfig: () => ({}), +})); +vi.mock("../../environments/primary/desktopAuth", () => ({ + readDesktopPrimaryBearerToken: vi.fn(), +})); +vi.mock("../../environments/primary/target", () => ({ resolvePrimaryEnvironmentHttpUrl: vi.fn() })); + +vi.mock("./settingsLayout", () => ({ SettingsRow: () => null, SettingsSection: () => null })); + +import { HostedServerAutoConnect } from "./SelfHostedServer"; + +let renderer: ReactTestRenderer | undefined; +const server = "https://knife.example.test"; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("window", { location: { origin: server } }); + state.connect.mockReset().mockResolvedValue({ _tag: "Success", value: undefined }); +}); + +afterEach(async () => { + await act(() => renderer?.unmount()); + renderer = undefined; + vi.unstubAllGlobals(); +}); + +it("registers the authenticated web server after StrictMode replaces the first pending effect", async () => { + const configuration: Array<(response: Response) => void> = []; + const fetch = vi.fn((url: string) => { + if (url.endsWith("/oidc/config")) + return new Promise((resolve) => configuration.push(resolve)); + return Promise.resolve(Response.json({ accountId: "owner", deviceId: "browser" })); + }); + vi.stubGlobal("fetch", fetch); + await act(() => { + renderer = create( + + + , + ); + }); + expect(configuration).toHaveLength(2); + await act(async () => { + for (const resolve of configuration) resolve(Response.json({ enabled: true })); + }); + expect(state.connect.mock.calls).toEqual([[{ httpBaseUrl: server }]]); + expect(fetch.mock.calls.filter(([url]) => url.endsWith("/account"))).toHaveLength(1); +}); + +it("does not register a server when its account response arrives after unmount", async () => { + let accountResponse: ((response: Response) => void) | undefined; + vi.stubGlobal( + "fetch", + vi.fn((url: string) => + url.endsWith("/oidc/config") + ? Promise.resolve(Response.json({ enabled: true })) + : new Promise((resolve) => { + accountResponse = resolve; + }), + ), + ); + await act(() => { + renderer = create(); + }); + expect(accountResponse).toBeDefined(); + await act(() => renderer?.unmount()); + renderer = undefined; + await act(async () => { + accountResponse!(Response.json({ accountId: "owner" })); + }); + expect(state.connect).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/components/settings/SelfHostedServer.tsx b/apps/web/src/components/settings/SelfHostedServer.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3f55e2c7705ba17341e346c9f044b582cf61644d --- /dev/null +++ b/apps/web/src/components/settings/SelfHostedServer.tsx @@ -0,0 +1,568 @@ +import { normalizeHostedServer } from "@t3tools/shared/hostedServerUrl"; +import { formatBytes } from "../../lib/formatBytes"; +import { ArchiveStatus, SelfHostedConnectionStatus } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { readDesktopPrimaryBearerToken } from "../../environments/primary/desktopAuth"; +import { resolvePrimaryEnvironmentHttpUrl } from "../../environments/primary/target"; +import type { HostedAuthDevice } from "@t3tools/contracts"; +import { + beginHostedSignIn, + hostedDevices, + hostedAuthHeaders, + hostedAccountRequest, + forgetHostedWebSession, +} from "../../connection/hostedAuth"; +import { useAtomValue } from "@effect/atom-react"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { CloudIcon, LinkIcon, ServerIcon } from "lucide-react"; +import { useEffect, useEffectEvent, useId, useRef, useState, type ReactNode } from "react"; + +import { activateCloudAuthentication } from "../../cloud/activation"; +import { hasCloudPublicConfig } from "../../cloud/publicConfig"; +import { hubConnection } from "../../connection/hub"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { Button } from "../ui/button"; +import { Input } from "../ui/input"; +import { Radio, RadioGroup } from "../ui/radio-group"; +import { SettingsRow, SettingsSection } from "./settingsLayout"; + +export function RemoteConnectionFlow({ + onDirect, + onConnected, + onCloudSelected, + cloudContent, +}: { + readonly onDirect: () => void; + readonly onConnected?: () => void; + readonly onCloudSelected?: () => void; + readonly cloudContent?: ReactNode; +}) { + const cloudAvailable = hasCloudPublicConfig(); + const [method, setMethod] = useState("self"); + const [step, setStep] = useState<"choose" | "self" | "cloud">("choose"); + const headingId = useId(); + const choicesRef = useRef(null); + const showChoices = () => { + setStep("choose"); + requestAnimationFrame(() => + choicesRef.current + ?.querySelector('[role="radio"][aria-checked="true"]') + ?.focus(), + ); + }; + if (step === "self") + return ( +
+ + { + showChoices(); + onConnected?.(); + }} + /> +
+ ); + if (step === "cloud") + return ( +
+ + {cloudContent} +
+ ); + return ( +
+

+ Choose how to reach your computers. +

+ + {cloudAvailable ? ( + + ) : null} + + +
+ + +
+
+ ); +} + +function SelfHostedServerForm({ onConnected }: { readonly onConnected: () => void }) { + const state = useAtomValue(hubConnection.stateValueAtom); + const connect = useAtomCommand(hubConnection.connectAuthenticated, { reportFailure: false }); + const addressId = useId(); + const errorId = useId(); + const connected = useEffectEvent(onConnected); + const [address, setAddress] = useState(state.connection?.httpBaseUrl ?? ""); + const [connecting, setConnecting] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + const bridge = window.desktopBridge; + if (!connecting || !bridge?.getHostedAuthState) return; + let stopped = false; + let checking = false; + const check = async () => { + if (checking) return; + checking = true; + try { + const next = await bridge.getHostedAuthState!(); + if (stopped) return; + if (next.phase === "error") { + setConnecting(false); + setError(next.error ?? "Sign-in did not finish. Sign in again."); + } else if ( + next.phase === "connected" && + next.serverUrl === normalizeHostedServer(address.trim()) + ) { + const result = await connect({ httpBaseUrl: next.serverUrl }); + if (stopped) return; + setConnecting(false); + if (result._tag === "Success") connected(); + else if (!isAtomCommandInterrupted(result)) { + const cause = squashAtomCommandFailure(result); + setError(cause instanceof Error ? cause.message : "Couldn't connect. Sign in again."); + } + } + } catch { + if (!stopped) { + setConnecting(false); + setError("Couldn't finish sign-in. Try again."); + } + } finally { + checking = false; + } + }; + void check(); + const timer = window.setInterval(() => void check(), 1000); + return () => { + stopped = true; + window.clearInterval(timer); + void bridge.cancelHostedAuth?.(); + }; + }, [connecting, address, connect]); + + return ( +
{ + event.preventDefault(); + if (connecting) return; + setError(null); + try { + const server = normalizeHostedServer(address.trim()); + setConnecting(true); + void beginHostedSignIn(server).catch(() => { + setConnecting(false); + setError("Couldn't start sign-in. Check the server address and try again."); + }); + } catch { + setError("Enter an HTTPS server address."); + } + }} + > +
+ + setAddress(event.target.value)} + placeholder="https://server.example.net" + autoComplete="url" + autoCapitalize="none" + spellCheck={false} + required + disabled={connecting} + aria-describedby={error ? errorId : undefined} + /> +
+ {connecting ? ( +

+ Finish signing in in your browser +

+ ) : null} + {error ? ( + + ) : null} +
+ {connecting && window.desktopBridge ? ( + + ) : null} + +
+
+ ); +} + +export function HostedServerAutoConnect() { + const state = useAtomValue(hubConnection.stateValueAtom); + const connect = useAtomCommand(hubConnection.connectAuthenticated, { reportFailure: false }); + useEffect(() => { + if (state.connection !== null) return; + let stopped = false; + void (async () => { + let server: string | null = null; + if (window.desktopBridge?.getHostedAuthState) { + const current = await window.desktopBridge.getHostedAuthState(); + if (current.phase === "connected") server = current.serverUrl; + } else { + const config = await fetch("/api/auth/oidc/config", { + credentials: "same-origin", + redirect: "error", + }); + if (stopped || !config.ok || !(await config.json()).enabled) return; + const response = await fetch("/api/auth/account", { + credentials: "same-origin", + redirect: "error", + }); + if (response.ok) server = window.location.origin; + } + if (!stopped && server !== null) await connect({ httpBaseUrl: server }); + })().catch(() => undefined); + return () => { + stopped = true; + }; + }, [state.connection, connect]); + useEffect(() => { + const connection = state.connection; + if (window.desktopBridge || connection?.authorization !== "hosted") return; + const renew = () => { + void hostedAuthHeaders(connection.httpBaseUrl).catch(() => undefined); + }; + renew(); + const timer = window.setInterval(renew, 30_000); + window.addEventListener("focus", renew); + return () => { + window.clearInterval(timer); + window.removeEventListener("focus", renew); + }; + }, [state.connection]); + return null; +} + +const decodeSyncStatus = Schema.decodeUnknownSync(SelfHostedConnectionStatus); +const decodeArchiveStatus = Schema.decodeUnknownSync(Schema.toCodecJson(ArchiveStatus)); + +const requestComputerSync = async (path: string, body?: unknown) => { + const token = await readDesktopPrimaryBearerToken(); + if (!token) throw new Error("This computer is unavailable."); + const response = await fetch(resolvePrimaryEnvironmentHttpUrl(path), { + method: body === undefined ? "GET" : "POST", + headers: { + authorization: `Bearer ${token}`, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + redirect: "error", + }); + if (!response.ok) throw new Error("Couldn't load sync status. Try again."); + return response; +}; + +function ComputerSyncStatus() { + const [status, setStatus] = useState<{ configured: boolean; connected: boolean } | null>(null); + const [archive, setArchive] = useState(null); + const [error, setError] = useState(null); + const [retrying, setRetrying] = useState(false); + useEffect(() => { + let stopped = false; + let running = false; + const refresh = async () => { + if (running) return; + running = true; + try { + const [sync, archiveResponse] = await Promise.all([ + requestComputerSync("/api/self-hosted/connection").then((response) => response.json()), + requestComputerSync("/api/archive/manage", { type: "archive.status" }).then((response) => + response.json(), + ), + ]); + if (!stopped) { + setStatus(decodeSyncStatus(sync)); + setArchive(decodeArchiveStatus(archiveResponse)); + setError(null); + } + } catch { + if (!stopped) setError("Couldn't load sync status. Reconnecting…"); + } finally { + running = false; + } + }; + void refresh(); + const timer = window.setInterval(() => void refresh(), 15_000); + return () => { + stopped = true; + window.clearInterval(timer); + }; + }, []); + return ( + <> + 0 + ? `${archive.pending} chats waiting to sync` + : `${archive.offloaded} chats archived`) + } + control={ + archive && (archive.pending > 0 || archive.failures.length > 0) ? ( + + ) : undefined + } + /> + {archive ? ( +

+ {formatBytes(archive.storage.allocatedDatabaseBytes + archive.storage.attachmentsBytes)}{" "} + local archive storage. {formatBytes(archive.storage.nativeProviderLogBytes)} provider logs + retained. + {archive.lastReclamation + ? ` Last cleanup freed ${formatBytes(Math.max(0, archive.lastReclamation.beforeAllocatedBytes - archive.lastReclamation.afterAllocatedBytes))}.` + : ""} + {archive.queue.length > 0 + ? ` Next retry ${new Date(Math.min(...archive.queue.map((item) => item.nextAttemptAt))).toLocaleTimeString()}.` + : ""} +

+ ) : null} + + ); +} + +export function SelfHostedServerSettings() { + const state = useAtomValue(hubConnection.stateValueAtom); + const refresh = useAtomCommand(hubConnection.refresh, { reportFailure: false }); + const disconnect = useAtomCommand(hubConnection.disconnect, { reportFailure: false }); + const [signingIn, setSigningIn] = useState(false); + const [deviceList, setDeviceList] = useState<{ + serverUrl: string; + items: readonly HostedAuthDevice[]; + } | null>(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const serverUrl = state.connection?.httpBaseUrl; + const hosted = state.connection?.authorization === "hosted"; + useEffect(() => { + if (!hosted || !serverUrl) return; + let stopped = false; + void hostedDevices(serverUrl) + .then((items) => { + if (!stopped) setDeviceList({ serverUrl, items }); + }) + .catch(() => { + if (!stopped) setError("Couldn't load connected devices. Sign in again to reconnect."); + }); + return () => { + stopped = true; + }; + }, [serverUrl, hosted]); + const devices = deviceList !== null && deviceList.serverUrl === serverUrl ? deviceList.items : []; + if (state.connection === null) return null; + const run = async (command: typeof refresh) => { + if (busy) return; + setBusy(true); + setError(null); + const result = await command(); + setBusy(false); + if (result._tag !== "Success" && !isAtomCommandInterrupted(result)) { + const cause = squashAtomCommandFailure(result); + setError( + cause instanceof Error ? cause.message : "Couldn't update this connection. Try again.", + ); + } + }; + return ( + + + {hosted && (state.error || error) ? ( + + ) : null} + + +
+ } + /> + {signingIn ? ( + { + setSigningIn(false); + setError(null); + if (serverUrl) + void hostedDevices(serverUrl) + .then((items) => setDeviceList({ serverUrl, items })) + .catch(() => undefined); + }} + /> + ) : null} + {hosted && window.desktopBridge ? : null} + {hosted + ? devices + .filter((device) => device.revokedAt === null) + .map(({ deviceId, current, client: presentation }) => ( + { + if (!serverUrl) return; + setBusy(true); + void hostedAccountRequest( + serverUrl, + `/api/auth/devices/${encodeURIComponent(deviceId)}/revoke`, + "POST", + ) + .then(async () => { + if (current) { + await window.desktopBridge?.disconnectHostedAuth?.(); + forgetHostedWebSession(); + await disconnect(); + } else + setDeviceList({ serverUrl, items: await hostedDevices(serverUrl) }); + }) + .catch(() => setError("Couldn't revoke this device. Try again.")) + .finally(() => setBusy(false)); + }} + > + Revoke + + } + /> + )) + : null} + {error || state.error ? ( +

+ {error ?? state.error} +

+ ) : null} + + ); +} diff --git a/apps/web/src/components/settings/pairingUrls.test.ts b/apps/web/src/components/settings/pairingUrls.test.ts index c5c45bcf1ce76ff1cfcd09cecbd0b52d7b048eed..ae8301ce33cee16249eb77f057c53149c4cfe128 100644 --- a/apps/web/src/components/settings/pairingUrls.test.ts +++ b/apps/web/src/components/settings/pairingUrls.test.ts @@ -14,6 +14,12 @@ describe("settings pairing URL helpers", () => { ); }); + it("keeps the environment prefix in direct pairing links", () => { + expect( + resolveDesktopPairingUrl("https://hub.example/prefix/environments/one/", "PAIRCODE"), + ).toBe("https://hub.example/prefix/environments/one/pair#token=PAIRCODE"); + }); + it("uses hosted pairing URLs for HTTPS endpoints", () => { vi.stubEnv("VITE_HOSTED_APP_URL", "https://preview.t3.codes"); diff --git a/apps/web/src/components/settings/pairingUrls.ts b/apps/web/src/components/settings/pairingUrls.ts index 891fe04ad6b11582db8a4f7af3d935aa489f2109..f0f7ffd24401d3ce9adf1a001046b08f139c84d6 100644 --- a/apps/web/src/components/settings/pairingUrls.ts +++ b/apps/web/src/components/settings/pairingUrls.ts @@ -1,9 +1,9 @@ +import { environmentEndpointUrl } from "@t3tools/shared/advertisedEndpoint"; import { buildHostedPairingUrl } from "../../hostedPairing"; import { setPairingTokenOnUrl } from "../../pairingUrl"; export function resolveDesktopPairingUrl(endpointUrl: string, credential: string): string { - const url = new URL(endpointUrl); - url.pathname = "/pair"; + const url = new URL(environmentEndpointUrl(endpointUrl, "/pair")); return setPairingTokenOnUrl(url, credential).toString(); } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index a63ef8209cf3d742e92afbbd4bbbeaccb9a0e3a3..e66849efff227fd6d3a3634c0df814bd70359d1c 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -998,7 +998,9 @@ export const SETTINGS_SEARCH_ITEMS = [ id: "remote-environments", title: "Environments", to: "/settings/connections", - searchTerms: ["add pair backend host code ssh config agent tunnel saved t3 connect"], + searchTerms: [ + "add pair backend host code ssh config agent tunnel saved t3 connect self hosted self-hosted server hub", + ], }, { id: "load-balancing", diff --git a/apps/web/src/components/threadActionMenu.logic.test.ts b/apps/web/src/components/threadActionMenu.logic.test.ts index 6bae63cffec6c0e7133f2440b3bff7e3d0e59a8c..315114481fc4dac70abc4ad1d66a54e24926ca87 100644 --- a/apps/web/src/components/threadActionMenu.logic.test.ts +++ b/apps/web/src/components/threadActionMenu.logic.test.ts @@ -8,6 +8,7 @@ import { const baseState: ThreadActionMenuState = { canOperate: true, + canOrganize: true, branch: null, projectFilter: null, isPinned: false, @@ -46,6 +47,7 @@ describe("buildThreadActionMenuItems", () => { const items = buildThreadActionMenuItems({ ...baseState, canOperate: false, + canOrganize: false, isPinned: reversed, isSettled: reversed, isSnoozed: reversed, @@ -80,7 +82,12 @@ describe("buildThreadActionMenuItems", () => { ); it("preserves local actions and restores mutations after a grant", () => { - const denied = buildThreadActionMenuItems({ ...baseState, canOperate: false, branch: "main" }); + const denied = buildThreadActionMenuItems({ + ...baseState, + canOperate: false, + canOrganize: false, + branch: "main", + }); expect(denied.filter((item) => !item.disabled).map((item) => item.id)).toEqual([ "new-thread-on-branch", "mark-unread", @@ -244,3 +251,13 @@ describe("buildDraftActionMenuItems", () => { expect(items.at(-1)).toMatchObject({ label: "Discard draft", destructive: true }); }); }); + +it("permits central organization while execution and deletion remain disabled", () => { + const items = buildThreadActionMenuItems({ ...baseState, canOperate: false, canOrganize: true }); + for (const id of ["pin", "settle", "snooze", "auto-settle"]) { + expect(items.find((item) => item.id === id)?.disabled).not.toBe(true); + } + for (const id of ["rename", "regenerate-title", "archive", "delete"]) { + expect(items.find((item) => item.id === id)?.disabled).toBe(true); + } +}); diff --git a/apps/web/src/components/threadActionMenu.logic.ts b/apps/web/src/components/threadActionMenu.logic.ts index f51328266cabd8eb63a4ea436db357743a491323..bb150f56706d099001a3bcb1cccf98eb29198577 100644 --- a/apps/web/src/components/threadActionMenu.logic.ts +++ b/apps/web/src/components/threadActionMenu.logic.ts @@ -71,6 +71,7 @@ export function buildDraftActionMenuItems(options: { export interface ThreadActionMenuState { readonly canOperate: boolean; + readonly canOrganize: boolean; readonly branch: string | null; /** * Project scoping for the thread list. Null on surfaces with no scoped @@ -114,6 +115,14 @@ export function threadActionRequiresOperate(action: ThreadActionMenuId): boolean ].includes(action); } +export function threadActionIsOrganization(action: ThreadActionMenuId): boolean { + return ( + ["pin", "unpin", "settle", "unsettle", "unsnooze", "snooze", "auto-settle"].includes(action) || + action.startsWith("snooze:") || + action.startsWith("auto-settle:") + ); +} + /** * Single source for the per-thread action menu: the sidebar row's right-click * menu and the chat header menu share labels, ordering, and capability gating. @@ -249,17 +258,16 @@ export function buildThreadActionMenuItems( icon: "trash", }, ]; - return state.canOperate - ? items - : items.map((item) => - threadActionRequiresOperate(item.id) - ? { - ...item, - disabled: true, - ...(item.children - ? { children: item.children.map((child) => ({ ...child, disabled: true })) } - : {}), - } - : item, - ); + return items.map((item) => + threadActionRequiresOperate(item.id) && + !(threadActionIsOrganization(item.id) ? state.canOrganize : state.canOperate) + ? { + ...item, + disabled: true, + ...(item.children + ? { children: item.children.map((child) => ({ ...child, disabled: true })) } + : {}), + } + : item, + ); } diff --git a/apps/web/src/connection/hostedAuth.test.ts b/apps/web/src/connection/hostedAuth.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..90099e415b0a10a430952ebcff2d4ab212960b0b --- /dev/null +++ b/apps/web/src/connection/hostedAuth.test.ts @@ -0,0 +1,201 @@ +import { normalizeHostedServer } from "@t3tools/shared/hostedServerUrl"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { forgetHostedWebSession, hostedAuthHeaders } from "./hostedAuth"; + +afterEach(() => { + forgetHostedWebSession(); + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +const server = "https://knife.example.test"; +const account = (csrfToken: string, expiresAt: string) => ({ + accountId: "owner", + deviceId: "browser", + csrfToken, + expiresAt, +}); + +describe("hosted account client custody", () => { + it("uses same-origin cookies and renews with CSRF without requesting a bearer token", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-17T00:00:00Z")); + vi.stubGlobal("window", { location: { origin: server } }); + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json(account("first-csrf", "2026-10-17T00:00:30Z"))) + .mockResolvedValueOnce(Response.json(account("renewed-csrf", "2026-10-17T01:00:00Z"))); + vi.stubGlobal("fetch", fetch); + expect(await hostedAuthHeaders(server)).toEqual({ "x-csrf-token": "first-csrf" }); + expect(await hostedAuthHeaders(server)).toEqual({ "x-csrf-token": "renewed-csrf" }); + expect(fetch.mock.calls[0]).toEqual([ + `${server}/api/auth/account`, + { credentials: "same-origin", redirect: "error" }, + ]); + expect(fetch.mock.calls[1]).toEqual([ + `${server}/api/auth/session/refresh`, + { + method: "POST", + credentials: "same-origin", + redirect: "error", + headers: { "x-csrf-token": "first-csrf" }, + }, + ]); + }); + + it("shares concurrent browser renewal requests", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-17T00:00:00Z")); + vi.stubGlobal("window", { location: { origin: server } }); + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json(account("first", "2026-10-17T00:15:00Z"))) + .mockResolvedValueOnce(Response.json(account("renewed", "2026-10-17T00:29:30Z"))); + vi.stubGlobal("fetch", fetch); + await hostedAuthHeaders(server); + vi.setSystemTime(new Date("2026-10-17T00:14:30Z")); + expect(await Promise.all([hostedAuthHeaders(server), hostedAuthHeaders(server)])).toEqual([ + { "x-csrf-token": "renewed" }, + { "x-csrf-token": "renewed" }, + ]); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it("recovers a second tab's stale CSRF after another tab rotates the shared cookie", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-17T00:00:00Z")); + vi.stubGlobal("window", { location: { origin: server } }); + let session = 1; + const calls: Array<{ path: string; csrf: string | null }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: RequestInit) => { + const path = new URL(url).pathname; + const csrf = new Headers(init.headers).get("x-csrf-token"); + calls.push({ path, csrf }); + if (path === "/api/auth/account") + return Response.json(account(`opaque-${session}`, "2026-10-17T00:15:00Z")); + if (csrf !== `opaque-${session}`) return new Response(null, { status: 401 }); + if (path === "/api/auth/session/refresh") { + session++; + return Response.json(account(`opaque-${session}`, "2026-10-17T00:30:00Z")); + } + return Response.json({ revoked: true }); + }), + ); + vi.resetModules(); + const firstTab = await import("./hostedAuth"); + vi.resetModules(); + const secondTab = await import("./hostedAuth"); + expect(firstTab).not.toBe(secondTab); + await firstTab.hostedAuthHeaders(server); + await secondTab.hostedAuthHeaders(server); + await firstTab.hostedAccountRequest(server, "/api/auth/session/refresh", "POST"); + expect( + (await secondTab.hostedAccountRequest(server, "/api/auth/devices/other/revoke", "POST")).ok, + ).toBe(true); + expect(calls.slice(-3)).toEqual([ + { path: "/api/auth/devices/other/revoke", csrf: "opaque-1" }, + { path: "/api/auth/account", csrf: null }, + { path: "/api/auth/devices/other/revoke", csrf: "opaque-2" }, + ]); + }); + + it("serializes each tab's concurrent renewals and repairs a shared-cookie rotation once", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-17T00:00:00Z")); + vi.stubGlobal("window", { location: { origin: server } }); + let session = 1; + const refreshProofs: string[] = []; + const reads: number[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: RequestInit) => { + if (new URL(url).pathname === "/api/auth/account") { + reads.push(session); + return Response.json( + account( + `opaque-${session}`, + session === 1 ? "2026-10-17T00:15:00Z" : "2026-10-17T00:30:00Z", + ), + ); + } + const csrf = new Headers(init.headers).get("x-csrf-token")!; + refreshProofs.push(csrf); + if (csrf !== `opaque-${session}`) return new Response(null, { status: 401 }); + session++; + return Response.json(account(`opaque-${session}`, "2026-10-17T00:30:00Z")); + }), + ); + vi.resetModules(); + const firstTab = await import("./hostedAuth"); + vi.resetModules(); + const secondTab = await import("./hostedAuth"); + await Promise.all([firstTab.hostedAuthHeaders(server), secondTab.hostedAuthHeaders(server)]); + vi.setSystemTime(new Date("2026-10-17T00:14:30Z")); + const results = await Promise.all([ + firstTab.hostedAuthHeaders(server), + firstTab.hostedAuthHeaders(server), + secondTab.hostedAuthHeaders(server), + secondTab.hostedAuthHeaders(server), + ]); + expect(results[0]).toEqual(results[1]); + expect(results[2]).toEqual(results[3]); + expect(refreshProofs).toEqual(["opaque-1", "opaque-1", "opaque-2"]); + expect(reads).toEqual([1, 1, 2]); + expect(results[2]).toEqual({ "x-csrf-token": "opaque-3" }); + }); + + it("bounds expired-cookie recovery to one metadata request without repeating a mutation", async () => { + vi.stubGlobal("window", { location: { origin: server } }); + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json(account("expired-proof", "2035-01-01T00:00:00Z"))) + .mockResolvedValueOnce(new Response(null, { status: 401 })) + .mockResolvedValueOnce(new Response(null, { status: 401 })); + vi.stubGlobal("fetch", fetch); + vi.resetModules(); + const tab = await import("./hostedAuth"); + await expect( + tab.hostedAccountRequest(server, "/api/auth/devices/other/revoke", "POST"), + ).rejects.toThrow("Sign in"); + expect(fetch).toHaveBeenCalledTimes(3); + }); + + it("never sends browser cookies or CSRF to another origin", async () => { + vi.stubGlobal("window", { location: { origin: server } }); + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + await expect(hostedAuthHeaders("https://other.example.test")).rejects.toThrow( + "Open this server", + ); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("obtains ephemeral native access from main and refreshes it on each request", async () => { + const getHostedAccessToken = vi + .fn() + .mockResolvedValueOnce("ephemeral-one") + .mockResolvedValueOnce("ephemeral-two"); + vi.stubGlobal("window", { + location: { origin: "http://localhost:3773" }, + desktopBridge: { getHostedAccessToken }, + }); + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + expect(await hostedAuthHeaders(server)).toEqual({ authorization: "Bearer ephemeral-one" }); + expect(await hostedAuthHeaders(server)).toEqual({ authorization: "Bearer ephemeral-two" }); + expect(fetch).not.toHaveBeenCalled(); + expect(getHostedAccessToken.mock.calls).toEqual([[server], [server]]); + }); + + it.each([ + "http://knife.example.test", + `${server}/path`, + "https://user:secret@knife.example.test", + `${server}?token=secret`, + `${server}#token`, + ])("rejects unsafe server addresses: %s", (value) => { + expect(() => normalizeHostedServer(value)).toThrow(); + }); +}); diff --git a/apps/web/src/connection/hostedAuth.ts b/apps/web/src/connection/hostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..511df6fcb398299d84377d26b09e6ffac342ab9b --- /dev/null +++ b/apps/web/src/connection/hostedAuth.ts @@ -0,0 +1,112 @@ +import { HostedAuthAccount, HostedAuthDevice } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Schema from "effect/Schema"; + +const decodeAccount = Schema.decodeUnknownSync(Schema.toCodecJson(HostedAuthAccount)); +const decodeDevices = Schema.decodeUnknownSync(Schema.toCodecJson(Schema.Array(HostedAuthDevice))); + +let account: { server: string; value: HostedAuthAccount } | null = null; +let browserRequest: { server: string; value: Promise } | null = null; + +export async function hostedAuthHeaders( + server: string, + rejectedCsrf?: string, +): Promise> { + const desktop = window.desktopBridge; + if (desktop?.getHostedAccessToken) { + return { authorization: `Bearer ${await desktop.getHostedAccessToken(server)}` }; + } + if (new URL(server).origin !== window.location.origin) { + throw new Error("Open this server in your browser to sign in."); + } + if ( + rejectedCsrf !== undefined && + account?.server === server && + account.value.csrfToken === rejectedCsrf + ) { + account = null; + } + if ( + account?.server === server && + DateTime.toEpochMillis(account.value.expiresAt) > Date.now() + 60_000 + ) { + return { "x-csrf-token": account.value.csrfToken }; + } + if (browserRequest?.server !== server) { + const previous = account?.server === server ? account.value : null; + const pending = (async () => { + let response = await fetch( + `${server}/api/auth/${previous === null ? "account" : "session/refresh"}`, + { + ...(previous === null + ? {} + : { method: "POST", headers: { "x-csrf-token": previous.csrfToken } }), + credentials: "same-origin", + redirect: "error", + }, + ); + if (response.status === 401 && previous !== null) { + const current = await fetch(`${server}/api/auth/account`, { + credentials: "same-origin", + redirect: "error", + }); + if (!current.ok) throw new Error("Sign in to your server again to reconnect."); + const refreshed = decodeAccount(await current.json()); + response = await fetch(`${server}/api/auth/session/refresh`, { + method: "POST", + headers: { "x-csrf-token": refreshed.csrfToken }, + credentials: "same-origin", + redirect: "error", + }); + } + if (!response.ok) throw new Error("Sign in to your server again to reconnect."); + return decodeAccount(await response.json()); + })(); + browserRequest = { server, value: pending }; + } + const current = browserRequest; + try { + const value = await current.value; + if (browserRequest === current) account = { server, value }; + return { "x-csrf-token": value.csrfToken }; + } finally { + if (browserRequest === current) browserRequest = null; + } +} + +export async function hostedAccountRequest(server: string, path: string, method = "GET") { + const request = (headers: Record) => + fetch(`${server}${path}`, { + method, + headers, + credentials: "same-origin", + redirect: "error", + }); + const headers = await hostedAuthHeaders(server); + let response = await request(headers); + if (response.status === 401 && headers["x-csrf-token"]) { + response = await request(await hostedAuthHeaders(server, headers["x-csrf-token"])); + } + if (!response.ok) throw new Error("Couldn't update this server connection. Try again."); + return response; +} + +export async function hostedDevices(server: string) { + const response = await hostedAccountRequest(server, "/api/auth/devices"); + return decodeDevices(await response.json()); +} + +export async function beginHostedSignIn(server: string) { + if (window.desktopBridge?.beginHostedAuth) { + await window.desktopBridge.beginHostedAuth(server); + return; + } + const url = new URL(`${server}/api/auth/oidc/start`); + url.searchParams.set("returnTo", "/"); + window.location.assign(url.href); +} + +export function forgetHostedWebSession() { + account = null; + browserRequest = null; +} diff --git a/apps/web/src/connection/hub.ts b/apps/web/src/connection/hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..20adbe9997c59050029d7597e10ceb82041853b6 --- /dev/null +++ b/apps/web/src/connection/hub.ts @@ -0,0 +1,5 @@ +import { createHubConnectionAtoms } from "@t3tools/client-runtime/state/hub"; + +import { connectionAtomRuntime } from "./runtime"; + +export const hubConnection = createHubConnectionAtoms(connectionAtomRuntime); diff --git a/apps/web/src/connection/platform.ts b/apps/web/src/connection/platform.ts index 63c6563aea700d1595009434a88dcc65e08f74bf..d975572d2fbddde2d5bdb8ff166a79eceab75e22 100644 --- a/apps/web/src/connection/platform.ts +++ b/apps/web/src/connection/platform.ts @@ -40,6 +40,7 @@ import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { FetchHttpClient } from "effect/http"; +import { hostedAuthHeaders } from "./hostedAuth"; import { APP_VERSION } from "../branding"; import { readDesktopPrimaryBearerToken } from "../environments/primary/desktopAuth"; import * as PrimaryEnvironmentHttpLayer from "../environments/primary/httpLayer"; @@ -69,6 +70,18 @@ function currentNetworkStatus(): "unknown" | "offline" | "online" { return navigator.onLine ? "online" : "offline"; } +const layerHostedAuthAccess = Layer.succeed(ClientCapabilities.HostedAuthAccess, { + headers: (serverUrl) => + Effect.tryPromise({ + try: () => hostedAuthHeaders(serverUrl), + catch: () => + new ConnectionBlockedError({ + reason: "authentication", + detail: "Sign in to your server again to reconnect.", + }), + }), +}); + const layerConnectivity = Connectivity.layer({ status: Effect.sync(currentNetworkStatus), changes: Stream.callback((queue) => @@ -745,9 +758,9 @@ const layerPlatformConnectionSource = Layer.effect( const layerEnvironmentOwnedDataCleanup = Layer.succeed( Persistence.EnvironmentOwnedDataCleanup, Persistence.EnvironmentOwnedDataCleanup.of({ - clear: (environmentId) => + clear: (environmentId, options) => Effect.sync(() => { - clearComposerDraftsEnvironment(environmentId); + if (!options?.retainDrafts) clearComposerDraftsEnvironment(environmentId); }), }), ); @@ -770,6 +783,7 @@ const layerRpcRequestObserver = Layer.succeed( type ConnectionPlatformLayerSource = | typeof ConnectionStorage.layer | typeof layerConnectivity + | typeof layerHostedAuthAccess | typeof layerWakeups | typeof layerCapabilities | typeof layerPlatformConnectionSource @@ -782,6 +796,7 @@ export const layer: Layer.Layer< Layer.Services > = Layer.mergeAll( ConnectionStorage.layer, + layerHostedAuthAccess, layerConnectivity, layerWakeups, layerCapabilities, diff --git a/apps/web/src/connection/storage.test.ts b/apps/web/src/connection/storage.test.ts index 87d78dddbf8c14b827b0c6261651718c1eea0a3c..8b2dbd71515c0c1c5114fa975b3dfdbd77a0bf28 100644 --- a/apps/web/src/connection/storage.test.ts +++ b/apps/web/src/connection/storage.test.ts @@ -1,4 +1,10 @@ import { + BearerConnectionCredential, + BearerConnectionProfile, + BearerConnectionRegistration, + BearerConnectionTarget, + CredentialStore, + HubConnectionCredential, ConnectionTransientError, PrimaryConnectionTarget, } from "@t3tools/client-runtime/connection"; @@ -31,6 +37,63 @@ afterEach(() => { }); describe("ConnectionStorage.makeCatalogStore", () => { + it.effect("restores hub route pointers and the single bearer after reopening storage", () => + Effect.gen(function* () { + let saved: string | null = null; + vi.stubGlobal("window", { + desktopBridge: { + getConnectionCatalog: async () => saved, + setConnectionCatalog: async (raw: string) => { + saved = raw; + return true; + }, + }, + }); + const connectionId = "hub:hub:source"; + const hubCredentialId = "hub-session:hub"; + const credential = new HubConnectionCredential({ connectionId: hubCredentialId }); + const bearer = new BearerConnectionCredential({ token: "single-hub-secret" }); + const registration = new BearerConnectionRegistration({ + target: new BearerConnectionTarget({ + connectionId, + environmentId: EnvironmentId.make("source"), + label: "Source", + }), + profile: new BearerConnectionProfile({ + connectionId, + environmentId: EnvironmentId.make("source"), + label: "Source", + httpBaseUrl: "https://hub.example/environments/source/", + wsBaseUrl: "wss://hub.example/environments/source/", + }), + credential, + }); + yield* Effect.gen(function* () { + const credentials = yield* CredentialStore.ConnectionCredentialStore; + const registrations = yield* Persistence.ConnectionRegistrationStore; + yield* credentials.put(hubCredentialId, bearer); + yield* registrations.register(registration, [registration.target]); + expect(yield* credentials.get(connectionId)).toEqual(Option.some(credential)); + }).pipe(Effect.provide(ConnectionStorage.layer), Effect.scoped); + + yield* Effect.gen(function* () { + const credentials = yield* CredentialStore.ConnectionCredentialStore; + expect(yield* credentials.get(connectionId)).toEqual(Option.some(credential)); + expect(yield* credentials.get(hubCredentialId)).toEqual(Option.some(bearer)); + }).pipe(Effect.provide(ConnectionStorage.layer), Effect.scoped); + expect(JSON.parse(saved!).credentials).toEqual([ + { + connectionId: hubCredentialId, + credential: { _tag: "BearerConnectionCredential", token: bearer.token }, + }, + { + connectionId, + credential: { _tag: "HubConnectionCredential", connectionId: hubCredentialId }, + }, + ]); + }), + ); + it.effect("quarantines malformed catalogs and starts from an empty document", () => Effect.gen(function* () { const writes: string[] = []; diff --git a/apps/web/src/environments/primary/bootstrap.test.ts b/apps/web/src/environments/primary/bootstrap.test.ts index c9da4dab7051279cad852ee5c2ab46fd2233ea22..6f8e4ea3e40ec96a7fdcdad4ec935ac1aa8889a2 100644 --- a/apps/web/src/environments/primary/bootstrap.test.ts +++ b/apps/web/src/environments/primary/bootstrap.test.ts @@ -127,6 +127,21 @@ describe("environmentBootstrap", () => { ); }); + it("keeps configured environment paths in primary descriptor and auth URLs", async () => { + vi.stubEnv("VITE_HTTP_URL", "https://hub.example/prefix/environments/one/?old=1#fragment"); + await installDescriptorApi(); + await expect(resolveInitialPrimaryEnvironmentDescriptor()).resolves.toEqual(BASE_ENVIRONMENT); + expect(resolvePrimaryEnvironmentHttpUrl("/.well-known/t3/environment")).toBe( + "https://hub.example/prefix/environments/one/.well-known/t3/environment", + ); + expect(readPrimaryEnvironmentTarget()?.target.wsBaseUrl).toBe( + "wss://hub.example/prefix/environments/one/", + ); + expect(resolvePrimaryEnvironmentHttpUrl("/api/auth/session", { token: "ticket" })).toBe( + "https://hub.example/prefix/environments/one/api/auth/session?token=ticket", + ); + }); + it("derives the websocket url when only VITE_HTTP_URL is configured", async () => { vi.stubEnv("VITE_HTTP_URL", "https://remote.example.com"); await installDescriptorApi(); @@ -155,6 +170,16 @@ describe("environmentBootstrap", () => { }); }); + it.each(["wss://hub.example/prefix/ws", "wss://hub.example/prefix/"])( + "derives HTTP base paths from a configured WebSocket address: %s", + (wsBaseUrl) => { + vi.stubEnv("VITE_WS_URL", wsBaseUrl); + expect(resolvePrimaryEnvironmentHttpUrl("/api/auth/session")).toBe( + "https://hub.example/prefix/api/auth/session", + ); + }, + ); + it("keeps an uppercase wss scheme secure when deriving the http url", () => { vi.stubEnv("VITE_WS_URL", "WSS://remote.example.com"); diff --git a/apps/web/src/environments/primary/target.ts b/apps/web/src/environments/primary/target.ts index 7e574bb7cd2098c82bf77ba7e15463d2784492ae..70834670b317e096890e72ce7b9fed23f827fd48 100644 --- a/apps/web/src/environments/primary/target.ts +++ b/apps/web/src/environments/primary/target.ts @@ -1,4 +1,9 @@ import { PRIMARY_LOCAL_ENVIRONMENT_ID, type DesktopEnvironmentBootstrap } from "@t3tools/contracts"; +import { + deriveWsBaseUrl, + environmentEndpointUrl, + normalizeHttpBaseUrl, +} from "@t3tools/shared/advertisedEndpoint"; import * as Schema from "effect/Schema"; import { isLocalEnvironmentDisabled } from "../../localEnvironment"; @@ -137,6 +142,9 @@ function swapBaseUrlProtocol( source: "configured", urlKind, }); + if ((url.protocol === "ws:" || url.protocol === "wss:") && !url.pathname.endsWith("/")) { + url.pathname = url.pathname.replace(/[^/]+$/, ""); + } url.protocol = nextProtocol; return url.toString(); } @@ -209,17 +217,19 @@ function resolveConfiguredPrimaryTarget(): PrimaryEnvironmentTarget | null { (configuredWsBaseUrl?.toLowerCase().startsWith("wss:") ? swapBaseUrlProtocol(configuredWsBaseUrl, "https:", "websocket-base-url") : swapBaseUrlProtocol(configuredWsBaseUrl!, "http:", "websocket-base-url")); + const normalizedHttpBaseUrl = normalizeHttpBaseUrl( + normalizeBaseUrl(resolvedHttpBaseUrl, "configured", "http-base-url"), + ); const resolvedWsBaseUrl = - configuredWsBaseUrl ?? - (configuredHttpBaseUrl?.toLowerCase().startsWith("https:") - ? swapBaseUrlProtocol(configuredHttpBaseUrl, "wss:", "http-base-url") - : swapBaseUrlProtocol(configuredHttpBaseUrl!, "ws:", "http-base-url")); + configuredWsBaseUrl === undefined + ? deriveWsBaseUrl(normalizedHttpBaseUrl) + : normalizeBaseUrl(configuredWsBaseUrl, "configured", "websocket-base-url"); return { source: "configured", target: { - httpBaseUrl: normalizeBaseUrl(resolvedHttpBaseUrl, "configured", "http-base-url"), - wsBaseUrl: normalizeBaseUrl(resolvedWsBaseUrl, "configured", "websocket-base-url"), + httpBaseUrl: normalizedHttpBaseUrl, + wsBaseUrl: resolvedWsBaseUrl, }, }; } @@ -296,11 +306,11 @@ export function resolvePrimaryEnvironmentHttpUrl( source: primaryTarget.source, urlKind: "http-base-url", }); - url.pathname = pathname; + const endpoint = new URL(environmentEndpointUrl(url.toString(), pathname)); if (searchParams) { - url.search = new URLSearchParams(searchParams).toString(); + endpoint.search = new URLSearchParams(searchParams).toString(); } - return url.toString(); + return endpoint.toString(); } // Null only when the desktop app runs with its local environment disabled; diff --git a/apps/web/src/hooks/useThreadActionMenu.ts b/apps/web/src/hooks/useThreadActionMenu.ts index 2fda09240b6b27e2d426d98d37a5e8758715ecd4..d0990547209c7bd76da3a6c5fa2494790dda3403 100644 --- a/apps/web/src/hooks/useThreadActionMenu.ts +++ b/apps/web/src/hooks/useThreadActionMenu.ts @@ -19,12 +19,13 @@ import { resolveSnoozePresets } from "../components/Sidebar.snooze"; import { buildThreadActionMenuItems, threadActionRequiresOperate, + threadActionIsOrganization, type ThreadActionMenuId, } from "../components/threadActionMenu.logic"; import { stackedThreadToast, toastManager } from "../components/ui/toast"; import { threadEnvironment } from "../state/threads"; import { useOrchestrationCommand } from "../state/use-orchestration-command"; -import { readEnvironmentScope } from "../state/session"; +import { readEnvironmentScope, readThreadOrganizationPermission } from "../state/session"; import { readEnvironmentSupportsAutoSettleOptOut, readEnvironmentSupportsPinning, @@ -148,6 +149,7 @@ export function useThreadActionMenu(input: { const isRegeneratingTitle = thread.titleRegeneration != null; const snoozePresets = resolveSnoozePresets(now, timestampFormat); const items = buildThreadActionMenuItems({ + canOrganize: readThreadOrganizationPermission(threadRef.environmentId), canOperate: readEnvironmentScope(threadRef.environmentId, AuthOrchestrationOperateScope), branch: thread.branch ?? null, projectFilter: null, @@ -166,7 +168,9 @@ export function useThreadActionMenu(input: { const action: ThreadActionMenuId = clicked.value; if ( threadActionRequiresOperate(action) && - !readEnvironmentScope(threadRef.environmentId, AuthOrchestrationOperateScope) + !(threadActionIsOrganization(action) + ? readThreadOrganizationPermission(threadRef.environmentId) + : readEnvironmentScope(threadRef.environmentId, AuthOrchestrationOperateScope)) ) { failureToast( "Thread action unavailable", diff --git a/apps/web/src/main.tsx b/apps/web/src/main.tsx index 81d1c2140af6b473f6f3f8bccf2fa52d094b6744..e28804233f995fa2ee31db88f893c82d7396d1e8 100644 --- a/apps/web/src/main.tsx +++ b/apps/web/src/main.tsx @@ -6,7 +6,6 @@ import "./index.css"; import { prepareProviderAuthDelivery } from "./providerAuthDelivery"; import { isElectron } from "./env"; -import { hasCloudPublicConfig } from "./cloud/publicConfig"; import { getRouter } from "./router"; import { syncDocumentElectronPlatformClasses, @@ -27,8 +26,6 @@ if (isElectron) { syncDocumentWindowControlsOverlayClass(); } -const clerkPublishableKey = import.meta.env.VITE_CLERK_PUBLISHABLE_KEY as string | undefined; - // A failed split-chunk fetch usually means the hashed assets went stale under // a deploy; one guarded reload picks up the fresh index.html. let chunkLoadFailed = false; @@ -43,41 +40,13 @@ window.addEventListener("vite:preloadError", (event) => { const app = ; -// Managed auth is cloud-only, and the Electron Clerk provider bundles the full -// clerk-js runtime. Loading only the selected runtime as a split chunk keeps -// every Clerk byte out of the startup graph for local-mode users, and keeps -// the bundled clerk-js out of the browser build entirely. -const managedAuthShellModule = - clerkPublishableKey && hasCloudPublicConfig() - ? isElectron - ? import("./components/clerk/ElectronManagedAuthShell") - : import("./components/clerk/BrowserManagedAuthShell") - : null; - -// The index.html boot splash lives inside #root, and React's first commit -// clears it. Resolve everything that first commit needs, the selected -// managed-auth runtime and the initial route's split chunks, before -// rendering, so the splash holds until real UI paints instead of dropping to -// a blank window while chunks download. -export const startup = Promise.all([ - managedAuthShellModule?.then((module) => module.default) ?? null, - router.load(), -]) - .then(([ManagedAuthShell]) => { - // A route chunk failure still resolves router.load(): the error is parked in - // the lazy component and surfaces through the route error boundary. Skip the - // paint when a reload is on its way, and only re-arm the guard after a boot - // that fetched every chunk it asked for. +export const startup = router + .load() + .then(() => { if (reloadScheduled) return; if (!chunkLoadFailed) clearChunkReloadGuard(); ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render( - - {ManagedAuthShell && clerkPublishableKey ? ( - {app} - ) : ( - app - )} - , + {app}, ); }) .catch((error: unknown) => { diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx index b94d979dfc09f6fbec2c6b0f6e61ebf28a7ae75a..d1a2fc8fe07cd466945058fa26b3ae2f96bc76d2 100644 --- a/apps/web/src/routes/__root.tsx +++ b/apps/web/src/routes/__root.tsx @@ -1,3 +1,4 @@ +import { HostedServerAutoConnect } from "../components/settings/SelfHostedServer"; import { usePersonalUiSettingsSync } from "../state/uiSettingsSync"; import { PermissionUpdateNotice } from "../components/PermissionUpdateNotice"; import { type ServerLifecycleWelcomePayload } from "@t3tools/contracts"; @@ -235,6 +236,7 @@ function RootRouteView() { + diff --git a/apps/web/src/state/entities.ts b/apps/web/src/state/entities.ts index 9e1c7f24d3e7b454ad4b53591ac6a778947c8549..38c06fa50ab4ceac1508de1a3d2880a10f31322a 100644 --- a/apps/web/src/state/entities.ts +++ b/apps/web/src/state/entities.ts @@ -14,7 +14,7 @@ import type { EnvironmentId, OrchestrationV2ProjectedTurnItem } from "@t3tools/c import { Atom } from "effect/reactivity"; import { appAtomRegistry } from "../rpc/atomRegistry"; import { environmentProjects } from "./projects"; -import { environmentServerConfigsAtom } from "./server"; +import { environmentCapabilitiesAtom, environmentServerConfigsAtom } from "./server"; import { allEnvironmentProjectSnapshotsReadyAtom, allEnvironmentShellsBootstrappedAtom, @@ -240,8 +240,8 @@ export function readEnvironmentSupportsServerBrowser(environmentId: EnvironmentI export function readEnvironmentSupportsTitleRegeneration(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadTitleRegeneration === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadTitleRegeneration === + true ); } @@ -249,8 +249,7 @@ export function readEnvironmentSupportsTitleRegeneration(environmentId: Environm Same version-skew contract as settlement. */ export function readEnvironmentSupportsPinning(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadPinning === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadPinning === true ); } @@ -258,8 +257,7 @@ export function readEnvironmentSupportsPinning(environmentId: EnvironmentId): bo orderKey on thread.pin). Same version-skew contract as settlement. */ export function readEnvironmentSupportsPinReorder(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadPinReorder === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadPinReorder === true ); } @@ -267,15 +265,15 @@ export function readEnvironmentSupportsPinReorder(environmentId: EnvironmentId): Same version-skew contract as settlement. */ export function readEnvironmentSupportsAutoSettleOptOut(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadAutoSettleOptOut === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadAutoSettleOptOut === + true ); } export function readEnvironmentSupportsActiveReorder(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadActiveReorder === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadActiveReorder === + true ); } @@ -284,18 +282,14 @@ export function readEnvironmentSupportsActiveReorder(environmentId: EnvironmentI so clients under version skew fall back instead of erroring. */ export function readEnvironmentSupportsSettlement(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadSettlement === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadSettlement === true ); } /** Whether the environment's server understands thread.snooze/unsnooze. Same version-skew contract as settlement. */ export function readEnvironmentSupportsSnooze(environmentId: EnvironmentId): boolean { - return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadSnooze === true - ); + return appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadSnooze === true; } /** Whether the environment's server understands thread.visit/mark-unread and @@ -304,8 +298,8 @@ export function readEnvironmentSupportsSnooze(environmentId: EnvironmentId): boo state instead. */ export function readEnvironmentSupportsVisitedTracking(environmentId: EnvironmentId): boolean { return ( - appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities - .threadVisitedTracking === true + appAtomRegistry.get(environmentCapabilitiesAtom).get(environmentId)?.threadVisitedTracking === + true ); } diff --git a/apps/web/src/state/server.ts b/apps/web/src/state/server.ts index 14cc31ae5e3b051e1205fd287915df896d1e25e5..f82a8465754783b8219a129d974376ddb073432b 100644 --- a/apps/web/src/state/server.ts +++ b/apps/web/src/state/server.ts @@ -1,3 +1,5 @@ +import { createHubEnvironmentCapabilitiesAtom } from "@t3tools/client-runtime/state/hub"; +import { hubConnection } from "../connection/hub"; import { DEFAULT_SERVER_SETTINGS, type EditorId, @@ -39,6 +41,10 @@ export const environmentServerConfigsAtom = createEnvironmentServerConfigsAtom({ catalogValueAtom: environmentCatalog.catalogValueAtom, serverConfigValueAtom: serverEnvironment.configValueAtom, }); +export const environmentCapabilitiesAtom = createHubEnvironmentCapabilitiesAtom( + environmentServerConfigsAtom, + hubConnection, +); interface PrimaryServerState { readonly config: ServerConfig | null; diff --git a/apps/web/src/state/session.ts b/apps/web/src/state/session.ts index 883a899ef2c0b328c9b1fc7ec8721b5366fdd8f6..5286fb3a82c336fcda64018b8fa54194da9d1e1c 100644 --- a/apps/web/src/state/session.ts +++ b/apps/web/src/state/session.ts @@ -1,6 +1,8 @@ +import { hubConnection } from "../connection/hub"; import { useAtomValue } from "@effect/atom-react"; import { createEnvironmentSessionAtoms } from "@t3tools/client-runtime/state/session"; import { + AuthOrchestrationOperateScope, type AuthEnvironmentScope, type AuthSessionState, type EnvironmentId, @@ -101,3 +103,16 @@ export function useEnvironmentSessionState(environmentId: EnvironmentId) { isPending: result.waiting, }; } + +export function useThreadOrganizationPermission(environmentId: EnvironmentId | null): boolean { + const source = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + const central = useAtomValue(hubConnection.organizationPermissionAtom(environmentId)); + return source || central; +} + +export function readThreadOrganizationPermission(environmentId: EnvironmentId): boolean { + return ( + readEnvironmentScope(environmentId, AuthOrchestrationOperateScope) || + appAtomRegistry.get(hubConnection.organizationPermissionAtom(environmentId)) + ); +} diff --git a/apps/web/src/state/use-orchestration-command.test.ts b/apps/web/src/state/use-orchestration-command.test.ts index 44941063a8dc8b526f95973dcadf183fa7e2d9a9..81d06a69845746bf112900ff3c35b59ecb88707d 100644 --- a/apps/web/src/state/use-orchestration-command.test.ts +++ b/apps/web/src/state/use-orchestration-command.test.ts @@ -10,12 +10,17 @@ import { assert, beforeEach, expect, it, vi } from "vite-plus/test"; const state = vi.hoisted(() => ({ allowed: new Set(), + organization: new Set(), readScope: vi.fn(), run: vi.fn(), })); vi.mock("react", () => ({ useCallback: (callback: unknown) => callback })); -vi.mock("./session", () => ({ readEnvironmentScope: state.readScope })); +vi.mock("./session", () => ({ + readEnvironmentScope: state.readScope, + readThreadOrganizationPermission: (id: string) => + state.organization.has(id) || state.allowed.has(id), +})); vi.mock("./use-atom-command", () => ({ useAtomCommand: () => state.run })); import { useOrchestrationCommand } from "./use-orchestration-command"; @@ -31,6 +36,7 @@ const target = (environmentId: EnvironmentId): Target => ({ beforeEach(() => { state.allowed.clear(); + state.organization.clear(); state.run.mockReset().mockResolvedValue(AsyncResult.success("receipt")); state.readScope .mockReset() @@ -83,3 +89,14 @@ it("does not dispatch a later mutation after access changes while awaiting a rec expect((await mutate(target(second)))._tag).toBe("Failure"); expect(state.run).toHaveBeenCalledTimes(1); }); + +it("accepts the central organization grant only for organization commands and rechecks revocation", async () => { + state.organization.add(second); + const organize = useOrchestrationCommand(command, { organization: true, reportFailure: false }); + const execute = useOrchestrationCommand(command, { reportFailure: false }); + expect((await organize(target(second)))._tag).toBe("Success"); + expect((await execute(target(second)))._tag).toBe("Failure"); + state.organization.clear(); + expect((await organize(target(second)))._tag).toBe("Failure"); + expect(state.run).toHaveBeenCalledTimes(1); +}); diff --git a/apps/web/src/state/use-orchestration-command.ts b/apps/web/src/state/use-orchestration-command.ts index 71bee0a35b5d23965a7f99a2dcb1b1621244b23d..41e20d995a51463b38969960404a994d1b2fd623 100644 --- a/apps/web/src/state/use-orchestration-command.ts +++ b/apps/web/src/state/use-orchestration-command.ts @@ -12,7 +12,7 @@ import * as Cause from "effect/Cause"; import { AsyncResult } from "effect/reactivity"; import { useCallback } from "react"; -import { readEnvironmentScope } from "./session"; +import { readEnvironmentScope, readThreadOrganizationPermission } from "./session"; import { useAtomCommand } from "./use-atom-command"; export const ORCHESTRATION_OPERATE_DENIED_MESSAGE = @@ -21,12 +21,16 @@ export const ORCHESTRATION_OPERATE_DENIED_MESSAGE = /** Recheck the target grant for every step of a thread or project mutation. */ export function useOrchestrationCommand( command: AtomCommand, - options?: string | AtomCommandOptions, + options?: string | (AtomCommandOptions & { readonly organization?: true }), ): (value: W) => Promise> { const run = useAtomCommand(command, options); + const organization = typeof options === "object" && options.organization === true; return useCallback( async (value: W): Promise> => { - if (!readEnvironmentScope(value.environmentId, AuthOrchestrationOperateScope)) { + const permitted = organization + ? readThreadOrganizationPermission(value.environmentId) + : readEnvironmentScope(value.environmentId, AuthOrchestrationOperateScope); + if (!permitted) { return AsyncResult.failure( Cause.fail( new EnvironmentAuthorizationError({ @@ -38,6 +42,6 @@ export function useOrchestrationCommand unexpectedEndpoint("auth.delegatedSession")) .handle("token", () => unexpectedEndpoint("auth.token")) .handle("webSocketTicket", () => unexpectedEndpoint("auth.webSocketTicket")) .handle( diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index cf343e26a4fcf9ffbe1e9d208753d319bdbc9581..44e07a8e1792844183399a9fdbabda88188ac01c 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -253,7 +253,9 @@ export default defineConfig(() => { { target: devProxyTarget, changeOrigin: !DEV_PROXIED_ORIGIN_PRESERVING_PREFIXES.has(prefix), - ...(prefix === "/ws" || prefix === "/api" ? { ws: true } : {}), + ...(prefix === "/ws" || prefix === "/api" || prefix === "/environments" + ? { ws: true } + : {}), }, ]), ), diff --git a/compose.sso.yaml b/compose.sso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f153dc53a9f3acb9feb8a40f669ca814bb46b77 --- /dev/null +++ b/compose.sso.yaml @@ -0,0 +1,12 @@ +services: + knife: + environment: + T3CODE_SSO_CONFIG: /run/secrets/knife-oidc.json + T3CODE_ARCHIVE_STORE_ID: ${KNIFE_ARCHIVE_STORE_ID:?Set the archive store identity} + volumes: + - type: bind + source: ${KNIFE_SSO_CONFIG:?Set the private OIDC configuration path} + target: /run/secrets/knife-oidc.json + read_only: true + bind: + create_host_path: false diff --git a/docs/operations/development.md b/docs/operations/development.md index ac427db9428a24a9c8efaf59540f09623ed1af1d..38267e42c617f0df0241843563765dd1b8ede189 100644 --- a/docs/operations/development.md +++ b/docs/operations/development.md @@ -369,7 +369,81 @@ four minutes without restarting or replacing the running code. A feed outage use baseline; a checksum or dependency mismatch stops startup. Rebuild the base image when the server's locked native dependency closure or container bootstrap changes. -Enroll execution hosts with the private authority using the existing -[`t3 shared` commands](../user/shared-server.md). Each machine gets its own -revocable session credential. The authority owns Codex token refreshes; Claude -accounts need setup tokens. Connected hosts synchronize accounts automatically. +Use hosted sign-in below for desktop enrollment and archive uploads. Optional +[shared provider accounts](../user/shared-server.md#share-agent-accounts) use separate +grants; signing into the server does not export local provider credentials. + +### Hosted sign-in + +Register a confidential OpenID Connect application with your identity provider: + +| Setting | Value | +| --------------------- | ------------------------------------------------------------------- | +| Grant types | `authorization_code`, `refresh_token` | +| Response type | `code` | +| Callback | `https://knife.example.com/api/auth/oidc/callback`, matched exactly | +| PKCE | Required, `S256` | +| Client authentication | `client_secret_basic` | +| ID token algorithm | `RS256` | +| Scopes | `openid offline_access`; allow refresh tokens | + +The issuer callback is HTTPS. `t3code://app/self-hosted/callback` is a separate, +proof-bound return to the desktop; do not register it with the issuer. There is no +public native client or dynamic client registration. + +Set `T3CODE_SSO_CONFIG` to a private JSON file readable by the server process: + +```json +{ + "publicUrl": "https://knife.example.com", + "issuer": "https://identity.example.com", + "clientId": "knife", + "clientSecret": "PRIVATE_REGISTERED_CLIENT_SECRET", + "ownerSub": "IMMUTABLE_OWNER_SUBJECT", + "maxInteractiveAgeSeconds": 2592000, + "issuerRevalidationSeconds": 3600 +} +``` + +Use the exact issuer string from discovery and its immutable subject for the +intended owner. `publicUrl` is the HTTPS origin without a trailing slash. Email, +display name, and the first person to visit never establish ownership. Missing +owner configuration prevents SSO from starting. Without `T3CODE_SSO_CONFIG`, +existing pairing remains available. + +Keep this file outside the checkout, restrict it to its owner (`chmod 600`), and +mount it read-only into containers. Pass its container path through +`T3CODE_SSO_CONFIG`. Never include it in an image, public build share, or browser +configuration. The same deployment needs persistent private server state and an +HTTPS proxy that forwards WebSocket upgrades. + +For Compose, set `KNIFE_SSO_CONFIG` to that host file's absolute path and +`KNIFE_ARCHIVE_STORE_ID` to the identity stored in the NAS archive directory's +pre-existing `.knife-archive-store-id` file. The server never creates this marker; +missing or mismatched storage identity prevents catalog publication and local +payload eviction. Use the optional overlay: + +```sh +docker compose -f compose.yaml -f compose.sso.yaml up -d --no-build +``` + +The overlay mounts the private configuration read-only and refuses to create a +missing host file. Existing deployments can keep using the base Compose file. + +Access credentials last fifteen minutes. Native devices renew with rotating +credentials kept in the operating system’s protected storage. By default, devices +must complete browser sign-in again after thirty days. The server revalidates the +issuer grant during renewal at least once per hour; issuer-side revocation is +noticed on that check, and previously issued access can last another fifteen +minutes. Local device revocation closes active connections and disables renewal. + +If the issuer is unavailable or a device is lost, run this on the server using its +existing local state directory: + +```sh +t3 auth devices revoke-all +``` + +This local recovery command revokes SSO devices and sessions without contacting +the issuer. Existing pairing credentials remain valid. It does not change the +configured owner or revoke identity-provider accounts. diff --git a/docs/user/shared-server.md b/docs/user/shared-server.md index e84036522add4253fe83e243029801e84b02bc7d..eb61696d52d373800fc611ac61b5b5b69c31bddf 100644 --- a/docs/user/shared-server.md +++ b/docs/user/shared-server.md @@ -1,4 +1,16 @@ -# Shared accounts +# Self-hosted server + +A self-hosted server gives your browser and desktop one address for your machines and archived chats. Each machine keeps its workspaces and running agents. T3 Connect remains optional. + +## Connect your machines + +On a current Knife desktop, open **Connections**, choose **Self-hosted server**, and enter its HTTPS address. Sign in through the system browser. The desktop enrolls this machine and connects it to the server; private network addresses and transferred token files are unnecessary. Provider credentials stay on the machine unless you explicitly share them. + +Open the same address in a web browser to sign in and see your machines. A sleeping machine stays listed. Synced chats can be settled, snoozed, woken, and pinned while it is offline. The machine must reconnect before it can run work; cleanup from settling waits until then. Disconnecting keeps unsent drafts and leaves direct connections and running agents in place. + +Device access renews automatically until the server requires sign-in again. Revoking a device closes its connections and stops renewal. Existing manually paired machines keep their identities and saved chats; their pairing and `t3 shared hub` commands remain available for recovery. + +## Share agent accounts A private T3 server can hold managed Codex and Claude accounts for your other hosts. Each host runs agents against its own workspaces. The shared server can run agents too. Provider homes and cached credentials live inside each host’s T3 home; existing CLI homes stay separate. This is opt-in and does not move native CLI history between machines. @@ -25,14 +37,12 @@ Use `t3 shared status` to list shared accounts, `t3 shared sync` for an immediat ## Run in a container -Build from this checkout with `docker build -t the-knife .`. The image includes the web client, server, Codex, and Claude. Mount persistent private storage and your workspaces: +The image includes the web client, server, Codex, and Claude. Follow the [container and hosted sign-in setup](../operations/development.md#private-server-container), including the Compose SSO overlay, private configuration mount, and archive storage identity. The process runs as UID 1000; give it access to the mounted directories. Put an HTTPS reverse proxy in front of port 3773, including WebSocket upgrades, then connect clients using the server’s HTTPS address. Provider account sharing remains optional. Back up `/data` and `/home/node`, including their private credentials, independently of `/archives`. -```sh -docker run --name the-knife -p 127.0.0.1:3773:3773 \ - -e KNIFE_UPDATE_URL=https://files.example.com/public-builds/ \ - -v /private/t3:/data -v /private/provider-home:/home/node \ - -v /private/archives:/archives \ - -v /private/workspaces:/workspaces the-knife -``` +## Offload archived chats -The process runs as UID 1000; give it access to these directories. Put an HTTPS reverse proxy in front of port 3773, including WebSocket upgrades. Pair clients using the normal T3 pairing flow. Use `docker exec the-knife knife auth session issue …` for server-side account setup. Back up `/data` and `/home/node`, including their private credentials, independently of `/archives`. +The container’s `T3CODE_ARCHIVE_DIR` points at the private archive mount. Signing a desktop into the self-hosted server also configures its background archive uploads while its local server runs. Keep the archive mount separate from the T3 state mount. Existing hosts using `t3 shared connect` and `T3CODE_ARCHIVE_OFFLOAD=true` keep their saved archive locations. + +Archived chats upload automatically in the background. Message and tool bodies are removed locally only after verified upload; titles, projects, timeline indexes, and control metadata stay local. Opening a chat fetches its bodies without retaining them. Unarchiving restores the bodies and attachments before the chat can run again. Local attachment bytes stay while an active chat or fork needs them. Workspace media is copied into the archive; your original files are left to the normal workspace lifecycle. + +Uploads retry automatically after failures, with delays increasing from one minute to one hour; the queue survives restarts. `t3 shared archive status` shows pending uploads and failures; `t3 shared archive retry [THREAD_ID]` requests an immediate attempt. A missing mount, failed verification, or concurrent chat change retains the local copy. Synced chats and attachments remain readable from the server while their source machine sleeps. Back up the archive mount: its verified objects become the authoritative copy of offloaded bodies. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 360c4bd0bd5f540c1e96f2d6fb78bcccdda1f127..c04217be3aede8254d1ba3a99bad024e7b563a8a 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -3,6 +3,10 @@ "private": true, "type": "module", "exports": { + "./state/hub": { + "types": "./src/state/hub.ts", + "default": "./src/state/hub.ts" + }, "./composerThreadItems": { "types": "./src/composerThreadItems.ts", "default": "./src/composerThreadItems.ts" diff --git a/packages/client-runtime/src/authorization/remote.test.ts b/packages/client-runtime/src/authorization/remote.test.ts index 54290c469d826e46043a24aaba11b8d0503b7881..44dbed210942aa331c49736ffc418615464212df 100644 --- a/packages/client-runtime/src/authorization/remote.test.ts +++ b/packages/client-runtime/src/authorization/remote.test.ts @@ -16,6 +16,7 @@ import { RemoteEnvironmentAuthInvalidJsonError, RemoteEnvironmentAuthTimeoutError, resolveRemoteWebSocketConnectionUrl, + resolveRemoteDpopWebSocketConnectionUrl, } from "./remote.ts"; import { fetchRemoteEnvironmentDescriptor } from "../environment/descriptor.ts"; import * as RpcHttp from "../rpc/http.ts"; @@ -90,6 +91,62 @@ const expectFetchCall = ( }; describe("remote environment authorization", () => { + it.effect("keeps prefixed descriptor, auth, proof, and socket URLs on one environment", () => + Effect.gen(function* () { + const base = "https://hub.example/prefix/environments/one/"; + const ticket = () => + Response.json({ ticket: "ws-ticket", expiresAt: "2026-05-01T12:05:00.000Z" }); + const fetch = recordedFetch( + Response.json({ + environmentId: "one", + label: "One", + platform: { os: "linux", arch: "x64" }, + serverVersion: "test", + capabilities: { repositoryIdentity: true }, + }), + Response.json({ + access_token: "access", + issued_token_type: "urn:ietf:params:oauth:token-type:access_token", + token_type: "DPoP", + expires_in: 3600, + scope: "orchestration:read", + }), + ticket(), + ticket(), + ); + yield* fetchRemoteEnvironmentDescriptor({ httpBaseUrl: base }).pipe( + provideRemoteHttp(fetch.fetchFn), + ); + yield* exchangeRemoteDpopAccessToken({ + httpBaseUrl: base, + credential: "bootstrap", + dpopProof: "proof", + }).pipe(provideRemoteHttp(fetch.fetchFn)); + const bearerSocket = yield* resolveRemoteWebSocketConnectionUrl({ + httpBaseUrl: base, + wsBaseUrl: base.replace("https:", "wss:"), + bearerToken: "bearer", + }).pipe(provideRemoteHttp(fetch.fetchFn)); + const dpopSocket = yield* resolveRemoteDpopWebSocketConnectionUrl({ + httpBaseUrl: base, + wsBaseUrl: base.replace("https:", "wss:"), + accessToken: "access", + dpopProof: "proof", + }).pipe(provideRemoteHttp(fetch.fetchFn)); + expect(fetch.calls.map(([url]) => String(url))).toEqual([ + `${base}.well-known/t3/environment`, + `${base}oauth/token`, + `${base}api/auth/websocket-ticket`, + `${base}api/auth/websocket-ticket`, + ]); + const urls = RpcHttp.makeEnvironmentHttpApiUrlBuilder(base); + expect(urls.auth.token()).toBe(`${base}oauth/token`); + expect(urls.auth.webSocketTicket()).toBe(`${base}api/auth/websocket-ticket`); + expect(bearerSocket).toBe("wss://hub.example/prefix/environments/one/ws?wsTicket=ws-ticket"); + expect(dpopSocket).toBe(bearerSocket); + }), + ); + it.effect("bootstraps bearer auth against a remote backend", () => Effect.gen(function* () { const fetch = recordedFetch( diff --git a/packages/client-runtime/src/authorization/remote.ts b/packages/client-runtime/src/authorization/remote.ts index 425d95d3ac170c4d4d73186d2cc1e95ae4e28d39..32a3cde3105fe33a42f0112ef392e61d1cc28368 100644 --- a/packages/client-runtime/src/authorization/remote.ts +++ b/packages/client-runtime/src/authorization/remote.ts @@ -8,7 +8,7 @@ import { } from "@t3tools/contracts"; import { encodeOAuthScope } from "@t3tools/shared/oauthScope"; import * as Effect from "effect/Effect"; -import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { environmentEndpointUrl, environmentWebSocketUrl } from "../environment/endpoint.ts"; import { executeEnvironmentHttpRequest, makeEnvironmentHttpApiGroupClient, @@ -214,10 +214,7 @@ export const resolveRemoteWebSocketConnectionUrl = Effect.fn( ...(input.timeoutMs ? { timeoutMs: input.timeoutMs } : {}), }); - const url = new URL(input.wsBaseUrl); - if (url.pathname === "" || url.pathname === "/") { - url.pathname = "/ws"; - } + const url = new URL(environmentWebSocketUrl(input.wsBaseUrl)); url.searchParams.set("wsTicket", issued.ticket); appendClientConnectionParams(url, input.clientMetadata, input.connectionMethod); return url.toString(); @@ -240,10 +237,7 @@ export const resolveRemoteDpopWebSocketConnectionUrl = Effect.fn( dpopProof: input.dpopProof, ...(input.timeoutMs ? { timeoutMs: input.timeoutMs } : {}), }); - const url = new URL(input.wsBaseUrl); - if (url.pathname === "" || url.pathname === "/") { - url.pathname = "/ws"; - } + const url = new URL(environmentWebSocketUrl(input.wsBaseUrl)); url.searchParams.set("wsTicket", issued.ticket); appendClientConnectionParams(url, input.clientMetadata, input.connectionMethod); return url.toString(); diff --git a/packages/client-runtime/src/connection/catalog.ts b/packages/client-runtime/src/connection/catalog.ts index b0b1526a450756ba6b80d8e5e324cfcd182e7717..2bbe45041bdb2fc7d211969c0d941aa29b94a1d8 100644 --- a/packages/client-runtime/src/connection/catalog.ts +++ b/packages/client-runtime/src/connection/catalog.ts @@ -32,7 +32,7 @@ export class BearerConnectionProfile extends Schema.TaggedClass()( + "HubConnectionCredential", + { + connectionId: Schema.String, + }, +) {} + +export class HostedConnectionCredential extends Schema.TaggedClass()( + "HostedConnectionCredential", + { serverUrl: Schema.String }, +) {} + +export const ConnectionCredential = Schema.Union([ + BearerConnectionCredential, + HubConnectionCredential, + HostedConnectionCredential, +]); export type ConnectionCredential = typeof ConnectionCredential.Type; export class PrimaryConnectionRegistration extends Schema.TaggedClass()( @@ -104,7 +120,7 @@ export class BearerConnectionRegistration extends Schema.TaggedClass ({ + environmentId: EnvironmentId.make(id), + label: id, + platform: { os: "linux" as const, arch: "x64" as const }, + serverVersion: "test", + orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + capabilities: { repositoryIdentity: true }, +}); + +function harness() { + const profiles = new Map(); + const credentials = new Map(); + const requests: Array<{ + url: string; + authorization: string | null; + method: string; + etag: string | null; + }> = []; + const writes: Array = []; + const commands: Array = []; + const cleanups: Array<{ id: EnvironmentId; retainDrafts?: boolean }> = []; + const control = { + machines: ["laptop", "desktop"], + offline: false, + badAddress: false, + protocolVersion: ORCHESTRATION_PROTOCOL_VERSION, + inboxStatus: 200, + notModified: false, + inboxEtag: '"inbox-v1"', + inboxSequence: 5, + token: "hub-secret", + hostedAuthorization: "Bearer hosted-ephemeral" as string | null, + failProfileWrite: false, + failCredentialWrite: false, + }; + const dependencies = Layer.mergeAll( + Layer.succeed(Capabilities.HostedAuthAccess, { + headers: () => + Effect.sync(() => + control.hostedAuthorization === null + ? { "x-csrf-token": "cookie-proof" } + : { authorization: control.hostedAuthorization }, + ), + }), + Http.layerRemoteHttpClient((async (input, init) => { + const url = String(input); + requests.push({ + url, + authorization: new Headers(init?.headers).get("authorization"), + method: init?.method ?? "GET", + etag: new Headers(init?.headers).get("if-none-match"), + }); + if (url.endsWith("/.well-known/t3/environment")) return Response.json(descriptor(HUB_ID)); + if (url.endsWith("/oauth/token")) + return Response.json({ + access_token: control.token, + issued_token_type: "urn:ietf:params:oauth:token-type:access_token", + token_type: "Bearer", + expires_in: 3600, + scope: AuthStandardClientScopes.join(" "), + }); + if (url === `${BASE}api/hub/environments`) { + if (control.offline) throw new Error("offline"); + return Response.json({ + hubEnvironmentId: HUB_ID, + environments: control.machines.map((id) => ({ + ...descriptor(id), + descriptor: { + ...descriptor(id), + orchestrationProtocolVersion: control.protocolVersion, + }, + scopes: AuthStandardClientScopes, + httpBaseUrl: `${control.badAddress ? "https://elsewhere.example/" : BASE}environments/${id}/`, + })), + }); + } + if (url === `${BASE}api/hub/inbox`) { + if (control.offline) throw new Error("offline"); + if (control.inboxStatus !== 200) return new Response(null, { status: control.inboxStatus }); + if ( + control.notModified && + init?.method !== "POST" && + new Headers(init?.headers).get("if-none-match") === control.inboxEtag + ) { + return new Response(null, { status: 304 }); + } + const entry: HubInboxEnvironment = { + environmentId: EnvironmentId.make("laptop"), + snapshot: { + schemaVersion: 2, + snapshotSequence: control.inboxSequence, + projects: [v2Project], + threads: [ + { + ...v2ThreadShell, + id: ThreadId.make("chat"), + lineage: { + rootThreadId: ThreadId.make("chat"), + parentThreadId: null, + relationshipToParent: null, + }, + snoozedAt: v2ThreadShell.updatedAt, + snoozedUntil: DateTime.makeUnsafe("2030-01-01T00:00:00.000Z"), + }, + ], + archivedThreads: [], + }, + pendingThreadIds: [], + }; + if (init?.method === "POST") { + commands.push(decodeInboxCommand(JSON.parse(await new Response(init.body).text()))); + } + const body = + init?.method === "POST" + ? encodeInboxEnvironment(entry) + : encodeInbox({ environments: [entry] }); + return Response.json(body, { + headers: { etag: control.inboxEtag }, + }); + } + throw new Error(`Unexpected URL ${url}`); + }) satisfies typeof fetch), + Layer.succeed(Capabilities.ClientPresentation, { + metadata: { label: "Phone", deviceType: "mobile", surface: "mobile" }, + }), + Layer.succeed(Profiles.ConnectionProfileStore, { + get: (id) => Effect.succeed(Option.fromUndefinedOr(profiles.get(id))), + put: (profile) => + Effect.suspend(() => + control.failProfileWrite + ? Effect.fail( + new ConnectionBlockedError({ + reason: "configuration", + detail: "Profile storage unavailable", + }), + ) + : Effect.sync(() => { + profiles.set(profile.connectionId, profile); + }), + ), + remove: (id) => + Effect.sync(() => { + profiles.delete(id); + }), + }), + Layer.succeed(Credentials.ConnectionCredentialStore, { + get: (id) => Effect.succeed(Option.fromUndefinedOr(credentials.get(id))), + put: (id, credential) => + Effect.suspend(() => + control.failCredentialWrite + ? Effect.fail( + new ConnectionBlockedError({ + reason: "configuration", + detail: "Credential storage unavailable", + }), + ) + : Effect.sync(() => { + credentials.set(id, credential); + }), + ), + remove: (id) => + Effect.sync(() => { + credentials.delete(id); + }), + }), + Layer.mock(Persistence.ConnectionTargetStore)({ + list: Effect.succeed([]), + listDisabled: Effect.succeed([]), + }), + Layer.mock(Persistence.ConnectionRegistrationStore)({ + register: (registration) => + Effect.sync(() => { + writes.push(registration.target.environmentId); + if (registration._tag === "BearerConnectionRegistration") { + profiles.set(registration.profile.connectionId, registration.profile); + credentials.set(registration.target.connectionId, registration.credential); + } + }), + setRoutes: () => Effect.void, + remove: () => Effect.void, + setEnabled: () => Effect.void, + }), + Layer.mock(Persistence.EnvironmentCacheStore)({ clear: () => Effect.void }), + Layer.succeed(Persistence.EnvironmentOwnedDataCleanup, { + clear: (id, options) => + Effect.sync(() => { + cleanups.push({ id, ...options }); + }), + }), + Layer.mock(Capabilities.SshEnvironmentGateway)({}), + Layer.succeed(Connectivity.Connectivity, { + status: Effect.succeed("online" as const), + changes: Stream.never, + }), + Layer.succeed(Wakeups.ConnectionWakeups, { changes: Stream.never }), + Layer.mock(Driver.ConnectionDriver)({ connect: () => Effect.never }), + ); + const registry = Registry.layer.pipe(Layer.provideMerge(dependencies)); + return { + profiles, + credentials, + requests, + writes, + commands, + cleanups, + control, + layer: Hub.layer.pipe(Layer.provideMerge(registry)), + }; +} + +describe("self-hosted server connections", () => { + it.effect( + "stores only a server marker for SSO and renews native headers through the broker", + () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connectAuthenticated({ httpBaseUrl: BASE }); + expect(test.credentials.get(`hub-session:${HUB_ID}`)).toEqual( + new HostedConnectionCredential({ serverUrl: BASE }), + ); + expect(test.requests.some((request) => request.url.endsWith("/oauth/token"))).toBe(false); + expect(yield* hub.headers).toEqual({ authorization: "Bearer hosted-ephemeral" }); + test.control.hostedAuthorization = "Bearer renewed-ephemeral"; + yield* hub.refreshInbox; + expect(test.requests.at(-1)?.authorization).toBe("Bearer renewed-ephemeral"); + test.control.hostedAuthorization = null; + expect(yield* hub.headers).toEqual({ "x-csrf-token": "cookie-proof" }); + yield* hub.refreshInbox; + expect(test.requests.at(-1)?.authorization).toBe(null); + }).pipe(Effect.provide(test.layer)); + }, + ); + + it.effect("preserves paired access if SSO connection validation or persistence fails", () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const previous = test.credentials.get(`hub-session:${HUB_ID}`); + test.control.offline = true; + yield* Effect.option(hub.connectAuthenticated({ httpBaseUrl: BASE })); + expect(test.credentials.get(`hub-session:${HUB_ID}`)).toEqual(previous); + test.control.offline = false; + test.control.failProfileWrite = true; + yield* Effect.option(hub.connectAuthenticated({ httpBaseUrl: BASE })); + expect(test.credentials.get(`hub-session:${HUB_ID}`)).toEqual(previous); + }).pipe(Effect.provide(test.layer)); + }); + + it.effect("round-trips timestamped inbox snapshots and organization commands as JSON", () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const thread = (yield* SubscriptionRef.get(hub.state)).inbox.environments[0]?.snapshot + .threads[0]; + expect(DateTime.formatIso(thread!.createdAt)).toBe("2026-06-20T00:00:00.000Z"); + expect(DateTime.formatIso(thread!.snoozedUntil!)).toBe("2030-01-01T00:00:00.000Z"); + const settledAt = DateTime.makeUnsafe("2026-06-21T12:34:56.000Z"); + yield* hub.organize({ + environmentId: EnvironmentId.make("laptop"), + command: { + type: "thread.settle", + commandId: CommandId.make("settle-with-timestamp"), + threadId: ThreadId.make("chat"), + settledAt, + }, + }); + expect(test.commands).toEqual([ + { + environmentId: EnvironmentId.make("laptop"), + command: { + type: "thread.settle", + commandId: CommandId.make("settle-with-timestamp"), + threadId: ThreadId.make("chat"), + settledAt, + }, + }, + ]); + const updated = (yield* SubscriptionRef.get(hub.state)).inbox.environments[0]?.snapshot + .threads[0]; + expect(DateTime.formatIso(updated!.updatedAt)).toBe("2026-06-20T00:00:00.000Z"); + expect(DateTime.formatIso(updated!.snoozedAt!)).toBe("2026-06-20T00:00:00.000Z"); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }); + it.effect("accepts organization through the hub while the source has no connection", () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const result = yield* hub.organize({ + environmentId: EnvironmentId.make("laptop"), + command: { + type: "thread.settle", + commandId: CommandId.make("settle-offline"), + threadId: ThreadId.make("chat"), + }, + }); + expect(Option.getOrNull(result)).toEqual({ sequence: 5 }); + expect(test.requests.at(-1)).toEqual({ + url: `${BASE}api/hub/inbox`, + authorization: "Bearer hub-secret", + method: "POST", + etag: null, + }); + const direct = yield* hub.organize({ + environmentId: EnvironmentId.make("direct-only"), + command: { + type: "thread.unpin", + commandId: CommandId.make("unpin-direct"), + threadId: ThreadId.make("chat"), + }, + }); + expect(Option.isNone(direct)).toBe(true); + test.control.inboxStatus = 403; + expect( + (yield* Effect.result( + hub.organize({ + environmentId: EnvironmentId.make("laptop"), + command: { + type: "thread.pin", + commandId: CommandId.make("denied-pin"), + threadId: ThreadId.make("chat"), + }, + }), + ))._tag, + ).toBe("Failure"); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }); + it.effect("revalidates unchanged inbox metadata without replacing the cached snapshot", () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const inbox = (yield* SubscriptionRef.get(hub.state)).inbox; + test.control.notModified = true; + yield* hub.refreshInbox; + expect(test.requests.at(-1)?.etag).toBe('"inbox-v1"'); + expect((yield* SubscriptionRef.get(hub.state)).inbox).toBe(inbox); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }); + it.effect("reuses the validated body after a reconnect fails to persist its profile", () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + test.control.inboxSequence = 6; + test.control.inboxEtag = '"inbox-v2"'; + test.control.token = "new-account-token"; + test.control.failProfileWrite = true; + yield* hub.connect({ host: BASE, pairingCode: "new-code" }).pipe(Effect.flip); + expect( + (yield* SubscriptionRef.get(hub.state)).inbox.environments[0]?.snapshot.snapshotSequence, + ).toBe(5); + test.control.notModified = true; + yield* hub.refreshInbox; + expect(test.requests.at(-1)?.etag).toBe('"inbox-v2"'); + expect( + (yield* SubscriptionRef.get(hub.state)).inbox.environments[0]?.snapshot.snapshotSequence, + ).toBe(6); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }); + + it.effect( + "never sends a validator fetched with another credential after failed pairing persistence", + () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + test.control.inboxSequence = 6; + test.control.inboxEtag = '"inbox-v2"'; + test.control.token = "new-account-token"; + test.control.failCredentialWrite = true; + yield* hub.connect({ host: BASE, pairingCode: "new-code" }).pipe(Effect.flip); + test.control.notModified = true; + yield* hub.refreshInbox; + expect(test.requests.at(-1)?.authorization).toBe("Bearer hub-secret"); + expect(test.requests.at(-1)?.etag).toBeNull(); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); + it.effect( + "clears an outdated machine's compatibility rejection after its advertised protocol updates", + () => { + const test = harness(); + test.control.protocolVersion = ORCHESTRATION_PROTOCOL_VERSION - 1; + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + const registry = yield* Registry.EnvironmentRegistry; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const id = EnvironmentId.make("laptop"); + expect( + (yield* SubscriptionRef.get(registry.entries)).get(id)?.unsupportedReason, + ).toBeDefined(); + test.control.protocolVersion = ORCHESTRATION_PROTOCOL_VERSION; + yield* hub.refresh; + expect( + (yield* SubscriptionRef.get(registry.entries)).get(id)?.unsupportedReason, + ).toBeUndefined(); + yield* registry.setEnabled(id, true); + expect((yield* SubscriptionRef.get(registry.entries)).get(id)?.enabled).toBe(true); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); + it.effect( + "pairs once for two machines, links one secret, and leaves unchanged routes running on refresh", + () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + const registry = yield* Registry.EnvironmentRegistry; + yield* hub.connect({ host: BASE, pairingCode: "one-use-code" }); + const entries = yield* SubscriptionRef.get(registry.entries); + expect([...entries.keys()]).toEqual(["laptop", "desktop"]); + expect( + test.requests.filter((request) => request.url.endsWith("/oauth/token")), + ).toHaveLength(1); + expect( + [...test.credentials.values()].filter( + (value) => value._tag === "BearerConnectionCredential", + ), + ).toEqual([new BearerConnectionCredential({ token: "hub-secret" })]); + expect(test.credentials.get("hub:hub:laptop")).toMatchObject({ + _tag: "HubConnectionCredential", + connectionId: "hub-session:hub", + }); + expect( + test.requests.find((request) => request.url.endsWith("/api/hub/environments")) + ?.authorization, + ).toBe("Bearer hub-secret"); + yield* hub.refresh; + expect(test.writes).toEqual(["laptop", "desktop"]); + expect(yield* SubscriptionRef.get(registry.entries)).toBe(entries); + const restored = yield* Hub.make; + expect((yield* SubscriptionRef.get(restored.state)).connection?.httpBaseUrl).toBe(BASE); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); + + it.effect( + "keeps saved machines during an outage and discovers later enrollment on refresh", + () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + const registry = yield* Registry.EnvironmentRegistry; + yield* hub.connect({ host: BASE, pairingCode: "one-use-code" }); + const entries = yield* SubscriptionRef.get(registry.entries); + test.control.offline = true; + expect((yield* Effect.result(hub.refresh))._tag).toBe("Failure"); + expect(yield* SubscriptionRef.get(registry.entries)).toBe(entries); + expect((yield* SubscriptionRef.get(hub.state)).error).toContain("Couldn't reach"); + test.control.offline = false; + test.control.machines.push("new-machine"); + yield* hub.refresh; + expect([...(yield* SubscriptionRef.get(registry.entries)).keys()]).toEqual([ + "laptop", + "desktop", + "new-machine", + ]); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); + + it.effect( + "rejects a directory that sends machine traffic outside the paired origin without saving credentials", + () => { + const test = harness(); + test.control.badAddress = true; + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + const registry = yield* Registry.EnvironmentRegistry; + expect((yield* Effect.result(hub.connect({ host: BASE, pairingCode: "code" })))._tag).toBe( + "Failure", + ); + expect(test.credentials.size).toBe(0); + expect((yield* SubscriptionRef.get(registry.entries)).size).toBe(0); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); + + it.effect( + "disconnects only hub-owned routes, preserving directly paired and T3 Connect routes", + () => { + const test = harness(); + return Effect.gen(function* () { + const hub = yield* Hub.HubConnection; + const registry = yield* Registry.EnvironmentRegistry; + yield* hub.connect({ host: BASE, pairingCode: "code" }); + const id = EnvironmentId.make("laptop"); + const connectionId = "direct-laptop"; + yield* registry.register( + new BearerConnectionRegistration({ + target: new BearerConnectionTarget({ + environmentId: id, + label: "Laptop", + connectionId, + }), + profile: new BearerConnectionProfile({ + environmentId: id, + label: "Laptop", + connectionId, + httpBaseUrl: "https://laptop.example/", + wsBaseUrl: "wss://laptop.example/", + }), + credential: new BearerConnectionCredential({ token: "direct-secret" }), + }), + ); + yield* registry.register( + new RelayConnectionRegistration({ + target: new RelayConnectionTarget({ environmentId: id, label: "Laptop" }), + }), + ); + yield* hub.disconnect; + const entries = yield* SubscriptionRef.get(registry.entries); + expect([...entries.keys()]).toEqual([id]); + expect( + connectionRoutes(entries.get(id)!).map((route) => connectionRouteId(route.target)), + ).toEqual([connectionId, "relay"]); + expect(test.cleanups).toEqual([{ id: "desktop", retainDrafts: true }]); + expect(test.credentials.has("hub-session:hub")).toBe(false); + expect((yield* SubscriptionRef.get(hub.state)).connection).toBeNull(); + }).pipe(Effect.provide(test.layer), Effect.scoped); + }, + ); +}); diff --git a/packages/client-runtime/src/connection/hub.ts b/packages/client-runtime/src/connection/hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..dfc1e1e174010fe15216cd5ded565e6bc3755a04 --- /dev/null +++ b/packages/client-runtime/src/connection/hub.ts @@ -0,0 +1,580 @@ +import { + EnvironmentId, + HubDirectory, + HubInbox, + HubInboxCommand, + HubInboxEnvironment, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Equal from "effect/Equal"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { HttpBody, HttpClient } from "effect/http"; + +import { + BearerConnectionCredential, + BearerConnectionProfile, + BearerConnectionRegistration, + HubConnectionCredential, + HostedConnectionCredential, +} from "./catalog.ts"; +import { + BearerConnectionTarget, + ConnectionBlockedError, + ConnectionTransientError, +} from "./model.ts"; +import type { ConnectionAttemptError } from "./model.ts"; +import { preparePairingRegistration, type PairingConnectionInput } from "./onboarding.ts"; +import * as CredentialStore from "./credentialStore.ts"; +import * as ProfileStore from "./profileStore.ts"; +import * as Registry from "./registry.ts"; +import { connectionRouteId, connectionRoutes } from "./routes.ts"; +import { orchestrationProtocolCompatibilityError } from "./compatibility.ts"; +import { fetchRemoteEnvironmentDescriptor } from "../environment/descriptor.ts"; +import { deriveWsBaseUrl, environmentEndpointUrl } from "../environment/endpoint.ts"; +import * as ClientCapabilities from "../platform/capabilities.ts"; + +const HUB_PROFILE_ID = "self-hosted-server"; +const isConnectionAttemptError = Schema.is( + Schema.Union([ConnectionBlockedError, ConnectionTransientError]), +); +const encodeInboxCommand = Schema.encodeEffect(Schema.toCodecJson(HubInboxCommand)); +const decodeInbox = Schema.decodeUnknownEffect(Schema.toCodecJson(HubInbox)); +const decodeInboxEnvironment = Schema.decodeUnknownEffect(Schema.toCodecJson(HubInboxEnvironment)); + +export interface HubServerConnection { + readonly environmentId: EnvironmentId; + readonly label: string; + readonly httpBaseUrl: string; + readonly authorization?: "t3-connect" | "hosted"; +} + +export interface HubConnectionState { + readonly connection: HubServerConnection | null; + readonly directory: HubDirectory | null; + readonly inbox: HubInbox; + readonly refreshing: boolean; + readonly error: string | null; +} + +export const EMPTY_HUB_CONNECTION_STATE: HubConnectionState = { + connection: null, + directory: null, + inbox: { environments: [] }, + refreshing: false, + error: null, +}; + +export class HubConnection extends Context.Service< + HubConnection, + { + readonly state: SubscriptionRef.SubscriptionRef; + readonly connect: ( + input: PairingConnectionInput, + ) => Effect.Effect; + readonly connectAuthenticated: (input: { + readonly httpBaseUrl: string; + }) => Effect.Effect; + readonly headers: Effect.Effect, ConnectionAttemptError>; + readonly refresh: Effect.Effect; + readonly refreshInbox: Effect.Effect; + readonly organize: ( + input: HubInboxCommand, + ) => Effect.Effect, ConnectionAttemptError>; + readonly disconnect: Effect.Effect; + } +>()("@t3tools/client-runtime/connection/hub/HubConnection") {} + +const hubCredentialId = (environmentId: EnvironmentId) => `hub-session:${environmentId}`; +const hubRoutePrefix = (environmentId: EnvironmentId) => `hub:${environmentId}:`; + +export const make = Effect.gen(function* () { + const profiles = yield* ProfileStore.ConnectionProfileStore; + const credentials = yield* CredentialStore.ConnectionCredentialStore; + const registry = yield* Registry.EnvironmentRegistry; + const http = yield* HttpClient.HttpClient; + const presentation = yield* ClientCapabilities.ClientPresentation; + const lock = yield* Semaphore.make(1); + const saved = Option.getOrNull(yield* profiles.get(HUB_PROFILE_ID)); + const connection = saved?._tag === "BearerConnectionProfile" ? saved : null; + const state = yield* SubscriptionRef.make({ + ...EMPTY_HUB_CONNECTION_STATE, + connection, + }); + let inboxCache: + | { + readonly url: string; + readonly token: string; + readonly etag: string | undefined; + readonly inbox: HubInbox; + } + | undefined; + + const currentCredential = Effect.fn("HubConnection.currentCredential")(function* ( + profile: HubServerConnection, + ) { + const credential = Option.getOrNull( + yield* credentials.get(hubCredentialId(profile.environmentId)), + ); + if ( + credential?._tag !== "BearerConnectionCredential" && + credential?._tag !== "HostedConnectionCredential" + ) { + return yield* new ConnectionBlockedError({ + reason: "authentication", + detail: "Sign in to this server to restore access.", + }); + } + return credential; + }); + + const headersFor = (credential: BearerConnectionCredential | HostedConnectionCredential) => + credential._tag === "BearerConnectionCredential" + ? Effect.succeed({ authorization: `Bearer ${credential.token}` } as Record) + : Effect.gen(function* () { + const access = yield* Effect.serviceOption(ClientCapabilities.HostedAuthAccess); + if (Option.isNone(access)) + return yield* new ConnectionBlockedError({ + reason: "authentication", + detail: "Sign in to this server to reconnect.", + }); + return yield* access.value.headers(credential.serverUrl); + }); + + const requestInbox = Effect.fn("HubConnection.inbox")( + function* ( + profile: HubServerConnection, + credential: BearerConnectionCredential | HostedConnectionCredential, + ) { + const url = environmentEndpointUrl(profile.httpBaseUrl, "/api/hub/inbox"); + const headers = yield* headersFor(credential); + const cacheIdentity = headers.authorization ?? headers["x-csrf-token"] ?? "cookie"; + const cached = + inboxCache?.url === url && inboxCache.token === cacheIdentity ? inboxCache : undefined; + const response = yield* http.get(url, { + headers: { + ...headers, + ...(cached?.etag === undefined ? {} : { "if-none-match": cached.etag }), + }, + }); + if (response.status === 304 && cached?.etag !== undefined) return cached.inbox; + if (response.status === 401 || response.status === 403) { + return yield* new ConnectionBlockedError({ + reason: "authentication", + detail: "Sign in to this server to restore access.", + }); + } + if (response.status !== 200) { + return yield* new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "Couldn't load chats from this server. Try again.", + }); + } + const inbox = yield* response.json.pipe(Effect.flatMap(decodeInbox)); + inboxCache = { url, token: cacheIdentity, etag: response.headers.etag, inbox }; + return inbox; + }, + Effect.timeout("15 seconds"), + Effect.mapError((error) => + isConnectionAttemptError(error) + ? error + : new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "Couldn't load chats from this server. Try again.", + }), + ), + ); + + const requestDirectory = Effect.fn("HubConnection.directory")( + function* ( + profile: HubServerConnection, + credential: BearerConnectionCredential | HostedConnectionCredential, + ) { + const response = yield* http + .get(environmentEndpointUrl(profile.httpBaseUrl, "/api/hub/environments"), { + headers: yield* headersFor(credential), + }) + .pipe( + Effect.mapError( + () => + new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "Couldn't reach this server. Check the address and try again.", + }), + ), + ); + if (response.status === 401 || response.status === 403) { + return yield* new ConnectionBlockedError({ + reason: "authentication", + detail: "Sign in to this server to restore access.", + }); + } + if (response.status === 404) { + return yield* new ConnectionBlockedError({ + reason: "unsupported", + detail: "This server can't share other machines. Pair it as a single machine.", + }); + } + if (response.status !== 200) { + return yield* new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "The server is unavailable. Try again.", + }); + } + const directory = yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(HubDirectory)), + Effect.mapError( + () => + new ConnectionBlockedError({ + reason: "configuration", + detail: "The server returned an invalid machine list.", + }), + ), + ); + if (directory.hubEnvironmentId !== profile.environmentId) { + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "This address reaches a different server. Check the address and pair again.", + }); + } + const seen = new Set(); + for (const machine of directory.environments) { + const expected = environmentEndpointUrl( + profile.httpBaseUrl, + `/environments/${encodeURIComponent(machine.environmentId)}/`, + ); + if ( + seen.has(machine.environmentId) || + machine.descriptor.environmentId !== machine.environmentId || + machine.httpBaseUrl !== expected + ) { + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "The server returned an invalid machine address.", + }); + } + seen.add(machine.environmentId); + } + return directory; + }, + Effect.timeoutOrElse({ + duration: "15 seconds", + orElse: () => + Effect.fail( + new ConnectionTransientError({ + reason: "timeout", + detail: "The server took too long to respond. Try again.", + }), + ), + }), + ); + + const removeRoutes = Effect.fn("HubConnection.removeRoutes")(function* ( + environmentId: EnvironmentId, + keep: ReadonlySet = new Set(), + ) { + const entries = yield* SubscriptionRef.get(registry.entries); + for (const [id, entry] of entries) { + for (const route of connectionRoutes(entry)) { + const routeId = connectionRouteId(route.target); + if (routeId.startsWith(hubRoutePrefix(environmentId)) && !keep.has(routeId)) { + yield* registry.removeRoute(id, routeId, { retainDrafts: true }); + } + } + } + }); + + const reconcile = Effect.fn("HubConnection.reconcile")(function* ( + profile: HubServerConnection, + directory: HubDirectory, + ) { + const keep = new Set(); + for (const machine of directory.environments) { + const connectionId = `${hubRoutePrefix(profile.environmentId)}${machine.environmentId}`; + keep.add(connectionId); + const registration = new BearerConnectionRegistration({ + target: new BearerConnectionTarget({ + connectionId, + environmentId: machine.environmentId, + label: machine.label, + }), + profile: new BearerConnectionProfile({ + connectionId, + environmentId: machine.environmentId, + label: machine.label, + httpBaseUrl: machine.httpBaseUrl, + wsBaseUrl: deriveWsBaseUrl(machine.httpBaseUrl), + }), + credential: new HubConnectionCredential({ + connectionId: hubCredentialId(profile.environmentId), + }), + }); + const entry = (yield* SubscriptionRef.get(registry.entries)).get(machine.environmentId); + const unchanged = + entry !== undefined && + connectionRoutes(entry).some( + (route) => + Equal.equals(route.target, registration.target) && + Option.isSome(route.profile) && + Equal.equals(route.profile.value, registration.profile), + ); + if (!unchanged) yield* registry.register(registration); + const current = (yield* SubscriptionRef.get(registry.entries)).get(machine.environmentId); + if ( + current !== undefined && + connectionRoutes(current).every((route) => connectionRouteId(route.target) === connectionId) + ) { + yield* registry.setCompatibility( + machine.environmentId, + orchestrationProtocolCompatibilityError(machine.descriptor), + ); + } + } + yield* removeRoutes(profile.environmentId, keep); + }); + + const reportError = (error: { readonly message: string }) => + SubscriptionRef.update(state, (current) => ({ ...current, error: error.message })); + const finish = SubscriptionRef.update(state, (current) => ({ ...current, refreshing: false })); + const mapPersistence = (error: { readonly message: string }) => + new ConnectionBlockedError({ reason: "configuration", detail: error.message }); + + const refresh = lock.withPermits(1)( + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if (current.connection === null) return; + yield* SubscriptionRef.update(state, (value) => ({ + ...value, + refreshing: true, + error: null, + })); + const credential = yield* currentCredential(current.connection); + const directory = yield* requestDirectory(current.connection, credential); + yield* reconcile(current.connection, directory).pipe(Effect.mapError(mapPersistence)); + const inbox = yield* requestInbox(current.connection, credential); + yield* SubscriptionRef.update(state, (value) => ({ ...value, directory, inbox })); + }).pipe(Effect.tapError(reportError), Effect.ensuring(finish)), + ); + + const connect = (input: PairingConnectionInput) => + lock.withPermits(1)( + Effect.gen(function* () { + yield* SubscriptionRef.update(state, (value) => ({ + ...value, + refreshing: true, + error: null, + })); + const registration = yield* preparePairingRegistration(input).pipe( + Effect.provideService(ClientCapabilities.ClientPresentation, presentation), + Effect.provideService(HttpClient.HttpClient, http), + Effect.mapError((error) => + error._tag === "ConnectionBlockedError" && error.reason === "authentication" + ? new ConnectionBlockedError({ + reason: "authentication", + detail: + "The pairing code is invalid or expired. Create a new code on the server.", + }) + : error, + ), + ); + const profile = registration.profile; + const credential = registration.credential; + if (credential._tag !== "BearerConnectionCredential") + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "The pairing credential is invalid.", + }); + const directory = yield* requestDirectory(profile, credential); + inboxCache = undefined; + const inbox = yield* requestInbox(profile, credential); + const previous = (yield* SubscriptionRef.get(state)).connection; + yield* credentials.put(hubCredentialId(profile.environmentId), credential); + yield* reconcile(profile, directory).pipe(Effect.mapError(mapPersistence)); + yield* profiles.put( + new BearerConnectionProfile({ ...profile, connectionId: HUB_PROFILE_ID }), + ); + yield* SubscriptionRef.set(state, { + connection: profile, + directory, + inbox, + refreshing: false, + error: null, + }); + for (const machine of directory.environments) { + yield* registry.retryNow(machine.environmentId); + } + if (previous !== null && previous.environmentId !== profile.environmentId) { + yield* removeRoutes(previous.environmentId).pipe(Effect.mapError(mapPersistence)); + yield* credentials.remove(hubCredentialId(previous.environmentId)); + } + }).pipe(Effect.tapError(reportError), Effect.ensuring(finish)), + ); + + const connectAuthenticated = (input: { readonly httpBaseUrl: string }) => + lock.withPermits(1)( + Effect.gen(function* () { + const descriptor = yield* fetchRemoteEnvironmentDescriptor(input).pipe( + Effect.provideService(HttpClient.HttpClient, http), + Effect.mapError(mapPersistence), + ); + const credential = new HostedConnectionCredential({ serverUrl: input.httpBaseUrl }); + const profile = new BearerConnectionProfile({ + connectionId: HUB_PROFILE_ID, + authorization: "hosted", + environmentId: descriptor.environmentId, + label: descriptor.label, + httpBaseUrl: input.httpBaseUrl, + wsBaseUrl: deriveWsBaseUrl(input.httpBaseUrl), + }); + const directory = yield* requestDirectory(profile, credential); + inboxCache = undefined; + const inbox = yield* requestInbox(profile, credential); + const previous = (yield* SubscriptionRef.get(state)).connection; + const credentialId = hubCredentialId(profile.environmentId); + const previousCredential = yield* credentials.get(credentialId); + let credentialWritten = false; + yield* Effect.gen(function* () { + yield* credentials.put(credentialId, credential); + credentialWritten = true; + yield* reconcile(profile, directory).pipe(Effect.mapError(mapPersistence)); + yield* profiles.put(profile); + }).pipe( + Effect.tapError(() => + !credentialWritten + ? Effect.void + : Option.isSome(previousCredential) + ? credentials.put(credentialId, previousCredential.value) + : credentials.remove(credentialId), + ), + ); + + yield* SubscriptionRef.set(state, { + connection: profile, + directory, + inbox, + refreshing: false, + error: null, + }); + if (previous !== null && previous.environmentId !== profile.environmentId) { + yield* removeRoutes(previous.environmentId).pipe(Effect.mapError(mapPersistence)); + yield* credentials.remove(hubCredentialId(previous.environmentId)); + } + }).pipe(Effect.tapError(reportError), Effect.ensuring(finish)), + ); + + const disconnect = lock.withPermits(1)( + Effect.gen(function* () { + const current = (yield* SubscriptionRef.get(state)).connection; + if (current === null) return; + yield* removeRoutes(current.environmentId).pipe(Effect.mapError(mapPersistence)); + yield* profiles.remove(HUB_PROFILE_ID); + yield* credentials.remove(hubCredentialId(current.environmentId)); + inboxCache = undefined; + yield* SubscriptionRef.set(state, EMPTY_HUB_CONNECTION_STATE); + }).pipe(Effect.tapError(reportError)), + ); + + const refreshInbox = lock.withPermit( + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if (current.connection === null) return; + const credential = yield* currentCredential(current.connection); + const inbox = yield* requestInbox(current.connection, credential); + yield* SubscriptionRef.update(state, (value) => ({ ...value, inbox, error: null })); + }).pipe(Effect.tapError(reportError)), + ); + + const organize = (input: HubInboxCommand) => + lock.withPermit( + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if ( + current.connection === null || + !current.directory?.environments.some( + (entry) => entry.environmentId === input.environmentId, + ) + ) + return Option.none(); + const credential = yield* currentCredential(current.connection); + const encoded = yield* encodeInboxCommand(input); + const response = yield* http.post( + environmentEndpointUrl(current.connection.httpBaseUrl, "/api/hub/inbox"), + { + headers: yield* headersFor(credential), + body: HttpBody.jsonUnsafe(encoded), + }, + ); + if (response.status === 401 || response.status === 403) { + return yield* new ConnectionBlockedError({ + reason: "authentication", + detail: + "This server connection cannot change chats. Pair again with access to change threads.", + }); + } + if (response.status !== 200) { + return yield* new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "Couldn't save this chat change on the server. Try again.", + }); + } + const entry = yield* response.json.pipe(Effect.flatMap(decodeInboxEnvironment)); + if (entry.environmentId !== input.environmentId) { + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "The server returned a different machine. Refresh and try again.", + }); + } + inboxCache = undefined; + yield* SubscriptionRef.update(state, (value) => ({ + ...value, + inbox: { + environments: [ + ...value.inbox.environments.filter( + (item) => item.environmentId !== entry.environmentId, + ), + entry, + ], + }, + error: null, + })); + return Option.some({ sequence: entry.snapshot.snapshotSequence }); + }).pipe( + Effect.timeout("15 seconds"), + Effect.mapError((error) => + isConnectionAttemptError(error) + ? error + : new ConnectionTransientError({ + reason: "remote-unavailable", + detail: "Couldn't save this chat change on the server. Try again.", + }), + ), + ), + ); + + const headers = Effect.gen(function* () { + const connection = (yield* SubscriptionRef.get(state)).connection; + if (connection === null) + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "Connect your server to open archived chats.", + }); + return yield* headersFor(yield* currentCredential(connection)); + }); + + return HubConnection.of({ + state, + connect, + connectAuthenticated, + headers, + refresh, + refreshInbox, + organize, + disconnect, + }); +}); + +export const layer = Layer.effect(HubConnection, make); diff --git a/packages/client-runtime/src/connection/index.ts b/packages/client-runtime/src/connection/index.ts index 90881817c6420eebc0643bbdbfc4f1e373abe201..73ad1d242c6e73968a39271e44fddc932112f47d 100644 --- a/packages/client-runtime/src/connection/index.ts +++ b/packages/client-runtime/src/connection/index.ts @@ -1,4 +1,5 @@ export * from "./catalog.ts"; +export { HubConnection } from "./hub.ts"; export * as Connectivity from "./connectivity.ts"; export * as CredentialStore from "./credentialStore.ts"; export { type ConnectionDriverProgress, type EnvironmentConnectionLease } from "./driver.ts"; diff --git a/packages/client-runtime/src/connection/layer.ts b/packages/client-runtime/src/connection/layer.ts index 7c6d1d6e9a498d68bf604c23cd406c27bedd42b8..50fd51240be99e6e92b649d169951dbfe56615a8 100644 --- a/packages/client-runtime/src/connection/layer.ts +++ b/packages/client-runtime/src/connection/layer.ts @@ -15,6 +15,8 @@ import * as PlatformConnectionSource from "../platform/source.ts"; import * as RelayEnvironmentDiscovery from "../relay/discovery.ts"; import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; import * as RpcSession from "../rpc/session.ts"; +import * as Hub from "./hub.ts"; +import * as Wakeups from "./wakeups.ts"; export const watchDiscoveredCompatibility = Effect.fn("connection.watchDiscoveredCompatibility")( function* () { @@ -78,10 +80,12 @@ export function layerWithOptions(options: RpcSession.RpcSessionOptions) { ); const layerRegistry = EnvironmentRegistry.layer.pipe(Layer.provide(layerDriver)); const layerOnboarding = ConnectionOnboarding.layer.pipe(Layer.provide(layerRegistry)); + const layerHub = Hub.layer.pipe(Layer.provide(layerRegistry)); const layerConnectionServices = Layer.mergeAll( layerRegistry, RelayEnvironmentDiscovery.layer, layerOnboarding, + layerHub, // Exposed for updating hosts too old to connect through the driver. ConnectionResolver.layer, ); @@ -89,8 +93,19 @@ export function layerWithOptions(options: RpcSession.RpcSessionOptions) { Effect.gen(function* () { const registry = yield* EnvironmentRegistry.EnvironmentRegistry; const platformSource = yield* PlatformConnectionSource.PlatformConnectionSource; + const hub = yield* Hub.HubConnection; + const wakeups = yield* Wakeups.ConnectionWakeups; yield* watchDiscoveredCompatibility().pipe(Effect.forkScoped); yield* registry.start; + yield* Stream.merge(Stream.tick("1 minute"), wakeups.changes).pipe( + Stream.runForEach(() => hub.refresh.pipe(Effect.ignore)), + Effect.forkScoped, + ); + yield* hub.refresh.pipe(Effect.ignore, Effect.forkScoped); + yield* Stream.tick("15 seconds").pipe( + Stream.runForEach(() => hub.refreshInbox.pipe(Effect.ignore)), + Effect.forkScoped, + ); yield* platformSource.registrations.pipe( Stream.runForEach(registry.reconcilePlatform), Effect.forkScoped, diff --git a/packages/client-runtime/src/connection/onboarding.test.ts b/packages/client-runtime/src/connection/onboarding.test.ts index e53420659c0535e639fbf1a94e872f6e35c327d9..8babb9e61ed6bbaac779558bab9e916e985e9c6b 100644 --- a/packages/client-runtime/src/connection/onboarding.test.ts +++ b/packages/client-runtime/src/connection/onboarding.test.ts @@ -271,6 +271,35 @@ describe("connection onboarding", () => { }), ); + it.effect("keeps routes for different prefixes on the same hub distinct", () => + Effect.gen(function* () { + const calls: Array<{ readonly url: string; readonly init: RequestInit }> = []; + const registrations = []; + for (const name of ["one", "two"]) { + registrations.push( + yield* preparePairingRegistration({ + host: `https://hub.example/prefix/environments/${name}/`, + pairingCode: "pairing-token", + }).pipe(Effect.provide(Layer.mergeAll(layerClientPresentation, layerPairingHttp(calls)))), + ); + } + expect(registrations.map((registration) => registration.target.connectionId)).toEqual([ + "bearer:environment-paired:https://hub.example/prefix/environments/one", + "bearer:environment-paired:https://hub.example/prefix/environments/two", + ]); + expect(registrations[0]?.profile).toMatchObject({ + httpBaseUrl: "https://hub.example/prefix/environments/one/", + wsBaseUrl: "wss://hub.example/prefix/environments/one/", + }); + expect(calls.map((call) => call.url)).toEqual([ + "https://hub.example/prefix/environments/one/.well-known/t3/environment", + "https://hub.example/prefix/environments/one/oauth/token", + "https://hub.example/prefix/environments/two/.well-known/t3/environment", + "https://hub.example/prefix/environments/two/oauth/token", + ]); + }), + ); + it.effect("turns a switched-off environment back on when it is paired again", () => Effect.gen(function* () { const registry = yield* EnvironmentRegistry.EnvironmentRegistry; @@ -434,6 +463,9 @@ describe("connection onboarding", () => { pairingCode: "pairing-token", }).pipe(Effect.provide(Layer.mergeAll(layerClientPresentation, httpLayer))); + if (registration.credential._tag !== "BearerConnectionCredential") { + throw new Error("Pairing must issue a bearer credential."); + } const session = yield* fetchRemoteSessionState({ httpBaseUrl: registration.profile.httpBaseUrl, bearerToken: registration.credential.token, @@ -589,8 +621,8 @@ describe("connection onboarding", () => { profile: { environmentId, label: "Renamed environment", - httpBaseUrl: "http://100.65.180.100:3773/", - wsBaseUrl: "ws://100.65.180.100:3773/", + httpBaseUrl: "http://100.65.180.100:3773/path/", + wsBaseUrl: "ws://100.65.180.100:3773/path/", }, credential: { token: "bearer-token" }, }); diff --git a/packages/client-runtime/src/connection/onboarding.ts b/packages/client-runtime/src/connection/onboarding.ts index a6ac9aa556e3ca03523db64ccd0c2c057d994556..dd22e191e466bd80f092de54c629783244312d55 100644 --- a/packages/client-runtime/src/connection/onboarding.ts +++ b/packages/client-runtime/src/connection/onboarding.ts @@ -98,7 +98,8 @@ const resolvePairingTarget = Effect.fn("clientRuntime.connection.onboarding.reso * reuses the id and replaces that route. */ function bearerConnectionId(environmentId: EnvironmentId, httpBaseUrl: string): string { - return `bearer:${environmentId}:${new URL(httpBaseUrl).origin}`; + const url = new URL(normalizeHttpBaseUrl(httpBaseUrl)); + return `bearer:${environmentId}:${url.origin}${url.pathname.replace(/\/+$/, "")}`; } function differentMachineError(label: string) { diff --git a/packages/client-runtime/src/connection/presentation.test.ts b/packages/client-runtime/src/connection/presentation.test.ts index 61d6278a5305158a80cc8ff97aadbe69ff991f38..107a65b55ffb9f918376a8084a35d19cdea7d37e 100644 --- a/packages/client-runtime/src/connection/presentation.test.ts +++ b/packages/client-runtime/src/connection/presentation.test.ts @@ -135,6 +135,22 @@ describe("connection presentation", () => { ).toBe("https://tunnel.example.test/mcp"); }); + it("keeps the selected environment prefix in MCP addresses", () => { + const entry = { + ...ENTRY, + profile: Option.some( + new BearerConnectionProfile({ + connectionId: TARGET.connectionId, + environmentId: TARGET.environmentId, + label: TARGET.label, + httpBaseUrl: "https://hub.example/prefix/environments/one/", + wsBaseUrl: "wss://hub.example/prefix/environments/one/", + }), + ), + }; + expect(environmentMcpUrl({ entry })).toBe("https://hub.example/prefix/environments/one/mcp"); + }); + it("distinguishes initial connection, reconnect, and retry errors", () => { expect(presentConnectionState(supervisorState({ phase: "connecting", attempt: 1 }))).toEqual({ phase: "connecting", diff --git a/packages/client-runtime/src/connection/presentation.ts b/packages/client-runtime/src/connection/presentation.ts index 9015a755133e5bc4d9740596d5ce81e76e165556..6dff002b887f56e3a6366b0d87b48f91e09c1198 100644 --- a/packages/client-runtime/src/connection/presentation.ts +++ b/packages/client-runtime/src/connection/presentation.ts @@ -1,6 +1,7 @@ import type { ServerConfig } from "@t3tools/contracts"; import * as Option from "effect/Option"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; import type { ConnectionCatalogEntry } from "./catalog.ts"; import type { ConnectionTarget, SupervisorConnectionState } from "./model.ts"; import { connectionRouteId, connectionRoutes, routeHttpBaseUrl } from "./routes.ts"; @@ -122,16 +123,11 @@ export function environmentMcpUrl(input: { } function mcpUrlFromBase(httpBaseUrl: string): string | null { - let url: URL; try { - url = new URL(httpBaseUrl); + return environmentEndpointUrl(httpBaseUrl, "/mcp"); } catch { return null; } - url.pathname = "/mcp"; - url.search = ""; - url.hash = ""; - return url.toString(); } export function connectionCatalogDisplayUrl(entry: ConnectionCatalogEntry): string | null { diff --git a/packages/client-runtime/src/connection/registry.test.ts b/packages/client-runtime/src/connection/registry.test.ts index 18d1f062f04394668f61a0dc9ba719ac08aba154..c790197eba0465b1ff381ef1bcd223ff6db574b3 100644 --- a/packages/client-runtime/src/connection/registry.test.ts +++ b/packages/client-runtime/src/connection/registry.test.ts @@ -393,9 +393,10 @@ const makeHarness = Effect.fn("TestEnvironmentRegistry.makeHarness")(function* ( ...PREPARED, environmentId: target.environmentId, label: target.label, - httpAuthorization: Option.isSome(credential) - ? { _tag: "Bearer", token: credential.value.token } - : null, + httpAuthorization: + Option.isSome(credential) && credential.value._tag === "BearerConnectionCredential" + ? { _tag: "Bearer", token: credential.value.token } + : null, target, }; if (options?.prepareError) return yield* options.prepareError; @@ -1375,11 +1376,13 @@ describe("EnvironmentRegistry", () => { target._tag !== "BearerConnectionTarget" || storedCredentials === undefined ? Effect.succeed(undefined) : Ref.get(storedCredentials).pipe( - Effect.map((current) => - current.get(target.connectionId)?.token === "old-bearer" + Effect.map((current) => { + const credential = current.get(target.connectionId); + return credential?._tag === "BearerConnectionCredential" && + credential.token === "old-bearer" ? new ConnectionBlockedError({ reason: "authentication", detail: "revoked" }) - : undefined, - ), + : undefined; + }), ), }); storedCredentials = harness.storedCredentials; diff --git a/packages/client-runtime/src/connection/registry.ts b/packages/client-runtime/src/connection/registry.ts index 501a19bb0b9e0be7533bcdfc0746a5f41b1b6ef9..a30e53075b3e633b777fff4534b75c3c87659583 100644 --- a/packages/client-runtime/src/connection/registry.ts +++ b/packages/client-runtime/src/connection/registry.ts @@ -126,6 +126,7 @@ export class EnvironmentRegistry extends Context.Service< readonly removeRoute: ( environmentId: EnvironmentId, routeId: string, + options?: { readonly retainDrafts?: boolean }, ) => Effect.Effect< void, | Persistence.ConnectionPersistenceError @@ -728,7 +729,15 @@ export const make = Effect.gen(function* () { state.lastFailure.reason === "authentication") ) { yield* scope.supervisor.retryNow; - } else if (state.phase === "connected") { + } else if ( + state.phase === "connected" && + registration.credential._tag === "HubConnectionCredential" + ) { + yield* scope.supervisor.retryNow; + } else if ( + state.phase === "connected" && + registration.credential._tag === "BearerConnectionCredential" + ) { const token = registration.credential.token; yield* SubscriptionRef.updateSome(scope.supervisor.prepared, (current) => Option.isSome(current) && @@ -844,6 +853,7 @@ export const make = Effect.gen(function* () { /** Forgets a user-saved environment. Callers hold its lease lock. */ const removeLocked = Effect.fn("EnvironmentRegistry.removeLocked")(function* ( environmentId: EnvironmentId, + options?: { readonly retainDrafts?: boolean }, ) { if ((yield* Ref.get(platformEnvironmentIds)).has(environmentId)) { return yield* new PlatformEnvironmentRemovalError({ @@ -875,7 +885,7 @@ export const make = Effect.gen(function* () { }), ), ), - ownedDataCleanup.clear(environmentId), + ownedDataCleanup.clear(environmentId, options), ], { concurrency: "unbounded", discard: true }, ); @@ -918,6 +928,10 @@ export const make = Effect.gen(function* () { readonly activeRoute: ConnectionRoute; readonly reported: ReadonlyArray; }) { + if (connectionRouteId(input.activeRoute.target).startsWith("hub:")) { + return Option.none(); + } + return yield* withLeaseLock( input.environmentId, Effect.gen(function* () { @@ -980,6 +994,7 @@ export const make = Effect.gen(function* () { const removeRoute = Effect.fn("EnvironmentRegistry.removeRoute")(function* ( environmentId: EnvironmentId, routeId: string, + options?: { readonly retainDrafts?: boolean }, ) { // One lock for the whole decision: a route registered between "this was // the last route" and the removal must not be deleted with it. @@ -991,7 +1006,7 @@ export const make = Effect.gen(function* () { const route = routes.find((candidate) => connectionRouteId(candidate.target) === routeId); if (route === undefined) return; const remaining = routesAfterRemoving(routes, routeId); - if (remaining.length === 0) return yield* removeLocked(environmentId); + if (remaining.length === 0) return yield* removeLocked(environmentId, options); yield* replaceRoutesLocked(entry, remaining); const profile = Option.getOrNull(route.profile); if (profile !== null && isSshConnectionProfile(profile)) { diff --git a/packages/client-runtime/src/connection/resolver.test.ts b/packages/client-runtime/src/connection/resolver.test.ts index 0d40db307c1f0a5bae88b0e91d91ceb4da70468e..5b73043bcaa02438d7ef095f9d4b3167bf38d88f 100644 --- a/packages/client-runtime/src/connection/resolver.test.ts +++ b/packages/client-runtime/src/connection/resolver.test.ts @@ -17,6 +17,7 @@ import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; import { BearerConnectionCredential, BearerConnectionProfile, + HubConnectionCredential, type ConnectionCatalogEntry, SshConnectionProfile, type ConnectionCredential, @@ -186,6 +187,35 @@ const makeDependencies = Effect.fn("TestConnectionResolver.makeDependencies")((o }); describe("ConnectionResolver", () => { + it.effect("resolves hub routes through one privately stored credential", () => + Effect.gen(function* () { + const connectionId = "hub:hub:environment-1"; + const profile = new BearerConnectionProfile({ + connectionId, + environmentId: ENVIRONMENT_ID, + label: "Machine", + httpBaseUrl: "https://hub.example/environments/environment-1/", + wsBaseUrl: "wss://hub.example/environments/environment-1/", + }); + const dependencies = yield* makeDependencies({ + credentials: [ + [connectionId, new HubConnectionCredential({ connectionId: "hub-session:hub" })], + ["hub-session:hub", new BearerConnectionCredential({ token: "single-hub-secret" })], + ], + }); + const resolver = yield* ConnectionResolver.ConnectionResolver.pipe( + Effect.provide(dependencies), + ); + const target = new BearerConnectionTarget({ + connectionId, + environmentId: ENVIRONMENT_ID, + label: "Machine", + }); + const prepared = yield* resolver.prepare(catalogEntry(target, Option.some(profile))); + expect(prepared.httpAuthorization).toEqual({ _tag: "Bearer", token: "single-hub-secret" }); + expect(prepared.httpBaseUrl).toBe(profile.httpBaseUrl); + }), + ); it.effect("blocks an old host during discovery before opening orchestration RPC", () => Effect.gen(function* () { const layerBroker = yield* makeDependencies({ descriptorProtocolVersion: null }); diff --git a/packages/client-runtime/src/connection/resolver.ts b/packages/client-runtime/src/connection/resolver.ts index a70a7531a79955500a6f289c4bbb06fd4ce9b4a4..02b4042b4a075331df783aa238b331e5b10ea283 100644 --- a/packages/client-runtime/src/connection/resolver.ts +++ b/packages/client-runtime/src/connection/resolver.ts @@ -45,6 +45,7 @@ import { orchestrationProtocolCompatibilityError, } from "./compatibility.ts"; import { credentialConnectionId } from "./routes.ts"; +import { deriveWsBaseUrl, environmentWebSocketUrl } from "../environment/endpoint.ts"; import { fetchRemoteEnvironmentDescriptor } from "../environment/descriptor.ts"; export class ConnectionResolver extends Context.Service< @@ -72,13 +73,10 @@ const isSshProfile = Schema.is(SshConnectionProfile); const isBearerCredential = Schema.is(BearerConnectionCredential); function primarySocketUrl( - target: PrimaryConnectionTarget, + target: { readonly wsBaseUrl: string }, clientMetadata: AuthClientPresentationMetadata | undefined, ): string { - const url = new URL(target.wsBaseUrl); - if (url.pathname === "" || url.pathname === "/") { - url.pathname = "/ws"; - } + const url = new URL(environmentWebSocketUrl(target.wsBaseUrl)); appendClientConnectionParams(url, clientMetadata, "direct"); return url.toString(); } @@ -120,6 +118,7 @@ const makePrimaryBroker = Effect.fn("clientRuntime.connection.broker.makePrimary const makeBearerBroker = Effect.fn("clientRuntime.connection.broker.makeBearer")(function* () { const credentials = yield* ConnectionCredentialStore.ConnectionCredentialStore; const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization; + const presentation = yield* ClientCapabilities.ClientPresentation; return Effect.fn("clientRuntime.connection.broker.bearer")(function* ( entry: ConnectionCatalogEntry & { readonly target: BearerConnectionTarget }, @@ -148,15 +147,60 @@ const makeBearerBroker = Effect.fn("clientRuntime.connection.broker.makeBearer") return { ...authorized, target } satisfies PreparedConnection; } // A learned route borrows the credential of the route it was learned from. - const credential = yield* credentials.get(credentialConnectionId(target.connectionId)).pipe( - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(credentialMissingError(target.connectionId)), - onSome: Effect.succeed, - }), - ), - ); - if (!isBearerCredential(credential)) { + const savedCredential = yield* credentials + .get(credentialConnectionId(target.connectionId)) + .pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.fail(credentialMissingError(target.connectionId)), + onSome: Effect.succeed, + }), + ), + ); + const credential = + savedCredential._tag === "HubConnectionCredential" + ? Option.getOrNull(yield* credentials.get(savedCredential.connectionId)) + : savedCredential; + if (credential?._tag === "HostedConnectionCredential") { + const service = yield* Effect.serviceOption(ClientCapabilities.HostedAuthAccess); + if (Option.isNone(service)) return yield* credentialMissingError(target.connectionId); + const destination = new URL(profile.httpBaseUrl); + const server = new URL(credential.serverUrl); + const prefix = server.pathname.replace(/\/+$/, ""); + if ( + destination.origin !== server.origin || + (destination.pathname !== prefix && !destination.pathname.startsWith(`${prefix}/`)) || + profile.wsBaseUrl !== deriveWsBaseUrl(profile.httpBaseUrl) + ) { + return yield* new ConnectionBlockedError({ + reason: "configuration", + detail: "This connection belongs to a different server. Sign in again.", + }); + } + const headers = yield* service.value.headers(credential.serverUrl); + const bearer = headers.authorization?.replace(/^Bearer /, ""); + if (bearer) { + return { + ...(yield* remote.authorizeBearer({ + expectedEnvironmentId: target.environmentId, + httpBaseUrl: profile.httpBaseUrl, + wsBaseUrl: profile.wsBaseUrl, + bearerToken: bearer, + connectionMethod: "direct", + })), + target, + } satisfies PreparedConnection; + } + return { + environmentId: target.environmentId, + label: profile.label, + httpBaseUrl: profile.httpBaseUrl, + socketUrl: primarySocketUrl(profile, presentation.metadata), + httpAuthorization: null, + target, + } satisfies PreparedConnection; + } + if (credential === null || !isBearerCredential(credential)) { return yield* credentialMissingError(target.connectionId); } const authorized = yield* remote.authorizeBearer({ diff --git a/packages/client-runtime/src/connection/routes.test.ts b/packages/client-runtime/src/connection/routes.test.ts index 3079294305c74b2c637ba704620070381d944a29..f9897a015e0c142db1d8894665f65a920a1e1bb8 100644 --- a/packages/client-runtime/src/connection/routes.test.ts +++ b/packages/client-runtime/src/connection/routes.test.ts @@ -104,6 +104,19 @@ describe("learned routes", () => { }); }); + it("retains distinct learned environment bases on the same origin", () => { + const reported = [ + { httpBaseUrl: "https://hub.example/prefix/environments/one/" }, + { httpBaseUrl: "https://hub.example/prefix/environments/two/" }, + ]; + const routes = mergeLearnedRoutes({ entry: relayOnly, activeRoute: RELAY, reported, allowInsecure: false })!; + expect(routes.slice(0, 2).map((route) => profileOf(route))).toMatchObject([ + { httpBaseUrl: reported[0]!.httpBaseUrl, wsBaseUrl: "wss://hub.example/prefix/environments/one/" }, + { httpBaseUrl: reported[1]!.httpBaseUrl, wsBaseUrl: "wss://hub.example/prefix/environments/two/" }, + ]); + expect(mergeLearnedRoutes({ entry: entryWithRoutes(relayOnly, routes), activeRoute: RELAY, reported, allowInsecure: false })).toBeNull(); + }); + it("replaces a learned LAN address when the server reports a new one", () => { const first = mergeLearnedRoutes({ entry: relayOnly, diff --git a/packages/client-runtime/src/connection/routes.ts b/packages/client-runtime/src/connection/routes.ts index 4691aa9389d8cb40c775a51de5e0c5821993ea6a..3a25da2bc41fecfa2c0407bddb83f59ddafab827 100644 --- a/packages/client-runtime/src/connection/routes.ts +++ b/packages/client-runtime/src/connection/routes.ts @@ -4,6 +4,7 @@ import { isSharedAddressSpaceHost, isTailnetHost, } from "@t3tools/shared/hostClassification"; +import { deriveWsBaseUrl, normalizeHttpBaseUrl } from "../environment/endpoint.ts"; import type { DesktopSshEnvironmentTarget } from "@t3tools/contracts"; import * as Option from "effect/Option"; @@ -261,13 +262,13 @@ export function mergeLearnedRoutes(input: { ? credentialConnectionId(active.connectionId) : undefined; - // Usable reported addresses, by origin. + // Usable reported environment bases. const reported = new Map(); const tailscaleOrigins = new Set(); for (const endpoint of input.reported) { let url: URL; try { - url = new URL(endpoint.httpBaseUrl); + url = new URL(normalizeHttpBaseUrl(endpoint.httpBaseUrl)); } catch { continue; } @@ -275,8 +276,9 @@ export function mergeLearnedRoutes(input: { if (url.protocol === "http:" && !input.allowInsecure) continue; // A loopback address names whichever device opens it, never the server. if (isLocalLoopbackHost(url.hostname)) continue; - reported.set(url.origin, url); - if (endpoint.kind === "tailnet") tailscaleOrigins.add(url.origin); + const address = url.href.replace(/\/+$/, ""); + reported.set(address, url); + if (endpoint.kind === "tailnet") tailscaleOrigins.add(address); } const normalized = (url: string) => url.replace(/\/+$/, ""); const known = new Set( @@ -300,11 +302,12 @@ export function mergeLearnedRoutes(input: { }); let next: ReadonlyArray = kept; for (const url of reported.values()) { - if (known.has(url.origin)) continue; - const httpBaseUrl = `${url.origin}/`; + const httpBaseUrl = url.href; + const address = httpBaseUrl.replace(/\/+$/, ""); + if (known.has(address)) continue; const connectionId = learnedConnectionId( entry.target.environmentId, - url.origin, + address, sharedCredential, ); next = insertRoute(next, { @@ -319,10 +322,10 @@ export function mergeLearnedRoutes(input: { environmentId: entry.target.environmentId, label: entry.target.label, httpBaseUrl, - wsBaseUrl: `${url.protocol === "https:" ? "wss:" : "ws:"}//${url.host}/`, + wsBaseUrl: deriveWsBaseUrl(httpBaseUrl), learned: true, ...(authorization === undefined ? {} : { authorization }), - ...(tailscaleOrigins.has(url.origin) ? { network: "tailscale" as const } : {}), + ...(tailscaleOrigins.has(address) ? { network: "tailscale" as const } : {}), }), ), }); diff --git a/packages/client-runtime/src/environment/endpoint.test.ts b/packages/client-runtime/src/environment/endpoint.test.ts index d26201dc4f72c688b9eeb8e28dc6d1d1836da943..3d26b7cdee01fbc94100549476f1b14bc52e8167 100644 --- a/packages/client-runtime/src/environment/endpoint.test.ts +++ b/packages/client-runtime/src/environment/endpoint.test.ts @@ -4,6 +4,8 @@ import { classifyHostedHttpsCompatibility, createAdvertisedEndpoint, deriveWsBaseUrl, + environmentEndpointUrl, + environmentWebSocketUrl, normalizeHttpBaseUrl, } from "./endpoint.ts"; @@ -16,12 +18,33 @@ const coreProvider = { describe("advertised endpoint helpers", () => { it("normalizes HTTP and WebSocket base URLs", () => { - expect(normalizeHttpBaseUrl("https://example.com/path?x=1#hash")).toBe("https://example.com/"); - expect(normalizeHttpBaseUrl("wss://example.com/socket")).toBe("https://example.com/"); - expect(deriveWsBaseUrl("https://example.com/api")).toBe("wss://example.com/"); + expect(normalizeHttpBaseUrl("https://example.com/path?x=1#hash")).toBe( + "https://example.com/path/", + ); + expect(normalizeHttpBaseUrl("wss://example.com/socket")).toBe("https://example.com/socket/"); + expect(deriveWsBaseUrl("https://example.com/api")).toBe("wss://example.com/api/"); expect(deriveWsBaseUrl("http://127.0.0.1:3773")).toBe("ws://127.0.0.1:3773/"); }); + it.each(["https", "http", "wss", "ws"])("preserves the base path for %s routes", (scheme) => { + const base = `${scheme}://hub.example/prefix/environments/one///?old=1#fragment`; + expect(environmentEndpointUrl(base, "/api/assets/a%2Fb?signature=abc#preview")).toBe( + `${scheme}://hub.example/prefix/environments/one/api/assets/a%2Fb?signature=abc#preview`, + ); + expect(environmentEndpointUrl(base, ".well-known/t3/environment")).toBe( + `${scheme}://hub.example/prefix/environments/one/.well-known/t3/environment`, + ); + }); + + it("distinguishes a canonical WebSocket base from an existing socket endpoint", () => { + expect(environmentWebSocketUrl("wss://hub.example/prefix/environments/one/")).toBe( + "wss://hub.example/prefix/environments/one/ws", + ); + expect(environmentWebSocketUrl("wss://remote.example/ws?transport=1")).toBe( + "wss://remote.example/ws?transport=1", + ); + }); + it("marks HTTP endpoints as blocked from hosted HTTPS apps", () => { expect(classifyHostedHttpsCompatibility("http://192.168.1.44:3773")).toBe( "mixed-content-blocked", diff --git a/packages/client-runtime/src/environment/endpoint.ts b/packages/client-runtime/src/environment/endpoint.ts index 4178259361e79940e729750415db285a18dee6d1..da7d766fa8066e9a621c5cfe2f5ff54ca0e1e2ad 100644 --- a/packages/client-runtime/src/environment/endpoint.ts +++ b/packages/client-runtime/src/environment/endpoint.ts @@ -1,9 +1 @@ export * from "@t3tools/shared/advertisedEndpoint"; - -export const environmentEndpointUrl = (httpBaseUrl: string, pathname: string): string => { - const url = new URL(httpBaseUrl); - url.pathname = pathname; - url.search = ""; - url.hash = ""; - return url.toString(); -}; diff --git a/packages/client-runtime/src/operations/commands.ts b/packages/client-runtime/src/operations/commands.ts index 83ca6b4f663a823374e6d7f84cfc9a359e7d0b58..bce6b98c421c4af2efc68e7ecbd632427e21b85f 100644 --- a/packages/client-runtime/src/operations/commands.ts +++ b/packages/client-runtime/src/operations/commands.ts @@ -4,6 +4,7 @@ import { CommandId, CheckpointId, CheckpointScopeId, + HubOrganizationCommand, ORCHESTRATION_V2_WS_METHODS, OrchestrationV2CheckpointUnavailableError, WS_METHODS, @@ -32,9 +33,15 @@ import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2Pend import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Hub from "../connection/hub.ts"; +import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import { getInitialServerConfig, request } from "../rpc/client.ts"; +const isHubOrganizationCommand = Schema.is(HubOrganizationCommand); + interface CommandMetadata { readonly commandId?: CommandId; readonly createdAt?: string; @@ -270,8 +277,22 @@ const allocateCommandId = Effect.fn("EnvironmentCommands.allocateCommandId")(fun return CommandId.make(yield* crypto.randomUUIDv4().pipe(Effect.orDie)); }); -const dispatch = (command: OrchestrationV2Command) => - request(ORCHESTRATION_V2_WS_METHODS.dispatchCommand, command); +const dispatch = Effect.fn("EnvironmentCommands.dispatch")(function* ( + command: OrchestrationV2Command, +) { + if (isHubOrganizationCommand(command)) { + const hub = yield* Effect.serviceOption(Hub.HubConnection); + if (Option.isSome(hub)) { + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; + const result = yield* hub.value.organize({ + environmentId: supervisor.target.environmentId, + command, + }); + if (Option.isSome(result)) return result.value; + } + } + return yield* request(ORCHESTRATION_V2_WS_METHODS.dispatchCommand, command); +}); const getProjection = (threadId: ThreadId) => request(ORCHESTRATION_V2_WS_METHODS.getThreadProjection, { threadId }); diff --git a/packages/client-runtime/src/platform/capabilities.ts b/packages/client-runtime/src/platform/capabilities.ts index 2de4522a592ec72eaf32ec46a9c472ce53b7270c..6f4d20341a87297dd9315f3723511f7f9f5b522e 100644 --- a/packages/client-runtime/src/platform/capabilities.ts +++ b/packages/client-runtime/src/platform/capabilities.ts @@ -76,3 +76,12 @@ export class SshEnvironmentGateway extends Context.Service< ) => Effect.Effect; } >()("@t3tools/client-runtime/platform/capabilities/SshEnvironmentGateway") {} + +export class HostedAuthAccess extends Context.Service< + HostedAuthAccess, + { + readonly headers: ( + serverUrl: string, + ) => Effect.Effect, ConnectionAttemptError>; + } +>()("@t3tools/client-runtime/platform/capabilities/HostedAuthAccess") {} diff --git a/packages/client-runtime/src/platform/persistence.ts b/packages/client-runtime/src/platform/persistence.ts index 29463ce6da585fb9219058fbfcf0634de7bd44d3..a929a29118dbeafab0afe84bf2ee57f6893c2bf0 100644 --- a/packages/client-runtime/src/platform/persistence.ts +++ b/packages/client-runtime/src/platform/persistence.ts @@ -146,7 +146,10 @@ export class EnvironmentCacheStore extends Context.Service< >()("@t3tools/client-runtime/platform/persistence/EnvironmentCacheStore") {} export class EnvironmentOwnedDataCleanup extends Context.Reference<{ - readonly clear: (environmentId: EnvironmentId) => Effect.Effect; + readonly clear: ( + environmentId: EnvironmentId, + options?: { readonly retainDrafts?: boolean }, + ) => Effect.Effect; }>("@t3tools/client-runtime/platform/persistence/EnvironmentOwnedDataCleanup", { defaultValue: () => ({ clear: () => Effect.void, diff --git a/packages/client-runtime/src/rpc/http.ts b/packages/client-runtime/src/rpc/http.ts index bccd0ef516060e5b16ebaacb3ed9d8faf681c359..64423edcb4665d4da6a69a32f37a044a81738fad 100644 --- a/packages/client-runtime/src/rpc/http.ts +++ b/packages/client-runtime/src/rpc/http.ts @@ -8,6 +8,7 @@ import { type EnvironmentResourceNotFoundError, type EnvironmentScopeRequiredError, } from "@t3tools/contracts"; +import { normalizeHttpBaseUrl } from "@t3tools/shared/advertisedEndpoint"; import * as HttpObservability from "@t3tools/shared/httpObservability"; import * as Data from "effect/Data"; import * as Duration from "effect/Duration"; @@ -86,17 +87,9 @@ export const layerRemoteHttpClient = ( HttpObservability.layer, ); -const remoteApiBaseUrl = (httpBaseUrl: string): string => { - const url = new URL(httpBaseUrl); - url.pathname = "/"; - url.search = ""; - url.hash = ""; - return url.toString(); -}; - export const makeEnvironmentHttpApiClient = (httpBaseUrl: string) => HttpApiClient.make(EnvironmentHttpApi, { - baseUrl: remoteApiBaseUrl(httpBaseUrl), + baseUrl: normalizeHttpBaseUrl(httpBaseUrl), }); export const makeEnvironmentHttpApiGroupClient = < @@ -109,14 +102,14 @@ export const makeEnvironmentHttpApiGroupClient = < HttpApiClient.group(EnvironmentHttpApi, { httpClient, group, - baseUrl: remoteApiBaseUrl(httpBaseUrl), + baseUrl: normalizeHttpBaseUrl(httpBaseUrl), }), ); /** Contract-derived request URLs for authentication proofs, tracing, and structured errors. */ export const makeEnvironmentHttpApiUrlBuilder = (httpBaseUrl: string) => HttpApiClient.urlBuilder(EnvironmentHttpApi, { - baseUrl: remoteApiBaseUrl(httpBaseUrl), + baseUrl: normalizeHttpBaseUrl(httpBaseUrl), }); const failRemoteRequest = ( diff --git a/packages/client-runtime/src/state/archiveAssetHttp.ts b/packages/client-runtime/src/state/archiveAssetHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..bba590a5711eadc9b6d0c9ce1dff15188062a65f --- /dev/null +++ b/packages/client-runtime/src/state/archiveAssetHttp.ts @@ -0,0 +1,41 @@ +import { + ArchiveAssetCreateUrlInput, + ArchiveError, + AssetCreateUrlResult, + type AssetResource, + type EnvironmentId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import * as HubConnection from "../connection/hub.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; + +const encodeInput = Schema.encodeEffect(Schema.toCodecJson(ArchiveAssetCreateUrlInput)); +export const fetchArchivedAssetUrl = Effect.fn("clientRuntime.state.fetchArchivedAssetUrl")( + function* (environmentId: EnvironmentId, resource: AssetResource) { + const hub = yield* HubConnection.HubConnection; + const connection = (yield* SubscriptionRef.get(hub.state)).connection; + if (!connection) return yield* new ArchiveError({ reason: "unavailable" }); + const http = yield* HttpClient.HttpClient; + const result = yield* HttpClientRequest.post( + `${connection.httpBaseUrl}/api/archive/assets`, + ).pipe( + HttpClientRequest.setHeaders(yield* hub.headers), + HttpClientRequest.bodyJson(yield* encodeInput({ environmentId, resource })), + Effect.flatMap(http.execute), + Effect.provideService(FetchHttpClient.RequestInit, { + credentials: "include", + redirect: "error", + }), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(AssetCreateUrlResult)), + Effect.timeout("10 seconds"), + ); + return { + ...result, + relativeUrl: environmentEndpointUrl(connection.httpBaseUrl, result.relativeUrl), + }; + }, +); diff --git a/packages/client-runtime/src/state/archiveThreadHttp.ts b/packages/client-runtime/src/state/archiveThreadHttp.ts new file mode 100644 index 0000000000000000000000000000000000000000..bcb66475234c50968cc119b785ec0bb3b3f98ec5 --- /dev/null +++ b/packages/client-runtime/src/state/archiveThreadHttp.ts @@ -0,0 +1,38 @@ +import { + ArchiveError, + OrchestrationV2ThreadDetailSnapshot, + type EnvironmentId, + type ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import * as HubConnection from "../connection/hub.ts"; + +export const fetchArchivedThreadSnapshot = Effect.fn( + "clientRuntime.state.fetchArchivedThreadSnapshot", +)(function* (environmentId: EnvironmentId, threadId: ThreadId) { + const hub = yield* HubConnection.HubConnection; + const connection = (yield* SubscriptionRef.get(hub.state)).connection; + if (!connection) return yield* new ArchiveError({ reason: "unavailable" }); + const http = yield* HttpClient.HttpClient; + return yield* HttpClientRequest.get( + `${connection.httpBaseUrl}/api/archive/catalog/${encodeURIComponent(environmentId)}/${encodeURIComponent(threadId)}`, + ).pipe( + HttpClientRequest.setHeaders(yield* hub.headers), + http.execute, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap( + HttpClientResponse.schemaBodyJson(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)), + ), + Effect.filterOrFail( + (snapshot) => + snapshot.projection.thread.id === threadId && + snapshot.projection.thread.archivedAt !== null, + () => new ArchiveError({ reason: "integrity" }), + ), + Effect.timeout("10 seconds"), + ); +}); diff --git a/packages/client-runtime/src/state/assets.test.ts b/packages/client-runtime/src/state/assets.test.ts index c572f51370e8d677955f6b9702d1f00451d0b751..61de79f7936c9905e3171e714c0c20102e559abc 100644 --- a/packages/client-runtime/src/state/assets.test.ts +++ b/packages/client-runtime/src/state/assets.test.ts @@ -11,6 +11,8 @@ import { ThreadId, WS_METHODS, } from "@t3tools/contracts"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import * as HubConnection from "../connection/hub.ts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Stream from "effect/Stream"; @@ -29,14 +31,29 @@ import { import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import type { RpcSession } from "../rpc/session.ts"; import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; +import { createOrchestrationEnvironmentAtoms } from "./orchestration.ts"; +import { v2Project, v2ThreadShell, v2Now } from "./orchestrationV2TestFixtures.ts"; import { createProjectFaviconCache } from "../projectFaviconCache.ts"; import { createAssetEnvironmentAtoms, createProjectFaviconUrlAtomFamily, InvalidAssetCollectionKeyError, parseAssetCollectionKey, + resolveAssetUrl, } from "./assets.ts"; +describe("asset addresses", () => { + it("resolves signed environment paths through the proxy while retaining external assets", () => { + const base = "https://hub.example/prefix/environments/one/"; + expect(resolveAssetUrl(base, "/api/assets/a%2Fb?signature=signed")).toBe( + `${base}api/assets/a%2Fb?signature=signed`, + ); + expect(resolveAssetUrl(base, "https://other.example/api/assets/signed")).toBe( + "https://other.example/api/assets/signed", + ); + }); +}); + describe("asset collection keys", () => { it("preserves malformed JSON and its native cause", () => { const key = "not-json"; @@ -60,6 +77,118 @@ describe("asset collection keys", () => { }); describe("createAssetEnvironmentAtoms", () => { + it.effect("issues NAS attachment URLs without waiting for a sleeping source session", () => + Effect.gen(function* () { + const environmentId = EnvironmentId.make("sleeping-laptop"); + const resource = { + _tag: "attachment" as const, + attachmentId: "stored-image", + fileName: "picture.png", + }; + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: new PrimaryConnectionTarget({ + environmentId, + label: "Laptop", + httpBaseUrl: "https://source.example.test", + wsBaseUrl: "wss://source.example.test", + }), + state: yield* SubscriptionRef.make({ + ...AVAILABLE_CONNECTION_STATE, + phase: "offline", + }), + session: yield* SubscriptionRef.make(Option.none()), + prepared: yield* SubscriptionRef.make(Option.none()), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + }); + const environments = EnvironmentRegistry.EnvironmentRegistry.of({ + run: (_id, effect) => + Effect.provideService(effect, EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + followStream: (_id, stream) => + Stream.provideService(stream, EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + } as EnvironmentRegistry.EnvironmentRegistry["Service"]); + const hubState = yield* SubscriptionRef.make({ + ...HubConnection.EMPTY_HUB_CONNECTION_STATE, + inbox: { + environments: [ + { + environmentId, + pendingThreadIds: [], + snapshot: { + schemaVersion: 2, + snapshotSequence: 17, + projects: [v2Project], + threads: [], + archivedThreads: [{ ...v2ThreadShell, archivedAt: v2Now }], + }, + }, + ], + }, + connection: { + environmentId: EnvironmentId.make("nas"), + label: "Archive", + httpBaseUrl: "https://archive.example.test", + }, + }); + const hub = HubConnection.HubConnection.of({ + state: hubState, + headers: Effect.succeed({ authorization: "Bearer ephemeral" }), + connect: () => Effect.void, + connectAuthenticated: () => Effect.void, + refresh: Effect.void, + refreshInbox: Effect.void, + organize: () => Effect.succeedNone, + disconnect: Effect.void, + }); + const client = HttpClient.make((request) => + Effect.sync(() => { + expect(request.method).toBe("POST"); + expect(request.url).toBe("https://archive.example.test/api/archive/assets"); + expect(request.headers.authorization).toBe("Bearer ephemeral"); + if (request.body._tag !== "Uint8Array") throw new Error("Expected JSON body"); + expect(JSON.parse(new TextDecoder().decode(request.body.body))).toEqual({ + environmentId, + resource, + }); + return HttpClientResponse.fromWeb( + request, + Response.json({ + relativeUrl: "/api/archive/assets/signed/picture.png", + expiresAt: 4_000_000_000_000, + }), + ); + }), + ); + const runtime = Atom.runtime( + Layer.mergeAll( + Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, environments), + Layer.succeed(HubConnection.HubConnection, hub), + Layer.succeed(HttpClient.HttpClient, client), + ), + ); + const registry = AtomRegistry.make(); + yield* Effect.addFinalizer(() => Effect.sync(() => registry.dispose())); + const query = createAssetEnvironmentAtoms(runtime).createUrl({ + environmentId, + input: { resource }, + }); + const result = yield* AtomRegistry.getResult(registry, query, { suspendOnWaiting: true }); + expect(result.relativeUrl).toBe( + "https://archive.example.test/api/archive/assets/signed/picture.png", + ); + const archiveQuery = createOrchestrationEnvironmentAtoms(runtime).archivedShellSnapshot({ + environmentId, + input: {}, + }); + const discovered = yield* AtomRegistry.getResult(registry, archiveQuery, { + suspendOnWaiting: true, + }); + expect(discovered.threads.map((thread) => thread.id)).toEqual([v2ThreadShell.id]); + expect(discovered.projects[0]?.id).toBe(v2Project.id); + }).pipe(Effect.scoped), + ); + it.effect.each([ { name: "missing video", path: "/tmp/clip.mp4", fallback: true }, { name: "literal filename characters", path: "/tmp/frame#one?two.png", fallback: true }, diff --git a/packages/client-runtime/src/state/assets.ts b/packages/client-runtime/src/state/assets.ts index 8543dd957731e5ac13ce1da518642e2d3d7d035e..42a9a6ef00ce8db8906b548cf35e63c16a991ec4 100644 --- a/packages/client-runtime/src/state/assets.ts +++ b/packages/client-runtime/src/state/assets.ts @@ -1,4 +1,5 @@ import { + ArchiveError, type AssetCreateUrlInput, type AssetCreateUrlResult, type AssetImageDimensions, @@ -13,6 +14,9 @@ import { getProjectFaviconResourceKey, isProjectFaviconFallbackUrl, } from "@t3tools/shared/projectFavicon"; +import { HttpClient } from "effect/http"; +import * as HubConnection from "../connection/hub.ts"; +import { fetchArchivedAssetUrl } from "./archiveAssetHttp.ts"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Result from "effect/Result"; @@ -21,9 +25,11 @@ import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; import * as EnvironmentRegistry from "../connection/registry.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; -import { request } from "../rpc/client.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { request, EnvironmentRpcUnavailableError } from "../rpc/client.ts"; import type { ProjectFaviconCache, ProjectFaviconTarget } from "../projectFaviconCache.ts"; import { createEnvironmentQueryAtomFamily } from "./runtime.ts"; +import { authorizeHubSignedUrl } from "./hubSignedUrl.ts"; const ASSET_URL_REFRESH_INTERVAL_MS = 30 * 60_000; const ASSET_URL_STALE_TIME_MS = 5 * 60_000; @@ -44,6 +50,7 @@ export class InvalidAssetCollectionKeyError extends Schema.TaggedError( readonly httpBaseUrl: string; } | null>, ) { + const fromArchive = Effect.fn("assets.fromArchive")(function* (input: AssetCreateUrlInput) { + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; + const hub = yield* Effect.serviceOption(HubConnection.HubConnection); + const http = yield* Effect.serviceOption(HttpClient.HttpClient); + if (Option.isNone(hub) || Option.isNone(http)) return Option.none(); + return yield* fetchArchivedAssetUrl(supervisor.target.environmentId, input.resource).pipe( + Effect.provideService(HubConnection.HubConnection, hub.value), + Effect.provideService(HttpClient.HttpClient, http.value), + Effect.option, + ); + }); const execute = Effect.fn("assets.createUrl")(function* (input: AssetCreateUrlInput) { + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; const result = yield* request(WS_METHODS.assetsCreateUrl, input).pipe(Effect.result); - if (Result.isSuccess(result)) return result.success; + if (Result.isSuccess(result)) return yield* authorizeHubSignedUrl(result.success); + const archived = yield* fromArchive(input); + if (Option.isSome(archived)) return archived.value; const error = result.failure; const resource = input.resource; const local = localMediaEnvironment ? (yield* AtomRegistry.AtomRegistry).get(localMediaEnvironment) : null; - const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; if ( !local || local.environmentId === supervisor.target.environmentId || @@ -170,20 +190,79 @@ export function createAssetEnvironmentAtoms( const registry = yield* EnvironmentRegistry.EnvironmentRegistry; const asset = yield* registry.run( local.environmentId, - request(WS_METHODS.assetsCreateUrl, input), + request(WS_METHODS.assetsCreateUrl, input).pipe(Effect.flatMap(authorizeHubSignedUrl)), ); // Callers resolve against the thread's server, so preserve the local server's origin. - return { ...asset, relativeUrl: new URL(asset.relativeUrl, local.httpBaseUrl).href }; + return { ...asset, relativeUrl: environmentEndpointUrl(local.httpBaseUrl, asset.relativeUrl) }; }); - const createUrl = createEnvironmentQueryAtomFamily(runtime, { + const createUrlQuery = createEnvironmentQueryAtomFamily(runtime, { label: "environment-data:assets:create-url", execute, + executeDisconnected: (input) => + fromArchive(input).pipe( + Effect.flatMap( + Option.match({ + onSome: Effect.succeed, + onNone: () => + EnvironmentSupervisor.EnvironmentSupervisor.pipe( + Effect.flatMap((supervisor) => + Effect.fail( + new EnvironmentRpcUnavailableError({ + environmentId: supervisor.target.environmentId, + message: new ArchiveError({ reason: "unavailable" }).message, + }), + ), + ), + ), + }), + ), + ), staleTimeMs: ASSET_URL_STALE_TIME_MS, idleTtlMs: ASSET_URL_IDLE_TTL_MS, refreshIntervalMs: ASSET_URL_REFRESH_INTERVAL_MS, refreshTrigger: ({ input }) => input.resource._tag === "media-file" ? localMediaEnvironment : undefined, }); + const signedUrlFamily = Atom.family((key: string) => { + const [environmentId, resource] = decodeAssetUrlKey(JSON.parse(key)); + const query = createUrlQuery({ environmentId, input: { resource } }); + return Atom.transform( + query, + (get) => { + const result = get(query); + if ( + AsyncResult.isSuccess(result) && + !result.waiting && + (new URL(result.value.relativeUrl, "http://relative.invalid").searchParams.has( + "wsTicket", + ) || + new URL(result.value.relativeUrl, "http://relative.invalid").pathname.includes( + "/api/archive/assets/", + )) + ) { + // @effect-diagnostics-next-line globalDate:off - Atom expiry schedules outside an Effect runtime. + const now = Date.now(); + const remaining = Math.min( + ASSET_URL_REFRESH_INTERVAL_MS, + Math.max(0, result.value.expiresAt - now - 60_000), + ); + if (remaining === 0) { + get.refresh(query); + return get.once(query); + } + // @effect-diagnostics-next-line globalTimers:off - Atom lifetime owns the refresh timer. + const handle = setTimeout(() => get.refresh(query), remaining); + get.addFinalizer(() => clearTimeout(handle)); + } + return result; + }, + { initialValueTarget: query }, + ).pipe(Atom.setIdleTTL(ASSET_URL_IDLE_TTL_MS)); + }); + const createUrl = (target: { + readonly environmentId: EnvironmentId; + readonly input: AssetCreateUrlInput; + }) => signedUrlFamily(JSON.stringify([target.environmentId, target.input.resource])); const createUrlsFamily = Atom.family((key: string) => { const [environmentId, resources] = parseAssetCollectionKey(key); return Atom.make((get) => diff --git a/packages/client-runtime/src/state/attachments.ts b/packages/client-runtime/src/state/attachments.ts index b58a8e34e5a95ab8927330bf385bd5b02617c7b3..d2fd65ec78a9b833979ee09dce9c7e66967efdf5 100644 --- a/packages/client-runtime/src/state/attachments.ts +++ b/packages/client-runtime/src/state/attachments.ts @@ -7,8 +7,11 @@ import { type EnvironmentId, } from "@t3tools/contracts"; import type { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; +import * as Effect from "effect/Effect"; import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { requestGuarded } from "../rpc/client.ts"; +import { authorizeHubSignedUrl } from "./hubSignedUrl.ts"; import { createEnvironmentRpcCommand, executeAtomQuery, @@ -29,6 +32,10 @@ export function createAttachmentEnvironmentAtoms( createUploadUrl: createEnvironmentRpcCommand(runtime, { label: "environment-command:attachments:create-upload-url", tag: WS_METHODS.attachmentsCreateUploadUrl, + execute: (input) => + requestGuarded(WS_METHODS.attachmentsCreateUploadUrl, input).pipe( + Effect.flatMap(authorizeHubSignedUrl), + ), }), remove: createEnvironmentRpcCommand(runtime, { label: "environment-command:attachments:delete", diff --git a/packages/client-runtime/src/state/environmentHttpAuth.test.ts b/packages/client-runtime/src/state/environmentHttpAuth.test.ts index 587d707b8a07bb916e2af62b7814361bd00b9ac9..2e1f6d22aa2bddc3d65453a1786109e8124974df 100644 --- a/packages/client-runtime/src/state/environmentHttpAuth.test.ts +++ b/packages/client-runtime/src/state/environmentHttpAuth.test.ts @@ -23,6 +23,7 @@ import * as Tracer from "effect/Tracer"; import { TestClock } from "effect/testing"; import type { HttpClient } from "effect/http"; +import { HostedAuthAccess } from "../platform/capabilities.ts"; import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; import { ConnectionTransientError, @@ -629,6 +630,31 @@ describe("authenticated environment HTTP requests", () => { }), ); + it.effect("adds current hosted CSRF to cookie-authenticated environment requests", () => + Effect.gen(function* () { + const harness = makeHarness(() => Response.json(SESSION)); + const servers: string[] = []; + yield* fetchEnvironmentSessionState({ + prepared: { ...PREPARED, httpAuthorization: null }, + signer: Option.none(), + }).pipe( + Effect.provide(harness.httpLayer), + Effect.provideService(HostedAuthAccess, { + headers: (server) => + Effect.sync(() => { + servers.push(server); + return { "x-csrf-token": "current-cookie-proof" }; + }), + }), + ); + expect(servers).toEqual([PREPARED.httpBaseUrl]); + expect(new Headers(harness.calls[0]!.init.headers).get("x-csrf-token")).toBe( + "current-cookie-proof", + ); + expect(harness.calls[0]!.init.credentials).toBe("include"); + }), + ); + it.effect("keeps the caller's timeout while waiting for renewal", () => Effect.gen(function* () { const harness = makeHarness(() => Response.json(SESSION)); diff --git a/packages/client-runtime/src/state/environmentHttpAuth.ts b/packages/client-runtime/src/state/environmentHttpAuth.ts index 9693c7215b3c611995d0741b320e5ef169cd18a8..6e0b389156f37082de1ceca48ee37b1d82942c04 100644 --- a/packages/client-runtime/src/state/environmentHttpAuth.ts +++ b/packages/client-runtime/src/state/environmentHttpAuth.ts @@ -8,6 +8,7 @@ import * as Result from "effect/Result"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { FetchHttpClient, type HttpMethod } from "effect/http"; +import { HostedAuthAccess } from "../platform/capabilities.ts"; import type { RemoteEnvironmentAuthorization } from "../authorization/service.ts"; import type { PreparedConnection, PreparedHttpAuthorization } from "../connection/model.ts"; import type { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; @@ -21,6 +22,7 @@ import { } from "../rpc/http.ts"; export interface EnvironmentHttpAuthHeaders { + readonly "x-csrf-token"?: string; readonly authorization?: string; readonly dpop?: string; } @@ -71,7 +73,11 @@ const buildEnvironmentAuthHeaders = ( ): Effect.Effect => Effect.gen(function* () { if (authorization === null) { - return {}; + const hosted = yield* Effect.serviceOption(HostedAuthAccess); + if (Option.isNone(hosted)) return {}; + return yield* hosted.value + .headers(new URL(url).origin) + .pipe(Effect.orElseSucceed(() => ({}))); } if (authorization._tag === "Bearer") { return { authorization: `Bearer ${authorization.token}` }; diff --git a/packages/client-runtime/src/state/hub-capabilities.test.ts b/packages/client-runtime/src/state/hub-capabilities.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..e4976fecf55d8c2c08581f41e933a142bad58df1 --- /dev/null +++ b/packages/client-runtime/src/state/hub-capabilities.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + AuthOrchestrationOperateScope, + EnvironmentId, + ExecutionEnvironmentDescriptor, + type ServerConfig, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { Atom, AtomRegistry } from "effect/reactivity"; + +import { HubConnection, type HubConnectionState } from "../connection/hub.ts"; +import { createHubConnectionAtoms, createHubEnvironmentCapabilitiesAtom } from "./hub.ts"; + +const hubId = EnvironmentId.make("hub"); +const id = EnvironmentId.make("offline"); +const descriptor = Schema.decodeUnknownSync(ExecutionEnvironmentDescriptor)({ + environmentId: id, + label: "Offline machine", + platform: { os: "darwin", arch: "arm64" }, + serverVersion: "test", + capabilities: {}, +}); +const initial = { + connection: { environmentId: hubId, label: "Hub", httpBaseUrl: "https://hub.example/" }, + directory: { + hubEnvironmentId: hubId, + environments: [ + { + environmentId: id, + label: "Offline machine", + httpBaseUrl: "https://hub.example/environments/offline/", + scopes: [AuthOrchestrationOperateScope], + descriptor, + }, + ], + }, + inbox: { + environments: [ + { + environmentId: id, + pendingThreadIds: [], + snapshot: { + schemaVersion: 1, + snapshotSequence: 0, + projects: [], + threads: [], + archivedThreads: [], + }, + }, + ], + }, + refreshing: false, + error: null, +} satisfies HubConnectionState; + +describe("central organization availability", () => { + it.effect( + "permits offline organization from the directory grant without a source configuration", + () => + Effect.gen(function* () { + const state = yield* SubscriptionRef.make(initial); + const runtime = Atom.runtime( + Layer.succeed(HubConnection, { + state, + connect: () => Effect.void, + connectAuthenticated: () => Effect.void, + headers: Effect.succeed({}), + refresh: Effect.void, + refreshInbox: Effect.void, + disconnect: Effect.void, + organize: () => Effect.succeedNone, + }), + ); + const hub = createHubConnectionAtoms(runtime); + const configs = Atom.make>(new Map()); + const capabilities = createHubEnvironmentCapabilitiesAtom(configs, hub); + const registry = AtomRegistry.make(); + yield* Effect.addFinalizer(() => Effect.sync(() => registry.dispose())); + registry.mount(hub.organizationPermissionAtom(id)); + yield* AtomRegistry.getResult(registry, hub.stateAtom); + + expect(registry.get(hub.organizationPermissionAtom(id))).toBe(true); + expect(registry.get(hub.organizationPermissionAtom(EnvironmentId.make("other")))).toBe( + false, + ); + expect(registry.get(configs).size).toBe(0); + expect(registry.get(capabilities).get(id)).toMatchObject({ + threadSettlement: true, + threadSnooze: true, + threadPinning: true, + threadPinReorder: true, + threadActiveReorder: true, + threadAutoSettleOptOut: true, + }); + + expect(registry.get(capabilities).get(id)?.threadTitleRegeneration).not.toBe(true); + const revoked = yield* Effect.forkChild( + Effect.callback((resume) => { + const unsubscribe = registry.subscribe( + hub.organizationPermissionAtom(id), + () => { + if (!registry.get(hub.organizationPermissionAtom(id))) resume(Effect.void); + }, + { immediate: true }, + ); + return Effect.sync(unsubscribe); + }), + ); + yield* SubscriptionRef.set(state, { + ...initial, + directory: { + ...initial.directory, + environments: initial.directory.environments.map((entry) => ({ ...entry, scopes: [] })), + }, + }); + yield* Fiber.join(revoked); + expect(registry.get(hub.organizationPermissionAtom(id))).toBe(false); + expect(registry.get(capabilities).get(id)?.threadSettlement).not.toBe(true); + }).pipe(Effect.scoped), + ); +}); diff --git a/packages/client-runtime/src/state/hub.ts b/packages/client-runtime/src/state/hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..c7b91a006bf2566cf0f807d9106af09100e8a30d --- /dev/null +++ b/packages/client-runtime/src/state/hub.ts @@ -0,0 +1,173 @@ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { AsyncResult, Atom } from "effect/reactivity"; +import { + AuthOrchestrationOperateScope, + HUB_THREAD_ORGANIZATION_FIELDS, + type EnvironmentId, + type ExecutionEnvironmentCapabilities, + type HubInboxEnvironment, + type ServerConfig, +} from "@t3tools/contracts"; +import * as Equal from "effect/Equal"; +import * as Struct from "effect/Struct"; + +import { HubConnection, EMPTY_HUB_CONNECTION_STATE } from "../connection/hub.ts"; +import type { PairingConnectionInput } from "../connection/onboarding.ts"; +import { createAtomCommandScheduler, createRuntimeCommand } from "./runtime.ts"; +import type { EnvironmentShellState } from "./shell.ts"; + +export type { HubConnectionState, HubServerConnection } from "../connection/hub.ts"; +export type { PairingConnectionInput } from "../connection/onboarding.ts"; + +export function applyHubInboxEnvironment( + state: EnvironmentShellState, + entry: HubInboxEnvironment | undefined, +): EnvironmentShellState { + if (entry === undefined) return state; + const local = Option.getOrNull(state.snapshot); + const snapshot = + local !== null && + (state.status === "live" || local.snapshotSequence > entry.snapshot.snapshotSequence) + ? local + : entry.snapshot; + const pending = new Set(entry.pendingThreadIds); + const organization = new Map(entry.snapshot.threads.map((thread) => [thread.id, thread])); + let changed = false; + const threads = snapshot.threads.map((thread) => { + const central = organization.get(thread.id); + if (!pending.has(thread.id) || central === undefined) return thread; + const fields = Struct.pick(central, HUB_THREAD_ORGANIZATION_FIELDS); + if (Equal.equals(fields, Struct.pick(thread, HUB_THREAD_ORGANIZATION_FIELDS))) return thread; + changed = true; + return { ...thread, ...fields }; + }); + const next = changed ? { ...snapshot, threads } : snapshot; + if (local === next) return state; + return { + ...state, + snapshot: Option.some(next), + status: state.status === "live" ? "live" : "cached", + }; +} + +export function createHubConnectionAtoms(runtime: Atom.AtomRuntime) { + const stateAtom = runtime.atom( + Stream.unwrap(HubConnection.pipe(Effect.map((hub) => SubscriptionRef.changes(hub.state)))), + { initialValue: EMPTY_HUB_CONNECTION_STATE }, + ); + const stateValueAtom = Atom.make((get) => + Option.getOrElse(AsyncResult.value(get(stateAtom)), () => EMPTY_HUB_CONNECTION_STATE), + ); + const inboxValueAtom = Atom.make((get) => get(stateValueAtom).inbox); + const organizationEnvironmentIdsAtom = Atom.make((get) => { + const state = get(stateValueAtom); + if (state.connection === null) return new Set(); + const permitted = new Set( + state.directory?.environments.flatMap((entry) => + entry.scopes.includes(AuthOrchestrationOperateScope) ? [entry.environmentId] : [], + ), + ); + return new Set( + state.inbox.environments.flatMap((entry) => + permitted.has(entry.environmentId) ? [entry.environmentId] : [], + ), + ); + }).pipe( + Atom.withEquality>( + (left, right) => left.size === right.size && [...left].every((id) => right.has(id)), + ), + ); + const organizationPermissionAtom = Atom.family((environmentId: EnvironmentId | null) => + Atom.make((get) => + environmentId === null ? false : get(organizationEnvironmentIdsAtom).has(environmentId), + ), + ); + const scheduler = createAtomCommandScheduler(); + const concurrency = { mode: "serial" as const, key: () => "hub" }; + const connect = createRuntimeCommand(runtime, { + label: "hub:connect", + scheduler, + concurrency, + execute: (input: PairingConnectionInput) => + HubConnection.pipe(Effect.flatMap((hub) => hub.connect(input))), + }); + const connectAuthenticated = createRuntimeCommand(runtime, { + label: "hub:sign-in", + scheduler, + concurrency, + execute: (input: { readonly httpBaseUrl: string }) => + HubConnection.pipe(Effect.flatMap((hub) => hub.connectAuthenticated(input))), + }); + const refresh = createRuntimeCommand(runtime, { + label: "hub:refresh", + scheduler, + concurrency, + execute: (_input: void) => HubConnection.pipe(Effect.flatMap((hub) => hub.refresh)), + }); + const disconnect = createRuntimeCommand(runtime, { + label: "hub:disconnect", + scheduler, + concurrency, + execute: (_input: void) => HubConnection.pipe(Effect.flatMap((hub) => hub.disconnect)), + }); + return { + stateAtom, + stateValueAtom, + inboxValueAtom, + organizationEnvironmentIdsAtom, + organizationPermissionAtom, + connect, + connectAuthenticated, + refresh, + disconnect, + }; +} + +export function createHubEnvironmentCapabilitiesAtom( + configsAtom: Atom.Atom>, + hub: Pick< + ReturnType, + "stateValueAtom" | "organizationEnvironmentIdsAtom" + >, +) { + return Atom.make((get): ReadonlyMap => { + const capabilities = new Map( + get(hub.stateValueAtom).directory?.environments.map((entry) => [ + entry.environmentId, + entry.descriptor.capabilities, + ]), + ); + for (const [id, config] of get(configsAtom)) + capabilities.set(id, config.environment.capabilities); + for (const id of get(hub.organizationEnvironmentIdsAtom)) { + const current = capabilities.get(id); + if (current === undefined) continue; + capabilities.set(id, { + ...current, + threadSettlement: true, + threadSnooze: true, + threadPinning: true, + threadPinReorder: true, + threadActiveReorder: true, + threadAutoSettleOptOut: true, + }); + } + return capabilities; + }).pipe( + Atom.withEquality>( + (left, right) => + left.size === right.size && + [...left].every(([id, capabilities]) => { + const other = right.get(id); + return ( + other !== undefined && + Object.keys(capabilities).length === Object.keys(other).length && + Object.entries(capabilities).every(([key, value]) => Reflect.get(other, key) === value) + ); + }), + ), + ); +} diff --git a/packages/client-runtime/src/state/hubSignedUrl.test.ts b/packages/client-runtime/src/state/hubSignedUrl.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..2057533dad08b98b1168a49e505e7d4555555eb4 --- /dev/null +++ b/packages/client-runtime/src/state/hubSignedUrl.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + AuthStandardClientScopes, + EnvironmentId, + EnvironmentAuthorizationError, + WS_METHODS, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as DateTime from "effect/DateTime"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { Atom, AtomRegistry } from "effect/reactivity"; +import { vi } from "vite-plus/test"; + +import * as EnvironmentRegistry from "../connection/registry.ts"; +import { + AVAILABLE_CONNECTION_STATE, + BearerConnectionTarget, + type PreparedConnection, + type SupervisorConnectionState, +} from "../connection/model.ts"; +import * as EnvironmentSupervisor from "../connection/supervisor.ts"; +import type { RpcSession } from "../rpc/session.ts"; +import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; +import { layerRemoteHttpClient } from "../rpc/http.ts"; +import { createAssetEnvironmentAtoms, resolveAssetUrl } from "./assets.ts"; +import { createAttachmentEnvironmentAtoms, runAttachmentUploadCycle } from "./attachments.ts"; +import { runAtomCommand } from "./runtime.ts"; + +const environmentId = EnvironmentId.make("laptop"); +const base = "https://hub.example/knife/environments/laptop/"; + +const harness = (hub: boolean, allowed = true) => + Effect.gen(function* () { + const tickets = new Map(); + const calls = { assets: 0, uploads: 0, tickets: 0 }; + const target = new BearerConnectionTarget({ + environmentId, + label: "Laptop", + connectionId: hub ? "hub:home:laptop" : "direct:laptop", + }); + const prepared: PreparedConnection = { + environmentId, + label: "Laptop", + httpBaseUrl: base, + socketUrl: `${base.replace(/^http/, "ws")}ws`, + httpAuthorization: { _tag: "Bearer", token: "root-session" }, + target, + }; + const fetcher: typeof fetch = async (input, init) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/api/auth/session")) + return Response.json({ + authenticated: true, + auth: { + policy: "remote-reachable", + bootstrapMethods: [], + sessionMethods: [], + sessionCookieName: "test", + }, + scopes: allowed ? AuthStandardClientScopes : [], + permissions: allowed ? AuthStandardClientScopes : [], + }); + if (url.pathname.endsWith("/api/auth/websocket-ticket")) { + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer root-session"); + const ticket = `media-${++calls.tickets}`; + const expiresAt = DateTime.toEpochMillis(DateTime.nowUnsafe()) + 5 * 60_000; + tickets.set(ticket, expiresAt); + return Response.json({ + ticket, + expiresAt: DateTime.formatIso(DateTime.makeUnsafe(expiresAt)), + }); + } + if ( + url.pathname.includes("/api/assets/") || + url.pathname.includes("/api/attachments/upload/") + ) { + const authorized = + !hub || + (tickets.get(url.searchParams.get("wsTicket") ?? "") ?? 0) > + DateTime.toEpochMillis(DateTime.nowUnsafe()); + return new Response(authorized ? "file bytes" : "unauthorized", { + status: authorized ? 200 : 401, + }); + } + throw new Error(`Unexpected URL ${url}`); + }; + const client = { + [WS_METHODS.assetsCreateUrl]: () => + Effect.sync(() => { + calls.assets++; + return { + relativeUrl: "/api/assets/leaf-signed/photo.png", + expiresAt: DateTime.toEpochMillis(DateTime.nowUnsafe()) + 3_600_000, + }; + }), + [WS_METHODS.attachmentsCreateUploadUrl]: () => + !allowed + ? Effect.fail( + new EnvironmentAuthorizationError({ + requiredScope: "orchestration:operate", + message: "Not allowed", + }), + ) + : Effect.sync(() => { + calls.uploads++; + return { + attachmentId: "pending", + relativeUrl: "/api/attachments/upload/leaf-signed", + expiresAt: DateTime.toEpochMillis(DateTime.nowUnsafe()) + 600_000, + }; + }), + } as unknown as WsRpcProtocolClient; + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target, + state: yield* SubscriptionRef.make({ + ...AVAILABLE_CONNECTION_STATE, + phase: "connected" as const, + }), + prepared: yield* SubscriptionRef.make(Option.some(prepared)), + session: yield* SubscriptionRef.make(Option.some({ client } as RpcSession)), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + }); + const environments = EnvironmentRegistry.EnvironmentRegistry.of({ + run: (_id, effect) => + Effect.provideService(effect, EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + followStream: (_id, stream) => + Stream.provideService(stream, EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + } as EnvironmentRegistry.EnvironmentRegistry["Service"]); + const runtime = Atom.runtime( + Layer.merge( + layerRemoteHttpClient(fetcher), + Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, environments), + ), + ); + const registry = AtomRegistry.make(); + yield* Effect.addFinalizer(() => Effect.sync(() => registry.dispose())); + const assets = createAssetEnvironmentAtoms(runtime); + const attachments = createAttachmentEnvironmentAtoms(runtime); + const query = assets.createUrl({ + environmentId, + input: { resource: { _tag: "attachment", attachmentId: "photo" } }, + }); + return { registry, assets, attachments, query, fetcher, calls }; + }); + +describe("hub signed media capabilities", () => { + it.effect("authorizes image reads and guarded byte uploads through the public hub prefix", () => + Effect.scoped( + Effect.gen(function* () { + const test = yield* harness(true); + const asset = yield* AtomRegistry.getResult(test.registry, test.query, { + suspendOnWaiting: true, + }); + const url = resolveAssetUrl(base, asset.relativeUrl)!; + expect(url).toContain(`${base}api/assets/leaf-signed/photo.png?wsTicket=`); + expect(asset.expiresAt).toBeLessThanOrEqual( + DateTime.toEpochMillis(DateTime.nowUnsafe()) + 5 * 60_000, + ); + expect((yield* Effect.promise(() => test.fetcher(url))).status).toBe(200); + expect((yield* Effect.promise(() => test.fetcher(url.split("?")[0]!))).status).toBe(401); + const upload = yield* Effect.promise(() => + runAttachmentUploadCycle({ + registry: test.registry, + createUploadUrl: test.attachments.createUploadUrl, + remove: test.attachments.remove, + environmentId, + upload: { type: "file", name: "notes.txt", mimeType: "text/plain", sizeBytes: 3 }, + resolveUploadUrl: (relativeUrl) => resolveAssetUrl(base, relativeUrl), + transport: (uploadUrl) => ({ + done: test.fetcher(uploadUrl, { method: "PUT", body: "abc" }).then((response) => { + expect(response.status).toBe(200); + }), + abort: () => {}, + }), + }), + ); + expect(upload).toEqual({ status: "uploaded", attachmentId: "pending" }); + expect(test.calls).toEqual({ assets: 1, uploads: 1, tickets: 2 }); + }), + ), + ); + + it.effect("does not mint a hub ticket when the server refuses the upload", () => + Effect.scoped( + Effect.gen(function* () { + const test = yield* harness(true, false); + const result = yield* Effect.promise(() => + runAtomCommand( + test.registry, + test.attachments.createUploadUrl, + { + environmentId, + input: { type: "file", name: "notes.txt", mimeType: "text/plain", sizeBytes: 3 }, + }, + { reportFailure: false }, + ), + ); + expect(result._tag).toBe("Failure"); + expect(test.calls).toEqual({ assets: 0, uploads: 0, tickets: 0 }); + }), + ), + ); + + it.effect.each([true, false])( + "refreshes ticket-bound images before expiry without shortening direct asset caching (hub=%s)", + (hub) => + Effect.scoped( + Effect.gen(function* () { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + yield* Effect.addFinalizer(() => Effect.sync(() => vi.useRealTimers())); + const test = yield* harness(hub); + const unmount = test.registry.mount(test.query); + yield* Effect.addFinalizer(() => Effect.sync(unmount)); + yield* AtomRegistry.getResult(test.registry, test.query, { suspendOnWaiting: true }); + vi.advanceTimersByTime(4 * 60_000); + yield* Effect.yieldNow; + const asset = yield* AtomRegistry.getResult(test.registry, test.query, { + suspendOnWaiting: true, + }); + expect(test.calls.assets).toBe(hub ? 2 : 1); + expect(test.calls.tickets).toBe(hub ? 2 : 0); + expect( + (yield* Effect.promise(() => test.fetcher(resolveAssetUrl(base, asset.relativeUrl)!))) + .status, + ).toBe(200); + }), + ), + ); +}); diff --git a/packages/client-runtime/src/state/hubSignedUrl.ts b/packages/client-runtime/src/state/hubSignedUrl.ts new file mode 100644 index 0000000000000000000000000000000000000000..772747fab7fa0d2d93196f6471eaabe5c43d0b08 --- /dev/null +++ b/packages/client-runtime/src/state/hubSignedUrl.ts @@ -0,0 +1,50 @@ +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import { HttpClient } from "effect/http"; + +import { issueRemoteWebSocketTicket } from "../authorization/remote.ts"; +import * as EnvironmentSupervisor from "../connection/supervisor.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { EnvironmentRpcUnavailableError } from "../rpc/client.ts"; + +export const authorizeHubSignedUrl = Effect.fn("clientRuntime.state.authorizeHubSignedUrl")( + function* (result: A) { + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; + const prepared = Option.getOrNull(yield* SubscriptionRef.get(supervisor.prepared)); + const target = prepared?.target ?? supervisor.target; + if (target._tag !== "BearerConnectionTarget" || !target.connectionId.startsWith("hub:")) { + return result; + } + + const unavailable = () => + new EnvironmentRpcUnavailableError({ + environmentId: target.environmentId, + message: "Reconnect to this server to access its files.", + }); + if (prepared?.httpAuthorization?._tag !== "Bearer") return yield* unavailable(); + + const client = yield* Effect.serviceOption(HttpClient.HttpClient); + if (Option.isNone(client)) return yield* unavailable(); + + const ticket = yield* issueRemoteWebSocketTicket({ + httpBaseUrl: prepared.httpBaseUrl, + bearerToken: prepared.httpAuthorization.token, + }).pipe( + Effect.provideService(HttpClient.HttpClient, client.value), + Effect.mapError(unavailable), + ); + const base = new URL(prepared.httpBaseUrl); + const url = new URL(environmentEndpointUrl(prepared.httpBaseUrl, result.relativeUrl)); + if (url.origin !== base.origin || !url.pathname.startsWith(base.pathname)) + return yield* unavailable(); + url.searchParams.set("wsTicket", ticket.ticket); + + return { + ...result, + relativeUrl: url.href, + expiresAt: Math.min(result.expiresAt, DateTime.toEpochMillis(ticket.expiresAt)), + }; + }, +); diff --git a/packages/client-runtime/src/state/orchestration.ts b/packages/client-runtime/src/state/orchestration.ts index 68495a181c9732d905567198b9403d64ba53406f..2f74ccef87e83303097389be3394edd3739ad37a 100644 --- a/packages/client-runtime/src/state/orchestration.ts +++ b/packages/client-runtime/src/state/orchestration.ts @@ -1,4 +1,4 @@ -import { ORCHESTRATION_V2_WS_METHODS, type ThreadId } from "@t3tools/contracts"; +import { ArchiveError, ORCHESTRATION_V2_WS_METHODS, type ThreadId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -7,6 +7,7 @@ import { Atom } from "effect/reactivity"; import { createEnvironmentRpcCommand, + createEnvironmentQueryAtomFamily, createEnvironmentRpcQueryAtomFamily, createEnvironmentRpcSubscriptionAtomFamily, createEnvironmentCommand, @@ -15,7 +16,12 @@ import type { EnvironmentRegistry } from "../connection/registry.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; import * as ManagedRelayDpopSigner from "../relay/managedRelay.ts"; -import { EnvironmentRpcUnavailableError } from "../rpc/client.ts"; +import * as HubConnection from "../connection/hub.ts"; +import { + request, + EnvironmentRpcUnavailableError, + type EnvironmentRpcInput, +} from "../rpc/client.ts"; import { fetchEnvironmentThreadTranscript } from "./threadTranscriptHttp.ts"; export function createOrchestrationEnvironmentAtoms( @@ -109,9 +115,33 @@ export function createOrchestrationEnvironmentAtoms( staleTimeMs: 30_000, idleTtlMs: 60_000, }), - archivedShellSnapshot: createEnvironmentRpcQueryAtomFamily(runtime, { + archivedShellSnapshot: createEnvironmentQueryAtomFamily(runtime, { label: "environment-data:orchestration:archived-shell-snapshot", - tag: ORCHESTRATION_V2_WS_METHODS.getArchivedShellSnapshot, + execute: ( + input: EnvironmentRpcInput, + ) => request(ORCHESTRATION_V2_WS_METHODS.getArchivedShellSnapshot, input), + executeDisconnected: () => + Effect.gen(function* () { + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; + const unavailable = () => + new EnvironmentRpcUnavailableError({ + environmentId: supervisor.target.environmentId, + message: new ArchiveError({ reason: "unavailable" }).message, + }); + const hub = yield* Effect.serviceOption(HubConnection.HubConnection); + if (Option.isNone(hub)) return yield* unavailable(); + yield* hub.value.refreshInbox.pipe(Effect.ignore); + const entry = (yield* SubscriptionRef.get(hub.value.state)).inbox.environments.find( + (entry) => entry.environmentId === supervisor.target.environmentId, + ); + if (!entry) return yield* unavailable(); + return { + schemaVersion: entry.snapshot.schemaVersion, + snapshotSequence: entry.snapshot.snapshotSequence, + projects: entry.snapshot.projects, + threads: entry.snapshot.archivedThreads, + }; + }), }), }; } diff --git a/packages/client-runtime/src/state/runtime.ts b/packages/client-runtime/src/state/runtime.ts index 95aabd6d843da0e3c3b8507a45adad09d5b7b7a9..148a16d6edac3ecc33123404b64e9ca33104fcae 100644 --- a/packages/client-runtime/src/state/runtime.ts +++ b/packages/client-runtime/src/state/runtime.ts @@ -66,6 +66,10 @@ interface EnvironmentQueryAtomOptions extends Omit< input: Input, emit: (value: A) => Effect.Effect, ) => Effect.Effect; + readonly executeDisconnected?: ( + input: Input, + emit: (value: A) => Effect.Effect, + ) => Effect.Effect; readonly persistence?: QueryPersistence; readonly staleTimeMs?: number; readonly idleTtlMs?: number; @@ -592,6 +596,14 @@ export function createEnvironmentQueryAtomFamily( > => { if (connection === null) return Effect.never; const [connectionState, session] = connection; + if ( + (connectionState.phase !== "connected" || Option.isNone(session)) && + options.executeDisconnected + ) + return runInEnvironment( + target.environmentId, + options.executeDisconnected(target.input, emit), + ); switch (connectionState.phase) { case "connected": return Option.isSome(session) diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts index dbc4405a9faa0311cb7d3099a4fab2dd9fe62d01..bca1b087ede1dfeb97579f7f03f35359efa563b4 100644 --- a/packages/client-runtime/src/state/shell.ts +++ b/packages/client-runtime/src/state/shell.ts @@ -23,6 +23,8 @@ import * as EnvironmentRegistry from "../connection/registry.ts"; import { connectionProjectionPhase } from "../connection/model.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import * as ConnectionWakeups from "../connection/wakeups.ts"; +import * as Hub from "../connection/hub.ts"; +import { applyHubInboxEnvironment } from "./hub.ts"; import { safeErrorLogAttributes } from "../errors/safeLog.ts"; import * as Persistence from "../platform/persistence.ts"; import { runCachePersistence } from "./cachePersistence.ts"; @@ -428,9 +430,23 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") }); function shellStateChanges(environmentId: EnvironmentId) { - return followStreamInEnvironment( - environmentId, - Stream.unwrap(makeEnvironmentShellState().pipe(Effect.map(SubscriptionRef.changes))), + return Stream.unwrap( + Effect.gen(function* () { + const source = followStreamInEnvironment( + environmentId, + Stream.unwrap(makeEnvironmentShellState().pipe(Effect.map(SubscriptionRef.changes))), + ); + const hub = yield* Effect.serviceOption(Hub.HubConnection); + if (Option.isNone(hub)) return source; + return Stream.zipLatest(source, SubscriptionRef.changes(hub.value.state)).pipe( + Stream.map(([state, central]) => + applyHubInboxEnvironment( + state, + central.inbox.environments.find((entry) => entry.environmentId === environmentId), + ), + ), + ); + }), ); } diff --git a/packages/client-runtime/src/state/threadCommands.test.ts b/packages/client-runtime/src/state/threadCommands.test.ts index cf36760baecb774b1ed8fe161d20178b5345d2b8..2fb860d339bffef5cb35d633f03421668e72a6a0 100644 --- a/packages/client-runtime/src/state/threadCommands.test.ts +++ b/packages/client-runtime/src/state/threadCommands.test.ts @@ -24,6 +24,7 @@ import * as EnvironmentRegistry from "../connection/registry.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import type { RpcSession } from "../rpc/session.ts"; import { createThreadEnvironmentAtoms } from "./threadCommands.ts"; +import { applyHubInboxEnvironment } from "./hub.ts"; const ENVIRONMENT_ID = EnvironmentId.make("remote"); const THREAD_ID = ThreadId.make("thread"); @@ -80,6 +81,47 @@ const SNAPSHOT: OrchestrationV2ShellSnapshot = { ], }; +describe("central inbox shell state", () => { + const entry = { + environmentId: ENVIRONMENT_ID, + pendingThreadIds: [THREAD_ID], + snapshot: { + ...SNAPSHOT, + threads: SNAPSHOT.threads.map((thread) => ({ + ...thread, + settledOverride: "settled" as const, + settledAt: NOW, + })), + }, + }; + + it("shows cached chat metadata on a new client with no source snapshot", () => { + const state = applyHubInboxEnvironment( + { + snapshot: Option.none(), + status: "empty", + error: Option.none(), + }, + entry, + ); + expect(state.status).toBe("cached"); + expect(Option.getOrNull(state.snapshot)?.threads[0]?.settledOverride).toBe("settled"); + }); + + it("applies pending organization without hiding live execution", () => { + const live = { + ...SNAPSHOT, + threads: SNAPSHOT.threads.map((thread) => ({ ...thread, status: "running" as const })), + }; + const state = { snapshot: Option.some(live), status: "live" as const, error: Option.none() }; + const merged = Option.getOrThrow(applyHubInboxEnvironment(state, entry).snapshot); + expect(merged.threads[0]?.settledOverride).toBe("settled"); + expect(merged.threads[0]?.status).toBe("running"); + expect(merged.snapshotSequence).toBe(live.snapshotSequence); + expect(applyHubInboxEnvironment(state, { ...entry, pendingThreadIds: [] })).toBe(state); + }); +}); + const makeHarness = Effect.fn("TestThreadCommands.makeHarness")(function* () { const requests = yield* Queue.unbounded<{ command: OrchestrationV2Command; diff --git a/packages/client-runtime/src/state/threads-sync.test.ts b/packages/client-runtime/src/state/threads-sync.test.ts index b78377b2b1ecafc3bb45500fd2b102ced510d7c7..6b5589577aa8f4244f484cd7a8cd7b02cd80c666 100644 --- a/packages/client-runtime/src/state/threads-sync.test.ts +++ b/packages/client-runtime/src/state/threads-sync.test.ts @@ -1,11 +1,11 @@ import { + OrchestrationV2ThreadDetailSnapshot, EnvironmentId, EventId, MessageId, ORCHESTRATION_V2_WS_METHODS, ThreadId, TurnItemId, - type OrchestrationV2ThreadDetailSnapshot, type OrchestrationV2ThreadProjection, type OrchestrationV2ThreadStreamItem, type OrchestrationV2TurnItem, @@ -19,6 +19,8 @@ import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as HubConnection from "../connection/hub.ts"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -46,6 +48,9 @@ import { } from "./threads.ts"; import * as ThreadSnapshotLoader from "./threadSnapshotHttp.ts"; +const encodeArchiveSnapshot = Schema.encodeEffect( + Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot), +); const TARGET = new PrimaryConnectionTarget({ environmentId: EnvironmentId.make("environment-1"), label: "Test environment", @@ -99,6 +104,8 @@ function awaitThreadState( } const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (options?: { + readonly hub?: HubConnection.HubConnection["Service"]; + readonly offline?: boolean; readonly cached?: OrchestrationV2ThreadProjection; readonly cachedHistory?: { readonly historyCursor: string | null; @@ -155,7 +162,7 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o ), } as unknown as WsRpcProtocolClient; const supervisorSession = yield* SubscriptionRef.make>( - Option.some(testSession(client, options)), + options?.offline ? Option.none() : Option.some(testSession(client, options)), ); const prepared = yield* SubscriptionRef.make>( Option.some(PREPARED), @@ -244,6 +251,10 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o Effect.provideService(ThreadHistoryController.ThreadHistoryController, historyController), ); } + if (options?.hub) + makeThreadState = makeThreadState.pipe( + Effect.provideService(HubConnection.HubConnection, options.hub), + ); const threadState = yield* makeThreadState; yield* SubscriptionRef.changes(threadState).pipe( Stream.runForEach((state) => @@ -320,6 +331,64 @@ const deleted = (sequence = 3): OrchestrationV2ThreadStreamItem => { }; describe("EnvironmentThreads", () => { + it.effect( + "opens a NAS archived transcript without a source session and resumes its source sequence after reconnect", + () => + Effect.gen(function* () { + const archived = { + ...BASE_PROJECTION, + thread: { + ...BASE_PROJECTION.thread, + archivedAt: DateTime.makeUnsafe("2026-06-20T02:00:00Z"), + }, + }; + const archiveSnapshot = { snapshotSequence: 81, projection: archived }; + const hubState = yield* SubscriptionRef.make({ + ...HubConnection.EMPTY_HUB_CONNECTION_STATE, + connection: { + environmentId: EnvironmentId.make("archive-hub"), + label: "Archive", + httpBaseUrl: "https://archive.example.test", + }, + }); + const hub = HubConnection.HubConnection.of({ + state: hubState, + headers: Effect.succeed({ authorization: "Bearer ephemeral-archive-access" }), + connect: () => Effect.void, + connectAuthenticated: () => Effect.void, + refresh: Effect.void, + refreshInbox: Effect.void, + organize: () => Effect.succeedNone, + disconnect: Effect.void, + }); + const json = yield* encodeArchiveSnapshot(archiveSnapshot); + const http = HttpClient.make((request) => + Effect.sync(() => { + expect(request.url).toBe( + `https://archive.example.test/api/archive/catalog/${TARGET.environmentId}/${THREAD_ID}`, + ); + expect(request.headers.authorization).toBe("Bearer ephemeral-archive-access"); + return HttpClientResponse.fromWeb(request, Response.json(json)); + }), + ); + const h = yield* makeHarness({ hub, offline: true, historyHttpClient: http }); + const state = yield* awaitThreadState(h.observed, (value) => Option.isSome(value.data)); + expect(state.status).toBe("cached"); + expect(Option.getOrThrow(state.data)).toEqual(archived); + expect(yield* Ref.get(h.subscriptionCount)).toBe(0); + expect(yield* Ref.get(h.loaderCalls)).toBe(0); + yield* h.replaceSession; + yield* Queue.offer(h.inputs, titleUpdated("Source reconnected", 82)); + const live = yield* awaitThreadState( + h.observed, + (value) => + Option.isSome(value.data) && value.data.value.thread.title === "Source reconnected", + ); + expect(live.status).toBe("live"); + expect(yield* Ref.get(h.lastSubscribeAfterSequence)).toBe(81); + }), + ); + it.effect("loads the server thread when its local cache read fails", () => Effect.gen(function* () { const h = yield* makeHarness({ diff --git a/packages/client-runtime/src/state/threads.ts b/packages/client-runtime/src/state/threads.ts index a6e31cf40c4a8f1e32cf69f73fde8ba7f722bd14..150b1c5a668f4af98989556ae0a4491e262d21c9 100644 --- a/packages/client-runtime/src/state/threads.ts +++ b/packages/client-runtime/src/state/threads.ts @@ -44,6 +44,8 @@ import { mergeOlderHistoryIntoProjection, type ThreadHistoryMeta, } from "./threadHistoryMerge.ts"; +import { fetchArchivedThreadSnapshot } from "./archiveThreadHttp.ts"; +import * as HubConnection from "../connection/hub.ts"; import * as ThreadSnapshotLoader from "./threadSnapshotHttp.ts"; import { EMPTY_ENVIRONMENT_THREAD_STATE, @@ -178,6 +180,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make ThreadHistoryController.ThreadHistoryController, ); const httpClient = yield* Effect.serviceOption(HttpClient.HttpClient); + const archiveHub = yield* Effect.serviceOption(HubConnection.HubConnection); const dpopSigner = yield* Effect.serviceOption(ManagedRelay.ManagedRelayDpopSigner); const remoteAuthorization = yield* Effect.serviceOption( RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization, @@ -359,6 +362,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make options?: { /** Socket/full snapshots: drop progressive meta with the new timeline. */ readonly resetHistory?: boolean; + readonly status?: "cached"; /** * Explicit progressive meta installed atomically with the projection * (bounded HTTP). Wins over resetHistory when both are supplied. @@ -381,11 +385,13 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make ...previous, data: Option.some(thread), // Buffered values from a failed attempt can arrive after its error. - status: Option.isSome(previous.error) - ? ("cached" as const) - : waiting - ? ("synchronizing" as const) - : ("live" as const), + status: + options?.status ?? + (Option.isSome(previous.error) + ? ("cached" as const) + : waiting + ? ("synchronizing" as const) + : ("live" as const)), error: previous.error, history, }; @@ -771,6 +777,43 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make Effect.forkScoped, ); + if (Option.isSome(archiveHub) && Option.isSome(httpClient)) { + yield* Stream.merge( + SubscriptionRef.changes(supervisor.state).pipe(Stream.map(() => undefined)), + SubscriptionRef.changes(archiveHub.value.state).pipe(Stream.map(() => undefined)), + ).pipe( + Stream.mapEffect(() => + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if (current.status === "live" || current.status === "deleted") return; + if (Option.isSome(current.data) && !current.history.hasMoreHistory) return; + const snapshot = yield* fetchArchivedThreadSnapshot(environmentId, threadId).pipe( + Effect.provideService(HubConnection.HubConnection, archiveHub.value), + Effect.provideService(HttpClient.HttpClient, httpClient.value), + Effect.option, + ); + if (Option.isNone(snapshot)) return; + yield* applyLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if ( + current.status === "live" || + current.status === "deleted" || + snapshot.value.snapshotSequence < (yield* SubscriptionRef.get(lastSequence)) + ) + return; + yield* SubscriptionRef.set(lastSequence, snapshot.value.snapshotSequence); + yield* setThread(snapshot.value.projection, { resetHistory: true, status: "cached" }); + yield* remember; + }), + ); + }), + ), + Stream.runDrain, + Effect.forkScoped, + ); + } + const foregroundResubscriptions = Option.match(wakeups, { onNone: () => Stream.never, onSome: (service) => diff --git a/packages/client-runtime/src/webhookAddress.test.ts b/packages/client-runtime/src/webhookAddress.test.ts index 2284ee8fb150d06fab3a72f6c39dcbd40dedb3a2..389245ae464a8795f3964c073a0bf219613603e2 100644 --- a/packages/client-runtime/src/webhookAddress.test.ts +++ b/packages/client-runtime/src/webhookAddress.test.ts @@ -21,6 +21,12 @@ describe("webhookAddress", () => { expect(result.note).toContain("Tailscale"); }); + it("keeps the environment prefix in direct webhook addresses", () => { + expect( + webhookAddress(endpoint(null), "https://hub.example/prefix/environments/one/").address, + ).toBe(`https://hub.example/prefix/environments/one${path}`); + }); + it("says only this computer can call a loopback address", () => { const result = webhookAddress(endpoint(null), "http://127.0.0.1:3773/"); expect(result.copyable).toBe(true); diff --git a/packages/client-runtime/src/webhookAddress.ts b/packages/client-runtime/src/webhookAddress.ts index cd1424fd6f867075a5e42748ad023050e250ae0b..eb229210841143448b10fdccffbc8cb61b9ee269 100644 --- a/packages/client-runtime/src/webhookAddress.ts +++ b/packages/client-runtime/src/webhookAddress.ts @@ -1,3 +1,4 @@ +import { environmentEndpointUrl } from "./environment/endpoint.ts"; import type { ScheduledTaskWebhookEndpoint } from "@t3tools/contracts"; import { isLocalLoopbackHost } from "@t3tools/shared/hostClassification"; @@ -29,7 +30,7 @@ export function webhookAddress( note: "Link this environment to T3 Connect for a public URL.", }; } - const url = new URL(endpoint.path, httpBaseUrl); + const url = new URL(environmentEndpointUrl(httpBaseUrl, endpoint.path)); return { address: url.href, copyable: true, diff --git a/packages/contracts/src/auth.ts b/packages/contracts/src/auth.ts index e206163869c48918dc634801975386873ac88264..ca7581d6b865804c5de32569e0a0132ad2a6c3f4 100644 --- a/packages/contracts/src/auth.ts +++ b/packages/contracts/src/auth.ts @@ -81,6 +81,9 @@ export type ServerAuthSessionMethod = typeof ServerAuthSessionMethod.Type; export const AuthOrchestrationReadScope = "orchestration:read" as const; export const AuthOrchestrationOperateScope = "orchestration:operate" as const; +export const AuthEnvironmentConnectScope = "environment:connect" as const; +export const AuthArchiveReadScope = "archive:read" as const; +export const AuthArchiveWriteScope = "archive:write" as const; export const AuthSettingsWriteScope = "settings:write" as const; export const AuthProvidersManageScope = "providers:manage" as const; export const AuthEnvironmentMaintainScope = "environment:maintain" as const; @@ -98,8 +101,11 @@ export const AuthAccessWriteScope = "access:write" as const; export const AuthRelayReadScope = "relay:read" as const; export const AuthRelayWriteScope = "relay:write" as const; export const AuthEnvironmentScope = Schema.Literals([ + AuthEnvironmentConnectScope, AuthOrchestrationReadScope, AuthOrchestrationOperateScope, + AuthArchiveReadScope, + AuthArchiveWriteScope, AuthSettingsWriteScope, AuthProvidersManageScope, AuthEnvironmentMaintainScope, @@ -152,6 +158,8 @@ const authScopeResponseFields = { // Only clients talking to an old server use these parent checks. Servers never // expand stored grants, and an explicitly empty permissions array grants nothing. const legacyParents: Partial> = { + [AuthArchiveReadScope]: AuthOrchestrationReadScope, + [AuthArchiveWriteScope]: AuthOrchestrationOperateScope, [AuthFilesystemReadScope]: AuthOrchestrationReadScope, [AuthDiagnosticsReadScope]: AuthOrchestrationReadScope, [AuthSettingsWriteScope]: AuthOrchestrationOperateScope, @@ -205,6 +213,8 @@ export function sessionHasLegacyPermissions(session: SessionGrantInput): boolean export const AuthStandardClientScopes = [ AuthOrchestrationReadScope, AuthOrchestrationOperateScope, + AuthArchiveReadScope, + AuthArchiveWriteScope, AuthSettingsWriteScope, AuthProvidersManageScope, AuthEnvironmentMaintainScope, @@ -218,6 +228,7 @@ export const AuthStandardClientScopes = [ AuthRelayReadScope, ] as const; export const AuthAdministrativeScopes = [ + AuthEnvironmentConnectScope, ...AuthStandardClientScopes, AuthAccessReadScope, AuthAccessWriteScope, @@ -458,6 +469,12 @@ export const AuthCreatePairingCredentialInput = Schema.Struct({ }); export type AuthCreatePairingCredentialInput = typeof AuthCreatePairingCredentialInput.Type; +export const AuthCreateDelegatedSessionInput = Schema.Struct({ + label: Schema.optionalKey(TrimmedNonEmptyString), + scopes: AuthGrantScopes, +}); +export type AuthCreateDelegatedSessionInput = typeof AuthCreateDelegatedSessionInput.Type; + export const AuthSessionState = Schema.Struct({ authenticated: Schema.Boolean, auth: ServerAuthDescriptor, diff --git a/packages/contracts/src/desktopHostedAuth.ts b/packages/contracts/src/desktopHostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..6a99dcf13d16d92e80bf25c9e37673076ecb35e3 --- /dev/null +++ b/packages/contracts/src/desktopHostedAuth.ts @@ -0,0 +1,15 @@ +import * as Schema from "effect/Schema"; + +export const DesktopHostedAuthState = Schema.Struct({ + serverUrl: Schema.NullOr(Schema.String), + phase: Schema.Literals(["idle", "waiting", "connected", "error"]), + error: Schema.NullOr(Schema.String), +}); +export type DesktopHostedAuthState = typeof DesktopHostedAuthState.Type; + +export const SelfHostedConnectionStatus = Schema.Struct({ + configured: Schema.Boolean, + connected: Schema.Boolean, + expiresAt: Schema.NullOr(Schema.String), +}); +export type SelfHostedConnectionStatus = typeof SelfHostedConnectionStatus.Type; diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index fc07bb48505fafed1d6532a1b79e5c8cd08a576a..1a411f857d8cefc4698b34e1445dcda390afc6e1 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -15,6 +15,7 @@ import { AuthBrowserSessionResult, AuthClientSession, AuthCreatePairingCredentialInput, + AuthCreateDelegatedSessionInput, AuthMcpApprovalDecisionRequest, AuthMcpApprovalDetails, AuthMcpApprovalError, @@ -74,17 +75,20 @@ import { } from "./relay.ts"; const OptionalBearerHeaders = Schema.Struct({ + "x-csrf-token": Schema.optionalKey(Schema.String), authorization: Schema.optionalKey(Schema.String), dpop: Schema.optionalKey(Schema.String), }); const OrchestrationProtocolHeaders = Schema.Struct({ + "x-csrf-token": Schema.optionalKey(Schema.String), authorization: Schema.optionalKey(Schema.String), dpop: Schema.optionalKey(Schema.String), [ORCHESTRATION_PROTOCOL_HEADER]: Schema.Literal(ORCHESTRATION_PROTOCOL_VERSION_TEXT), }); const OptionalDpopProofHeaders = Schema.Struct({ + "x-csrf-token": Schema.optionalKey(Schema.String), dpop: Schema.optionalKey(Schema.String), }); @@ -374,6 +378,7 @@ const EnvironmentProjectMutationErrors = [ ] as const; export interface EnvironmentSessionPrincipalShape { + readonly parentSessionId?: AuthSessionId; readonly sessionId: AuthSessionId; readonly subject: string; readonly method: ServerAuthSessionMethod; @@ -496,6 +501,14 @@ class EnvironmentAuthHttpApi extends HttpApiGroup.make("auth") error: EnvironmentPairingCredentialErrors, }).middleware(EnvironmentAuthenticatedAuth), ) + .add( + HttpApiEndpoint.post("delegatedSession", "/api/auth/delegated-session", { + headers: OptionalBearerHeaders, + payload: AuthCreateDelegatedSessionInput, + success: AuthAccessTokenResult, + error: EnvironmentPairingCredentialErrors, + }).middleware(EnvironmentAuthenticatedAuth), + ) .add( HttpApiEndpoint.get("pairingLinks", "/api/auth/pairing-links", { headers: OptionalBearerHeaders, diff --git a/packages/contracts/src/hostedAuth.ts b/packages/contracts/src/hostedAuth.ts new file mode 100644 index 0000000000000000000000000000000000000000..ca9c9949e92071a325b67bfaf3259fce023ba4df --- /dev/null +++ b/packages/contracts/src/hostedAuth.ts @@ -0,0 +1,70 @@ +import * as Schema from "effect/Schema"; +import { AuthClientPresentationMetadata, AuthGrantScopes } from "./auth.ts"; +import { EnvironmentId } from "./baseSchemas.ts"; + +const ProofChallenge = Schema.String.check(Schema.isPattern(/^[A-Za-z0-9_-]{43}$/u)); +const ProofVerifier = Schema.String.check(Schema.isPattern(/^[A-Za-z0-9._~-]{43,128}$/u)); + +export const HostedAuthDeviceStart = Schema.Struct({ + codeChallenge: ProofChallenge, + state: ProofChallenge, + client: AuthClientPresentationMetadata, +}); +export type HostedAuthDeviceStart = typeof HostedAuthDeviceStart.Type; + +export const HostedAuthDeviceStartResult = Schema.Struct({ authorizationUrl: Schema.String }); +export type HostedAuthDeviceStartResult = typeof HostedAuthDeviceStartResult.Type; + +export const HostedAuthDeviceExchange = Schema.Struct({ + code: ProofChallenge, + codeVerifier: ProofVerifier, +}); +export type HostedAuthDeviceExchange = typeof HostedAuthDeviceExchange.Type; + +export const HostedAuthDeviceRefresh = Schema.Struct({ refreshToken: Schema.NonEmptyString }); +export type HostedAuthDeviceRefresh = typeof HostedAuthDeviceRefresh.Type; + +export const HostedAuthGrant = Schema.Struct({ + deviceId: Schema.String, + accessToken: Schema.String, + refreshToken: Schema.String, + expiresAt: Schema.DateTimeUtc, +}); +export type HostedAuthGrant = typeof HostedAuthGrant.Type; + +export const HostedAuthDevice = Schema.Struct({ + deviceId: Schema.String, + client: AuthClientPresentationMetadata, + createdAt: Schema.DateTimeUtc, + expiresAt: Schema.DateTimeUtc, + revokedAt: Schema.NullOr(Schema.DateTimeUtc), + current: Schema.Boolean, + environmentId: Schema.optionalKey(EnvironmentId), +}); +export type HostedAuthDevice = typeof HostedAuthDevice.Type; + +export const HostedAuthAccount = Schema.Struct({ + accountId: Schema.String, + deviceId: Schema.String, + csrfToken: Schema.String, + expiresAt: Schema.DateTimeUtc, +}); +export type HostedAuthAccount = typeof HostedAuthAccount.Type; + +export const HostedAuthTokenRequest = Schema.Struct({ scopes: AuthGrantScopes }); +export type HostedAuthTokenRequest = typeof HostedAuthTokenRequest.Type; + +export class HostedAuthError extends Schema.TaggedError()("HostedAuthError", { + reason: Schema.Literals([ + "disabled", + "configuration", + "identity", + "transaction", + "credential", + "forbidden", + "reauthentication", + "upstream", + "storage", + ]), + cause: Schema.optionalKey(Schema.Redacted(Schema.Defect())), +}) {} diff --git a/packages/contracts/src/hub.ts b/packages/contracts/src/hub.ts new file mode 100644 index 0000000000000000000000000000000000000000..d84b335a5df77f88dce491112832a30518711656 --- /dev/null +++ b/packages/contracts/src/hub.ts @@ -0,0 +1,116 @@ +import * as Schema from "effect/Schema"; +import { AuthEnvironmentScopes } from "./auth.ts"; +import { EnvironmentId, ThreadId } from "./baseSchemas.ts"; +import { ExecutionEnvironmentDescriptor } from "./environment.ts"; +import { OrchestrationV2Command, OrchestrationV2ShellSnapshot } from "./orchestrationV2.ts"; + +export const HUB_THREAD_ORGANIZATION_FIELDS = [ + "settledOverride", + "settledAt", + "unsettledAt", + "snoozedUntil", + "snoozedAt", + "pinnedAt", + "pinOrderKey", + "activeOrderKey", + "autoSettleDisabledAt", +] as const; + +const organizationCommandTypes = [ + "thread.settle", + "thread.unsettle", + "thread.snooze", + "thread.unsnooze", + "thread.pin", + "thread.unpin", + "thread.pin.reorder", + "thread.active.reorder", + "thread.auto-settle.set", +] as const; + +export const HubOrganizationCommand = OrchestrationV2Command.pipe( + Schema.refine( + ( + command, + ): command is Extract< + OrchestrationV2Command, + { readonly type: (typeof organizationCommandTypes)[number] } + > => organizationCommandTypes.some((type) => type === command.type), + ), +); +export type HubOrganizationCommand = typeof HubOrganizationCommand.Type; + +export const HubInboxCommand = Schema.Struct({ + environmentId: EnvironmentId, + command: HubOrganizationCommand, +}); +export type HubInboxCommand = typeof HubInboxCommand.Type; + +export const HubInboxEnvironment = Schema.Struct({ + environmentId: EnvironmentId, + snapshot: OrchestrationV2ShellSnapshot, + pendingThreadIds: Schema.Array(ThreadId), +}); +export type HubInboxEnvironment = typeof HubInboxEnvironment.Type; + +export const HubInbox = Schema.Struct({ environments: Schema.Array(HubInboxEnvironment) }); +export type HubInbox = typeof HubInbox.Type; + +export const HubDirectory = Schema.Struct({ + hubEnvironmentId: EnvironmentId, + environments: Schema.Array( + Schema.Struct({ + environmentId: EnvironmentId, + label: Schema.NonEmptyString, + httpBaseUrl: Schema.NonEmptyString, + scopes: AuthEnvironmentScopes, + descriptor: ExecutionEnvironmentDescriptor, + }), + ), +}); +export type HubDirectory = typeof HubDirectory.Type; + +export const HubManagementRequest = Schema.Union([ + Schema.Struct({ type: Schema.Literal("hub.configure"), publicUrl: Schema.NonEmptyString }), + Schema.Struct({ type: Schema.Literal("hub.status") }), + Schema.Struct({ + type: Schema.Literal("hub.enroll"), + url: Schema.NonEmptyString, + environmentId: EnvironmentId, + tokenFile: Schema.NonEmptyString, + }), + Schema.Struct({ type: Schema.Literal("hub.remove"), environmentId: EnvironmentId }), +]); +export type HubManagementRequest = typeof HubManagementRequest.Type; + +export class HubError extends Schema.TaggedError()("HubError", { + reason: Schema.Literals([ + "configuration", + "address", + "identity", + "credential", + "not-found", + "upstream", + "storage", + ]), + cause: Schema.optionalKey(Schema.Redacted(Schema.Defect())), +}) { + override get message(): string { + switch (this.reason) { + case "configuration": + return "Configure the server's public URL before enrolling machines."; + case "address": + return "Use an HTTP or HTTPS address without a password, query, or fragment."; + case "identity": + return "The machine identity changed. Check its address before enrolling it."; + case "credential": + return "The machine access grant is unavailable. Enroll it with a new grant."; + case "not-found": + return "This machine was removed. Refresh the server's machine list."; + case "upstream": + return "The machine is unreachable. Check its private network connection."; + case "storage": + return "The server could not save its machines. Check its storage permissions."; + } + } +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 8e385387b0ec78716d0cf9d1d6e790fc63addc8e..58f832664ae17e71ae297acd7bdfc8d21fd0d3ad 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -70,3 +70,9 @@ export * from "./scriptProgress.ts"; export * from "./uiSettings.ts"; export * from "./sharedAccounts.ts"; + +export * from "./transcriptArchive.ts"; +export * from "./hub.ts"; +export * from "./hostedAuth.ts"; +export * from "./desktopHostedAuth.ts"; +export * from "./outbound.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f05f11c1d1a8f04b9429c2d7a05701149bbb4ac3..b150dbb62ecc64c9dbd211d17a0f88c9bdda6d96 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1,3 +1,4 @@ +import type { DesktopHostedAuthState } from "./desktopHostedAuth.ts"; import * as Schema from "effect/Schema"; import { SnapShotSource } from "./chatAttachment.ts"; @@ -1127,6 +1128,11 @@ export const SystemSettingsPaneSchema = Schema.Literals(["full-disk-access"]); export type SystemSettingsPane = typeof SystemSettingsPaneSchema.Type; export interface DesktopBridge { + beginHostedAuth?: (server: string) => Promise; + getHostedAuthState?: () => Promise; + getHostedAccessToken?: (server: string) => Promise; + cancelHostedAuth?: () => Promise; + disconnectHostedAuth?: () => Promise; supportsLocalTerminal?: (input: { readonly httpBaseUrl: string; readonly environmentId: string; diff --git a/packages/contracts/src/outbound.ts b/packages/contracts/src/outbound.ts new file mode 100644 index 0000000000000000000000000000000000000000..05bb39db2fe7a26d3f62c77a5cccc91a9cb5425f --- /dev/null +++ b/packages/contracts/src/outbound.ts @@ -0,0 +1,32 @@ +import * as Schema from "effect/Schema"; +import { AuthEnvironmentScopes } from "./auth.ts"; +import { ExecutionEnvironmentDescriptor } from "./environment.ts"; + +export const OutboundSourceMessage = Schema.Union([ + Schema.Struct({ type: Schema.Literal("register"), descriptor: ExecutionEnvironmentDescriptor }), + Schema.Struct({ + type: Schema.Literal("grant"), + id: Schema.String.check(Schema.isUUID()), + token: Schema.NonEmptyString.check(Schema.isMaxLength(16_384)), + }), + Schema.Struct({ type: Schema.Literal("pong") }), +]); +export type OutboundSourceMessage = typeof OutboundSourceMessage.Type; + +export const OutboundHubMessage = Schema.Union([ + Schema.Struct({ type: Schema.Literal("connect"), id: Schema.String.check(Schema.isUUID()) }), + Schema.Struct({ + type: Schema.Literal("grant"), + id: Schema.String.check(Schema.isUUID()), + scopes: AuthEnvironmentScopes, + }), + Schema.Struct({ type: Schema.Literal("ping") }), +]); +export type OutboundHubMessage = typeof OutboundHubMessage.Type; + +export const HostedSourceConnection = Schema.Struct({ + serverUrl: Schema.NonEmptyString.check(Schema.isMaxLength(2_048)), + authorization: Schema.NonEmptyString.check(Schema.isMaxLength(16_384)), + expiresAt: Schema.DateTimeUtc, +}); +export type HostedSourceConnection = typeof HostedSourceConnection.Type; diff --git a/packages/contracts/src/transcriptArchive.ts b/packages/contracts/src/transcriptArchive.ts new file mode 100644 index 0000000000000000000000000000000000000000..f5fb6a224690fa0412d1e153bc15495ad10391ab --- /dev/null +++ b/packages/contracts/src/transcriptArchive.ts @@ -0,0 +1,76 @@ +import * as Schema from "effect/Schema"; +import { OrchestrationV2ThreadShell } from "./orchestrationV2.ts"; +import { AssetResource } from "./assets.ts"; +import { OrchestrationProjectShell } from "./orchestrationProject.ts"; +import { EnvironmentId, ThreadId } from "./baseSchemas.ts"; + +export const ArchiveObjectHash = Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/u)); +export const ArchiveObjectReceipt = Schema.Struct({ + hash: ArchiveObjectHash, + size: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), +}); +export const ArchiveManagementRequest = Schema.Union([ + Schema.Struct({ type: Schema.Literal("archive.status") }), + Schema.Struct({ type: Schema.Literal("archive.retry"), threadId: Schema.optionalKey(ThreadId) }), +]); +export type ArchiveManagementRequest = typeof ArchiveManagementRequest.Type; +export const ArchiveStatus = Schema.Struct({ + enabled: Schema.Boolean, + offloaded: Schema.Int, + pending: Schema.Int, + queue: Schema.Array(Schema.Struct({ threadId: ThreadId, nextAttemptAt: Schema.Number })), + storage: Schema.Struct({ + allocatedDatabaseBytes: Schema.Number, + attachmentsBytes: Schema.Number, + nativeProviderLogBytes: Schema.Number, + }), + lastReclamation: Schema.NullOr( + Schema.Struct({ + beforeAllocatedBytes: Schema.Number, + afterAllocatedBytes: Schema.Number, + }), + ), + failures: Schema.Array( + Schema.Struct({ threadId: ThreadId, attempts: Schema.Int, reason: Schema.String }), + ), +}); +export type ArchiveStatus = typeof ArchiveStatus.Type; + +export const ArchiveAssetCreateUrlInput = Schema.Struct({ + environmentId: EnvironmentId, + resource: AssetResource, +}); + +export const ArchivePublication = Schema.Struct({ + environmentId: EnvironmentId, + manifestHash: ArchiveObjectHash, + project: OrchestrationProjectShell, +}); +export type ArchivePublication = typeof ArchivePublication.Type; +export const ArchiveCatalogEntry = Schema.Struct({ + environmentId: EnvironmentId, + manifestHash: ArchiveObjectHash, + throughSequence: Schema.Number, + thread: OrchestrationV2ThreadShell, + project: OrchestrationProjectShell, +}); +export type ArchiveCatalogEntry = typeof ArchiveCatalogEntry.Type; +export const ArchiveCatalogResult = Schema.Struct({ entries: Schema.Array(ArchiveCatalogEntry) }); + +export class ArchiveError extends Schema.TaggedError()("ArchiveError", { + reason: Schema.Literals(["unavailable", "integrity", "changed", "storage"]), + cause: Schema.optionalKey(Schema.Redacted(Schema.Defect())), +}) { + override get message() { + switch (this.reason) { + case "unavailable": + return "The transcript archive is unavailable. Reconnect to its server and retry."; + case "integrity": + return "The archived transcript failed verification. Retry or check the archive storage."; + case "changed": + return "The chat changed while it was being archived. Its local copy was retained."; + case "storage": + return "The transcript archive could not be saved. Check its mounted storage and retry."; + } + } +} diff --git a/packages/shared/package.json b/packages/shared/package.json index b453dad3f3a998b64e8fe4252cebaf146d6800ab..f3a07e22b032c4f704d1d380d658648a7d0bf46b 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -3,6 +3,10 @@ "private": true, "type": "module", "exports": { + "./hostedServerUrl": { + "types": "./src/hostedServerUrl.ts", + "import": "./src/hostedServerUrl.ts" + }, "./forkUpdateDirectory": { "types": "./src/forkUpdateDirectory.ts", "import": "./src/forkUpdateDirectory.ts" diff --git a/packages/shared/src/advertisedEndpoint.ts b/packages/shared/src/advertisedEndpoint.ts index 314d8272c81652a35e593b7a1df7f4f7d63aa088..007c3bb7d0d3f3eb8f77866dc2f616965213a292 100644 --- a/packages/shared/src/advertisedEndpoint.ts +++ b/packages/shared/src/advertisedEndpoint.ts @@ -33,12 +33,26 @@ export function normalizeHttpBaseUrl(rawValue: string): string { throw new Error(`Endpoint must use HTTP or HTTPS. Received ${url.protocol}`); } - url.pathname = "/"; + url.pathname = `${url.pathname.replace(/\/+$/, "")}/`; url.search = ""; url.hash = ""; return url.toString(); } +export function environmentEndpointUrl(baseUrl: string, path: string): string { + const url = new URL(baseUrl); + url.pathname = `${url.pathname.replace(/\/+$/, "")}/`; + url.search = ""; + url.hash = ""; + return new URL(path.replace(/^\/(?!\/)/, ""), url).toString(); +} + +/** Canonical bases end in a slash; existing socket endpoints already include their route. */ +export function environmentWebSocketUrl(wsBaseUrl: string): string { + const url = new URL(wsBaseUrl); + return url.pathname.endsWith("/") ? environmentEndpointUrl(wsBaseUrl, "/ws") : url.toString(); +} + export function deriveWsBaseUrl(httpBaseUrl: string): string { const url = new URL(normalizeHttpBaseUrl(httpBaseUrl)); url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; diff --git a/packages/shared/src/devProxy.ts b/packages/shared/src/devProxy.ts index 709a5449c22e989d2e6211b44a8f870e10480272..3c90b9f58e64670911ff08039ffc8a339b2da35c 100644 --- a/packages/shared/src/devProxy.ts +++ b/packages/shared/src/devProxy.ts @@ -8,7 +8,14 @@ * prefix only Vite knows gets answered with index.html; a prefix only the * server knows redirect-loops through the proxy. */ -export const DEV_PROXIED_PATH_PREFIXES = ["/api", "/oauth", "/.well-known", "/ws", "/mcp"] as const; +export const DEV_PROXIED_PATH_PREFIXES = [ + "/api", + "/oauth", + "/.well-known", + "/ws", + "/mcp", + "/environments", +] as const; /** * Prefixes the proxy must forward with the browser's own Host. MCP OAuth @@ -22,6 +29,7 @@ export const DEV_PROXIED_ORIGIN_PRESERVING_PREFIXES: ReadonlySet = new S "/.well-known", "/mcp", "/ws", + "/environments", ]); export function isDevProxiedPath(pathname: string): boolean { diff --git a/packages/shared/src/hostedServerUrl.ts b/packages/shared/src/hostedServerUrl.ts new file mode 100644 index 0000000000000000000000000000000000000000..74c988c1e5932328382247d4ba939e753c57b002 --- /dev/null +++ b/packages/shared/src/hostedServerUrl.ts @@ -0,0 +1,14 @@ +export function normalizeHostedServer(value: string): string { + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/" + ) { + throw new Error("Enter an HTTPS server address without a path."); + } + return url.origin; +} diff --git a/packages/shared/src/remote.test.ts b/packages/shared/src/remote.test.ts index 5b7a9973652a5e1d261fd38df4fecca8c87a40af..85f40aadb24d106fb64db6d2518cf53aa8fea376 100644 --- a/packages/shared/src/remote.test.ts +++ b/packages/shared/src/remote.test.ts @@ -21,6 +21,25 @@ describe("remote", () => { }); }); + it.each([ + { pairingUrl: "https://hub.example/prefix/environments/one/pair#token=pairing-token" }, + { pairingUrl: "https://hub.example/prefix/environments/one/#token=pairing-token" }, + { + pairingUrl: + "https://app.example/pair?host=https%3A%2F%2Fhub.example%2Fprefix%2Fenvironments%2Fone%2F#token=pairing-token", + }, + { + host: "wss://hub.example/prefix/environments/one///?old=1#fragment", + pairingCode: "pairing-token", + }, + ])("preserves a proxied environment base through pairing: %o", (input) => { + expect(resolveRemotePairingTarget(input)).toEqual({ + credential: "pairing-token", + httpBaseUrl: "https://hub.example/prefix/environments/one/", + wsBaseUrl: "wss://hub.example/prefix/environments/one/", + }); + }); + it("accepts pairing urls that still use a query token", () => { expect( resolveRemotePairingTarget({ diff --git a/packages/shared/src/remote.ts b/packages/shared/src/remote.ts index 55698568fb9efe1e39a05a38760820cd4f671368..baf1d73915976eb4d38fcdbaf14aea5333340961 100644 --- a/packages/shared/src/remote.ts +++ b/packages/shared/src/remote.ts @@ -1,3 +1,4 @@ +import { deriveWsBaseUrl, normalizeHttpBaseUrl } from "./advertisedEndpoint.ts"; import * as Schema from "effect/Schema"; const PAIRING_TOKEN_PARAM = "token"; @@ -97,38 +98,9 @@ const normalizeRemoteBaseUrl = ( protocol: url.protocol, }); } - url.pathname = "/"; - url.search = ""; - url.hash = ""; return url; }; -const toHttpBaseUrl = (url: URL): string => { - const next = new URL(url.toString()); - if (next.protocol === "ws:") { - next.protocol = "http:"; - } else if (next.protocol === "wss:") { - next.protocol = "https:"; - } - next.pathname = "/"; - next.search = ""; - next.hash = ""; - return next.toString(); -}; - -const toWsBaseUrl = (url: URL): string => { - const next = new URL(url.toString()); - if (next.protocol === "http:") { - next.protocol = "ws:"; - } else if (next.protocol === "https:") { - next.protocol = "wss:"; - } - next.pathname = "/"; - next.search = ""; - next.hash = ""; - return next.toString(); -}; - export interface ResolvedRemotePairingTarget { readonly credential: string; readonly httpBaseUrl: string; @@ -211,8 +183,8 @@ export const resolveRemotePairingTarget = (input: { ); return { credential: hostedPairingRequest.token, - httpBaseUrl: toHttpBaseUrl(hostedBackendUrl), - wsBaseUrl: toWsBaseUrl(hostedBackendUrl), + httpBaseUrl: normalizeHttpBaseUrl(hostedBackendUrl.toString()), + wsBaseUrl: deriveWsBaseUrl(hostedBackendUrl.toString()), }; } @@ -220,10 +192,11 @@ export const resolveRemotePairingTarget = (input: { if (!credential) { throw new RemotePairingTokenMissingError({ host: url.host }); } + url.pathname = url.pathname.replace(/\/pair\/?$/, "/"); return { credential, - httpBaseUrl: toHttpBaseUrl(url), - wsBaseUrl: toWsBaseUrl(url), + httpBaseUrl: normalizeHttpBaseUrl(url.toString()), + wsBaseUrl: deriveWsBaseUrl(url.toString()), }; } @@ -239,7 +212,7 @@ export const resolveRemotePairingTarget = (input: { return { credential: pairingCode, - httpBaseUrl: toHttpBaseUrl(normalizedHost), - wsBaseUrl: toWsBaseUrl(normalizedHost), + httpBaseUrl: normalizeHttpBaseUrl(normalizedHost.toString()), + wsBaseUrl: deriveWsBaseUrl(normalizedHost.toString()), }; }; -- 2.54.0