authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 18:48:38-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 18:48:38-07:00
log9422e2ee12fd3e3bfdf6fe5c377672badf7b20ca
treeecef44a0be2f9c7c10ba05c29e1fe25a7b2b7033
parentecfdda5fa804582066c1aa62afea419677c24883
signature Signed by PGP key B5690EEEBB952194

refactor(server): T3 Connect link capabilities live in a CloudLink service (#16265)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

9 files changed, 2215 insertions(+), 2041 deletions(-)

apps/server/src/auth/replayMarkers.ts+1-1
......@@ -7,7 +7,7 @@ import * as Option from "effect/Option";
77import * as Path from "effect/Path";
88import * as Schedule from "effect/Schedule";
99
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts";
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/CloudLink.ts";
1111import * as ServerConfig from "../config.ts";
1212import { forkParked } from "../serverActivation.ts";
1313import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts";
apps/server/src/cloud/CloudLink.test.ts created+268
......@@ -0,0 +1,268 @@
1import { EnvironmentId } from "@t3tools/contracts";
2import { assert, it } from "@effect/vitest";
3import * as Effect from "effect/Effect";
4import * as Fiber from "effect/Fiber";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as FetchHttpClient from "effect/http/FetchHttpClient";
8import { HttpClient, HttpServer } from "effect/http";
9import * as NetAddress from "effect/net/NetAddress";
10import * as NodeServices from "@effect/platform-node/NodeServices";
11
12import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
13import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
14import * as ServerConfig from "../config.ts";
15import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
16import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
17import * as CliTokenManager from "./CliTokenManager.ts";
18import * as CloudLink from "./CloudLink.ts";
19import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
20import {
21 HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
22 PUBLISH_AGENT_ACTIVITY_SECRET,
23 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
24 RELAY_URL_SECRET,
25} from "./config.ts";
26
27const encode = (value: string) => new TextEncoder().encode(value);
28
29/** A linked environment whose secret store can refuse writes, and the relay calls it made. */
30const withService = <A, E>(
31 options: {
32 readonly failHoldWrite?: boolean;
33 readonly failActivityWrite?: boolean;
34 readonly relayFails?: boolean;
35 readonly failActivityRead?: boolean;
36 readonly failHoldRead?: boolean;
37 /** The first relay call reports itself, then waits for this before answering. */
38 readonly holdFirstRelayCall?: {
39 readonly started: () => void;
40 readonly released: Promise<void>;
41 };
42 },
43 body: (input: {
44 readonly preferences: { readonly update: CloudLink.CloudLink["Service"]["updatePreferences"] };
45 readonly stored: Map<string, Uint8Array>;
46 readonly relayCalls: Array<boolean>;
47 }) => Effect.Effect<A, E, never>,
48) =>
49 Effect.gen(function* () {
50 const stored = new Map<string, Uint8Array>([
51 [RELAY_URL_SECRET, encode("https://relay.test")],
52 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, encode("credential")],
53 [HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, encode("false")],
54 [PUBLISH_AGENT_ACTIVITY_SECRET, encode("false")],
55 ]);
56 const relayCalls: Array<boolean> = [];
57 const fetch: typeof globalThis.fetch = Object.assign(
58 (_input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => {
59 const payload = JSON.parse(String(init?.body)) as { holdWebhooksWhileOffline: boolean };
60 relayCalls.push(payload.holdWebhooksWhileOffline);
61 if (relayCalls.length === 1 && options.holdFirstRelayCall) {
62 options.holdFirstRelayCall.started();
63 return options.holdFirstRelayCall.released.then(() => Response.json(payload));
64 }
65 return options.relayFails
66 ? Promise.resolve(new Response("unavailable", { status: 503 }))
67 : Promise.resolve(Response.json(payload));
68 },
69 { preconnect: () => {} },
70 );
71 const layerDependencies = Layer.mergeAll(
72 Layer.mock(ServerSecretStore.ServerSecretStore)({
73 get: (name) =>
74 (options.failActivityRead && name === PUBLISH_AGENT_ACTIVITY_SECRET) ||
75 (options.failHoldRead && name === HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)
76 ? Effect.fail(
77 new ServerSecretStore.SecretStoreReadError({
78 resource: name,
79 cause: new Error("busy"),
80 }),
81 )
82 : Effect.succeed(Option.fromNullishOr(stored.get(name))),
83 set: (name, value) =>
84 (options.failHoldWrite && name === HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET) ||
85 (options.failActivityWrite && name === PUBLISH_AGENT_ACTIVITY_SECRET)
86 ? Effect.fail(
87 new ServerSecretStore.SecretStorePersistError({
88 resource: name,
89 cause: new Error("read-only"),
90 }),
91 )
92 : Effect.sync(() => void stored.set(name, value)),
93 }),
94 Layer.mock(ServerEnvironment.ServerEnvironment)({
95 getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-1")),
96 }),
97 Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ requestCatchUp: () => Effect.void }),
98 // Saving preferences touches none of the link's other dependencies.
99 Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({}),
100 Layer.mock(EnvironmentAuth.EnvironmentAuth)({}),
101 Layer.mock(CliTokenManager.CloudCliTokenManager)({}),
102 Layer.mock(HttpServer.HttpServer)({
103 address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773),
104 }),
105 Layer.mock(ServerConfig.ServerConfig)({} as ServerConfig.ServerConfig["Service"]),
106 Layer.succeed(
107 HttpClient.HttpClient,
108 HttpClient.make(() => Effect.die("unused")),
109 ),
110 NodeServices.layer,
111 );
112 return yield* Effect.gen(function* () {
113 const link = yield* CloudLink.CloudLink;
114 return yield* body({ preferences: { update: link.updatePreferences }, stored, relayCalls });
115 }).pipe(
116 Effect.provide(CloudLink.layer.pipe(Layer.provide(layerDependencies))),
117 Effect.provideService(FetchHttpClient.Fetch, fetch),
118 );
119 });
120
121it.effect("tells the relay before saving the hold setting locally", () =>
122 withService({}, ({ preferences, stored, relayCalls }) =>
123 Effect.gen(function* () {
124 yield* preferences.update({ publishAgentActivity: true, holdWebhooksWhileOffline: true });
125 assert.deepEqual(relayCalls, [true]);
126 assert.equal(
127 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
128 "true",
129 );
130 }),
131 ),
132);
133
134it.effect("puts the relay back when the local save fails", () =>
135 withService({ failHoldWrite: true }, ({ preferences, stored, relayCalls }) =>
136 Effect.gen(function* () {
137 const error = yield* preferences
138 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
139 .pipe(Effect.flip);
140 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
141 assert.deepEqual(relayCalls, [true, false]);
142 assert.equal(
143 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
144 "false",
145 );
146 }),
147 ),
148);
149
150it.effect("leaves the relay untouched when the activity setting can't be saved", () =>
151 withService({ failActivityWrite: true }, ({ preferences, stored, relayCalls }) =>
152 Effect.gen(function* () {
153 const error = yield* preferences
154 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
155 .pipe(Effect.flip);
156 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
157 assert.deepEqual(relayCalls, []);
158 assert.equal(
159 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
160 "false",
161 );
162 }),
163 ),
164);
165
166it.effect("keeps both settings unchanged when the relay refuses the hold change", () =>
167 withService({ relayFails: true }, ({ preferences, stored }) =>
168 Effect.gen(function* () {
169 yield* preferences
170 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
171 .pipe(Effect.flip);
172 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
173 assert.equal(
174 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
175 "false",
176 );
177 }),
178 ),
179);
180
181it.effect("changes nothing when the current activity setting can't be read", () =>
182 withService({ failActivityRead: true, relayFails: true }, ({ preferences, stored, relayCalls }) =>
183 Effect.gen(function* () {
184 const error = yield* preferences
185 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
186 .pipe(Effect.flip);
187 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
188 assert.deepEqual(relayCalls, []);
189 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
190 }),
191 ),
192);
193
194it.effect("two overlapping updates are applied one after the other", () => {
195 let release!: () => void;
196 let started!: () => void;
197 const released = new Promise<void>((resolve) => {
198 release = resolve;
199 });
200 const firstAtRelay = new Promise<void>((resolve) => {
201 started = resolve;
202 });
203 return withService({ holdFirstRelayCall: { started, released } }, ({ preferences, stored }) =>
204 Effect.gen(function* () {
205 const first = yield* preferences
206 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
207 .pipe(Effect.forkChild);
208 yield* Effect.promise(() => firstAtRelay);
209 const second = yield* preferences
210 .update({ publishAgentActivity: false, holdWebhooksWhileOffline: false })
211 .pipe(Effect.forkChild);
212 // Give the second update every chance to run; it must wait for the first.
213 yield* Effect.yieldNow;
214 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "true");
215 release();
216 yield* Fiber.join(first);
217 yield* Fiber.join(second);
218 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
219 assert.equal(
220 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
221 "false",
222 );
223 }),
224 );
225});
226
227it.effect("changes nothing when the current hold setting can't be read", () =>
228 withService({ failHoldRead: true, failHoldWrite: true }, ({ preferences, stored, relayCalls }) =>
229 Effect.gen(function* () {
230 stored.set(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, encode("true"));
231 const error = yield* preferences
232 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: false })
233 .pipe(Effect.flip);
234 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
235 assert.deepEqual(relayCalls, []);
236 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
237 }),
238 ),
239);
240
241it.effect("an update the client walks away from still finishes", () => {
242 let release!: () => void;
243 let started!: () => void;
244 const released = new Promise<void>((resolve) => {
245 release = resolve;
246 });
247 const atRelay = new Promise<void>((resolve) => {
248 started = resolve;
249 });
250 return withService({ holdFirstRelayCall: { started, released } }, ({ preferences, stored }) =>
251 Effect.gen(function* () {
252 const update = yield* preferences
253 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
254 .pipe(Effect.forkChild);
255 yield* Effect.promise(() => atRelay);
256 // The interrupt is in flight before the relay answers.
257 const interrupted = yield* Fiber.interrupt(update).pipe(Effect.forkChild);
258 yield* Effect.yieldNow;
259 release();
260 yield* Fiber.join(interrupted);
261 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "true");
262 assert.equal(
263 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
264 "true",
265 );
266 }),
267 );
268});
apps/server/src/cloud/CloudLink.ts created+1820
......@@ -0,0 +1,1820 @@
1/**
2 * The T3 Connect link lifecycle of this environment: linking it to the relay,
3 * applying and reading the link, unlinking, answering the relay's signed health
4 * and mint requests, and keeping the managed tunnel registered, recovered and
5 * released. HTTP handlers, server startup and shutdown all go through it.
6 */
7import * as NodeCrypto from "node:crypto";
8import {
9 AuthStandardClientScopes,
10 DESKTOP_UPDATE_RESTART_MARKER_FILE,
11 EnvironmentCloudEndpointUnavailableError,
12 type EnvironmentCloudLinkStateResult,
13 type EnvironmentCloudPreferencesRequest,
14 type EnvironmentCloudRelayConfigResult,
15 EnvironmentHttpBadRequestError,
16 EnvironmentHttpConflictError,
17 type EnvironmentHttpForbiddenError,
18 EnvironmentHttpInternalServerError,
19 EnvironmentHttpUnauthorizedError,
20} from "@t3tools/contracts";
21import {
22 RelayCloudEnvironmentHealthProofPayload,
23 type RelayCloudEnvironmentHealthRequest,
24 RelayCloudMintCredentialProofPayload,
25 type RelayCloudMintCredentialRequest,
26 type RelayEnvironmentConfigRequest,
27 type RelayEnvironmentHealthResponse,
28 type RelayEnvironmentHealthResponseProofPayload,
29 type RelayEnvironmentLinkProof,
30 type RelayEnvironmentLinkProofPayload,
31 RelayEnvironmentLinkChallengeResponse,
32 RelayEnvironmentLinkResponse,
33 type RelayEnvironmentMintResponse,
34 type RelayEnvironmentMintResponseProofPayload,
35 type RelayLinkProofRequest,
36 type RelayManagedEndpointOrigin,
37 type RelayManagedEndpointRecoveryProofPayload,
38 RelayManagedEndpointRecoveryRegistrationResponse,
39 RelayManagedEndpointRecoveryResponse,
40 type RelayManagedEndpointRuntimeConfig,
41 RelayOkResponse,
42} from "@t3tools/contracts/relay";
43import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
44import {
45 normalizeRelayIssuer,
46 RELAY_HEALTH_REQUEST_TYP,
47 RELAY_HEALTH_RESPONSE_TYP,
48 RELAY_LINK_PROOF_TYP,
49 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
50 RELAY_MINT_REQUEST_TYP,
51 RELAY_MINT_RESPONSE_TYP,
52 signRelayJwt,
53 verifyRelayJwt,
54} from "@t3tools/shared/relayJwt";
55import { isSecureRelayUrl } from "@t3tools/shared/relayUrl";
56import * as Clock from "effect/Clock";
57import * as Context from "effect/Context";
58import * as Crypto from "effect/Crypto";
59import * as DateTime from "effect/DateTime";
60import * as Duration from "effect/Duration";
61import * as Effect from "effect/Effect";
62import * as FileSystem from "effect/FileSystem";
63import * as Layer from "effect/Layer";
64import * as Option from "effect/Option";
65import * as Path from "effect/Path";
66import type * as PlatformError from "effect/PlatformError";
67import * as Schedule from "effect/Schedule";
68import * as Schema from "effect/Schema";
69import * as Semaphore from "effect/Semaphore";
70import {
71 HttpClient,
72 HttpClientRequest,
73 HttpClientResponse,
74 type HttpServerRequest,
75} from "effect/http";
76import type * as HttpClientError from "effect/http/HttpClientError";
77import * as HttpServer from "effect/http/HttpServer";
78
79import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
80import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
81import * as ServerConfig from "../config.ts";
82import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
83import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
84import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts";
85import {
86 type CliDesiredLinkMode,
87 readCliDesiredCloudLink,
88 readCliDesiredLinkMode,
89 setCliDesiredCloudLink,
90} from "./CliState.ts";
91import * as CliTokenManager from "./CliTokenManager.ts";
92import {
93 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
94 CLOUD_ENDPOINT_RUNTIME_CONFIG,
95 CLOUD_LINKED_USER_ID,
96 CLOUD_MINT_PUBLIC_KEY,
97 decodeConfirmedOrigin,
98 decodeRuntimeConfig,
99 encodeConfirmedOriginJson,
100 encodeEndpointRuntimeConfigJson,
101 HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
102 PUBLISH_AGENT_ACTIVITY_SECRET,
103 readRelayConnection,
104 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
105 RELAY_ISSUER_SECRET,
106 RELAY_URL_SECRET,
107} from "./config.ts";
108import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
109import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
110import { relayUrlConfig } from "./publicConfig.ts";
111import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
112import {
113 SERVICE_STATE_FILE,
114 SERVICE_STOP_MARKER_FILE,
115 serviceStateHasPendingUpdate,
116} from "./serviceProtocol.ts";
117
118const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
119const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
120const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
121const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
122/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
123export const CLOUD_REPLAY_MARKER_PREFIXES = [
124 CLOUD_MINT_NONCE_PREFIX,
125 CLOUD_MINT_JTI_PREFIX,
126 CLOUD_HEALTH_NONCE_PREFIX,
127 CLOUD_HEALTH_JTI_PREFIX,
128] as const;
129const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
130const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
131// The desktop app stops its backends within seconds of writing the marker.
132const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
133const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
134const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
135
136const failEnvironmentCloudInternalError =
137 (message: string) =>
138 (cause: unknown): Effect.Effect<never, EnvironmentHttpInternalServerError> =>
139 Effect.logError(message, { cause }).pipe(
140 Effect.flatMap(() => Effect.fail(new EnvironmentHttpInternalServerError({ message }))),
141 );
142
143const failCloudCliTokenManagerError = (error: CliTokenManager.CloudCliTokenManagerError) =>
144 failEnvironmentCloudInternalError(error.message)(error);
145
146/** A failed rollback leaves a setting changed; it is logged, not hidden. */
147const rollbackFailed = (cause: unknown) =>
148 Effect.logWarning("Could not roll back a T3 Connect preference", { cause });
149
150const requireRelayUrl = relayUrlConfig.pipe(
151 Effect.mapError(
152 () =>
153 new EnvironmentHttpInternalServerError({
154 message: "T3CODE_RELAY_URL must be configured as a secure absolute HTTPS origin.",
155 }),
156 ),
157);
158
159function bytesToString(bytes: Uint8Array): string {
160 return new TextDecoder().decode(bytes);
161}
162
163function stringToBytes(value: string): Uint8Array {
164 return new TextEncoder().encode(value);
165}
166
167export function consumeCloudReplayGuards(input: {
168 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
169 readonly names: ReadonlyArray<string>;
170 readonly value: Uint8Array;
171}) {
172 return Effect.forEach(
173 input.names,
174 (name) =>
175 input.secrets.create(name, input.value).pipe(
176 Effect.as(true),
177 Effect.catchIf(ServerSecretStore.isSecretStoreError, (error) =>
178 ServerSecretStore.isSecretAlreadyExistsError(error)
179 ? Effect.succeed(false)
180 : Effect.fail(error),
181 ),
182 ),
183 { concurrency: input.names.length },
184 ).pipe(Effect.map((created) => created.every(Boolean)));
185}
186
187function normalizePemForSignedPayload(value: string): string {
188 return value.trim();
189}
190
191function normalizeHostname(hostname: string): string {
192 return hostname
193 .trim()
194 .toLowerCase()
195 .replace(/^\[(.*)\]$/, "$1");
196}
197
198function validateCloudMintPublicKey(
199 publicKey: string,
200): Effect.Effect<void, EnvironmentHttpBadRequestError> {
201 return Effect.try({
202 try: () => NodeCrypto.createPublicKey(publicKey.replace(/\\n/g, "\n")),
203 catch: () =>
204 new EnvironmentHttpBadRequestError({
205 message: "Cloud mint public key must be a valid Ed25519 public key.",
206 }),
207 }).pipe(
208 Effect.flatMap((key) =>
209 key.asymmetricKeyType === "ed25519"
210 ? Effect.void
211 : Effect.fail(
212 new EnvironmentHttpBadRequestError({
213 message: "Cloud mint public key must be a valid Ed25519 public key.",
214 }),
215 ),
216 ),
217 );
218}
219
220function validateRelayConfigPayload(
221 payload: RelayEnvironmentConfigRequest,
222): Effect.Effect<void, EnvironmentHttpBadRequestError> {
223 if (!isSecureRelayUrl(payload.relayUrl)) {
224 return Effect.fail(
225 new EnvironmentHttpBadRequestError({
226 message: "Relay URL must be a secure absolute HTTPS URL.",
227 }),
228 );
229 }
230 if (payload.relayIssuer !== undefined && !isSecureRelayUrl(payload.relayIssuer)) {
231 return Effect.fail(
232 new EnvironmentHttpBadRequestError({
233 message: "Relay issuer must be a secure absolute HTTPS URL.",
234 }),
235 );
236 }
237 if (payload.environmentCredential.trim().length === 0) {
238 return Effect.fail(
239 new EnvironmentHttpBadRequestError({
240 message: "Relay environment credential is required.",
241 }),
242 );
243 }
244 if (payload.cloudUserId.trim().length === 0) {
245 return Effect.fail(
246 new EnvironmentHttpBadRequestError({
247 message: "Cloud user id is required.",
248 }),
249 );
250 }
251 return Effect.void;
252}
253
254function isLoopbackHostname(hostname: string): boolean {
255 return LOOPBACK_HOSTNAMES.has(normalizeHostname(hostname));
256}
257
258function firstForwardedHeaderValue(value: string | undefined): string | undefined {
259 const first = value?.split(",")[0]?.trim();
260 return first && first.length > 0 ? first : undefined;
261}
262
263function requestAbsoluteUrl(request: HttpServerRequest.HttpServerRequest): string | null {
264 try {
265 return new URL(request.originalUrl).href;
266 } catch {
267 const host = firstForwardedHeaderValue(request.headers.host) ?? "127.0.0.1";
268 try {
269 return new URL(request.originalUrl, `http://${host}`).href;
270 } catch {
271 return null;
272 }
273 }
274}
275
276function hasForwardedAuthorityHeaders(request: HttpServerRequest.HttpServerRequest): boolean {
277 return (
278 firstForwardedHeaderValue(request.headers["x-forwarded-host"]) !== undefined ||
279 firstForwardedHeaderValue(request.headers["x-forwarded-proto"]) !== undefined
280 );
281}
282
283function endpointRequestPort(url: URL): number {
284 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
285}
286
287export function parseManagedEndpointLocalOrigin(localOrigin: string) {
288 const url = new URL(localOrigin);
289 if (
290 localOrigin !== localOrigin.trim() ||
291 (url.protocol !== "http:" && url.protocol !== "https:") ||
292 url.username !== "" ||
293 url.password !== "" ||
294 url.pathname !== "/" ||
295 url.search !== "" ||
296 url.hash !== "" ||
297 localOrigin.includes("?") ||
298 localOrigin.includes("#")
299 ) {
300 throw new Error("Invalid local origin");
301 }
302 const wsUrl = new URL(url.origin);
303 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
304 return {
305 httpBaseUrl: url.origin,
306 wsBaseUrl: wsUrl.origin,
307 origin: {
308 localHttpHost: url.hostname,
309 localHttpPort: endpointRequestPort(url),
310 } satisfies RelayManagedEndpointOrigin,
311 };
312}
313
314function isAllowedEndpointOrigin(input: {
315 readonly origin: RelayManagedEndpointOrigin;
316 readonly requestUrl: string;
317}): boolean {
318 if (!isLoopbackHostname(input.origin.localHttpHost)) {
319 return false;
320 }
321
322 const url = new URL(input.requestUrl);
323 if (!isLoopbackHostname(url.hostname)) {
324 return false;
325 }
326
327 return input.origin.localHttpPort === endpointRequestPort(url);
328}
329
330// A managed (Cloudflare tunnel) endpoint is provisioned by the relay and must
331// point at a loopback origin. A manual endpoint is reached out of band (e.g.
332// Tailscale) or not advertised at all for publish-only links, so it is not
333// tied to the managed-tunnel scope.
334export function isSupportedLinkProviderKind(request: RelayLinkProofRequest): boolean {
335 return (
336 request.endpoint.providerKind === "cloudflare_tunnel" ||
337 request.endpoint.providerKind === "manual"
338 );
339}
340
341export function linkProofScopes(
342 request: RelayLinkProofRequest,
343): RelayEnvironmentLinkProofPayload["scopes"] {
344 return request.endpoint.providerKind === "cloudflare_tunnel"
345 ? ["agent_activity_notifications", "managed_tunnels"]
346 : ["agent_activity_notifications"];
347}
348
349function hasExactScope(input: {
350 readonly scopes: ReadonlyArray<string>;
351 readonly expected: string;
352}): boolean {
353 return input.scopes.length === 1 && input.scopes[0] === input.expected;
354}
355
356function hasBoundedCloudProofLifetime(input: {
357 readonly iat: number;
358 readonly exp: number;
359 readonly nowSeconds: number;
360}): boolean {
361 return (
362 input.exp > input.iat &&
363 input.exp - input.iat <= CLOUD_PROOF_MAX_LIFETIME_SECONDS &&
364 input.iat <= input.nowSeconds + CLOUD_PROOF_CLOCK_SKEW_SECONDS
365 );
366}
367
368function managedEndpointRuntimeConfigsMatch(
369 left: RelayManagedEndpointRuntimeConfig,
370 right: RelayManagedEndpointRuntimeConfig,
371): boolean {
372 return (
373 left.providerKind === right.providerKind &&
374 left.connectorToken === right.connectorToken &&
375 left.tunnelId === right.tunnelId &&
376 left.tunnelName === right.tunnelName
377 );
378}
379
380const decodeCloudHealthProof = Schema.decodeUnknownEffect(RelayCloudEnvironmentHealthProofPayload);
381const decodeCloudMintProof = Schema.decodeUnknownEffect(RelayCloudMintCredentialProofPayload);
382
383// The launcher owns this durable state, so read it directly both when a trial
384// decides whether it owns pre-activation cleanup and while a server tears down.
385export const pendingServiceUpdateExists = Effect.gen(function* () {
386 const config = yield* ServerConfig.ServerConfig;
387 const fs = yield* FileSystem.FileSystem;
388 const path = yield* Path.Path;
389 const runtimeDir = path.join(config.baseDir, "runtime");
390 const stateText = yield* fs
391 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
392 .pipe(Effect.option);
393 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
394});
395
396// A pending update alone is not proof a replacement server is coming: an
397// explicit launcher stop (`t3 service uninstall`, `systemctl stop`,
398// `launchctl bootout`) during
399// the pending window also tears this server down. The launcher marks that case
400// just before it signals the child, so pending + no marker is the handoff.
401const pendingUpdateHandoffExists = Effect.gen(function* () {
402 if (!(yield* pendingServiceUpdateExists)) {
403 return false;
404 }
405 const config = yield* ServerConfig.ServerConfig;
406 const fs = yield* FileSystem.FileSystem;
407 const path = yield* Path.Path;
408 const runtimeDir = path.join(config.baseDir, "runtime");
409 const stopping = yield* fs
410 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
411 .pipe(Effect.orElseSucceed(() => false));
412 return !stopping;
413});
414
415// The desktop app writes its marker right before it stops this server to
416// install an update, whether a remote client or the local app started it.
417// Reading consumes it, so shutdown checks it first. Only a fresh marker counts,
418// so a marker the server never read (a hard kill) cannot keep the tunnel on a
419// later quit.
420const desktopUpdateRestartPending = Effect.gen(function* () {
421 const config = yield* ServerConfig.ServerConfig;
422 const fs = yield* FileSystem.FileSystem;
423 const path = yield* Path.Path;
424 const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE);
425 const marker = yield* fs.stat(markerPath).pipe(Effect.option);
426 if (Option.isNone(marker)) {
427 return false;
428 }
429 yield* fs.remove(markerPath).pipe(Effect.ignore);
430 const now = yield* Clock.currentTimeMillis;
431 return Option.match(marker.value.mtime, {
432 onNone: () => false,
433 onSome: (writtenAt) =>
434 now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL),
435 });
436});
437
438type ManagedTunnelRecoveryProofInput = {
439 readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"];
440 readonly cloudUserId: string;
441 readonly relayUrl: string;
442} & (
443 | {
444 readonly action: "register";
445 readonly tunnelId: string;
446 readonly origin: RelayManagedEndpointOrigin;
447 }
448 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
449);
450
451/** Failures of the link work that runs outside a request: startup, recovery and shutdown. */
452type CloudLinkBackgroundError =
453 | EnvironmentCloudEndpointUnavailableError
454 | EnvironmentHttpBadRequestError
455 | EnvironmentHttpForbiddenError
456 | EnvironmentHttpInternalServerError
457 | EnvironmentHttpUnauthorizedError
458 | ServerSecretStore.SecretStoreError
459 | Schema.SchemaError
460 | PlatformError.PlatformError;
461
462type ManagedTunnelRegistration =
463 | { readonly status: "not_linked" | "superseded" }
464 | { readonly status: "recovery_required"; readonly config: RelayManagedEndpointRuntimeConfig }
465 | {
466 readonly status: "ready";
467 readonly endpointRuntimeStatus: ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus;
468 };
469
470export class CloudLink extends Context.Service<
471 CloudLink,
472 {
473 /**
474 * Signs the proof the relay asks for to link this environment. The request
475 * must reach this server directly on the loopback origin it names.
476 */
477 readonly linkProof: (
478 request: RelayLinkProofRequest,
479 httpRequest: HttpServerRequest.HttpServerRequest,
480 ) => Effect.Effect<
481 RelayEnvironmentLinkProof,
482 EnvironmentHttpBadRequestError | EnvironmentHttpInternalServerError
483 >;
484 /** Installs the link the relay returned and, for a managed tunnel, confirms its origin. */
485 readonly applyRelayConfig: (
486 payload: RelayEnvironmentConfigRequest,
487 ) => Effect.Effect<
488 EnvironmentCloudRelayConfigResult,
489 | EnvironmentCloudEndpointUnavailableError
490 | EnvironmentHttpBadRequestError
491 | EnvironmentHttpConflictError
492 | EnvironmentHttpForbiddenError
493 | EnvironmentHttpInternalServerError
494 | EnvironmentHttpUnauthorizedError
495 >;
496 readonly linkState: () => Effect.Effect<
497 EnvironmentCloudLinkStateResult,
498 EnvironmentHttpInternalServerError
499 >;
500 /** Stops the tunnel and forgets the link, including the CLI's wish to keep it. */
501 readonly unlink: () => Effect.Effect<
502 EnvironmentCloudRelayConfigResult,
503 EnvironmentHttpInternalServerError
504 >;
505 /**
506 * Saves this environment's T3 Connect preferences, all or nothing, and
507 * returns the link state. The activity setting is saved first. Holding
508 * webhooks while offline is decided by the relay, so the relay is told
509 * before the local copy is saved. If either step fails, the activity
510 * setting is put back, and so is the relay when only the local save failed.
511 */
512 readonly updatePreferences: (
513 input: EnvironmentCloudPreferencesRequest,
514 ) => Effect.Effect<
515 EnvironmentCloudLinkStateResult,
516 EnvironmentHttpBadRequestError | EnvironmentHttpInternalServerError
517 >;
518 /** Answers the relay's signed health check once per proof. */
519 readonly answerHealthRequest: (
520 request: RelayCloudEnvironmentHealthRequest,
521 ) => Effect.Effect<
522 RelayEnvironmentHealthResponse,
523 | EnvironmentHttpConflictError
524 | EnvironmentHttpInternalServerError
525 | EnvironmentHttpUnauthorizedError
526 >;
527 /** Issues a short-lived pairing credential for a client the relay vouched for, once per proof. */
528 readonly mintCredential: (
529 request: RelayCloudMintCredentialRequest,
530 ) => Effect.Effect<
531 RelayEnvironmentMintResponse,
532 | EnvironmentHttpConflictError
533 | EnvironmentHttpInternalServerError
534 | EnvironmentHttpUnauthorizedError
535 >;
536 /** Links this environment with the stored CLI authorization and records the CLI's wish. */
537 readonly reconcileDesiredLink: (
538 localOrigin: string,
539 ) => Effect.Effect<
540 CliDesiredLinkMode,
541 | CloudLinkBackgroundError
542 | EnvironmentHttpConflictError
543 | EnvironmentAuth.ServerAuthInternalError
544 | CliTokenManager.CloudCliAuthorizationDeniedError
545 >;
546 /** As `reconcileDesiredLink`, but returns null when the CLI no longer wants a link. */
547 readonly reconcileDesiredLinkIfStillDesired: (
548 localOrigin: string,
549 ) => Effect.Effect<
550 CliDesiredLinkMode | null,
551 | CloudLinkBackgroundError
552 | EnvironmentHttpConflictError
553 | EnvironmentAuth.ServerAuthInternalError
554 | CliTokenManager.CloudCliAuthorizationDeniedError
555 >;
556 /** Tells the relay which local origin the stored tunnel serves, then starts it. */
557 readonly registerManagedTunnelRecovery: (
558 localOrigin: string,
559 options?: { readonly retryRuntimeFailures?: boolean },
560 ) => Effect.Effect<ManagedTunnelRegistration, CloudLinkBackgroundError>;
561 /** Asks the relay for a replacement tunnel, unless the stored one changed meanwhile. */
562 readonly recoverManagedTunnel: (
563 localOrigin: string,
564 expectedConfig?: RelayManagedEndpointRuntimeConfig,
565 options?: { readonly retryRuntimeFailures?: boolean },
566 ) => Effect.Effect<boolean, CloudLinkBackgroundError>;
567 /**
568 * Starts the stored tunnel. By default only one the relay already confirmed
569 * on this origin; startup falls back to any stored tunnel when the relay
570 * stays unreachable.
571 */
572 readonly startManagedTunnelIfOriginConfirmed: (
573 localOrigin: string,
574 options?: { readonly requireConfirmedOrigin?: boolean },
575 ) => Effect.Effect<
576 boolean,
577 | EnvironmentCloudEndpointUnavailableError
578 | EnvironmentHttpBadRequestError
579 | ServerSecretStore.SecretStoreError
580 >;
581 /** Deletes a CLI-managed tunnel when this server goes offline for good. */
582 readonly releaseManagedTunnelOnShutdown: () => Effect.Effect<
583 boolean,
584 | CloudLinkBackgroundError
585 | CliTokenManager.CloudCliTokenManagerError
586 | HttpClientError.HttpClientError
587 >;
588 }
589>()("t3/cloud/CloudLink") {}
590
591const make = Effect.gen(function* () {
592 const secrets = yield* ServerSecretStore.ServerSecretStore;
593 const environment = yield* ServerEnvironment.ServerEnvironment;
594 const endpointRuntime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime;
595 const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth;
596 const cliTokenManager = yield* CliTokenManager.CloudCliTokenManager;
597 const httpClient = yield* HttpClient.HttpClient;
598 const httpServer = yield* HttpServer.HttpServer;
599 const awarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
600 const crypto = yield* Crypto.Crypto;
601 const config = yield* ServerConfig.ServerConfig;
602 const fileSystem = yield* FileSystem.FileSystem;
603 const path = yield* Path.Path;
604
605 const withSecrets = Effect.provideService(ServerSecretStore.ServerSecretStore, secrets);
606 const withRuntimeFiles = <A, E>(
607 effect: Effect.Effect<A, E, ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path>,
608 ) =>
609 effect.pipe(
610 Effect.provideService(ServerConfig.ServerConfig, config),
611 Effect.provideService(FileSystem.FileSystem, fileSystem),
612 Effect.provideService(Path.Path, path),
613 );
614
615 const validateLinkedCloudUser = (
616 cloudUserId: string,
617 ): Effect.Effect<void, EnvironmentAuth.ServerAuthInternalError | EnvironmentHttpConflictError> =>
618 secrets.get(CLOUD_LINKED_USER_ID).pipe(
619 Effect.mapError(
620 (cause) =>
621 new EnvironmentAuth.ServerAuthLinkedCloudAccountVerificationError({
622 cause,
623 }),
624 ),
625 Effect.flatMap((existing) => {
626 if (Option.isNone(existing)) {
627 return Effect.void;
628 }
629 const existingCloudUserId = bytesToString(existing.value);
630 return existingCloudUserId === cloudUserId
631 ? Effect.void
632 : Effect.fail(
633 new EnvironmentHttpConflictError({
634 message:
635 "This environment is already linked to a different cloud account. Unlink it before switching accounts.",
636 }),
637 );
638 }),
639 );
640
641 const readInstalledCloudUserId: Effect.Effect<string, EnvironmentAuth.ServerAuthInternalError> =
642 secrets.get(CLOUD_LINKED_USER_ID).pipe(
643 Effect.mapError(
644 (cause) =>
645 new EnvironmentAuth.ServerAuthLinkedCloudAccountReadError({
646 cause,
647 }),
648 ),
649 Effect.flatMap((bytes) =>
650 Option.isSome(bytes)
651 ? Effect.succeed(bytesToString(bytes.value))
652 : Effect.fail(new EnvironmentAuth.ServerAuthLinkedCloudAccountMissingError({})),
653 ),
654 );
655
656 const makeCloudLinkProof = Effect.fn("environment.cloud.makeLinkProof")(function* (
657 request: RelayLinkProofRequest,
658 requestUrl: string,
659 ) {
660 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
661 if (
662 !isSupportedLinkProviderKind(request) ||
663 !isAllowedEndpointOrigin({
664 origin: request.origin,
665 requestUrl,
666 })
667 ) {
668 return yield* new EnvironmentHttpBadRequestError({
669 message: "Invalid managed endpoint origin.",
670 });
671 }
672 const now = yield* DateTime.now;
673 const expiresAt = DateTime.add(now, { minutes: 5 });
674 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
675 const descriptor = yield* environment.getDescriptor;
676 const payload = {
677 iss: `t3-env:${descriptor.environmentId}`,
678 aud: normalizeRelayIssuer(request.relayIssuer),
679 sub: descriptor.environmentId,
680 jti: yield* crypto.randomUUIDv4,
681 iat: nowSeconds,
682 exp: Math.floor(expiresAt.epochMilliseconds / 1_000),
683 challenge: request.challenge,
684 descriptor,
685 environmentId: descriptor.environmentId,
686 environmentPublicKey: normalizePemForSignedPayload(keyPair.publicKey),
687 endpoint: request.endpoint,
688 origin: request.origin,
689 scopes: linkProofScopes(request),
690 } satisfies RelayEnvironmentLinkProofPayload;
691 return yield* signRelayJwt({
692 privateKey: keyPair.privateKey,
693 typ: RELAY_LINK_PROOF_TYP,
694 payload,
695 }).pipe(
696 Effect.mapError(
697 (cause) =>
698 new EnvironmentAuth.ServerAuthCloudLinkJwtSigningError({
699 cause,
700 }),
701 ),
702 );
703 });
704
705 const linkProof = Effect.fn("environment.cloud.linkProof")(
706 function* (request: RelayLinkProofRequest, httpRequest: HttpServerRequest.HttpServerRequest) {
707 const requestUrl = requestAbsoluteUrl(httpRequest);
708 if (requestUrl === null || hasForwardedAuthorityHeaders(httpRequest)) {
709 return yield* new EnvironmentHttpBadRequestError({
710 message: "Invalid managed endpoint origin.",
711 });
712 }
713 const proof = yield* makeCloudLinkProof(request, requestUrl);
714 return proof satisfies RelayEnvironmentLinkProof;
715 },
716 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
717 failEnvironmentCloudInternalError(error.message)(error),
718 ),
719 Effect.catchIf(
720 ServerSecretStore.isSecretStoreError,
721 failEnvironmentCloudInternalError("Could not generate environment link proof."),
722 ),
723 Effect.catchTag(
724 "PlatformError",
725 failEnvironmentCloudInternalError("Could not generate environment link proof."),
726 ),
727 );
728
729 const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(
730 function* (input: {
731 readonly config: RelayManagedEndpointRuntimeConfig;
732 readonly configJson: string;
733 readonly origin: RelayManagedEndpointOrigin;
734 }) {
735 return yield* endpointRuntime.withLinkStateLock(
736 Effect.gen(function* () {
737 const currentConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
738 if (
739 Option.isNone(currentConfig) ||
740 bytesToString(currentConfig.value) !== input.configJson
741 ) {
742 return null;
743 }
744 const status = yield* endpointRuntime.applyConfig(input.config);
745 if (status.status !== "running") {
746 return yield* new EnvironmentCloudEndpointUnavailableError({
747 message: "Managed endpoint runtime could not be started.",
748 endpointRuntimeStatus: status,
749 });
750 }
751 const marker = yield* encodeConfirmedOriginJson({
752 config: input.config,
753 origin: input.origin,
754 });
755 yield* secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
756 return status;
757 }),
758 );
759 },
760 );
761
762 const activateManagedTunnelWithRetry = (
763 input: {
764 readonly config: RelayManagedEndpointRuntimeConfig;
765 readonly configJson: string;
766 readonly origin: RelayManagedEndpointOrigin;
767 },
768 retryRuntimeFailures: boolean,
769 ) => {
770 const activate = activateManagedTunnel(input);
771 return retryRuntimeFailures
772 ? activate.pipe(
773 Effect.retry({
774 while: (error) =>
775 error._tag === "EnvironmentCloudEndpointUnavailableError" &&
776 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus(
777 error.endpointRuntimeStatus,
778 ),
779 schedule: Schedule.exponential("1 second").pipe(
780 Schedule.modifyDelay(({ duration }) =>
781 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
782 ),
783 Schedule.jittered,
784 ),
785 }),
786 )
787 : activate;
788 };
789
790 const startManagedTunnelIfOriginConfirmed = Effect.fn(
791 "environment.cloud.startManagedCloudTunnelIfOriginConfirmed",
792 )(function* (localOrigin: string, options?: { readonly requireConfirmedOrigin?: boolean }) {
793 const requireConfirmedOrigin = options?.requireConfirmedOrigin ?? true;
794 const parsedOrigin = yield* Effect.try({
795 try: () => parseManagedEndpointLocalOrigin(localOrigin),
796 catch: () =>
797 new EnvironmentHttpBadRequestError({
798 message: "Could not resolve local environment origin.",
799 }),
800 });
801 return yield* endpointRuntime.withLinkStateLock(
802 Effect.gen(function* () {
803 const [runtimeBytes, markerBytes] = yield* Effect.all([
804 secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
805 secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
806 ]);
807 if (Option.isNone(runtimeBytes)) return false;
808 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value)));
809 if (config === null || config.providerKind !== "cloudflare_tunnel") return false;
810 // With the marker required, only a config the relay already confirmed on
811 // this port may start. Without it, startup is falling back after the
812 // relay stayed unreachable: an unconfirmed origin may send traffic to a
813 // stale port, but that beats no remote access at all.
814 if (requireConfirmedOrigin) {
815 if (Option.isNone(markerBytes)) return false;
816 const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value)));
817 if (
818 marker === null ||
819 !managedEndpointRuntimeConfigsMatch(marker.config, config) ||
820 marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost ||
821 marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort
822 ) {
823 return false;
824 }
825 }
826 const status = yield* endpointRuntime.applyConfig(config);
827 if (status.status !== "running") {
828 return yield* new EnvironmentCloudEndpointUnavailableError({
829 message: "Managed endpoint runtime could not be started.",
830 endpointRuntimeStatus: status,
831 });
832 }
833 return true;
834 }),
835 );
836 });
837
838 const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
839 payload: RelayEnvironmentConfigRequest,
840 options?: {
841 readonly lockHeld?: boolean;
842 readonly confirmedOrigin?: RelayManagedEndpointOrigin;
843 },
844 ) {
845 const apply = Effect.gen(function* () {
846 yield* validateRelayConfigPayload(payload);
847 yield* validateLinkedCloudUser(payload.cloudUserId);
848 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
849 // Reject unsupported runtimes before touching the connector so a bad
850 // payload cannot stop a healthy tunnel on its way to a 503.
851 if (
852 payload.endpointRuntime !== null &&
853 payload.endpointRuntime.providerKind !== "cloudflare_tunnel"
854 ) {
855 return yield* new EnvironmentCloudEndpointUnavailableError({
856 message: "Managed endpoint runtime could not be started.",
857 endpointRuntimeStatus: {
858 status: "unsupported",
859 providerKind: payload.endpointRuntime.providerKind,
860 },
861 });
862 }
863 yield* endpointRuntime.applyConfig(null);
864 yield* secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
865
866 yield* secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
867 yield* secrets.set(
868 RELAY_ISSUER_SECRET,
869 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
870 );
871 yield* secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
872 yield* secrets.set(
873 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
874 stringToBytes(payload.environmentCredential),
875 );
876 yield* secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey));
877 yield* awarenessRelay.requestCatchUp();
878 if (payload.endpointRuntime) {
879 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
880 yield* secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson));
881 } else {
882 yield* secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
883 }
884 if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) {
885 return {
886 ok: true,
887 endpointRuntimeStatus: { status: "disabled" },
888 } satisfies EnvironmentCloudRelayConfigResult;
889 }
890 const endpointRuntimeStatus = yield* endpointRuntime.applyConfig(payload.endpointRuntime);
891 if (endpointRuntimeStatus.status !== "running") {
892 return yield* new EnvironmentCloudEndpointUnavailableError({
893 message: "Managed endpoint runtime could not be started.",
894 endpointRuntimeStatus,
895 });
896 }
897 const marker = yield* encodeConfirmedOriginJson({
898 config: payload.endpointRuntime,
899 origin: options.confirmedOrigin,
900 });
901 yield* secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
902 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
903 });
904 return yield* options?.lockHeld ? apply : endpointRuntime.withLinkStateLock(apply);
905 });
906
907 const relayClientRequest = <A>(input: {
908 readonly url: string;
909 readonly token: string;
910 readonly payload: unknown;
911 readonly schema: Schema.Decoder<A>;
912 readonly timeout?: Duration.Input;
913 }) =>
914 HttpClientRequest.post(input.url).pipe(
915 HttpClientRequest.bearerToken(input.token),
916 HttpClientRequest.bodyJson(input.payload),
917 Effect.flatMap(httpClient.execute),
918 Effect.flatMap(filterRelayResponse),
919 Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
920 Effect.timeout(input.timeout ?? "10 seconds"),
921 Effect.mapError(relayRequestError),
922 withRelayClientTracing,
923 );
924
925 const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
926 function* (localOrigin: string) {
927 const parsedOrigin = yield* Effect.try({
928 try: () => parseManagedEndpointLocalOrigin(localOrigin),
929 catch: () =>
930 new EnvironmentHttpBadRequestError({
931 message: "Could not resolve local environment origin.",
932 }),
933 });
934 const token = yield* cliTokenManager.getExisting.pipe(
935 Effect.flatMap(
936 Option.match({
937 onNone: () =>
938 Effect.fail(
939 new EnvironmentHttpUnauthorizedError({
940 message: "Run `t3 connect link` to authorize this environment.",
941 }),
942 ),
943 onSome: Effect.succeed,
944 }),
945 ),
946 );
947 const mode = yield* readCliDesiredLinkMode.pipe(withSecrets);
948 const managedTunnelsEnabled = mode !== "publish_only";
949 const relayUrl = yield* requireRelayUrl;
950 const challenge = yield* relayClientRequest({
951 url: `${relayUrl}/v1/client/environment-link-challenges`,
952 token: token.accessToken,
953 payload: {
954 notificationsEnabled: true,
955 liveActivitiesEnabled: true,
956 managedTunnelsEnabled,
957 },
958 schema: RelayEnvironmentLinkChallengeResponse,
959 });
960 const proof = yield* makeCloudLinkProof(
961 {
962 challenge: challenge.challenge,
963 relayIssuer: relayUrl,
964 endpoint: {
965 httpBaseUrl: parsedOrigin.httpBaseUrl,
966 wsBaseUrl: parsedOrigin.wsBaseUrl,
967 providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",
968 },
969 origin: parsedOrigin.origin,
970 },
971 parsedOrigin.httpBaseUrl,
972 );
973 const link = yield* relayClientRequest({
974 url: `${relayUrl}/v1/client/environment-links`,
975 token: token.accessToken,
976 payload: {
977 proof,
978 notificationsEnabled: true,
979 liveActivitiesEnabled: true,
980 managedTunnelsEnabled,
981 },
982 schema: RelayEnvironmentLinkResponse,
983 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
984 });
985 yield* setCliDesiredCloudLink(true, mode).pipe(withSecrets);
986 yield* applyCloudRelayConfig(
987 {
988 relayUrl,
989 relayIssuer: link.relayIssuer,
990 cloudUserId: link.cloudUserId,
991 environmentCredential: link.environmentCredential,
992 cloudMintPublicKey: link.cloudMintPublicKey,
993 endpointRuntime: link.endpointRuntime,
994 },
995 {
996 lockHeld: true,
997 confirmedOrigin: parsedOrigin.origin,
998 },
999 );
1000 // Callers decide on managed tunnel recovery from the mode this link
1001 // actually used, not from a value read before the relay round trip.
1002 return mode;
1003 },
1004 Effect.catchIf(
1005 ServerSecretStore.isSecretStoreError,
1006 failEnvironmentCloudInternalError("Could not persist desired T3 Connect link state."),
1007 ),
1008 Effect.catchTags({
1009 CloudCliCredentialRemovalError: failCloudCliTokenManagerError,
1010 CloudCliCredentialRefreshError: failCloudCliTokenManagerError,
1011 CloudCliCredentialReadError: failCloudCliTokenManagerError,
1012 CloudCliAuthorizationError: failCloudCliTokenManagerError,
1013 CloudCliAuthorizationTimeoutError: failCloudCliTokenManagerError,
1014 }),
1015 );
1016
1017 const reconcileDesiredLink = Effect.fn("environment.cloud.reconcileDesiredLink")(function* (
1018 localOrigin: string,
1019 ) {
1020 return yield* endpointRuntime.withLinkStateLock(reconcileDesiredCloudLinkWith(localOrigin));
1021 });
1022
1023 const reconcileDesiredLinkIfStillDesired = Effect.fn(
1024 "environment.cloud.reconcileDesiredLinkIfStillDesired",
1025 )(function* (localOrigin: string) {
1026 return yield* endpointRuntime.withLinkStateLock(
1027 Effect.gen(function* () {
1028 if (!(yield* readCliDesiredCloudLink.pipe(withSecrets))) {
1029 return null;
1030 }
1031 return yield* reconcileDesiredCloudLinkWith(localOrigin);
1032 }),
1033 );
1034 });
1035
1036 const makeManagedTunnelRecoveryProof = Effect.fn(
1037 "environment.cloud.makeManagedTunnelRecoveryProof",
1038 )(function* (input: ManagedTunnelRecoveryProofInput) {
1039 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
1040 const configuredIssuer = yield* secrets.get(RELAY_ISSUER_SECRET);
1041 const now = yield* DateTime.now;
1042 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
1043 const claims = {
1044 iss: `t3-env:${input.environmentId}`,
1045 aud: normalizeRelayIssuer(
1046 Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl,
1047 ),
1048 sub: input.environmentId,
1049 jti: yield* crypto.randomUUIDv4,
1050 iat: issuedAt,
1051 exp: issuedAt + 60,
1052 environmentId: input.environmentId,
1053 cloudUserId: input.cloudUserId,
1054 };
1055 const payload =
1056 input.action === "register"
1057 ? {
1058 ...claims,
1059 action: "register" as const,
1060 tunnelId: input.tunnelId,
1061 origin: input.origin,
1062 }
1063 : { ...claims, action: "recover" as const, origin: input.origin };
1064
1065 return yield* signRelayJwt({
1066 privateKey: keyPair.privateKey,
1067 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
1068 payload,
1069 }).pipe(
1070 Effect.mapError(
1071 () =>
1072 new EnvironmentHttpInternalServerError({
1073 message: "Could not sign the managed tunnel recovery request.",
1074 }),
1075 ),
1076 );
1077 });
1078
1079 const registerManagedTunnelRecovery = Effect.fn(
1080 "environment.cloud.registerManagedCloudTunnelRecovery",
1081 )(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) {
1082 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
1083 secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1084 secrets.get(RELAY_URL_SECRET),
1085 secrets.get(CLOUD_LINKED_USER_ID),
1086 secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1087 ]);
1088 if (
1089 Option.isNone(runtimeConfig) ||
1090 Option.isNone(relayUrl) ||
1091 Option.isNone(cloudUserId) ||
1092 Option.isNone(environmentCredential)
1093 ) {
1094 return { status: "not_linked" as const };
1095 }
1096
1097 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
1098 if (config?.providerKind !== "cloudflare_tunnel") {
1099 return { status: "not_linked" as const };
1100 }
1101
1102 const parsedOrigin = yield* Effect.try({
1103 try: () => parseManagedEndpointLocalOrigin(localOrigin),
1104 catch: () =>
1105 new EnvironmentHttpBadRequestError({
1106 message: "Could not resolve local environment origin.",
1107 }),
1108 });
1109 if (config.tunnelId === undefined) {
1110 return { status: "recovery_required" as const, config };
1111 }
1112 const origin = parsedOrigin.origin;
1113 const environmentId = yield* environment.getEnvironmentId;
1114 const relayUrlValue = bytesToString(relayUrl.value);
1115 const cloudUserIdValue = bytesToString(cloudUserId.value);
1116 const proof = yield* makeManagedTunnelRecoveryProof({
1117 action: "register",
1118 environmentId,
1119 cloudUserId: cloudUserIdValue,
1120 relayUrl: relayUrlValue,
1121 tunnelId: config.tunnelId,
1122 origin,
1123 });
1124 const registered = yield* relayClientRequest({
1125 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`,
1126 token: bytesToString(environmentCredential.value),
1127 payload: {
1128 cloudUserId: cloudUserIdValue,
1129 tunnelId: config.tunnelId,
1130 origin,
1131 proof,
1132 },
1133 schema: RelayManagedEndpointRecoveryRegistrationResponse,
1134 });
1135 if (registered.status === "recovery_required") {
1136 return { status: registered.status, config };
1137 }
1138 const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry(
1139 {
1140 config,
1141 configJson: bytesToString(runtimeConfig.value),
1142 origin,
1143 },
1144 options?.retryRuntimeFailures === true,
1145 );
1146 return endpointRuntimeStatus === null
1147 ? { status: "superseded" as const }
1148 : { status: "ready" as const, endpointRuntimeStatus };
1149 });
1150
1151 const recoverManagedTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")(function* (
1152 localOrigin: string,
1153 expectedConfig?: RelayManagedEndpointRuntimeConfig,
1154 options?: { readonly retryRuntimeFailures?: boolean },
1155 ) {
1156 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
1157 secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1158 secrets.get(RELAY_URL_SECRET),
1159 secrets.get(CLOUD_LINKED_USER_ID),
1160 secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1161 ]);
1162 if (
1163 Option.isNone(runtimeConfig) ||
1164 Option.isNone(relayUrl) ||
1165 Option.isNone(cloudUserId) ||
1166 Option.isNone(environmentCredential)
1167 ) {
1168 return false;
1169 }
1170 if (expectedConfig !== undefined) {
1171 const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
1172 if (
1173 current === null ||
1174 current.providerKind !== expectedConfig.providerKind ||
1175 current.connectorToken !== expectedConfig.connectorToken ||
1176 current.tunnelId !== expectedConfig.tunnelId ||
1177 current.tunnelName !== expectedConfig.tunnelName
1178 ) {
1179 return false;
1180 }
1181 }
1182
1183 const parsedOrigin = yield* Effect.try({
1184 try: () => parseManagedEndpointLocalOrigin(localOrigin),
1185 catch: () =>
1186 new EnvironmentHttpBadRequestError({
1187 message: "Could not resolve local environment origin.",
1188 }),
1189 });
1190
1191 const environmentId = yield* environment.getEnvironmentId;
1192 const relayUrlValue = bytesToString(relayUrl.value);
1193 const cloudUserIdValue = bytesToString(cloudUserId.value);
1194 const origin = parsedOrigin.origin;
1195 const proof = yield* makeManagedTunnelRecoveryProof({
1196 action: "recover",
1197 environmentId,
1198 cloudUserId: cloudUserIdValue,
1199 relayUrl: relayUrlValue,
1200 origin,
1201 });
1202 const recovered = yield* relayClientRequest({
1203 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`,
1204 token: bytesToString(environmentCredential.value),
1205 payload: {
1206 cloudUserId: cloudUserIdValue,
1207 origin,
1208 proof,
1209 },
1210 schema: RelayManagedEndpointRecoveryResponse,
1211 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
1212 });
1213 if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") {
1214 return yield* new EnvironmentHttpInternalServerError({
1215 message: "T3 Connect returned an unsupported managed tunnel configuration.",
1216 });
1217 }
1218
1219 const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe(
1220 Effect.mapError(
1221 () =>
1222 new EnvironmentHttpInternalServerError({
1223 message: "Could not persist the recovered managed tunnel configuration.",
1224 }),
1225 ),
1226 );
1227 const stored = yield* endpointRuntime.withLinkStateLock(
1228 Effect.gen(function* () {
1229 const currentConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1230 if (
1231 Option.isNone(currentConfig) ||
1232 bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value)
1233 ) {
1234 return false;
1235 }
1236 yield* secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded));
1237 yield* secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1238 return true;
1239 }),
1240 );
1241 if (!stored) return false;
1242 const status = yield* activateManagedTunnelWithRetry(
1243 {
1244 config: recovered.endpointRuntime,
1245 configJson: encoded,
1246 origin,
1247 },
1248 options?.retryRuntimeFailures === true,
1249 );
1250 return status !== null;
1251 });
1252
1253 const applyRelayConfig = Effect.fn("environment.cloud.relayConfig")(
1254 function* (payload: RelayEnvironmentConfigRequest) {
1255 const result = yield* applyCloudRelayConfig(payload);
1256 if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") {
1257 const address = httpServer.address;
1258 if (typeof address === "string" || !("port" in address)) {
1259 return yield* new EnvironmentHttpInternalServerError({
1260 message: "Could not resolve the local server origin.",
1261 });
1262 }
1263 const registration = yield* registerManagedTunnelRecovery(
1264 `http://127.0.0.1:${address.port}`,
1265 ).pipe(
1266 Effect.retry({
1267 times: 2,
1268 while: (error) =>
1269 shouldRetryCloudLink(error) &&
1270 error._tag !== "EnvironmentCloudEndpointUnavailableError",
1271 }),
1272 );
1273 if (registration.status === "superseded") {
1274 return yield* new EnvironmentHttpConflictError({
1275 message: "The managed tunnel configuration changed during registration.",
1276 });
1277 }
1278 if (registration.status === "recovery_required") {
1279 yield* endpointRuntime.requestRecovery(registration.config);
1280 }
1281 if (registration.status !== "ready") {
1282 return yield* new EnvironmentCloudEndpointUnavailableError({
1283 message: "Managed endpoint origin could not be confirmed.",
1284 endpointRuntimeStatus: { status: "disabled" },
1285 });
1286 }
1287 return {
1288 ok: true,
1289 endpointRuntimeStatus: registration.endpointRuntimeStatus,
1290 } satisfies EnvironmentCloudRelayConfigResult;
1291 }
1292 return result;
1293 },
1294 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
1295 failEnvironmentCloudInternalError(error.message)(error),
1296 ),
1297 Effect.catchIf(
1298 ServerSecretStore.isSecretStoreError,
1299 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
1300 ),
1301 Effect.catchTags({
1302 SchemaError: failEnvironmentCloudInternalError(
1303 "Could not persist environment relay configuration.",
1304 ),
1305 PlatformError: failEnvironmentCloudInternalError(
1306 "Could not register the managed endpoint origin.",
1307 ),
1308 }),
1309 );
1310
1311 // Cloudflare bills per provisioned tunnel, so an environment that goes offline
1312 // must not leave its tunnel behind. Releasing deletes only the tunnel — the
1313 // relay keeps the link and its hostname reservation, and the next startup's
1314 // link reconcile provisions a replacement tunnel under the same URL.
1315 const releaseManagedTunnelOnShutdown = Effect.fn(
1316 "environment.cloud.releaseManagedTunnelOnShutdown",
1317 )(function* () {
1318 // Only a managed link stores a runtime config; publish-only links have no
1319 // tunnel to release.
1320 const runtimeConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1321 if (Option.isNone(runtimeConfig)) {
1322 return false;
1323 }
1324 // Only CLI-desired managed links release eagerly because this request uses
1325 // CLI authorization. Web/mobile links register startup recovery with their
1326 // environment credential, and the relay reaper removes them after they are
1327 // down for the configured grace period. Unlink still deletes either kind.
1328 if (
1329 !(yield* readCliDesiredCloudLink.pipe(withSecrets)) ||
1330 (yield* readCliDesiredLinkMode.pipe(withSecrets)) !== "managed"
1331 ) {
1332 return false;
1333 }
1334 // A shutdown that hands off to a pending update is not the environment
1335 // going offline: the service launcher or the desktop app immediately brings
1336 // a server back (the new version, or the old one after a rollback). Deleting
1337 // the tunnel here forces that server to provision a replacement UUID, and the
1338 // public hostname's route to the new tunnel takes 1-2 minutes to propagate —
1339 // the dominant cost of an update restart. Keep the tunnel instead: the next
1340 // boot respawns the connector from the stored config and is reachable as
1341 // soon as it connects, and the reconcile confirms the still-live tunnel
1342 // without replacing it.
1343 if (
1344 (yield* withRuntimeFiles(desktopUpdateRestartPending)) ||
1345 (yield* withRuntimeFiles(pendingUpdateHandoffExists))
1346 ) {
1347 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
1348 return false;
1349 }
1350 const token = yield* cliTokenManager.getExisting;
1351 if (Option.isNone(token)) {
1352 return false;
1353 }
1354 // The link belongs to the relay it was installed against, so target the
1355 // persisted URL: T3CODE_RELAY_URL may have changed since the link was made.
1356 const relayUrl = yield* secrets.get(RELAY_URL_SECRET);
1357 if (Option.isNone(relayUrl)) {
1358 return false;
1359 }
1360 const environmentId = yield* environment.getEnvironmentId;
1361 // Stop the local connector before the relay deletes the tunnel it serves.
1362 yield* endpointRuntime.applyConfig(null);
1363 const response = yield* HttpClientRequest.delete(
1364 `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`,
1365 ).pipe(
1366 HttpClientRequest.bearerToken(token.value.accessToken),
1367 httpClient.execute,
1368 Effect.flatMap(filterRelayResponse),
1369 Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
1370 withRelayClientTracing,
1371 );
1372 // ok:false means the relay skipped deletion because a concurrent provision
1373 // owns the recorded tunnel now — leave the stored config alone.
1374 if (!response.ok) {
1375 return false;
1376 }
1377 // The connector token died with the tunnel. Drop the stored config so the
1378 // next start waits for the link reconcile instead of respawning the relay
1379 // client with a dead token. Kept when the release request fails: the tunnel
1380 // still exists, so the stored token keeps working across the restart.
1381 // Only dropped while it is still the config this shutdown released — a fast
1382 // restart may already have reconciled and stored a fresh config for its
1383 // replacement tunnel, and that one must survive this finalizer.
1384 const storedConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1385 if (
1386 Option.isSome(storedConfig) &&
1387 bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)
1388 ) {
1389 yield* secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1390 yield* secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1391 }
1392 return true;
1393 });
1394
1395 const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* () {
1396 const [
1397 cloudUserId,
1398 relayUrl,
1399 relayIssuer,
1400 endpointRuntimeConfig,
1401 publishAgentActivity,
1402 holdWebhooks,
1403 ] = yield* Effect.all(
1404 [
1405 secrets.get(CLOUD_LINKED_USER_ID),
1406 secrets.get(RELAY_URL_SECRET),
1407 secrets.get(RELAY_ISSUER_SECRET),
1408 secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1409 secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
1410 secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1411 ],
1412 { concurrency: 6 },
1413 );
1414 return {
1415 linked: Option.isSome(cloudUserId),
1416 cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
1417 relayUrl: Option.isSome(relayUrl) ? bytesToString(relayUrl.value) : null,
1418 relayIssuer: Option.isSome(relayIssuer) ? bytesToString(relayIssuer.value) : null,
1419 // The managed tunnel runtime config is only stored for managed links; a
1420 // publish-only link leaves it absent.
1421 managedTunnelActive: Option.isSome(endpointRuntimeConfig),
1422 publishAgentActivity: Option.isSome(publishAgentActivity)
1423 ? bytesToString(publishAgentActivity.value) === "true"
1424 : false,
1425 holdWebhooksWhileOffline:
1426 Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true",
1427 } satisfies EnvironmentCloudLinkStateResult;
1428 });
1429
1430 const linkState = Effect.fn("environment.cloud.linkState")(
1431 function* () {
1432 return yield* readCloudLinkState();
1433 },
1434 Effect.catchIf(
1435 ServerSecretStore.isSecretStoreError,
1436 failEnvironmentCloudInternalError("Could not read environment relay configuration."),
1437 ),
1438 );
1439
1440 const unlink = Effect.fn("environment.cloud.unlink")(
1441 function* () {
1442 return yield* endpointRuntime.withLinkStateLock(
1443 Effect.gen(function* () {
1444 const endpointRuntimeStatus = yield* endpointRuntime.applyConfig(null);
1445 yield* Effect.all(
1446 [
1447 secrets.remove(CLOUD_LINKED_USER_ID),
1448 secrets.remove(RELAY_URL_SECRET),
1449 secrets.remove(RELAY_ISSUER_SECRET),
1450 secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1451 secrets.remove(CLOUD_MINT_PUBLIC_KEY),
1452 secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1453 secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1454 secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1455 secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1456 ],
1457 { concurrency: 9 },
1458 );
1459 yield* setCliDesiredCloudLink(false).pipe(withSecrets);
1460 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
1461 }),
1462 );
1463 },
1464 Effect.catchIf(
1465 ServerSecretStore.isSecretStoreError,
1466 failEnvironmentCloudInternalError("Could not remove environment relay configuration."),
1467 ),
1468 );
1469
1470 const pushHoldWebhooksWhileOffline = Effect.fn("CloudPreferences.pushHoldWebhooksWhileOffline")(
1471 function* (holdWebhooksWhileOffline: boolean) {
1472 const connection = yield* readRelayConnection.pipe(withSecrets);
1473 if (connection === null) {
1474 return yield* new EnvironmentHttpBadRequestError({
1475 message: "Link this environment to T3 Connect first.",
1476 });
1477 }
1478 const environmentId = yield* environment.getEnvironmentId;
1479 const client = yield* makeRelayEnvironmentClient(connection);
1480 yield* client.server
1481 .updateLinkPreferences({
1482 params: { environmentId },
1483 payload: { holdWebhooksWhileOffline },
1484 })
1485 .pipe(
1486 Effect.timeout("10 seconds"),
1487 Effect.catch(
1488 failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."),
1489 ),
1490 );
1491 },
1492 );
1493
1494 const savePreference = (name: string, value: boolean) =>
1495 secrets
1496 .set(name, stringToBytes(String(value)))
1497 .pipe(
1498 Effect.catch(
1499 failEnvironmentCloudInternalError("Could not persist environment cloud preferences."),
1500 ),
1501 );
1502
1503 // One update at a time, so two requests can't each leave one setting behind.
1504 const preferencesLock = yield* Semaphore.make(1);
1505
1506 const savePreferences = Effect.fn("CloudPreferences.update")(
1507 function* (input: EnvironmentCloudPreferencesRequest) {
1508 // All or nothing: the activity setting is saved first, before the relay
1509 // is told anything, and put back if the hold change then fails. Both
1510 // current values are read up front; a failed read stops here, before
1511 // anything changes, because a guessed value would be the rollback target.
1512 const readCurrent = (name: string) =>
1513 secrets
1514 .get(name)
1515 .pipe(
1516 Effect.catch(
1517 failEnvironmentCloudInternalError("Could not read environment cloud preferences."),
1518 ),
1519 );
1520 const previousActivity = yield* readCurrent(PUBLISH_AGENT_ACTIVITY_SECRET);
1521 const previousHold = yield* readCurrent(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET);
1522 yield* savePreference(PUBLISH_AGENT_ACTIVITY_SECRET, input.publishAgentActivity);
1523 if (input.holdWebhooksWhileOffline !== undefined) {
1524 const next = input.holdWebhooksWhileOffline;
1525 const previous = Option.match(previousHold, {
1526 onNone: () => false,
1527 onSome: (bytes) => bytesToString(bytes) === "true",
1528 });
1529 yield* pushHoldWebhooksWhileOffline(next).pipe(
1530 Effect.andThen(
1531 savePreference(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, next).pipe(
1532 Effect.tapError(() =>
1533 previous === next
1534 ? Effect.void
1535 : pushHoldWebhooksWhileOffline(previous).pipe(Effect.catch(rollbackFailed)),
1536 ),
1537 ),
1538 ),
1539 Effect.tapError(() =>
1540 Option.match(previousActivity, {
1541 onNone: () => secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1542 onSome: (bytes) => secrets.set(PUBLISH_AGENT_ACTIVITY_SECRET, bytes),
1543 }).pipe(Effect.catch(rollbackFailed)),
1544 ),
1545 );
1546 }
1547 yield* awarenessRelay.requestCatchUp();
1548 },
1549 // A client that disconnects mid-update must not stop it between a save and
1550 // its rollback, so an update always finishes or undoes itself.
1551 Effect.uninterruptible,
1552 preferencesLock.withPermits(1),
1553 );
1554
1555 const updatePreferences = Effect.fn("environment.cloud.preferences")(
1556 function* (input: EnvironmentCloudPreferencesRequest) {
1557 yield* savePreferences(input);
1558 return yield* readCloudLinkState();
1559 },
1560 Effect.catchIf(
1561 ServerSecretStore.isSecretStoreError,
1562 failEnvironmentCloudInternalError("Could not read environment cloud preferences."),
1563 ),
1564 );
1565
1566 const answerHealthRequest = Effect.fn("environment.cloud.health")(
1567 function* (request: RelayCloudEnvironmentHealthRequest) {
1568 const cloudMintPublicKey = yield* secrets
1569 .get(CLOUD_MINT_PUBLIC_KEY)
1570 .pipe(
1571 Effect.flatMap((bytes) =>
1572 Option.isSome(bytes)
1573 ? Effect.succeed(bytesToString(bytes.value))
1574 : Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
1575 ),
1576 );
1577 const relayIssuer = yield* secrets
1578 .get(RELAY_ISSUER_SECRET)
1579 .pipe(
1580 Effect.flatMap((bytes) =>
1581 Option.isSome(bytes)
1582 ? Effect.succeed(bytesToString(bytes.value))
1583 : secrets
1584 .get(RELAY_URL_SECRET)
1585 .pipe(
1586 Effect.flatMap((fallbackBytes) =>
1587 Option.isSome(fallbackBytes)
1588 ? Effect.succeed(bytesToString(fallbackBytes.value))
1589 : Effect.fail(
1590 new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({}),
1591 ),
1592 ),
1593 ),
1594 ),
1595 );
1596 const environmentId = yield* environment.getEnvironmentId;
1597 const linkedCloudUserId = yield* readInstalledCloudUserId;
1598 const now = yield* DateTime.now;
1599 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
1600 const proofOption = yield* verifyRelayJwt({
1601 publicKey: cloudMintPublicKey,
1602 token: request.proof,
1603 typ: RELAY_HEALTH_REQUEST_TYP,
1604 issuer: normalizeRelayIssuer(relayIssuer),
1605 audience: `t3-env:${environmentId}`,
1606 nowEpochSeconds: nowSeconds,
1607 }).pipe(Effect.flatMap(decodeCloudHealthProof), Effect.option);
1608 if (
1609 Option.isNone(proofOption) ||
1610 proofOption.value.environmentId !== environmentId ||
1611 proofOption.value.sub !== linkedCloudUserId ||
1612 !hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
1613 !hasExactScope({ scopes: proofOption.value.scope, expected: "environment:status" })
1614 ) {
1615 return yield* new EnvironmentHttpUnauthorizedError({
1616 message: "Invalid cloud health request.",
1617 });
1618 }
1619 const proof = proofOption.value;
1620
1621 const jtiSecretName = `${CLOUD_HEALTH_JTI_PREFIX}${proof.jti}`;
1622 const nonceSecretName = `${CLOUD_HEALTH_NONCE_PREFIX}${proof.nonce}`;
1623 const consumedReplayGuards = yield* consumeCloudReplayGuards({
1624 secrets,
1625 names: [jtiSecretName, nonceSecretName],
1626 value: stringToBytes(DateTime.formatIso(now)),
1627 });
1628 if (!consumedReplayGuards) {
1629 return yield* new EnvironmentHttpConflictError({
1630 message: "Cloud health request was already consumed.",
1631 });
1632 }
1633
1634 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
1635 const descriptor = yield* environment.getDescriptor;
1636 const responseExpiresAt = DateTime.add(now, { minutes: 5 });
1637 const responsePayload = {
1638 iss: `t3-env:${environmentId}`,
1639 aud: normalizeRelayIssuer(relayIssuer),
1640 sub: environmentId,
1641 jti: yield* crypto.randomUUIDv4,
1642 iat: nowSeconds,
1643 exp: Math.floor(responseExpiresAt.epochMilliseconds / 1_000),
1644 environmentId,
1645 requestNonce: proof.nonce,
1646 status: "online",
1647 descriptor,
1648 checkedAt: DateTime.formatIso(now),
1649 } satisfies RelayEnvironmentHealthResponseProofPayload;
1650 const responseProof = yield* signRelayJwt({
1651 privateKey: keyPair.privateKey,
1652 typ: RELAY_HEALTH_RESPONSE_TYP,
1653 payload: responsePayload,
1654 }).pipe(
1655 Effect.mapError(
1656 (cause) =>
1657 new EnvironmentAuth.ServerAuthCloudHealthJwtSigningError({
1658 cause,
1659 }),
1660 ),
1661 );
1662 return {
1663 environmentId,
1664 status: "online",
1665 descriptor,
1666 checkedAt: responsePayload.checkedAt,
1667 proof: responseProof,
1668 } satisfies RelayEnvironmentHealthResponse;
1669 },
1670 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
1671 failEnvironmentCloudInternalError(error.message)(error),
1672 ),
1673 Effect.catchIf(
1674 ServerSecretStore.isSecretStoreError,
1675 failEnvironmentCloudInternalError("Could not answer cloud health request."),
1676 ),
1677 Effect.catchTag(
1678 "PlatformError",
1679 failEnvironmentCloudInternalError("Could not answer cloud health request."),
1680 ),
1681 );
1682
1683 const mintCredential = Effect.fn("environment.cloud.mintCredential")(
1684 function* (request: RelayCloudMintCredentialRequest) {
1685 const cloudMintPublicKey = yield* secrets
1686 .get(CLOUD_MINT_PUBLIC_KEY)
1687 .pipe(
1688 Effect.flatMap((bytes) =>
1689 Option.isSome(bytes)
1690 ? Effect.succeed(bytesToString(bytes.value))
1691 : Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
1692 ),
1693 );
1694 const relayIssuer = yield* secrets
1695 .get(RELAY_ISSUER_SECRET)
1696 .pipe(
1697 Effect.flatMap((bytes) =>
1698 Option.isSome(bytes)
1699 ? Effect.succeed(bytesToString(bytes.value))
1700 : secrets
1701 .get(RELAY_URL_SECRET)
1702 .pipe(
1703 Effect.flatMap((fallbackBytes) =>
1704 Option.isSome(fallbackBytes)
1705 ? Effect.succeed(bytesToString(fallbackBytes.value))
1706 : Effect.fail(
1707 new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({}),
1708 ),
1709 ),
1710 ),
1711 ),
1712 );
1713 const environmentId = yield* environment.getEnvironmentId;
1714 const linkedCloudUserId = yield* readInstalledCloudUserId;
1715 const now = yield* DateTime.now;
1716 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
1717 const proofOption = yield* verifyRelayJwt({
1718 publicKey: cloudMintPublicKey,
1719 token: request.proof,
1720 typ: RELAY_MINT_REQUEST_TYP,
1721 issuer: normalizeRelayIssuer(relayIssuer),
1722 audience: `t3-env:${environmentId}`,
1723 nowEpochSeconds: nowSeconds,
1724 }).pipe(Effect.flatMap(decodeCloudMintProof), Effect.option);
1725 if (
1726 Option.isNone(proofOption) ||
1727 proofOption.value.environmentId !== environmentId ||
1728 proofOption.value.sub !== linkedCloudUserId ||
1729 proofOption.value.cnf.jkt !== proofOption.value.clientProofKeyThumbprint ||
1730 !hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
1731 !hasExactScope({ scopes: proofOption.value.scope, expected: "environment:connect" })
1732 ) {
1733 return yield* new EnvironmentHttpUnauthorizedError({
1734 message: "Invalid cloud mint request.",
1735 });
1736 }
1737 const proof = proofOption.value;
1738
1739 const jtiSecretName = `${CLOUD_MINT_JTI_PREFIX}${proof.jti}`;
1740 const nonceSecretName = `${CLOUD_MINT_NONCE_PREFIX}${proof.nonce}`;
1741 const consumedReplayGuards = yield* consumeCloudReplayGuards({
1742 secrets,
1743 names: [jtiSecretName, nonceSecretName],
1744 value: stringToBytes(DateTime.formatIso(now)),
1745 });
1746 if (!consumedReplayGuards) {
1747 return yield* new EnvironmentHttpConflictError({
1748 message: "Cloud mint request was already consumed.",
1749 });
1750 }
1751
1752 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
1753 const issued = yield* environmentAuth.createPairingLink({
1754 scopes: AuthStandardClientScopes,
1755 subject: "cloud-connect",
1756 ttl: Duration.minutes(2),
1757 label: "T3 Connect connect",
1758 proofKeyThumbprint: proof.clientProofKeyThumbprint,
1759 });
1760 const responsePayload = {
1761 iss: `t3-env:${environmentId}`,
1762 aud: normalizeRelayIssuer(relayIssuer),
1763 sub: environmentId,
1764 jti: yield* crypto.randomUUIDv4,
1765 iat: nowSeconds,
1766 exp: Math.floor(issued.expiresAt.epochMilliseconds / 1_000),
1767 environmentId,
1768 clientProofKeyThumbprint: proof.clientProofKeyThumbprint,
1769 requestNonce: proof.nonce,
1770 credential: issued.credential,
1771 } satisfies RelayEnvironmentMintResponseProofPayload;
1772 const responseProof = yield* signRelayJwt({
1773 privateKey: keyPair.privateKey,
1774 typ: RELAY_MINT_RESPONSE_TYP,
1775 payload: responsePayload,
1776 }).pipe(
1777 Effect.mapError(
1778 (cause) =>
1779 new EnvironmentAuth.ServerAuthCloudMintJwtSigningError({
1780 cause,
1781 }),
1782 ),
1783 );
1784 return {
1785 credential: issued.credential,
1786 expiresAt: DateTime.formatIso(issued.expiresAt),
1787 proof: responseProof,
1788 } satisfies RelayEnvironmentMintResponse;
1789 },
1790 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
1791 failEnvironmentCloudInternalError(error.message)(error),
1792 ),
1793 Effect.catchIf(
1794 ServerSecretStore.isSecretStoreError,
1795 failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
1796 ),
1797 Effect.catchTag(
1798 "PlatformError",
1799 failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
1800 ),
1801 );
1802
1803 return CloudLink.of({
1804 linkProof,
1805 applyRelayConfig,
1806 linkState,
1807 unlink,
1808 updatePreferences,
1809 answerHealthRequest,
1810 mintCredential,
1811 reconcileDesiredLink,
1812 reconcileDesiredLinkIfStillDesired,
1813 registerManagedTunnelRecovery,
1814 recoverManagedTunnel,
1815 startManagedTunnelIfOriginConfirmed,
1816 releaseManagedTunnelOnShutdown,
1817 });
1818});
1819
1820export const layer = Layer.effect(CloudLink, make);
apps/server/src/cloud/CloudPreferences.test.ts deleted-248
......@@ -1,248 +0,0 @@
1import { EnvironmentId } from "@t3tools/contracts";
2import { assert, it } from "@effect/vitest";
3import * as Effect from "effect/Effect";
4import * as Fiber from "effect/Fiber";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as FetchHttpClient from "effect/http/FetchHttpClient";
8
9import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
10import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
11import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
12import * as CloudPreferences from "./CloudPreferences.ts";
13import {
14 HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
15 PUBLISH_AGENT_ACTIVITY_SECRET,
16 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
17 RELAY_URL_SECRET,
18} from "./config.ts";
19
20const encode = (value: string) => new TextEncoder().encode(value);
21
22/** A linked environment whose secret store can refuse writes, and the relay calls it made. */
23const withService = <A, E>(
24 options: {
25 readonly failHoldWrite?: boolean;
26 readonly failActivityWrite?: boolean;
27 readonly relayFails?: boolean;
28 readonly failActivityRead?: boolean;
29 readonly failHoldRead?: boolean;
30 /** The first relay call reports itself, then waits for this before answering. */
31 readonly holdFirstRelayCall?: {
32 readonly started: () => void;
33 readonly released: Promise<void>;
34 };
35 },
36 body: (input: {
37 readonly preferences: CloudPreferences.CloudPreferences["Service"];
38 readonly stored: Map<string, Uint8Array>;
39 readonly relayCalls: Array<boolean>;
40 }) => Effect.Effect<A, E, never>,
41) =>
42 Effect.gen(function* () {
43 const stored = new Map<string, Uint8Array>([
44 [RELAY_URL_SECRET, encode("https://relay.test")],
45 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, encode("credential")],
46 [HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, encode("false")],
47 [PUBLISH_AGENT_ACTIVITY_SECRET, encode("false")],
48 ]);
49 const relayCalls: Array<boolean> = [];
50 const fetch: typeof globalThis.fetch = Object.assign(
51 (_input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => {
52 const payload = JSON.parse(String(init?.body)) as { holdWebhooksWhileOffline: boolean };
53 relayCalls.push(payload.holdWebhooksWhileOffline);
54 if (relayCalls.length === 1 && options.holdFirstRelayCall) {
55 options.holdFirstRelayCall.started();
56 return options.holdFirstRelayCall.released.then(() => Response.json(payload));
57 }
58 return options.relayFails
59 ? Promise.resolve(new Response("unavailable", { status: 503 }))
60 : Promise.resolve(Response.json(payload));
61 },
62 { preconnect: () => {} },
63 );
64 const layerDependencies = Layer.mergeAll(
65 Layer.mock(ServerSecretStore.ServerSecretStore)({
66 get: (name) =>
67 (options.failActivityRead && name === PUBLISH_AGENT_ACTIVITY_SECRET) ||
68 (options.failHoldRead && name === HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)
69 ? Effect.fail(
70 new ServerSecretStore.SecretStoreReadError({
71 resource: name,
72 cause: new Error("busy"),
73 }),
74 )
75 : Effect.succeed(Option.fromNullishOr(stored.get(name))),
76 set: (name, value) =>
77 (options.failHoldWrite && name === HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET) ||
78 (options.failActivityWrite && name === PUBLISH_AGENT_ACTIVITY_SECRET)
79 ? Effect.fail(
80 new ServerSecretStore.SecretStorePersistError({
81 resource: name,
82 cause: new Error("read-only"),
83 }),
84 )
85 : Effect.sync(() => void stored.set(name, value)),
86 }),
87 Layer.mock(ServerEnvironment.ServerEnvironment)({
88 getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-1")),
89 }),
90 Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ requestCatchUp: () => Effect.void }),
91 );
92 return yield* Effect.gen(function* () {
93 const preferences = yield* CloudPreferences.CloudPreferences;
94 return yield* body({ preferences, stored, relayCalls });
95 }).pipe(
96 Effect.provide(CloudPreferences.layer.pipe(Layer.provide(layerDependencies))),
97 Effect.provideService(FetchHttpClient.Fetch, fetch),
98 );
99 });
100
101it.effect("tells the relay before saving the hold setting locally", () =>
102 withService({}, ({ preferences, stored, relayCalls }) =>
103 Effect.gen(function* () {
104 yield* preferences.update({ publishAgentActivity: true, holdWebhooksWhileOffline: true });
105 assert.deepEqual(relayCalls, [true]);
106 assert.equal(
107 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
108 "true",
109 );
110 }),
111 ),
112);
113
114it.effect("puts the relay back when the local save fails", () =>
115 withService({ failHoldWrite: true }, ({ preferences, stored, relayCalls }) =>
116 Effect.gen(function* () {
117 const error = yield* preferences
118 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
119 .pipe(Effect.flip);
120 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
121 assert.deepEqual(relayCalls, [true, false]);
122 assert.equal(
123 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
124 "false",
125 );
126 }),
127 ),
128);
129
130it.effect("leaves the relay untouched when the activity setting can't be saved", () =>
131 withService({ failActivityWrite: true }, ({ preferences, stored, relayCalls }) =>
132 Effect.gen(function* () {
133 const error = yield* preferences
134 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
135 .pipe(Effect.flip);
136 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
137 assert.deepEqual(relayCalls, []);
138 assert.equal(
139 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
140 "false",
141 );
142 }),
143 ),
144);
145
146it.effect("keeps both settings unchanged when the relay refuses the hold change", () =>
147 withService({ relayFails: true }, ({ preferences, stored }) =>
148 Effect.gen(function* () {
149 yield* preferences
150 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
151 .pipe(Effect.flip);
152 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
153 assert.equal(
154 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
155 "false",
156 );
157 }),
158 ),
159);
160
161it.effect("changes nothing when the current activity setting can't be read", () =>
162 withService({ failActivityRead: true, relayFails: true }, ({ preferences, stored, relayCalls }) =>
163 Effect.gen(function* () {
164 const error = yield* preferences
165 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
166 .pipe(Effect.flip);
167 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
168 assert.deepEqual(relayCalls, []);
169 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
170 }),
171 ),
172);
173
174it.effect("two overlapping updates are applied one after the other", () => {
175 let release!: () => void;
176 let started!: () => void;
177 const released = new Promise<void>((resolve) => {
178 release = resolve;
179 });
180 const firstAtRelay = new Promise<void>((resolve) => {
181 started = resolve;
182 });
183 return withService({ holdFirstRelayCall: { started, released } }, ({ preferences, stored }) =>
184 Effect.gen(function* () {
185 const first = yield* preferences
186 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
187 .pipe(Effect.forkChild);
188 yield* Effect.promise(() => firstAtRelay);
189 const second = yield* preferences
190 .update({ publishAgentActivity: false, holdWebhooksWhileOffline: false })
191 .pipe(Effect.forkChild);
192 // Give the second update every chance to run; it must wait for the first.
193 yield* Effect.yieldNow;
194 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "true");
195 release();
196 yield* Fiber.join(first);
197 yield* Fiber.join(second);
198 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
199 assert.equal(
200 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
201 "false",
202 );
203 }),
204 );
205});
206
207it.effect("changes nothing when the current hold setting can't be read", () =>
208 withService({ failHoldRead: true, failHoldWrite: true }, ({ preferences, stored, relayCalls }) =>
209 Effect.gen(function* () {
210 stored.set(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, encode("true"));
211 const error = yield* preferences
212 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: false })
213 .pipe(Effect.flip);
214 assert.equal(error._tag, "EnvironmentHttpInternalServerError");
215 assert.deepEqual(relayCalls, []);
216 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "false");
217 }),
218 ),
219);
220
221it.effect("an update the client walks away from still finishes", () => {
222 let release!: () => void;
223 let started!: () => void;
224 const released = new Promise<void>((resolve) => {
225 release = resolve;
226 });
227 const atRelay = new Promise<void>((resolve) => {
228 started = resolve;
229 });
230 return withService({ holdFirstRelayCall: { started, released } }, ({ preferences, stored }) =>
231 Effect.gen(function* () {
232 const update = yield* preferences
233 .update({ publishAgentActivity: true, holdWebhooksWhileOffline: true })
234 .pipe(Effect.forkChild);
235 yield* Effect.promise(() => atRelay);
236 // The interrupt is in flight before the relay answers.
237 const interrupted = yield* Fiber.interrupt(update).pipe(Effect.forkChild);
238 yield* Effect.yieldNow;
239 release();
240 yield* Fiber.join(interrupted);
241 assert.equal(new TextDecoder().decode(stored.get(PUBLISH_AGENT_ACTIVITY_SECRET)), "true");
242 assert.equal(
243 new TextDecoder().decode(stored.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET)),
244 "true",
245 );
246 }),
247 );
248});
apps/server/src/cloud/CloudPreferences.ts deleted-133
......@@ -1,133 +0,0 @@
1import {
2 type EnvironmentCloudPreferencesRequest,
3 EnvironmentHttpBadRequestError,
4 EnvironmentHttpInternalServerError,
5} from "@t3tools/contracts";
6import * as Context from "effect/Context";
7import * as Effect from "effect/Effect";
8import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
10import * as Semaphore from "effect/Semaphore";
11
12import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
13import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
14import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
15import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts";
16import {
17 HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
18 PUBLISH_AGENT_ACTIVITY_SECRET,
19 readRelayConnection,
20} from "./config.ts";
21
22const encode = (value: boolean) => new TextEncoder().encode(String(value));
23
24/** A failed rollback leaves a setting changed; it is logged, not hidden. */
25const rollbackFailed = (cause: unknown) =>
26 Effect.logWarning("Could not roll back a T3 Connect preference", { cause });
27
28const internalError = (message: string) => (cause: unknown) =>
29 Effect.logError(message, { cause }).pipe(
30 Effect.andThen(Effect.fail(new EnvironmentHttpInternalServerError({ message }))),
31 );
32
33export class CloudPreferences extends Context.Service<
34 CloudPreferences,
35 {
36 /**
37 * Saves this environment's T3 Connect preferences, all or nothing. The
38 * activity setting is saved first. Holding webhooks while offline is decided
39 * by the relay, so the relay is told before the local copy is saved. If
40 * either step fails, the activity setting is put back, and so is the relay
41 * when only the local save failed.
42 */
43 readonly update: (
44 input: EnvironmentCloudPreferencesRequest,
45 ) => Effect.Effect<void, EnvironmentHttpBadRequestError | EnvironmentHttpInternalServerError>;
46 }
47>()("t3/cloud/CloudPreferences") {}
48
49const make = Effect.gen(function* () {
50 const secrets = yield* ServerSecretStore.ServerSecretStore;
51 const environment = yield* ServerEnvironment.ServerEnvironment;
52 const awarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
53 const withSecrets = Effect.provideService(ServerSecretStore.ServerSecretStore, secrets);
54
55 const pushHoldWebhooksWhileOffline = Effect.fn("CloudPreferences.pushHoldWebhooksWhileOffline")(
56 function* (holdWebhooksWhileOffline: boolean) {
57 const connection = yield* readRelayConnection.pipe(withSecrets);
58 if (connection === null) {
59 return yield* new EnvironmentHttpBadRequestError({
60 message: "Link this environment to T3 Connect first.",
61 });
62 }
63 const environmentId = yield* environment.getEnvironmentId;
64 const client = yield* makeRelayEnvironmentClient(connection);
65 yield* client.server
66 .updateLinkPreferences({
67 params: { environmentId },
68 payload: { holdWebhooksWhileOffline },
69 })
70 .pipe(
71 Effect.timeout("10 seconds"),
72 Effect.catch(internalError("Could not update T3 Connect webhook settings.")),
73 );
74 },
75 );
76
77 const save = (name: string, value: boolean) =>
78 secrets
79 .set(name, encode(value))
80 .pipe(Effect.catch(internalError("Could not persist environment cloud preferences.")));
81
82 // One update at a time, so two requests can't each leave one setting behind.
83 const updateLock = yield* Semaphore.make(1);
84
85 const update: CloudPreferences["Service"]["update"] = Effect.fn("CloudPreferences.update")(
86 function* (input) {
87 // All or nothing: the activity setting is saved first, before the relay
88 // is told anything, and put back if the hold change then fails. Both
89 // current values are read up front; a failed read stops here, before
90 // anything changes, because a guessed value would be the rollback target.
91 const readCurrent = (name: string) =>
92 secrets
93 .get(name)
94 .pipe(Effect.catch(internalError("Could not read environment cloud preferences.")));
95 const previousActivity = yield* readCurrent(PUBLISH_AGENT_ACTIVITY_SECRET);
96 const previousHold = yield* readCurrent(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET);
97 yield* save(PUBLISH_AGENT_ACTIVITY_SECRET, input.publishAgentActivity);
98 if (input.holdWebhooksWhileOffline !== undefined) {
99 const next = input.holdWebhooksWhileOffline;
100 const previous = Option.match(previousHold, {
101 onNone: () => false,
102 onSome: (bytes) => new TextDecoder().decode(bytes) === "true",
103 });
104 yield* pushHoldWebhooksWhileOffline(next).pipe(
105 Effect.andThen(
106 save(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, next).pipe(
107 Effect.tapError(() =>
108 previous === next
109 ? Effect.void
110 : pushHoldWebhooksWhileOffline(previous).pipe(Effect.catch(rollbackFailed)),
111 ),
112 ),
113 ),
114 Effect.tapError(() =>
115 Option.match(previousActivity, {
116 onNone: () => secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
117 onSome: (bytes) => secrets.set(PUBLISH_AGENT_ACTIVITY_SECRET, bytes),
118 }).pipe(Effect.catch(rollbackFailed)),
119 ),
120 );
121 }
122 yield* awarenessRelay.requestCatchUp();
123 },
124 // A client that disconnects mid-update must not stop it between a save and
125 // its rollback, so an update always finishes or undoes itself.
126 Effect.uninterruptible,
127 updateLock.withPermits(1),
128 );
129
130 return CloudPreferences.of({ update });
131});
132
133export const layer = Layer.effect(CloudPreferences, make);
apps/server/src/cloud/http.test.ts+35-7
......@@ -16,9 +16,11 @@ import * as Stream from "effect/Stream";
1616import {
1717 HttpClient,
1818 HttpClientResponse,
19 HttpServer,
1920 HttpServerRequest,
2021 type HttpClientRequest,
2122} from "effect/http";
23import * as NetAddress from "effect/net/NetAddress";
2224
2325import { DESKTOP_UPDATE_RESTART_MARKER_FILE, EnvironmentId } from "@t3tools/contracts";
2426import { RelayClientTracer } from "@t3tools/shared/relayTracing";
......@@ -49,19 +51,14 @@ import {
4951 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
5052 RELAY_URL_SECRET,
5153} from "./config.ts";
54import * as CloudLink from "./CloudLink.ts";
5255import {
5356 consumeCloudReplayGuards,
5457 isSupportedLinkProviderKind,
5558 linkProofScopes,
5659 pendingServiceUpdateExists,
5760 parseManagedEndpointLocalOrigin,
58 reconcileDesiredCloudLink,
59 reconcileDesiredCloudLinkIfStillDesired,
60 recoverManagedCloudTunnel,
61 registerManagedCloudTunnelRecovery,
62 releaseManagedTunnelOnShutdown,
63 startManagedCloudTunnelIfOriginConfirmed,
64} from "./http.ts";
61} from "./CloudLink.ts";
6562import {
6663 managedTunnelStartupAction,
6764 retryManagedTunnelRegistration,
......@@ -89,6 +86,29 @@ const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({
8986 requestCatchUp: () => Effect.void,
9087 start: () => Effect.void,
9188});
89// The connect routes register recovery against the listening port; these tests
90// call the service directly.
91const idleHttpServer = HttpServer.HttpServer.of({
92 address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773),
93 serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],
94});
95
96const reconcileDesiredCloudLink = (localOrigin: string) =>
97 CloudLink.CloudLink.use((link) => link.reconcileDesiredLink(localOrigin));
98const reconcileDesiredCloudLinkIfStillDesired = (localOrigin: string) =>
99 CloudLink.CloudLink.use((link) => link.reconcileDesiredLinkIfStillDesired(localOrigin));
100const recoverManagedCloudTunnel = (
101 ...args: Parameters<CloudLink.CloudLink["Service"]["recoverManagedTunnel"]>
102) => CloudLink.CloudLink.use((link) => link.recoverManagedTunnel(...args));
103const registerManagedCloudTunnelRecovery = (
104 ...args: Parameters<CloudLink.CloudLink["Service"]["registerManagedTunnelRecovery"]>
105) => CloudLink.CloudLink.use((link) => link.registerManagedTunnelRecovery(...args));
106const releaseManagedTunnelOnShutdown = () =>
107 CloudLink.CloudLink.use((link) => link.releaseManagedTunnelOnShutdown());
108const startManagedCloudTunnelIfOriginConfirmed = (
109 ...args: Parameters<CloudLink.CloudLink["Service"]["startManagedTunnelIfOriginConfirmed"]>
110) => CloudLink.CloudLink.use((link) => link.startManagedTunnelIfOriginConfirmed(...args));
111
92112const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
93113 Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
94114);
......@@ -232,6 +252,12 @@ describe("reconcileDesiredCloudLink", () => {
232252 message: "Run `t3 connect link` to authorize this environment.",
233253 });
234254 }).pipe(
255 Effect.provide(CloudLink.layer),
256 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
257 Effect.provideService(
258 ServerConfigModule.ServerConfig,
259 ServerConfigModule.ServerConfig.of({} as ServerConfigModule.ServerConfig["Service"]),
260 ),
235261 Effect.provideService(
236262 ServerSecretStore.ServerSecretStore,
237263 makeSecretStore(unusedSecretStoreOperation),
......@@ -384,6 +410,8 @@ describe("releaseManagedTunnelOnShutdown", () => {
384410 (harness: ReleaseHarness) =>
385411 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
386412 effect.pipe(
413 Effect.provide(CloudLink.layer),
414 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
387415 Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store),
388416 Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
389417 Effect.provideService(
apps/server/src/cloud/http.ts+41-1599
......@@ -1,125 +1,13 @@
1import * as NodeCrypto from "node:crypto";
2import {
3 AuthRelayReadScope,
4 AuthRelayWriteScope,
5 AuthStandardClientScopes,
6 EnvironmentCloudEndpointUnavailableError,
7 EnvironmentCloudLinkStateResult,
8 EnvironmentCloudRelayConfigResult,
9 EnvironmentHttpApi,
10 EnvironmentHttpBadRequestError,
11 EnvironmentHttpConflictError,
12 EnvironmentHttpInternalServerError,
13 EnvironmentHttpUnauthorizedError,
14 DESKTOP_UPDATE_RESTART_MARKER_FILE,
15} from "@t3tools/contracts";
16import {
17 RelayCloudEnvironmentHealthProofPayload,
18 RelayCloudEnvironmentHealthRequest,
19 RelayCloudMintCredentialProofPayload,
20 RelayCloudMintCredentialRequest,
21 RelayEnvironmentHealthResponseProofPayload,
22 type RelayEnvironmentHealthResponse as RelayEnvironmentHealthResponseShape,
23 RelayEnvironmentConfigRequest,
24 RelayEnvironmentLinkChallengeResponse,
25 RelayEnvironmentLinkResponse,
26 RelayEnvironmentMintResponseProofPayload,
27 type RelayEnvironmentMintResponse as RelayEnvironmentMintResponseShape,
28 RelayEnvironmentLinkProof,
29 RelayEnvironmentLinkProofPayload,
30 RelayLinkProofRequest,
31 RelayManagedEndpointOrigin,
32 RelayManagedEndpointRecoveryProofPayload,
33 RelayManagedEndpointRecoveryRegistrationResponse,
34 RelayManagedEndpointRecoveryResponse,
35 type RelayManagedEndpointRuntimeConfig,
36 RelayOkResponse,
37} from "@t3tools/contracts/relay";
38import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
39import {
40 normalizeRelayIssuer,
41 RELAY_HEALTH_REQUEST_TYP,
42 RELAY_HEALTH_RESPONSE_TYP,
43 RELAY_LINK_PROOF_TYP,
44 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
45 RELAY_MINT_REQUEST_TYP,
46 RELAY_MINT_RESPONSE_TYP,
47 signRelayJwt,
48 verifyRelayJwt,
49} from "@t3tools/shared/relayJwt";
50import { isSecureRelayUrl } from "@t3tools/shared/relayUrl";
51import * as Clock from "effect/Clock";
52import * as DateTime from "effect/DateTime";
53import * as Crypto from "effect/Crypto";
54import * as Duration from "effect/Duration";
1import { AuthRelayReadScope, AuthRelayWriteScope, EnvironmentHttpApi } from "@t3tools/contracts";
552import * as Effect from "effect/Effect";
56import * as FileSystem from "effect/FileSystem";
57import * as Option from "effect/Option";
58import * as Path from "effect/Path";
59import * as Schema from "effect/Schema";
60import * as Schedule from "effect/Schedule";
613import * as HttpEffect from "effect/http/HttpEffect";
624import { HttpServerRequest, HttpServerResponse } from "effect/http";
63import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
645import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
65import * as HttpServer from "effect/http/HttpServer";
666
67import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
68import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
697import { requireEnvironmentScope } from "../auth/http.ts";
70import * as ServerConfig from "../config.ts";
71import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
72import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
73import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
74import {
75 SERVICE_STATE_FILE,
76 SERVICE_STOP_MARKER_FILE,
77 serviceStateHasPendingUpdate,
78} from "./serviceProtocol.ts";
79import {
80 CLOUD_ENDPOINT_RUNTIME_CONFIG,
81 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
82 decodeConfirmedOrigin,
83 CLOUD_LINKED_USER_ID,
84 CLOUD_MINT_PUBLIC_KEY,
85 decodeRuntimeConfig,
86 encodeEndpointRuntimeConfigJson,
87 encodeConfirmedOriginJson,
88 PUBLISH_AGENT_ACTIVITY_SECRET,
89 HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
90 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
91 RELAY_ISSUER_SECRET,
92 RELAY_URL_SECRET,
93} from "./config.ts";
94import { relayUrlConfig } from "./publicConfig.ts";
95import {
96 readCliDesiredCloudLink,
97 readCliDesiredLinkMode,
98 setCliDesiredCloudLink,
99} from "./CliState.ts";
100import * as CliTokenManager from "./CliTokenManager.ts";
101import * as CloudPreferences from "./CloudPreferences.ts";
102import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
8import * as CloudLink from "./CloudLink.ts";
1039import { traceRelayRequest } from "./traceRelayRequest.ts";
104import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
10510
106const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
107const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
108const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
109const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
110/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
111export const CLOUD_REPLAY_MARKER_PREFIXES = [
112 CLOUD_MINT_NONCE_PREFIX,
113 CLOUD_MINT_JTI_PREFIX,
114 CLOUD_HEALTH_NONCE_PREFIX,
115 CLOUD_HEALTH_JTI_PREFIX,
116] as const;
117const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
118const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
119// The desktop app stops its backends within seconds of writing the marker.
120const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
121const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
122const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
12311const CLOUD_CREDENTIAL_RESPONSE_HEADERS = {
12412 "cache-control": "no-store",
12513 pragma: "no-cache",
......@@ -130,1499 +18,53 @@ const appendCloudCredentialResponseHeaders = HttpEffect.appendPreResponseHandler
13018 Effect.succeed(HttpServerResponse.setHeaders(response, CLOUD_CREDENTIAL_RESPONSE_HEADERS)),
13119);
13220
133const failEnvironmentCloudInternalError =
134 (message: string) =>
135 (cause: unknown): Effect.Effect<never, EnvironmentHttpInternalServerError> =>
136 Effect.logError(message, { cause }).pipe(
137 Effect.flatMap(() => Effect.fail(new EnvironmentHttpInternalServerError({ message }))),
138 );
139
140const failCloudCliTokenManagerError = (error: CliTokenManager.CloudCliTokenManagerError) =>
141 failEnvironmentCloudInternalError(error.message)(error);
142
143const requireRelayUrl = relayUrlConfig.pipe(
144 Effect.mapError(
145 () =>
146 new EnvironmentHttpInternalServerError({
147 message: "T3CODE_RELAY_URL must be configured as a secure absolute HTTPS origin.",
148 }),
149 ),
150);
151
152function bytesToString(bytes: Uint8Array): string {
153 return new TextDecoder().decode(bytes);
154}
155
156function stringToBytes(value: string): Uint8Array {
157 return new TextEncoder().encode(value);
158}
159
160export function consumeCloudReplayGuards(input: {
161 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
162 readonly names: ReadonlyArray<string>;
163 readonly value: Uint8Array;
164}) {
165 return Effect.forEach(
166 input.names,
167 (name) =>
168 input.secrets.create(name, input.value).pipe(
169 Effect.as(true),
170 Effect.catchIf(ServerSecretStore.isSecretStoreError, (error) =>
171 ServerSecretStore.isSecretAlreadyExistsError(error)
172 ? Effect.succeed(false)
173 : Effect.fail(error),
174 ),
175 ),
176 { concurrency: input.names.length },
177 ).pipe(Effect.map((created) => created.every(Boolean)));
178}
179
180function normalizePemForSignedPayload(value: string): string {
181 return value.trim();
182}
183
184function normalizeHostname(hostname: string): string {
185 return hostname
186 .trim()
187 .toLowerCase()
188 .replace(/^\[(.*)\]$/, "$1");
189}
190
191function validateCloudMintPublicKey(
192 publicKey: string,
193): Effect.Effect<void, EnvironmentHttpBadRequestError> {
194 return Effect.try({
195 try: () => NodeCrypto.createPublicKey(publicKey.replace(/\\n/g, "\n")),
196 catch: () =>
197 new EnvironmentHttpBadRequestError({
198 message: "Cloud mint public key must be a valid Ed25519 public key.",
199 }),
200 }).pipe(
201 Effect.flatMap((key) =>
202 key.asymmetricKeyType === "ed25519"
203 ? Effect.void
204 : Effect.fail(
205 new EnvironmentHttpBadRequestError({
206 message: "Cloud mint public key must be a valid Ed25519 public key.",
207 }),
208 ),
209 ),
210 );
211}
212
213function validateRelayConfigPayload(
214 payload: RelayEnvironmentConfigRequest,
215): Effect.Effect<void, EnvironmentHttpBadRequestError> {
216 if (!isSecureRelayUrl(payload.relayUrl)) {
217 return Effect.fail(
218 new EnvironmentHttpBadRequestError({
219 message: "Relay URL must be a secure absolute HTTPS URL.",
220 }),
221 );
222 }
223 if (payload.relayIssuer !== undefined && !isSecureRelayUrl(payload.relayIssuer)) {
224 return Effect.fail(
225 new EnvironmentHttpBadRequestError({
226 message: "Relay issuer must be a secure absolute HTTPS URL.",
227 }),
228 );
229 }
230 if (payload.environmentCredential.trim().length === 0) {
231 return Effect.fail(
232 new EnvironmentHttpBadRequestError({
233 message: "Relay environment credential is required.",
234 }),
235 );
236 }
237 if (payload.cloudUserId.trim().length === 0) {
238 return Effect.fail(
239 new EnvironmentHttpBadRequestError({
240 message: "Cloud user id is required.",
241 }),
242 );
243 }
244 return Effect.void;
245}
246
247function validateLinkedCloudUser(input: {
248 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
249 readonly cloudUserId: string;
250}): Effect.Effect<void, EnvironmentAuth.ServerAuthInternalError | EnvironmentHttpConflictError> {
251 return input.secrets.get(CLOUD_LINKED_USER_ID).pipe(
252 Effect.mapError(
253 (cause) =>
254 new EnvironmentAuth.ServerAuthLinkedCloudAccountVerificationError({
255 cause,
256 }),
257 ),
258 Effect.flatMap((existing) => {
259 if (Option.isNone(existing)) {
260 return Effect.void;
261 }
262 const existingCloudUserId = bytesToString(existing.value);
263 return existingCloudUserId === input.cloudUserId
264 ? Effect.void
265 : Effect.fail(
266 new EnvironmentHttpConflictError({
267 message:
268 "This environment is already linked to a different cloud account. Unlink it before switching accounts.",
269 }),
270 );
271 }),
272 );
273}
274
275function readInstalledCloudUserId(
276 secrets: ServerSecretStore.ServerSecretStore["Service"],
277): Effect.Effect<string, EnvironmentAuth.ServerAuthInternalError> {
278 return secrets.get(CLOUD_LINKED_USER_ID).pipe(
279 Effect.mapError(
280 (cause) =>
281 new EnvironmentAuth.ServerAuthLinkedCloudAccountReadError({
282 cause,
283 }),
284 ),
285 Effect.flatMap((bytes) =>
286 Option.isSome(bytes)
287 ? Effect.succeed(bytesToString(bytes.value))
288 : Effect.fail(new EnvironmentAuth.ServerAuthLinkedCloudAccountMissingError({})),
289 ),
290 );
291}
292
293function isLoopbackHostname(hostname: string): boolean {
294 return LOOPBACK_HOSTNAMES.has(normalizeHostname(hostname));
295}
296
297function firstForwardedHeaderValue(value: string | undefined): string | undefined {
298 const first = value?.split(",")[0]?.trim();
299 return first && first.length > 0 ? first : undefined;
300}
301
302function requestAbsoluteUrl(request: HttpServerRequest.HttpServerRequest): string | null {
303 try {
304 return new URL(request.originalUrl).href;
305 } catch {
306 const host = firstForwardedHeaderValue(request.headers.host) ?? "127.0.0.1";
307 try {
308 return new URL(request.originalUrl, `http://${host}`).href;
309 } catch {
310 return null;
311 }
312 }
313}
314
315function hasForwardedAuthorityHeaders(request: HttpServerRequest.HttpServerRequest): boolean {
316 return (
317 firstForwardedHeaderValue(request.headers["x-forwarded-host"]) !== undefined ||
318 firstForwardedHeaderValue(request.headers["x-forwarded-proto"]) !== undefined
319 );
320}
321
322function endpointRequestPort(url: URL): number {
323 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
324}
325
326export function parseManagedEndpointLocalOrigin(localOrigin: string) {
327 const url = new URL(localOrigin);
328 if (
329 localOrigin !== localOrigin.trim() ||
330 (url.protocol !== "http:" && url.protocol !== "https:") ||
331 url.username !== "" ||
332 url.password !== "" ||
333 url.pathname !== "/" ||
334 url.search !== "" ||
335 url.hash !== "" ||
336 localOrigin.includes("?") ||
337 localOrigin.includes("#")
338 ) {
339 throw new Error("Invalid local origin");
340 }
341 const wsUrl = new URL(url.origin);
342 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
343 return {
344 httpBaseUrl: url.origin,
345 wsBaseUrl: wsUrl.origin,
346 origin: {
347 localHttpHost: url.hostname,
348 localHttpPort: endpointRequestPort(url),
349 } satisfies RelayManagedEndpointOrigin,
350 };
351}
352
353function isAllowedEndpointOrigin(input: {
354 readonly origin: RelayManagedEndpointOrigin;
355 readonly requestUrl: string;
356}): boolean {
357 if (!isLoopbackHostname(input.origin.localHttpHost)) {
358 return false;
359 }
360
361 const url = new URL(input.requestUrl);
362 if (!isLoopbackHostname(url.hostname)) {
363 return false;
364 }
365
366 return input.origin.localHttpPort === endpointRequestPort(url);
367}
368
369// A managed (Cloudflare tunnel) endpoint is provisioned by the relay and must
370// point at a loopback origin. A manual endpoint is reached out of band (e.g.
371// Tailscale) or not advertised at all for publish-only links, so it is not
372// tied to the managed-tunnel scope.
373export function isSupportedLinkProviderKind(request: RelayLinkProofRequest): boolean {
374 return (
375 request.endpoint.providerKind === "cloudflare_tunnel" ||
376 request.endpoint.providerKind === "manual"
377 );
378}
379
380export function linkProofScopes(
381 request: RelayLinkProofRequest,
382): RelayEnvironmentLinkProofPayload["scopes"] {
383 return request.endpoint.providerKind === "cloudflare_tunnel"
384 ? ["agent_activity_notifications", "managed_tunnels"]
385 : ["agent_activity_notifications"];
386}
387
388function hasExactScope(input: {
389 readonly scopes: ReadonlyArray<string>;
390 readonly expected: string;
391}): boolean {
392 return input.scopes.length === 1 && input.scopes[0] === input.expected;
393}
394
395function hasBoundedCloudProofLifetime(input: {
396 readonly iat: number;
397 readonly exp: number;
398 readonly nowSeconds: number;
399}): boolean {
400 return (
401 input.exp > input.iat &&
402 input.exp - input.iat <= CLOUD_PROOF_MAX_LIFETIME_SECONDS &&
403 input.iat <= input.nowSeconds + CLOUD_PROOF_CLOCK_SKEW_SECONDS
404 );
405}
406
407const decodeCloudHealthProof = Schema.decodeUnknownEffect(RelayCloudEnvironmentHealthProofPayload);
408const decodeCloudMintProof = Schema.decodeUnknownEffect(RelayCloudMintCredentialProofPayload);
409
410interface CloudHttpDependencies {
411 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
412 readonly environment: ServerEnvironment.ServerEnvironment["Service"];
413 readonly endpointRuntime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"];
414 readonly environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"];
415 readonly cliTokenManager: CliTokenManager.CloudCliTokenManager["Service"];
416 readonly httpClient: HttpClient.HttpClient;
417 readonly awarenessRelay: AgentAwarenessRelay.AgentAwarenessRelay["Service"];
418}
419
420const cloudHttpDependencies = Effect.gen(function* () {
421 return {
422 secrets: yield* ServerSecretStore.ServerSecretStore,
423 environment: yield* ServerEnvironment.ServerEnvironment,
424 endpointRuntime: yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime,
425 environmentAuth: yield* EnvironmentAuth.EnvironmentAuth,
426 cliTokenManager: yield* CliTokenManager.CloudCliTokenManager,
427 httpClient: yield* HttpClient.HttpClient,
428 awarenessRelay: yield* AgentAwarenessRelay.AgentAwarenessRelay,
429 } satisfies CloudHttpDependencies;
430});
431
432const makeCloudLinkProof = Effect.fn("environment.cloud.makeLinkProof")(function* (
433 dependencies: CloudHttpDependencies,
434 request: RelayLinkProofRequest,
435 requestUrl: string,
436) {
437 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
438 if (
439 !isSupportedLinkProviderKind(request) ||
440 !isAllowedEndpointOrigin({
441 origin: request.origin,
442 requestUrl,
443 })
444 ) {
445 return yield* new EnvironmentHttpBadRequestError({
446 message: "Invalid managed endpoint origin.",
447 });
448 }
449 const now = yield* DateTime.now;
450 const expiresAt = DateTime.add(now, { minutes: 5 });
451 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
452 const descriptor = yield* dependencies.environment.getDescriptor;
453 const payload = {
454 iss: `t3-env:${descriptor.environmentId}`,
455 aud: normalizeRelayIssuer(request.relayIssuer),
456 sub: descriptor.environmentId,
457 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
458 iat: nowSeconds,
459 exp: Math.floor(expiresAt.epochMilliseconds / 1_000),
460 challenge: request.challenge,
461 descriptor,
462 environmentId: descriptor.environmentId,
463 environmentPublicKey: normalizePemForSignedPayload(keyPair.publicKey),
464 endpoint: request.endpoint,
465 origin: request.origin,
466 scopes: linkProofScopes(request),
467 } satisfies RelayEnvironmentLinkProofPayload;
468 return yield* signRelayJwt({
469 privateKey: keyPair.privateKey,
470 typ: RELAY_LINK_PROOF_TYP,
471 payload,
472 }).pipe(
473 Effect.mapError(
474 (cause) =>
475 new EnvironmentAuth.ServerAuthCloudLinkJwtSigningError({
476 cause,
477 }),
478 ),
479 );
480});
481
482const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")(
483 function* (dependencies: CloudHttpDependencies, request: RelayLinkProofRequest) {
484 yield* requireEnvironmentScope(AuthRelayWriteScope);
485 const httpRequest = yield* HttpServerRequest.HttpServerRequest;
486 const requestUrl = requestAbsoluteUrl(httpRequest);
487 if (requestUrl === null || hasForwardedAuthorityHeaders(httpRequest)) {
488 return yield* new EnvironmentHttpBadRequestError({
489 message: "Invalid managed endpoint origin.",
490 });
491 }
492 const proof = yield* makeCloudLinkProof(dependencies, request, requestUrl);
493 yield* appendCloudCredentialResponseHeaders;
494 return proof satisfies RelayEnvironmentLinkProof;
495 },
496 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
497 failEnvironmentCloudInternalError(error.message)(error),
498 ),
499 Effect.catchIf(
500 ServerSecretStore.isSecretStoreError,
501 failEnvironmentCloudInternalError("Could not generate environment link proof."),
502 ),
503 Effect.catchTag(
504 "PlatformError",
505 failEnvironmentCloudInternalError("Could not generate environment link proof."),
506 ),
507);
508
509function managedEndpointRuntimeConfigsMatch(
510 left: RelayManagedEndpointRuntimeConfig,
511 right: RelayManagedEndpointRuntimeConfig,
512): boolean {
513 return (
514 left.providerKind === right.providerKind &&
515 left.connectorToken === right.connectorToken &&
516 left.tunnelId === right.tunnelId &&
517 left.tunnelName === right.tunnelName
518 );
519}
520
521const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* (
522 dependencies: CloudHttpDependencies,
523 input: {
524 readonly config: RelayManagedEndpointRuntimeConfig;
525 readonly configJson: string;
526 readonly origin: RelayManagedEndpointOrigin;
527 },
528) {
529 return yield* dependencies.endpointRuntime.withLinkStateLock(
530 Effect.gen(function* () {
531 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
532 if (Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson) {
533 return null;
534 }
535 const status = yield* dependencies.endpointRuntime.applyConfig(input.config);
536 if (status.status !== "running") {
537 return yield* new EnvironmentCloudEndpointUnavailableError({
538 message: "Managed endpoint runtime could not be started.",
539 endpointRuntimeStatus: status,
540 });
541 }
542 const marker = yield* encodeConfirmedOriginJson({
543 config: input.config,
544 origin: input.origin,
545 });
546 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
547 return status;
548 }),
549 );
550});
551
552const activateManagedTunnelWithRetry = (
553 dependencies: CloudHttpDependencies,
554 input: {
555 readonly config: RelayManagedEndpointRuntimeConfig;
556 readonly configJson: string;
557 readonly origin: RelayManagedEndpointOrigin;
558 },
559 retryRuntimeFailures: boolean,
560) => {
561 const activate = activateManagedTunnel(dependencies, input);
562 return retryRuntimeFailures
563 ? activate.pipe(
564 Effect.retry({
565 while: (error) =>
566 error._tag === "EnvironmentCloudEndpointUnavailableError" &&
567 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus(
568 error.endpointRuntimeStatus,
569 ),
570 schedule: Schedule.exponential("1 second").pipe(
571 Schedule.modifyDelay(({ duration }) =>
572 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
573 ),
574 Schedule.jittered,
575 ),
576 }),
577 )
578 : activate;
579};
580
581export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn(
582 "environment.cloud.startManagedCloudTunnelIfOriginConfirmed",
583)(function* (localOrigin: string, options?: { readonly requireConfirmedOrigin?: boolean }) {
584 const dependencies = yield* cloudHttpDependencies;
585 const requireConfirmedOrigin = options?.requireConfirmedOrigin ?? true;
586 const parsedOrigin = yield* Effect.try({
587 try: () => parseManagedEndpointLocalOrigin(localOrigin),
588 catch: () =>
589 new EnvironmentHttpBadRequestError({
590 message: "Could not resolve local environment origin.",
591 }),
592 });
593 return yield* dependencies.endpointRuntime.withLinkStateLock(
594 Effect.gen(function* () {
595 const [runtimeBytes, markerBytes] = yield* Effect.all([
596 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
597 dependencies.secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
598 ]);
599 if (Option.isNone(runtimeBytes)) return false;
600 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value)));
601 if (config === null || config.providerKind !== "cloudflare_tunnel") return false;
602 // With the marker required, only a config the relay already confirmed on
603 // this port may start. Without it, startup is falling back after the
604 // relay stayed unreachable: an unconfirmed origin may send traffic to a
605 // stale port, but that beats no remote access at all.
606 if (requireConfirmedOrigin) {
607 if (Option.isNone(markerBytes)) return false;
608 const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value)));
609 if (
610 marker === null ||
611 !managedEndpointRuntimeConfigsMatch(marker.config, config) ||
612 marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost ||
613 marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort
614 ) {
615 return false;
616 }
617 }
618 const status = yield* dependencies.endpointRuntime.applyConfig(config);
619 if (status.status !== "running") {
620 return yield* new EnvironmentCloudEndpointUnavailableError({
621 message: "Managed endpoint runtime could not be started.",
622 endpointRuntimeStatus: status,
623 });
624 }
625 return true;
626 }),
627 );
628});
629
630const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
631 dependencies: CloudHttpDependencies,
632 payload: RelayEnvironmentConfigRequest,
633 options?: {
634 readonly lockHeld?: boolean;
635 readonly confirmedOrigin?: RelayManagedEndpointOrigin;
636 },
637) {
638 const apply = Effect.gen(function* () {
639 yield* validateRelayConfigPayload(payload);
640 yield* validateLinkedCloudUser({
641 secrets: dependencies.secrets,
642 cloudUserId: payload.cloudUserId,
643 });
644 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
645 // Reject unsupported runtimes before touching the connector so a bad
646 // payload cannot stop a healthy tunnel on its way to a 503.
647 if (
648 payload.endpointRuntime !== null &&
649 payload.endpointRuntime.providerKind !== "cloudflare_tunnel"
650 ) {
651 return yield* new EnvironmentCloudEndpointUnavailableError({
652 message: "Managed endpoint runtime could not be started.",
653 endpointRuntimeStatus: {
654 status: "unsupported",
655 providerKind: payload.endpointRuntime.providerKind,
656 },
657 });
658 }
659 yield* dependencies.endpointRuntime.applyConfig(null);
660 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
661
662 yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
663 yield* dependencies.secrets.set(
664 RELAY_ISSUER_SECRET,
665 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
666 );
667 yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
668 yield* dependencies.secrets.set(
669 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
670 stringToBytes(payload.environmentCredential),
671 );
672 yield* dependencies.secrets.set(
673 CLOUD_MINT_PUBLIC_KEY,
674 stringToBytes(payload.cloudMintPublicKey),
675 );
676 yield* dependencies.awarenessRelay.requestCatchUp();
677 if (payload.endpointRuntime) {
678 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
679 yield* dependencies.secrets.set(
680 CLOUD_ENDPOINT_RUNTIME_CONFIG,
681 stringToBytes(endpointRuntimeJson),
682 );
683 } else {
684 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
685 }
686 if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) {
687 return {
688 ok: true,
689 endpointRuntimeStatus: { status: "disabled" },
690 } satisfies EnvironmentCloudRelayConfigResult;
691 }
692 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(
693 payload.endpointRuntime,
694 );
695 if (endpointRuntimeStatus.status !== "running") {
696 return yield* new EnvironmentCloudEndpointUnavailableError({
697 message: "Managed endpoint runtime could not be started.",
698 endpointRuntimeStatus,
699 });
700 }
701 const marker = yield* encodeConfirmedOriginJson({
702 config: payload.endpointRuntime,
703 origin: options.confirmedOrigin,
704 });
705 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
706 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
707 });
708 return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply);
709});
710
711const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
712 function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) {
713 yield* requireEnvironmentScope(AuthRelayWriteScope);
714 const result = yield* applyCloudRelayConfig(dependencies, payload);
715 if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") {
716 const server = yield* HttpServer.HttpServer;
717 const address = server.address;
718 if (typeof address === "string" || !("port" in address)) {
719 return yield* new EnvironmentHttpInternalServerError({
720 message: "Could not resolve the local server origin.",
721 });
722 }
723 const registration = yield* registerManagedCloudTunnelRecovery(
724 `http://127.0.0.1:${address.port}`,
725 ).pipe(
726 Effect.retry({
727 times: 2,
728 while: (error) =>
729 shouldRetryCloudLink(error) &&
730 error._tag !== "EnvironmentCloudEndpointUnavailableError",
731 }),
732 );
733 if (registration.status === "superseded") {
734 return yield* new EnvironmentHttpConflictError({
735 message: "The managed tunnel configuration changed during registration.",
736 });
737 }
738 if (registration.status === "recovery_required") {
739 yield* dependencies.endpointRuntime.requestRecovery(registration.config);
740 }
741 if (registration.status !== "ready") {
742 return yield* new EnvironmentCloudEndpointUnavailableError({
743 message: "Managed endpoint origin could not be confirmed.",
744 endpointRuntimeStatus: { status: "disabled" },
745 });
746 }
747 return {
748 ok: true,
749 endpointRuntimeStatus: registration.endpointRuntimeStatus,
750 } satisfies EnvironmentCloudRelayConfigResult;
751 }
752 return result;
753 },
754 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
755 failEnvironmentCloudInternalError(error.message)(error),
756 ),
757 Effect.catchIf(
758 ServerSecretStore.isSecretStoreError,
759 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
760 ),
761 Effect.catchTags({
762 SchemaError: failEnvironmentCloudInternalError(
763 "Could not persist environment relay configuration.",
764 ),
765 PlatformError: failEnvironmentCloudInternalError(
766 "Could not register the managed endpoint origin.",
767 ),
768 }),
769);
770
771const relayClientRequest = <A>(
772 dependencies: CloudHttpDependencies,
773 input: {
774 readonly url: string;
775 readonly token: string;
776 readonly payload: unknown;
777 readonly schema: Schema.Decoder<A>;
778 readonly timeout?: Duration.Input;
779 },
780) =>
781 HttpClientRequest.post(input.url).pipe(
782 HttpClientRequest.bearerToken(input.token),
783 HttpClientRequest.bodyJson(input.payload),
784 Effect.flatMap(dependencies.httpClient.execute),
785 Effect.flatMap(filterRelayResponse),
786 Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
787 Effect.timeout(input.timeout ?? "10 seconds"),
788 Effect.mapError(relayRequestError),
789 withRelayClientTracing,
790 );
791
792const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
793 function* (dependencies: CloudHttpDependencies, localOrigin: string) {
794 const parsedOrigin = yield* Effect.try({
795 try: () => parseManagedEndpointLocalOrigin(localOrigin),
796 catch: () =>
797 new EnvironmentHttpBadRequestError({
798 message: "Could not resolve local environment origin.",
799 }),
800 });
801 const token = yield* dependencies.cliTokenManager.getExisting.pipe(
802 Effect.flatMap(
803 Option.match({
804 onNone: () =>
805 Effect.fail(
806 new EnvironmentHttpUnauthorizedError({
807 message: "Run `t3 connect link` to authorize this environment.",
808 }),
809 ),
810 onSome: Effect.succeed,
811 }),
812 ),
813 );
814 const mode = yield* readCliDesiredLinkMode;
815 const managedTunnelsEnabled = mode !== "publish_only";
816 const relayUrl = yield* requireRelayUrl;
817 const challenge = yield* relayClientRequest(dependencies, {
818 url: `${relayUrl}/v1/client/environment-link-challenges`,
819 token: token.accessToken,
820 payload: {
821 notificationsEnabled: true,
822 liveActivitiesEnabled: true,
823 managedTunnelsEnabled,
824 },
825 schema: RelayEnvironmentLinkChallengeResponse,
826 });
827 const proof = yield* makeCloudLinkProof(
828 dependencies,
829 {
830 challenge: challenge.challenge,
831 relayIssuer: relayUrl,
832 endpoint: {
833 httpBaseUrl: parsedOrigin.httpBaseUrl,
834 wsBaseUrl: parsedOrigin.wsBaseUrl,
835 providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",
836 },
837 origin: parsedOrigin.origin,
838 },
839 parsedOrigin.httpBaseUrl,
840 );
841 const link = yield* relayClientRequest(dependencies, {
842 url: `${relayUrl}/v1/client/environment-links`,
843 token: token.accessToken,
844 payload: {
845 proof,
846 notificationsEnabled: true,
847 liveActivitiesEnabled: true,
848 managedTunnelsEnabled,
849 },
850 schema: RelayEnvironmentLinkResponse,
851 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
852 });
853 yield* setCliDesiredCloudLink(true, mode);
854 yield* applyCloudRelayConfig(
855 dependencies,
856 {
857 relayUrl,
858 relayIssuer: link.relayIssuer,
859 cloudUserId: link.cloudUserId,
860 environmentCredential: link.environmentCredential,
861 cloudMintPublicKey: link.cloudMintPublicKey,
862 endpointRuntime: link.endpointRuntime,
863 },
864 {
865 lockHeld: true,
866 confirmedOrigin: parsedOrigin.origin,
867 },
868 );
869 // Callers decide on managed tunnel recovery from the mode this link
870 // actually used, not from a value read before the relay round trip.
871 return mode;
872 },
873 Effect.catchIf(
874 ServerSecretStore.isSecretStoreError,
875 failEnvironmentCloudInternalError("Could not persist desired T3 Connect link state."),
876 ),
877 Effect.catchTags({
878 CloudCliCredentialRemovalError: failCloudCliTokenManagerError,
879 CloudCliCredentialRefreshError: failCloudCliTokenManagerError,
880 CloudCliCredentialReadError: failCloudCliTokenManagerError,
881 CloudCliAuthorizationError: failCloudCliTokenManagerError,
882 CloudCliAuthorizationTimeoutError: failCloudCliTokenManagerError,
883 }),
884);
885
886export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")(
887 function* (localOrigin: string) {
888 const dependencies = yield* cloudHttpDependencies;
889 return yield* dependencies.endpointRuntime.withLinkStateLock(
890 reconcileDesiredCloudLinkWith(dependencies, localOrigin),
891 );
892 },
893);
894
895export const reconcileDesiredCloudLinkIfStillDesired = Effect.fn(
896 "environment.cloud.reconcileDesiredLinkIfStillDesired",
897)(function* (localOrigin: string) {
898 const dependencies = yield* cloudHttpDependencies;
899 return yield* dependencies.endpointRuntime.withLinkStateLock(
900 Effect.gen(function* () {
901 if (!(yield* readCliDesiredCloudLink)) {
902 return null;
903 }
904 return yield* reconcileDesiredCloudLinkWith(dependencies, localOrigin);
905 }),
906 );
907});
908
909type ManagedTunnelRecoveryProofInput = {
910 readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"];
911 readonly cloudUserId: string;
912 readonly relayUrl: string;
913} & (
914 | {
915 readonly action: "register";
916 readonly tunnelId: string;
917 readonly origin: RelayManagedEndpointOrigin;
918 }
919 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
920);
921
922const makeManagedTunnelRecoveryProof = Effect.fn(
923 "environment.cloud.makeManagedTunnelRecoveryProof",
924)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) {
925 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
926 const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET);
927 const now = yield* DateTime.now;
928 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
929 const claims = {
930 iss: `t3-env:${input.environmentId}`,
931 aud: normalizeRelayIssuer(
932 Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl,
933 ),
934 sub: input.environmentId,
935 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
936 iat: issuedAt,
937 exp: issuedAt + 60,
938 environmentId: input.environmentId,
939 cloudUserId: input.cloudUserId,
940 };
941 const payload =
942 input.action === "register"
943 ? {
944 ...claims,
945 action: "register" as const,
946 tunnelId: input.tunnelId,
947 origin: input.origin,
948 }
949 : { ...claims, action: "recover" as const, origin: input.origin };
950
951 return yield* signRelayJwt({
952 privateKey: keyPair.privateKey,
953 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
954 payload,
955 }).pipe(
956 Effect.mapError(
957 () =>
958 new EnvironmentHttpInternalServerError({
959 message: "Could not sign the managed tunnel recovery request.",
960 }),
961 ),
962 );
963});
964
965export const registerManagedCloudTunnelRecovery = Effect.fn(
966 "environment.cloud.registerManagedCloudTunnelRecovery",
967)(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) {
968 const dependencies = yield* cloudHttpDependencies;
969 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
970 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
971 dependencies.secrets.get(RELAY_URL_SECRET),
972 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
973 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
974 ]);
975 if (
976 Option.isNone(runtimeConfig) ||
977 Option.isNone(relayUrl) ||
978 Option.isNone(cloudUserId) ||
979 Option.isNone(environmentCredential)
980 ) {
981 return { status: "not_linked" as const };
982 }
983
984 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
985 if (config?.providerKind !== "cloudflare_tunnel") {
986 return { status: "not_linked" as const };
987 }
988
989 const parsedOrigin = yield* Effect.try({
990 try: () => parseManagedEndpointLocalOrigin(localOrigin),
991 catch: () =>
992 new EnvironmentHttpBadRequestError({
993 message: "Could not resolve local environment origin.",
994 }),
995 });
996 if (config.tunnelId === undefined) {
997 return { status: "recovery_required" as const, config };
998 }
999 const origin = parsedOrigin.origin;
1000 const environmentId = yield* dependencies.environment.getEnvironmentId;
1001 const relayUrlValue = bytesToString(relayUrl.value);
1002 const cloudUserIdValue = bytesToString(cloudUserId.value);
1003 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
1004 action: "register",
1005 environmentId,
1006 cloudUserId: cloudUserIdValue,
1007 relayUrl: relayUrlValue,
1008 tunnelId: config.tunnelId,
1009 origin,
1010 });
1011 const registered = yield* relayClientRequest(dependencies, {
1012 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`,
1013 token: bytesToString(environmentCredential.value),
1014 payload: {
1015 cloudUserId: cloudUserIdValue,
1016 tunnelId: config.tunnelId,
1017 origin,
1018 proof,
1019 },
1020 schema: RelayManagedEndpointRecoveryRegistrationResponse,
1021 });
1022 if (registered.status === "recovery_required") {
1023 return { status: registered.status, config };
1024 }
1025 const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry(
1026 dependencies,
1027 {
1028 config,
1029 configJson: bytesToString(runtimeConfig.value),
1030 origin,
1031 },
1032 options?.retryRuntimeFailures === true,
1033 );
1034 return endpointRuntimeStatus === null
1035 ? { status: "superseded" as const }
1036 : { status: "ready" as const, endpointRuntimeStatus };
1037});
1038
1039export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")(
1040 function* (
1041 localOrigin: string,
1042 expectedConfig?: RelayManagedEndpointRuntimeConfig,
1043 options?: { readonly retryRuntimeFailures?: boolean },
1044 ) {
1045 const dependencies = yield* cloudHttpDependencies;
1046 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
1047 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1048 dependencies.secrets.get(RELAY_URL_SECRET),
1049 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1050 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1051 ]);
1052 if (
1053 Option.isNone(runtimeConfig) ||
1054 Option.isNone(relayUrl) ||
1055 Option.isNone(cloudUserId) ||
1056 Option.isNone(environmentCredential)
1057 ) {
1058 return false;
1059 }
1060 if (expectedConfig !== undefined) {
1061 const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
1062 if (
1063 current === null ||
1064 current.providerKind !== expectedConfig.providerKind ||
1065 current.connectorToken !== expectedConfig.connectorToken ||
1066 current.tunnelId !== expectedConfig.tunnelId ||
1067 current.tunnelName !== expectedConfig.tunnelName
1068 ) {
1069 return false;
1070 }
1071 }
1072
1073 const parsedOrigin = yield* Effect.try({
1074 try: () => parseManagedEndpointLocalOrigin(localOrigin),
1075 catch: () =>
1076 new EnvironmentHttpBadRequestError({
1077 message: "Could not resolve local environment origin.",
1078 }),
1079 });
1080
1081 const environmentId = yield* dependencies.environment.getEnvironmentId;
1082 const relayUrlValue = bytesToString(relayUrl.value);
1083 const cloudUserIdValue = bytesToString(cloudUserId.value);
1084 const origin = parsedOrigin.origin;
1085 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
1086 action: "recover",
1087 environmentId,
1088 cloudUserId: cloudUserIdValue,
1089 relayUrl: relayUrlValue,
1090 origin,
1091 });
1092 const recovered = yield* relayClientRequest(dependencies, {
1093 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`,
1094 token: bytesToString(environmentCredential.value),
1095 payload: {
1096 cloudUserId: cloudUserIdValue,
1097 origin,
1098 proof,
1099 },
1100 schema: RelayManagedEndpointRecoveryResponse,
1101 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
1102 });
1103 if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") {
1104 return yield* new EnvironmentHttpInternalServerError({
1105 message: "T3 Connect returned an unsupported managed tunnel configuration.",
1106 });
1107 }
1108
1109 const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe(
1110 Effect.mapError(
1111 () =>
1112 new EnvironmentHttpInternalServerError({
1113 message: "Could not persist the recovered managed tunnel configuration.",
1114 }),
1115 ),
1116 );
1117 const stored = yield* dependencies.endpointRuntime.withLinkStateLock(
1118 Effect.gen(function* () {
1119 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1120 if (
1121 Option.isNone(currentConfig) ||
1122 bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value)
1123 ) {
1124 return false;
1125 }
1126 yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded));
1127 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1128 return true;
1129 }),
1130 );
1131 if (!stored) return false;
1132 const status = yield* activateManagedTunnelWithRetry(
1133 dependencies,
1134 {
1135 config: recovered.endpointRuntime,
1136 configJson: encoded,
1137 origin,
1138 },
1139 options?.retryRuntimeFailures === true,
1140 );
1141 return status !== null;
1142 },
1143);
1144
1145// The launcher owns this durable state, so read it directly both when a trial
1146// decides whether it owns pre-activation cleanup and while a server tears down.
1147export const pendingServiceUpdateExists = Effect.gen(function* () {
1148 const config = yield* ServerConfig.ServerConfig;
1149 const fs = yield* FileSystem.FileSystem;
1150 const path = yield* Path.Path;
1151 const runtimeDir = path.join(config.baseDir, "runtime");
1152 const stateText = yield* fs
1153 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
1154 .pipe(Effect.option);
1155 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
1156});
1157
1158// A pending update alone is not proof a replacement server is coming: an
1159// explicit launcher stop (`t3 service uninstall`, `systemctl stop`,
1160// `launchctl bootout`) during
1161// the pending window also tears this server down. The launcher marks that case
1162// just before it signals the child, so pending + no marker is the handoff.
1163const pendingUpdateHandoffExists = Effect.gen(function* () {
1164 if (!(yield* pendingServiceUpdateExists)) {
1165 return false;
1166 }
1167 const config = yield* ServerConfig.ServerConfig;
1168 const fs = yield* FileSystem.FileSystem;
1169 const path = yield* Path.Path;
1170 const runtimeDir = path.join(config.baseDir, "runtime");
1171 const stopping = yield* fs
1172 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
1173 .pipe(Effect.orElseSucceed(() => false));
1174 return !stopping;
1175});
1176
1177// The desktop app writes its marker right before it stops this server to
1178// install an update, whether a remote client or the local app started it.
1179// Reading consumes it, so shutdown checks it first. Only a fresh marker counts,
1180// so a marker the server never read (a hard kill) cannot keep the tunnel on a
1181// later quit.
1182const desktopUpdateRestartPending = Effect.gen(function* () {
1183 const config = yield* ServerConfig.ServerConfig;
1184 const fs = yield* FileSystem.FileSystem;
1185 const path = yield* Path.Path;
1186 const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE);
1187 const marker = yield* fs.stat(markerPath).pipe(Effect.option);
1188 if (Option.isNone(marker)) {
1189 return false;
1190 }
1191 yield* fs.remove(markerPath).pipe(Effect.ignore);
1192 const now = yield* Clock.currentTimeMillis;
1193 return Option.match(marker.value.mtime, {
1194 onNone: () => false,
1195 onSome: (writtenAt) =>
1196 now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL),
1197 });
1198});
1199
1200// Cloudflare bills per provisioned tunnel, so an environment that goes offline
1201// must not leave its tunnel behind. Releasing deletes only the tunnel — the
1202// relay keeps the link and its hostname reservation, and the next startup's
1203// link reconcile provisions a replacement tunnel under the same URL.
1204export const releaseManagedTunnelOnShutdown = Effect.fn(
1205 "environment.cloud.releaseManagedTunnelOnShutdown",
1206)(function* () {
1207 const dependencies = yield* cloudHttpDependencies;
1208 // Only a managed link stores a runtime config; publish-only links have no
1209 // tunnel to release.
1210 const runtimeConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1211 if (Option.isNone(runtimeConfig)) {
1212 return false;
1213 }
1214 // Only CLI-desired managed links release eagerly because this request uses
1215 // CLI authorization. Web/mobile links register startup recovery with their
1216 // environment credential, and the relay reaper removes them after they are
1217 // down for the configured grace period. Unlink still deletes either kind.
1218 if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
1219 return false;
1220 }
1221 // A shutdown that hands off to a pending update is not the environment
1222 // going offline: the service launcher or the desktop app immediately brings
1223 // a server back (the new version, or the old one after a rollback). Deleting
1224 // the tunnel here forces that server to provision a replacement UUID, and the
1225 // public hostname's route to the new tunnel takes 1-2 minutes to propagate —
1226 // the dominant cost of an update restart. Keep the tunnel instead: the next
1227 // boot respawns the connector from the stored config and is reachable as
1228 // soon as it connects, and the reconcile confirms the still-live tunnel
1229 // without replacing it.
1230 if ((yield* desktopUpdateRestartPending) || (yield* pendingUpdateHandoffExists)) {
1231 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
1232 return false;
1233 }
1234 const token = yield* dependencies.cliTokenManager.getExisting;
1235 if (Option.isNone(token)) {
1236 return false;
1237 }
1238 // The link belongs to the relay it was installed against, so target the
1239 // persisted URL: T3CODE_RELAY_URL may have changed since the link was made.
1240 const relayUrl = yield* dependencies.secrets.get(RELAY_URL_SECRET);
1241 if (Option.isNone(relayUrl)) {
1242 return false;
1243 }
1244 const environmentId = yield* dependencies.environment.getEnvironmentId;
1245 // Stop the local connector before the relay deletes the tunnel it serves.
1246 yield* dependencies.endpointRuntime.applyConfig(null);
1247 const response = yield* HttpClientRequest.delete(
1248 `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`,
1249 ).pipe(
1250 HttpClientRequest.bearerToken(token.value.accessToken),
1251 dependencies.httpClient.execute,
1252 Effect.flatMap(filterRelayResponse),
1253 Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
1254 withRelayClientTracing,
1255 );
1256 // ok:false means the relay skipped deletion because a concurrent provision
1257 // owns the recorded tunnel now — leave the stored config alone.
1258 if (!response.ok) {
1259 return false;
1260 }
1261 // The connector token died with the tunnel. Drop the stored config so the
1262 // next start waits for the link reconcile instead of respawning the relay
1263 // client with a dead token. Kept when the release request fails: the tunnel
1264 // still exists, so the stored token keeps working across the restart.
1265 // Only dropped while it is still the config this shutdown released — a fast
1266 // restart may already have reconciled and stored a fresh config for its
1267 // replacement tunnel, and that one must survive this finalizer.
1268 const storedConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1269 if (
1270 Option.isSome(storedConfig) &&
1271 bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)
1272 ) {
1273 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1274 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1275 }
1276 return true;
1277});
1278
1279const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (
1280 dependencies: CloudHttpDependencies,
1281) {
1282 const [
1283 cloudUserId,
1284 relayUrl,
1285 relayIssuer,
1286 endpointRuntimeConfig,
1287 publishAgentActivity,
1288 holdWebhooks,
1289 ] = yield* Effect.all(
1290 [
1291 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1292 dependencies.secrets.get(RELAY_URL_SECRET),
1293 dependencies.secrets.get(RELAY_ISSUER_SECRET),
1294 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1295 dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
1296 dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1297 ],
1298 { concurrency: 6 },
1299 );
1300 return {
1301 linked: Option.isSome(cloudUserId),
1302 cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
1303 relayUrl: Option.isSome(relayUrl) ? bytesToString(relayUrl.value) : null,
1304 relayIssuer: Option.isSome(relayIssuer) ? bytesToString(relayIssuer.value) : null,
1305 // The managed tunnel runtime config is only stored for managed links; a
1306 // publish-only link leaves it absent.
1307 managedTunnelActive: Option.isSome(endpointRuntimeConfig),
1308 publishAgentActivity: Option.isSome(publishAgentActivity)
1309 ? bytesToString(publishAgentActivity.value) === "true"
1310 : false,
1311 holdWebhooksWhileOffline:
1312 Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true",
1313 } satisfies EnvironmentCloudLinkStateResult;
1314});
1315
1316const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")(
1317 function* (dependencies: CloudHttpDependencies) {
1318 yield* requireEnvironmentScope(AuthRelayReadScope);
1319 return yield* readCloudLinkState(dependencies);
1320 },
1321 Effect.catchIf(
1322 ServerSecretStore.isSecretStoreError,
1323 failEnvironmentCloudInternalError("Could not read environment relay configuration."),
1324 ),
1325);
1326
1327const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
1328 function* (dependencies: CloudHttpDependencies) {
1329 yield* requireEnvironmentScope(AuthRelayWriteScope);
1330 return yield* dependencies.endpointRuntime.withLinkStateLock(
1331 Effect.gen(function* () {
1332 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);
1333 yield* Effect.all(
1334 [
1335 dependencies.secrets.remove(CLOUD_LINKED_USER_ID),
1336 dependencies.secrets.remove(RELAY_URL_SECRET),
1337 dependencies.secrets.remove(RELAY_ISSUER_SECRET),
1338 dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1339 dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),
1340 dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1341 dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1342 dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1343 dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1344 ],
1345 { concurrency: 9 },
1346 );
1347 yield* setCliDesiredCloudLink(false);
1348 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
1349 }),
1350 );
1351 },
1352 Effect.catchIf(
1353 ServerSecretStore.isSecretStoreError,
1354 failEnvironmentCloudInternalError("Could not remove environment relay configuration."),
1355 ),
1356);
1357
1358const cloudEnvironmentHealthHandler = Effect.fn("environment.cloud.health")(
1359 function* (dependencies: CloudHttpDependencies, request: RelayCloudEnvironmentHealthRequest) {
1360 const cloudMintPublicKey = yield* dependencies.secrets
1361 .get(CLOUD_MINT_PUBLIC_KEY)
1362 .pipe(
1363 Effect.flatMap((bytes) =>
1364 Option.isSome(bytes)
1365 ? Effect.succeed(bytesToString(bytes.value))
1366 : Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
1367 ),
1368 );
1369 const relayIssuer = yield* dependencies.secrets
1370 .get(RELAY_ISSUER_SECRET)
1371 .pipe(
1372 Effect.flatMap((bytes) =>
1373 Option.isSome(bytes)
1374 ? Effect.succeed(bytesToString(bytes.value))
1375 : dependencies.secrets
1376 .get(RELAY_URL_SECRET)
1377 .pipe(
1378 Effect.flatMap((fallbackBytes) =>
1379 Option.isSome(fallbackBytes)
1380 ? Effect.succeed(bytesToString(fallbackBytes.value))
1381 : Effect.fail(new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({})),
1382 ),
1383 ),
1384 ),
1385 );
1386 const environmentId = yield* dependencies.environment.getEnvironmentId;
1387 const linkedCloudUserId = yield* readInstalledCloudUserId(dependencies.secrets);
1388 const now = yield* DateTime.now;
1389 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
1390 const proofOption = yield* verifyRelayJwt({
1391 publicKey: cloudMintPublicKey,
1392 token: request.proof,
1393 typ: RELAY_HEALTH_REQUEST_TYP,
1394 issuer: normalizeRelayIssuer(relayIssuer),
1395 audience: `t3-env:${environmentId}`,
1396 nowEpochSeconds: nowSeconds,
1397 }).pipe(Effect.flatMap(decodeCloudHealthProof), Effect.option);
1398 if (
1399 Option.isNone(proofOption) ||
1400 proofOption.value.environmentId !== environmentId ||
1401 proofOption.value.sub !== linkedCloudUserId ||
1402 !hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
1403 !hasExactScope({ scopes: proofOption.value.scope, expected: "environment:status" })
1404 ) {
1405 return yield* new EnvironmentHttpUnauthorizedError({
1406 message: "Invalid cloud health request.",
1407 });
1408 }
1409 const proof = proofOption.value;
1410
1411 const jtiSecretName = `${CLOUD_HEALTH_JTI_PREFIX}${proof.jti}`;
1412 const nonceSecretName = `${CLOUD_HEALTH_NONCE_PREFIX}${proof.nonce}`;
1413 const consumedReplayGuards = yield* consumeCloudReplayGuards({
1414 secrets: dependencies.secrets,
1415 names: [jtiSecretName, nonceSecretName],
1416 value: stringToBytes(DateTime.formatIso(now)),
1417 });
1418 if (!consumedReplayGuards) {
1419 return yield* new EnvironmentHttpConflictError({
1420 message: "Cloud health request was already consumed.",
1421 });
1422 }
1423
1424 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
1425 const descriptor = yield* dependencies.environment.getDescriptor;
1426 const responseExpiresAt = DateTime.add(now, { minutes: 5 });
1427 const responsePayload = {
1428 iss: `t3-env:${environmentId}`,
1429 aud: normalizeRelayIssuer(relayIssuer),
1430 sub: environmentId,
1431 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
1432 iat: nowSeconds,
1433 exp: Math.floor(responseExpiresAt.epochMilliseconds / 1_000),
1434 environmentId,
1435 requestNonce: proof.nonce,
1436 status: "online",
1437 descriptor,
1438 checkedAt: DateTime.formatIso(now),
1439 } satisfies RelayEnvironmentHealthResponseProofPayload;
1440 const responseProof = yield* signRelayJwt({
1441 privateKey: keyPair.privateKey,
1442 typ: RELAY_HEALTH_RESPONSE_TYP,
1443 payload: responsePayload,
1444 }).pipe(
1445 Effect.mapError(
1446 (cause) =>
1447 new EnvironmentAuth.ServerAuthCloudHealthJwtSigningError({
1448 cause,
1449 }),
1450 ),
1451 );
1452 const response = {
1453 environmentId,
1454 status: "online",
1455 descriptor,
1456 checkedAt: responsePayload.checkedAt,
1457 proof: responseProof,
1458 } satisfies RelayEnvironmentHealthResponseShape;
1459
1460 yield* appendCloudCredentialResponseHeaders;
1461 return response;
1462 },
1463 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
1464 failEnvironmentCloudInternalError(error.message)(error),
1465 ),
1466 Effect.catchIf(
1467 ServerSecretStore.isSecretStoreError,
1468 failEnvironmentCloudInternalError("Could not answer cloud health request."),
1469 ),
1470 Effect.catchTag(
1471 "PlatformError",
1472 failEnvironmentCloudInternalError("Could not answer cloud health request."),
1473 ),
1474);
1475
1476const cloudMintCredentialHandler = Effect.fn("environment.cloud.mintCredential")(
1477 function* (dependencies: CloudHttpDependencies, request: RelayCloudMintCredentialRequest) {
1478 const cloudMintPublicKey = yield* dependencies.secrets
1479 .get(CLOUD_MINT_PUBLIC_KEY)
1480 .pipe(
1481 Effect.flatMap((bytes) =>
1482 Option.isSome(bytes)
1483 ? Effect.succeed(bytesToString(bytes.value))
1484 : Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
1485 ),
1486 );
1487 const relayIssuer = yield* dependencies.secrets
1488 .get(RELAY_ISSUER_SECRET)
1489 .pipe(
1490 Effect.flatMap((bytes) =>
1491 Option.isSome(bytes)
1492 ? Effect.succeed(bytesToString(bytes.value))
1493 : dependencies.secrets
1494 .get(RELAY_URL_SECRET)
1495 .pipe(
1496 Effect.flatMap((fallbackBytes) =>
1497 Option.isSome(fallbackBytes)
1498 ? Effect.succeed(bytesToString(fallbackBytes.value))
1499 : Effect.fail(new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({})),
1500 ),
1501 ),
1502 ),
1503 );
1504 const environmentId = yield* dependencies.environment.getEnvironmentId;
1505 const linkedCloudUserId = yield* readInstalledCloudUserId(dependencies.secrets);
1506 const now = yield* DateTime.now;
1507 const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
1508 const proofOption = yield* verifyRelayJwt({
1509 publicKey: cloudMintPublicKey,
1510 token: request.proof,
1511 typ: RELAY_MINT_REQUEST_TYP,
1512 issuer: normalizeRelayIssuer(relayIssuer),
1513 audience: `t3-env:${environmentId}`,
1514 nowEpochSeconds: nowSeconds,
1515 }).pipe(Effect.flatMap(decodeCloudMintProof), Effect.option);
1516 if (
1517 Option.isNone(proofOption) ||
1518 proofOption.value.environmentId !== environmentId ||
1519 proofOption.value.sub !== linkedCloudUserId ||
1520 proofOption.value.cnf.jkt !== proofOption.value.clientProofKeyThumbprint ||
1521 !hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
1522 !hasExactScope({ scopes: proofOption.value.scope, expected: "environment:connect" })
1523 ) {
1524 return yield* new EnvironmentHttpUnauthorizedError({
1525 message: "Invalid cloud mint request.",
1526 });
1527 }
1528 const proof = proofOption.value;
1529
1530 const jtiSecretName = `${CLOUD_MINT_JTI_PREFIX}${proof.jti}`;
1531 const nonceSecretName = `${CLOUD_MINT_NONCE_PREFIX}${proof.nonce}`;
1532 const consumedReplayGuards = yield* consumeCloudReplayGuards({
1533 secrets: dependencies.secrets,
1534 names: [jtiSecretName, nonceSecretName],
1535 value: stringToBytes(DateTime.formatIso(now)),
1536 });
1537 if (!consumedReplayGuards) {
1538 return yield* new EnvironmentHttpConflictError({
1539 message: "Cloud mint request was already consumed.",
1540 });
1541 }
1542
1543 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
1544 const issued = yield* dependencies.environmentAuth.createPairingLink({
1545 scopes: AuthStandardClientScopes,
1546 subject: "cloud-connect",
1547 ttl: Duration.minutes(2),
1548 label: "T3 Connect connect",
1549 proofKeyThumbprint: proof.clientProofKeyThumbprint,
1550 });
1551 const responsePayload = {
1552 iss: `t3-env:${environmentId}`,
1553 aud: normalizeRelayIssuer(relayIssuer),
1554 sub: environmentId,
1555 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
1556 iat: nowSeconds,
1557 exp: Math.floor(issued.expiresAt.epochMilliseconds / 1_000),
1558 environmentId,
1559 clientProofKeyThumbprint: proof.clientProofKeyThumbprint,
1560 requestNonce: proof.nonce,
1561 credential: issued.credential,
1562 } satisfies RelayEnvironmentMintResponseProofPayload;
1563 const responseProof = yield* signRelayJwt({
1564 privateKey: keyPair.privateKey,
1565 typ: RELAY_MINT_RESPONSE_TYP,
1566 payload: responsePayload,
1567 }).pipe(
1568 Effect.mapError(
1569 (cause) =>
1570 new EnvironmentAuth.ServerAuthCloudMintJwtSigningError({
1571 cause,
1572 }),
1573 ),
1574 );
1575 const response = {
1576 credential: issued.credential,
1577 expiresAt: DateTime.formatIso(issued.expiresAt),
1578 proof: responseProof,
1579 } satisfies RelayEnvironmentMintResponseShape;
1580
1581 yield* appendCloudCredentialResponseHeaders;
1582 return response;
1583 },
1584 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
1585 failEnvironmentCloudInternalError(error.message)(error),
1586 ),
1587 Effect.catchIf(
1588 ServerSecretStore.isSecretStoreError,
1589 failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
1590 ),
1591 Effect.catchTag(
1592 "PlatformError",
1593 failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
1594 ),
1595);
1596
159721export const layer = HttpApiBuilder.group(
159822 EnvironmentHttpApi,
159923 "connect",
160024 Effect.fnUntraced(function* (handlers) {
1601 const dependencies = yield* cloudHttpDependencies;
1602 const preferences = yield* CloudPreferences.CloudPreferences;
25 const cloudLink = yield* CloudLink.CloudLink;
160326 return handlers
1604 .handle("linkProof", ({ payload }) => cloudLinkProofHandler(dependencies, payload))
1605 .handle("relayConfig", ({ payload }) => cloudRelayConfigHandler(dependencies, payload))
1606 .handle("linkState", () => cloudLinkStateHandler(dependencies))
1607 .handle("unlink", () => cloudUnlinkHandler(dependencies))
1608 .handle(
1609 "preferences",
1610 Effect.fn("environment.cloud.preferences")(
1611 function* ({ payload }) {
1612 yield* requireEnvironmentScope(AuthRelayWriteScope);
1613 yield* preferences.update(payload);
1614 return yield* readCloudLinkState(dependencies);
1615 },
1616 Effect.catchIf(
1617 ServerSecretStore.isSecretStoreError,
1618 failEnvironmentCloudInternalError("Could not read environment cloud preferences."),
1619 ),
27 .handle("linkProof", ({ payload }) =>
28 Effect.gen(function* () {
29 yield* requireEnvironmentScope(AuthRelayWriteScope);
30 const request = yield* HttpServerRequest.HttpServerRequest;
31 const proof = yield* cloudLink.linkProof(payload, request);
32 yield* appendCloudCredentialResponseHeaders;
33 return proof;
34 }),
35 )
36 .handle("relayConfig", ({ payload }) =>
37 requireEnvironmentScope(AuthRelayWriteScope).pipe(
38 Effect.andThen(cloudLink.applyRelayConfig(payload)),
162039 ),
162140 )
1622 .handle("health", ({ payload }) => cloudEnvironmentHealthHandler(dependencies, payload))
1623 .handle("mintCredential", ({ payload }) => cloudMintCredentialHandler(dependencies, payload))
41 .handle("linkState", () =>
42 requireEnvironmentScope(AuthRelayReadScope).pipe(Effect.andThen(cloudLink.linkState())),
43 )
44 .handle("unlink", () =>
45 requireEnvironmentScope(AuthRelayWriteScope).pipe(Effect.andThen(cloudLink.unlink())),
46 )
47 .handle("preferences", ({ payload }) =>
48 requireEnvironmentScope(AuthRelayWriteScope).pipe(
49 Effect.andThen(cloudLink.updatePreferences(payload)),
50 ),
51 )
52 .handle("health", ({ payload }) =>
53 cloudLink
54 .answerHealthRequest(payload)
55 .pipe(Effect.tap(() => appendCloudCredentialResponseHeaders)),
56 )
57 .handle("mintCredential", ({ payload }) =>
58 cloudLink
59 .mintCredential(payload)
60 .pipe(Effect.tap(() => appendCloudCredentialResponseHeaders)),
61 )
162462 .handle("t3MintCredential", ({ payload }) =>
1625 traceRelayRequest(cloudMintCredentialHandler(dependencies, payload)),
63 traceRelayRequest(
64 cloudLink
65 .mintCredential(payload)
66 .pipe(Effect.tap(() => appendCloudCredentialResponseHeaders)),
67 ),
162668 );
162769 }),
162870);
apps/server/src/server.ts+48-51
......@@ -112,7 +112,6 @@ import * as ReplayMarkers from "./auth/replayMarkers.ts";
112112import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
113113import * as WebhookRoute from "./scheduledTasks/webhookRoute.ts";
114114import * as RelayDeliveryProof from "./scheduledTasks/RelayDeliveryProof.ts";
115import * as CloudPreferences from "./cloud/CloudPreferences.ts";
116115import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts";
117116import {
118117 relayHookBaseUrl,
......@@ -124,15 +123,8 @@ import {
124123 RELAY_URL_SECRET,
125124} from "./cloud/config.ts";
126125import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
127import {
128 pendingServiceUpdateExists,
129 reconcileDesiredCloudLinkIfStillDesired,
130 recoverManagedCloudTunnel,
131 registerManagedCloudTunnelRecovery,
132 startManagedCloudTunnelIfOriginConfirmed,
133 releaseManagedTunnelOnShutdown,
134} from "./cloud/http.ts";
135126import * as CloudHttp from "./cloud/http.ts";
127import * as CloudLink from "./cloud/CloudLink.ts";
136128import * as RelayTracing from "./cloud/relayTracing.ts";
137129import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";
138130import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
......@@ -658,7 +650,7 @@ const layerMakeRoutes = Layer.mergeAll(
658650 Layer.mergeAll(
659651 HttpApiBuilder.layer(EnvironmentHttpApi).pipe(
660652 Layer.provide(AuthHttp.layer),
661 Layer.provide(CloudHttp.layer.pipe(Layer.provide(CloudPreferences.layer))),
653 Layer.provide(CloudHttp.layer),
662654 Layer.provide(OrchestrationHttp.layer),
663655 Layer.provide(PullRequestHttp.layer),
664656 Layer.provide(ProjectHttp.layer),
......@@ -802,7 +794,8 @@ const layerMakeServer = Layer.unwrap(
802794 : Layer.empty;
803795 const layerCloudDesiredLinkReconcile = Layer.effectDiscard(
804796 Effect.gen(function* () {
805 const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(
797 const cloudLink = yield* CloudLink.CloudLink;
798 const releaseManagedTunnel = cloudLink.releaseManagedTunnelOnShutdown().pipe(
806799 Effect.timeout("10 seconds"),
807800 Effect.tap((released) =>
808801 released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void,
......@@ -821,7 +814,7 @@ const layerMakeServer = Layer.unwrap(
821814 // while the launcher's explicit-stop marker allows it to be released.
822815 // Other runtimes wait for activation so a failed standby cannot tear
823816 // down the active runtime's tunnel.
824 const cleanupBeforeActivation = yield* pendingServiceUpdateExists;
817 const cleanupBeforeActivation = yield* CloudLink.pendingServiceUpdateExists;
825818 if (cleanupBeforeActivation) {
826819 yield* Effect.addFinalizer(() => releaseManagedTunnel);
827820 }
......@@ -846,7 +839,7 @@ const layerMakeServer = Layer.unwrap(
846839 lastRecoveryAtMillis = yield* Clock.currentTimeMillis;
847840 }).pipe(
848841 Effect.andThen(
849 recoverManagedCloudTunnel(localOrigin, config, {
842 cloudLink.recoverManagedTunnel(localOrigin, config, {
850843 retryRuntimeFailures: true,
851844 }),
852845 ),
......@@ -909,30 +902,32 @@ const layerMakeServer = Layer.unwrap(
909902 const startedConfirmed =
910903 desiredCliLinkMode === "publish_only"
911904 ? false
912 : yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin).pipe(
905 : yield* cloudLink.startManagedTunnelIfOriginConfirmed(localOrigin).pipe(
913906 Effect.catch((cause) =>
914907 Effect.logWarning("Failed to start the confirmed T3 Connect tunnel", {
915908 cause,
916909 }).pipe(Effect.as(false)),
917910 ),
918911 );
919 const startStoredManagedTunnel = startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
920 requireConfirmedOrigin: false,
921 }).pipe(
922 Effect.tap((started) =>
923 started
924 ? Effect.logWarning(
925 "T3 Connect started the stored tunnel without relay confirmation",
926 )
927 : Effect.void,
928 ),
929 Effect.catch((cause) =>
930 Effect.logWarning("Failed to start the stored T3 Connect tunnel", { cause }),
931 ),
932 Effect.asVoid,
933 );
912 const startStoredManagedTunnel = cloudLink
913 .startManagedTunnelIfOriginConfirmed(localOrigin, {
914 requireConfirmedOrigin: false,
915 })
916 .pipe(
917 Effect.tap((started) =>
918 started
919 ? Effect.logWarning(
920 "T3 Connect started the stored tunnel without relay confirmation",
921 )
922 : Effect.void,
923 ),
924 Effect.catch((cause) =>
925 Effect.logWarning("Failed to start the stored T3 Connect tunnel", { cause }),
926 ),
927 Effect.asVoid,
928 );
934929 const registerManagedTunnel = retryManagedTunnelRegistration(
935 registerManagedCloudTunnelRecovery(localOrigin, {
930 cloudLink.registerManagedTunnelRecovery(localOrigin, {
936931 retryRuntimeFailures: true,
937932 }),
938933 (error) =>
......@@ -978,29 +973,29 @@ const layerMakeServer = Layer.unwrap(
978973 yield* endpointRuntime.requestRecovery(startupAction.config);
979974 }
980975 if (startupAction.action === "reconcile_link") {
981 const reconciledMode = yield* reconcileDesiredCloudLinkIfStillDesired(
982 localOrigin,
983 ).pipe(
984 Effect.retry({
985 while: shouldRetryCloudLink,
986 schedule: Schedule.exponential("1 second").pipe(
987 Schedule.modifyDelay(({ duration }) =>
988 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
976 const reconciledMode = yield* cloudLink
977 .reconcileDesiredLinkIfStillDesired(localOrigin)
978 .pipe(
979 Effect.retry({
980 while: shouldRetryCloudLink,
981 schedule: Schedule.exponential("1 second").pipe(
982 Schedule.modifyDelay(({ duration }) =>
983 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
984 ),
985 Schedule.upTo({ duration: "10 minutes" }),
989986 ),
990 Schedule.upTo({ duration: "10 minutes" }),
987 }),
988 Effect.tap((mode) =>
989 mode === null
990 ? Effect.void
991 : Effect.logInfo("T3 Connect desired link reconciled on startup"),
991992 ),
992 }),
993 Effect.tap((mode) =>
994 mode === null
995 ? Effect.void
996 : Effect.logInfo("T3 Connect desired link reconciled on startup"),
997 ),
998 Effect.catch((cause) =>
999 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
1000 cause,
1001 }).pipe(Effect.as(null)),
1002 ),
1003 );
993 Effect.catch((cause) =>
994 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
995 cause,
996 }).pipe(Effect.as(null)),
997 ),
998 );
1004999 if (reconciledMode === "managed") {
10051000 const afterReconcile = yield* registerManagedTunnel;
10061001 if (afterReconcile.status === "recovery_required") {
......@@ -1045,6 +1040,8 @@ const layerMakeServer = Layer.unwrap(
10451040 );
10461041
10471042 return layerServerApplication.pipe(
1043 // The connect routes and the startup/shutdown link work share one instance.
1044 Layer.provide(CloudLink.layer),
10481045 Layer.provideMerge(layerRuntimeServices),
10491046 Layer.provideMerge(
10501047 McpSessionRegistry.layer.pipe(
docs/internals/t3-connect.md+2-2
......@@ -48,7 +48,7 @@ mint responses also bind the credential to the client proof key. The relay
4848verifies those bindings before returning a credential. This prevents a different
4949process behind the tunnel from impersonating the linked environment. The checks
5050meet in the
51[environment cloud handlers](../../apps/server/src/cloud/http.ts) and
51[environment link service](../../apps/server/src/cloud/CloudLink.ts) and
5252[relay connector](../../infra/relay/src/environments/EnvironmentConnector.ts).
5353
5454The relay holds the signing authority for mint requests. DPoP protects an honest
......@@ -80,7 +80,7 @@ Two cases must retain the tunnel across shutdown. A link installed through a
8080client has no startup provisioning path and depends on its stored connector
8181token. An update handoff immediately starts a replacement server, and replacing
8282the tunnel would add routing propagation delay to every update. These exceptions
83belong to [shutdown handling](../../apps/server/src/cloud/http.ts).
83belong to [shutdown handling](../../apps/server/src/cloud/CloudLink.ts).
8484
8585Release and unlink claim the allocation generation before deleting external
8686resources. A delayed cleanup must not delete a tunnel reused by a concurrent