1import { AuthStandardClientScopes, EnvironmentId, type ServerConfig } from "@t3tools/contracts";
2import { describe, expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
4import * as Effect from "effect/Effect";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as Random from "effect/Random";
8import * as Ref from "effect/Ref";
9import * as Stream from "effect/Stream";
10import * as SubscriptionRef from "effect/SubscriptionRef";
11import * as TestClock from "effect/testing/TestClock";
12import * as Tracer from "effect/Tracer";
13
14import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
15import * as TokenStore from "../authorization/tokenStore.ts";
16import * as ClientCapabilities from "../platform/capabilities.ts";
17import * as ManagedRelay from "../relay/managedRelay.ts";
18import * as RpcHttp from "../rpc/http.ts";
19import type { WsRpcProtocolClient } from "../rpc/protocol.ts";
20import { fetchEnvironmentSessionState } from "../state/session.ts";
21import type { ConnectionCatalogEntry, ConnectionRoute } from "./catalog.ts";
22import * as Connectivity from "./connectivity.ts";
23import * as ConnectionDriver from "./driver.ts";
24import { BearerConnectionProfile } from "./catalog.ts";
25import {
26 BearerConnectionTarget,
27 ConnectionBlockedError,
28 ConnectionTransientError,
29 PrimaryConnectionTarget,
30 RelayConnectionTarget,
31 type ConnectionAttemptError,
32 type ConnectionTarget,
33 type NetworkStatus,
34 type PreparedConnection,
35 type SupervisorConnectionState,
36} from "./model.ts";
37import * as RpcSession from "../rpc/session.ts";
38import * as EnvironmentSupervisor from "./supervisor.ts";
39import * as ConnectionWakeups from "./wakeups.ts";
40import { connectionRouteId, entryWithRoutes, isLearned, mergeLearnedRoutes } from "./routes.ts";
41import { NETWORK_BLOCKING_HINT } from "../errors/network.ts";
42
43const TARGET = new PrimaryConnectionTarget({
44 environmentId: EnvironmentId.make("environment-1"),
45 label: "Test environment",
46 httpBaseUrl: "https://environment.example.test",
47 wsBaseUrl: "wss://environment.example.test",
48});
49
50const RELAY_TARGET = new RelayConnectionTarget({
51 environmentId: TARGET.environmentId,
52 label: TARGET.label,
53});
54
55const TARGET_ENTRY: ConnectionCatalogEntry = {
56 target: TARGET,
57 profile: Option.none(),
58 enabled: true,
59};
60
61const RELAY_ENTRY: ConnectionCatalogEntry = {
62 target: RELAY_TARGET,
63 profile: Option.none(),
64 enabled: true,
65};
66
67const PREPARED_CONNECTION: PreparedConnection = {
68 environmentId: TARGET.environmentId,
69 label: TARGET.label,
70 httpBaseUrl: TARGET.httpBaseUrl,
71 socketUrl: "wss://environment.example.test/ws",
72 httpAuthorization: null,
73 target: TARGET,
74};
75
76const TEST_RPC_CLIENT = {} as WsRpcProtocolClient;
77
78const LAN_TARGET = new BearerConnectionTarget({
79 environmentId: TARGET.environmentId,
80 label: TARGET.label,
81 connectionId: "bearer:lan",
82});
83const LAN_ROUTE: ConnectionRoute = {
84 target: LAN_TARGET,
85 profile: Option.some(
86 new BearerConnectionProfile({
87 connectionId: LAN_TARGET.connectionId,
88 environmentId: TARGET.environmentId,
89 label: TARGET.label,
90 httpBaseUrl: "http://192.168.1.10:3773/",
91 wsBaseUrl: "ws://192.168.1.10:3773/",
92 }),
93 ),
94};
95// LAN first, T3 Connect as the fallback.
96const LAN_THEN_RELAY_ENTRY: ConnectionCatalogEntry = {
97 target: LAN_ROUTE.target,
98 profile: LAN_ROUTE.profile,
99 alternateRoutes: [{ target: RELAY_TARGET, profile: Option.none() }],
100 enabled: true,
101};
102
103function preparedFor(target: ConnectionTarget): PreparedConnection {
104 return { ...PREPARED_CONNECTION, target };
105}
106
107function transient(message = "Connection failed.") {
108 return new ConnectionTransientError({
109 reason: "transport",
110 detail: message,
111 });
112}
113
114function blocked(message = "Authentication required.") {
115 return new ConnectionBlockedError({
116 reason: "authentication",
117 detail: message,
118 });
119}
120
121function awaitState(
122 state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>,
123 predicate: (value: SupervisorConnectionState) => boolean,
124) {
125 return SubscriptionRef.changes(state).pipe(
126 Stream.filter(predicate),
127 Stream.runHead,
128 Effect.map(Option.getOrThrow),
129 );
130}
131
132const eventuallyState = Effect.fn("TestConnectionHarness.eventuallyState")(function* (
133 state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>,
134 predicate: (value: SupervisorConnectionState) => boolean,
135) {
136 let lastState = yield* SubscriptionRef.get(state);
137 for (let iteration = 0; iteration < 100; iteration += 1) {
138 lastState = yield* SubscriptionRef.get(state);
139 if (predicate(lastState)) {
140 return lastState;
141 }
142 yield* Effect.yieldNow;
143 }
144 return yield* Effect.die(
145 new Error(
146 `Expected supervisor state was not observed. Last state: phase=${lastState.phase}, stage=${lastState.stage ?? "none"}, attempt=${lastState.attempt}, generation=${lastState.generation}`,
147 ),
148 );
149});
150
151const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: {
152 readonly networkStatus?: NetworkStatus;
153 readonly prepare?: (
154 attempt: number,
155 target: ConnectionTarget,
156 ) => Effect.Effect<PreparedConnection, ConnectionAttemptError>;
157 readonly ready?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
158 readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
159 readonly initialConfig?: (attempt: number) => Effect.Effect<ServerConfig, ConnectionAttemptError>;
160 readonly checkRoute?: (route: ConnectionRoute) => Effect.Effect<ConnectionDriver.RouteCheck>;
161}) {
162 const networkStatus = yield* SubscriptionRef.make<NetworkStatus>(
163 options?.networkStatus ?? "online",
164 );
165 const prepareCount = yield* Ref.make(0);
166 const sessionCount = yield* Ref.make(0);
167 const releaseCount = yield* Ref.make(0);
168 const wakeups = yield* SubscriptionRef.make<{
169 readonly sequence: number;
170 readonly reason: ConnectionWakeups.ConnectionWakeup;
171 }>({
172 sequence: 0,
173 reason: "application-active",
174 });
175 const closedSessions = yield* Ref.make<
176 ReadonlyArray<Deferred.Deferred<never, ConnectionTransientError>>
177 >([]);
178
179 const connectivity = Connectivity.Connectivity.of({
180 status: SubscriptionRef.get(networkStatus),
181 changes: SubscriptionRef.changes(networkStatus),
182 });
183
184 const prepare = Effect.fn("TestConnectionDriver.prepare")(function* (target: ConnectionTarget) {
185 const attempt = yield* Ref.updateAndGet(prepareCount, (count) => count + 1);
186 if (options?.prepare) {
187 return yield* options.prepare(attempt, target);
188 }
189 return PREPARED_CONNECTION;
190 });
191
192 const checkRoute = (route: ConnectionRoute) =>
193 options?.checkRoute?.(route) ?? Effect.succeed<ConnectionDriver.RouteCheck>("unchecked");
194
195 const connectRoute = Effect.fn("TestConnectionDriver.connectRoute")(function* (
196 target: ConnectionTarget,
197 reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>,
198 ) {
199 const prepared = yield* prepare(target);
200 yield* reportProgress({ stage: "opening", prepared });
201
202 const attempt = yield* Ref.updateAndGet(sessionCount, (count) => count + 1);
203 const closed = yield* Deferred.make<never, ConnectionTransientError>();
204 yield* Ref.update(closedSessions, (sessions) => [...sessions, closed]);
205
206 const session = yield* Effect.acquireRelease(
207 Effect.succeed({
208 client: TEST_RPC_CLIENT,
209 initialConfig:
210 options?.initialConfig?.(attempt) ??
211 Effect.die(new Error("Initial config is not used by supervisor tests.")),
212 subscribeServerConfig: (input) => TEST_RPC_CLIENT.subscribeServerConfig(input),
213 ready: options?.ready?.(attempt) ?? Effect.void,
214 probe: options?.probe?.(attempt) ?? Effect.void,
215 closed: Deferred.await(closed),
216 } satisfies RpcSession.RpcSession),
217 () => Ref.update(releaseCount, (count) => count + 1),
218 );
219
220 yield* reportProgress({ stage: "synchronizing", prepared });
221 yield* session.ready;
222 return { prepared, session } satisfies ConnectionDriver.EnvironmentConnectionLease;
223 });
224
225 const connect = Effect.fn("TestConnectionDriver.connect")(function* (
226 entry: ConnectionCatalogEntry,
227 reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>,
228 ) {
229 yield* reportProgress({ stage: "preparing" });
230 return yield* ConnectionDriver.connectOverRoutes(entry, checkRoute, (route) =>
231 connectRoute(route.target, reportProgress),
232 );
233 });
234
235 const layerDependencies = Layer.mergeAll(
236 // Jitter at its maximum, so each retry waits exactly its ceiling: 2s, 4s, 8s...
237 Layer.succeed(Random.Random, {
238 nextDoubleUnsafe: () => 1 - Number.EPSILON,
239 nextIntUnsafe: () => 0,
240 }),
241 Layer.succeed(Connectivity.Connectivity, connectivity),
242 Layer.succeed(
243 ConnectionWakeups.ConnectionWakeups,
244 ConnectionWakeups.ConnectionWakeups.of({
245 changes: SubscriptionRef.changes(wakeups).pipe(
246 Stream.drop(1),
247 Stream.map((event) => event.reason),
248 ),
249 }),
250 ),
251 Layer.succeed(
252 ConnectionDriver.ConnectionDriver,
253 ConnectionDriver.ConnectionDriver.of({
254 connect,
255 checkRoute: (_entry, route) => checkRoute(route),
256 preflight: (_entry, route) =>
257 checkRoute(route).pipe(Effect.map((check) => check === "answered")),
258 }),
259 ),
260 );
261
262 return {
263 dependencies: layerDependencies,
264 prepareCount,
265 sessionCount,
266 releaseCount,
267 setNetworkStatus: (status: NetworkStatus) => SubscriptionRef.set(networkStatus, status),
268 wake: (reason: ConnectionWakeups.ConnectionWakeup) =>
269 SubscriptionRef.update(wakeups, (event) => ({
270 sequence: event.sequence + 1,
271 reason,
272 })),
273 closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* (
274 error = transient("Session closed."),
275 ) {
276 const sessions = yield* Ref.get(closedSessions);
277 const latest = sessions.at(-1);
278 if (latest) {
279 yield* Deferred.fail(latest, error);
280 }
281 }),
282 };
283});
284
285describe("retryDelayMs", () => {
286 it("doubles from 2 seconds to a 5 minute cap, jittered within the upper half of each step", () => {
287 const ceilings = [2_000, 4_000, 8_000, 16_000, 32_000, 64_000, 128_000, 256_000, 300_000];
288 for (const [failureCount, ceiling] of [...ceilings, 300_000].entries()) {
289 expect(EnvironmentSupervisor.retryDelayMs(failureCount, 0)).toBe(ceiling / 2);
290 expect(EnvironmentSupervisor.retryDelayMs(failureCount, 0.5)).toBe((ceiling * 3) / 4);
291 expect(EnvironmentSupervisor.retryDelayMs(failureCount, 1 - Number.EPSILON)).toBe(ceiling);
292 }
293 });
294});
295
296describe("EnvironmentSupervisor", () => {
297 it.effect("records each relay setup as a local linked trace that ends at readiness", () =>
298 Effect.gen(function* () {
299 const spans: Array<Tracer.NativeSpan> = [];
300 const tracer = Tracer.make({
301 span: (options) => {
302 const span = new Tracer.NativeSpan(options);
303 const end = span.end.bind(span);
304 span.end = (endTime, exit) => {
305 end(endTime, exit);
306 spans.push(span);
307 };
308 return span;
309 },
310 });
311 const harness = yield* makeHarness({
312 prepare: (attempt) =>
313 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
314 });
315 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
316 initiallyDesired: true,
317 }).pipe(Effect.provide(harness.dependencies), Effect.withTracer(tracer));
318
319 yield* awaitState(
320 supervisor.state,
321 (state) => state.phase === "backoff" && state.attempt === 1,
322 );
323 const firstAttempt = spans.find((span) => span.name === "relay.connection.attempt");
324 expect(firstAttempt).toBeDefined();
325
326 yield* TestClock.adjust("3 seconds");
327 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
328
329 const attempts = spans.filter((span) => span.name === "relay.connection.attempt");
330 expect(attempts).toHaveLength(2);
331 expect(attempts[0]?.traceId).not.toBe(attempts[1]?.traceId);
332 expect(attempts[1]?.links.map((link) => link.span.spanId)).toContain(attempts[0]?.spanId);
333 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
334 }).pipe(Effect.provide(TestClock.layer())),
335 );
336
337 it.effect("does not attempt a connection until it is desired", () =>
338 Effect.gen(function* () {
339 const harness = yield* makeHarness();
340 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe(
341 Effect.provide(harness.dependencies),
342 );
343
344 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("available");
345 expect(yield* Ref.get(harness.prepareCount)).toBe(0);
346 }),
347 );
348
349 it.effect("does not let the initial connect signal cancel the first attempt", () =>
350 Effect.gen(function* () {
351 const harness = yield* makeHarness();
352 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe(
353 Effect.provide(harness.dependencies),
354 );
355
356 yield* supervisor.connect;
357 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
358
359 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
360 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
361 }),
362 );
363
364 it.effect("waits while offline and connects immediately when the network returns", () =>
365 Effect.gen(function* () {
366 const harness = yield* makeHarness({ networkStatus: "offline" });
367 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
368 initiallyDesired: true,
369 }).pipe(Effect.provide(harness.dependencies));
370
371 yield* awaitState(supervisor.state, (state) => state.phase === "offline");
372 expect(yield* Ref.get(harness.prepareCount)).toBe(0);
373
374 yield* harness.setNetworkStatus("online");
375 const ready = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
376
377 expect(ready).toMatchObject({
378 desired: true,
379 network: "online",
380 phase: "connected",
381 attempt: 1,
382 generation: 1,
383 lastFailure: null,
384 });
385 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
386 }),
387 );
388
389 it.effect("resets retries when activation arrives before the network returns", () =>
390 Effect.gen(function* () {
391 const harness = yield* makeHarness();
392 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
393 initiallyDesired: true,
394 }).pipe(Effect.provide(harness.dependencies));
395
396 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
397 yield* harness.closeLatestSession();
398 yield* awaitState(
399 supervisor.state,
400 (state) => state.phase === "backoff" && state.attempt === 1,
401 );
402 yield* harness.setNetworkStatus("offline");
403 yield* awaitState(
404 supervisor.state,
405 (state) => state.phase === "offline" && state.attempt === 2,
406 );
407
408 yield* harness.wake("application-active-reconnect");
409 yield* awaitState(
410 supervisor.state,
411 (state) => state.phase === "offline" && state.attempt === 1,
412 );
413 yield* harness.setNetworkStatus("online");
414 yield* awaitState(
415 supervisor.state,
416 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
417 );
418 }),
419 );
420
421 it.effect("retries forever with exponential backoff capped at five minutes", () =>
422 Effect.gen(function* () {
423 const harness = yield* makeHarness({
424 prepare: () => Effect.fail(transient()),
425 });
426 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
427 initiallyDesired: true,
428 }).pipe(Effect.provide(harness.dependencies));
429
430 yield* awaitState(
431 supervisor.state,
432 (state) => state.phase === "backoff" && state.attempt === 1,
433 );
434 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
435
436 const delays = [
437 2_000, 4_000, 8_000, 16_000, 32_000, 64_000, 128_000, 256_000, 300_000, 300_000,
438 ];
439 for (const [index, delay] of delays.entries()) {
440 yield* TestClock.adjust(delay - 1);
441 expect(yield* Ref.get(harness.prepareCount)).toBe(index + 1);
442 yield* TestClock.adjust(1);
443 yield* eventuallyState(
444 supervisor.state,
445 (state) => state.phase === "backoff" && state.attempt === index + 2,
446 );
447 }
448
449 expect(yield* Ref.get(harness.prepareCount)).toBe(delays.length + 1);
450 }).pipe(Effect.provide(TestClock.layer())),
451 );
452
453 it.effect("keeps the latest failure visible throughout the next connection attempt", () =>
454 Effect.gen(function* () {
455 const harness = yield* makeHarness({
456 prepare: (attempt) =>
457 attempt === 1 ? Effect.fail(transient("Relay connection timed out.")) : Effect.never,
458 });
459 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
460 initiallyDesired: true,
461 }).pipe(Effect.provide(harness.dependencies));
462
463 yield* awaitState(
464 supervisor.state,
465 (state) => state.phase === "backoff" && state.attempt === 1,
466 );
467 yield* TestClock.adjust("3 seconds");
468
469 const retrying = yield* awaitState(
470 supervisor.state,
471 (state) =>
472 state.phase === "connecting" && state.stage === "preparing" && state.attempt === 2,
473 );
474 expect(retrying).toMatchObject({
475 phase: "connecting",
476 stage: "preparing",
477 attempt: 2,
478 lastFailure: {
479 _tag: "ConnectionTransientError",
480 reason: "transport",
481 message: "Relay connection timed out.",
482 },
483 });
484 }).pipe(Effect.provide(TestClock.layer())),
485 );
486
487 it.effect("keeps a working socket open while a slow server config snapshot arrives", () =>
488 Effect.gen(function* () {
489 const harness = yield* makeHarness({
490 ready: () => Effect.sleep("40 seconds"),
491 });
492 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
493 initiallyDesired: true,
494 }).pipe(Effect.provide(harness.dependencies));
495
496 yield* awaitState(
497 supervisor.state,
498 (state) => state.phase === "connecting" && state.stage === "synchronizing",
499 );
500 yield* TestClock.adjust("39 seconds");
501 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connecting");
502
503 yield* TestClock.adjust("1 second");
504 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
505 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
506 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
507 }).pipe(Effect.provide(TestClock.layer())),
508 );
509
510 it.effect("retries when a session never becomes ready", () =>
511 Effect.gen(function* () {
512 const harness = yield* makeHarness({
513 ready: () => Effect.never,
514 });
515 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
516 initiallyDesired: true,
517 }).pipe(Effect.provide(harness.dependencies));
518
519 yield* awaitState(
520 supervisor.state,
521 (state) => state.phase === "connecting" && state.stage === "synchronizing",
522 );
523 // 15 seconds of setup plus 75 for the snapshot; setup finished instantly.
524 yield* TestClock.adjust("89 seconds");
525 expect((yield* SubscriptionRef.get(supervisor.state)).stage).toBe("synchronizing");
526
527 yield* TestClock.adjust("1 second");
528 const retrying = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
529
530 expect(retrying).toMatchObject({
531 phase: "backoff",
532 lastFailure: {
533 _tag: "ConnectionTransientError",
534 reason: "timeout",
535 message: "Test environment did not respond during connection setup.",
536 },
537 });
538 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
539 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
540 }).pipe(Effect.provide(TestClock.layer())),
541 );
542
543 it.effect("interrupts and releases a connection attempt when setup times out", () =>
544 Effect.gen(function* () {
545 const harness = yield* makeHarness({
546 prepare: () => Effect.never,
547 });
548 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
549 initiallyDesired: true,
550 }).pipe(Effect.provide(harness.dependencies));
551
552 yield* awaitState(
553 supervisor.state,
554 (state) => state.phase === "connecting" && state.stage === "preparing",
555 );
556 yield* TestClock.adjust("15 seconds");
557 const retrying = yield* eventuallyState(
558 supervisor.state,
559 (state) => state.phase === "backoff" && state.attempt === 1,
560 );
561
562 expect(retrying).toMatchObject({
563 lastFailure: {
564 _tag: "ConnectionTransientError",
565 reason: "timeout",
566 message: "Test environment did not respond during connection setup.",
567 },
568 });
569 }).pipe(Effect.provide(TestClock.layer())),
570 );
571
572 it.effect(
573 "shows a network hint for a stalled relay connection and clears it after recovery",
574 () =>
575 Effect.gen(function* () {
576 const harness = yield* makeHarness({
577 prepare: (attempt) =>
578 attempt === 1 ? Effect.never : Effect.succeed(PREPARED_CONNECTION),
579 });
580 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
581 initiallyDesired: true,
582 }).pipe(Effect.provide(harness.dependencies));
583
584 yield* awaitState(supervisor.state, (state) => state.phase === "connecting");
585 yield* TestClock.adjust("15 seconds");
586 const failed = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
587 expect(failed.lastFailure?.message).toBe(
588 `Test environment did not respond during connection setup. ${NETWORK_BLOCKING_HINT}`,
589 );
590
591 yield* TestClock.adjust("3 seconds");
592 const recovered = yield* awaitState(
593 supervisor.state,
594 (state) => state.phase === "connected",
595 );
596 expect(recovered.lastFailure).toBeNull();
597 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
598 }).pipe(Effect.provide(TestClock.layer())),
599 );
600
601 it.effect("converts unexpected driver defects into retryable failures", () =>
602 Effect.gen(function* () {
603 const harness = yield* makeHarness({
604 prepare: (attempt) =>
605 attempt === 1
606 ? Effect.die(new Error("Native transport defect."))
607 : Effect.succeed(PREPARED_CONNECTION),
608 });
609 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
610 initiallyDesired: true,
611 }).pipe(Effect.provide(harness.dependencies));
612
613 const failed = yield* awaitState(
614 supervisor.state,
615 (state) => state.phase === "backoff" && state.attempt === 1,
616 );
617 expect(failed).toMatchObject({
618 lastFailure: {
619 _tag: "ConnectionTransientError",
620 reason: "transport",
621 message: "Test environment connection failed unexpectedly.",
622 },
623 });
624
625 yield* TestClock.adjust("3 seconds");
626 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
627 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
628 }).pipe(Effect.provide(TestClock.layer())),
629 );
630
631 it.effect("explicit retry interrupts the current backoff", () =>
632 Effect.gen(function* () {
633 const harness = yield* makeHarness({
634 prepare: (attempt) =>
635 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
636 });
637 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
638 initiallyDesired: true,
639 }).pipe(Effect.provide(harness.dependencies));
640
641 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
642 yield* supervisor.retryNow;
643 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
644
645 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
646 }),
647 );
648
649 it.effect("explicit retry starts a fresh backoff sequence", () =>
650 Effect.gen(function* () {
651 const harness = yield* makeHarness({
652 prepare: () => Effect.fail(transient()),
653 });
654 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
655 initiallyDesired: true,
656 }).pipe(Effect.provide(harness.dependencies));
657
658 yield* awaitState(
659 supervisor.state,
660 (state) => state.phase === "backoff" && state.attempt === 1,
661 );
662 yield* TestClock.adjust("3 seconds");
663 yield* eventuallyState(
664 supervisor.state,
665 (state) => state.phase === "backoff" && state.attempt === 2,
666 );
667
668 yield* supervisor.retryNow;
669 yield* eventuallyState(
670 supervisor.state,
671 (state) => state.phase === "backoff" && state.attempt === 1,
672 );
673 expect(yield* Ref.get(harness.prepareCount)).toBe(3);
674
675 yield* TestClock.adjust("1999 millis");
676 expect(yield* Ref.get(harness.prepareCount)).toBe(3);
677 yield* TestClock.adjust("1 milli");
678 yield* eventuallyState(
679 supervisor.state,
680 (state) => state.phase === "backoff" && state.attempt === 2,
681 );
682 expect(yield* Ref.get(harness.prepareCount)).toBe(4);
683 }).pipe(Effect.provide(TestClock.layer())),
684 );
685
686 it.effect("keeps blocked failures idle until an external signal requests another attempt", () =>
687 Effect.gen(function* () {
688 const harness = yield* makeHarness({
689 prepare: (attempt) =>
690 attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION),
691 });
692 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
693 initiallyDesired: true,
694 }).pipe(Effect.provide(harness.dependencies));
695
696 yield* awaitState(supervisor.state, (state) => state.phase === "blocked");
697 yield* TestClock.adjust("1 hour");
698 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
699
700 yield* supervisor.retryNow;
701 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
702 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
703 }).pipe(Effect.provide(TestClock.layer())),
704 );
705
706 it.effect("resets retries when activation wakes a blocked connection", () =>
707 Effect.gen(function* () {
708 const harness = yield* makeHarness({
709 prepare: (attempt) =>
710 attempt === 1
711 ? Effect.fail(transient())
712 : attempt === 2
713 ? Effect.fail(blocked())
714 : Effect.succeed(PREPARED_CONNECTION),
715 });
716 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
717 initiallyDesired: true,
718 }).pipe(Effect.provide(harness.dependencies));
719
720 yield* awaitState(
721 supervisor.state,
722 (state) => state.phase === "backoff" && state.attempt === 1,
723 );
724 yield* TestClock.adjust("3 seconds");
725 yield* awaitState(
726 supervisor.state,
727 (state) => state.phase === "blocked" && state.attempt === 2,
728 );
729
730 yield* harness.wake("application-active-reconnect");
731 yield* awaitState(
732 supervisor.state,
733 (state) => state.phase === "connected" && state.attempt === 1,
734 );
735 }).pipe(Effect.provide(TestClock.layer())),
736 );
737
738 it.effect("keeps a session that still answers when the network reports offline", () =>
739 Effect.gen(function* () {
740 const probeCount = yield* Ref.make(0);
741 const harness = yield* makeHarness({
742 probe: () => Ref.update(probeCount, (count) => count + 1),
743 });
744 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
745 initiallyDesired: true,
746 }).pipe(Effect.provide(harness.dependencies));
747
748 yield* awaitState(
749 supervisor.state,
750 (state) => state.phase === "connected" && state.generation === 1,
751 );
752 // A loopback server, or a flap shorter than the probe, keeps working.
753 yield* harness.setNetworkStatus("offline");
754 for (let attempt = 0; attempt < 100; attempt += 1) {
755 if ((yield* Ref.get(probeCount)) > 0) break;
756 yield* Effect.yieldNow;
757 }
758 yield* harness.setNetworkStatus("online");
759 yield* Effect.yieldNow;
760
761 expect(yield* Ref.get(probeCount)).toBe(1);
762 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
763 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
764 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
765 phase: "connected",
766 generation: 1,
767 });
768 }),
769 );
770
771 it.effect("replaces the session on a long resume while the network reports offline", () =>
772 Effect.gen(function* () {
773 const probeCount = yield* Ref.make(0);
774 const harness = yield* makeHarness({
775 probe: () => Ref.update(probeCount, (count) => count + 1),
776 });
777 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
778 initiallyDesired: true,
779 }).pipe(Effect.provide(harness.dependencies));
780
781 yield* awaitState(
782 supervisor.state,
783 (state) => state.phase === "connected" && state.generation === 1,
784 );
785 // A wrong offline report: the probe answers, so the session stays.
786 yield* harness.setNetworkStatus("offline");
787 for (let attempt = 0; attempt < 100; attempt += 1) {
788 if ((yield* Ref.get(probeCount)) > 0) break;
789 yield* Effect.yieldNow;
790 }
791 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
792
793 // The replacement connects although the network still reports offline.
794 yield* harness.wake("application-active-reconnect");
795 const replaced = yield* awaitState(
796 supervisor.state,
797 (state) => state.phase === "connected" && state.generation === 2,
798 );
799
800 expect(replaced.attempt).toBe(1);
801 expect(yield* Ref.get(probeCount)).toBe(1);
802 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
803 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
804 }),
805 );
806
807 it.effect(
808 "releases a session that stops answering while offline and reconnects when online",
809 () =>
810 Effect.gen(function* () {
811 const harness = yield* makeHarness({
812 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
813 });
814 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
815 initiallyDesired: true,
816 }).pipe(Effect.provide(harness.dependencies));
817
818 yield* awaitState(
819 supervisor.state,
820 (state) => state.phase === "connected" && state.generation === 1,
821 );
822 yield* harness.setNetworkStatus("offline");
823 yield* TestClock.adjust("3 seconds");
824 yield* awaitState(supervisor.state, (state) => state.phase === "offline");
825
826 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
827 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
828
829 yield* harness.setNetworkStatus("online");
830 yield* awaitState(
831 supervisor.state,
832 (state) => state.phase === "connected" && state.generation === 2,
833 );
834 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
835 }).pipe(Effect.provide(TestClock.layer())),
836 );
837
838 it.effect("probes instead of replacing a healthy session on an explicit retry", () =>
839 Effect.gen(function* () {
840 const probeCount = yield* Ref.make(0);
841 const harness = yield* makeHarness({
842 probe: () => Ref.update(probeCount, (count) => count + 1),
843 });
844 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
845 initiallyDesired: true,
846 }).pipe(Effect.provide(harness.dependencies));
847
848 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
849 yield* supervisor.retryNow;
850 for (let attempt = 0; attempt < 100; attempt += 1) {
851 if ((yield* Ref.get(probeCount)) > 0) break;
852 yield* Effect.yieldNow;
853 }
854
855 expect(yield* Ref.get(probeCount)).toBe(1);
856 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
857 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
858 }),
859 );
860
861 it.effect("keeps the backoff ladder after an explicit retry finds a healthy session", () =>
862 Effect.gen(function* () {
863 const probeCount = yield* Ref.make(0);
864 const harness = yield* makeHarness({
865 probe: () => Ref.update(probeCount, (count) => count + 1),
866 });
867 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
868 initiallyDesired: true,
869 }).pipe(Effect.provide(harness.dependencies));
870
871 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
872 yield* harness.closeLatestSession();
873 yield* awaitState(
874 supervisor.state,
875 (state) => state.phase === "backoff" && state.attempt === 1,
876 );
877 yield* TestClock.adjust("2 seconds");
878 yield* awaitState(
879 supervisor.state,
880 (state) => state.phase === "connected" && state.generation === 2,
881 );
882
883 yield* supervisor.retryNow;
884 for (let attempt = 0; attempt < 100; attempt += 1) {
885 if ((yield* Ref.get(probeCount)) > 0) break;
886 yield* Effect.yieldNow;
887 }
888 expect(yield* Ref.get(probeCount)).toBe(1);
889
890 // The flapping session keeps climbing the ladder: the answered retry does
891 // not reset it after this unrelated close.
892 yield* harness.closeLatestSession();
893 yield* awaitState(
894 supervisor.state,
895 (state) => state.phase === "backoff" && state.attempt === 2,
896 );
897 yield* TestClock.adjust("4 seconds");
898 const reconnected = yield* awaitState(
899 supervisor.state,
900 (state) => state.phase === "connected" && state.generation === 3,
901 );
902 expect(reconnected.attempt).toBe(3);
903 }).pipe(Effect.provide(TestClock.layer())),
904 );
905
906 it.effect("retries a blocked connection when platform credentials change", () =>
907 Effect.gen(function* () {
908 const harness = yield* makeHarness({
909 prepare: (attempt) =>
910 attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION),
911 });
912 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
913 initiallyDesired: true,
914 }).pipe(Effect.provide(harness.dependencies));
915
916 yield* awaitState(supervisor.state, (state) => state.phase === "blocked");
917 yield* harness.wake("credentials-changed");
918 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
919
920 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
921 }),
922 );
923
924 it.effect("does not let platform wakeups reset an in-flight attempt", () =>
925 Effect.gen(function* () {
926 const firstAttemptStarted = yield* Deferred.make<void>();
927 const harness = yield* makeHarness({
928 prepare: () =>
929 Deferred.succeed(firstAttemptStarted, undefined).pipe(Effect.andThen(Effect.never)),
930 });
931 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
932 initiallyDesired: true,
933 }).pipe(Effect.provide(harness.dependencies));
934
935 yield* Deferred.await(firstAttemptStarted);
936 yield* Effect.all(
937 [
938 harness.wake("credentials-changed"),
939 harness.wake("application-active"),
940 harness.wake("credentials-changed"),
941 ],
942 { concurrency: "unbounded" },
943 );
944 yield* Effect.yieldNow;
945
946 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
947
948 yield* TestClock.adjust("15 seconds");
949 const retrying = yield* eventuallyState(
950 supervisor.state,
951 (state) => state.phase === "backoff" && state.attempt === 1,
952 );
953
954 expect(retrying).toMatchObject({
955 lastFailure: {
956 _tag: "ConnectionTransientError",
957 reason: "timeout",
958 message: "Test environment did not respond during connection setup.",
959 },
960 });
961 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
962 expect(yield* Ref.get(harness.sessionCount)).toBe(0);
963 }).pipe(Effect.provide(TestClock.layer())),
964 );
965
966 it.effect("treats an involuntary session close as transient and reconnects", () =>
967 Effect.gen(function* () {
968 const harness = yield* makeHarness();
969 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
970 initiallyDesired: true,
971 }).pipe(Effect.provide(harness.dependencies));
972
973 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
974 yield* harness.closeLatestSession();
975 yield* awaitState(
976 supervisor.state,
977 (state) => state.phase === "backoff" && state.attempt === 1,
978 );
979 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
980
981 yield* TestClock.adjust("3 seconds");
982 yield* awaitState(
983 supervisor.state,
984 (state) => state.phase === "connected" && state.generation === 2,
985 );
986
987 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
988 expect(Option.isSome(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
989 }).pipe(Effect.provide(TestClock.layer())),
990 );
991
992 it.effect("keeps escalating backoff when a newly opened session flaps", () =>
993 Effect.gen(function* () {
994 const harness = yield* makeHarness();
995 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
996 initiallyDesired: true,
997 }).pipe(Effect.provide(harness.dependencies));
998
999 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1000 yield* harness.closeLatestSession();
1001 yield* awaitState(
1002 supervisor.state,
1003 (state) => state.phase === "backoff" && state.attempt === 1,
1004 );
1005
1006 yield* TestClock.adjust("3 seconds");
1007 yield* awaitState(
1008 supervisor.state,
1009 (state) => state.phase === "connected" && state.generation === 2,
1010 );
1011 yield* harness.closeLatestSession();
1012 const secondFailure = yield* awaitState(
1013 supervisor.state,
1014 (state) => state.phase === "backoff" && state.attempt === 2,
1015 );
1016
1017 expect(secondFailure.retryAt).not.toBeNull();
1018
1019 yield* TestClock.adjust("3 seconds");
1020 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1021
1022 yield* TestClock.adjust("1 second");
1023 yield* awaitState(
1024 supervisor.state,
1025 (state) => state.phase === "connected" && state.generation === 3,
1026 );
1027 expect(yield* Ref.get(harness.sessionCount)).toBe(3);
1028 }).pipe(Effect.provide(TestClock.layer())),
1029 );
1030
1031 it.effect("restarts the retry ladder when mobile returns to the foreground", () =>
1032 Effect.gen(function* () {
1033 const harness = yield* makeHarness();
1034 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1035 initiallyDesired: true,
1036 }).pipe(Effect.provide(harness.dependencies));
1037
1038 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1039 yield* harness.closeLatestSession();
1040 yield* awaitState(
1041 supervisor.state,
1042 (state) => state.phase === "backoff" && state.attempt === 1,
1043 );
1044 yield* TestClock.adjust("3 seconds");
1045 yield* awaitState(
1046 supervisor.state,
1047 (state) => state.phase === "connected" && state.generation === 2,
1048 );
1049 yield* harness.closeLatestSession();
1050 yield* awaitState(
1051 supervisor.state,
1052 (state) => state.phase === "backoff" && state.attempt === 2,
1053 );
1054
1055 yield* harness.wake("application-active-reconnect");
1056 yield* awaitState(
1057 supervisor.state,
1058 (state) => state.phase === "connected" && state.generation === 3 && state.attempt === 1,
1059 );
1060 yield* harness.closeLatestSession();
1061 yield* awaitState(
1062 supervisor.state,
1063 (state) => state.phase === "backoff" && state.attempt === 1,
1064 );
1065
1066 expect(yield* Ref.get(harness.sessionCount)).toBe(3);
1067 }).pipe(Effect.provide(TestClock.layer())),
1068 );
1069
1070 it.effect("restarts the retry ladder when a long resume replaces a connected session", () =>
1071 Effect.gen(function* () {
1072 const harness = yield* makeHarness();
1073 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1074 initiallyDesired: true,
1075 }).pipe(Effect.provide(harness.dependencies));
1076
1077 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1078 yield* harness.closeLatestSession();
1079 yield* awaitState(
1080 supervisor.state,
1081 (state) => state.phase === "backoff" && state.attempt === 1,
1082 );
1083 yield* TestClock.adjust("2 seconds");
1084 yield* awaitState(
1085 supervisor.state,
1086 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 2,
1087 );
1088
1089 yield* harness.wake("application-active-reconnect");
1090 yield* awaitState(
1091 supervisor.state,
1092 (state) => state.phase === "connected" && state.generation === 3 && state.attempt === 1,
1093 );
1094 }).pipe(Effect.provide(TestClock.layer())),
1095 );
1096
1097 it.effect("restarts the retry ladder when a long resume interrupts connection setup", () =>
1098 Effect.gen(function* () {
1099 const harness = yield* makeHarness({
1100 prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)),
1101 });
1102 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1103 initiallyDesired: true,
1104 }).pipe(Effect.provide(harness.dependencies));
1105
1106 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1107 yield* harness.closeLatestSession();
1108 yield* awaitState(
1109 supervisor.state,
1110 (state) => state.phase === "backoff" && state.attempt === 1,
1111 );
1112 yield* TestClock.adjust("3 seconds");
1113 yield* awaitState(
1114 supervisor.state,
1115 (state) => state.phase === "connecting" && state.attempt === 2,
1116 );
1117
1118 yield* harness.wake("application-active-reconnect");
1119 yield* awaitState(
1120 supervisor.state,
1121 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1122 );
1123 }).pipe(Effect.provide(TestClock.layer())),
1124 );
1125
1126 it.effect("probes the active session without reconnecting on application activation", () =>
1127 Effect.gen(function* () {
1128 const probeCount = yield* Ref.make(0);
1129 const probeCalled = yield* Deferred.make<void>();
1130 const harness = yield* makeHarness({
1131 probe: () =>
1132 Ref.update(probeCount, (count) => count + 1).pipe(
1133 Effect.andThen(Deferred.succeed(probeCalled, undefined)),
1134 ),
1135 });
1136 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1137 initiallyDesired: true,
1138 }).pipe(Effect.provide(harness.dependencies));
1139
1140 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1141 yield* harness.wake("application-active");
1142 yield* Deferred.await(probeCalled);
1143
1144 expect(yield* Ref.get(probeCount)).toBe(1);
1145 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1146 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1147 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
1148 }),
1149 );
1150
1151 it.effect("immediately replaces a mobile session after a long background resume", () =>
1152 Effect.gen(function* () {
1153 const probeCount = yield* Ref.make(0);
1154 const harness = yield* makeHarness({
1155 probe: () => Ref.update(probeCount, (count) => count + 1),
1156 });
1157 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1158 initiallyDesired: true,
1159 }).pipe(Effect.provide(harness.dependencies));
1160
1161 yield* awaitState(
1162 supervisor.state,
1163 (state) => state.phase === "connected" && state.generation === 1,
1164 );
1165 yield* harness.wake("application-active-reconnect");
1166 yield* awaitState(
1167 supervisor.state,
1168 (state) => state.phase === "connected" && state.generation === 2,
1169 );
1170
1171 expect(yield* Ref.get(probeCount)).toBe(0);
1172 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1173 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1174 }),
1175 );
1176
1177 it.effect("replaces a mobile session when a long resume interrupts an active probe", () =>
1178 Effect.gen(function* () {
1179 const harness = yield* makeHarness({
1180 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1181 });
1182 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1183 initiallyDesired: true,
1184 }).pipe(Effect.provide(harness.dependencies));
1185
1186 yield* awaitState(
1187 supervisor.state,
1188 (state) => state.phase === "connected" && state.generation === 1,
1189 );
1190 yield* harness.wake("application-active-probe");
1191 yield* Effect.yieldNow;
1192 yield* harness.wake("application-active-reconnect");
1193 yield* awaitState(
1194 supervisor.state,
1195 (state) => state.phase === "connected" && state.generation === 2,
1196 );
1197
1198 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1199 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1200 }),
1201 );
1202
1203 it.effect("reconnects immediately when the session closes during a resume probe", () =>
1204 Effect.gen(function* () {
1205 const probeStarted = yield* Deferred.make<void>();
1206 const harness = yield* makeHarness({
1207 probe: (attempt) =>
1208 attempt === 1
1209 ? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never))
1210 : Effect.void,
1211 });
1212 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1213 initiallyDesired: true,
1214 }).pipe(Effect.provide(harness.dependencies));
1215
1216 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1217 yield* harness.wake("application-active-probe");
1218 yield* Deferred.await(probeStarted);
1219 // The OS reports the suspended socket's close before the probe answers.
1220 yield* harness.closeLatestSession();
1221
1222 // No TestClock advance: the unanswered probe skips the first backoff rung.
1223 const reconnected = yield* awaitState(
1224 supervisor.state,
1225 (state) => state.phase === "connected" && state.generation === 2,
1226 );
1227 expect(reconnected.attempt).toBe(1);
1228 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1229 }).pipe(Effect.provide(TestClock.layer())),
1230 );
1231
1232 it.effect("reconnects immediately when the foreground liveness probe fails", () =>
1233 Effect.gen(function* () {
1234 const allowReconnect = yield* Deferred.make<void>();
1235 const harness = yield* makeHarness({
1236 prepare: (attempt) =>
1237 attempt === 2
1238 ? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION))
1239 : Effect.succeed(PREPARED_CONNECTION),
1240 probe: (attempt) =>
1241 attempt === 1 ? Effect.fail(transient("The live session is stale.")) : Effect.void,
1242 });
1243 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1244 initiallyDesired: true,
1245 }).pipe(Effect.provide(harness.dependencies));
1246
1247 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1248 yield* harness.wake("application-active");
1249 const reconnecting = yield* awaitState(
1250 supervisor.state,
1251 (state) => state.phase === "connecting",
1252 );
1253 expect(reconnecting.attempt).toBe(1);
1254 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
1255
1256 // No TestClock advance: a failed wake probe skips the first backoff rung.
1257 yield* Deferred.succeed(allowReconnect, undefined);
1258 yield* awaitState(
1259 supervisor.state,
1260 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1261 );
1262
1263 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1264 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1265 }).pipe(Effect.provide(TestClock.layer())),
1266 );
1267
1268 it.effect("keeps normal backoff when a reconnect after a failed wake probe also fails", () =>
1269 Effect.gen(function* () {
1270 const harness = yield* makeHarness({
1271 prepare: (attempt) =>
1272 attempt === 2 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
1273 probe: (attempt) =>
1274 attempt === 1 ? Effect.fail(transient("The live session is stale.")) : Effect.void,
1275 });
1276 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1277 initiallyDesired: true,
1278 }).pipe(Effect.provide(harness.dependencies));
1279
1280 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1281 yield* harness.wake("application-active");
1282 // The immediate follow-up attempt fails: only the first attempt after
1283 // the wake probe skips the ladder, so this failure backs off normally.
1284 yield* awaitState(
1285 supervisor.state,
1286 (state) => state.phase === "backoff" && state.attempt === 1,
1287 );
1288 yield* TestClock.adjust("1999 millis");
1289 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1290 yield* TestClock.adjust("1 milli");
1291 yield* eventuallyState(
1292 supervisor.state,
1293 (state) => state.phase === "connected" && state.generation === 2,
1294 );
1295
1296 expect(yield* Ref.get(harness.prepareCount)).toBe(3);
1297 }).pipe(Effect.provide(TestClock.layer())),
1298 );
1299
1300 it.effect("uses the full tolerance window for a stalled desktop foreground probe", () =>
1301 Effect.gen(function* () {
1302 const harness = yield* makeHarness({
1303 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1304 });
1305 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1306 initiallyDesired: true,
1307 }).pipe(Effect.provide(harness.dependencies));
1308
1309 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1310 yield* harness.wake("application-active");
1311 yield* TestClock.adjust("14999 millis");
1312 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1313 yield* TestClock.adjust("1 milli");
1314 yield* awaitState(
1315 supervisor.state,
1316 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1317 );
1318
1319 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1320 }).pipe(Effect.provide(TestClock.layer())),
1321 );
1322
1323 it.effect("an explicit retry shortens a stalled desktop foreground probe", () =>
1324 Effect.gen(function* () {
1325 const harness = yield* makeHarness({
1326 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1327 });
1328 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1329 initiallyDesired: true,
1330 }).pipe(Effect.provide(harness.dependencies));
1331
1332 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1333 yield* harness.wake("application-active");
1334 yield* TestClock.adjust("5 seconds");
1335 yield* supervisor.retryNow;
1336 // The retry's 3 second limit applies, not the 10 seconds left of the 15.
1337 yield* TestClock.adjust("2999 millis");
1338 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1339 yield* TestClock.adjust("1 milli");
1340 yield* awaitState(
1341 supervisor.state,
1342 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1343 );
1344
1345 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1346 }).pipe(Effect.provide(TestClock.layer())),
1347 );
1348
1349 it.effect("quickly times out a stalled mobile foreground liveness probe", () =>
1350 Effect.gen(function* () {
1351 const harness = yield* makeHarness({
1352 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1353 });
1354 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1355 initiallyDesired: true,
1356 }).pipe(Effect.provide(harness.dependencies));
1357
1358 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1359 yield* harness.wake("application-active-probe");
1360 yield* TestClock.adjust("3 seconds");
1361 // The timed-out wake probe reconnects immediately without a backoff
1362 // sleep: no further clock advance is needed.
1363 yield* awaitState(
1364 supervisor.state,
1365 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1366 );
1367
1368 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1369 }).pipe(Effect.provide(TestClock.layer())),
1370 );
1371
1372 it.effect("honors an explicit disconnect while a foreground probe is stalled", () =>
1373 Effect.gen(function* () {
1374 const probeStarted = yield* Deferred.make<void>();
1375 const harness = yield* makeHarness({
1376 probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)),
1377 });
1378 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1379 initiallyDesired: true,
1380 }).pipe(Effect.provide(harness.dependencies));
1381
1382 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1383 yield* harness.wake("application-active");
1384 yield* Deferred.await(probeStarted);
1385 yield* supervisor.disconnect;
1386 yield* awaitState(supervisor.state, (state) => state.phase === "available");
1387
1388 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1389 }),
1390 );
1391
1392 it.effect("does not churn a healthy session when credentials change", () =>
1393 Effect.gen(function* () {
1394 const harness = yield* makeHarness();
1395 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1396 initiallyDesired: true,
1397 }).pipe(Effect.provide(harness.dependencies));
1398
1399 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1400 yield* harness.wake("credentials-changed");
1401 yield* Effect.yieldNow;
1402
1403 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1404 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1405 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
1406 }),
1407 );
1408
1409 it.effect("releases and reconnects a relay session when credentials change", () =>
1410 Effect.gen(function* () {
1411 const harness = yield* makeHarness({
1412 prepare: () => Effect.succeed(preparedFor(RELAY_TARGET)),
1413 });
1414 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1415 initiallyDesired: true,
1416 }).pipe(Effect.provide(harness.dependencies));
1417
1418 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1419 yield* harness.wake("credentials-changed");
1420 yield* awaitState(
1421 supervisor.state,
1422 (state) => state.phase === "connected" && state.generation === 2,
1423 );
1424
1425 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1426 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1427 }),
1428 );
1429
1430 it.effect("keeps a healthy relay session when its HTTP access token expires", () =>
1431 Effect.gen(function* () {
1432 const harness = yield* makeHarness({
1433 prepare: (attempt) =>
1434 Effect.succeed({
1435 ...PREPARED_CONNECTION,
1436 target: RELAY_TARGET,
1437 httpAuthorization: {
1438 _tag: "Dpop",
1439 accessToken: `access-token-${attempt}`,
1440 expiresAtEpochMs: 3_600_000 * attempt,
1441 },
1442 }),
1443 });
1444 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1445 initiallyDesired: true,
1446 }).pipe(Effect.provide(harness.dependencies));
1447
1448 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1449 const session = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
1450
1451 yield* TestClock.adjust("2 hours");
1452
1453 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1454 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1455 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session);
1456 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1457 phase: "connected",
1458 generation: 1,
1459 });
1460 }).pipe(Effect.provide(TestClock.layer())),
1461 );
1462
1463 it.effect("refreshes HTTP authorization without replacing the active relay session", () =>
1464 Effect.gen(function* () {
1465 const endpoint = {
1466 httpBaseUrl: TARGET.httpBaseUrl,
1467 wsBaseUrl: TARGET.wsBaseUrl,
1468 providerKind: "cloudflare_tunnel" as const,
1469 };
1470 const token = yield* Ref.make(
1471 Option.some(
1472 new TokenStore.RemoteDpopAccessToken({
1473 environmentId: TARGET.environmentId,
1474 accountId: "test-account",
1475 label: TARGET.label,
1476 endpoint,
1477 accessToken: "access-token-1",
1478 expiresAtEpochMs: 3_600_000,
1479 dpopThumbprint: "test-thumbprint",
1480 }),
1481 ),
1482 );
1483 const bootstrapFails = yield* Ref.make(false);
1484 const bootstrapCalls = yield* Ref.make(0);
1485 const httpPaths: Array<string> = [];
1486 const sessionAuthorizations: Array<string | null> = [];
1487 const fetchFn = ((input, init) => {
1488 const request = new Request(input, init);
1489 const pathname = new URL(request.url).pathname;
1490 httpPaths.push(pathname);
1491 switch (pathname) {
1492 case "/.well-known/t3/environment":
1493 return Promise.resolve(
1494 Response.json({
1495 environmentId: TARGET.environmentId,
1496 label: TARGET.label,
1497 platform: { os: "linux", arch: "x64" },
1498 serverVersion: "0.0.0-test",
1499 capabilities: { repositoryIdentity: true },
1500 }),
1501 );
1502 case "/oauth/token":
1503 return Promise.resolve(
1504 Response.json({
1505 access_token: "access-token-2",
1506 issued_token_type: "urn:ietf:params:oauth:token-type:access_token",
1507 token_type: "DPoP",
1508 expires_in: 3_600,
1509 scope: AuthStandardClientScopes.join(" "),
1510 }),
1511 );
1512 case "/api/auth/websocket-ticket":
1513 return Promise.resolve(
1514 Response.json({
1515 ticket: "ws-ticket",
1516 expiresAt: "2026-09-04T01:00:00.000Z",
1517 }),
1518 );
1519 case "/api/auth/session": {
1520 const authorization = request.headers.get("authorization");
1521 sessionAuthorizations.push(authorization);
1522 return Promise.resolve(
1523 Response.json({
1524 authenticated: authorization === "DPoP access-token-2",
1525 auth: {
1526 policy: "loopback-browser",
1527 bootstrapMethods: ["one-time-token"],
1528 sessionMethods: ["dpop-access-token"],
1529 sessionCookieName: "t3_session_test",
1530 },
1531 scopes: AuthStandardClientScopes,
1532 }),
1533 );
1534 }
1535 default:
1536 return Promise.reject(new Error(`Unexpected HTTP request to ${request.url}`));
1537 }
1538 }) satisfies typeof fetch;
1539 const signer = ManagedRelay.ManagedRelayDpopSigner.of({
1540 thumbprint: Effect.succeed("test-thumbprint"),
1541 createProof: () => Effect.succeed("test-proof"),
1542 });
1543 const unused = () => Effect.die("Unexpected relay operation.");
1544 const relay = ManagedRelay.ManagedRelayClient.of({
1545 relayUrl: "https://relay.example.test",
1546 listEnvironments: unused,
1547 listDevices: unused,
1548 createEnvironmentLinkChallenge: unused,
1549 linkEnvironment: unused,
1550 unlinkEnvironment: unused,
1551 getEnvironmentStatus: unused,
1552 connectEnvironment: Effect.fn("TestConnectionHttp.connectEnvironment")(function* () {
1553 yield* Ref.update(bootstrapCalls, (count) => count + 1);
1554 if (yield* Ref.get(bootstrapFails)) {
1555 return yield* new ManagedRelay.ManagedRelayRequestTimeoutError({
1556 activity: "Relay environment connection",
1557 timeoutMs: 6_000,
1558 traceId: null,
1559 });
1560 }
1561 return {
1562 environmentId: TARGET.environmentId,
1563 endpoint,
1564 credential: "relay-bootstrap",
1565 expiresAt: "2026-09-04T01:00:00.000Z",
1566 };
1567 }),
1568 registerDevice: unused,
1569 unregisterDevice: unused,
1570 registerLiveActivity: unused,
1571 getAgentActivitySnapshot: unused,
1572 resetTokenCache: Effect.void,
1573 });
1574 const layerHttp = RpcHttp.layerRemoteHttpClient(fetchFn);
1575 const remoteAuthorization = yield* RemoteEnvironmentAuthorization.make.pipe(
1576 Effect.provide(
1577 Layer.mergeAll(
1578 layerHttp,
1579 Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, signer),
1580 Layer.succeed(ManagedRelay.ManagedRelayClient, relay),
1581 Layer.succeed(ClientCapabilities.CloudSession, {
1582 identity: Effect.succeedSome({ accountId: "test-account" }),
1583 clerkToken: Effect.succeed("clerk-token"),
1584 }),
1585 Layer.succeed(ClientCapabilities.RelayDeviceIdentity, {
1586 deviceId: Effect.succeedNone,
1587 }),
1588 TokenStore.layer({
1589 get: () => Ref.get(token),
1590 put: (value) => Ref.set(token, Option.some(value)),
1591 remove: () => Ref.set(token, Option.none()),
1592 }),
1593 Layer.succeed(ClientCapabilities.ClientPresentation, {
1594 metadata: { label: "Test client", deviceType: "desktop" },
1595 }),
1596 ),
1597 ),
1598 );
1599 const harness = yield* makeHarness({
1600 prepare: () =>
1601 remoteAuthorization
1602 .authorizeDpop({ expectedEnvironmentId: TARGET.environmentId })
1603 .pipe(Effect.map((prepared) => ({ ...prepared, target: RELAY_TARGET }))),
1604 });
1605 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1606 initiallyDesired: true,
1607 }).pipe(Effect.provide(harness.dependencies));
1608 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1609 const session = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
1610 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
1611 const readSession = fetchEnvironmentSessionState({
1612 prepared,
1613 signer: Option.some(signer),
1614 remoteAuthorization: Option.some(remoteAuthorization),
1615 }).pipe(Effect.provide(layerHttp));
1616
1617 yield* TestClock.adjust("2 hours");
1618 expect((yield* readSession).authenticated).toBe(true);
1619 expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]);
1620 expect(yield* Ref.get(bootstrapCalls)).toBe(1);
1621 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session);
1622 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1623 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1624 phase: "connected",
1625 generation: 1,
1626 });
1627
1628 yield* TestClock.adjust("2 hours");
1629 yield* Ref.set(bootstrapFails, true);
1630 const failure = yield* readSession.pipe(Effect.flip);
1631 expect(failure._tag).toBe("RemoteEnvironmentAuthFetchError");
1632 expect(yield* Ref.get(bootstrapCalls)).toBe(2);
1633 expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]);
1634 expect(httpPaths.filter((path) => path === "/api/auth/websocket-ticket")).toHaveLength(1);
1635 expect(httpPaths.filter((path) => path === "/oauth/token")).toHaveLength(1);
1636 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1637 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1638 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session);
1639 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1640 phase: "connected",
1641 generation: 1,
1642 });
1643 }).pipe(Effect.provide(TestClock.layer())),
1644 );
1645
1646 it.effect("interrupts relay setup when credentials change", () =>
1647 Effect.gen(function* () {
1648 const firstAttemptStarted = yield* Deferred.make<void>();
1649 const harness = yield* makeHarness({
1650 prepare: (attempt) =>
1651 attempt === 1
1652 ? Deferred.succeed(firstAttemptStarted, undefined).pipe(Effect.andThen(Effect.never))
1653 : Effect.succeed(PREPARED_CONNECTION),
1654 });
1655 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1656 initiallyDesired: true,
1657 }).pipe(Effect.provide(harness.dependencies));
1658
1659 yield* Deferred.await(firstAttemptStarted);
1660 yield* harness.wake("credentials-changed");
1661 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1662
1663 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1664 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1665 }),
1666 );
1667
1668 it.effect("explicit disconnect releases the session and returns to available", () =>
1669 Effect.gen(function* () {
1670 const harness = yield* makeHarness();
1671 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1672 initiallyDesired: true,
1673 }).pipe(Effect.provide(harness.dependencies));
1674
1675 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1676 yield* supervisor.disconnect;
1677 yield* awaitState(supervisor.state, (state) => state.phase === "available");
1678
1679 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1680 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
1681 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
1682 }),
1683 );
1684
1685 it.effect("does not lose an explicit disconnect among concurrent wakeup signals", () =>
1686 Effect.gen(function* () {
1687 const harness = yield* makeHarness();
1688 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1689 initiallyDesired: true,
1690 }).pipe(Effect.provide(harness.dependencies));
1691
1692 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1693 yield* Effect.all(
1694 [
1695 supervisor.disconnect,
1696 harness.wake("credentials-changed"),
1697 harness.wake("application-active"),
1698 harness.wake("credentials-changed"),
1699 ],
1700 { concurrency: "unbounded" },
1701 );
1702 yield* awaitState(supervisor.state, (state) => state.phase === "available");
1703
1704 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1705 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
1706 }),
1707 );
1708});
1709
1710describe("EnvironmentSupervisor routes", () => {
1711 it.effect("skips a silent LAN route and connects over T3 Connect", () =>
1712 Effect.gen(function* () {
1713 const harness = yield* makeHarness({
1714 checkRoute: (route) =>
1715 Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"),
1716 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1717 });
1718 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1719 initiallyDesired: true,
1720 }).pipe(Effect.provide(harness.dependencies));
1721
1722 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1723 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
1724 expect(prepared.target._tag).toBe("RelayConnectionTarget");
1725 // The silent route never reached the resolver.
1726 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
1727 }),
1728 );
1729
1730 it.effect("moves to a blocked route's fallback instead of stopping", () =>
1731 Effect.gen(function* () {
1732 const harness = yield* makeHarness({
1733 checkRoute: () => Effect.succeed("answered"),
1734 prepare: (_attempt, target) =>
1735 target._tag === "BearerConnectionTarget"
1736 ? Effect.fail(blocked("The environment credential is invalid."))
1737 : Effect.succeed(preparedFor(target)),
1738 });
1739 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1740 initiallyDesired: true,
1741 }).pipe(Effect.provide(harness.dependencies));
1742
1743 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1744 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
1745 expect(prepared.target._tag).toBe("RelayConnectionTarget");
1746 }),
1747 );
1748
1749 it.effect("tries every route when none answers the reachability check", () =>
1750 Effect.gen(function* () {
1751 const harness = yield* makeHarness({
1752 checkRoute: () => Effect.succeed("silent"),
1753 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1754 });
1755 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1756 initiallyDesired: true,
1757 }).pipe(Effect.provide(harness.dependencies));
1758
1759 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1760 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
1761 expect(prepared.target._tag).toBe("BearerConnectionTarget");
1762 }),
1763 );
1764
1765 it.effect(
1766 "gives the next route its own snapshot budget after a route fails while synchronizing",
1767 () =>
1768 Effect.gen(function* () {
1769 const harness = yield* makeHarness({
1770 checkRoute: () => Effect.succeed("answered"),
1771 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1772 // The LAN session fails 50 seconds into its snapshot; T3 Connect needs 40.
1773 ready: (attempt) =>
1774 attempt === 1
1775 ? Effect.sleep("50 seconds").pipe(Effect.andThen(Effect.fail(transient())))
1776 : Effect.sleep("40 seconds"),
1777 });
1778 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1779 initiallyDesired: true,
1780 }).pipe(Effect.provide(harness.dependencies));
1781
1782 yield* awaitState(
1783 supervisor.state,
1784 (state) => state.phase === "connecting" && state.stage === "synchronizing",
1785 );
1786 yield* TestClock.adjust("50 seconds");
1787 yield* TestClock.adjust("40 seconds");
1788 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1789 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1790 "RelayConnectionTarget",
1791 );
1792 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1793 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1794 }).pipe(Effect.provide(TestClock.layer())),
1795 );
1796
1797 it.effect("still tries a silent LAN route after the T3 Connect route fails", () =>
1798 Effect.gen(function* () {
1799 const harness = yield* makeHarness({
1800 checkRoute: (route) =>
1801 Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"),
1802 // Signed out of T3 Connect; the LAN is up but its check timed out.
1803 prepare: (_attempt, target) =>
1804 target._tag === "RelayConnectionTarget"
1805 ? Effect.fail(blocked("Sign in to T3 Connect."))
1806 : Effect.succeed(preparedFor(target)),
1807 });
1808 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1809 initiallyDesired: true,
1810 }).pipe(Effect.provide(harness.dependencies));
1811
1812 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1813 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1814 "BearerConnectionTarget",
1815 );
1816 }),
1817 );
1818
1819 it.effect("keeps retrying when one route is blocked and another failed transiently", () =>
1820 Effect.gen(function* () {
1821 const harness = yield* makeHarness({
1822 checkRoute: () => Effect.succeed("answered"),
1823 prepare: (_attempt, target) =>
1824 target._tag === "RelayConnectionTarget"
1825 ? Effect.fail(blocked("Sign in to T3 Connect."))
1826 : Effect.fail(transient("LAN socket refused.")),
1827 });
1828 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1829 initiallyDesired: true,
1830 }).pipe(Effect.provide(harness.dependencies));
1831
1832 // A blocked state would stop retrying; the LAN may come back.
1833 const state = yield* awaitState(supervisor.state, (value) => value.phase === "backoff");
1834 expect(state.lastFailure?._tag).toBe("ConnectionTransientError");
1835 }),
1836 );
1837
1838 it.effect("moves back to the LAN route when the network changes and it answers again", () =>
1839 Effect.gen(function* () {
1840 const lanReachable = yield* Ref.make(false);
1841 const harness = yield* makeHarness({
1842 checkRoute: (route) =>
1843 route.target._tag === "BearerConnectionTarget"
1844 ? Ref.get(lanReachable).pipe(
1845 Effect.map((reachable) => (reachable ? "answered" : "silent")),
1846 )
1847 : Effect.succeed("unchecked"),
1848 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1849 });
1850 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1851 initiallyDesired: true,
1852 }).pipe(Effect.provide(harness.dependencies));
1853
1854 yield* awaitState(
1855 supervisor.state,
1856 (state) => state.phase === "connected" && state.generation === 1,
1857 );
1858 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1859 "RelayConnectionTarget",
1860 );
1861
1862 // Back home: the LAN answers, and a network change asks for a better route.
1863 yield* Ref.set(lanReachable, true);
1864 yield* harness.setNetworkStatus("unknown");
1865 yield* harness.setNetworkStatus("online");
1866
1867 yield* awaitState(
1868 supervisor.state,
1869 (state) => state.phase === "connected" && state.generation === 2,
1870 );
1871 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1872 "BearerConnectionTarget",
1873 );
1874 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1875 }),
1876 );
1877
1878 it.effect("checks for a better route periodically while on a fallback", () =>
1879 Effect.gen(function* () {
1880 const lanReachable = yield* Ref.make(false);
1881 const harness = yield* makeHarness({
1882 checkRoute: (route) =>
1883 route.target._tag === "BearerConnectionTarget"
1884 ? Ref.get(lanReachable).pipe(
1885 Effect.map((reachable) => (reachable ? "answered" : "silent")),
1886 )
1887 : Effect.succeed("unchecked"),
1888 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1889 });
1890 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1891 initiallyDesired: true,
1892 }).pipe(Effect.provide(harness.dependencies));
1893
1894 yield* awaitState(
1895 supervisor.state,
1896 (state) => state.phase === "connected" && state.generation === 1,
1897 );
1898 yield* Ref.set(lanReachable, true);
1899 yield* TestClock.adjust("60 seconds");
1900
1901 yield* awaitState(
1902 supervisor.state,
1903 (state) => state.phase === "connected" && state.generation === 2,
1904 );
1905 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1906 "BearerConnectionTarget",
1907 );
1908 }),
1909 );
1910
1911 it.effect("does not retry a better route that answered but failed to connect", () =>
1912 Effect.gen(function* () {
1913 const lanAnswers = yield* Ref.make(false);
1914 const harness = yield* makeHarness({
1915 checkRoute: (route) =>
1916 route.target._tag === "BearerConnectionTarget"
1917 ? Ref.get(lanAnswers).pipe(Effect.map((answers) => (answers ? "answered" : "silent")))
1918 : Effect.succeed("unchecked"),
1919 // The LAN answers its check but the socket never opens.
1920 prepare: (_attempt, target) =>
1921 target._tag === "BearerConnectionTarget"
1922 ? Effect.fail(transient("Socket refused."))
1923 : Effect.succeed(preparedFor(target)),
1924 });
1925 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1926 initiallyDesired: true,
1927 }).pipe(Effect.provide(harness.dependencies));
1928
1929 yield* awaitState(
1930 supervisor.state,
1931 (state) => state.phase === "connected" && state.generation === 1,
1932 );
1933 yield* Ref.set(lanAnswers, true);
1934 yield* TestClock.adjust("60 seconds");
1935 // The switch lands back on T3 Connect.
1936 yield* awaitState(
1937 supervisor.state,
1938 (state) => state.phase === "connected" && state.generation === 2,
1939 );
1940 const preparesAfterSwitch = yield* Ref.get(harness.prepareCount);
1941
1942 // The failed LAN is cooling down, so the next periodic check leaves the
1943 // session alone.
1944 yield* TestClock.adjust("60 seconds");
1945 yield* Effect.yieldNow;
1946 expect(yield* Ref.get(harness.prepareCount)).toBe(preparesAfterSwitch);
1947 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1948 phase: "connected",
1949 generation: 2,
1950 });
1951 }),
1952 );
1953
1954 it.effect("stays on the preferred route without checking other routes", () =>
1955 Effect.gen(function* () {
1956 const checks = yield* Ref.make(0);
1957 const harness = yield* makeHarness({
1958 checkRoute: () => Ref.update(checks, (count) => count + 1).pipe(Effect.as("answered")),
1959 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1960 });
1961 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1962 initiallyDesired: true,
1963 }).pipe(Effect.provide(harness.dependencies));
1964
1965 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1966 const checksAtConnect = yield* Ref.get(checks);
1967 yield* harness.setNetworkStatus("unknown");
1968 yield* harness.setNetworkStatus("online");
1969 yield* TestClock.adjust("5 minutes");
1970 yield* Effect.yieldNow;
1971
1972 expect(yield* Ref.get(checks)).toBe(checksAtConnect);
1973 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1974 }),
1975 );
1976
1977 it.effect("moves off a dead LAN socket when the phone switches to cellular", () =>
1978 Effect.gen(function* () {
1979 const lanReachable = yield* Ref.make(true);
1980 const harness = yield* makeHarness({
1981 checkRoute: (route) =>
1982 route.target._tag === "BearerConnectionTarget"
1983 ? Ref.get(lanReachable).pipe(
1984 Effect.map((reachable) => (reachable ? "answered" : "silent")),
1985 )
1986 : Effect.succeed("unchecked"),
1987 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1988 // The LAN socket never answers once the phone has left the network.
1989 probe: () =>
1990 Ref.get(lanReachable).pipe(
1991 Effect.flatMap((reachable) => (reachable ? Effect.void : Effect.never)),
1992 ),
1993 });
1994 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1995 initiallyDesired: true,
1996 }).pipe(Effect.provide(harness.dependencies));
1997
1998 yield* awaitState(
1999 supervisor.state,
2000 (state) => state.phase === "connected" && state.generation === 1,
2001 );
2002 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2003 "BearerConnectionTarget",
2004 );
2005
2006 yield* Ref.set(lanReachable, false);
2007 yield* harness.wake("network-changed");
2008 yield* TestClock.adjust("3 seconds");
2009
2010 yield* awaitState(
2011 supervisor.state,
2012 (state) => state.phase === "connected" && state.generation === 2,
2013 );
2014 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2015 "RelayConnectionTarget",
2016 );
2017 }),
2018 );
2019
2020 it.effect("keeps a LAN session when the T3 Connect account changes", () =>
2021 Effect.gen(function* () {
2022 const harness = yield* makeHarness({
2023 checkRoute: () => Effect.succeed("answered"),
2024 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2025 });
2026 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2027 initiallyDesired: true,
2028 }).pipe(Effect.provide(harness.dependencies));
2029
2030 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2031 yield* harness.wake("credentials-changed");
2032 yield* Effect.yieldNow;
2033
2034 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
2035 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
2036 phase: "connected",
2037 generation: 1,
2038 });
2039 }),
2040 );
2041
2042 it.effect("learns the LAN address over T3 Connect and moves to it", () =>
2043 Effect.gen(function* () {
2044 const relayEntry: ConnectionCatalogEntry = {
2045 target: RELAY_TARGET,
2046 profile: Option.none(),
2047 enabled: true,
2048 };
2049 const lanAddress = yield* Ref.make("http://192.168.1.10:3773/");
2050 const learned = yield* Ref.make<ReadonlyArray<string>>([]);
2051 const harness = yield* makeHarness({
2052 checkRoute: (route) =>
2053 route.target._tag === "BearerConnectionTarget"
2054 ? Effect.succeed("answered")
2055 : Effect.succeed("unchecked"),
2056 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2057 // Only the reported endpoints matter to the supervisor.
2058 initialConfig: () =>
2059 Ref.get(lanAddress).pipe(
2060 Effect.map((httpBaseUrl): Pick<ServerConfig, "directEndpoints"> => ({
2061 directEndpoints: [{ kind: "lan", httpBaseUrl }],
2062 })),
2063 Effect.map((config) => config as ServerConfig),
2064 ),
2065 });
2066 const current = yield* Ref.make(relayEntry);
2067 const supervisor = yield* EnvironmentSupervisor.make(relayEntry, {
2068 initiallyDesired: true,
2069 learnRoutes: ({ activeRoute, reported }) =>
2070 Effect.gen(function* () {
2071 const entry = yield* Ref.get(current);
2072 const routes = mergeLearnedRoutes({
2073 entry,
2074 activeRoute,
2075 reported,
2076 allowInsecure: true,
2077 });
2078 if (routes === null) return Option.none();
2079 const next = entryWithRoutes(entry, routes);
2080 yield* Ref.set(current, next);
2081 yield* Ref.update(learned, (all) => [
2082 ...all,
2083 ...routes.filter(isLearned).map((route) => connectionRouteId(route.target)),
2084 ]);
2085 return Option.some(next);
2086 }),
2087 }).pipe(Effect.provide(harness.dependencies));
2088
2089 // Connected over T3 Connect, the server reports its LAN address; the
2090 // learned route ranks first, answers, and the session moves to it.
2091 yield* awaitState(
2092 supervisor.state,
2093 (state) => state.phase === "connected" && state.generation === 2,
2094 );
2095 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2096 "BearerConnectionTarget",
2097 );
2098 expect(yield* Ref.get(learned)).toEqual([
2099 `learned:${TARGET.environmentId}:http://192.168.1.10:3773`,
2100 ]);
2101 }),
2102 );
2103});