authorgravatar for saphid@gmail.comAlex Southwell <saphid@gmail.com> 2026-10-14 19:04:09+11:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 01:04:09-07:00
log5bba764432366a41fa14ea0df694c2a7fca35c4c
treed634cb2d0c045a58d84870ab5e2a3e18a53ef3ff
parent94ca023bf488ac2d13e3233a7e2382ac67bda66d
signature Signed by PGP key B5690EEEBB952194

fix(client-runtime): slow config snapshots no longer drop working connections (#14518)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: SAPHID <saphid@users.noreply.github.com>

2 files changed, 98 insertions(+), 9 deletions(-)

packages/client-runtime/src/connection/supervisor.test.ts+57-1
......@@ -488,6 +488,29 @@ describe("EnvironmentSupervisor", () => {
488488 }).pipe(Effect.provide(TestClock.layer())),
489489 );
490490
491 it.effect("keeps a working socket open while a slow server config snapshot arrives", () =>
492 Effect.gen(function* () {
493 const harness = yield* makeHarness({
494 ready: () => Effect.sleep("40 seconds"),
495 });
496 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
497 initiallyDesired: true,
498 }).pipe(Effect.provide(harness.dependencies));
499
500 yield* awaitState(
501 supervisor.state,
502 (state) => state.phase === "connecting" && state.stage === "synchronizing",
503 );
504 yield* TestClock.adjust("39 seconds");
505 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connecting");
506
507 yield* TestClock.adjust("1 second");
508 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
509 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
510 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
511 }).pipe(Effect.provide(TestClock.layer())),
512 );
513
491514 it.effect("retries when a session never becomes ready", () =>
492515 Effect.gen(function* () {
493516 const harness = yield* makeHarness({
......@@ -501,7 +524,8 @@ describe("EnvironmentSupervisor", () => {
501524 supervisor.state,
502525 (state) => state.phase === "connecting" && state.stage === "synchronizing",
503526 );
504 yield* TestClock.adjust("14 seconds");
527 // 15 seconds of setup plus 75 for the snapshot; setup finished instantly.
528 yield* TestClock.adjust("89 seconds");
505529 expect((yield* SubscriptionRef.get(supervisor.state)).stage).toBe("synchronizing");
506530
507531 yield* TestClock.adjust("1 second");
......@@ -1742,6 +1766,38 @@ describe("EnvironmentSupervisor routes", () => {
17421766 }),
17431767 );
17441768
1769 it.effect(
1770 "gives the next route its own snapshot budget after a route fails while synchronizing",
1771 () =>
1772 Effect.gen(function* () {
1773 const harness = yield* makeHarness({
1774 checkRoute: () => Effect.succeed("answered"),
1775 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1776 // The LAN session fails 50 seconds into its snapshot; T3 Connect needs 40.
1777 ready: (attempt) =>
1778 attempt === 1
1779 ? Effect.sleep("50 seconds").pipe(Effect.andThen(Effect.fail(transient())))
1780 : Effect.sleep("40 seconds"),
1781 });
1782 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1783 initiallyDesired: true,
1784 }).pipe(Effect.provide(harness.dependencies));
1785
1786 yield* awaitState(
1787 supervisor.state,
1788 (state) => state.phase === "connecting" && state.stage === "synchronizing",
1789 );
1790 yield* TestClock.adjust("50 seconds");
1791 yield* TestClock.adjust("40 seconds");
1792 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1793 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1794 "RelayConnectionTarget",
1795 );
1796 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1797 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1798 }).pipe(Effect.provide(TestClock.layer())),
1799 );
1800
17451801 it.effect("still tries a silent LAN route after the T3 Connect route fails", () =>
17461802 Effect.gen(function* () {
17471803 const harness = yield* makeHarness({
packages/client-runtime/src/connection/supervisor.ts+41-8
......@@ -39,8 +39,15 @@ import {
3939
4040const RETRY_BASE_DELAY_MS = 1_000;
4141const RETRY_MAX_DELAY_MS = 300_000;
42// Per-route budget to resolve credentials and create the session, which ends
43// when the driver reports the synchronizing stage. Each route that gets that
44// far adds its own budget for the socket to open (bounded at 15 seconds by the
45// RPC session) and the server to deliver the first config snapshot. A cold
46// server with a large database can take longer than 15 seconds to answer, and
47// that is not a reason to drop a working connection.
4248const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
4349const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT);
50const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");
4451const CONNECTION_PROBE_TIMEOUT = "15 seconds";
4552// Mobile resumes, explicit retries, and offline events want a fast answer:
4653// the user is waiting, or the network may be gone.
......@@ -428,9 +435,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
428435 attempt: number,
429436 generation: number,
430437 lastFailure: ConnectionAttemptError | null,
438 synchronizedRoutes: Ref.Ref<number>,
431439 ) {
432440 return yield* driver.connect(yield* attemptEntry, (progress) =>
433 reportProgress(attempt, generation, lastFailure, progress),
441 (progress.stage === "synchronizing"
442 ? Ref.update(synchronizedRoutes, (count) => count + 1)
443 : Effect.void
444 ).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))),
434445 );
435446 });
436447
......@@ -481,16 +492,17 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
481492 generation: number,
482493 lastFailure: ConnectionAttemptError | null,
483494 pendingRetry: Option.Option<PendingRetryTrace>,
495 synchronizedRoutes: Ref.Ref<number>,
484496 ) {
485497 if (usesRelay) {
486498 return yield* traceRelayEstablishment(
487 establishConnection(attempt, generation, lastFailure),
499 establishConnection(attempt, generation, lastFailure, synchronizedRoutes),
488500 attempt,
489501 generation,
490502 pendingRetry,
491503 );
492504 }
493 return yield* establishConnection(attempt, generation, lastFailure).pipe(
505 return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe(
494506 Effect.map((lease) => ({
495507 attemptSpan: Option.none<Tracer.Span>(),
496508 lease,
......@@ -684,9 +696,16 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
684696 ) {
685697 const switchingTo = yield* Ref.get(preferredRouteId);
686698 yield* SubscriptionRef.set(prepared, Option.none());
699 const synchronizedRoutes = yield* Ref.make(0);
687700 const establishment = yield* Effect.raceAllFirst([
688701 exitUnlessInterrupted(
689 establishTracedConnection(attempt, generation, lastFailure, pendingRetry),
702 establishTracedConnection(
703 attempt,
704 generation,
705 lastFailure,
706 pendingRetry,
707 synchronizedRoutes,
708 ),
690709 ).pipe(
691710 Effect.map((exit): EstablishmentEvent => ({
692711 _tag: "Completed",
......@@ -699,10 +718,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
699718 resetRetry,
700719 })),
701720 ),
702 // Each route may use the full setup time before the next is tried.
703 Effect.sleep(
704 Duration.times(establishmentTimeout, connectionRoutes(yield* Ref.get(currentEntry)).length),
705 ).pipe(Effect.as<EstablishmentEvent>({ _tag: "TimedOut" })),
721 // Each route may use the full setup time before the next is tried, and
722 // each route that reaches synchronizing adds its own snapshot budget, so
723 // a route that fails late does not eat the next route's time.
724 Effect.gen(function* () {
725 const routeCount = connectionRoutes(yield* Ref.get(currentEntry)).length;
726 const startedAt = yield* Clock.monotonicTimeNanos;
727 const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routeCount));
728 while (true) {
729 const synchronized = BigInt(yield* Ref.get(synchronizedRoutes));
730 const deadline =
731 startedAt + setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout);
732 const remaining = deadline - (yield* Clock.monotonicTimeNanos);
733 if (remaining <= 0n) {
734 return { _tag: "TimedOut" } as const satisfies EstablishmentEvent;
735 }
736 yield* Effect.sleep(Duration.nanos(remaining));
737 }
738 }),
706739 ]);
707740
708741 if (establishment._tag === "Interrupted") {