authorgravatar for saphid@gmail.comAlex Southwell <saphid@gmail.com> 2026-10-14 19:04:09+11:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 01:04:09-07:00
log5bba764432366a41fa14ea0df694c2a7fca35c4c
treed634cb2d0c045a58d84870ab5e2a3e18a53ef3ff
parent94ca023bf488ac2d13e3233a7e2382ac67bda66d
signature Signed by PGP key B5690EEEBB952194

fix(client-runtime): slow config snapshots no longer drop working connections (#14518)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: SAPHID <saphid@users.noreply.github.com>

2 files changed, 98 insertions(+), 9 deletions(-)

packages/client-runtime/src/connection/supervisor.test.ts+57-1
...@@ -488,6 +488,29 @@ describe("EnvironmentSupervisor", () => {...@@ -488,6 +488,29 @@ describe("EnvironmentSupervisor", () => {
488 }).pipe(Effect.provide(TestClock.layer())),488 }).pipe(Effect.provide(TestClock.layer())),
489 );489 );
490490
491 it.effect("keeps a working socket open while a slow server config snapshot arrives", () =>
492 Effect.gen(function* () {
493 const harness = yield* makeHarness({
494 ready: () => Effect.sleep("40 seconds"),
495 });
496 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
497 initiallyDesired: true,
498 }).pipe(Effect.provide(harness.dependencies));
499
500 yield* awaitState(
501 supervisor.state,
502 (state) => state.phase === "connecting" && state.stage === "synchronizing",
503 );
504 yield* TestClock.adjust("39 seconds");
505 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connecting");
506
507 yield* TestClock.adjust("1 second");
508 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
509 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
510 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
511 }).pipe(Effect.provide(TestClock.layer())),
512 );
513
491 it.effect("retries when a session never becomes ready", () =>514 it.effect("retries when a session never becomes ready", () =>
492 Effect.gen(function* () {515 Effect.gen(function* () {
493 const harness = yield* makeHarness({516 const harness = yield* makeHarness({
...@@ -501,7 +524,8 @@ describe("EnvironmentSupervisor", () => {...@@ -501,7 +524,8 @@ describe("EnvironmentSupervisor", () => {
501 supervisor.state,524 supervisor.state,
502 (state) => state.phase === "connecting" && state.stage === "synchronizing",525 (state) => state.phase === "connecting" && state.stage === "synchronizing",
503 );526 );
504 yield* TestClock.adjust("14 seconds");527 // 15 seconds of setup plus 75 for the snapshot; setup finished instantly.
528 yield* TestClock.adjust("89 seconds");
505 expect((yield* SubscriptionRef.get(supervisor.state)).stage).toBe("synchronizing");529 expect((yield* SubscriptionRef.get(supervisor.state)).stage).toBe("synchronizing");
506530
507 yield* TestClock.adjust("1 second");531 yield* TestClock.adjust("1 second");
...@@ -1742,6 +1766,38 @@ describe("EnvironmentSupervisor routes", () => {...@@ -1742,6 +1766,38 @@ describe("EnvironmentSupervisor routes", () => {
1742 }),1766 }),
1743 );1767 );
17441768
1769 it.effect(
1770 "gives the next route its own snapshot budget after a route fails while synchronizing",
1771 () =>
1772 Effect.gen(function* () {
1773 const harness = yield* makeHarness({
1774 checkRoute: () => Effect.succeed("answered"),
1775 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1776 // The LAN session fails 50 seconds into its snapshot; T3 Connect needs 40.
1777 ready: (attempt) =>
1778 attempt === 1
1779 ? Effect.sleep("50 seconds").pipe(Effect.andThen(Effect.fail(transient())))
1780 : Effect.sleep("40 seconds"),
1781 });
1782 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1783 initiallyDesired: true,
1784 }).pipe(Effect.provide(harness.dependencies));
1785
1786 yield* awaitState(
1787 supervisor.state,
1788 (state) => state.phase === "connecting" && state.stage === "synchronizing",
1789 );
1790 yield* TestClock.adjust("50 seconds");
1791 yield* TestClock.adjust("40 seconds");
1792 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1793 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1794 "RelayConnectionTarget",
1795 );
1796 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1797 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1798 }).pipe(Effect.provide(TestClock.layer())),
1799 );
1800
1745 it.effect("still tries a silent LAN route after the T3 Connect route fails", () =>1801 it.effect("still tries a silent LAN route after the T3 Connect route fails", () =>
1746 Effect.gen(function* () {1802 Effect.gen(function* () {
1747 const harness = yield* makeHarness({1803 const harness = yield* makeHarness({
packages/client-runtime/src/connection/supervisor.ts+41-8
...@@ -39,8 +39,15 @@ import {...@@ -39,8 +39,15 @@ import {
3939
40const RETRY_BASE_DELAY_MS = 1_000;40const RETRY_BASE_DELAY_MS = 1_000;
41const RETRY_MAX_DELAY_MS = 300_000;41const RETRY_MAX_DELAY_MS = 300_000;
42// Per-route budget to resolve credentials and create the session, which ends
43// when the driver reports the synchronizing stage. Each route that gets that
44// far adds its own budget for the socket to open (bounded at 15 seconds by the
45// RPC session) and the server to deliver the first config snapshot. A cold
46// server with a large database can take longer than 15 seconds to answer, and
47// that is not a reason to drop a working connection.
42const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";48const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
43const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT);49const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT);
50const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");
44const CONNECTION_PROBE_TIMEOUT = "15 seconds";51const CONNECTION_PROBE_TIMEOUT = "15 seconds";
45// Mobile resumes, explicit retries, and offline events want a fast answer:52// Mobile resumes, explicit retries, and offline events want a fast answer:
46// the user is waiting, or the network may be gone.53// the user is waiting, or the network may be gone.
...@@ -428,9 +435,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (...@@ -428,9 +435,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
428 attempt: number,435 attempt: number,
429 generation: number,436 generation: number,
430 lastFailure: ConnectionAttemptError | null,437 lastFailure: ConnectionAttemptError | null,
438 synchronizedRoutes: Ref.Ref<number>,
431 ) {439 ) {
432 return yield* driver.connect(yield* attemptEntry, (progress) =>440 return yield* driver.connect(yield* attemptEntry, (progress) =>
433 reportProgress(attempt, generation, lastFailure, progress),441 (progress.stage === "synchronizing"
442 ? Ref.update(synchronizedRoutes, (count) => count + 1)
443 : Effect.void
444 ).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))),
434 );445 );
435 });446 });
436447
...@@ -481,16 +492,17 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (...@@ -481,16 +492,17 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
481 generation: number,492 generation: number,
482 lastFailure: ConnectionAttemptError | null,493 lastFailure: ConnectionAttemptError | null,
483 pendingRetry: Option.Option<PendingRetryTrace>,494 pendingRetry: Option.Option<PendingRetryTrace>,
495 synchronizedRoutes: Ref.Ref<number>,
484 ) {496 ) {
485 if (usesRelay) {497 if (usesRelay) {
486 return yield* traceRelayEstablishment(498 return yield* traceRelayEstablishment(
487 establishConnection(attempt, generation, lastFailure),499 establishConnection(attempt, generation, lastFailure, synchronizedRoutes),
488 attempt,500 attempt,
489 generation,501 generation,
490 pendingRetry,502 pendingRetry,
491 );503 );
492 }504 }
493 return yield* establishConnection(attempt, generation, lastFailure).pipe(505 return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe(
494 Effect.map((lease) => ({506 Effect.map((lease) => ({
495 attemptSpan: Option.none<Tracer.Span>(),507 attemptSpan: Option.none<Tracer.Span>(),
496 lease,508 lease,
...@@ -684,9 +696,16 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (...@@ -684,9 +696,16 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
684 ) {696 ) {
685 const switchingTo = yield* Ref.get(preferredRouteId);697 const switchingTo = yield* Ref.get(preferredRouteId);
686 yield* SubscriptionRef.set(prepared, Option.none());698 yield* SubscriptionRef.set(prepared, Option.none());
699 const synchronizedRoutes = yield* Ref.make(0);
687 const establishment = yield* Effect.raceAllFirst([700 const establishment = yield* Effect.raceAllFirst([
688 exitUnlessInterrupted(701 exitUnlessInterrupted(
689 establishTracedConnection(attempt, generation, lastFailure, pendingRetry),702 establishTracedConnection(
703 attempt,
704 generation,
705 lastFailure,
706 pendingRetry,
707 synchronizedRoutes,
708 ),
690 ).pipe(709 ).pipe(
691 Effect.map((exit): EstablishmentEvent => ({710 Effect.map((exit): EstablishmentEvent => ({
692 _tag: "Completed",711 _tag: "Completed",
...@@ -699,10 +718,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (...@@ -699,10 +718,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
699 resetRetry,718 resetRetry,
700 })),719 })),
701 ),720 ),
702 // Each route may use the full setup time before the next is tried.721 // Each route may use the full setup time before the next is tried, and
703 Effect.sleep(722 // each route that reaches synchronizing adds its own snapshot budget, so
704 Duration.times(establishmentTimeout, connectionRoutes(yield* Ref.get(currentEntry)).length),723 // a route that fails late does not eat the next route's time.
705 ).pipe(Effect.as<EstablishmentEvent>({ _tag: "TimedOut" })),724 Effect.gen(function* () {
725 const routeCount = connectionRoutes(yield* Ref.get(currentEntry)).length;
726 const startedAt = yield* Clock.monotonicTimeNanos;
727 const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routeCount));
728 while (true) {
729 const synchronized = BigInt(yield* Ref.get(synchronizedRoutes));
730 const deadline =
731 startedAt + setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout);
732 const remaining = deadline - (yield* Clock.monotonicTimeNanos);
733 if (remaining <= 0n) {
734 return { _tag: "TimedOut" } as const satisfies EstablishmentEvent;
735 }
736 yield* Effect.sleep(Duration.nanos(remaining));
737 }
738 }),
706 ]);739 ]);
707740
708 if (establishment._tag === "Interrupted") {741 if (establishment._tag === "Interrupted") {