authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-05 22:12:52-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-05 19:12:52-07:00
log990bb0b689b23a33f8e53f12962fe2542b3f4005
tree6467139147b298037087d3b9545cc827d80d4c37
parent7251f1a1f14afdf626e7c7f8664da423594732a3
signature Signed by PGP key B5690EEEBB952194

fix: reconnect faster after remote server updates (#5404)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

3 files changed, 98 insertions(+), 14 deletions(-)

apps/server/src/server.ts+7-2
......@@ -583,8 +583,13 @@ export const makeServerLayer = Layer.unwrap(
583583 const server = yield* HttpServer.HttpServer;
584584 const address = server.address;
585585 if (typeof address === "string" || !("port" in address)) return;
586 yield* Effect.sleep("250 millis").pipe(
587 Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),
586 // No settling delay before the first attempt: routes are already
587 // serving by the time activation opens this gate (the startup
588 // sequence awaits routesReady), and the retry schedule below
589 // covers anything this sleep used to hedge against. Every
590 // millisecond here is dead time on the path to remote
591 // reachability after a restart.
592 yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(
588593 Effect.retry({
589594 while: (error) =>
590595 error._tag !== "EnvironmentHttpBadRequestError" &&
packages/client-runtime/src/state/server.test.ts+55
......@@ -7,12 +7,16 @@ import {
77} from "@t3tools/contracts";
88import { describe, expect, it } from "@effect/vitest";
99import * as Cause from "effect/Cause";
10import * as Duration from "effect/Duration";
1011import * as Effect from "effect/Effect";
1112import * as Exit from "effect/Exit";
13import * as Fiber from "effect/Fiber";
1214import * as Option from "effect/Option";
1315import * as Queue from "effect/Queue";
16import * as Ref from "effect/Ref";
1417import * as Stream from "effect/Stream";
1518import * as SubscriptionRef from "effect/SubscriptionRef";
19import * as TestClock from "effect/testing/TestClock";
1620import { RpcClientError } from "effect/unstable/rpc";
1721import * as Socket from "effect/unstable/socket/Socket";
1822
......@@ -30,6 +34,7 @@ import {
3034 makeEnvironmentServerConfigState,
3135 isLegacyUpdateHandoffLoss,
3236 matchesServerUpdateReadyEvent,
37 nudgeReconnectDuringUpdateRestart,
3338 projectServerWelcome,
3439 resolveServerConfigValue,
3540 resolveServerUpdateProgressResult,
......@@ -71,6 +76,56 @@ function session(client: WsRpcProtocolClient): RpcSession {
7176 };
7277}
7378
79describe("update restart reconnect nudges", () => {
80 it.effect("retries once per backoff entry instead of only the first", () =>
81 Effect.gen(function* () {
82 const retries = yield* Ref.make(0);
83 const states = [
84 { phase: "backoff" },
85 { phase: "connecting" },
86 { phase: "backoff" },
87 { phase: "backoff" },
88 ];
89
90 yield* nudgeReconnectDuringUpdateRestart({
91 stateChanges: Stream.fromIterable(states),
92 retryNow: Ref.update(retries, (count) => count + 1),
93 interval: Duration.zero,
94 });
95
96 // Three backoff entries, three nudges. The old one-shot behavior fired
97 // once and then let the supervisor's ladder stretch to 16-second gaps.
98 expect(yield* Ref.get(retries)).toBe(3);
99 }),
100 );
101
102 it.effect("paces nudges so a fast-failing connection cannot spin", () =>
103 Effect.gen(function* () {
104 const retries = yield* Ref.make(0);
105
106 const fiber = yield* Effect.forkChild(
107 nudgeReconnectDuringUpdateRestart({
108 stateChanges: Stream.fromIterable([{ phase: "backoff" }, { phase: "backoff" }]),
109 retryNow: Ref.update(retries, (count) => count + 1),
110 }),
111 { startImmediately: true },
112 );
113
114 // Each nudge waits out the interval first, so nothing fires immediately.
115 yield* TestClock.adjust(Duration.millis(999));
116 expect(yield* Ref.get(retries)).toBe(0);
117
118 yield* TestClock.adjust(Duration.millis(1));
119 expect(yield* Ref.get(retries)).toBe(1);
120
121 yield* TestClock.adjust(Duration.seconds(1));
122 expect(yield* Ref.get(retries)).toBe(2);
123
124 yield* Fiber.join(fiber);
125 }).pipe(Effect.provide(TestClock.layer())),
126 );
127});
128
74129describe("server state projection", () => {
75130 it("only treats a legacy transport interruption as an unacknowledged handoff", () => {
76131 expect(isLegacyUpdateHandoffLoss(Cause.interrupt(1))).toBe(true);
packages/client-runtime/src/state/server.ts+36-12
......@@ -149,6 +149,35 @@ export function validateServerUpdateReadyEvent(
149149 );
150150}
151151
152/**
153 * Keeps reconnect attempts ~1s apart for the whole update restart.
154 *
155 * A restart takes the server down for ~15 seconds, but the supervisor's normal
156 * backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts
157 * at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a
158 * 33-second "Resuming". Nudging on every backoff entry (not just the first)
159 * holds the retry cadence flat until the server answers again. The sleep before
160 * each nudge is the pacer: a connection that fails instantly re-enters backoff
161 * immediately and would otherwise spin a tight retry loop.
162 *
163 * Callers fork this as a child of the update command so it is interrupted as
164 * soon as the update settles, whether it succeeds, fails, or times out.
165 */
166export function nudgeReconnectDuringUpdateRestart(input: {
167 readonly stateChanges: Stream.Stream<{ readonly phase: string }, unknown>;
168 readonly retryNow: Effect.Effect<void>;
169 readonly interval?: Duration.Duration;
170}): Effect.Effect<void> {
171 return input.stateChanges.pipe(
172 Stream.filter((state) => state.phase === "backoff"),
173 Stream.runForEach(() =>
174 Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)),
175 ),
176 Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT),
177 Effect.ignore,
178 );
179}
180
152181export function serverUpdateStateForProgressEvent(
153182 fromVersion: string,
154183 targetVersion: string,
......@@ -589,18 +618,13 @@ export function createServerEnvironmentAtoms<R, E>(
589618 }),
590619 );
591620
592 // The update restart is intentional. As soon as the supervisor sees
593 // that first failed connection, discard any prior backoff debt and
594 // retry immediately instead of carrying an old 16-second delay.
595 yield* environmentRegistry.stateChanges(target.environmentId).pipe(
596 Stream.filter((state) => state.phase === "backoff"),
597 Stream.take(1),
598 Stream.runDrain,
599 Effect.andThen(environmentRegistry.retryNow(target.environmentId)),
600 Effect.timeoutOption(Duration.seconds(30)),
601 Effect.ignore,
602 Effect.forkChild,
603 );
621 // The update restart is intentional and the server stays unreachable
622 // for the whole restart, so hold the retry cadence flat instead of
623 // letting the supervisor climb its backoff ladder.
624 yield* nudgeReconnectDuringUpdateRestart({
625 stateChanges: environmentRegistry.stateChanges(target.environmentId),
626 retryNow: environmentRegistry.retryNow(target.environmentId),
627 }).pipe(Effect.forkChild);
604628
605629 const resumed = yield* environmentRegistry
606630 .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {}))