authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 02:17:30-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 02:17:30-07:00
logcd45be3b57dc34796c21e7bbad9ae55fef5a583b
tree3436178c09b6560849522bd65bc92d76d5f45a18
parent9a5dc75648de725c737069379600570e391ec8b5
signature Signed by PGP key B5690EEEBB952194

fix(server): T3 Connect no longer stays down after a network change (#19335)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

2 files changed, 203 insertions(+), 6 deletions(-)

apps/server/src/cloud/ManagedEndpointRuntime.test.ts+122-3
......@@ -3,6 +3,7 @@ import { vi } from "vite-plus/test";
33import * as Deferred from "effect/Deferred";
44import * as Duration from "effect/Duration";
55import * as Effect from "effect/Effect";
6import * as Exit from "effect/Exit";
67import * as Fiber from "effect/Fiber";
78import * as Layer from "effect/Layer";
89import * as Option from "effect/Option";
......@@ -12,6 +13,7 @@ import * as Sink from "effect/Sink";
1213import * as Stream from "effect/Stream";
1314import * as TestClock from "effect/testing/TestClock";
1415import { ChildProcess, ChildProcessSpawner } from "effect/process";
16import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
1517import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1618import * as RelayClient from "@t3tools/shared/relayClient";
1719
......@@ -33,12 +35,17 @@ const layerRelayClientAvailable = Layer.succeed(
3335 }),
3436);
3537
38// Only connectors that log a metrics address are polled for readiness.
39const unusedHttpClient = HttpClient.make(() => Effect.die("unused"));
40
3641const runtimeDependencies = (
3742 spawner: ReturnType<typeof ChildProcessSpawner.make>,
3843 relayClientLayer = layerRelayClientAvailable,
44 httpClient = unusedHttpClient,
3945) =>
4046 Layer.mergeAll(
4147 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
48 Layer.succeed(HttpClient.HttpClient, httpClient),
4249 relayClientLayer,
4350 Layer.mock(ServerSecretStore.ServerSecretStore)({
4451 get: () => Effect.succeedNone,
......@@ -48,11 +55,12 @@ const runtimeDependencies = (
4855const buildCloudManagedEndpointRuntime = (
4956 spawner: ReturnType<typeof ChildProcessSpawner.make>,
5057 relayClientLayer = layerRelayClientAvailable,
58 httpClient = unusedHttpClient,
5159) =>
5260 Effect.gen(function* () {
5361 const context = yield* Layer.build(
5462 ManagedEndpointRuntime.layer.pipe(
55 Layer.provide(runtimeDependencies(spawner, relayClientLayer)),
63 Layer.provide(runtimeDependencies(spawner, relayClientLayer, httpClient)),
5664 ),
5765 );
5866 return yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe(
......@@ -410,6 +418,97 @@ describe("CloudManagedEndpointRuntime", () => {
410418 }).pipe(Effect.provide(TestClock.layer())),
411419 );
412420
421 it.effect(
422 "restarts a running connector whose readiness check keeps reporting no connections",
423 () =>
424 Effect.gen(function* () {
425 const spawned: Array<number> = [];
426 const killed: Array<number> = [];
427 const exits: Array<Deferred.Deferred<ChildProcessSpawner.ExitCode>> = [];
428 const secondSpawned = yield* Deferred.make<void>();
429 const spawner = ChildProcessSpawner.make(() =>
430 Effect.gen(function* () {
431 const index = spawned.length;
432 const pid = 1000 + index;
433 const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
434 spawned.push(pid);
435 exits.push(exit);
436 if (index === 1) yield* Deferred.succeed(secondSpawned, undefined);
437 const handle = makeHandle({
438 pid,
439 exitCode: Deferred.await(exit),
440 onKill: () => {
441 if (!killed.includes(pid)) killed.push(pid);
442 Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143)));
443 },
444 output: Stream.concat(
445 Stream.make(
446 new TextEncoder().encode(
447 `2026-10-14T08:49:04Z INF Starting metrics server on 127.0.0.1:${40000 + index}/metrics\n`,
448 ),
449 ),
450 Stream.never,
451 ),
452 });
453 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
454 return handle;
455 }),
456 );
457 // Each readiness check takes the next scripted answer, then 503. It
458 // records which connectors were killed before it was sent.
459 const answers: Array<number | "refused"> = [];
460 const checks = yield* Queue.unbounded<{ readonly url: string; readonly killed: string }>();
461 const httpClient = HttpClient.make((request) => {
462 Queue.offerUnsafe(checks, { url: request.url, killed: killed.join(",") });
463 const status = answers.shift() ?? 503;
464 return status === "refused"
465 ? Effect.fail(
466 new HttpClientError.HttpClientError({
467 reason: new HttpClientError.TransportError({ request }),
468 }),
469 )
470 : Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status })));
471 });
472 // Advances the test clock until the watchdog checks readiness again.
473 const nextCheck = Effect.gen(function* () {
474 for (let second = 0; second < 60; second += 1) {
475 yield* TestClock.adjust(Duration.seconds(1));
476 const check = yield* Queue.poll(checks);
477 if (Option.isSome(check)) return check.value;
478 }
479 return yield* Effect.die("No readiness check within a minute");
480 });
481 const runtime = yield* buildCloudManagedEndpointRuntime(
482 spawner,
483 layerRelayClientAvailable,
484 httpClient,
485 );
486 yield* runtime.applyConfig({
487 providerKind: "cloudflare_tunnel",
488 connectorToken: "token",
489 tunnelId: "tunnel-1",
490 });
491
492 // Five 503s, a failed request that does not count, a ready answer that
493 // clears the streak, then six 503s in a row.
494 answers.push(503, 503, 503, 503, 503, "refused", 200);
495 for (let check = 0; check < 13; check += 1) {
496 expect(yield* nextCheck).toEqual({ url: "http://127.0.0.1:40000/ready", killed: "" });
497 }
498 yield* Deferred.await(secondSpawned);
499 expect(killed).toEqual([1000]);
500 expect(spawned).toEqual([1000, 1001]);
501
502 // The replacement follows a restart that never got a ready answer, so
503 // it is killed only after twelve 503s.
504 for (let check = 0; check < 12; check += 1) {
505 expect(yield* nextCheck).toEqual({ url: "http://127.0.0.1:40001/ready", killed: "1000" });
506 }
507 yield* Deferred.await(exits[1]!);
508 expect(killed).toEqual([1000, 1001]);
509 }).pipe(Effect.provide(TestClock.layer())),
510 );
511
413512 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
414513 Effect.gen(function* () {
415514 const spawned: Array<ChildProcess.StandardCommand> = [];
......@@ -457,8 +556,28 @@ describe("CloudManagedEndpointRuntime", () => {
457556
458557 expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);
459558 expect(spawned.map((command) => command.args)).toEqual([
460 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
461 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
559 [
560 "tunnel",
561 "--no-autoupdate",
562 "--loglevel",
563 "info",
564 "--output",
565 "default",
566 "--metrics",
567 "127.0.0.1:0",
568 "run",
569 ],
570 [
571 "tunnel",
572 "--no-autoupdate",
573 "--loglevel",
574 "info",
575 "--output",
576 "default",
577 "--metrics",
578 "127.0.0.1:0",
579 "run",
580 ],
462581 ]);
463582 expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([
464583 "token-1",
apps/server/src/cloud/ManagedEndpointRuntime.ts+81-3
......@@ -7,12 +7,14 @@ import * as Duration from "effect/Duration";
77import * as Effect from "effect/Effect";
88import * as Exit from "effect/Exit";
99import * as Layer from "effect/Layer";
10import * as Option from "effect/Option";
1011import * as Queue from "effect/Queue";
1112import * as Ref from "effect/Ref";
1213import * as Result from "effect/Result";
1314import * as Semaphore from "effect/Semaphore";
1415import * as Scope from "effect/Scope";
1516import * as Stream from "effect/Stream";
17import { HttpClient } from "effect/http";
1618import * as ChildProcess from "effect/process/ChildProcess";
1719import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
1820
......@@ -63,6 +65,8 @@ interface ActiveConnector {
6365 readonly startedAtMillis: number;
6466 /** Completes when the connector first registers a tunnel connection. */
6567 readonly registered: Deferred.Deferred<void>;
68 /** Completes with the loopback address of the connector's metrics server. */
69 readonly metricsAddress: Deferred.Deferred<string>;
6670}
6771
6872// A connector that exits before running this long is treated as part of a
......@@ -80,6 +84,18 @@ const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
8084// Ask for recovery after this long, and again at this interval while it stays
8185// unconnected; the relay hands back the same tunnel when it is still live.
8286const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
87// cloudflared resolves the edge only at startup, so after a network change it
88// can stay alive with zero connections, dialing addresses that no longer route.
89// Its /ready endpoint answers 503 then. After this many 503s in a row (about
90// three minutes), the connector is restarted. Each restart that is never
91// followed by a ready answer doubles the count, up to about thirty minutes, so
92// a host that is really offline does not restart every few minutes.
93const CONNECTOR_READINESS_INTERVAL = Duration.seconds(30);
94const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5);
95const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6;
96const CONNECTOR_NOT_READY_CHECKS_MAX = 60;
97// The connector binds its metrics server to a random loopback port and logs it.
98const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u;
8399// A failed background install of the pinned relay client is retried no sooner
84100// than this, so a crash-looping connector or an offline host does not redownload
85101// on every reconcile.
......@@ -162,6 +178,7 @@ const stopConnector = (connector: ActiveConnector | null) =>
162178export const make = Effect.gen(function* () {
163179 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
164180 const relayClient = yield* RelayClient.RelayClient;
181 const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
165182 const activeRef = yield* Ref.make<ActiveConnector | null>(null);
166183 const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
167184 const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
......@@ -174,6 +191,7 @@ export const make = Effect.gen(function* () {
174191 const lastInstallFailureAtRef = yield* Ref.make<number | null>(null);
175192 const prunedRef = yield* Ref.make(false);
176193 const retryLoopRunningRef = yield* Ref.make(false);
194 const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
177195 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
178196
179197 const stopActive = Effect.gen(function* () {
......@@ -393,6 +411,48 @@ export const make = Effect.gen(function* () {
393411 return false;
394412 }).pipe(Effect.repeat({ until: (connected) => connected }));
395413
414 // Kills a connector whose /ready keeps answering 503, so the supervisor
415 // restarts it like any exited connector. Only an explicit 503 counts: a failed
416 // request says nothing about the tunnel. A connector that never logs its
417 // metrics address is not polled.
418 const watchConnectorReadiness = (connector: ActiveConnector) =>
419 Effect.gen(function* () {
420 const readyUrl = `http://${yield* Deferred.await(connector.metricsAddress)}/ready`;
421 let notReadyChecks = 0;
422 while (true) {
423 yield* Effect.sleep(CONNECTOR_READINESS_INTERVAL);
424 const status = yield* httpClient.get(readyUrl).pipe(
425 Effect.map((response) => response.status),
426 Effect.scoped,
427 Effect.withTracerEnabled(false),
428 Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT),
429 Effect.orElseSucceed(() => Option.none<number>()),
430 );
431 if (Option.isNone(status)) continue;
432 if (status.value === 200) {
433 notReadyChecks = 0;
434 yield* Ref.set(notReadyChecksBeforeRestartRef, CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
435 continue;
436 }
437 if (status.value !== 503) continue;
438 notReadyChecks += 1;
439 const threshold = yield* Ref.get(notReadyChecksBeforeRestartRef);
440 if (notReadyChecks < threshold) continue;
441 notReadyChecks = 0;
442 yield* Ref.set(
443 notReadyChecksBeforeRestartRef,
444 Math.min(threshold * 2, CONNECTOR_NOT_READY_CHECKS_MAX),
445 );
446 yield* Effect.logWarning("Relay client has no tunnel connection; restarting it", {
447 pid: Number(connector.child.pid),
448 notReadyChecks: threshold,
449 tunnelId: connector.config.tunnelId,
450 tunnelName: connector.config.tunnelName,
451 });
452 yield* connector.child.kill().pipe(Effect.ignore);
453 }
454 });
455
396456 const observeConnectorOutput = (connector: ActiveConnector) => {
397457 let rejectedRegistrations = 0;
398458
......@@ -443,8 +503,14 @@ export const make = Effect.gen(function* () {
443503 }
444504 }
445505 return Effect.logWarning("Relay client reported a transport warning", attributes);
446 case "debug":
447 return Effect.logDebug("Relay client output", attributes);
506 case "debug": {
507 const metricsAddress = RELAY_CLIENT_METRICS_LINE.exec(line)?.[1];
508 return metricsAddress
509 ? Deferred.succeed(connector.metricsAddress, metricsAddress).pipe(
510 Effect.andThen(Effect.logDebug("Relay client output", attributes)),
511 )
512 : Effect.logDebug("Relay client output", attributes);
513 }
448514 }
449515 }),
450516 Effect.catchCause((cause) =>
......@@ -507,7 +573,17 @@ export const make = Effect.gen(function* () {
507573 .spawn(
508574 ChildProcess.make(
509575 executable.executablePath,
510 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
576 [
577 "tunnel",
578 "--no-autoupdate",
579 "--loglevel",
580 "info",
581 "--output",
582 "default",
583 "--metrics",
584 "127.0.0.1:0",
585 "run",
586 ],
511587 {
512588 detached: false,
513589 env: {
......@@ -563,11 +639,13 @@ export const make = Effect.gen(function* () {
563639 config,
564640 startedAtMillis: yield* Clock.currentTimeMillis,
565641 registered: yield* Deferred.make<void>(),
642 metricsAddress: yield* Deferred.make<string>(),
566643 } satisfies ActiveConnector;
567644 yield* Ref.set(activeRef, connector);
568645 yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);
569646 yield* Effect.forkIn(superviseConnector(connector), connectorScope);
570647 yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope);
648 yield* Effect.forkIn(watchConnectorReadiness(connector), connectorScope);
571649 return {
572650 status: "running",
573651 providerKind: "cloudflare_tunnel",