1import * as NodeNet from "node:net";
2
3import { assert, describe, it } from "@effect/vitest";
4import * as NodeServices from "@effect/platform-node/NodeServices";
5import * as NetService from "@t3tools/shared/Net";
6import * as HostProcess from "@t3tools/shared/HostProcess";
7import * as Deferred from "effect/Deferred";
8import * as Duration from "effect/Duration";
9import * as Effect from "effect/Effect";
10import * as Fiber from "effect/Fiber";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as Option from "effect/Option";
14import * as Result from "effect/Result";
15import * as Sink from "effect/Sink";
16import * as Stream from "effect/Stream";
17import { TestClock } from "effect/testing";
18import { HttpClient, HttpClientResponse } from "effect/http";
19import { ChildProcess, ChildProcessSpawner } from "effect/process";
20
21import * as SshAuth from "./auth.ts";
22import { SshCommandError } from "./errors.ts";
23import * as SshTunnel from "./tunnel.ts";
24
25const TEST_NODE_ENGINE_RANGE = "^22.16 || ^23.11 || >=24.10";
26
27const makeSuccessfulProcess = (stdout: string) => {
28 const stdoutStream = Stream.make(new TextEncoder().encode(stdout));
29 return ChildProcessSpawner.makeHandle({
30 pid: ChildProcessSpawner.ProcessId(123),
31 stdout: stdoutStream,
32 stderr: Stream.empty,
33 all: stdoutStream,
34 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)),
35 isRunning: Effect.succeed(false),
36 kill: () => Effect.void,
37 stdin: Sink.drain,
38 getInputFd: () => Sink.drain,
39 getOutputFd: () => Stream.empty,
40 unref: Effect.succeed(Effect.void),
41 });
42};
43
44const makeDelayedSuccessfulProcess = (stdout: string, delayMs: number) => {
45 const process = makeSuccessfulProcess(stdout);
46 return {
47 ...process,
48 exitCode: Effect.sleep(Duration.millis(delayMs)).pipe(
49 Effect.as(ChildProcessSpawner.ExitCode(0)),
50 ),
51 };
52};
53
54const makeRunningProcess = (onKill: () => void) => {
55 let finish: ((exitCode: ChildProcessSpawner.ExitCode) => void) | null = null;
56 return ChildProcessSpawner.makeHandle({
57 pid: ChildProcessSpawner.ProcessId(123),
58 stdout: Stream.empty,
59 stderr: Stream.empty,
60 all: Stream.empty,
61 exitCode: Effect.callback<ChildProcessSpawner.ExitCode>((resume) => {
62 finish = (exitCode) => resume(Effect.succeed(exitCode));
63 return Effect.sync(() => {
64 finish = null;
65 });
66 }),
67 isRunning: Effect.succeed(true),
68 kill: () =>
69 Effect.sync(() => {
70 onKill();
71 finish?.(ChildProcessSpawner.ExitCode(143));
72 }),
73 stdin: Sink.drain,
74 getInputFd: () => Sink.drain,
75 getOutputFd: () => Stream.empty,
76 unref: Effect.succeed(Effect.void),
77 });
78};
79
80const testHttpClient = HttpClient.make((request) =>
81 Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))),
82);
83
84const hangingHttpClient = HttpClient.make(() => Effect.never);
85
86const testNetService = NetService.NetService.of({
87 canListenOnHost: () => Effect.succeed(true),
88 isPortAvailableOnLoopback: () => Effect.succeed(true),
89 hasListenerOnHost: () => Effect.succeed(false),
90 reserveLoopbackPort: () => Effect.succeed(41_773),
91 findAvailablePort: (preferred) => Effect.succeed(preferred),
92});
93
94function commandArgs(command: ChildProcess.Command): ReadonlyArray<string> {
95 return command._tag === "StandardCommand" ? command.args : [];
96}
97
98const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
99const NODE_SCRIPT = {
100 nodeScriptPath: "/Users/julius/Development/Work/codething-mvp/apps/server/dist/bin.mjs",
101} as const;
102
103describe("ssh tunnel scripts", () => {
104 it("installs and runs the release archive without Node, npm, or npx", () => {
105 const script = SshTunnel.buildRemoteT3RunnerScript(ARCHIVE);
106
107 assert.include(script, "T3_ARCHIVE_VERSION='1.2.3-preview.20260911.4'");
108 assert.include(script, "T3_NODE_SCRIPT_PATH=''");
109 assert.include(
110 script,
111 "T3_RELEASE_BASE_URL='https://github.com/pingdotgg/t3code/releases/download'",
112 );
113 assert.include(script, 'T3_RUNTIME_DIR="$HOME/.t3/runtime/versions/$T3_ARCHIVE_VERSION"');
114 assert.include(script, 'T3_ARCHIVE="t3-$T3_ARCHIVE_VERSION-$T3_PLATFORM-$T3_ARCH.tar.gz"');
115 assert.include(script, "SHA256SUMS");
116 assert.include(script, 'exec "$T3_RUNTIME_DIR/t3" "$@"');
117 assert.notInclude(script, "npx");
118 assert.notInclude(script, "npm exec");
119 assert.notInclude(script, "t3@latest");
120 assert.notInclude(script, 'exec t3 "$@"');
121 // Concurrent launches serialize on a per-version mkdir lock and recheck
122 // the completion marker after acquiring it.
123 assert.include(
124 script,
125 'T3_LOCK="$HOME/.t3/runtime/versions/.$T3_ARCHIVE_VERSION.install.lock"',
126 );
127 // mkdir is the exclusive create; the pid follows atomically. A dead owner
128 // is reclaimed at once, a never-published owner after a short grace.
129 assert.include(script, 'while ! mkdir "$T3_LOCK" 2>/dev/null; do');
130 assert.include(script, 'mv "$T3_LOCK/pid.tmp" "$T3_LOCK/pid"');
131 assert.include(script, 'if ! kill -0 "$T3_LOCK_OWNER" 2>/dev/null; then');
132 assert.include(script, 'if [ "$T3_LOCK_UNOWNED" -ge 5 ]; then');
133 assert.include(script, 'if [ "$T3_LOCK_WAITED" -ge 360 ]; then');
134 assert.include(script, '"$T3_STAGING/SHA256SUMS" 30');
135 assert.include(script, '"$T3_STAGING/$T3_ARCHIVE" 240');
136 assert.notInclude(script, "T3_LOCK_CANDIDATE");
137 assert.notInclude(script, "-mmin");
138 assert.equal(script.split("if ! t3_runtime_ready; then").length - 1, 2);
139 assert.isBelow(
140 script.indexOf('"$T3_STAGING/t3" --version'),
141 script.indexOf('> "$T3_STAGING/.install-complete"'),
142 );
143 // Node discovery is defined for the dev path but only ever invoked inside
144 // the node-script branch, which the archive path skips entirely.
145 assert.equal(script.split("ensure_remote_node_path || true").length - 1, 1);
146 assert.isBelow(
147 script.indexOf("ensure_remote_node_path || true"),
148 script.indexOf('exec node "$T3_NODE_SCRIPT_PATH" "$@"'),
149 );
150 assert.isBelow(
151 script.indexOf('exec node "$T3_NODE_SCRIPT_PATH" "$@"'),
152 script.indexOf("T3_ARCHIVE_VERSION="),
153 );
154
155 const launch = SshTunnel.buildRemoteLaunchScript({
156 ...ARCHIVE,
157 releaseBaseUrl: "https://mirror.example/t3/",
158 });
159 assert.include(launch, "T3_ARCHIVE_MODE=1");
160 assert.include(launch, "T3_RELEASE_BASE_URL='https://mirror.example/t3'");
161 assert.include(launch, '"$RUNNER_FILE" __ssh-helper pick-port "$PORT_FILE"');
162 assert.include(launch, '"$RUNNER_FILE" __ssh-helper wait-ready "$REMOTE_PORT"');
163 assert.include(launch, '"$RUNNER_FILE" __ssh-helper runtime-port "$DEFAULT_RUNTIME_FILE"');
164 assert.include(SshTunnel.buildRemoteLaunchScript(NODE_SCRIPT), "T3_ARCHIVE_MODE=0");
165 });
166
167 it("rejects archive versions that are not a single exact version segment", () => {
168 for (const archiveVersion of [
169 "../other",
170 "1.2.3/evil",
171 "1.2.3\\evil",
172 "1.2.3-preview.1 x",
173 "1.2.3-preview.1\nrm -rf /",
174 "v1.2.3",
175 ]) {
176 assert.throws(
177 () => SshTunnel.buildRemoteT3RunnerScript({ archiveVersion }),
178 SshTunnel.SshInvalidArchiveVersionError,
179 undefined,
180 archiveVersion,
181 );
182 }
183 assert.include(
184 SshTunnel.buildRemoteT3RunnerScript(ARCHIVE),
185 "T3_ARCHIVE_VERSION='1.2.3-preview.20260911.4'",
186 );
187 });
188
189 it("refuses to build a runner with neither an archive version nor a node script", () => {
190 for (const input of [undefined, {}, { archiveVersion: " " }, { nodeScriptPath: null }]) {
191 assert.throws(
192 () => SshTunnel.buildRemoteT3RunnerScript(input),
193 SshTunnel.SshMissingRunnerError,
194 );
195 }
196 assert.throws(() => SshTunnel.buildRemoteLaunchScript(), SshTunnel.SshMissingRunnerError);
197 });
198
199 it("does not hard-code a remote node engine range", () => {
200 const script = SshTunnel.buildRemoteT3RunnerScript(NODE_SCRIPT);
201
202 assert.include(script, "T3_NODE_ENGINE_RANGE=''");
203 assert.notInclude(script, TEST_NODE_ENGINE_RANGE);
204 });
205
206 it("builds the remote t3 runner with a node script override", () => {
207 const script = SshTunnel.buildRemoteT3RunnerScript({
208 ...NODE_SCRIPT,
209 nodeEngineRange: TEST_NODE_ENGINE_RANGE,
210 });
211
212 assert.include(
213 script,
214 "T3_NODE_SCRIPT_PATH='/Users/julius/Development/Work/codething-mvp/apps/server/dist/bin.mjs'",
215 );
216 assert.include(script, 'exec node "$T3_NODE_SCRIPT_PATH" "$@"');
217 assert.include(script, "T3_ARCHIVE_VERSION=''");
218 assert.include(script, 'prepend_path_if_dir "$HOME/.local/bin"');
219 assert.include(script, `T3_NODE_ENGINE_RANGE='${TEST_NODE_ENGINE_RANGE}'`);
220 assert.include(script, "remote_node_satisfies_engine()");
221 assert.include(script, "function satisfiesSemverRange");
222 assert.include(script, "satisfiesSemverRange(rawVersion, range)");
223 assert.include(script, 'prepend_path_if_dir "/home/linuxbrew/.linuxbrew/bin"');
224 assert.include(script, 'prepend_path_if_dir "$VOLTA_HOME/bin"');
225 assert.include(script, 'prepend_path_if_dir "$HOME/.asdf/shims"');
226 assert.include(script, 'prepend_path_if_dir "$HOME/.local/share/mise/shims"');
227 assert.include(script, 'eval "$(fnm env --shell bash)"');
228 assert.include(script, "fnm use --silent-if-unchanged");
229 assert.include(script, "fnm use default");
230 assert.include(script, 'prepend_path_if_dir "$HOME/.nodenv/shims"');
231 assert.include(script, 'NVM_DIR="$HOME/.nvm"');
232 assert.include(script, "nvm use --silent default");
233 assert.include(script, 'for T3_NODE_BIN in "$NVM_DIR"/versions/node/*/bin');
234 assert.notInclude(script, "ensure $NVM_DIR/nvm.sh is available");
235 assert.notInclude(script, "npx");
236 });
237
238 it("embeds the runner in the launch script byte for byte", () => {
239 const runner = SshTunnel.buildRemoteT3RunnerScript(ARCHIVE);
240 assert.include(runner, 'printf \'%s\\n\' "$$" > "$T3_LOCK/pid.tmp"');
241 assert.include(SshTunnel.buildRemoteLaunchScript(ARCHIVE), runner);
242 });
243
244 it("uses the remote t3 runner for launch and pairing scripts", () => {
245 const stateKey = "711bc738002d72fd";
246 const launch = SshTunnel.buildRemoteLaunchScript(ARCHIVE);
247 const devLaunch = SshTunnel.buildRemoteLaunchScript({
248 ...NODE_SCRIPT,
249 nodeEngineRange: TEST_NODE_ENGINE_RANGE,
250 });
251
252 assert.include(
253 launch,
254 '[ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && server_running "$REMOTE_PID"',
255 );
256 assert.include(launch, "RUNNER_CHANGED=1");
257 assert.include(launch, "ensure_remote_node_path()");
258 assert.include(launch, "if ! ensure_remote_node_path; then");
259 assert.include(devLaunch, `T3_NODE_ENGINE_RANGE='${TEST_NODE_ENGINE_RANGE}'`);
260 assert.include(devLaunch, "does not satisfy required range ");
261 assert.include(launch, 'kill "$REMOTE_PID" 2>/dev/null || true');
262 assert.include(launch, "wait_ready");
263 assert.include(launch, '"$RUNNER_FILE" serve --host 127.0.0.1');
264 assert.include(launch, '--base-dir "$DEFAULT_SERVER_HOME"');
265 assert.notInclude(launch, "server-home");
266 assert.include(launch, "Remote T3 server did not become ready");
267 assert.include(launch, 'wait_ready "60000"');
268 assert.include(launch, 'if [ -s "$LOG_FILE" ]; then');
269 assert.include(launch, "It wrote nothing to %s");
270 assert.include(launch, "T3_ARCHIVE_VERSION='1.2.3-preview.20260911.4'");
271 assert.include(
272 SshTunnel.buildRemotePairingScript(stateKey, ARCHIVE),
273 '"$RUNNER_FILE" auth pairing create --base-dir "$PAIRING_BASE_DIR" --json',
274 );
275 assert.include(
276 SshTunnel.buildRemotePairingScript(stateKey, ARCHIVE),
277 'PAIRING_BASE_DIR="$DEFAULT_SERVER_HOME"',
278 );
279 assert.notInclude(SshTunnel.buildRemotePairingScript(stateKey, ARCHIVE), "server-home");
280 assert.include(
281 SshTunnel.buildRemotePairingScript(stateKey, ARCHIVE),
282 "T3_ARCHIVE_VERSION='1.2.3-preview.20260911.4'",
283 );
284 assert.include(
285 SshTunnel.buildRemoteStopScript(stateKey),
286 'if [ "$REMOTE_MANAGED" != "external" ] && [ -n "$REMOTE_PID" ]',
287 );
288 assert.include(
289 SshTunnel.buildRemoteStopScript(stateKey),
290 'kill "$REMOTE_PID" 2>/dev/null || true',
291 );
292 assert.include(
293 SshTunnel.buildRemoteStopScript(stateKey),
294 'rm -f "$PID_FILE" "$PORT_FILE" "$MANAGED_FILE"',
295 );
296 assert.include(
297 launch,
298 'DEFAULT_RUNTIME_FILE="$DEFAULT_SERVER_HOME/userdata/server-runtime.json"',
299 );
300 assert.include(launch, "resolve_default_runtime_port()");
301 assert.include(launch, 'DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port');
302 assert.include(launch, "if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(port))");
303 assert.include(
304 launch,
305 'if [ -n "$DEFAULT_REMOTE_PORT" ] && [ "$REMOTE_MANAGED" = "managed" ] && [ "$DEFAULT_RUNTIME_PID" = "$REMOTE_PID" ]; then',
306 );
307 assert.include(launch, 'REMOTE_PORT="$DEFAULT_REMOTE_PORT"');
308 assert.include(launch, 'rm -f "$PID_FILE"');
309 assert.include(launch, "printf 'external\\n' >\"$MANAGED_FILE\"");
310 assert.include(launch, 'if [ -z "$REMOTE_PORT" ]; then');
311 assert.isBelow(
312 launch.indexOf('if [ "$REMOTE_MANAGED" = "managed" ]'),
313 launch.indexOf("printf 'external\\n' >\"$MANAGED_FILE\""),
314 );
315 assert.isBelow(
316 launch.indexOf('DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port'),
317 launch.indexOf('elif [ -n "$REMOTE_PID" ]'),
318 );
319 });
320
321 it.effect("accepts launch JSON after remote shell startup noise", () => {
322 const target = {
323 alias: "devbox",
324 hostname: "devbox.example.com",
325 username: "julius",
326 port: 2222,
327 } as const;
328 const spawnedCommands: Array<ReadonlyArray<string>> = [];
329 const spawner = ChildProcessSpawner.make((command) =>
330 Effect.sync(() => {
331 spawnedCommands.push(commandArgs(command));
332 return makeSuccessfulProcess('loaded nvm default\n{"remotePort":3774}\n');
333 }),
334 );
335 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
336 const layerProcess = Layer.merge(NodeServices.layer, layerSpawner);
337
338 return Effect.gen(function* () {
339 const result = yield* SshTunnel.launchOrReuseRemoteServer(target, undefined, ARCHIVE);
340 assert.equal(result.remotePort, 3774);
341 assert.deepEqual(spawnedCommands[0]?.slice(-5, -1), ["sh", "-l", "-s", "--"]);
342 }).pipe(Effect.provide(layerProcess));
343 });
344
345 it.effect("allows cold remote launches to exceed the default SSH command timeout", () => {
346 const target = {
347 alias: "devbox",
348 hostname: "devbox.example.com",
349 username: "julius",
350 port: 2222,
351 } as const;
352 const spawner = ChildProcessSpawner.make(() =>
353 Effect.succeed(makeDelayedSuccessfulProcess('{"remotePort":3774}\n', 75_000)),
354 );
355 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
356 const layerProcess = Layer.mergeAll(NodeServices.layer, layerSpawner, TestClock.layer());
357
358 return Effect.gen(function* () {
359 const fiber = yield* Effect.forkChild(
360 SshTunnel.launchOrReuseRemoteServer(target, undefined, NODE_SCRIPT),
361 );
362 yield* Effect.yieldNow;
363 yield* TestClock.adjust(Duration.seconds(75));
364
365 const result = yield* Fiber.join(fiber);
366 assert.equal(result.remotePort, 3774);
367 }).pipe(Effect.provide(layerProcess));
368 });
369
370 it.effect("gives cold archive launches a larger budget than node-script launches", () => {
371 const target = {
372 alias: "devbox",
373 hostname: "devbox.example.com",
374 username: "julius",
375 port: 2222,
376 } as const;
377 const spawner = ChildProcessSpawner.make(() =>
378 Effect.succeed(makeDelayedSuccessfulProcess('{"remotePort":3774}\n', 800_000)),
379 );
380 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
381 const layerProcess = Layer.mergeAll(NodeServices.layer, layerSpawner, TestClock.layer());
382
383 return Effect.gen(function* () {
384 const fiber = yield* Effect.forkChild(
385 SshTunnel.launchOrReuseRemoteServer(target, undefined, ARCHIVE),
386 );
387 yield* Effect.yieldNow;
388 yield* TestClock.adjust(Duration.seconds(800));
389
390 const result = yield* Fiber.join(fiber);
391 assert.equal(result.remotePort, 3774);
392 }).pipe(Effect.provide(layerProcess));
393 });
394
395 it("allows the remote port picker to run without a state file path", () => {
396 assert.include(SshTunnel.REMOTE_PICK_PORT_SCRIPT, 'const filePath = process.argv[2] ?? "";');
397 });
398
399 it.effect("bounds each HTTP readiness probe so retries cannot hang on one request", () =>
400 Effect.gen(function* () {
401 const fiber = yield* Effect.forkChild(
402 Effect.result(
403 SshTunnel.waitForHttpReady({
404 baseUrl: "http://127.0.0.1:41773/",
405 timeoutMs: 1_000,
406 intervalMs: 100,
407 probeTimeoutMs: 250,
408 }),
409 ),
410 );
411 yield* Effect.yieldNow;
412 yield* TestClock.adjust(Duration.millis(1_000));
413
414 const result = yield* Fiber.join(fiber);
415
416 assert.isTrue(Result.isFailure(result));
417 if (Result.isFailure(result)) {
418 assert.include(result.failure.message, "Timed out waiting 1000ms");
419 }
420 }).pipe(
421 Effect.provide(
422 Layer.merge(TestClock.layer(), Layer.succeed(HttpClient.HttpClient, hangingHttpClient)),
423 ),
424 ),
425 );
426
427 it("preserves primitive readiness reason values in diagnostic output", () => {
428 assert.deepEqual(
429 SshTunnel.describeReadinessCause({
430 _tag: "HttpClientError",
431 message: "Backend readiness probe failed.",
432 reason: "authentication failed",
433 cause: "upstream closed",
434 }),
435 {
436 _tag: "HttpClientError",
437 message: "Backend readiness probe failed.",
438 reason: "authentication failed",
439 cause: "upstream closed",
440 },
441 );
442 });
443
444 it.effect("accepts pretty-printed pairing JSON from the remote CLI", () => {
445 const target = {
446 alias: "devbox",
447 hostname: "devbox.example.com",
448 username: "julius",
449 port: 2222,
450 } as const;
451 const spawner = ChildProcessSpawner.make(() =>
452 Effect.succeed(
453 makeSuccessfulProcess(`{
454 "id": "88941235-6ed5-4184-a2ff-5339e2075958",
455 "credential": "LCL4R2TPHDKQ",
456 "scopes": ["orchestration:read"],
457 "expiresAt": "2026-04-29T01:01:20.994Z"
458}
459
460`),
461 ),
462 );
463 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
464 const layerProcess = Layer.merge(NodeServices.layer, layerSpawner);
465 return Effect.gen(function* () {
466 const result = yield* SshTunnel.issueRemotePairingToken(target, undefined, ARCHIVE);
467 assert.equal(result.credential, "LCL4R2TPHDKQ");
468 }).pipe(Effect.provide(layerProcess));
469 });
470
471 it.effect("accepts pretty-printed pairing JSON after remote shell startup noise", () => {
472 const target = {
473 alias: "devbox",
474 hostname: "devbox.example.com",
475 username: "julius",
476 port: 2222,
477 } as const;
478 const spawner = ChildProcessSpawner.make(() =>
479 Effect.succeed(
480 makeSuccessfulProcess(`loaded nvm default
481{
482 "id": "88941235-6ed5-4184-a2ff-5339e2075958",
483 "credential": "LCL4R2TPHDKQ",
484 "scopes": ["orchestration:read"],
485 "expiresAt": "2026-04-29T01:01:20.994Z"
486}
487
488`),
489 ),
490 );
491 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
492 const layerProcess = Layer.merge(NodeServices.layer, layerSpawner);
493 return Effect.gen(function* () {
494 const result = yield* SshTunnel.issueRemotePairingToken(target, undefined, ARCHIVE);
495 assert.equal(result.credential, "LCL4R2TPHDKQ");
496 }).pipe(Effect.provide(layerProcess));
497 });
498
499 it.effect("replaces a stale SSH forward without stopping the remote server", () =>
500 Effect.gen(function* () {
501 const probeStarted = yield* Deferred.make<void>();
502 let stale = false;
503 let forwards = 0;
504 let stops = 0;
505 let kills = 0;
506 const spawner = ChildProcessSpawner.make((command) =>
507 Effect.sync(() => {
508 const args = commandArgs(command);
509 if (args.includes("-N")) {
510 forwards += 1;
511 stale = false;
512 return makeRunningProcess(() => {
513 kills += 1;
514 });
515 }
516 if (args.includes("--")) return makeSuccessfulProcess('{"remotePort":3773}\n');
517 if (args.includes("sh")) stops += 1;
518 return makeSuccessfulProcess("");
519 }),
520 );
521 const httpClient = HttpClient.make((request) =>
522 stale
523 ? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never))
524 : Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))),
525 );
526 const layer = Layer.mergeAll(
527 NodeServices.layer,
528 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
529 Layer.succeed(HttpClient.HttpClient, httpClient),
530 Layer.succeed(NetService.NetService, testNetService),
531 SshAuth.SshPasswordPrompt.disabledLayer,
532 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
533 );
534 yield* Effect.gen(function* () {
535 const manager = yield* SshTunnel.SshEnvironmentManager;
536 const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
537 yield* manager.ensureEnvironment(target);
538 stale = true;
539 const reconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
540 yield* Deferred.await(probeStarted);
541 yield* TestClock.adjust("2 seconds");
542 yield* Fiber.join(reconnect);
543 assert.equal(forwards, 2);
544 assert.equal(kills, 1);
545 assert.equal(stops, 0);
546 yield* manager.disconnectEnvironment(target);
547 assert.equal(stops, 1);
548 }).pipe(Effect.provide(layer), Effect.scoped);
549 }),
550 );
551
552 it.effect.each(["successful stop", "failed stop"] as const)(
553 "closes the tunnel scope and starts fresh after a %s",
554 (mode) => {
555 const spawnedCommands: Array<ReadonlyArray<string>> = [];
556 let tunnelKillCount = 0;
557 let stopCommandCount = 0;
558 const spawner = ChildProcessSpawner.make((command) =>
559 Effect.sync(() => {
560 const args = commandArgs(command);
561 spawnedCommands.push(args);
562 if (args.includes("-N")) {
563 return makeRunningProcess(() => {
564 tunnelKillCount += 1;
565 });
566 }
567 if (args.includes("sh") && args.includes("--")) {
568 return makeSuccessfulProcess('{"remotePort":3773}\n');
569 }
570 if (args.includes("sh")) {
571 stopCommandCount += 1;
572 if (mode === "failed stop" && stopCommandCount === 1) {
573 return {
574 ...makeSuccessfulProcess(""),
575 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)),
576 stderr: Stream.make(
577 new TextEncoder().encode("Remote T3 server did not stop within 2 seconds.\n"),
578 ),
579 };
580 }
581 return makeSuccessfulProcess('{"stopped":true}\n');
582 }
583 return makeSuccessfulProcess("\n");
584 }),
585 );
586 const layer = Layer.mergeAll(
587 NodeServices.layer,
588 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
589 Layer.succeed(HttpClient.HttpClient, testHttpClient),
590 Layer.succeed(NetService.NetService, testNetService),
591 SshAuth.SshPasswordPrompt.disabledLayer,
592 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
593 );
594 const target = {
595 alias: "devbox",
596 hostname: "devbox.example.com",
597 username: "julius",
598 port: 2222,
599 } as const;
600
601 return Effect.gen(function* () {
602 const manager = yield* SshTunnel.SshEnvironmentManager;
603
604 const first = yield* manager.ensureEnvironment(target);
605 assert.equal(first.httpBaseUrl, "http://127.0.0.1:41773/");
606 const firstTunnelArgs = spawnedCommands.find((args) => args.includes("-N"));
607 assert.isDefined(firstTunnelArgs);
608 assert.include(firstTunnelArgs, "ControlMaster=no");
609 assert.include(firstTunnelArgs, "ControlPath=none");
610 assert.include(firstTunnelArgs, "ControlPersist=no");
611
612 const disconnected = yield* Effect.result(manager.disconnectEnvironment(target));
613 if (mode === "failed stop") {
614 assert.isTrue(Result.isFailure(disconnected));
615 if (Result.isFailure(disconnected)) {
616 assert.instanceOf(disconnected.failure, SshCommandError);
617 assert.equal(
618 disconnected.failure.message,
619 "Remote T3 server did not stop within 2 seconds.",
620 );
621 }
622 } else {
623 assert.isTrue(Result.isSuccess(disconnected));
624 }
625 assert.equal(tunnelKillCount, 1);
626 assert.equal(stopCommandCount, 1);
627
628 if (mode === "failed stop") {
629 yield* manager.disconnectEnvironment(target);
630 assert.equal(tunnelKillCount, 1);
631 assert.equal(stopCommandCount, 2);
632 }
633
634 yield* manager.ensureEnvironment(target);
635
636 assert.equal(spawnedCommands.filter((args) => args.includes("-N")).length, 2);
637 assert.equal(tunnelKillCount, 1);
638 }).pipe(
639 Effect.provide(layer),
640 Effect.scoped,
641 Effect.andThen(
642 Effect.sync(() => {
643 assert.equal(tunnelKillCount, 2);
644 assert.equal(stopCommandCount, mode === "failed stop" ? 3 : 2);
645 }),
646 ),
647 );
648 },
649 );
650
651 it.effect.each(["local tunnel", "remote server"] as const)(
652 "waits for %s shutdown before reconnecting the same target",
653 (stalledStep) =>
654 Effect.gen(function* () {
655 const shutdownStarted = yield* Deferred.make<void>();
656 const finishShutdown = yield* Deferred.make<void>();
657 const reconnectsStarted = yield* Deferred.make<void>();
658 const pauseShutdown = Deferred.succeed(shutdownStarted, undefined).pipe(
659 Effect.andThen(Deferred.await(finishShutdown)),
660 );
661 let resolutions = 0;
662 let launches = 0;
663 let tunnels = 0;
664 let stops = 0;
665 let remoteRunning = false;
666 const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
667 const spawner = ChildProcessSpawner.make((command) =>
668 Effect.gen(function* () {
669 const args = commandArgs(command);
670 const isTarget = args.includes(target.alias);
671 if (args.includes("-G")) {
672 if (isTarget && ++resolutions === 4) {
673 yield* Deferred.succeed(reconnectsStarted, undefined);
674 }
675 return makeSuccessfulProcess("");
676 }
677 if (args.includes("-N")) {
678 const tunnel = makeRunningProcess(() => undefined);
679 if (isTarget && ++tunnels === 1 && stalledStep === "local tunnel") {
680 return {
681 ...tunnel,
682 kill: (options?: ChildProcess.KillOptions) =>
683 pauseShutdown.pipe(Effect.andThen(tunnel.kill(options))),
684 };
685 }
686 return tunnel;
687 }
688 if (args.includes("--")) {
689 if (isTarget) {
690 launches += 1;
691 remoteRunning = true;
692 }
693 return makeSuccessfulProcess('{"remotePort":3773}\n');
694 }
695 const stop = makeSuccessfulProcess('{"stopped":true}\n');
696 if (!isTarget) return stop;
697 const pause = ++stops === 1 && stalledStep === "remote server";
698 return {
699 ...stop,
700 exitCode: (pause ? pauseShutdown : Effect.void).pipe(
701 Effect.andThen(
702 Effect.sync(() => {
703 remoteRunning = false;
704 return ChildProcessSpawner.ExitCode(0);
705 }),
706 ),
707 ),
708 };
709 }),
710 );
711 const layer = Layer.mergeAll(
712 NodeServices.layer,
713 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
714 Layer.succeed(HttpClient.HttpClient, testHttpClient),
715 Layer.succeed(NetService.NetService, testNetService),
716 SshAuth.SshPasswordPrompt.disabledLayer,
717 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
718 );
719 yield* Effect.gen(function* () {
720 const manager = yield* SshTunnel.SshEnvironmentManager;
721 yield* manager.ensureEnvironment(target);
722 const disconnect = yield* Effect.forkChild(manager.disconnectEnvironment(target));
723 yield* Deferred.await(shutdownStarted);
724 const firstReconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
725 const secondReconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
726 yield* Deferred.await(reconnectsStarted);
727
728 yield* manager.ensureEnvironment({
729 alias: "other",
730 hostname: "other",
731 username: null,
732 port: null,
733 });
734 yield* TestClock.adjust(Duration.zero);
735 const launchesBeforeShutdown = launches;
736 yield* Deferred.succeed(finishShutdown, undefined);
737 yield* Fiber.join(disconnect);
738 const first = yield* Fiber.join(firstReconnect);
739 const second = yield* Fiber.join(secondReconnect);
740
741 assert.equal(launchesBeforeShutdown, 1);
742 assert.equal(launches, 2);
743 assert.equal(tunnels, 2);
744 assert.isTrue(remoteRunning);
745 assert.equal(first.httpBaseUrl, second.httpBaseUrl);
746 }).pipe(
747 Effect.ensuring(Deferred.succeed(finishShutdown, undefined)),
748 Effect.provide(layer),
749 Effect.scoped,
750 );
751 }),
752 );
753});
754
755// On macOS, Node can bind 127.0.0.1:<port> while another process (such as the
756// desktop app with network access on) listens on *:<port>. Linux refuses that
757// bind unless both sockets set SO_REUSEPORT, so on Linux the preload adds it to
758// the picker's probe and the holder sets it too. macOS needs neither.
759const MACOS_BIND_PRELOAD = `const net = require("node:net");
760const listen = net.Server.prototype.listen;
761net.Server.prototype.listen = function (port, host, callback) {
762 if (typeof port !== "number" || typeof host !== "string") return listen.apply(this, arguments);
763 return listen.call(this, { port, host, reusePort: true }, callback);
764};
765`;
766
767describe("remote port picker", () => {
768 const hostPlatform = HostProcess.Platform.defaultValue();
769 const linuxHost = hostPlatform === "linux";
770 const windowsHost = hostPlatform === "win32";
771
772 const holdWildcardPort = Effect.acquireRelease(
773 Effect.callback<NodeNet.Server>((resume) => {
774 const server = NodeNet.createServer((socket) => socket.destroy());
775 server.once("error", (error) => resume(Effect.die(error)));
776 server.listen({ port: 0, host: "0.0.0.0", reusePort: linuxHost }, () =>
777 resume(Effect.succeed(server)),
778 );
779 }),
780 (server) =>
781 Effect.callback<void>((resume) => {
782 server.close(() => resume(Effect.void));
783 }),
784 );
785
786 it.effect.skipIf(windowsHost)("skips a port another server answers on loopback", () =>
787 Effect.gen(function* () {
788 const fs = yield* FileSystem.FileSystem;
789 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
790 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pick-port-" });
791 yield* fs.writeFileString(`${root}/preload.cjs`, MACOS_BIND_PRELOAD);
792 yield* fs.writeFileString(`${root}/pick-port.cjs`, SshTunnel.REMOTE_PICK_PORT_SCRIPT);
793 const holder = yield* holdWildcardPort;
794 const heldPort = (holder.address() as NodeNet.AddressInfo).port;
795
796 const child = yield* spawner.spawn(
797 ChildProcess.make(process.execPath, [
798 ...(linuxHost ? ["--require", `${root}/preload.cjs`] : []),
799 `${root}/pick-port.cjs`,
800 "",
801 String(heldPort),
802 "20",
803 ]),
804 );
805 const [stdout, exitCode] = yield* Effect.all(
806 [
807 child.stdout.pipe(
808 Stream.decodeText(),
809 Stream.runFold(
810 () => "",
811 (acc, chunk) => acc + chunk,
812 ),
813 ),
814 child.exitCode.pipe(Effect.map(Number)),
815 ],
816 { concurrency: "unbounded" },
817 );
818
819 assert.equal(exitCode, 0);
820 const picked = Number(stdout);
821 assert.isAbove(picked, heldPort);
822 assert.isBelow(picked, heldPort + 20);
823 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
824 );
825});
826
827// Runs a generated script with an isolated HOME, as an SSH remote would.
828const runScript = (home: string, args: ReadonlyArray<string>) =>
829 Effect.gen(function* () {
830 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
831 const child = yield* spawner.spawn(
832 ChildProcess.make("sh", args, {
833 env: { PATH: process.env.PATH ?? "", HOME: home },
834 extendEnv: false,
835 }),
836 );
837 const [stdout, stderr, exitCode] = yield* Effect.all(
838 [
839 child.stdout.pipe(
840 Stream.decodeText(),
841 Stream.runFold(
842 () => "",
843 (acc, chunk) => acc + chunk,
844 ),
845 ),
846 child.stderr.pipe(
847 Stream.decodeText(),
848 Stream.runFold(
849 () => "",
850 (acc, chunk) => acc + chunk,
851 ),
852 ),
853 child.exitCode.pipe(Effect.map(Number)),
854 ],
855 { concurrency: "unbounded" },
856 );
857 return { stdout, stderr, exitCode };
858 });
859
860// The archive runner is generated shell; string assertions cannot prove the
861// lock excludes concurrent installers. Run the real script against a tiny
862// fake archive served from a file:// mirror.
863describe("archive runner script", () => {
864 const hostPlatform = HostProcess.Platform.defaultValue();
865 const hostArch = HostProcess.Architecture.defaultValue();
866 const windowsHost = hostPlatform === "win32";
867 const archiveVersion = "1.2.3-preview.20260911.4";
868
869 const runRunner = (home: string, runner: string) => runScript(home, [runner, "--version"]);
870
871 // A fake "executable" that answers --version, packed the way the release
872 // workflow packs the real archive: one top-level directory named after the
873 // stem, checksummed in SHA256SUMS.
874 const makeMirror = Effect.fn("makeMirror")(function* (root: string) {
875 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
876 const platform = hostPlatform === "darwin" ? "darwin" : "linux";
877 const arch = hostArch === "arm64" ? "arm64" : "x64";
878 const stem = `t3-${archiveVersion}-${platform}-${arch}`;
879 const stage = `${root}/stage/${stem}`;
880 const release = `${root}/mirror/v${archiveVersion}`;
881 const script = [
882 "set -eu",
883 `mkdir -p '${stage}' '${release}'`,
884 `printf '#!/bin/sh\\necho t3 v${archiveVersion}\\n' > '${stage}/t3'`,
885 `chmod +x '${stage}/t3'`,
886 `tar -czf '${release}/${stem}.tar.gz' -C '${root}/stage' '${stem}'`,
887 `cd '${release}' && (sha256sum '${stem}.tar.gz' 2>/dev/null || shasum -a 256 '${stem}.tar.gz') > SHA256SUMS`,
888 ].join("\n");
889 const child = yield* spawner.spawn(ChildProcess.make("sh", ["-c", script]));
890 assert.equal(Number(yield* child.exitCode), 0);
891 return `file://${root}/mirror`;
892 });
893
894 it.effect.skipIf(windowsHost)(
895 "installs once when several launches race, and reclaims stale locks",
896 () =>
897 Effect.gen(function* () {
898 const fs = yield* FileSystem.FileSystem;
899 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-archive-runner-" });
900 const releaseBaseUrl = yield* makeMirror(root);
901 const runner = `${root}/run-t3.sh`;
902 yield* fs.writeFileString(
903 runner,
904 SshTunnel.buildRemoteT3RunnerScript({ archiveVersion, releaseBaseUrl }),
905 );
906 const home = `${root}/home`;
907 yield* fs.makeDirectory(home, { recursive: true });
908
909 const results = yield* Effect.all(
910 [runRunner(home, runner), runRunner(home, runner), runRunner(home, runner)],
911 { concurrency: "unbounded" },
912 );
913 for (const result of results) {
914 assert.equal(result.exitCode, 0, result.stderr);
915 assert.include(result.stdout, `t3 v${archiveVersion}`);
916 }
917 const versionsDir = `${home}/.t3/runtime/versions`;
918 assert.deepEqual(yield* fs.readDirectory(versionsDir), [archiveVersion]);
919 assert.equal(
920 (yield* fs.readFileString(`${versionsDir}/${archiveVersion}/.install-complete`)).trim(),
921 archiveVersion,
922 );
923
924 // A lock left by a crashed installer (dead pid) must not block the
925 // next launch, and neither must one that never published a pid.
926 const lock = `${versionsDir}/.${archiveVersion}.install.lock`;
927 yield* fs.remove(`${versionsDir}/${archiveVersion}`, { recursive: true });
928 yield* fs.makeDirectory(lock);
929 yield* fs.writeFileString(`${lock}/pid`, "999999\n");
930 const afterDead = yield* runRunner(home, runner);
931 assert.equal(afterDead.exitCode, 0, afterDead.stderr);
932
933 yield* fs.remove(`${versionsDir}/${archiveVersion}`, { recursive: true });
934 yield* fs.makeDirectory(lock);
935 const afterUnowned = yield* runRunner(home, runner);
936 assert.equal(afterUnowned.exitCode, 0, afterUnowned.stderr);
937 assert.isFalse(yield* fs.exists(lock));
938 }).pipe(Effect.provide(NodeServices.layer)),
939 60_000,
940 );
941});
942
943// Reuse decisions also live in generated shell. Drive the real launch script
944// against a fake server that answers readiness probes and writes the default
945// runtime file the way `t3 serve` does.
946describe("launch script", () => {
947 const windowsHost = HostProcess.Platform.defaultValue() === "win32";
948 const FAKE_SERVER = `const fs = require("node:fs");
949const http = require("node:http");
950const path = require("node:path");
951const args = process.argv.slice(2);
952const flag = (name) => args[args.indexOf(name) + 1];
953// --answer-once stalls every request after the first, like a busy server;
954// --exit-after-answer exits right after the first answer.
955const answerOnce = args.includes("--answer-once");
956const exitAfterAnswer = args.includes("--exit-after-answer");
957let answered = false;
958const server = http.createServer((_request, response) => {
959 if (answerOnce && answered) return;
960 answered = true;
961 response.end("ok", () => {
962 if (exitAfterAnswer) process.exit(0);
963 });
964});
965server.listen(Number(flag("--port")), "127.0.0.1", () => {
966 const port = server.address().port;
967 const userdata = path.join(flag("--base-dir"), "userdata");
968 fs.mkdirSync(userdata, { recursive: true });
969 fs.writeFileSync(
970 path.join(userdata, "server-runtime.json"),
971 JSON.stringify({ version: 1, pid: process.pid, port, origin: "http://127.0.0.1:" + port }),
972 );
973 console.log(port);
974});
975`;
976
977 const isAlive = (pid: number) => {
978 try {
979 process.kill(pid, 0);
980 return true;
981 } catch {
982 return false;
983 }
984 };
985
986 const freeLoopbackPort = Effect.callback<number>((resume) => {
987 const server = NodeNet.createServer();
988 server.listen(0, "127.0.0.1", () => {
989 const address = server.address();
990 server.close(() =>
991 resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)),
992 );
993 });
994 });
995
996 // Whether anything still accepts connections on the port. Unlike a PID
997 // check, this cannot mistake an unreaped, exited server for a running one.
998 const isListening = (port: number) =>
999 Effect.callback<boolean>((resume) => {
1000 const socket = NodeNet.connect(port, "127.0.0.1");
1001 socket.once("connect", () => {
1002 socket.destroy();
1003 resume(Effect.succeed(true));
1004 });
1005 socket.once("error", () => resume(Effect.succeed(false)));
1006 });
1007
1008 const launchResult = (stdout: string) => JSON.parse(stdout.trim().split("\n").at(-1) ?? "");
1009
1010 // Writes the launch script for a fake server into a temporary HOME. The
1011 // seeded remembered port keeps the test off the default 3773, which a live
1012 // T3 on this machine may own.
1013 const prepareLaunch = Effect.gen(function* () {
1014 const fs = yield* FileSystem.FileSystem;
1015 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launch-script-" });
1016 const home = `${root}/home`;
1017 const stateDir = `${home}/.t3/ssh-launch/test-state`;
1018 const fakeServer = `${root}/fake-t3.cjs`;
1019 const launch = `${root}/launch.sh`;
1020 yield* fs.makeDirectory(stateDir, { recursive: true });
1021 yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
1022 yield* fs.writeFileString(fakeServer, FAKE_SERVER);
1023 yield* fs.writeFileString(
1024 launch,
1025 SshTunnel.buildRemoteLaunchScript({ nodeScriptPath: fakeServer }),
1026 );
1027 const readManagedPid = fs
1028 .readFileString(`${stateDir}/pid`)
1029 .pipe(Effect.map((pid) => Number(pid.trim())));
1030 // Launched servers outlive the script, so stop each PID the script
1031 // recorded, including one it recorded just before a failed assertion.
1032 // SIGKILL also ends a server a test left stopped.
1033 const launchedPids: Array<number> = [];
1034 yield* Effect.addFinalizer(() =>
1035 readManagedPid.pipe(
1036 Effect.map((pid) => [...launchedPids, pid]),
1037 Effect.orElseSucceed(() => launchedPids),
1038 Effect.map((pids) => {
1039 for (const pid of pids) {
1040 if (isAlive(pid)) process.kill(pid, "SIGKILL");
1041 }
1042 }),
1043 ),
1044 );
1045 const runLaunchScript = runScript(home, [launch, "test-state"]);
1046 const runLaunch = runLaunchScript.pipe(
1047 Effect.map((result) => {
1048 assert.equal(result.exitCode, 0, result.stderr);
1049 return launchResult(result.stdout);
1050 }),
1051 );
1052 const launchManaged = Effect.gen(function* () {
1053 const launched = yield* runLaunch;
1054 const pid = yield* readManagedPid;
1055 launchedPids.push(pid);
1056 assert.equal(launched.serverKind, "managed");
1057 return { launched, pid };
1058 });
1059 // Starts a separate server on the default home, as a service would, and
1060 // returns the port it reports. An unreaped server runs under a parent that
1061 // never waits on it, so once it exits it lingers as a zombie.
1062 const startDefaultHomeServer = (
1063 flags: ReadonlyArray<string>,
1064 options?: { readonly unreaped?: boolean },
1065 ) =>
1066 Effect.gen(function* () {
1067 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1068 const args = [fakeServer, "serve", "--port", "0", "--base-dir", `${home}/.t3`, ...flags];
1069 const server = yield* spawner.spawn(
1070 options?.unreaped
1071 ? ChildProcess.make("sh", ["-c", 'node "$@" & exec sleep 600', "sh", ...args])
1072 : ChildProcess.make("node", args),
1073 );
1074 const port = yield* server.stdout.pipe(
1075 Stream.decodeText(),
1076 Stream.splitLines,
1077 Stream.runHead,
1078 );
1079 return Number(Option.getOrThrow(port));
1080 });
1081 return {
1082 home,
1083 stateDir,
1084 launch,
1085 readManagedPid,
1086 runLaunchScript,
1087 runLaunch,
1088 launchManaged,
1089 startDefaultHomeServer,
1090 };
1091 });
1092
1093 it.effect.skipIf(windowsHost)(
1094 "reuses its own managed server and yields only to a different default-home server",
1095 () =>
1096 Effect.gen(function* () {
1097 const fs = yield* FileSystem.FileSystem;
1098 const { stateDir, readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
1099 yield* prepareLaunch;
1100 const { launched, pid: managedPid } = yield* launchManaged;
1101
1102 // The managed server wrote the default runtime file itself, so a
1103 // reconnect must keep it rather than stop it as an external server.
1104 assert.deepEqual(yield* runLaunch, launched);
1105 assert.equal(yield* readManagedPid, managedPid);
1106 assert.isTrue(isAlive(managedPid));
1107
1108 // A remembered port that is stale must not hide our own server: the
1109 // runtime file names its port.
1110 yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
1111 assert.deepEqual(yield* runLaunch, launched);
1112 assert.equal(yield* readManagedPid, managedPid);
1113 assert.equal(
1114 (yield* fs.readFileString(`${stateDir}/port`)).trim(),
1115 `${launched.remotePort}`,
1116 );
1117
1118 // A different server on the default home, such as a service, still
1119 // takes over from the managed one.
1120 const servicePort = yield* startDefaultHomeServer([]);
1121 assert.deepEqual(yield* runLaunch, {
1122 remotePort: servicePort,
1123 serverKind: "external",
1124 });
1125 assert.isFalse(yield* isListening(launched.remotePort));
1126 assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
1127 }).pipe(Effect.provide(NodeServices.layer)),
1128 60_000,
1129 );
1130
1131 // A stopped process keeps its listener but answers nothing, like a server
1132 // stuck in a long SQLite stall.
1133 it.effect.skipIf(windowsHost)(
1134 "waits for its busy managed server instead of replacing it",
1135 () =>
1136 Effect.gen(function* () {
1137 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1138 const { readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1139 const { launched, pid } = yield* launchManaged;
1140
1141 // It resumes after the two 2-second probes that used to replace it.
1142 process.kill(pid, "SIGSTOP");
1143 yield* spawner.spawn(ChildProcess.make("sh", ["-c", `sleep 5; kill -CONT ${pid}`]));
1144 assert.deepEqual(yield* runLaunch, launched);
1145 assert.equal(yield* readManagedPid, pid);
1146 assert.isTrue(isAlive(pid));
1147 }).pipe(Effect.provide(NodeServices.layer)),
1148 60_000,
1149 );
1150
1151 it.effect.skipIf(windowsHost)(
1152 "fails without stopping a managed server that never becomes ready",
1153 () =>
1154 Effect.gen(function* () {
1155 const fs = yield* FileSystem.FileSystem;
1156 const { stateDir, launch, readManagedPid, runLaunchScript, launchManaged } =
1157 yield* prepareLaunch;
1158 const { launched, pid } = yield* launchManaged;
1159
1160 // Shorten the one-minute readiness budget so the test stays fast.
1161 const script = yield* fs.readFileString(launch);
1162 assert.include(script, 'wait_ready "60000"');
1163 yield* fs.writeFileString(
1164 launch,
1165 script.replaceAll('wait_ready "60000"', 'wait_ready "1500"'),
1166 );
1167 process.kill(pid, "SIGSTOP");
1168 const result = yield* runLaunchScript;
1169 assert.equal(result.exitCode, 1, result.stdout);
1170 assert.include(result.stderr, "the server was left running");
1171 assert.isTrue(isAlive(pid));
1172 assert.equal(yield* readManagedPid, pid);
1173 assert.equal(
1174 (yield* fs.readFileString(`${stateDir}/port`)).trim(),
1175 `${launched.remotePort}`,
1176 );
1177 assert.equal((yield* fs.readFileString(`${stateDir}/managed`)).trim(), "managed");
1178 }).pipe(Effect.provide(NodeServices.layer)),
1179 60_000,
1180 );
1181
1182 it.effect.skipIf(windowsHost)(
1183 "still replaces its managed server when the runner changes",
1184 () =>
1185 Effect.gen(function* () {
1186 const fs = yield* FileSystem.FileSystem;
1187 const { stateDir, readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1188 const { pid: previousPid } = yield* launchManaged;
1189
1190 // A different runner on disk means the client updated: replace the server.
1191 yield* fs.writeFileString(`${stateDir}/run-t3.sh`, "# previous runner\n");
1192 assert.equal((yield* runLaunch).serverKind, "managed");
1193 const replacementPid = yield* readManagedPid;
1194 assert.notEqual(replacementPid, previousPid);
1195 assert.isFalse(isAlive(previousPid));
1196 assert.isTrue(isAlive(replacementPid));
1197 }).pipe(Effect.provide(NodeServices.layer)),
1198 60_000,
1199 );
1200
1201 it.effect.skipIf(windowsHost)(
1202 "replaces a managed server that exited but was not reaped",
1203 () =>
1204 Effect.gen(function* () {
1205 const fs = yield* FileSystem.FileSystem;
1206 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1207 const { home, stateDir, readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1208 const { launched, pid } = yield* launchManaged;
1209 process.kill(pid, "SIGKILL");
1210
1211 // A crashed server whose parent never reaps it stays a zombie, which
1212 // kill -0 still reports. Its pid file and runtime record keep naming it.
1213 const parent = yield* spawner.spawn(
1214 ChildProcess.make("sh", ["-c", "sleep 600 & echo $!; exec sleep 600"]),
1215 );
1216 const zombie = Number(
1217 Option.getOrThrow(
1218 yield* parent.stdout.pipe(Stream.decodeText(), Stream.splitLines, Stream.runHead),
1219 ),
1220 );
1221 process.kill(zombie, "SIGKILL");
1222 assert.include((yield* runScript(home, ["-c", `ps -o stat= -p ${zombie}`])).stdout, "Z");
1223 yield* fs.writeFileString(`${stateDir}/pid`, `${zombie}\n`);
1224 yield* fs.writeFileString(
1225 `${home}/.t3/userdata/server-runtime.json`,
1226 JSON.stringify({
1227 version: 1,
1228 pid: zombie,
1229 port: launched.remotePort,
1230 origin: `http://127.0.0.1:${launched.remotePort}`,
1231 }),
1232 );
1233
1234 assert.equal((yield* runLaunch).serverKind, "managed");
1235 const replacementPid = yield* readManagedPid;
1236 assert.notEqual(replacementPid, zombie);
1237 assert.isTrue(isAlive(replacementPid));
1238 }).pipe(Effect.provide(NodeServices.layer)),
1239 60_000,
1240 );
1241
1242 it.effect.skipIf(windowsHost)(
1243 "adopts a server it has no managed record for instead of restarting it",
1244 () =>
1245 Effect.gen(function* () {
1246 const fs = yield* FileSystem.FileSystem;
1247 const { stateDir, launchManaged, runLaunch } = yield* prepareLaunch;
1248 const { launched, pid } = yield* launchManaged;
1249
1250 // An interrupted launch, or a recycled PID, can leave a PID that matches
1251 // the runtime file without the managed marker. A changed runner must not
1252 // turn that into a restart of a server this launcher cannot claim.
1253 yield* fs.remove(`${stateDir}/managed`);
1254 yield* fs.writeFileString(`${stateDir}/run-t3.sh`, "# superseded runner\n");
1255 assert.deepEqual(yield* runLaunch, {
1256 remotePort: launched.remotePort,
1257 serverKind: "external",
1258 });
1259 assert.isTrue(isAlive(pid));
1260 }).pipe(Effect.provide(NodeServices.layer)),
1261 60_000,
1262 );
1263
1264 it.effect.skipIf(windowsHost)(
1265 "keeps an adopted default-home server without launching another beside it",
1266 () =>
1267 Effect.gen(function* () {
1268 const fs = yield* FileSystem.FileSystem;
1269 const { stateDir, runLaunch, startDefaultHomeServer } = yield* prepareLaunch;
1270 // Ownership files from an earlier managed launch that lost its PID.
1271 yield* fs.writeFileString(`${stateDir}/managed`, "managed\n");
1272 const servicePort = yield* startDefaultHomeServer(["--answer-once"]);
1273
1274 // The server answers adoption and then stalls. A second readiness
1275 // probe would launch another server on the same home.
1276 assert.deepEqual(yield* runLaunch, { remotePort: servicePort, serverKind: "external" });
1277 assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
1278 }).pipe(Effect.provide(NodeServices.layer)),
1279 60_000,
1280 );
1281
1282 it.effect.skipIf(windowsHost)(
1283 "launches a replacement when the adopted server exits during the handover",
1284 () =>
1285 Effect.gen(function* () {
1286 const { readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
1287 yield* prepareLaunch;
1288 const { pid: previousPid } = yield* launchManaged;
1289 yield* startDefaultHomeServer(["--exit-after-answer"], { unreaped: true });
1290
1291 // The service answers adoption and then exits while the managed
1292 // server is being stopped, staying a zombie, so the script must launch
1293 // a new server rather than return the dead one. The freed port may be
1294 // reused.
1295 const result = yield* runLaunch;
1296 assert.equal(result.serverKind, "managed");
1297 const replacementPid = yield* readManagedPid;
1298 assert.notEqual(replacementPid, previousPid);
1299 assert.isTrue(isAlive(replacementPid));
1300 }).pipe(Effect.provide(NodeServices.layer)),
1301 60_000,
1302 );
1303});