Commit 99e3b721c5255ded20b00ba1798f848bfc0f1f65 | the-knife

author	Julius Marminge <julius0216@outlook.com> 2026-09-04 00:43:47-07:00
committer	GitHub <noreply@github.com> 2026-09-04 00:43:47-07:00
log	99e3b721c5255ded20b00ba1798f848bfc0f1f65
tree	ac689cba47b84e4fe480eb9248655c9639d2fa0d
parent	c3b8825bf476cbce5e061c0f99570cf1f6723b89
signature	Signed by PGP key B5690EEEBB952194

fix(connect): diagnose incomplete headless server setup (#9602)

11 files changed, 614 insertions(+), 42 deletions(-)

apps
- server
  - src
    cli
    connect.ts
    service.test.ts
    service.ts
    cloud
    bootService.test.ts
    bootService.ts
    http.ts
    relayResponse.test.ts
    relayResponse.ts
    server.ts
docs
- user
  - background-service.md
  - remote-access.md

apps/server/src/cli/connect.ts+9-6

...	...	@@ -35,6 +35,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
35	35	import * as BootService from "../cloud/bootService.ts";
36	36	import * as CliState from "../cloud/CliState.ts";
37	37	import * as CliTokenManager from "../cloud/CliTokenManager.ts";
	38	import { filterRelayResponse } from "../cloud/relayResponse.ts";
38	39	import {
39	40	CLOUD_LINKED_USER_ID,
40	41	isAgentActivityPublishingEnabledValue,
...	...	@@ -208,6 +209,8 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b
208	209	` Relay: ${status.relayUrl ?? "not provisioned"}`,
209	210	` Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`,
210	211	...formatRelayClientStatus(status.relayClient),
	212	"",
	213	"This is saved setup, not a live connection check. Check the background service with `t3 service status`.",
211	214	...(nextStep ? ["", `Next: ${nextStep}`] : []),
212	215	].join("\n");
213	216	}
...	...	@@ -346,7 +349,7 @@ const unlinkRelayEnvironment = Effect.fn("cloud.cli.unlink_relay_environment")(f
346	349	).pipe(
347	350	HttpClientRequest.bearerToken(token.value.accessToken),
348	351	httpClient.execute,
349		Effect.flatMap(HttpClientResponse.filterStatusOk),
	352	Effect.flatMap(filterRelayResponse),
350	353	Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
351	354	withRelayClientTracing,
352	355	);
...	...	@@ -689,17 +692,17 @@ export const connectCommand = Command.make("connect", {
689	692	// Show which account was linked so an unexpected identity (an
690	693	// authorization code for a different account) is visible before the
691	694	// machine is brought online.
692		yield* Console.log(`✓ Connected${connectedAs(linked.identity)}`);
	695	yield* Console.log(`✓ Authorized${connectedAs(linked.identity)}`);
693	696
694		// Connect itself already succeeded; a boot-service failure must not
695		// fail the command, just tell the user what happened and move on.
	697	// Authorization is stored. If service setup fails, preserve it and
	698	// show how to run the server manually.
696	699	const background = yield* recoverServiceOnboardingOffer(offerServiceDuringOnboarding);
697	700	if (background) {
698	701	const platform = yield* HostProcessPlatform;
699	702	yield* Console.log(
700	703	platform === "darwin"
701		? "\n✓ Background service ready\n\nT3 Code will stay reachable while you are logged in to this Mac."
702		: "\n✓ Background service ready\n\nT3 Code will stay reachable after you log out.",
	704	? "\n✓ Background service ready\n\nT3 Code is set to run while you are logged in to this Mac. The server establishes the T3 Connect link on startup."
	705	: "\n✓ Background service ready\n\nT3 Code is set to keep running after you log out. The server establishes the T3 Connect link on startup.",
703	706	);
704	707	return;
705	708	}

apps/server/src/cli/service.test.ts+47-1

...	...	@@ -45,10 +45,38 @@ it("reports the installed service version and host paths", () => {
45	45	it("gives a direct repair command for a stale service", () => {
46	46	assert.include(
47	47	formatServiceStatus({ ...status, current: false }, "0.0.29"),
48		"Next: Run `npx t3@latest service update`.",
	48	"Next: Run `npx t3@0.0.29 service update`.",
49	49	);
50	50	});
51	51
	52	it("explains an incomplete nightly installation and keeps repair on its installed version", () => {
	53	const output = formatServiceStatus(
	54	{
	55	...status,
	56	current: false,
	57	installedVersion: "0.0.32-nightly.1",
	58	problems: ["linger-disabled", "service-stopped"],
	59	},
	60	"0.0.32-nightly.1",
	61	);
	62
	63	expect(output).toContain("[linger-disabled]");
	64	expect(output).toContain("last login session ends");
	65	expect(output).toContain('sudo loginctl enable-linger "$(id -un)"');
	66	expect(output).toContain("[service-stopped]");
	67	expect(output).toContain("npx t3@0.0.32-nightly.1 service update");
	68	expect(output).not.toContain("t3@latest");
	69	});
	70
	71	it("suggests the newer CLI version when the installed service needs an update", () => {
	72	const output = formatServiceStatus(
	73	{ ...status, current: false, installedVersion: "0.0.28" },
	74	"0.0.29",
	75	);
	76	expect(output).toContain("npx t3@0.0.29 service update");
	77	expect(output).not.toContain("npx t3@0.0.28 service update");
	78	});
	79
52	80	it("explains where the service is supported", () => {
53	81	assert.include(
54	82	formatServiceStatus({ ...status, supported: false, installed: false }, "0.0.29"),
...	...	@@ -151,6 +179,15 @@ it.effect.each([
151	179	name: "the same version",
152	180	state: { ...status, current: false, installedVersion: packageJson.version },
153	181	},
	182	{
	183	name: "an incomplete install of the same version",
	184	state: {
	185	...status,
	186	current: false,
	187	installedVersion: packageJson.version,
	188	problems: ["linger-disabled"] as const,
	189	},
	190	},
154	191	{ name: "an unknown version", state: { ...status, current: false } },
155	192	])("installs or repairs $name without an override", ({ state }) =>
156	193	Effect.gen(function* () {
...	...	@@ -201,3 +238,12 @@ it.effect("keeps onboarding successful when a newer version appears before insta
201	238	expect(ready).toBe(false);
202	239	}),
203	240	);
	241
	242	it.effect("keeps the manual-server fallback when background prerequisites fail", () =>
	243	Effect.gen(function* () {
	244	const ready = yield* recoverServiceOnboardingOffer(
	245	Effect.fail(new BootService.BootServicePrerequisiteError({ problem: "linger-disabled" })),
	246	);
	247	expect(ready).toBe(false);
	248	}),
	249	);

apps/server/src/cli/service.ts+11-1

...	...	@@ -68,6 +68,9 @@ export function formatServiceStatus(
68	68	return "T3 Code service\n Status: not installed\n Next: Run `t3 service install`.";
69	69	}
70	70	const installedVersion = status.installedVersion ?? cliVersion;
	71	const problems = (status.problems ?? []).map(
	72	(problem) => ` [${problem}] ${BootService.formatBootServiceProblem(problem)}`,
	73	);
71	74	if (
72	75	!status.current &&
73	76	status.installedVersion !== undefined &&
...	...	@@ -78,6 +81,7 @@ export function formatServiceStatus(
78	81	` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`,
79	82	` Unit: ${status.unitPath}`,
80	83	` Logs: ${status.logPath}`,
	84	...problems,
81	85	` Next: Use \`npx t3@${installedVersion} service update\` to repair it, or pass \`--allow-downgrade\` explicitly.`,
82	86	].join("\n");
83	87	}
...	...	@@ -86,7 +90,8 @@ export function formatServiceStatus(
86	90	` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`,
87	91	` Unit: ${status.unitPath}`,
88	92	` Logs: ${status.logPath}`,
89		...(status.current ? [] : [" Next: Run `npx t3@latest service update`."]),
	93	...problems,
	94	...(status.current ? [] : [` Next: Run \`npx t3@${cliVersion} service update\`.`]),
90	95	].join("\n");
91	96	}
92	97
...	...	@@ -189,6 +194,9 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () {
189	194	yield* Console.log("T3 Code is already set up to run in the background on this machine.");
190	195	return true;
191	196	}
	197	for (const problem of status.problems ?? []) {
	198	yield* Console.warn(`[${problem}] ${BootService.formatBootServiceProblem(problem)}`);
	199	}
192	200	if (
193	201	installed &&
194	202	status.installedVersion !== undefined &&
...	...	@@ -239,6 +247,8 @@ export const recoverServiceOnboardingOffer = <R>(
239	247	Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
240	248	BootServiceInstallError: (error) =>
241	249	Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
	250	BootServicePrerequisiteError: (error) =>
	251	Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
242	252	BootServiceUpdatePendingError: (error) =>
243	253	Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
244	254	BootServiceDowngradeRefusedError: (error) =>

apps/server/src/cloud/bootService.test.ts+136-8

...	...	@@ -127,8 +127,17 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
127	127
128	128	const commands: string[] = [];
129	129	const timeouts = new Map<string, unknown>();
130		const control: { failCommand: string \| undefined; stateAfterStop?: string } = {
	130	const control: {
	131	failCommand: string \| undefined;
	132	stateAfterStop?: string;
	133	linger: string;
	134	enabled: boolean;
	135	active: boolean;
	136	} = {
131	137	failCommand: undefined,
	138	linger: "yes",
	139	enabled: true,
	140	active: true,
132	141	};
133	142	const runner = ProcessRunner.ProcessRunner.of({
134	143	run: Effect.fn("test.run_boot_service_command")(function* (
...	...	@@ -137,6 +146,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
137	146	const command = `${input.command} ${input.args.join(" ")}`;
138	147	commands.push(command);
139	148	timeouts.set(command, input.timeout);
	149	const failed = command === control.failCommand;
	150	if (!failed && command === "loginctl enable-linger --no-ask-password 501")
	151	control.linger = "yes";
	152	if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true;
	153	if (!failed && command === "systemctl --user restart t3code.service") control.active = true;
140	154	if (
141	155	control.stateAfterStop !== undefined &&
142	156	(command === "systemctl --user stop t3code.service" \|\|
...	...	@@ -145,9 +159,20 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
145	159	yield* fs.writeFileString(statePath, control.stateAfterStop).pipe(Effect.orDie);
146	160	}
147	161	return {
148		stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "",
	162	stdout:
	163	input.args[1] === "--version"
	164	? "t3 v1.2.3\n"
	165	: input.command === "loginctl" && input.args[0] === "show-user"
	166	? `${control.linger}\n`
	167	: input.args[1] === "is-enabled"
	168	? control.enabled
	169	? "enabled\n"
	170	: "disabled\n"
	171	: "",
149	172	stderr: "",
150		code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0),
	173	code: ChildProcessSpawner.ExitCode(
	174	failed \|\| (input.args[1] === "is-active" && !control.active) ? 1 : 0,
	175	),
151	176	timedOut: false,
152	177	stdoutTruncated: false,
153	178	stderrTruncated: false,
...	...	@@ -182,10 +207,103 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
182	207	),
183	208	);
184	209	const service = yield* makeService();
185		return { service, makeService, fs, statePath, commands, timeouts, control };
	210	return { service, makeService, fs, statePath, commands, timeouts, control, runtime };
186	211	});
187	212
188	213	it.layer(NodeServices.layer)("boot service install", (it) => {
	214	it.effect(
	215	"fails before installing files or validating a runtime when lingering needs an administrator",
	216	() =>
	217	Effect.gen(function* () {
	218	const { service, fs, statePath, commands, control, runtime } = yield* makeHarness();
	219	const before = yield* service.status;
	220	control.linger = "no";
	221	control.failCommand = "loginctl enable-linger --no-ask-password 501";
	222	yield* fs.remove(runtime.sentinelPath);
	223
	224	const error = yield* service.install().pipe(Effect.flip);
	225
	226	expect(error).toMatchObject({
	227	_tag: "BootServicePrerequisiteError",
	228	problem: "linger-disabled",
	229	});
	230	expect(error.message).toContain('sudo loginctl enable-linger "$(id -un)"');
	231	expect(error.message).toContain("last login session ends");
	232	expect(yield* fs.exists(before.unitPath)).toBe(false);
	233	expect(yield* fs.exists(statePath)).toBe(false);
	234	expect(
	235	commands.some((command) => command.startsWith("npm ") \|\| command.includes("--version")),
	236	).toBe(false);
	237	expect(
	238	commands.some(
	239	(command) => command.includes("daemon-reload") \|\| command.includes("restart"),
	240	),
	241	).toBe(false);
	242	expect(yield* fs.readFileString(before.logPath)).toContain("[linger-disabled]");
	243	}),
	244	);
	245
	246	it.effect(
	247	"detects a partial install and preserves the running service when repair lacks permission",
	248	() =>
	249	Effect.gen(function* () {
	250	const { service, fs, statePath, commands, control } = yield* makeHarness();
	251	const plan = yield* service.install();
	252	const before = yield* fs.readFileString(statePath);
	253	const unit = yield* fs.readFileString(plan.unitPath);
	254	control.linger = "no";
	255	control.failCommand = "loginctl enable-linger --no-ask-password 501";
	256
	257	expect(yield* service.status).toMatchObject({
	258	current: false,
	259	problems: ["linger-disabled"],
	260	});
	261	commands.length = 0;
	262	expect((yield* service.install().pipe(Effect.flip))._tag).toBe(
	263	"BootServicePrerequisiteError",
	264	);
	265	expect(yield* fs.readFileString(statePath)).toBe(before);
	266	expect(yield* fs.readFileString(plan.unitPath)).toBe(unit);
	267	expect(commands).not.toContain("systemctl --user stop t3code.service");
	268	}),
	269	);
	270
	271	it.effect("enables lingering before installing and repairs stopped or disabled services", () =>
	272	Effect.gen(function* () {
	273	const { service, commands, control } = yield* makeHarness();
	274	control.linger = "no";
	275	yield* service.install();
	276	expect(control.linger).toBe("yes");
	277	expect(commands.indexOf("loginctl enable-linger --no-ask-password 501")).toBeLessThan(
	278	commands.indexOf("systemctl --user daemon-reload"),
	279	);
	280
	281	control.enabled = false;
	282	control.active = false;
	283	expect(yield* service.status).toMatchObject({
	284	current: false,
	285	problems: ["service-disabled", "service-stopped"],
	286	});
	287	yield* service.install();
	288	expect((yield* service.status).current).toBe(true);
	289	}),
	290	);
	291
	292	it.effect.each([
	293	{ command: "systemctl --user show-environment", problem: "user-manager-unavailable" },
	294	{ command: "loginctl show-user 501 --property=Linger --value", problem: "linger-unavailable" },
	295	])("reports failed prerequisite probes without installing: $command", ({ command, problem }) =>
	296	Effect.gen(function* () {
	297	const { service, fs, statePath, control } = yield* makeHarness();
	298	control.failCommand = command;
	299	expect(yield* service.install().pipe(Effect.flip)).toMatchObject({
	300	_tag: "BootServicePrerequisiteError",
	301	problem,
	302	});
	303	expect(yield* fs.exists(statePath)).toBe(false);
	304	}),
	305	);
	306
189	307	it.effect("installs, reports current state, and uninstalls", () =>
190	308	Effect.gen(function* () {
191	309	const { service, fs, statePath, commands, timeouts } = yield* makeHarness();
...	...	@@ -286,8 +404,10 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
286	404	expect(yield* fs.readFileString(plan.launcherPath)).toBe(launcher);
287	405	expect(yield* fs.readFileString(plan.unitPath)).toBe(unit);
288	406	expect(
289		commands.filter((command) =>
290		command.startsWith(platform === "linux" ? "systemctl " : "launchctl "),
	407	commands.filter(
	408	(command) =>
	409	command.startsWith(platform === "linux" ? "systemctl " : "launchctl ") &&
	410	!command.includes("show-environment"),
291	411	),
292	412	).toEqual(
293	413	platform === "linux"
...	...	@@ -351,7 +471,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
351	471
352	472	const error = yield* service.install().pipe(Effect.flip);
353	473	expect(error._tag).toBe("BootServiceCommandError");
354		expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([
	474	expect(
	475	commands.filter(
	476	(command) => command.startsWith("systemctl ") && !command.includes("show-environment"),
	477	),
	478	).toEqual([
355	479	"systemctl --user stop t3code.service",
356	480	"systemctl --user daemon-reload",
357	481	"systemctl --user restart t3code.service",
...	...	@@ -382,7 +506,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
382	506	"BootServiceUpdatePendingError",
383	507	);
384	508	expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true);
385		expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([
	509	expect(
	510	commands.filter(
	511	(command) => command.startsWith("systemctl ") && !command.includes("show-environment"),
	512	),
	513	).toEqual([
386	514	"systemctl --user stop t3code.service",
387	515	"systemctl --user restart t3code.service",
388	516	]);

apps/server/src/cloud/bootService.ts+113-11

...	...	@@ -235,7 +235,6 @@ function systemdManager(input: {
235	235	command: "systemctl",
236	236	args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE],
237	237	},
238		{ step: "enabling lingering for this user", command: "loginctl", args: ["enable-linger"] },
239	238	// Start last. No administrative state write occurs after this succeeds.
240	239	{
241	240	step: "starting the service",
...	...	@@ -409,6 +408,40 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass<BootService
409	408	}
410	409	}
411	410
	411	const BootServiceProblem = Schema.Literals([
	412	"user-manager-unavailable",
	413	"linger-unavailable",
	414	"linger-disabled",
	415	"service-disabled",
	416	"service-stopped",
	417	]);
	418	type BootServiceProblem = typeof BootServiceProblem.Type;
	419
	420	/** These codes and recovery steps are documented in docs/user/background-service.md. */
	421	export function formatBootServiceProblem(problem: BootServiceProblem): string {
	422	switch (problem) {
	423	case "user-manager-unavailable":
	424	return "Cannot reach the systemd user manager. Run `systemctl --user status` in a login session for the service user. Install your distribution's systemd user-session support if it is missing; do not run T3 with sudo.";
	425	case "linger-unavailable":
	426	return 'Cannot check whether this user can run services after logout. Run `loginctl show-user "$(id -un)" --property=Linger` and check that systemd-logind is available.';
	427	case "linger-disabled":
	428	return 'Lingering is disabled. T3 Code will stop when your last login session ends and will not start at boot. Run `sudo loginctl enable-linger "$(id -un)"` on this machine, then retry the service command as your normal user.';
	429	case "service-disabled":
	430	return "The service is not enabled to start automatically. Run `t3 service update` to repair it.";
	431	case "service-stopped":
	432	return "The service is not running. Check the service log and `systemctl --user status t3code.service`, then run `t3 service update`.";
	433	}
	434	}
	435
	436	export class BootServicePrerequisiteError extends Schema.TaggedErrorClass<BootServicePrerequisiteError>()(
	437	"BootServicePrerequisiteError",
	438	{ problem: BootServiceProblem, cause: Schema.optional(Schema.Defect()) },
	439	) {
	440	override get message(): string {
	441	return `[${this.problem}] ${formatBootServiceProblem(this.problem)}`;
	442	}
	443	}
	444
412	445	export class BootServiceUpdatePendingError extends Schema.TaggedErrorClass<BootServiceUpdatePendingError>()(
413	446	"BootServiceUpdatePendingError",
414	447	{},
...	...	@@ -434,6 +467,7 @@ export type BootServiceError =
434	467	\| BootServiceUnsupportedError
435	468	\| BootServiceCommandError
436	469	\| BootServiceInstallError
	470	\| BootServicePrerequisiteError
437	471	\| BootServiceUpdatePendingError
438	472	\| BootServiceDowngradeRefusedError;
439	473
...	...	@@ -442,6 +476,7 @@ export interface BootServiceStatus {
442	476	readonly installed: boolean;
443	477	readonly current: boolean;
444	478	readonly installedVersion?: string;
	479	readonly problems?: ReadonlyArray<BootServiceProblem>;
445	480	readonly unitPath: string;
446	481	readonly logPath: string;
447	482	}
...	...	@@ -539,6 +574,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
539	574	: Effect.succeed(detectedManager),
540	575	);
541	576
	577	const logFailure = (error: { readonly message: string }) =>
	578	DateTime.now.pipe(
	579	Effect.flatMap((now) =>
	580	fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { flag: "a" }),
	581	),
	582	Effect.ignore,
	583	);
	584
542	585	const runStep = Effect.fn("cloud.boot_service.run_step")(function* (
543	586	step: string,
544	587	command: string,
...	...	@@ -557,16 +600,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
557	600	stderrLength: result.stderr.length,
558	601	}),
559	602	),
560		Effect.tapError((error) =>
561		DateTime.now.pipe(
562		Effect.flatMap((now) =>
563		fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, {
564		flag: "a",
565		}),
566		),
567		Effect.ignore,
568		),
569		),
	603	Effect.tapError(logFailure),
570	604	);
571	605	});
572	606
...	...	@@ -587,6 +621,66 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
587	621	{ discard: true },
588	622	);
589	623
	624	const probe = (command: string, args: ReadonlyArray<string>) =>
	625	runner.run({ command, args, timeout: Duration.seconds(5) }).pipe(Effect.option);
	626	const succeeded = (result: Option.Option<ProcessRunner.ProcessRunOutput>) =>
	627	Option.isSome(result) && result.value.code === 0;
	628	const lingerArgs = [
	629	"show-user",
	630	...(uid === undefined ? [] : [String(uid)]),
	631	"--property=Linger",
	632	"--value",
	633	];
	634	const readSystemdProblems = Effect.fn("cloud.boot_service.read_systemd_problems")(function* (
	635	includeService: boolean,
	636	) {
	637	const [manager, linger] = yield* Effect.all(
	638	[probe("systemctl", ["--user", "show-environment"]), probe("loginctl", lingerArgs)],
	639	{ concurrency: "unbounded" },
	640	);
	641	const problems: BootServiceProblem[] = [];
	642	if (!succeeded(manager)) problems.push("user-manager-unavailable");
	643	const lingering = succeeded(linger) && Option.isSome(linger) ? linger.value.stdout.trim() : "";
	644	if (lingering !== "yes") {
	645	problems.push(lingering === "no" ? "linger-disabled" : "linger-unavailable");
	646	}
	647	if (includeService && succeeded(manager)) {
	648	const [enabled, active] = yield* Effect.all(
	649	[
	650	probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]),
	651	probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]),
	652	],
	653	{ concurrency: "unbounded" },
	654	);
	655	if (
	656	!succeeded(enabled) \|\|
	657	(Option.isSome(enabled) && enabled.value.stdout.trim() !== "enabled")
	658	) {
	659	problems.push("service-disabled");
	660	}
	661	if (!succeeded(active)) problems.push("service-stopped");
	662	}
	663	return problems;
	664	});
	665
	666	const requireSystemdPrerequisites = Effect.gen(function* () {
	667	const problems = yield* readSystemdProblems(false);
	668	const unavailable = problems.find((problem) => problem !== "linger-disabled");
	669	if (unavailable) return yield* new BootServicePrerequisiteError({ problem: unavailable });
	670	if (!problems.includes("linger-disabled")) return;
	671	yield* runStep("enabling lingering for this user", "loginctl", [
	672	"enable-linger",
	673	"--no-ask-password",
	674	...(uid === undefined ? [] : [String(uid)]),
	675	]).pipe(
	676	Effect.mapError(
	677	(cause) => new BootServicePrerequisiteError({ problem: "linger-disabled", cause }),
	678	),
	679	);
	680	const remaining = yield* readSystemdProblems(false);
	681	if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] });
	682	});
	683
590	684	const install = Effect.fn("cloud.boot_service.install")(function* (options?: {
591	685	readonly allowDowngrade?: boolean;
592	686	}) {
...	...	@@ -595,6 +689,11 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
595	689	.makeDirectory(input.logsDir, { recursive: true })
596	690	.pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
597	691
	692	// A permissions failure must not leave a partial install or stop a working server.
	693	if (manager.kind === "systemd") {
	694	yield* requireSystemdPrerequisites.pipe(Effect.tapError(logFailure));
	695	}
	696
598	697	// Prepare every immutable artifact before stopping the installed unit.
599	698	yield* ensurePinnedRuntimeInstalled({
600	699	baseDir: input.baseDir,
...	...	@@ -743,11 +842,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
743	842	detectedManager.kind === "launchd"
744	843	? contents.replace(/(<key>PATH<\/key>\n\s<string>)[^<](<\/string>)/, "$1$2")
745	844	: contents;
	845	const problems = detectedManager.kind === "systemd" ? yield* readSystemdProblems(true) : [];
746	846	return {
747	847	supported: true,
748	848	installed: true,
749	849	...(installedVersion === undefined ? {} : { installedVersion }),
	850	problems,
750	851	current:
	852	problems.length === 0 &&
751	853	normalizeUnit(unit) === normalizeUnit(detectedManager.render(plan)) &&
752	854	launcherExists &&
753	855	runtimeEntryExists &&

apps/server/src/cloud/http.ts+4-8

...	...	@@ -85,6 +85,7 @@ import {
85	85	import * as CliTokenManager from "./CliTokenManager.ts";
86	86	import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
87	87	import { traceRelayRequest } from "./traceRelayRequest.ts";
	88	import { filterRelayResponse, relayRequestError } from "./relayResponse.ts";
88	89
89	90	const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
90	91	const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
...	...	@@ -526,14 +527,9 @@ const relayClientRequest = <A>(
526	527	HttpClientRequest.bearerToken(input.token),
527	528	HttpClientRequest.bodyJson(input.payload),
528	529	Effect.flatMap(dependencies.httpClient.execute),
529		Effect.flatMap(HttpClientResponse.filterStatusOk),
	530	Effect.flatMap(filterRelayResponse),
530	531	Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
531		Effect.mapError(
532		(cause) =>
533		new EnvironmentHttpInternalServerError({
534		message: `T3 Connect relay request failed: ${String(cause)}`,
535		}),
536		),
	532	Effect.mapError(relayRequestError),
537	533	withRelayClientTracing,
538	534	);
539	535
...	...	@@ -720,7 +716,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
720	716	).pipe(
721	717	HttpClientRequest.bearerToken(token.value.accessToken),
722	718	dependencies.httpClient.execute,
723		Effect.flatMap(HttpClientResponse.filterStatusOk),
	719	Effect.flatMap(filterRelayResponse),
724	720	Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
725	721	withRelayClientTracing,
726	722	);

apps/server/src/cloud/relayResponse.test.ts created+145

...	...	@@ -0,0 +1,145 @@
	1	import { expect, it } from "@effect/vitest";
	2	import * as Effect from "effect/Effect";
	3	import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
	4	import * as HttpClientError from "effect/unstable/http/HttpClientError";
	5
	6	import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
	7
	8	const response = (
	9	status: number,
	10	body: string \| Record<string, unknown>,
	11	headers?: Record<string, string>,
	12	) =>
	13	HttpClientResponse.fromWeb(
	14	HttpClientRequest.post("https://relay.example.test/v1/client/environment-links"),
	15	typeof body === "string"
	16	? new Response(body, { status, ...(headers ? { headers } : {}) })
	17	: Response.json(body, { status, ...(headers ? { headers } : {}) }),
	18	);
	19
	20	it("reports a transport failure category without exposing request or cause details", () => {
	21	const error = relayRequestError(
	22	new HttpClientError.HttpClientError({
	23	reason: new HttpClientError.TransportError({
	24	request: HttpClientRequest.post("https://relay.example.test/link?token=private-token"),
	25	description: "private transport details",
	26	cause: new Error("private cause details"),
	27	}),
	28	}),
	29	);
	30
	31	expect(error._tag).toBe("EnvironmentHttpInternalServerError");
	32	expect(error.message).toContain("TransportError");
	33	expect(error.message).toContain("network connection");
	34	expect(error.message).not.toContain("relay.example.test");
	35	expect(error.message).not.toContain("private");
	36	expect(shouldRetryCloudLink(error)).toBe(true);
	37	});
	38
	39	it.effect("reports the tunnel limit and relay trace instead of a generic 403", () =>
	40	Effect.gen(function* () {
	41	const error = yield* filterRelayResponse(
	42	response(403, {
	43	_tag: "RelayEnvironmentLinkLimitExceededError",
	44	code: "environment_link_limit_exceeded",
	45	maxTunnels: 3,
	46	traceId: "trace-limit",
	47	}),
	48	).pipe(Effect.mapError(relayRequestError), Effect.flip);
	49
	50	expect(error._tag).toBe("EnvironmentHttpForbiddenError");
	51	expect(error.message).toContain("at most 3 tunnels");
	52	expect(error.message).toContain("Unlink an unused environment");
	53	expect(error.message).toContain("Trace ID: trace-limit");
	54	}),
	55	);
	56
	57	it.effect("makes revoked authorization actionable and non-retryable", () =>
	58	Effect.gen(function* () {
	59	const error = yield* filterRelayResponse(
	60	response(401, {
	61	_tag: "RelayAuthInvalidError",
	62	code: "auth_invalid",
	63	reason: "invalid_bearer",
	64	traceId: "trace-auth",
	65	}),
	66	).pipe(Effect.mapError(relayRequestError), Effect.flip);
	67
	68	expect(error._tag).toBe("EnvironmentHttpUnauthorizedError");
	69	expect(error.message).toContain("invalid_bearer");
	70	expect(error.message).toContain("t3 connect login");
	71	expect(error.message).toContain("Trace ID: trace-auth");
	72	}),
	73	);
	74
	75	it.effect("reports an unrecognized access denial without printing its response body", () =>
	76	Effect.gen(function* () {
	77	const error = yield* filterRelayResponse(
	78	response(403, "<html>private upstream details</html>", {
	79	"content-type": "text/html",
	80	"cf-ray": "abcdef1234-IAD",
	81	}),
	82	).pipe(Effect.flip);
	83
	84	expect(error._tag).toBe("EnvironmentHttpForbiddenError");
	85	expect(error.message).toContain("HTTP 403");
	86	expect(error.message).toContain("proxy or firewall");
	87	expect(error.message).toContain("Cloudflare Ray ID: abcdef1234-IAD");
	88	expect(error.message).not.toContain("private upstream details");
	89	}),
	90	);
	91
	92	it.effect.each([408, 429, 500, 502, 503, 504])(
	93	"keeps transient HTTP %s failures retryable",
	94	(status) =>
	95	Effect.gen(function* () {
	96	const error = yield* filterRelayResponse(response(status, "unavailable")).pipe(Effect.flip);
	97	expect(error._tag).toBe("EnvironmentHttpInternalServerError");
	98	expect(error.message).toContain(`HTTP ${status}`);
	99	}),
	100	);
	101
	102	it.effect.each([
	103	{ status: 401, attempts: 1 },
	104	{ status: 403, attempts: 1 },
	105	{ status: 429, attempts: 2 },
	106	{ status: 503, attempts: 2 },
	107	])("stops rejected startup links but retries temporary failures: $status", ({ status, attempts }) =>
	108	Effect.gen(function* () {
	109	let requests = 0;
	110	const result = yield* Effect.suspend(() => {
	111	requests++;
	112	return filterRelayResponse(response(requests === 1 ? status : 200, "{}"));
	113	}).pipe(
	114	Effect.mapError(relayRequestError),
	115	Effect.retry({ while: shouldRetryCloudLink, times: 1 }),
	116	Effect.result,
	117	);
	118	expect(requests).toBe(attempts);
	119	expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success");
	120	}),
	121	);
	122
	123	it.effect("keeps the relay failure reason and trace when tunnel cleanup fails", () =>
	124	Effect.gen(function* () {
	125	const error = yield* filterRelayResponse(
	126	response(500, {
	127	_tag: "RelayInternalError",
	128	code: "internal_error",
	129	reason: "upstream_unavailable",
	130	traceId: "trace-cleanup",
	131	}),
	132	).pipe(Effect.flip);
	133
	134	expect(error._tag).toBe("EnvironmentHttpInternalServerError");
	135	expect(error.message).toContain("upstream_unavailable");
	136	expect(error.message).toContain("Trace ID: trace-cleanup");
	137	}),
	138	);
	139
	140	it.effect("leaves successful response bodies available to their decoder", () =>
	141	Effect.gen(function* () {
	142	const result = yield* filterRelayResponse(response(200, '{"ok":true}'));
	143	expect(yield* result.json).toEqual({ ok: true });
	144	}),
	145	);

apps/server/src/cloud/relayResponse.ts created+82

...	...	@@ -0,0 +1,82 @@
	1	import {
	2	EnvironmentHttpBadRequestError,
	3	EnvironmentHttpConflictError,
	4	EnvironmentHttpForbiddenError,
	5	EnvironmentHttpInternalServerError,
	6	EnvironmentHttpUnauthorizedError,
	7	} from "@t3tools/contracts";
	8	import { RelayProtectedError } from "@t3tools/contracts/relay";
	9	import * as Effect from "effect/Effect";
	10	import * as Option from "effect/Option";
	11	import * as Schema from "effect/Schema";
	12	import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
	13	import { isHttpClientError } from "effect/unstable/http/HttpClientError";
	14
	15	const isRelayResponseError = Schema.is(
	16	Schema.Union([
	17	EnvironmentHttpBadRequestError,
	18	EnvironmentHttpForbiddenError,
	19	EnvironmentHttpInternalServerError,
	20	EnvironmentHttpUnauthorizedError,
	21	]),
	22	);
	23
	24	export function relayRequestError(cause: unknown) {
	25	return isRelayResponseError(cause)
	26	? cause
	27	: new EnvironmentHttpInternalServerError({
	28	message: `Could not complete the T3 Connect relay request. ${isHttpClientError(cause) ? `The relay request failed (${cause.reason._tag}).` : "The relay returned an unexpected response."} Check this machine's network connection and relay availability, then retry.`,
	29	});
	30	}
	31
	32	const isPermanentCloudLinkError = Schema.is(
	33	Schema.Union([
	34	EnvironmentHttpBadRequestError,
	35	EnvironmentHttpForbiddenError,
	36	EnvironmentHttpUnauthorizedError,
	37	EnvironmentHttpConflictError,
	38	]),
	39	);
	40
	41	export const shouldRetryCloudLink = (error: unknown): boolean => !isPermanentCloudLinkError(error);
	42
	43	function recoveryHint(error: RelayProtectedError): string {
	44	switch (error._tag) {
	45	case "RelayEnvironmentLinkLimitExceededError":
	46	return "Unlink an unused environment in T3 Connect, then restart T3 Code on this machine.";
	47	case "RelayAuthInvalidError":
	48	return "Run `t3 connect login` to check this machine's authorization. If the stored credential was revoked, sign out with `t3 connect logout`, then run `t3 connect` again. Restart T3 Code after signing in.";
	49	case "RelayEnvironmentLinkProofExpiredError":
	50	case "RelayEnvironmentLinkProofInvalidError":
	51	return "Check this machine's date and time, update T3 Code, then restart it.";
	52	default:
	53	return "Retry when the relay is available. If this continues, include the trace ID when reporting it.";
	54	}
	55	}
	56
	57	/** Preserve relay diagnostics before converting permanent rejections into non-retryable errors. */
	58	export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(function* (
	59	response: HttpClientResponse.HttpClientResponse,
	60	) {
	61	if (response.status >= 200 && response.status < 300) return response;
	62	const decoded = yield* HttpClientResponse.schemaBodyJson(RelayProtectedError)(response).pipe(
	63	Effect.option,
	64	);
	65	const ray = response.headers["cf-ray"];
	66	const requestId = ray && /^[a-zA-Z0-9-]{1,128}$/.test(ray) ? ` Cloudflare Ray ID: ${ray}.` : "";
	67	const message = Option.isSome(decoded)
	68	? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.`
	69	: `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`;
	70
	71	if (response.status === 401) return yield* new EnvironmentHttpUnauthorizedError({ message });
	72	if (response.status === 403) return yield* new EnvironmentHttpForbiddenError({ message });
	73	if (
	74	response.status >= 400 &&
	75	response.status < 500 &&
	76	response.status !== 408 &&
	77	response.status !== 429
	78	) {
	79	return yield* new EnvironmentHttpBadRequestError({ message });
	80	}
	81	return yield* new EnvironmentHttpInternalServerError({ message });
	82	});

apps/server/src/server.ts+5-6

...	...	@@ -1,4 +1,5 @@
1	1	import { EnvironmentHttpApi, ProviderDriverKind } from "@t3tools/contracts";
	2	import * as Cause from "effect/Cause";
2	3	import * as Duration from "effect/Duration";
3	4	import * as Deferred from "effect/Deferred";
4	5	import * as Effect from "effect/Effect";
...	...	@@ -106,6 +107,7 @@ import {
106	107	releaseManagedTunnelOnShutdown,
107	108	} from "./cloud/http.ts";
108	109	import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts";
	110	import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";
109	111	import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
110	112	import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
111	113	import * as CloudCliState from "./cloud/CliState.ts";
...	...	@@ -675,7 +677,7 @@ export const makeServerLayer = Layer.unwrap(
675	677	Effect.catchCause((cause) =>
676	678	Effect.logWarning(
677	679	"Failed to release the managed tunnel on shutdown; the next link reuses it",
678		{ cause },
	680	{ errors: Cause.prettyErrors(cause).map((error) => error.message) },
679	681	),
680	682	),
681	683	Effect.asVoid,
...	...	@@ -707,10 +709,7 @@ export const makeServerLayer = Layer.unwrap(
707	709	// reachability after a restart.
708	710	yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(
709	711	Effect.retry({
710		while: (error) =>
711		error._tag !== "EnvironmentHttpBadRequestError" &&
712		error._tag !== "EnvironmentHttpUnauthorizedError" &&
713		error._tag !== "EnvironmentHttpConflictError",
	712	while: shouldRetryCloudLink,
714	713	schedule: Schedule.exponential("1 second").pipe(
715	714	Schedule.modifyDelay(({ duration }) =>
716	715	Effect.succeed(Duration.min(duration, Duration.seconds(30))),
...	...	@@ -721,7 +720,7 @@ export const makeServerLayer = Layer.unwrap(
721	720	Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),
722	721	Effect.catch((cause) =>
723	722	Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
724		cause,
	723	message: cause.message,
725	724	}),
726	725	),
727	726	);

docs/user/background-service.md+35-1

...	...	@@ -11,7 +11,8 @@ Install it with the latest T3 Code release:
11	11	npx t3@latest service install
12	12	```
13	13
14		Check whether it is installed:
	14	Check whether it is installed. On Linux this also checks whether the service is running, enabled
	15	at startup, and allowed to keep running after logout:
15	16
16	17	```sh
17	18	npx t3@latest service status
...	...	@@ -58,6 +59,8 @@ updates roll back with the server version. An older launcher may require one loc
58	59
59	60	Linux uses a systemd user unit at `~/.config/systemd/user/t3code.service`. The service starts
60	61	when the machine boots and keeps running after you log out (lingering is enabled during install).
	62	Setup checks the systemd user manager and enables lingering before installing a runtime or stopping
	63	an existing service. If that requires administrator permission, setup stops with a recovery command.
61	64
62	65	macOS uses a launch agent at `~/Library/LaunchAgents/com.t3tools.t3code.service.plist`. It
63	66	starts when you log in, not when the Mac boots, and it stops when you log out; macOS has no
...	...	@@ -87,3 +90,34 @@ background. This is only an onboarding shortcut: the service and T3 Connect are
87	90
88	91	Signing out of T3 Connect does not remove the service. Use `t3 service uninstall` when you no longer
89	92	want T3 Code to start in the background.
	93
	94	## Troubleshooting
	95
	96	Run `t3 service status` on the server machine. An installed version alone does not mean the service
	97	is running or will survive logout. Linux status reports these problems:
	98
	99	\| Code \| What it means \| Recovery \|
	100	\| -------------------------- \| -------------------------------------------------------------------------------- \| ------------------------------------------------------------------------------------------------------------------------------------------ \|
	101	\| `linger-disabled` \| The service stops after your last login session ends and does not start at boot. \| Run `sudo loginctl enable-linger "$(id -un)"`, then retry setup as your normal user. \|
	102	\| `linger-unavailable` \| T3 Code could not verify the logout setting. \| Run `loginctl show-user "$(id -un)" --property=Linger` and check that systemd-logind is available. \|
	103	\| `user-manager-unavailable` \| T3 Code cannot reach your systemd user manager. \| Run `systemctl --user status` in a login session for the service user. Install your distribution's systemd user-session support if needed. \|
	104	\| `service-disabled` \| The service is not enabled to start automatically. \| Run the repair command shown by `t3 service status`. \|
	105	\| `service-stopped` \| The service is installed but is not running. \| Read the service log and `systemctl --user status t3code.service`, then run the displayed repair command. \|
	106
	107	For an SSH host, run the administrator command in an interactive terminal so sudo can prompt for
	108	your password:
	109
	110	```sh
	111	ssh -t your-server 'sudo loginctl enable-linger "$(id -un)"'
	112	```
	113
	114	Run only the `loginctl` command with sudo. Running `t3` with sudo creates a separate installation and
	115	Connect identity for root. If an administrator is unavailable, run `t3 serve` in a terminal and
	116	keep that session open.
	117
	118	The repair command shown by status uses the CLI version, or the installed service version if that
	119	is newer. An older stable CLI therefore does not recommend downgrading a nightly installation.
	120	Setup leaves an existing service running if the user-manager or lingering check fails.
	121
	122	`t3 service status` prints the log path. The adjacent `server.trace.ndjson` file contains detailed
	123	server traces. For failures after authorization, see [T3 Connect troubleshooting](./remote-access.md#t3-connect-troubleshooting).

docs/user/remote-access.md+27

...	...	@@ -2,6 +2,33 @@
2	2
3	3	Use this when you want to connect to a T3 Code server from another device such as a phone, tablet, or separate desktop app.
4	4
	5	## T3 Connect troubleshooting
	6
	7	Run `t3 connect` on the server machine to authorize it and optionally install the background service.
	8	The authorization message means your sign-in was saved. The server must then start and establish its
	9	relay link before the machine is reachable.
	10
	11	`t3 connect status` reports saved authorization and link configuration, not a live reachability
	12	check. If the machine appears offline, run `t3 service status` on it and read the displayed log.
	13	On Linux, a service that works while SSH is open but stops after logout usually has lingering
	14	disabled. See [background service troubleshooting](./background-service.md#troubleshooting).
	15
	16	Relay errors include the returned reason and trace ID when available:
	17
	18	\| Error \| Next step \|
	19	\| ---------------------------------------------------------------- \| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- \|
	20	\| `environment_link_limit_exceeded` / managed tunnel limit reached \| Unlink an unused environment in T3 Connect, then restart T3 Code on this machine. \|
	21	\| `auth_invalid` / `invalid_bearer` \| Run `t3 connect login`. If the stored credential was revoked, run `t3 connect logout`, then `t3 connect` and restart the server. \|
	22	\| Expired or invalid link proof \| Check the server's date and time, update T3 Code, and restart it. Include the reason and trace ID if it still fails. \|
	23	\| HTTP 403 without a recognized error response \| Check relay access and any proxy or firewall restrictions. Include the Cloudflare Ray ID if one was returned; an HTTP status alone does not identify the cause. \|
	24	\| HTTP 408, 429, or 5xx \| The server retries temporary failures during startup for up to ten minutes. Check network and relay availability; include the trace ID when reporting a persistent failure. \|
	25
	26	Authorization and other permanent 4xx rejections stop the startup link attempt immediately.
	27	After correcting them, restart the server. For the Linux background service, use
	28	`systemctl --user restart t3code.service`; for a foreground server, stop it and run `t3 serve` again.
	29	Keep the diagnostic message and trace ID when reporting a problem. Do not post authorization codes,
	30	pairing URLs, or the contents of the secrets directory.
	31
5	32	## Quick Pairing for a Running Server
6	33
7	34	If a server is already running on this machine, mint a fresh pairing token and QR code without restarting anything: