authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-09 21:13:12-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-09 21:13:12-07:00
loga0a601c262a8a7b260d18fd806302183ea529036
treec71905916b5157b95f057318a43417542ab2a932
parent968ac2e82b8fbc3ce6d87fbb4159a20a4971d0d3
signature Signed by PGP key B5690EEEBB952194

fix(server): agents run in their own systemd scopes so an OOM kill spares the server (#17662)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

19 files changed, 827 insertions(+), 23 deletions(-)

apps/server/src/cloud/bootService.ts+13-4
...@@ -106,13 +106,22 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {...@@ -106,13 +106,22 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
106 // Let the launcher mark an explicit stop before it signals the server.106 // Let the launcher mark an explicit stop before it signals the server.
107 // systemd still SIGKILLs the whole cgroup if graceful shutdown times out.107 // systemd still SIGKILLs the whole cgroup if graceful shutdown times out.
108 "KillMode=mixed",108 "KillMode=mixed",
109 // Agent tool calls run as children of the server, so they share this cgroup.109 // Agents and terminals run in their own scopes in app-t3code-agents.slice
110 // With the systemd default of OOMPolicy=stop, the kernel killing one greedy110 // (see process/agentScope.ts). Short helper commands still share this
111 // child stops the whole unit: the server, every live agent, and the user's111 // cgroup, and with the systemd default of OOMPolicy=stop, the kernel
112 // connection. Keep running and let Restart=always cover the main process.112 // killing one of them stops the whole unit. Keep running and let
113 // Restart=always cover the main process.
113 "OOMPolicy=continue",114 "OOMPolicy=continue",
114 "Restart=always",115 "Restart=always",
115 "RestartSec=5",116 "RestartSec=5",
117 // The agents slice is a sibling in app.slice, so these weights keep the
118 // server responsive while agents saturate CPU or disk.
119 "CPUWeight=1000",
120 "IOWeight=1000",
121 // systemd-oomd only honors this when the cgroup it watches is owned by the
122 // same user. Distros watch user-1000.slice, which root owns, so there it is
123 // ignored. The agent scopes are what keep oomd off the server.
124 "ManagedOOMPreference=avoid",
116 `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,125 `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,
117 `StandardError=append:${escapeSystemdSpecifiers(plan.logPath)}`,126 `StandardError=append:${escapeSystemdSpecifiers(plan.logPath)}`,
118 "",127 "",
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+29-1
...@@ -31,6 +31,7 @@ import type {...@@ -31,6 +31,7 @@ import type {
31 WebSearchOutput,31 WebSearchOutput,
32} from "@anthropic-ai/claude-agent-sdk/sdk-tools";32} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
33import { parseCliArgs } from "@t3tools/shared/cliArgs";33import { parseCliArgs } from "@t3tools/shared/cliArgs";
34import { AgentScope } from "@t3tools/shared/AgentScope";
34import * as HostProcess from "@t3tools/shared/HostProcess";35import * as HostProcess from "@t3tools/shared/HostProcess";
35import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model";36import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model";
36import {37import {
...@@ -178,6 +179,8 @@ export function claudeProviderTurnTokenUsage(...@@ -178,6 +179,8 @@ export function claudeProviderTurnTokenUsage(
178}179}
179export const CLAUDE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CLAUDE_PROVIDER);180export const CLAUDE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CLAUDE_PROVIDER);
180const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({});181const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({});
182// Entry extensions the Claude SDK runs through node instead of executing.
183const CLAUDE_SCRIPT_EXTENSIONS = [".js", ".mjs", ".tsx", ".ts", ".jsx"];
181184
182export const ClaudeProviderCapabilitiesV2 = {185export const ClaudeProviderCapabilitiesV2 = {
183 sessions: {186 sessions: {
...@@ -605,6 +608,7 @@ export const layerQueryRunner: Layer.Layer<...@@ -605,6 +608,7 @@ export const layerQueryRunner: Layer.Layer<
605 Effect.gen(function* () {608 Effect.gen(function* () {
606 const crypto = yield* Crypto.Crypto;609 const crypto = yield* Crypto.Crypto;
607 const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;610 const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
611 const agentScope = yield* AgentScope;
608612
609 return ClaudeAgentSdkQueryRunner.of({613 return ClaudeAgentSdkQueryRunner.of({
610 allocateSessionId: crypto.randomUUIDv4.pipe(614 allocateSessionId: crypto.randomUUIDv4.pipe(
...@@ -627,11 +631,35 @@ export const layerQueryRunner: Layer.Layer<...@@ -627,11 +631,35 @@ export const layerQueryRunner: Layer.Layer<
627 ),631 ),
628 Stream.toAsyncIterable,632 Stream.toAsyncIterable,
629 );633 );
634 // The SDK spawns a native binary as `path ...executableArgs ...sdkArgs`,
635 // which lets the agent scope wrapper go in front of the CLI while the
636 // SDK keeps its own stderr capture and exit reporting. The SDK runs a
637 // script entry through node instead, so those launch unwrapped.
638 const binaryPath = input.options.pathToClaudeCodeExecutable;
639 const launch =
640 binaryPath === undefined ||
641 CLAUDE_SCRIPT_EXTENSIONS.some((ext) => binaryPath.endsWith(ext))
642 ? undefined
643 : yield* agentScope.wrap({
644 command: binaryPath,
645 args: [],
646 name: "claude",
647 threadId: input.threadId,
648 env: input.options.env,
649 });
650 const options =
651 launch === undefined || launch.args.length === 0
652 ? input.options
653 : {
654 ...input.options,
655 pathToClaudeCodeExecutable: launch.command,
656 executableArgs: [...launch.args, ...(input.options.executableArgs ?? [])],
657 };
630 const queryRuntime = yield* Effect.try({658 const queryRuntime = yield* Effect.try({
631 try: () =>659 try: () =>
632 query({660 query({
633 prompt,661 prompt,
634 options: input.options,662 options,
635 }),663 }),
636 catch: (cause) => queryRunnerError(cause, "query"),664 catch: (cause) => queryRunnerError(cause, "query"),
637 });665 });
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts+13-2
...@@ -30,6 +30,7 @@ import {...@@ -30,6 +30,7 @@ import {
30 type ProviderSetupError,30 type ProviderSetupError,
31} from "@t3tools/contracts";31} from "@t3tools/contracts";
32import { SKILL_MENTION_PATTERN } from "@t3tools/shared/composerInlineTokens";32import { SKILL_MENTION_PATTERN } from "@t3tools/shared/composerInlineTokens";
33import { AgentScope } from "@t3tools/shared/AgentScope";
33import * as HostProcess from "@t3tools/shared/HostProcess";34import * as HostProcess from "@t3tools/shared/HostProcess";
34import { dynamicToolTitle } from "@t3tools/shared/toolActivity";35import { dynamicToolTitle } from "@t3tools/shared/toolActivity";
35import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";36import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
...@@ -1357,8 +1358,16 @@ export const makeCodexAppServerSpawnCommand = Effect.fn(...@@ -1357,8 +1358,16 @@ export const makeCodexAppServerSpawnCommand = Effect.fn(
1357 readonly cwd?: string | undefined;1358 readonly cwd?: string | undefined;
1358 readonly env?: NodeJS.ProcessEnv | undefined;1359 readonly env?: NodeJS.ProcessEnv | undefined;
1359 readonly extendEnv?: boolean | undefined;1360 readonly extendEnv?: boolean | undefined;
1361 readonly threadId?: string | undefined;
1360}) {1362}) {
1361 const spawnCommand = yield* resolveSpawnCommand(input.command, input.args, {1363 const launch = yield* (yield* AgentScope).wrap({
1364 command: input.command,
1365 args: input.args,
1366 name: "codex",
1367 threadId: input.threadId,
1368 env: input.env,
1369 });
1370 const spawnCommand = yield* resolveSpawnCommand(launch.command, launch.args, {
1362 ...(input.env === undefined ? {} : { env: input.env }),1371 ...(input.env === undefined ? {} : { env: input.env }),
1363 ...(input.extendEnv === undefined ? {} : { extendEnv: input.extendEnv }),1372 ...(input.extendEnv === undefined ? {} : { extendEnv: input.extendEnv }),
1364 });1373 });
...@@ -1495,6 +1504,7 @@ export const layerAppServerClientFactory: Layer.Layer<...@@ -1495,6 +1504,7 @@ export const layerAppServerClientFactory: Layer.Layer<
1495 Effect.gen(function* () {1504 Effect.gen(function* () {
1496 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;1505 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1497 const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;1506 const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
1507 const agentScope = yield* AgentScope;
14981508
1499 return CodexAppServerClientFactory.of({1509 return CodexAppServerClientFactory.of({
1500 open: (input) =>1510 open: (input) =>
...@@ -1510,7 +1520,8 @@ export const layerAppServerClientFactory: Layer.Layer<...@@ -1510,7 +1520,8 @@ export const layerAppServerClientFactory: Layer.Layer<
1510 resolveCodexLaunchArgs(input.settings.launchArgs, input.environment),1520 resolveCodexLaunchArgs(input.settings.launchArgs, input.environment),
1511 ),1521 ),
1512 env: environment,1522 env: environment,
1513 });1523 threadId: input.threadId,
1524 }).pipe(Effect.provideService(AgentScope, agentScope));
1514 const handle = yield* spawner.spawn(command).pipe(1525 const handle = yield* spawner.spawn(command).pipe(
1515 Effect.provideService(Scope.Scope, scope),1526 Effect.provideService(Scope.Scope, scope),
1516 Effect.mapError(1527 Effect.mapError(
apps/server/src/orchestration-v2/ProviderEventIngestor.ts+8-1
...@@ -26,6 +26,7 @@ import * as Effect from "effect/Effect";...@@ -26,6 +26,7 @@ import * as Effect from "effect/Effect";
26import * as Layer from "effect/Layer";26import * as Layer from "effect/Layer";
27import * as Schema from "effect/Schema";27import * as Schema from "effect/Schema";
2828
29import { AgentScope } from "@t3tools/shared/AgentScope";
29import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";30import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
30import * as AnalyticsService from "../telemetry/AnalyticsService.ts";31import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
31import * as EventSink from "./EventSink.ts";32import * as EventSink from "./EventSink.ts";
...@@ -316,6 +317,7 @@ export const layer: Layer.Layer<...@@ -316,6 +317,7 @@ export const layer: Layer.Layer<
316 const idAllocator = yield* IdAllocator.IdAllocatorV2;317 const idAllocator = yield* IdAllocator.IdAllocatorV2;
317 const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor;318 const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor;
318 const analytics = yield* ProviderTurnAnalytics;319 const analytics = yield* ProviderTurnAnalytics;
320 const agentScope = yield* AgentScope;
319 const completedTurnAnalytics = new Set<string>();321 const completedTurnAnalytics = new Set<string>();
320322
321 const makeDomainEvent = (323 const makeDomainEvent = (
...@@ -652,6 +654,11 @@ export const layer: Layer.Layer<...@@ -652,6 +654,11 @@ export const layer: Layer.Layer<
652 return dismissed;654 return dismissed;
653 }655 }
654 const occurredAt = yield* DateTime.now;656 const occurredAt = yield* DateTime.now;
657 // A turn that failed because systemd killed the agent's scope for
658 // memory would otherwise read as a plain provider crash.
659 const failure = (yield* agentScope.oomKilled(input.threadId))
660 ? { ...input.event.failure, message: "Killed: out of memory.", code: "oom_kill" }
661 : input.event.failure;
655 return [662 return [
656 ...dismissed,663 ...dismissed,
657 yield* makeDomainEvent(input, {664 yield* makeDomainEvent(input, {
...@@ -664,7 +671,7 @@ export const layer: Layer.Layer<...@@ -664,7 +671,7 @@ export const layer: Layer.Layer<
664 providerThreadId: input.event.providerThreadId,671 providerThreadId: input.event.providerThreadId,
665 providerTurnId: input.event.providerTurnId,672 providerTurnId: input.event.providerTurnId,
666 itemOrdinal: input.event.failureItemOrdinal,673 itemOrdinal: input.event.failureItemOrdinal,
667 failure: input.event.failure,674 failure,
668 ...(input.event.retry === undefined ? {} : { retry: input.event.retry }),675 ...(input.event.retry === undefined ? {} : { retry: input.event.retry }),
669 ...(input.event.retryStartedAt === undefined676 ...(input.event.retryStartedAt === undefined
670 ? {}677 ? {}
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+53
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1import * as NetAddress from "effect/net/NetAddress";1import * as NetAddress from "effect/net/NetAddress";
2import * as NodeServices from "@effect/platform-node/NodeServices";2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { assert, it } from "@effect/vitest";3import { assert, it } from "@effect/vitest";
4import { AgentScope } from "@t3tools/shared/AgentScope";
4import {5import {
5 EnvironmentId,6 EnvironmentId,
6 type ModelSelection,7 type ModelSelection,
...@@ -1092,6 +1093,58 @@ it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () =>...@@ -1092,6 +1093,58 @@ it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () =>
1092 }),1093 }),
1093);1094);
10941095
1096it.effect(
1097 "ProviderSessionManagerV2 starts a thread's session without its last agent's OOM kill",
1098 () =>
1099 Effect.gen(function* () {
1100 const state = yield* Ref.make(emptyState);
1101 const owner = ThreadId.make("thread-provider-session-manager-oom-owner");
1102 const joiner = ThreadId.make("thread-provider-session-manager-oom-joiner");
1103 // Both threads' previous agents were OOM-killed. A new session may run
1104 // without a scope (Cursor, OpenCode, a shared Codex session), so it must
1105 // not inherit that answer.
1106 const oomKilledThreads = new Set<string>([owner, joiner]);
1107 const agentScope = {
1108 ...AgentScope.defaultValue(),
1109 oomKilled: (id: string) => Effect.sync(() => oomKilledThreads.has(id)),
1110 clear: (id: string) => Effect.sync(() => void oomKilledThreads.delete(id)),
1111 };
1112
1113 const effect = Effect.gen(function* () {
1114 const eventSink = yield* EventSink.EventSinkV2;
1115 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1116 const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1117 const now = yield* DateTime.now;
1118 const providerSessionId = yield* idAllocator.allocate.providerSession({
1119 providerInstanceId: modelSelection.instanceId,
1120 threadId: owner,
1121 });
1122 yield* eventSink.write({
1123 events: [
1124 yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }),
1125 yield* makeThreadCreatedEvent({ idAllocator, threadId: joiner, now }),
1126 ],
1127 });
1128 yield* manager.open({ threadId: owner, providerSessionId, modelSelection, runtimePolicy });
1129 assert.isFalse(yield* agentScope.oomKilled(owner));
1130
1131 yield* manager.open({ threadId: joiner, providerSessionId, modelSelection, runtimePolicy });
1132 assert.isFalse(yield* agentScope.oomKilled(joiner));
1133
1134 // Reopening a session the thread already uses keeps its answer.
1135 oomKilledThreads.add(owner);
1136 yield* manager.open({ threadId: owner, providerSessionId, modelSelection, runtimePolicy });
1137 assert.isTrue(yield* agentScope.oomKilled(owner));
1138 assert.equal((yield* Ref.get(state)).openCount, 1);
1139 });
1140
1141 yield* effect.pipe(
1142 Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })),
1143 Effect.provideService(AgentScope, agentScope),
1144 );
1145 }),
1146);
1147
1095it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake", () =>1148it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake", () =>
1096 Effect.gen(function* () {1149 Effect.gen(function* () {
1097 const state = yield* Ref.make(emptyState);1150 const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts+9-4
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import { AgentScope } from "@t3tools/shared/AgentScope";
1import * as KeyedLock from "@t3tools/shared/KeyedLock";2import * as KeyedLock from "@t3tools/shared/KeyedLock";
2import { resolveProjectSettings } from "@t3tools/shared/projectSettings";3import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3import {4import {
...@@ -358,6 +359,7 @@ export const layerWithOptions = (...@@ -358,6 +359,7 @@ export const layerWithOptions = (
358 const idAllocator = yield* IdAllocator.IdAllocatorV2;359 const idAllocator = yield* IdAllocator.IdAllocatorV2;
359 const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;360 const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
360 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;361 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
362 const agentScope = yield* AgentScope;
361 const agentAccessSettings = Effect.fn("ProviderSessionManagerV2.agentAccessSettings")(363 const agentAccessSettings = Effect.fn("ProviderSessionManagerV2.agentAccessSettings")(
362 function* (threadId: ThreadId) {364 function* (threadId: ThreadId) {
363 if (Option.isNone(serverSettings)) return { browser: true, device: false };365 if (Option.isNone(serverSettings)) return { browser: true, device: false };
...@@ -2037,16 +2039,16 @@ export const layerWithOptions = (...@@ -2037,16 +2039,16 @@ export const layerWithOptions = (
2037 const key = sessionKey(input.providerSessionId);2039 const key = sessionKey(input.providerSessionId);
2038 const existing = (yield* Ref.get(sessions)).get(key);2040 const existing = (yield* Ref.get(sessions)).get(key);
2039 if (existing !== undefined) {2041 if (existing !== undefined) {
2040 if (2042 const attached = existing.attachedThreadIds.has(input.threadId);
2041 !existing.attachedThreadIds.has(input.threadId) &&2043 if (!attached && !existing.supportsMultipleProviderThreads) {
2042 !existing.supportsMultipleProviderThreads
2043 ) {
2044 return yield* new ProviderSessionOpenError({2044 return yield* new ProviderSessionOpenError({
2045 instanceId: input.modelSelection.instanceId,2045 instanceId: input.modelSelection.instanceId,
2046 providerSessionId: input.providerSessionId,2046 providerSessionId: input.providerSessionId,
2047 cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`,2047 cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`,
2048 });2048 });
2049 }2049 }
2050 // Joining a shared session is a new session for this thread.
2051 if (!attached) yield* agentScope.clear(input.threadId);
2050 yield* ensureThreadAttached({2052 yield* ensureThreadAttached({
2051 providerSessionId: input.providerSessionId,2053 providerSessionId: input.providerSessionId,
2052 threadId: input.threadId,2054 threadId: input.threadId,
...@@ -2083,6 +2085,9 @@ export const layerWithOptions = (...@@ -2083,6 +2085,9 @@ export const layerWithOptions = (
2083 }2085 }
2084 });2086 });
2085 const sessionScope = yield* Scope.fork(sessionScopes);2087 const sessionScope = yield* Scope.fork(sessionScopes);
2088 // The previous session's agent scope must not explain this
2089 // session's failures, even when this provider runs without one.
2090 yield* agentScope.clear(input.threadId);
2086 const runtime = yield* adapter2091 const runtime = yield* adapter
2087 .openSession({2092 .openSession({
2088 threadId: input.threadId,2093 threadId: input.threadId,
apps/server/src/process/agentScope.test.ts created+145
...@@ -0,0 +1,145 @@
1// @effect-diagnostics nodeBuiltinImport:off -- fake systemd binaries live in a temp dir.
2import * as NodeFS from "node:fs";
3import * as NodeOS from "node:os";
4import * as NodePath from "node:path";
5
6import * as NodeServices from "@effect/platform-node/NodeServices";
7import { describe, expect, it } from "@effect/vitest";
8import * as HostProcess from "@t3tools/shared/HostProcess";
9import * as Effect from "effect/Effect";
10import { ChildProcessSpawner } from "effect/process";
11
12import * as ProcessRunner from "../processRunner.ts";
13import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts";
14
15const GiB = 1024 ** 3;
16
17const running: ScopeState = {
18 loadState: "loaded",
19 activeState: "active",
20 result: "success",
21 oomKills: 0,
22 populated: true,
23};
24
25describe("classifyScope", () => {
26 it("reports an OOM kill only once systemd records it", () => {
27 expect(classifyScope({ ...running, activeState: "failed", result: "oom-kill" })).toBe(
28 "oom-killed",
29 );
30 // The agent pipe can close before systemd handles the kill.
31 expect(classifyScope({ ...running, oomKills: 1 })).toBe("stopping");
32 expect(classifyScope({ ...running, populated: false })).toBe("stopping");
33 expect(classifyScope({ ...running, activeState: "deactivating" })).toBe("stopping");
34 });
35
36 it("does not wait on a live agent or a scope that ended for another reason", () => {
37 expect(classifyScope(running)).toBe("running");
38 expect(classifyScope({ ...running, activeState: "inactive", populated: undefined })).toBe(
39 "gone",
40 );
41 expect(classifyScope({ ...running, loadState: "not-found", activeState: "inactive" })).toBe(
42 "gone",
43 );
44 });
45});
46
47describe("agentSliceMemoryLimits", () => {
48 it("leaves 6 GB before throttling and 4 GB before a kill on large machines", () => {
49 expect(agentSliceMemoryLimits(64 * GiB)).toEqual({ high: 58 * GiB, max: 60 * GiB });
50 });
51
52 it("still gives agents most of a small machine", () => {
53 expect(agentSliceMemoryLimits(8 * GiB)).toEqual({ high: 4 * GiB, max: 6 * GiB });
54 });
55});
56
57// Runs the real service against a fake user manager. `units` holds the
58// ActiveState and Result that `systemctl show` reports for each scope.
59const withFakeSystemd = <A, E>(
60 body: (input: {
61 readonly scope: Effect.Success<typeof make>;
62 readonly units: Map<string, { activeState: string; result: string }>;
63 }) => Effect.Effect<A, E>,
64) =>
65 Effect.gen(function* () {
66 const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
67 for (const name of ["systemd-run", "systemctl", "agent"]) {
68 NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
69 }
70 const units = new Map<string, { activeState: string; result: string }>();
71 const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({
72 stdout,
73 stderr: "",
74 code: ChildProcessSpawner.ExitCode(0),
75 timedOut: false,
76 stdoutTruncated: false,
77 stderrTruncated: false,
78 stdoutInvalidUtf8: false,
79 stderrInvalidUtf8: false,
80 });
81 const runner = ProcessRunner.ProcessRunner.of({
82 run: ({ args }) =>
83 Effect.sync(() => {
84 const [, verb, unit = ""] = args;
85 if (verb === "reset-failed") units.delete(unit);
86 if (verb !== "show") return output("");
87 const state = units.get(unit);
88 return output(
89 state === undefined
90 ? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
91 : `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n`,
92 );
93 }),
94 });
95 const scope = yield* make.pipe(
96 Effect.provideService(ProcessRunner.ProcessRunner, runner),
97 Effect.provideService(HostProcess.Platform, "linux"),
98 Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
99 );
100 return yield* body({ scope, units }).pipe(
101 Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
102 );
103 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer));
104
105const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>
106 launch.args.find((arg) => arg.startsWith("--unit="))?.slice("--unit=".length) ?? "";
107
108describe("AgentScope service", () => {
109 it.effect("labels a thread from its newest scope until its next session clears it", () =>
110 withFakeSystemd(({ scope, units }) =>
111 Effect.gen(function* () {
112 const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
113 units.set(unitOf(first), { activeState: "failed", result: "oom-kill" });
114 const second = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
115 units.set(unitOf(second), { activeState: "active", result: "success" });
116 // The older scope was OOM-killed, but the newest one is still running.
117 expect(yield* scope.oomKilled("a")).toBe(false);
118
119 units.set(unitOf(second), { activeState: "failed", result: "oom-kill" });
120 expect(yield* scope.oomKilled("a")).toBe(true);
121 // A retried failure reads the same, even after the failed unit is reset.
122 units.delete(unitOf(second));
123 expect(yield* scope.oomKilled("a")).toBe(true);
124
125 // The next session may run without a scope, for example on Cursor.
126 yield* scope.clear("a");
127 expect(yield* scope.oomKilled("a")).toBe(false);
128 }),
129 ),
130 );
131
132 it.effect("keeps a running scope while hundreds of other threads launch", () =>
133 withFakeSystemd(({ scope, units }) =>
134 Effect.gen(function* () {
135 const live = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "live" });
136 units.set(unitOf(live), { activeState: "active", result: "success" });
137 for (let index = 0; index < 600; index++) {
138 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: `other-${index}` });
139 }
140 units.set(unitOf(live), { activeState: "failed", result: "oom-kill" });
141 expect(yield* scope.oomKilled("live")).toBe(true);
142 }),
143 ),
144 );
145});
apps/server/src/process/agentScope.ts created+318
...@@ -0,0 +1,318 @@
1// @effect-diagnostics nodeBuiltinImport:off -- unit names and RAM size come from the host.
2import * as NodeCrypto from "node:crypto";
3import * as NodeOS from "node:os";
4
5import {
6 AgentScope,
7 agentScopeCommand,
8 type AgentScopeShape,
9 type AgentSystemdScope,
10} from "@t3tools/shared/AgentScope";
11import * as HostProcess from "@t3tools/shared/HostProcess";
12import { resolveCommandPath } from "@t3tools/shared/shell";
13import * as Effect from "effect/Effect";
14import * as FileSystem from "effect/FileSystem";
15import * as Layer from "effect/Layer";
16import * as Path from "effect/Path";
17
18import * as ProcessRunner from "../processRunner.ts";
19
20/**
21 * Agents and terminals share this slice. It sits in app.slice next to the
22 * server's own unit, so the server's CPUWeight and IOWeight compete with the
23 * agents directly.
24 */
25const AGENT_SLICE = "app-t3code-agents.slice";
26
27const GiB = 1024 ** 3;
28
29/**
30 * Memory limits for the agents slice. Agents together leave about 6 GB free
31 * before they are throttled and 4 GB before the kernel kills one, so the
32 * server, sshd, and tailscaled keep working. Small machines still give agents
33 * at least half of RAM before throttling and three quarters before a kill.
34 */
35export function agentSliceMemoryLimits(totalBytes: number) {
36 return {
37 high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)),
38 max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)),
39 };
40}
41
42export interface ScopeState {
43 readonly loadState: string;
44 readonly activeState: string;
45 readonly result: string;
46 /** `oom_kill` from the scope's memory.events, while its cgroup exists. */
47 readonly oomKills: number;
48 /** `populated` from the scope's cgroup.events, while its cgroup exists. */
49 readonly populated: boolean | undefined;
50}
51
52/**
53 * Reads one scope as the server sees it right after the agent process exits.
54 * Our pipe can close before systemd handles the kill, so a scope that is
55 * still stopping, is empty, or already counted an OOM kill is "stopping":
56 * systemd records `Result=oom-kill` for it in a moment.
57 */
58export function classifyScope(state: ScopeState): "oom-killed" | "gone" | "running" | "stopping" {
59 if (state.result === "oom-kill") return "oom-killed";
60 if (
61 state.loadState !== "loaded" ||
62 state.activeState === "inactive" ||
63 state.activeState === "failed"
64 ) {
65 return "gone";
66 }
67 if (state.activeState === "deactivating" || state.oomKills > 0 || state.populated === false) {
68 return "stopping";
69 }
70 return "running";
71}
72
73function parseKeyValues(text: string): Map<string, string> {
74 const values = new Map<string, string>();
75 for (const line of text.split("\n")) {
76 const index = line.indexOf("=");
77 if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim());
78 }
79 return values;
80}
81
82function parseCgroupCounter(text: string, key: string): number | undefined {
83 const match = new RegExp(`^${key} (\\d+)$`, "m").exec(text);
84 return match === null ? undefined : Number(match[1]);
85}
86
87const STOPPING_POLL_INTERVAL = "100 millis";
88const STOPPING_POLL_ATTEMPTS = 20;
89// A stopping scope can wait out TimeoutStopSec (90 s by default) for a stuck process.
90const CLEAR_FAILED_INTERVAL = "1 second";
91const CLEAR_FAILED_ATTEMPTS = 120;
92const MAX_TRACKED_THREADS = 500;
93
94export const make = Effect.gen(function* () {
95 const platform = yield* HostProcess.Platform;
96 if (platform !== "linux") return AgentScope.defaultValue();
97
98 const runner = yield* ProcessRunner.ProcessRunner;
99 const fileSystem = yield* FileSystem.FileSystem;
100 const path = yield* Path.Path;
101 const hostEnvironment = yield* HostProcess.Environment;
102 const layerScope = yield* Effect.scope;
103
104 const resolve = (command: string, env: NodeJS.ProcessEnv) =>
105 resolveCommandPath(command, { env }).pipe(
106 Effect.provideService(FileSystem.FileSystem, fileSystem),
107 Effect.provideService(Path.Path, path),
108 Effect.provideService(HostProcess.Platform, platform),
109 Effect.option,
110 );
111
112 const run = (command: string, args: ReadonlyArray<string>) =>
113 runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option);
114
115 // Runs once, on the first agent launch. It proves this process can reach a
116 // systemd user manager, creates the agents slice, and sets its limits.
117 const systemd = yield* Effect.cached(
118 Effect.gen(function* () {
119 const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;
120 if (runtimeDir === undefined || runtimeDir.length === 0) return undefined;
121 const systemdRun = yield* resolve("systemd-run", hostEnvironment);
122 const systemctl = yield* resolve("systemctl", hostEnvironment);
123 if (systemdRun._tag === "None" || systemctl._tag === "None") return undefined;
124 const probe = (options: ReadonlyArray<string>) =>
125 run(systemdRun.value, [
126 "--user",
127 "--scope",
128 "--quiet",
129 "--collect",
130 `--slice=${AGENT_SLICE}`,
131 ...options,
132 "--",
133 "true",
134 ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0));
135 // Stop the whole scope when the kernel kills one of its processes, and
136 // record the result as oom-kill. Scopes take OOMPolicy from systemd 253;
137 // older user managers still get plain scopes, without the OOM label.
138 const oomOptions = ["--property=OOMPolicy=stop"];
139 const scopeOptions = (yield* probe(oomOptions))
140 ? oomOptions
141 : (yield* probe([]))
142 ? []
143 : undefined;
144 if (scopeOptions === undefined) {
145 yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
146 return undefined;
147 }
148 const limits = agentSliceMemoryLimits(NodeOS.totalmem());
149 const limited = yield* run(systemctl.value, [
150 "--user",
151 "set-property",
152 "--runtime",
153 AGENT_SLICE,
154 `MemoryHigh=${limits.high}`,
155 `MemoryMax=${limits.max}`,
156 ]);
157 if (limited._tag === "None" || limited.value.code !== 0) {
158 yield* Effect.logWarning("Could not set memory limits on the agents slice", {
159 stderr: limited._tag === "Some" ? limited.value.stderr.trim() : undefined,
160 });
161 }
162 yield* Effect.logInfo("Agent scopes are on", {
163 slice: AGENT_SLICE,
164 ...limits,
165 oomPolicy: scopeOptions.length > 0,
166 });
167 return {
168 systemdRun: systemdRun.value,
169 systemctl: systemctl.value,
170 runtimeDir,
171 scopeOptions,
172 };
173 }),
174 );
175
176 // The newest scope per thread, oldest thread first. A thread runs one agent
177 // at a time, so only its latest scope can explain a failure.
178 const scopeByThread = new Map<string, { unit: string; oomKilled: boolean }>();
179
180 const readCgroupFile = (controlGroup: string, file: string) =>
181 fileSystem
182 .readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
183 .pipe(Effect.orElseSucceed(() => ""));
184
185 const readScope = (systemctl: string, unit: string) =>
186 Effect.gen(function* () {
187 const shown = yield* run(systemctl, [
188 "--user",
189 "show",
190 unit,
191 "--property=LoadState,ActiveState,Result,ControlGroup",
192 ]);
193 // A failed query says nothing about the scope; the caller keeps the unit.
194 if (shown._tag === "None" || shown.value.code !== 0) return undefined;
195 const values = parseKeyValues(shown.value.stdout);
196 const controlGroup = values.get("ControlGroup") ?? "";
197 const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, "memory.events") : "";
198 const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
199 const populated = parseCgroupCounter(cgroupEvents, "populated");
200 return {
201 loadState: values.get("LoadState") ?? "",
202 activeState: values.get("ActiveState") ?? "",
203 result: values.get("Result") ?? "",
204 oomKills: parseCgroupCounter(memoryEvents, "oom_kill") ?? 0,
205 populated: populated === undefined ? undefined : populated === 1,
206 } satisfies ScopeState;
207 });
208
209 const clearFailed = (systemctl: string, unit: string) =>
210 Effect.gen(function* () {
211 for (let attempt = 0; attempt < CLEAR_FAILED_ATTEMPTS; attempt++) {
212 const state = yield* readScope(systemctl, unit);
213 if (state === undefined) return;
214 if (state.activeState === "failed") {
215 yield* run(systemctl, ["--user", "reset-failed", unit]);
216 return;
217 }
218 if (state.activeState !== "deactivating") return;
219 yield* Effect.sleep(CLEAR_FAILED_INTERVAL);
220 }
221 });
222
223 // Failed scopes stay loaded so we can read their result. Clear them once
224 // systemd finishes stopping them.
225 const forget = (systemctl: string, unit: string) =>
226 clearFailed(systemctl, unit).pipe(Effect.forkIn(layerScope), Effect.asVoid);
227
228 // Drops the oldest thread's scope once it has ended. A scope that still
229 // runs moves to the back, so its thread keeps its OOM answer and cleanup.
230 const evictOldest = (systemctl: string) =>
231 Effect.gen(function* () {
232 const oldest = scopeByThread.entries().next();
233 if (oldest.done === true) return;
234 const [threadId, tracked] = oldest.value;
235 const state = yield* readScope(systemctl, tracked.unit);
236 // A newer launch replaced it while we read, and forgets it itself.
237 if (scopeByThread.get(threadId) !== tracked) return;
238 scopeByThread.delete(threadId);
239 const status = state === undefined ? "running" : classifyScope(state);
240 if (status === "running" || status === "stopping") {
241 scopeByThread.set(threadId, tracked);
242 return;
243 }
244 yield* forget(systemctl, tracked.unit);
245 });
246
247 // Records a thread's newest scope, or forgets the thread when `unit` is
248 // undefined. The replaced scope has its failed state cleared.
249 const track = (systemctl: string, threadId: string, unit: string | undefined) =>
250 Effect.gen(function* () {
251 const previous = scopeByThread.get(threadId);
252 scopeByThread.delete(threadId);
253 if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false });
254 if (previous !== undefined) yield* forget(systemctl, previous.unit);
255 if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);
256 });
257
258 const oomKilled: AgentScopeShape["oomKilled"] = (threadId) =>
259 Effect.gen(function* () {
260 const tracked = scopeByThread.get(threadId);
261 if (tracked === undefined) return false;
262 if (tracked.oomKilled) return true;
263 const scope = yield* systemd;
264 if (scope === undefined) return false;
265 for (let attempt = 0; attempt < STOPPING_POLL_ATTEMPTS; attempt++) {
266 const state = yield* readScope(scope.systemctl, tracked.unit);
267 if (state === undefined) return false;
268 const status = classifyScope(state);
269 if (status === "running") return false;
270 if (status === "stopping") {
271 yield* Effect.sleep(STOPPING_POLL_INTERVAL);
272 continue;
273 }
274 yield* forget(scope.systemctl, tracked.unit);
275 if (status === "gone") return false;
276 yield* Effect.logWarning("Agent scope was killed: out of memory", {
277 threadId,
278 unit: tracked.unit,
279 });
280 tracked.oomKilled = true;
281 return true;
282 }
283 return false;
284 });
285
286 const clear: AgentScopeShape["clear"] = (threadId) =>
287 Effect.gen(function* () {
288 if (!scopeByThread.has(threadId)) return;
289 const scope = yield* systemd;
290 if (scope !== undefined) yield* track(scope.systemctl, threadId, undefined);
291 });
292
293 return {
294 wrap: Effect.fn("AgentScope.wrap")(function* ({ command, args, name, threadId, env }) {
295 const resolved = yield* resolve(command, env ?? hostEnvironment);
296 // Leave unresolved commands alone so the spawn reports the missing binary.
297 if (resolved._tag === "None") return { command, args };
298 const scope = yield* systemd;
299 if (scope === undefined) return agentScopeCommand({ command: resolved.value, args });
300 const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`;
301 if (threadId !== undefined) yield* track(scope.systemctl, threadId, unit);
302 const systemdScope: AgentSystemdScope = {
303 systemdRun: scope.systemdRun,
304 runtimeDir: scope.runtimeDir,
305 slice: AGENT_SLICE,
306 unit,
307 // Without a thread nothing reads the result, so let systemd unload
308 // the unit even when it fails.
309 options: threadId === undefined ? [...scope.scopeOptions, "--collect"] : scope.scopeOptions,
310 };
311 return agentScopeCommand({ command: resolved.value, args, scope: systemdScope });
312 }),
313 oomKilled,
314 clear,
315 } satisfies AgentScopeShape;
316});
317
318export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer));
apps/server/src/server.ts+3
...@@ -31,6 +31,7 @@ import * as ServerHttp from "./http.ts";...@@ -31,6 +31,7 @@ import * as ServerHttp from "./http.ts";
31import { guardHttpResponseWriteErrors } from "./httpResponseErrorGuard.ts";31import { guardHttpResponseWriteErrors } from "./httpResponseErrorGuard.ts";
32import { fixPath } from "./os-jank.ts";32import { fixPath } from "./os-jank.ts";
33import * as Ws from "./ws.ts";33import * as Ws from "./ws.ts";
34import * as AgentScopeLive from "./process/agentScope.ts";
34import * as ExternalLauncher from "./process/externalLauncher.ts";35import * as ExternalLauncher from "./process/externalLauncher.ts";
35import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts";36import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts";
36import * as PullRequestHttp from "./pullRequest/http.ts";37import * as PullRequestHttp from "./pullRequest/http.ts";
...@@ -1100,6 +1101,8 @@ const layerMakeServer = Layer.unwrap(...@@ -1100,6 +1101,8 @@ const layerMakeServer = Layer.unwrap(
1100 Layer.provideMerge(FetchHttpClient.layer),1101 Layer.provideMerge(FetchHttpClient.layer),
1101 // PR reads, Git operations, and WebSocket discovery share one process limiter.1102 // PR reads, Git operations, and WebSocket discovery share one process limiter.
1102 Layer.provide(VcsProcess.layer),1103 Layer.provide(VcsProcess.layer),
1104 // Every agent and terminal spawn reads this, so it sits below everything.
1105 Layer.provideMerge(AgentScopeLive.layer),
1103 Layer.provideMerge(layerPlatformServices),1106 Layer.provideMerge(layerPlatformServices),
1104 );1107 );
1105 }),1108 }),
apps/server/src/terminal/Manager.ts+10-2
...@@ -42,6 +42,7 @@ import {...@@ -42,6 +42,7 @@ import {
42} from "@t3tools/contracts";42} from "@t3tools/contracts";
43import { makeKeyedCoalescingWorker } from "@t3tools/shared/KeyedCoalescingWorker";43import { makeKeyedCoalescingWorker } from "@t3tools/shared/KeyedCoalescingWorker";
44import * as KeyedLock from "@t3tools/shared/KeyedLock";44import * as KeyedLock from "@t3tools/shared/KeyedLock";
45import { AgentScope } from "@t3tools/shared/AgentScope";
45import * as HostProcess from "@t3tools/shared/HostProcess";46import * as HostProcess from "@t3tools/shared/HostProcess";
46import { mergePathEntries } from "@t3tools/shared/shell";47import { mergePathEntries } from "@t3tools/shared/shell";
4748
...@@ -1480,6 +1481,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func...@@ -1480,6 +1481,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
1480 const historyByteLimit = options.historyByteLimit ?? DEFAULT_HISTORY_BYTE_LIMIT;1481 const historyByteLimit = options.historyByteLimit ?? DEFAULT_HISTORY_BYTE_LIMIT;
1481 const platform = yield* HostProcess.Platform;1482 const platform = yield* HostProcess.Platform;
1482 const architecture = yield* HostProcess.Architecture;1483 const architecture = yield* HostProcess.Architecture;
1484 const agentScope = yield* AgentScope;
1483 // Terminals must inherit the user's full environment (minus the blocklist1485 // Terminals must inherit the user's full environment (minus the blocklist
1484 // applied in createTerminalSpawnEnv) — an allowlist here silently strips1486 // applied in createTerminalSpawnEnv) — an allowlist here silently strips
1485 // things like PSModulePath, DISPLAY, proxies, and toolchain variables.1487 // things like PSModulePath, DISPLAY, proxies, and toolchain variables.
...@@ -2180,10 +2182,16 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func...@@ -2180,10 +2182,16 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
2180 );2182 );
2181 }2183 }
21822184
2185 const launch = yield* agentScope.wrap({
2186 command: candidate.shell,
2187 args: candidate.args ?? [],
2188 name: "terminal",
2189 env: spawnEnv,
2190 });
2183 const attempt = yield* Effect.result(2191 const attempt = yield* Effect.result(
2184 options.ptyAdapter.spawn({2192 options.ptyAdapter.spawn({
2185 shell: candidate.shell,2193 shell: launch.command,
2186 ...(candidate.args ? { args: candidate.args } : {}),2194 ...(launch.args.length > 0 ? { args: [...launch.args] } : {}),
2187 cwd: session.cwd,2195 cwd: session.cwd,
2188 cols: session.cols,2196 cols: session.cols,
2189 rows: session.rows,2197 rows: session.rows,
packages/provider-acp/src/server/AcpSessionRuntime.ts+18-1
...@@ -27,6 +27,7 @@ import * as EffectAcpErrors from "effect-acp/errors";...@@ -27,6 +27,7 @@ import * as EffectAcpErrors from "effect-acp/errors";
27import type * as EffectAcpSchema from "effect-acp/compat";27import type * as EffectAcpSchema from "effect-acp/compat";
28import type * as EffectAcpProtocol from "effect-acp/protocol";28import type * as EffectAcpProtocol from "effect-acp/protocol";
29import { resolveSpawnCommand } from "@t3tools/shared/shell";29import { resolveSpawnCommand } from "@t3tools/shared/shell";
30import { AgentScope, AgentScopeThreadId, RAISE_OOM_SCORE_LINE } from "@t3tools/shared/AgentScope";
30import * as HostProcess from "@t3tools/shared/HostProcess";31import * as HostProcess from "@t3tools/shared/HostProcess";
3132
32import { signalProcessGroup } from "@t3tools/provider-core/server/processGroup";33import { signalProcessGroup } from "@t3tools/provider-core/server/processGroup";
...@@ -275,6 +276,7 @@ export function wrapCommandForLinuxCgroup(...@@ -275,6 +276,7 @@ export function wrapCommandForLinuxCgroup(
275 "-c",276 "-c",
276 [277 [
277 "lease_path=$1; expected=$2; shift 2",278 "lease_path=$1; expected=$2; shift 2",
279 RAISE_OOM_SCORE_LINE,
278 'printf "%s\\n" "$$" > "$lease_path/cgroup.procs" || exit 125',280 'printf "%s\\n" "$$" > "$lease_path/cgroup.procs" || exit 125',
279 "actual=",281 "actual=",
280 "while IFS= read -r line; do",282 "while IFS= read -r line; do",
...@@ -1552,9 +1554,24 @@ export const make = (...@@ -1552,9 +1554,24 @@ export const make = (
1552 cause: new Error("Contained ACP command was not found on PATH"),1554 cause: new Error("Contained ACP command was not found on PATH"),
1553 });1555 });
1554 });1556 });
1557 // A cgroup lease is already its own leaf cgroup, so it only needs the
1558 // raised OOM score (in its wrapper). Other agents get their own scope.
1559 const scopedSpawnCommand =
1560 linuxCgroupLease !== undefined || spawnCommand.shell
1561 ? spawnCommand
1562 : {
1563 ...(yield* (yield* AgentScope).wrap({
1564 command: spawnCommand.command,
1565 args: spawnCommand.args,
1566 name: "acp",
1567 threadId: yield* AgentScopeThreadId,
1568 env: { ...process.env, ...options.spawn.env },
1569 })),
1570 shell: false,
1571 };
1555 const containedSpawnCommand =1572 const containedSpawnCommand =
1556 linuxCgroupLease === undefined1573 linuxCgroupLease === undefined
1557 ? spawnCommand1574 ? scopedSpawnCommand
1558 : {1575 : {
1559 ...wrapCommandForLinuxCgroup(1576 ...wrapCommandForLinuxCgroup(
1560 linuxCgroupLease,1577 linuxCgroupLease,
packages/provider-acp/src/server/adapter.ts+6
...@@ -31,6 +31,7 @@ import {...@@ -31,6 +31,7 @@ import {
31 type ThreadTokenUsageSnapshot,31 type ThreadTokenUsageSnapshot,
32 type ThreadId,32 type ThreadId,
33} from "@t3tools/contracts";33} from "@t3tools/contracts";
34import { AgentScope, AgentScopeThreadId } from "@t3tools/shared/AgentScope";
34import { modelSelectionsEqual } from "@t3tools/shared/model";35import { modelSelectionsEqual } from "@t3tools/shared/model";
35import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";36import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";
36import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";37import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";
...@@ -1486,6 +1487,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (...@@ -1486,6 +1487,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
1486 const crypto = yield* Crypto.Crypto;1487 const crypto = yield* Crypto.Crypto;
1487 const fileSystem = yield* FileSystem.FileSystem;1488 const fileSystem = yield* FileSystem.FileSystem;
1488 const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;1489 const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1490 const agentScope = yield* AgentScope;
1489 const idAllocator = yield* IdAllocator.IdAllocatorV2;1491 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1490 const host = yield* ProviderHost.ProviderHost;1492 const host = yield* ProviderHost.ProviderHost;
1491 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;1493 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
...@@ -6020,6 +6022,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (...@@ -6020,6 +6022,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
6020 Effect.provideService(Scope.Scope, runtimeScope),6022 Effect.provideService(Scope.Scope, runtimeScope),
6021 Effect.provideService(Crypto.Crypto, crypto),6023 Effect.provideService(Crypto.Crypto, crypto),
6022 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner),6024 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner),
6025 Effect.provideService(AgentScope, agentScope),
6026 Effect.provideService(AgentScopeThreadId, threadId ?? undefined),
6023 );6027 );
6024 });6028 });
60256029
...@@ -6099,6 +6103,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (...@@ -6099,6 +6103,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
6099 Effect.provideService(Scope.Scope, replacementScope),6103 Effect.provideService(Scope.Scope, replacementScope),
6100 Effect.provideService(Crypto.Crypto, crypto),6104 Effect.provideService(Crypto.Crypto, crypto),
6101 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner),6105 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner),
6106 Effect.provideService(AgentScope, agentScope),
6107 Effect.provideService(AgentScopeThreadId, threadId ?? undefined),
6102 );6108 );
6103 // Session setup may publish commands before it returns. Buffer those6109 // Session setup may publish commands before it returns. Buffer those
6104 // notifications, but do not expose request or extension handlers6110 // notifications, but do not expose request or extension handlers
packages/provider-muse/src/server/adapter.ts+16-5
...@@ -22,6 +22,7 @@ import {...@@ -22,6 +22,7 @@ import {
22 type ServerProviderModel,22 type ServerProviderModel,
23} from "@t3tools/contracts";23} from "@t3tools/contracts";
24import type { MuseSettings } from "../settings.ts";24import type { MuseSettings } from "../settings.ts";
25import { AgentScope } from "@t3tools/shared/AgentScope";
25import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";26import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
26import * as Cause from "effect/Cause";27import * as Cause from "effect/Cause";
27import * as DateTime from "effect/DateTime";28import * as DateTime from "effect/DateTime";
...@@ -250,6 +251,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (...@@ -250,6 +251,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
250 const providerHost = yield* ProviderHost.ProviderHost;251 const providerHost = yield* ProviderHost.ProviderHost;
251 const fileSystem = yield* FileSystem.FileSystem;252 const fileSystem = yield* FileSystem.FileSystem;
252 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;253 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
254 const agentScope = yield* AgentScope;
253255
254 const protocolError = (detail: string, payload?: unknown) =>256 const protocolError = (detail: string, payload?: unknown) =>
255 new ProviderAdapter.ProviderAdapterProtocolError({257 new ProviderAdapter.ProviderAdapterProtocolError({
...@@ -1367,15 +1369,24 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (...@@ -1367,15 +1369,24 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1367 const launchHost = Effect.fnUntraced(function* () {1369 const launchHost = Effect.fnUntraced(function* () {
1368 const epoch = ++hostEpoch;1370 const epoch = ++hostEpoch;
1369 const mcpSession = yield* mcpSessions.read(input.threadId);1371 const mcpSession = yield* mcpSessions.read(input.threadId);
1372 const environment = McpProviderSession.withAgentDeviceEnvironment(
1373 options.environment,
1374 mcpSession,
1375 );
1376 const launch = yield* agentScope.wrap({
1377 command: options.settings.binaryPath || "muse",
1378 args: [],
1379 name: "muse",
1380 threadId: input.threadId,
1381 env: environment,
1382 });
1370 const created = yield* Effect.acquireRelease(1383 const created = yield* Effect.acquireRelease(
1371 createMuseSdkHostEffect(1384 createMuseSdkHostEffect(
1372 {1385 {
1373 binaryPath: options.settings.binaryPath || "muse",1386 binaryPath: launch.command,
1387 launchArgs: launch.args,
1374 cwd,1388 cwd,
1375 environment: McpProviderSession.withAgentDeviceEnvironment(1389 environment,
1376 options.environment,
1377 mcpSession,
1378 ),
1379 runtimeMode: input.runtimePolicy.runtimeMode,1390 runtimeMode: input.runtimePolicy.runtimeMode,
1380 },1391 },
1381 options.createHost,1392 options.createHost,
packages/provider-muse/src/server/sdk.ts+3-1
...@@ -29,6 +29,8 @@ export interface MuseSdkHost {...@@ -29,6 +29,8 @@ export interface MuseSdkHost {
2929
30export interface MuseSdkHostOptions {30export interface MuseSdkHostOptions {
31 readonly binaryPath: string;31 readonly binaryPath: string;
32 /** Arguments before the serve arguments, for a launcher in front of the binary. */
33 readonly launchArgs?: ReadonlyArray<string>;
32 readonly cwd?: string;34 readonly cwd?: string;
33 readonly environment?: NodeJS.ProcessEnv;35 readonly environment?: NodeJS.ProcessEnv;
34 readonly runtimeMode?: RuntimeMode;36 readonly runtimeMode?: RuntimeMode;
...@@ -89,7 +91,7 @@ export async function createMuseSdkHost(...@@ -89,7 +91,7 @@ export async function createMuseSdkHost(
89 options.signal?.throwIfAborted();91 options.signal?.throwIfAborted();
90 const handshake = spawn({92 const handshake = spawn({
91 command: options.binaryPath,93 command: options.binaryPath,
92 args: museServeArgs(options),94 args: [...(options.launchArgs ?? []), ...museServeArgs(options)],
93 ...(options.cwd ? { cwd: options.cwd } : {}),95 ...(options.cwd ? { cwd: options.cwd } : {}),
94 // Callers pass an environment already built with makeMuseEnvironment.96 // Callers pass an environment already built with makeMuseEnvironment.
95 env: options.environment ?? makeMuseEnvironment(),97 env: options.environment ?? makeMuseEnvironment(),
packages/provider-opencode/src/server/OpenCodeRuntime.ts+10-1
...@@ -36,6 +36,7 @@ import { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshot...@@ -36,6 +36,7 @@ import { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshot
36import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";36import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";
37import { collectStreamAsString } from "@t3tools/provider-core/server/snapshotProbe";37import { collectStreamAsString } from "@t3tools/provider-core/server/snapshotProbe";
38import * as NetService from "@t3tools/shared/Net";38import * as NetService from "@t3tools/shared/Net";
39import { AgentScope } from "@t3tools/shared/AgentScope";
39import * as HostProcess from "@t3tools/shared/HostProcess";40import * as HostProcess from "@t3tools/shared/HostProcess";
40import { compareSemverVersions, parseSemver } from "@t3tools/shared/semver";41import { compareSemverVersions, parseSemver } from "@t3tools/shared/semver";
41import { resolveSpawnCommand } from "@t3tools/shared/shell";42import { resolveSpawnCommand } from "@t3tools/shared/shell";
...@@ -597,6 +598,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () {...@@ -597,6 +598,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () {
597 const netService = yield* NetService.NetService;598 const netService = yield* NetService.NetService;
598 const hostPlatform = yield* HostProcess.Platform;599 const hostPlatform = yield* HostProcess.Platform;
599 const serverLedger = yield* OpenCodeServerLedger.OpenCodeServerLedger;600 const serverLedger = yield* OpenCodeServerLedger.OpenCodeServerLedger;
601 const agentScope = yield* AgentScope;
600 const resolveCommand = (command: string, args: ReadonlyArray<string>, env?: NodeJS.ProcessEnv) =>602 const resolveCommand = (command: string, args: ReadonlyArray<string>, env?: NodeJS.ProcessEnv) =>
601 resolveSpawnCommand(command, args, env ? { env } : {});603 resolveSpawnCommand(command, args, env ? { env } : {});
602604
...@@ -693,7 +695,14 @@ const makeOpenCodeRuntime = Effect.gen(function* () {...@@ -693,7 +695,14 @@ const makeOpenCodeRuntime = Effect.gen(function* () {
693 ));695 ));
694 const timeoutMs = input.timeoutMs ?? DEFAULT_OPENCODE_SERVER_TIMEOUT_MS;696 const timeoutMs = input.timeoutMs ?? DEFAULT_OPENCODE_SERVER_TIMEOUT_MS;
695 const args = ["serve", `--hostname=${hostname}`, `--port=${port}`];697 const args = ["serve", `--hostname=${hostname}`, `--port=${port}`];
696 const spawnCommand = yield* resolveCommand(input.binaryPath, args, input.environment);698 // One server runs every session of the instance, so its scope has no thread.
699 const launch = yield* agentScope.wrap({
700 command: input.binaryPath,
701 args,
702 name: "opencode",
703 env: input.environment,
704 });
705 const spawnCommand = yield* resolveCommand(launch.command, launch.args, input.environment);
697 const serverPassword = resolveOpenCodeServerPassword({706 const serverPassword = resolveOpenCodeServerPassword({
698 external: false,707 external: false,
699 ...(input.serverPassword !== undefined ? { serverPassword: input.serverPassword } : {}),708 ...(input.serverPassword !== undefined ? { serverPassword: input.serverPassword } : {}),
packages/provider-pi/src/server/adapter.ts+10-1
...@@ -24,6 +24,7 @@...@@ -24,6 +24,7 @@
24 * updates has no matching T3 surface and is ignored.24 * updates has no matching T3 surface and is ignored.
25 */25 */
26import * as HostProcess from "@t3tools/shared/HostProcess";26import * as HostProcess from "@t3tools/shared/HostProcess";
27import { AgentScope } from "@t3tools/shared/AgentScope";
27import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";28import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
28import {29import {
29 defaultInstanceIdForDriver,30 defaultInstanceIdForDriver,
...@@ -387,6 +388,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (...@@ -387,6 +388,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
387 options: PiAdapterV2Options,388 options: PiAdapterV2Options,
388) {389) {
389 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;390 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
391 const agentScope = yield* AgentScope;
390 const fileSystem = yield* FileSystem.FileSystem;392 const fileSystem = yield* FileSystem.FileSystem;
391 const idAllocator = yield* IdAllocator.IdAllocatorV2;393 const idAllocator = yield* IdAllocator.IdAllocatorV2;
392 const host = yield* ProviderHost.ProviderHost;394 const host = yield* ProviderHost.ProviderHost;
...@@ -440,9 +442,16 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (...@@ -440,9 +442,16 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
440 extensionPath,442 extensionPath,
441 runtimeMode: input.runtimePolicy.runtimeMode,443 runtimeMode: input.runtimePolicy.runtimeMode,
442 });444 });
443 const connection: PiRpcConnection = yield* makePiRpcConnection({445 const scopedLaunch = yield* agentScope.wrap({
444 command: options.settings.binaryPath || "pi",446 command: options.settings.binaryPath || "pi",
445 args: launch.args,447 args: launch.args,
448 name: "pi",
449 threadId: input.threadId,
450 env: launch.env,
451 });
452 const connection: PiRpcConnection = yield* makePiRpcConnection({
453 command: scopedLaunch.command,
454 args: scopedLaunch.args,
446 cwd,455 cwd,
447 env: launch.env,456 env: launch.env,
448 }).pipe(457 }).pipe(
packages/shared/package.json+4
...@@ -311,6 +311,10 @@...@@ -311,6 +311,10 @@
311 "types": "./src/nodeRuntime.ts",311 "types": "./src/nodeRuntime.ts",
312 "import": "./src/nodeRuntime.ts"312 "import": "./src/nodeRuntime.ts"
313 },313 },
314 "./AgentScope": {
315 "types": "./src/AgentScope.ts",
316 "import": "./src/AgentScope.ts"
317 },
314 "./HostProcess": {318 "./HostProcess": {
315 "types": "./src/HostProcess.ts",319 "types": "./src/HostProcess.ts",
316 "import": "./src/HostProcess.ts"320 "import": "./src/HostProcess.ts"
packages/shared/src/AgentScope.test.ts created+30
...@@ -0,0 +1,30 @@
1// @effect-diagnostics nodeBuiltinImport:off
2import * as NodeChildProcess from "node:child_process";
3
4import { describe, expect, it } from "@effect/vitest";
5
6import { AGENT_OOM_SCORE_ADJ, agentScopeCommand } from "./AgentScope.ts";
7import * as HostProcess from "./HostProcess.ts";
8
9describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("agentScopeCommand", () => {
10 it("raises the OOM score and execs the command with its arguments in the same process", () => {
11 const launch = agentScopeCommand({
12 command: "/bin/sh",
13 args: [
14 "-c",
15 'printf "%s\\n" "$$" "$(cat /proc/self/oom_score_adj)" "$@"',
16 "agent",
17 "a b",
18 "$HOME",
19 ],
20 });
21 const child = NodeChildProcess.spawnSync(launch.command, [...launch.args], {
22 encoding: "utf8",
23 });
24 const [pid, score, ...args] = child.stdout.trimEnd().split("\n");
25 // The command runs as the spawned process, so signals and PIDs still reach it.
26 expect(Number(pid)).toBe(child.pid);
27 expect(Number(score)).toBe(AGENT_OOM_SCORE_ADJ);
28 expect(args).toEqual(["a b", "$HOME"]);
29 });
30});
packages/shared/src/AgentScope.ts created+129
...@@ -0,0 +1,129 @@
1import * as Context from "effect/Context";
2import * as Effect from "effect/Effect";
3
4/**
5 * Isolates long-lived agent and terminal processes from the server on Linux.
6 *
7 * Each wrapped command runs with a high `oom_score_adj`, so the kernel OOM
8 * killer picks it before the server. Under a systemd user manager it also runs
9 * in its own transient scope, so systemd-oomd (which kills whole leaf cgroups)
10 * kills one agent instead of the server and every other agent.
11 *
12 * The server provides the real implementation. The default leaves commands
13 * unchanged, which is also the behavior on macOS, Windows, and in tests.
14 */
15export interface AgentScopeShape {
16 /**
17 * Wraps the command that starts an agent or terminal. `name` labels the
18 * scope unit, for example "claude" or "terminal". `env` is the environment
19 * the command will run with, used to resolve it on PATH. A command that does
20 * not resolve stays unwrapped, so a missing binary still fails at spawn.
21 */
22 readonly wrap: (input: {
23 readonly command: string;
24 readonly args: ReadonlyArray<string>;
25 readonly name: string;
26 readonly threadId?: string | undefined;
27 readonly env?: NodeJS.ProcessEnv | undefined;
28 }) => Effect.Effect<AgentScopeCommand>;
29 /**
30 * Whether the latest agent scope of this thread was killed because the
31 * machine or the agents slice ran out of memory. The answer holds until the
32 * thread launches its next agent, so a retried failure reads the same.
33 */
34 readonly oomKilled: (threadId: string) => Effect.Effect<boolean>;
35 /**
36 * Forgets the thread's scope. The session manager calls it before it opens
37 * a provider session, so a scope only answers for the session that launched
38 * it, even when the next session runs without one (Cursor, OpenCode).
39 */
40 readonly clear: (threadId: string) => Effect.Effect<void>;
41}
42
43export interface AgentScopeCommand {
44 readonly command: string;
45 readonly args: ReadonlyArray<string>;
46}
47
48export const AgentScope = Context.Reference<AgentScopeShape>("@t3tools/shared/AgentScope", {
49 defaultValue: () => ({
50 wrap: ({ command, args }) => Effect.succeed({ command, args }),
51 oomKilled: () => Effect.succeed(false),
52 clear: () => Effect.void,
53 }),
54});
55
56/**
57 * The thread an ACP runtime is spawned for. ACP runtimes are shared by several
58 * provider flavors and do not take a thread id, so the ACP adapter provides it.
59 */
60export const AgentScopeThreadId = Context.Reference<string | undefined>(
61 "@t3tools/shared/AgentScope/ThreadId",
62 { defaultValue: () => undefined },
63);
64
65/**
66 * Agents get this `oom_score_adj` so the kernel kills them before the server.
67 * An unprivileged process may raise its own score, never lower it.
68 */
69export const AGENT_OOM_SCORE_ADJ = 800;
70
71/** Shell line that raises the score of the current process, ignoring failure. */
72export const RAISE_OOM_SCORE_LINE = `{ echo ${AGENT_OOM_SCORE_ADJ} >/proc/self/oom_score_adj; } 2>/dev/null`;
73
74const LAUNCH_SCRIPT = [RAISE_OOM_SCORE_LINE, 'exec "$@"'].join("\n");
75
76// systemd-run needs XDG_RUNTIME_DIR to reach the user manager. Fill it in when
77// the agent environment does not carry it; it is the same user either way.
78const SCOPED_LAUNCH_SCRIPT = [
79 RAISE_OOM_SCORE_LINE,
80 ': "${XDG_RUNTIME_DIR:=$1}"',
81 "export XDG_RUNTIME_DIR",
82 "shift",
83 'exec "$@"',
84].join("\n");
85
86export interface AgentSystemdScope {
87 readonly systemdRun: string;
88 readonly slice: string;
89 readonly unit: string;
90 readonly runtimeDir: string;
91 /** Extra `systemd-run` options, such as `--property=OOMPolicy=stop`. */
92 readonly options: ReadonlyArray<string>;
93}
94
95/**
96 * Builds the Linux launch for an agent command. Every step `exec`s, so the
97 * spawned PID, stdio, and signals all belong to the agent itself.
98 * `systemd-run --scope` registers its own PID as the scope and then runs the
99 * command, so the agent and all of its children start inside the scope.
100 */
101export function agentScopeCommand(input: {
102 readonly command: string;
103 readonly args: ReadonlyArray<string>;
104 readonly scope?: AgentSystemdScope | undefined;
105}): AgentScopeCommand {
106 const { command, args, scope } = input;
107 if (scope === undefined) {
108 return { command: "/bin/sh", args: ["-c", LAUNCH_SCRIPT, "t3-agent", command, ...args] };
109 }
110 return {
111 command: "/bin/sh",
112 args: [
113 "-c",
114 SCOPED_LAUNCH_SCRIPT,
115 "t3-agent",
116 scope.runtimeDir,
117 scope.systemdRun,
118 "--user",
119 "--scope",
120 "--quiet",
121 `--slice=${scope.slice}`,
122 `--unit=${scope.unit}`,
123 ...scope.options,
124 "--",
125 command,
126 ...args,
127 ],
128 };
129}