authorgravatar for 62337003+Bil0000@users.noreply.github.comBilal Bakr <62337003+Bil0000@users.noreply.github.com> 2026-10-15 06:20:20+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-15 03:20:20+00:00
log21bedfb175b9eec8652adc35d7882bdc3c3c9d59
treeb7c484f3a589fd093f16aeb89cafa848fa0c81cb
parentc814e51af2ba962c9291c9459ee64b4f57e0b5ce
signature Signed by PGP key B5690EEEBB952194

feat(storage): clean up settled thread worktrees (#15208)

Co-authored-by: maria-rcks <maria@kuuro.net>

28 files changed, 1594 insertions(+), 473 deletions(-)

apps/server/src/environment/ServerEnvironment.test.ts+7
...@@ -118,6 +118,13 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => {...@@ -118,6 +118,13 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => {
118 expect(descriptor.capabilities.serverSelfUpdate).toBe(118 expect(descriptor.capabilities.serverSelfUpdate).toBe(
119 mode === "web" ? undefined : "desktop-managed",119 mode === "web" ? undefined : "desktop-managed",
120 );120 );
121 expect(descriptor.capabilities).toMatchObject({
122 settledWorktreeCleanup: true,
123 storageCleanupRun: true,
124 projectWorktreeCleanup: true,
125 worktreesDirectory: true,
126 threadAutoSettlementScope: true,
127 });
121 }128 }
122 }),129 }),
123 );130 );
apps/server/src/environment/ServerEnvironment.ts+1
...@@ -231,6 +231,7 @@ export const make = Effect.gen(function* () {...@@ -231,6 +231,7 @@ export const make = Effect.gen(function* () {
231 storageCleanupRun: true,231 storageCleanupRun: true,
232 otlpEndpointCheck: true,232 otlpEndpointCheck: true,
233 projectWorktreeCleanup: true,233 projectWorktreeCleanup: true,
234 settledWorktreeCleanup: true,
234 worktreesDirectory: true,235 worktreesDirectory: true,
235 threadAutoSettlementScope: true,236 threadAutoSettlementScope: true,
236 threadRestartContinuation: true,237 threadRestartContinuation: true,
apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts+25
...@@ -16,7 +16,9 @@ import * as Layer from "effect/Layer";...@@ -16,7 +16,9 @@ import * as Layer from "effect/Layer";
16import * as Option from "effect/Option";16import * as Option from "effect/Option";
17import * as Path from "effect/Path";17import * as Path from "effect/Path";
18import { symlinksSupported } from "@t3tools/shared/testing/symlinks";18import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
19import * as GitWorkflowService from "../git/GitWorkflowService.ts";
19import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";20import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
21import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts";
20import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";22import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
21import { CheckpointServiceV2 } from "./CheckpointService.ts";23import { CheckpointServiceV2 } from "./CheckpointService.ts";
22import { EventSinkV2 } from "./EventSink.ts";24import { EventSinkV2 } from "./EventSink.ts";
...@@ -105,6 +107,7 @@ it.effect.each([...@@ -105,6 +107,7 @@ it.effect.each([
105 NodeServices.layer,107 NodeServices.layer,
106 ThreadCommandExecutor.layer,108 ThreadCommandExecutor.layer,
107 IdAllocator.layer,109 IdAllocator.layer,
110 Layer.mock(GitWorkflowService.GitWorkflowService)({}),
108 Layer.mock(ProjectStore.ProjectStoreV2)({111 Layer.mock(ProjectStore.ProjectStoreV2)({
109 get: () => Effect.succeed(Option.some({ workspaceRoot: parent } as never)),112 get: () => Effect.succeed(Option.some({ workspaceRoot: parent } as never)),
110 }),113 }),
...@@ -197,3 +200,25 @@ it.effect.each([...@@ -197,3 +200,25 @@ it.effect.each([
197 }200 }
198 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),201 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
199);202);
203
204it.effect("lets rollback recreate a worktree storage cleanup removed", () =>
205 Effect.gen(function* () {
206 const fileSystem = yield* FileSystem.FileSystem;
207 const path = yield* Path.Path;
208 const parent = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-v2-restore-removed-" });
209 const worktreePath = path.join(parent, "worktree");
210 const isolated = yield* isCheckpointRestoreIsolated(
211 { id: ThreadId.make("restore-removed"), worktreePath },
212 { cwd: worktreePath },
213 {
214 fileSystem,
215 path,
216 projections: {
217 getShellSnapshot: () => Effect.succeed({ threads: [], archivedThreads: [] }),
218 } as never,
219 projects: {} as never,
220 },
221 );
222 assert.isTrue(isolated);
223 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
224);
apps/server/src/orchestration-v2/CheckpointRestoreSafety.ts+13-2
...@@ -33,8 +33,19 @@ export const isCheckpointRestoreIsolated = Effect.fn("orchestrationV2.isCheckpoi...@@ -33,8 +33,19 @@ export const isCheckpointRestoreIsolated = Effect.fn("orchestrationV2.isCheckpoi
33 const worktreePath = thread.worktreePath;33 const worktreePath = thread.worktreePath;
34 let shared = worktreePath == null;34 let shared = worktreePath == null;
35 if (!shared && worktreePath !== null) {35 if (!shared && worktreePath !== null) {
36 const cwd = yield* fileSystem.realPath(scope.cwd);36 // Storage cleanup may have removed the worktree; rollback recreates it.
37 const worktreeCwd = yield* fileSystem.realPath(worktreePath);37 const resolve = (target: string) =>
38 fileSystem
39 .realPath(target)
40 .pipe(
41 Effect.catch((error) =>
42 error.reason._tag === "NotFound"
43 ? Effect.succeed(path.resolve(target))
44 : Effect.fail(error),
45 ),
46 );
47 const cwd = yield* resolve(scope.cwd);
48 const worktreeCwd = yield* resolve(worktreePath);
38 shared = cwd !== worktreeCwd;49 shared = cwd !== worktreeCwd;
39 if (!shared) {50 if (!shared) {
40 const shell = yield* projections.getShellSnapshot();51 const shell = yield* projections.getShellSnapshot();
apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts+436-206
...@@ -4,18 +4,24 @@ import {...@@ -4,18 +4,24 @@ import {
4 CheckpointId,4 CheckpointId,
5 CheckpointScopeId,5 CheckpointScopeId,
6 type OrchestrationV2ThreadProjection,6 type OrchestrationV2ThreadProjection,
7 ProjectId,
7 ProviderInstanceId,8 ProviderInstanceId,
8 ProviderSessionId,9 ProviderSessionId,
9 ProviderThreadId,10 ProviderThreadId,
10 ThreadId,11 ThreadId,
11} from "@t3tools/contracts";12} from "@t3tools/contracts";
12import * as Effect from "effect/Effect";13import * as Effect from "effect/Effect";
14import * as Deferred from "effect/Deferred";
15import * as Fiber from "effect/Fiber";
13import * as FileSystem from "effect/FileSystem";16import * as FileSystem from "effect/FileSystem";
14import * as Path from "effect/Path";17import * as Path from "effect/Path";
15import * as Option from "effect/Option";18import * as Option from "effect/Option";
16import { symlinksSupported } from "@t3tools/shared/testing/symlinks";19import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
17import * as Layer from "effect/Layer";20import * as Layer from "effect/Layer";
1821
22import * as GitWorkflow from "../git/GitWorkflowService.ts";
23import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
24import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
19import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";25import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
20import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts";26import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts";
21import * as CheckpointService from "./CheckpointService.ts";27import * as CheckpointService from "./CheckpointService.ts";
...@@ -38,76 +44,121 @@ const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe(...@@ -38,76 +44,121 @@ const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe(
38 Layer.provide(44 Layer.provide(
39 Layer.mergeAll(45 Layer.mergeAll(
40 NodeServices.layer,46 NodeServices.layer,
47 Layer.mock(GitWorkflow.GitWorkflowService)({}),
41 Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject) }),48 Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject) }),
42 ),49 ),
43 ),50 ),
44);51);
4552
46it.effect("rejects a non-ready checkpoint before opening a session or restoring files", () => {53it.effect.each(["initial", "waiting"])(
47 const threadId = ThreadId.make("thread:rollback-non-ready");54 "rejects a non-ready checkpoint from %s before opening a session or restoring files",
48 const providerThreadId = ProviderThreadId.make("provider-thread:rollback-non-ready");55 (state) => {
49 const providerSessionId = ProviderSessionId.make("provider-session:rollback-non-ready");56 const threadId = ThreadId.make("thread:rollback-non-ready");
50 const checkpointId = CheckpointId.make("checkpoint:rollback-non-ready");57 const providerThreadId = ProviderThreadId.make("provider-thread:rollback-non-ready");
51 const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-non-ready");58 const providerSessionId = ProviderSessionId.make("provider-session:rollback-non-ready");
52 const providerInstanceId = ProviderInstanceId.make("provider_rollback_non_ready");59 const checkpointId = CheckpointId.make("checkpoint:rollback-non-ready");
53 const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));60 const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-non-ready");
54 const open = vi.fn(() => Effect.die("provider session open must not run"));61 const providerInstanceId = ProviderInstanceId.make("provider_rollback_non_ready");
55 const resolveRuntimePolicy = vi.fn(() => Effect.die("runtime policy resolution must not run"));62 const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));
56 const projection = {63 const open = vi.fn(() => Effect.die("provider session open must not run"));
57 thread: {64 const resolveRuntimePolicy = vi.fn(() => Effect.die("runtime policy resolution must not run"));
58 worktreePath: process.cwd(),65 const readEntered = Deferred.makeUnsafe<void>();
59 activeProviderThreadId: providerThreadId,66 const release = Deferred.makeUnsafe<void>();
60 modelSelection: { instanceId: providerInstanceId, model: "test-model" },67 let readCount = 0;
61 },68 let projection = {
62 providerThreads: [{ id: providerThreadId, providerSessionId, providerInstanceId }],69 thread: {
63 checkpoints: [{ id: checkpointId, scopeId, status: "stale" }],70 worktreePath: process.cwd(),
64 checkpointScopes: [{ id: scopeId, cwd: process.cwd() }],71 activeProviderThreadId: providerThreadId,
65 } as unknown as OrchestrationV2ThreadProjection;72 modelSelection: { instanceId: providerInstanceId, model: "test-model" },
66 const layerTest = layerCheckpointRollbackService.pipe(73 },
67 Layer.provide(74 providerThreads: [{ id: providerThreadId, providerSessionId, providerInstanceId }],
68 Layer.mergeAll(75 checkpoints: [
69 ThreadCommandExecutor.layer,76 {
70 Layer.mock(CheckpointService.CheckpointServiceV2)({ restore }),77 id: checkpointId,
71 Layer.mock(EventSink.EventSinkV2)({}),78 scopeId,
72 IdAllocator.layer,79 status: state === "waiting" ? "ready" : "stale",
73 Layer.mock(ProjectionStore.ProjectionStoreV2)({80 appRunOrdinal: 0,
74 getThreadRecords: () => Effect.succeed(projection),81 },
75 getShellSnapshot: () =>82 ],
76 Effect.succeed({83 checkpointScopes: [{ id: scopeId, cwd: process.cwd() }],
77 schemaVersion: 1,84 } as unknown as OrchestrationV2ThreadProjection;
78 snapshotSequence: 0,85 const getThreadRecords = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getThreadRecords"]>(
79 threads: [],86 () => {
80 archivedThreads: [],87 const current = projection;
81 }),88 readCount++;
82 }),89 return readCount === 1
83 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),90 ? Deferred.succeed(readEntered, undefined).pipe(Effect.as(current))
84 Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: resolveRuntimePolicy }),91 : Effect.succeed(current);
92 },
93 );
94 const testLayer = layerCheckpointRollbackService.pipe(
95 Layer.provide(
96 Layer.mergeAll(
97 ThreadCommandExecutor.layer,
98 Layer.mock(CheckpointService.CheckpointServiceV2)({ restore }),
99 Layer.mock(EventSink.EventSinkV2)({}),
100 IdAllocator.layer,
101 Layer.mock(ProjectionStore.ProjectionStoreV2)({
102 getThreadRecords,
103 getShellSnapshot: () =>
104 Effect.succeed({
105 schemaVersion: 1,
106 snapshotSequence: 0,
107 threads: [],
108 archivedThreads: [],
109 }),
110 }),
111 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),
112 Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: resolveRuntimePolicy }),
113 ),
85 ),114 ),
86 ),115 );
87 );
88116
89 return Effect.gen(function* () {117 return Effect.gen(function* () {
90 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;118 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
91 const error = yield* service119 const holder =
92 .execute({120 state === "waiting"
93 threadId,121 ? yield* withWorkspaceLease(process.cwd(), Deferred.await(release)).pipe(
94 providerThreadId,122 Effect.forkChild({ startImmediately: true }),
95 checkpointId,123 )
96 scopeId,124 : undefined;
97 })125 const rollback = yield* service
98 .pipe(Effect.flip);126 .execute({
127 threadId,
128 providerThreadId,
129 checkpointId,
130 scopeId,
131 })
132 .pipe(Effect.flip, Effect.forkChild({ startImmediately: true }));
133 if (state === "waiting") {
134 yield* Deferred.await(readEntered);
135 projection = {
136 ...projection,
137 checkpoints: projection.checkpoints.map((checkpoint) => ({
138 ...checkpoint,
139 status: "stale",
140 })),
141 };
142 yield* Deferred.succeed(release, undefined);
143 if (holder !== undefined) yield* Fiber.join(holder);
144 }
145 const error = yield* Fiber.join(rollback);
99146
100 assert.equal(error.reason, "rollback-target-invalid");147 assert.equal(getThreadRecords.mock.calls.length, 2);
101 assert.equal(148 assert.deepEqual(getThreadRecords.mock.calls[0]?.[1], []);
102 error.message,149 assert.equal(getThreadRecords.mock.calls[1]?.[1].includes("checkpoints"), true);
103 `Rollback target ${checkpointId} for provider thread ${providerThreadId} on thread ${threadId} is incomplete or invalid.`,150 assert.equal(error.reason, "rollback-target-invalid");
104 );151 assert.equal(
105 assert.equal(error.cause, undefined);152 error.message,
106 assert.equal(resolveRuntimePolicy.mock.calls.length, 0);153 `Rollback target ${checkpointId} for provider thread ${providerThreadId} on thread ${threadId} is incomplete or invalid.`,
107 assert.equal(open.mock.calls.length, 0);154 );
108 assert.equal(restore.mock.calls.length, 0);155 assert.equal(error.cause, undefined);
109 }).pipe(Effect.provide(layerTest));156 assert.equal(resolveRuntimePolicy.mock.calls.length, 0);
110});157 assert.equal(open.mock.calls.length, 0);
158 assert.equal(restore.mock.calls.length, 0);
159 }).pipe(Effect.provide(testLayer));
160 },
161);
111162
112it.effect("rejects a rollback when another provider thread became active", () => {163it.effect("rejects a rollback when another provider thread became active", () => {
113 const threadId = ThreadId.make("thread:rollback-inactive-provider-thread");164 const threadId = ThreadId.make("thread:rollback-inactive-provider-thread");
...@@ -128,6 +179,7 @@ it.effect("rejects a rollback when another provider thread became active", () =>...@@ -128,6 +179,7 @@ it.effect("rejects a rollback when another provider thread became active", () =>
128 const resolveRuntimePolicy = vi.fn(() => Effect.die("runtime policy resolution must not run"));179 const resolveRuntimePolicy = vi.fn(() => Effect.die("runtime policy resolution must not run"));
129 const projection = {180 const projection = {
130 thread: {181 thread: {
182 worktreePath: null,
131 activeProviderThreadId,183 activeProviderThreadId,
132 modelSelection: { instanceId: providerInstanceId, model: "test-model" },184 modelSelection: { instanceId: providerInstanceId, model: "test-model" },
133 },185 },
...@@ -268,7 +320,7 @@ it.effect("rejects a rollback when provider selection changed before execution",...@@ -268,7 +320,7 @@ it.effect("rejects a rollback when provider selection changed before execution",
268 }).pipe(Effect.provide(layerTest));320 }).pipe(Effect.provide(layerTest));
269});321});
270322
271it.effect("reports a missing provider turn as a structured rollback failure", () => {323it.effect("rejects an unavailable rollback target before opening a session", () => {
272 const threadId = ThreadId.make("thread:rollback-provider-turn-unavailable");324 const threadId = ThreadId.make("thread:rollback-provider-turn-unavailable");
273 const providerThreadId = ProviderThreadId.make(325 const providerThreadId = ProviderThreadId.make(
274 "provider-thread:rollback-provider-turn-unavailable",326 "provider-thread:rollback-provider-turn-unavailable",
...@@ -280,9 +332,11 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()...@@ -280,9 +332,11 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
280 const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-provider-turn-unavailable");332 const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-provider-turn-unavailable");
281 const providerInstanceId = ProviderInstanceId.make("provider_rollback_provider_turn_unavailable");333 const providerInstanceId = ProviderInstanceId.make("provider_rollback_provider_turn_unavailable");
282 const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));334 const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));
335 const open = vi.fn(() => Effect.die("provider session open must not run"));
283 const projection = {336 const projection = {
284 thread: {337 thread: {
285 worktreePath: process.cwd(),338 worktreePath: "/tmp/missing-invalid-rollback-worktree",
339 branch: "feature/rollback",
286 activeProviderThreadId: providerThreadId,340 activeProviderThreadId: providerThreadId,
287 modelSelection: { instanceId: providerInstanceId, model: "test-model" },341 modelSelection: { instanceId: providerInstanceId, model: "test-model" },
288 },342 },
...@@ -312,7 +366,7 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()...@@ -312,7 +366,7 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
312 }),366 }),
313 }),367 }),
314 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({368 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
315 open: () => Effect.succeed({} as never),369 open,
316 }),370 }),
317 Layer.mock(RuntimePolicy.RuntimePolicyV2)({371 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
318 resolve: () => Effect.succeed({} as never),372 resolve: () => Effect.succeed({} as never),
...@@ -324,14 +378,9 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()...@@ -324,14 +378,9 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
324 return Effect.gen(function* () {378 return Effect.gen(function* () {
325 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;379 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
326 const error = yield* service380 const error = yield* service
327 .execute({381 .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles: false })
328 threadId,
329 providerThreadId,
330 checkpointId,
331 scopeId,
332 })
333 .pipe(Effect.flip);382 .pipe(Effect.flip);
334383 assert.equal(open.mock.calls.length, 0);
335 assert.equal(error.reason, "provider-turn-unavailable");384 assert.equal(error.reason, "provider-turn-unavailable");
336 assert.equal(385 assert.equal(
337 error.message,386 error.message,
...@@ -342,6 +391,169 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()...@@ -342,6 +391,169 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
342 }).pipe(Effect.provide(layerTest));391 }).pipe(Effect.provide(layerTest));
343});392});
344393
394it.effect.each([
395 { order: "normal", restoreFiles: true },
396 { order: "cleanup-first", restoreFiles: true },
397 { order: "cleanup-first", restoreFiles: false },
398 { order: "rollback-first", restoreFiles: true },
399])("locks rollback with %s", ({ order, restoreFiles }) =>
400 Effect.gen(function* () {
401 const fileSystem = yield* FileSystem.FileSystem;
402 const path = yield* Path.Path;
403 const workspaceRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-rollback-" });
404 const worktreePath = path.join(workspaceRoot, "worktree");
405 yield* fileSystem.makeDirectory(worktreePath);
406 const threadId = ThreadId.make("rollback-worktree");
407 const projectId = ProjectId.make("rollback-worktree-project");
408 const providerThreadId = ProviderThreadId.make("rollback-worktree-provider-thread");
409 const providerSessionId = ProviderSessionId.make("rollback-worktree-provider-session");
410 const providerInstanceId = ProviderInstanceId.make("rollback-worktree-provider-instance");
411 const checkpointId = CheckpointId.make("rollback-worktree-checkpoint");
412 const scopeId = CheckpointScopeId.make("rollback-worktree-scope");
413 const branch = "feature/rollback";
414 const release = Deferred.makeUnsafe<void>();
415 const openEntered = Deferred.makeUnsafe<void>();
416 const calls: string[] = [];
417 let cleanupEntered = false;
418 const providerThread = { id: providerThreadId, providerSessionId, providerInstanceId };
419 const projection = {
420 thread: {
421 id: threadId,
422 projectId,
423 worktreePath,
424 branch,
425 activeProviderThreadId: providerThreadId,
426 modelSelection: { instanceId: providerInstanceId, model: "test-model" },
427 },
428 providerThreads: [providerThread],
429 providerSessions: [],
430 checkpoints: [{ id: checkpointId, scopeId, status: "ready", appRunOrdinal: 0 }],
431 checkpointScopes: [{ id: scopeId, cwd: worktreePath }],
432 runs: [],
433 attempts: [],
434 nodes: [],
435 providerTurns: [],
436 } as unknown as OrchestrationV2ThreadProjection;
437 const testLayer = CheckpointRollbackService.layer.pipe(
438 Layer.provide(
439 Layer.mergeAll(
440 ThreadCommandExecutor.layer,
441 NodeServices.layer,
442 Layer.mock(ProjectStore.ProjectStoreV2)({
443 get: (id) => {
444 assert.equal(id, projectId);
445 return Effect.succeed(Option.some({ workspaceRoot } as never));
446 },
447 }),
448 Layer.mock(GitWorkflow.GitWorkflowService)({
449 pruneWorktrees: (input) =>
450 Effect.sync(() => {
451 assert.deepEqual(input, { cwd: workspaceRoot });
452 calls.push("prune");
453 }),
454 createWorktree: (input) =>
455 Effect.gen(function* () {
456 assert.deepEqual(input, {
457 cwd: workspaceRoot,
458 refName: branch,
459 path: worktreePath,
460 });
461 calls.push("create");
462 yield* fileSystem.makeDirectory(worktreePath).pipe(Effect.orDie);
463 return {} as never;
464 }),
465 }),
466 Layer.mock(CheckpointService.CheckpointServiceV2)({
467 restore: () =>
468 Effect.gen(function* () {
469 assert.isTrue(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie));
470 calls.push("files");
471 }),
472 }),
473 Layer.mock(EventSink.EventSinkV2)({
474 write: () =>
475 Effect.sync(() => {
476 calls.push("projection");
477 return [];
478 }),
479 }),
480 IdAllocator.layer,
481 Layer.mock(ProjectionStore.ProjectionStoreV2)({
482 getThreadRecords: () => Effect.succeed(projection),
483 getShellSnapshot: () =>
484 Effect.succeed({
485 schemaVersion: 1,
486 snapshotSequence: 0,
487 threads: [],
488 archivedThreads: [],
489 }),
490 }),
491 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
492 open: (input) =>
493 Effect.gen(function* () {
494 assert.equal(input.runtimePolicy.cwd, worktreePath);
495 if (!(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie))) {
496 return yield* new ProviderWorkspaceMissingError({ threadId, cwd: worktreePath });
497 }
498 calls.push("session");
499 yield* Deferred.succeed(openEntered, undefined);
500 if (order === "rollback-first") yield* Deferred.await(release);
501 return {} as never;
502 }),
503 }),
504 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
505 resolve: ({ thread }) => {
506 assert.equal(thread.worktreePath, worktreePath);
507 return Effect.succeed({ cwd: thread.worktreePath } as never);
508 },
509 }),
510 ),
511 ),
512 );
513 yield* Effect.gen(function* () {
514 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
515 const cleanupEffect = withWorkspaceLease(
516 worktreePath,
517 Effect.gen(function* () {
518 cleanupEntered = true;
519 if (order === "cleanup-first") yield* Deferred.await(release);
520 yield* fileSystem.remove(worktreePath, { recursive: true, force: true });
521 calls.push("cleanup");
522 }),
523 );
524 const cleanup =
525 order === "cleanup-first"
526 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
527 : undefined;
528 const rollback = yield* service
529 .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles })
530 .pipe(Effect.forkChild({ startImmediately: true }));
531 if (order === "cleanup-first") assert.deepEqual(calls, []);
532 if (order === "rollback-first") yield* Deferred.await(openEntered);
533 const laterCleanup =
534 order === "rollback-first"
535 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
536 : undefined;
537 if (order === "rollback-first") assert.isFalse(cleanupEntered);
538 yield* Deferred.succeed(release, undefined);
539 if (cleanup !== undefined) yield* Fiber.join(cleanup);
540 yield* Fiber.join(rollback);
541 if (laterCleanup !== undefined) yield* Fiber.join(laterCleanup);
542 const rollbackCalls = restoreFiles
543 ? ["session", "files", "projection"]
544 : ["session", "projection"];
545 assert.deepEqual(
546 calls,
547 order === "cleanup-first"
548 ? ["cleanup", "prune", "create", ...rollbackCalls]
549 : order === "rollback-first"
550 ? [...rollbackCalls, "cleanup"]
551 : rollbackCalls,
552 );
553 }).pipe(Effect.provide(testLayer));
554 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
555);
556
345it.effect.each([557it.effect.each([
346 { restoreFiles: true, shared: "none" },558 { restoreFiles: true, shared: "none" },
347 { restoreFiles: false, shared: "root" },559 { restoreFiles: false, shared: "root" },
...@@ -350,143 +562,161 @@ it.effect.each([...@@ -350,143 +562,161 @@ it.effect.each([
350 { restoreFiles: false, shared: "worktree" },562 { restoreFiles: false, shared: "worktree" },
351 { restoreFiles: true, shared: "historical" },563 { restoreFiles: true, shared: "historical" },
352 { restoreFiles: false, shared: "none", targetOrdinal: 1 },564 { restoreFiles: false, shared: "none", targetOrdinal: 1 },
353])("rewinds safely with %s", ({ restoreFiles, shared, targetOrdinal = 0 }) => {565 { restoreFiles: false, shared: "none", targetOrdinal: 1, multipleTurns: true },
354 const threadId = ThreadId.make("rewind-files");566])(
355 const providerThreadId = ProviderThreadId.make("rewind-provider");567 "rewinds safely with %s",
356 const providerSessionId = ProviderSessionId.make("rewind-session");568 ({ restoreFiles, shared, targetOrdinal = 0, multipleTurns = false }) => {
357 const instanceId = ProviderInstanceId.make("rewind-instance");569 const threadId = ThreadId.make("rewind-files");
358 const checkpointId = CheckpointId.make("rewind-start");570 const providerThreadId = ProviderThreadId.make("rewind-provider");
359 const scopeId = CheckpointScopeId.make("rewind-scope");571 const providerSessionId = ProviderSessionId.make("rewind-session");
360 const calls: string[] = [];572 const instanceId = ProviderInstanceId.make("rewind-instance");
361 let lockedThreads = Effect.succeed<ReadonlyArray<ThreadId>>([]);573 const checkpointId = CheckpointId.make("rewind-start");
362 const layerThreadCommands = Layer.effect(574 const scopeId = CheckpointScopeId.make("rewind-scope");
363 ThreadCommandExecutor.ThreadCommandExecutor,575 const calls: string[] = [];
364 Effect.tap(KeyedLock.make<ThreadId>(), (lock) =>576 let lockedThreads = Effect.succeed<ReadonlyArray<ThreadId>>([]);
365 Effect.sync(() => {577 const layerThreadCommands = Layer.effect(
366 lockedThreads = lock.activeKeys;578 ThreadCommandExecutor.ThreadCommandExecutor,
367 }),579 Effect.tap(KeyedLock.make<ThreadId>(), (lock) =>
368 ),580 Effect.sync(() => {
369 );581 lockedThreads = lock.activeKeys;
370 const providerThread = {
371 id: providerThreadId,
372 providerSessionId,
373 providerInstanceId: instanceId,
374 };
375 const projection = {
376 thread: {
377 worktreePath: shared === "root" ? null : process.cwd(),
378 activeProviderThreadId: providerThreadId,
379 modelSelection: { instanceId, model: "test" },
380 },
381 providerThreads: [providerThread],
382 providerSessions: [],
383 // Turn 3 remains in the audit history after an earlier rollback.
384 providerTurns: [1, 2, 3].map((ordinal) => ({
385 id: `turn-${ordinal}`,
386 providerThreadId,
387 runAttemptId: `attempt-${ordinal}`,
388 ordinal,
389 status: "completed",
390 })),
391 nodes: [],
392 attempts: [1, 2, 3].map((ordinal) => ({ id: `attempt-${ordinal}`, runId: `run-${ordinal}` })),
393 checkpoints: [
394 { id: checkpointId, scopeId, status: "ready", appRunOrdinal: targetOrdinal || null },
395 ],
396 checkpointScopes: [{ id: scopeId, cwd: process.cwd() }],
397 runs: [1, 2, 3].map((ordinal) => ({
398 id: `run-${ordinal}`,
399 ordinal,
400 status: ordinal === 3 ? "rolled_back" : "completed",
401 rootNodeId: null,
402 activeAttemptId: `attempt-${ordinal}`,
403 })),
404 } as unknown as OrchestrationV2ThreadProjection;
405 const layerTest = layerCheckpointRollbackService.pipe(
406 Layer.provide(
407 Layer.mergeAll(
408 layerThreadCommands,
409 Layer.mock(CheckpointService.CheckpointServiceV2)({
410 restore: () =>
411 Effect.sync(() => {
412 calls.push("files");
413 }),
414 }),
415 Layer.mock(EventSink.EventSinkV2)({
416 write: ({ events }) =>
417 Effect.map(lockedThreads, (locked) => {
418 // A thread command that planned against the old provider thread
419 // must not commit after this and undo the rollback.
420 assert.include(locked, threadId);
421 assert.ok(
422 events.some(
423 (event) => event.type === "run.updated" && event.payload.status === "rolled_back",
424 ),
425 );
426 calls.push("projection");
427 return [];
428 }),
429 }),
430 IdAllocator.layer,
431 Layer.mock(ProjectionStore.ProjectionStoreV2)({
432 getThreadRecords: () => Effect.succeed(projection),
433 getThreadProviderContext: () => Effect.succeed({ providerSessions: [] } as never),
434 getCheckpointContext: () =>
435 Effect.succeed({
436 checkpointScopes: [{ cwd: process.cwd() }],
437 runs: [],
438 checkpoints: [],
439 } as never),
440 getShellSnapshot: () =>
441 Effect.succeed({
442 schemaVersion: 1,
443 snapshotSequence: 0,
444 threads: [],
445 archivedThreads:
446 shared === "worktree" || shared === "historical"
447 ? [
448 {
449 id: ThreadId.make("other-thread"),
450 deletedAt: null,
451 worktreePath: shared === "worktree" ? process.cwd() : null,
452 } as never,
453 ]
454 : [],
455 }),
456 }),
457 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
458 open: () =>
459 Effect.succeed({
460 rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) =>
461 Effect.gen(function* () {
462 const count = yield* resolveCodexRollbackTurnCount(input);
463 assert.equal(count, 2 - targetOrdinal);
464 calls.push("provider");
465 return { providerThread };
466 }),
467 } as never),
468 }),582 }),
469 Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never) }),
470 ),583 ),
471 ),
472 );
473 return Effect.gen(function* () {
474 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
475 if (restoreFiles && shared !== "none") {
476 const error = yield* Effect.flip(
477 service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }),
478 );
479 assert.equal(error.reason, "shared-workspace");
480 assert.deepEqual(calls, []);
481 return;
482 }
483 yield* service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles });
484 assert.deepEqual(
485 calls,
486 restoreFiles ? ["provider", "files", "projection"] : ["provider", "projection"],
487 );584 );
488 }).pipe(Effect.provide(layerTest));585 const providerThread = {
489});586 id: providerThreadId,
587 providerSessionId,
588 providerInstanceId: instanceId,
589 };
590 const projection = {
591 thread: {
592 worktreePath: shared === "root" ? null : process.cwd(),
593 activeProviderThreadId: providerThreadId,
594 modelSelection: { instanceId, model: "test" },
595 },
596 providerThreads: [providerThread],
597 providerSessions: [],
598 // Turn 3 remains in the audit history after an earlier rollback.
599 providerTurns: [
600 ...(multipleTurns
601 ? [
602 {
603 id: "turn-0",
604 providerThreadId,
605 runAttemptId: "attempt-1",
606 ordinal: 0,
607 status: "completed",
608 },
609 ]
610 : []),
611 ...[1, 2, 3].map((ordinal) => ({
612 id: `turn-${ordinal}`,
613 providerThreadId,
614 runAttemptId: `attempt-${ordinal}`,
615 ordinal,
616 status: "completed",
617 })),
618 ],
619 nodes: [],
620 attempts: [1, 2, 3].map((ordinal) => ({ id: `attempt-${ordinal}`, runId: `run-${ordinal}` })),
621 checkpoints: [
622 { id: checkpointId, scopeId, status: "ready", appRunOrdinal: targetOrdinal || null },
623 ],
624 checkpointScopes: [{ id: scopeId, cwd: process.cwd() }],
625 runs: [1, 2, 3].map((ordinal) => ({
626 id: `run-${ordinal}`,
627 ordinal,
628 status: ordinal === 3 ? "rolled_back" : "completed",
629 rootNodeId: null,
630 activeAttemptId: `attempt-${ordinal}`,
631 })),
632 } as unknown as OrchestrationV2ThreadProjection;
633 const layerTest = layerCheckpointRollbackService.pipe(
634 Layer.provide(
635 Layer.mergeAll(
636 layerThreadCommands,
637 Layer.mock(CheckpointService.CheckpointServiceV2)({
638 restore: () =>
639 Effect.sync(() => {
640 calls.push("files");
641 }),
642 }),
643 Layer.mock(EventSink.EventSinkV2)({
644 write: ({ events }) =>
645 Effect.map(lockedThreads, (locked) => {
646 // A thread command that planned against the old provider thread
647 // must not commit after this and undo the rollback.
648 assert.include(locked, threadId);
649 assert.ok(
650 events.some(
651 (event) =>
652 event.type === "run.updated" && event.payload.status === "rolled_back",
653 ),
654 );
655 calls.push("projection");
656 return [];
657 }),
658 }),
659 IdAllocator.layer,
660 Layer.mock(ProjectionStore.ProjectionStoreV2)({
661 getThreadRecords: () => Effect.succeed(projection),
662 getThreadProviderContext: () => Effect.succeed({ providerSessions: [] } as never),
663 getCheckpointContext: () =>
664 Effect.succeed({
665 checkpointScopes: [{ cwd: process.cwd() }],
666 runs: [],
667 checkpoints: [],
668 } as never),
669 getShellSnapshot: () =>
670 Effect.succeed({
671 schemaVersion: 1,
672 snapshotSequence: 0,
673 threads: [],
674 archivedThreads:
675 shared === "worktree" || shared === "historical"
676 ? [
677 {
678 id: ThreadId.make("other-thread"),
679 deletedAt: null,
680 worktreePath: shared === "worktree" ? process.cwd() : null,
681 } as never,
682 ]
683 : [],
684 }),
685 }),
686 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
687 open: () =>
688 Effect.succeed({
689 rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) =>
690 Effect.gen(function* () {
691 const count = yield* resolveCodexRollbackTurnCount(input);
692 assert.equal(count, 2 - targetOrdinal);
693 calls.push("provider");
694 return { providerThread };
695 }),
696 } as never),
697 }),
698 Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never) }),
699 ),
700 ),
701 );
702 return Effect.gen(function* () {
703 const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
704 if (restoreFiles && shared !== "none") {
705 const error = yield* Effect.flip(
706 service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }),
707 );
708 assert.equal(error.reason, "shared-workspace");
709 assert.deepEqual(calls, []);
710 return;
711 }
712 yield* service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles });
713 assert.deepEqual(
714 calls,
715 restoreFiles ? ["provider", "files", "projection"] : ["provider", "projection"],
716 );
717 }).pipe(Effect.provide(layerTest));
718 },
719);
490720
491it.effect.skipIf(!symlinksSupported)(721it.effect.skipIf(!symlinksSupported)(
492 "rejects an archived thread sharing a worktree through a symlink",722 "rejects an archived thread sharing a worktree through a symlink",
apps/server/src/orchestration-v2/CheckpointRollbackService.ts+58-33
...@@ -11,9 +11,12 @@ import * as DateTime from "effect/DateTime";...@@ -11,9 +11,12 @@ import * as DateTime from "effect/DateTime";
11import * as Effect from "effect/Effect";11import * as Effect from "effect/Effect";
12import * as FileSystem from "effect/FileSystem";12import * as FileSystem from "effect/FileSystem";
13import * as Layer from "effect/Layer";13import * as Layer from "effect/Layer";
14import * as Option from "effect/Option";
14import * as Path from "effect/Path";15import * as Path from "effect/Path";
15import * as Schema from "effect/Schema";16import * as Schema from "effect/Schema";
1617
18import * as GitWorkflowService from "../git/GitWorkflowService.ts";
19import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
17import {20import {
18 isCheckpointRestoreIsolated,21 isCheckpointRestoreIsolated,
19 SHARED_WORKSPACE_RESTORE_MESSAGE,22 SHARED_WORKSPACE_RESTORE_MESSAGE,
...@@ -84,6 +87,7 @@ export const layer: Layer.Layer<...@@ -84,6 +87,7 @@ export const layer: Layer.Layer<
84 CheckpointRollbackServiceV2,87 CheckpointRollbackServiceV2,
85 never,88 never,
86 | CheckpointServiceV289 | CheckpointServiceV2
90 | GitWorkflowService.GitWorkflowService
87 | EventSinkV291 | EventSinkV2
88 | IdAllocator.IdAllocatorV292 | IdAllocator.IdAllocatorV2
89 | ProjectionStoreV293 | ProjectionStoreV2
...@@ -97,6 +101,7 @@ export const layer: Layer.Layer<...@@ -97,6 +101,7 @@ export const layer: Layer.Layer<
97 CheckpointRollbackServiceV2,101 CheckpointRollbackServiceV2,
98 Effect.gen(function* () {102 Effect.gen(function* () {
99 const checkpoints = yield* CheckpointServiceV2;103 const checkpoints = yield* CheckpointServiceV2;
104 const gitWorkflow = yield* GitWorkflowService.GitWorkflowService;
100 const eventSink = yield* EventSinkV2;105 const eventSink = yield* EventSinkV2;
101 const ids = yield* IdAllocator.IdAllocatorV2;106 const ids = yield* IdAllocator.IdAllocatorV2;
102 const projections = yield* ProjectionStoreV2;107 const projections = yield* ProjectionStoreV2;
...@@ -158,6 +163,52 @@ export const layer: Layer.Layer<...@@ -158,6 +163,52 @@ export const layer: Layer.Layer<
158 });163 });
159 }164 }
160165
166 const targetOrdinal = checkpoint.appRunOrdinal ?? 0;
167 const rollbackTarget: ProviderAdapter.ProviderAdapterV2RollbackTarget =
168 targetOrdinal === 0
169 ? {
170 type: "thread_start",
171 checkpointId: checkpoint.id,
172 appRunOrdinal: 0,
173 }
174 : yield* Effect.gen(function* () {
175 const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal);
176 const targetAttempt = projection.attempts.find(
177 (attempt) => attempt.id === targetRun?.activeAttemptId,
178 );
179 // A goal run can span several native turns; roll back to its last.
180 const targetTurn =
181 latestProviderTurnForAttempt(projection.providerTurns, targetAttempt?.id) ??
182 projection.providerTurns.find((turn) => turn.id === targetAttempt?.providerTurnId);
183 if (targetTurn === undefined || targetTurn.providerThreadId !== providerThread.id) {
184 return yield* new CheckpointRollbackExecutionError({
185 reason: "provider-turn-unavailable",
186 threadId: input.threadId,
187 providerThreadId: input.providerThreadId,
188 checkpointId: input.checkpointId,
189 });
190 }
191 return {
192 type: "provider_turn" as const,
193 checkpointId: checkpoint.id,
194 appRunOrdinal: targetOrdinal,
195 providerTurn: targetTurn,
196 };
197 });
198
199 const { worktreePath, branch } = projection.thread;
200 if (worktreePath != null && branch != null && !(yield* fileSystem.exists(worktreePath))) {
201 const project = yield* projects.get(projection.thread.projectId);
202 if (Option.isSome(project)) {
203 yield* gitWorkflow.pruneWorktrees({ cwd: project.value.workspaceRoot });
204 yield* gitWorkflow.createWorktree({
205 cwd: project.value.workspaceRoot,
206 refName: branch,
207 path: worktreePath,
208 });
209 }
210 }
211
161 if (212 if (
162 input.restoreFiles !== false &&213 input.restoreFiles !== false &&
163 !(yield* isCheckpointRestoreIsolated(projection.thread, scope, {214 !(yield* isCheckpointRestoreIsolated(projection.thread, scope, {
...@@ -199,7 +250,6 @@ export const layer: Layer.Layer<...@@ -199,7 +250,6 @@ export const layer: Layer.Layer<
199 }),250 }),
200 });251 });
201252
202 const targetOrdinal = checkpoint.appRunOrdinal ?? 0;
203 // Stopped and failed runs after the target leave the provider253 // Stopped and failed runs after the target leave the provider
204 // conversation too, so they must not stay visible.254 // conversation too, so they must not stay visible.
205 const runsToRollback = projection.runs.filter(255 const runsToRollback = projection.runs.filter(
...@@ -225,37 +275,6 @@ export const layer: Layer.Layer<...@@ -225,37 +275,6 @@ export const layer: Layer.Layer<
225 turn.providerThreadId === providerThread.id &&275 turn.providerThreadId === providerThread.id &&
226 (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)),276 (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)),
227 );277 );
228 const rollbackTarget: ProviderAdapter.ProviderAdapterV2RollbackTarget =
229 targetOrdinal === 0
230 ? {
231 type: "thread_start",
232 checkpointId: checkpoint.id,
233 appRunOrdinal: 0,
234 }
235 : yield* Effect.gen(function* () {
236 const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal);
237 const targetAttempt = projection.attempts.find(
238 (attempt) => attempt.id === targetRun?.activeAttemptId,
239 );
240 // A goal run can span several native turns; roll back to its last.
241 const targetTurn =
242 latestProviderTurnForAttempt(projection.providerTurns, targetAttempt?.id) ??
243 projection.providerTurns.find((turn) => turn.id === targetAttempt?.providerTurnId);
244 if (targetTurn === undefined || targetTurn.providerThreadId !== providerThread.id) {
245 return yield* new CheckpointRollbackExecutionError({
246 reason: "provider-turn-unavailable",
247 threadId: input.threadId,
248 providerThreadId: input.providerThreadId,
249 checkpointId: input.checkpointId,
250 });
251 }
252 return {
253 type: "provider_turn" as const,
254 checkpointId: checkpoint.id,
255 appRunOrdinal: targetOrdinal,
256 providerTurn: targetTurn,
257 };
258 });
259278
260 const snapshot =279 const snapshot =
261 runsToRollback.length === 0280 runsToRollback.length === 0
...@@ -355,7 +374,13 @@ export const layer: Layer.Layer<...@@ -355,7 +374,13 @@ export const layer: Layer.Layer<
355374
356 return CheckpointRollbackServiceV2.of({375 return CheckpointRollbackServiceV2.of({
357 execute: (input) =>376 execute: (input) =>
358 execute(input).pipe(377 Effect.gen(function* () {
378 // Storage cleanup removes worktrees under the same lease.
379 const { thread } = yield* projections.getThreadRecords(input.threadId, []);
380 yield* thread.worktreePath === null
381 ? execute(input)
382 : withWorkspaceLease(path.resolve(thread.worktreePath), execute(input));
383 }).pipe(
359 Effect.mapError((cause) =>384 Effect.mapError((cause) =>
360 isCheckpointRollbackExecutionError(cause)385 isCheckpointRollbackExecutionError(cause)
361 ? cause386 ? cause
apps/server/src/orchestration-v2/EffectWorker.test.ts+1
...@@ -105,6 +105,7 @@ function layerExecutorFor(input: {...@@ -105,6 +105,7 @@ function layerExecutorFor(input: {
105 shutdown: Effect.void,105 shutdown: Effect.void,
106 open: () => Effect.die("unused open"),106 open: () => Effect.die("unused open"),
107 get: () => Effect.succeed(Option.none()),107 get: () => Effect.succeed(Option.none()),
108 isLive: () => Effect.succeed(false),
108 close: () => Effect.void,109 close: () => Effect.void,
109 closeInstance: () => Effect.void,110 closeInstance: () => Effect.void,
110 release: () => record("release"),111 release: () => record("release"),
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+19
...@@ -2899,6 +2899,25 @@ it.effect(...@@ -2899,6 +2899,25 @@ it.effect(
2899 }),2899 }),
2900);2900);
29012901
2902it.effect("ProviderSessionManagerV2 checks liveness without delaying idle release", () =>
2903 Effect.gen(function* () {
2904 const state = yield* Ref.make(emptyState);
2905 const effect = Effect.gen(function* () {
2906 const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2907 const { providerSessionId } = yield* openSessionWithBackgroundCommand(
2908 "provider-session-manager-is-live",
2909 );
2910 yield* TestClock.adjust("600 millis");
2911 assert.isTrue(yield* manager.isLive(providerSessionId));
2912 yield* TestClock.adjust("600 millis");
2913 yield* Effect.yieldNow;
2914 assert.isFalse(yield* manager.isLive(providerSessionId));
2915 });
2916
2917 yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
2918 }),
2919);
2920
2902it.effect("ProviderSessionManagerV2 clears the background roster of an idle-released session", () =>2921it.effect("ProviderSessionManagerV2 clears the background roster of an idle-released session", () =>
2903 Effect.gen(function* () {2922 Effect.gen(function* () {
2904 const state = yield* Ref.make(emptyState);2923 const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts+6
...@@ -169,6 +169,8 @@ export interface ProviderSessionManagerV2Shape {...@@ -169,6 +169,8 @@ export interface ProviderSessionManagerV2Shape {
169 Option.Option<ProviderAdapter.ProviderAdapterV2SessionRuntime>,169 Option.Option<ProviderAdapter.ProviderAdapterV2SessionRuntime>,
170 ProviderSessionManagerV2Error170 ProviderSessionManagerV2Error
171 >;171 >;
172 /** Whether the session's runtime is running. Unlike `get`, not activity. */
173 readonly isLive: (providerSessionId: ProviderSessionId) => Effect.Effect<boolean>;
172 readonly close: (174 readonly close: (
173 providerSessionId: ProviderSessionId,175 providerSessionId: ProviderSessionId,
174 ) => Effect.Effect<void, ProviderSessionManagerV2Error>;176 ) => Effect.Effect<void, ProviderSessionManagerV2Error>;
...@@ -2346,6 +2348,10 @@ export const layerWithOptions = (...@@ -2346,6 +2348,10 @@ export const layerWithOptions = (
2346 return exposedRuntime;2348 return exposedRuntime;
2347 }),2349 }),
2348 ),2350 ),
2351 isLive: (providerSessionId) =>
2352 Ref.get(sessions).pipe(
2353 Effect.map((current) => current.has(sessionKey(providerSessionId))),
2354 ),
2349 get: (providerSessionId) =>2355 get: (providerSessionId) =>
2350 Effect.gen(function* () {2356 Effect.gen(function* () {
2351 const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId));2357 const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId));
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts+1
...@@ -273,6 +273,7 @@ it.effect(...@@ -273,6 +273,7 @@ it.effect(
273 Effect.succeed(273 Effect.succeed(
274 providerSessionId === oldSessionId ? Option.some(runtime) : Option.none(),274 providerSessionId === oldSessionId ? Option.some(runtime) : Option.none(),
275 ),275 ),
276 isLive: (providerSessionId) => Effect.succeed(providerSessionId === oldSessionId),
276 close: () => Effect.void,277 close: () => Effect.void,
277 closeInstance: () => Effect.void,278 closeInstance: () => Effect.void,
278 release: () => Effect.void,279 release: () => Effect.void,
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts+214-17
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";
1import { expect, it, vi } from "vite-plus/test";2import { expect, it, vi } from "vite-plus/test";
2import { it as effectIt } from "@effect/vitest";3import { it as effectIt } from "@effect/vitest";
3import {4import {
...@@ -18,12 +19,16 @@ import {...@@ -18,12 +19,16 @@ import {
18} from "@t3tools/contracts";19} from "@t3tools/contracts";
19import * as Cause from "effect/Cause";20import * as Cause from "effect/Cause";
20import * as DateTime from "effect/DateTime";21import * as DateTime from "effect/DateTime";
22import * as Deferred from "effect/Deferred";
21import * as Effect from "effect/Effect";23import * as Effect from "effect/Effect";
22import * as FileSystem from "effect/FileSystem";24import * as FileSystem from "effect/FileSystem";
25import * as Fiber from "effect/Fiber";
23import * as Layer from "effect/Layer";26import * as Layer from "effect/Layer";
24import * as Option from "effect/Option";27import * as Option from "effect/Option";
28import * as Path from "effect/Path";
25import * as Schema from "effect/Schema";29import * as Schema from "effect/Schema";
2630
31import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
27import * as GitWorkflow from "../git/GitWorkflowService.ts";32import * as GitWorkflow from "../git/GitWorkflowService.ts";
28import * as ProjectService from "../project/ProjectService.ts";33import * as ProjectService from "../project/ProjectService.ts";
29import * as ProviderAuthService from "../provider/ProviderAuthService.ts";34import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
...@@ -95,7 +100,8 @@ it("does not commit running state when inherited background routing cannot be re...@@ -95,7 +100,8 @@ it("does not commit running state when inherited background routing cannot be re
95 Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}),100 Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}),
96 Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),101 Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
97 IdAllocator.layer,102 IdAllocator.layer,
98 Layer.succeed(FileSystem.FileSystem, { exists: () => Effect.succeed(false) } as never),103 Path.layer,
104 FileSystem.layerNoop({ exists: () => Effect.succeed(false) }),
99 Layer.mock(GitWorkflow.GitWorkflowService)({ pruneWorktrees, createWorktree }),105 Layer.mock(GitWorkflow.GitWorkflowService)({ pruneWorktrees, createWorktree }),
100 Layer.mock(ProjectService.ProjectService)({106 Layer.mock(ProjectService.ProjectService)({
101 getById: () =>107 getById: () =>
...@@ -104,6 +110,10 @@ it("does not commit running state when inherited background routing cannot be re...@@ -104,6 +110,10 @@ it("does not commit running state when inherited background routing cannot be re
104 ),110 ),
105 }),111 }),
106 Layer.mock(ProjectionStore.ProjectionStoreV2)({112 Layer.mock(ProjectionStore.ProjectionStoreV2)({
113 getThreadRecords: () => {
114 projectionReadCount += 1;
115 return Effect.succeed(projection);
116 },
107 getTurnStartContext: () => {117 getTurnStartContext: () => {
108 projectionReadCount += 1;118 projectionReadCount += 1;
109 return Effect.succeed({119 return Effect.succeed({
...@@ -141,7 +151,7 @@ it("does not commit running state when inherited background routing cannot be re...@@ -141,7 +151,7 @@ it("does not commit running state when inherited background routing cannot be re
141 .pipe(Effect.flip);151 .pipe(Effect.flip);
142152
143 expect(error._tag).toBe("ProviderTurnStartError");153 expect(error._tag).toBe("ProviderTurnStartError");
144 expect(projectionReadCount).toBe(2);154 expect(projectionReadCount).toBe(3);
145 expect(pruneWorktrees).toHaveBeenCalledWith({ cwd: "/tmp/provider-turn-start-project" });155 expect(pruneWorktrees).toHaveBeenCalledWith({ cwd: "/tmp/provider-turn-start-project" });
146 expect(createWorktree).toHaveBeenCalledWith({156 expect(createWorktree).toHaveBeenCalledWith({
147 cwd: "/tmp/provider-turn-start-project",157 cwd: "/tmp/provider-turn-start-project",
...@@ -155,6 +165,8 @@ it("does not commit running state when inherited background routing cannot be re...@@ -155,6 +165,8 @@ it("does not commit running state when inherited background routing cannot be re
155165
156function makeLocalCommandHarness(input: {166function makeLocalCommandHarness(input: {
157 readonly text: string;167 readonly text: string;
168 readonly worktreePath?: string;
169 readonly fileSystem?: FileSystem.FileSystem;
158 readonly previousNativeSession?: boolean;170 readonly previousNativeSession?: boolean;
159 readonly previousMessages?: ReadonlyArray<string>;171 readonly previousMessages?: ReadonlyArray<string>;
160 readonly logoutFailure?: string;172 readonly logoutFailure?: string;
...@@ -236,8 +248,9 @@ function makeLocalCommandHarness(input: {...@@ -236,8 +248,9 @@ function makeLocalCommandHarness(input: {
236 thread: {248 thread: {
237 id: threadId,249 id: threadId,
238 activeProviderThreadId: providerThreadId,250 activeProviderThreadId: providerThreadId,
239 branch: null,251 projectId: ProjectId.make("project-native-account-command"),
240 worktreePath: null,252 branch: input.worktreePath === undefined ? null : "feature/restore",
253 worktreePath: input.worktreePath ?? null,
241 } as OrchestrationV2ThreadProjection["thread"],254 } as OrchestrationV2ThreadProjection["thread"],
242 runs: [255 runs: [
243 ...(input.previousNativeSession256 ...(input.previousNativeSession
...@@ -483,6 +496,24 @@ function makeLocalCommandHarness(input: {...@@ -483,6 +496,24 @@ function makeLocalCommandHarness(input: {
483 return { committed, storedEvents: [] };496 return { committed, storedEvents: [] };
484 }),497 }),
485 );498 );
499 const resolveRuntimePolicy = vi.fn(
500 ({ thread }: Parameters<RuntimePolicy.RuntimePolicyV2Shape["resolve"]>[0]) =>
501 Effect.succeed({ cwd: thread.worktreePath ?? "/tmp/native-account-command" } as never),
502 );
503 const createWorktree = vi.fn(() => Effect.succeed({} as never));
504 const getThreadRecords = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getThreadRecords"]>(() =>
505 Effect.succeed(projection),
506 );
507 const getTurnStartContext = vi.fn(() =>
508 Effect.succeed({
509 ...projection,
510 hasConversation: projection.messages.some(
511 (m) =>
512 m.role === "user" &&
513 (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
514 ),
515 }),
516 );
486 const layer = ProviderTurnStart.layer.pipe(517 const layer = ProviderTurnStart.layer.pipe(
487 Layer.provide(518 Layer.provide(
488 Layer.mergeAll(519 Layer.mergeAll(
...@@ -491,19 +522,21 @@ function makeLocalCommandHarness(input: {...@@ -491,19 +522,21 @@ function makeLocalCommandHarness(input: {
491 }),522 }),
492 Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),523 Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
493 IdAllocator.layer,524 IdAllocator.layer,
494 FileSystem.layerNoop({}),525 Path.layer,
495 Layer.mock(GitWorkflow.GitWorkflowService)({}),526 input.fileSystem === undefined
496 Layer.mock(ProjectService.ProjectService)({}),527 ? FileSystem.layerNoop({ exists: () => Effect.succeed(true) })
528 : Layer.succeed(FileSystem.FileSystem, input.fileSystem),
529 Layer.mock(GitWorkflow.GitWorkflowService)({
530 pruneWorktrees: () => Effect.void,
531 createWorktree,
532 }),
533 Layer.mock(ProjectService.ProjectService)({
534 getById: () =>
535 Effect.succeed(Option.some({ workspaceRoot: "/tmp/native-account-project" } as never)),
536 }),
497 Layer.mock(ProjectionStore.ProjectionStoreV2)({537 Layer.mock(ProjectionStore.ProjectionStoreV2)({
498 getTurnStartContext: () =>538 getThreadRecords,
499 Effect.succeed({539 getTurnStartContext,
500 ...projection,
501 hasConversation: projection.messages.some(
502 (m) =>
503 m.role === "user" &&
504 (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
505 ),
506 }),
507 getRuntimeRecoveryProjection: () =>540 getRuntimeRecoveryProjection: () =>
508 Effect.as(failReadIfRunning, {541 Effect.as(failReadIfRunning, {
509 ...projection,542 ...projection,
...@@ -525,12 +558,19 @@ function makeLocalCommandHarness(input: {...@@ -525,12 +558,19 @@ function makeLocalCommandHarness(input: {
525 Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),558 Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
526 Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),559 Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
527 Layer.mock(RuntimePolicy.RuntimePolicyV2)({560 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
528 resolve: () => Effect.succeed({} as never),561 resolve: resolveRuntimePolicy,
529 }),562 }),
530 ),563 ),
531 ),564 ),
532 );565 );
533 return {566 return {
567 getThreadRecords,
568 getTurnStartContext,
569 setProjection: (next: OrchestrationV2ThreadProjection) => {
570 projection = next;
571 },
572 resolveRuntimePolicy,
573 createWorktree,
534 open,574 open,
535 writeIfRunCurrent,575 writeIfRunCurrent,
536 startRootRun,576 startRootRun,
...@@ -553,6 +593,163 @@ function makeLocalCommandHarness(input: {...@@ -553,6 +593,163 @@ function makeLocalCommandHarness(input: {
553 };593 };
554}594}
555595
596effectIt.effect.each(["cleanup-first", "startup-first"])(
597 "coordinates provider startup with %s",
598 (order) =>
599 Effect.gen(function* () {
600 const fileSystem = yield* FileSystem.FileSystem;
601 const path = yield* Path.Path;
602 const workspaceRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-start-" });
603 const worktreePath = path.join(workspaceRoot, "worktree");
604 yield* fileSystem.makeDirectory(worktreePath);
605 const release = yield* Deferred.make<void>();
606 const openEntered = yield* Deferred.make<void>();
607 const harness = makeLocalCommandHarness({
608 text: "Continue",
609 worktreePath,
610 fileSystem,
611 failReadsAfterRunning: true,
612 });
613 harness.createWorktree.mockImplementation(() =>
614 fileSystem.makeDirectory(worktreePath).pipe(Effect.as({} as never), Effect.orDie),
615 );
616 const open = harness.open.getMockImplementation()!;
617 harness.open.mockImplementation(() =>
618 Effect.gen(function* () {
619 expect(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie)).toBe(true);
620 yield* Deferred.succeed(openEntered, undefined);
621 if (order === "startup-first") yield* Deferred.await(release);
622 return yield* open();
623 }),
624 );
625 let cleanupEntered = false;
626 const cleanupEffect = withWorkspaceLease(
627 worktreePath,
628 Effect.gen(function* () {
629 cleanupEntered = true;
630 if (order === "cleanup-first") yield* Deferred.await(release);
631 else {
632 expect(harness.projection().providerSessions).toHaveLength(1);
633 expect(harness.projection().runs.at(-1)?.status).toBe("running");
634 }
635 yield* fileSystem.remove(worktreePath, { recursive: true, force: true });
636 }),
637 );
638 const cleanup =
639 order === "cleanup-first"
640 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
641 : undefined;
642 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
643 if (order === "cleanup-first") expect(harness.open).not.toHaveBeenCalled();
644 if (order === "startup-first") yield* Deferred.await(openEntered);
645 const laterCleanup =
646 order === "startup-first"
647 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
648 : undefined;
649 if (order === "startup-first") expect(cleanupEntered).toBe(false);
650 yield* Deferred.succeed(release, undefined);
651 if (cleanup !== undefined) yield* Fiber.join(cleanup);
652 yield* Fiber.join(startup);
653 if (laterCleanup !== undefined) yield* Fiber.join(laterCleanup);
654 if (order === "cleanup-first") {
655 expect(harness.createWorktree).toHaveBeenCalledWith({
656 cwd: "/tmp/native-account-project",
657 refName: "feature/restore",
658 path: worktreePath,
659 });
660 }
661 expect(harness.resolveRuntimePolicy).toHaveBeenCalledWith(
662 expect.objectContaining({
663 thread: expect.objectContaining({ worktreePath }),
664 }),
665 );
666 expect(harness.open).toHaveBeenCalledWith(
667 expect.objectContaining({
668 runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
669 }),
670 );
671 expect(harness.startRootRun).toHaveBeenCalledWith(
672 expect.objectContaining({
673 appThread: expect.objectContaining({ worktreePath }),
674 runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
675 }),
676 );
677 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
678);
679
680effectIt.effect("reads only thread state before taking the startup lease", () =>
681 Effect.gen(function* () {
682 const harness = makeLocalCommandHarness({ text: "/logout", previousNativeSession: true });
683 yield* harness.start;
684 expect(harness.getThreadRecords).toHaveBeenCalledExactlyOnceWith(
685 harness.projection().thread.id,
686 [],
687 );
688 expect(harness.getTurnStartContext).toHaveBeenCalledOnce();
689 expect(harness.projection().runs.at(-1)?.status).toBe("completed");
690 }),
691);
692
693effectIt.effect("reads current startup state after waiting for the workspace lease", () =>
694 Effect.gen(function* () {
695 const worktreePath = "/tmp/provider-start-fresh-state";
696 const release = yield* Deferred.make<void>();
697 const readEntered = yield* Deferred.make<void>();
698 const harness = makeLocalCommandHarness({
699 text: "Continue",
700 worktreePath,
701 failReadsAfterRunning: true,
702 });
703 harness.getThreadRecords.mockImplementationOnce(() =>
704 Deferred.succeed(readEntered, undefined).pipe(Effect.as(harness.projection())),
705 );
706 const holder = yield* withWorkspaceLease(worktreePath, Deferred.await(release)).pipe(
707 Effect.forkChild({ startImmediately: true }),
708 );
709 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
710 yield* Deferred.await(readEntered);
711 expect(harness.getTurnStartContext).not.toHaveBeenCalled();
712 const projection = harness.projection();
713 harness.setProjection({
714 ...projection,
715 runs: projection.runs.map((run) => ({ ...run, status: "running" })),
716 });
717 yield* Deferred.succeed(release, undefined);
718 yield* Fiber.join(holder);
719 yield* Fiber.join(startup);
720 expect(harness.open).not.toHaveBeenCalled();
721 expect(harness.startRootRun).not.toHaveBeenCalled();
722 }),
723);
724
725effectIt.effect("starts another provider in the same checkout while a turn is pending", () =>
726 Effect.gen(function* () {
727 const worktreePath = "/tmp/provider-start-shared-checkout";
728 const turnEntered = yield* Deferred.make<void>();
729 const releaseTurn = yield* Deferred.make<void>();
730 const first = makeLocalCommandHarness({
731 text: "Continue",
732 worktreePath,
733 failReadsAfterRunning: true,
734 });
735 const second = makeLocalCommandHarness({
736 text: "Continue",
737 worktreePath,
738 failReadsAfterRunning: true,
739 });
740 first.startRootRun.mockImplementation(() =>
741 Deferred.succeed(turnEntered, undefined).pipe(Effect.andThen(Deferred.await(releaseTurn))),
742 );
743 const pending = yield* first.start.pipe(Effect.forkChild({ startImmediately: true }));
744 yield* Deferred.await(turnEntered);
745 yield* second.start;
746 expect(second.startRootRun).toHaveBeenCalledOnce();
747 expect(second.projection().runs.at(-1)?.status).toBe("running");
748 yield* Deferred.succeed(releaseTurn, undefined);
749 yield* Fiber.join(pending);
750 }),
751);
752
556effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>753effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>
557 Effect.gen(function* () {754 Effect.gen(function* () {
558 const harness = makeLocalCommandHarness({755 const harness = makeLocalCommandHarness({
apps/server/src/orchestration-v2/ProviderTurnStartService.ts+14-2
...@@ -19,9 +19,11 @@ import * as Effect from "effect/Effect";...@@ -19,9 +19,11 @@ import * as Effect from "effect/Effect";
19import * as FileSystem from "effect/FileSystem";19import * as FileSystem from "effect/FileSystem";
20import * as Layer from "effect/Layer";20import * as Layer from "effect/Layer";
21import * as Option from "effect/Option";21import * as Option from "effect/Option";
22import * as Path from "effect/Path";
22import * as Predicate from "effect/Predicate";23import * as Predicate from "effect/Predicate";
23import * as Schema from "effect/Schema";24import * as Schema from "effect/Schema";
2425
26import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
25import * as GitWorkflowService from "../git/GitWorkflowService.ts";27import * as GitWorkflowService from "../git/GitWorkflowService.ts";
26import * as ProjectService from "../project/ProjectService.ts";28import * as ProjectService from "../project/ProjectService.ts";
27import * as ProviderAuthService from "../provider/ProviderAuthService.ts";29import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
...@@ -90,6 +92,7 @@ export const layer: Layer.Layer<...@@ -90,6 +92,7 @@ export const layer: Layer.Layer<
90 | ContextHandoffService.ContextHandoffServiceV292 | ContextHandoffService.ContextHandoffServiceV2
91 | IdAllocator.IdAllocatorV293 | IdAllocator.IdAllocatorV2
92 | FileSystem.FileSystem94 | FileSystem.FileSystem
95 | Path.Path
93 | GitWorkflowService.GitWorkflowService96 | GitWorkflowService.GitWorkflowService
94 | ProjectService.ProjectService97 | ProjectService.ProjectService
95 | ProviderAuthService.ProviderAuthService98 | ProviderAuthService.ProviderAuthService
...@@ -104,6 +107,7 @@ export const layer: Layer.Layer<...@@ -104,6 +107,7 @@ export const layer: Layer.Layer<
104 const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;107 const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;
105 const idAllocator = yield* IdAllocator.IdAllocatorV2;108 const idAllocator = yield* IdAllocator.IdAllocatorV2;
106 const fileSystem = yield* FileSystem.FileSystem;109 const fileSystem = yield* FileSystem.FileSystem;
110 const path = yield* Path.Path;
107 const gitWorkflow = yield* GitWorkflowService.GitWorkflowService;111 const gitWorkflow = yield* GitWorkflowService.GitWorkflowService;
108 const projects = yield* ProjectService.ProjectService;112 const projects = yield* ProjectService.ProjectService;
109 const providerAuth = yield* ProviderAuthService.ProviderAuthService;113 const providerAuth = yield* ProviderAuthService.ProviderAuthService;
...@@ -1224,7 +1228,7 @@ export const layer: Layer.Layer<...@@ -1224,7 +1228,7 @@ export const layer: Layer.Layer<
1224 !noteContinuation1228 !noteContinuation
1225 ? session1229 ? session
1226 : makeDeliverySession(session, startWithHandoffs);1230 : makeDeliverySession(session, startWithHandoffs);
1227 yield* runExecution.startRootRun({1231 return runExecution.startRootRun({
1228 commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`),1232 commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`),
1229 appThread: projection.thread,1233 appThread: projection.thread,
1230 providerSessionId,1234 providerSessionId,
...@@ -1279,7 +1283,15 @@ export const layer: Layer.Layer<...@@ -1279,7 +1283,15 @@ export const layer: Layer.Layer<
12791283
1280 return ProviderTurnStartServiceV2.of({1284 return ProviderTurnStartServiceV2.of({
1281 start: (input) =>1285 start: (input) =>
1282 start(input).pipe(1286 Effect.gen(function* () {
1287 // Storage cleanup removes worktrees under the same lease. Only startup
1288 // holds it; the provider turn runs after it is released.
1289 const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1290 const providerTurn = yield* thread.worktreePath === null
1291 ? start(input)
1292 : withWorkspaceLease(path.resolve(thread.worktreePath), start(input));
1293 if (providerTurn !== undefined) yield* providerTurn;
1294 }).pipe(
1283 Effect.mapError((cause) =>1295 Effect.mapError((cause) =>
1284 isProviderTurnStartError(cause)1296 isProviderTurnStartError(cause)
1285 ? cause1297 ? cause
apps/server/src/orchestration-v2/testkit/ProviderTurnStartMemory.fixture.mjs+13-1
...@@ -7,10 +7,11 @@ const mode = process.argv[3];...@@ -7,10 +7,11 @@ const mode = process.argv[3];
7const withHandoff = mode.startsWith("handoff");7const withHandoff = mode.startsWith("handoff");
8const require = NodeModule.createRequire(root + "/apps/server/package.json");8const require = NodeModule.createRequire(root + "/apps/server/package.json");
9const load = (name) => import(NodeURL.pathToFileURL(require.resolve("effect/" + name)));9const load = (name) => import(NodeURL.pathToFileURL(require.resolve("effect/" + name)));
10const [Effect, Layer, FileSystem] = await Promise.all([10const [Effect, Layer, FileSystem, Path] = await Promise.all([
11 load("Effect"),11 load("Effect"),
12 load("Layer"),12 load("Layer"),
13 load("FileSystem"),13 load("FileSystem"),
14 load("Path"),
14]);15]);
15const app = (file) => import(NodeURL.pathToFileURL(root + "/apps/server/src/" + file + ".ts"));16const app = (file) => import(NodeURL.pathToFileURL(root + "/apps/server/src/" + file + ".ts"));
16const providerCore = (file) =>17const providerCore = (file) =>
...@@ -31,6 +32,7 @@ const [Start, Projection, Run, Sessions, Policy, Id, Sink, Handoff, Git, Project...@@ -31,6 +32,7 @@ const [Start, Projection, Run, Sessions, Policy, Id, Sink, Handoff, Git, Project
31 ]);32 ]);
32let current;33let current;
33let fullReads = 0;34let fullReads = 0;
35let threadReads = 0;
34const liveRuns = [];36const liveRuns = [];
35const refs = [];37const refs = [];
36const checkpoints = [];38const checkpoints = [];
...@@ -48,6 +50,7 @@ const dependencies = Layer.mergeAll(...@@ -48,6 +50,7 @@ const dependencies = Layer.mergeAll(
48 Layer.mock(Handoff.ContextHandoffServiceV2)({}),50 Layer.mock(Handoff.ContextHandoffServiceV2)({}),
49 Id.layer,51 Id.layer,
50 FileSystem.layerNoop({}),52 FileSystem.layerNoop({}),
53 Path.layer,
51 Layer.mock(Git.GitWorkflowService)({}),54 Layer.mock(Git.GitWorkflowService)({}),
52 Layer.mock(Project.ProjectService)({}),55 Layer.mock(Project.ProjectService)({}),
53 Layer.mock(Auth.ProviderAuthService)({}),56 Layer.mock(Auth.ProviderAuthService)({}),
...@@ -58,6 +61,13 @@ const dependencies = Layer.mergeAll(...@@ -58,6 +61,13 @@ const dependencies = Layer.mergeAll(
58 throw new Error("full transcript read");61 throw new Error("full transcript read");
59 }),62 }),
60 hasUnpairedRunInterruptRequest: () => Effect.succeed(false),63 hasUnpairedRunInterruptRequest: () => Effect.succeed(false),
64 getThreadRecords: (threadId, fields) =>
65 Effect.sync(() => {
66 NodeAssert.equal(threadId, current.thread.id);
67 NodeAssert.deepEqual(fields, []);
68 threadReads++;
69 return { thread: current.thread };
70 }),
61 getTurnStartContext: () =>71 getTurnStartContext: () =>
62 Effect.sync(() => {72 Effect.sync(() => {
63 fullReads++;73 fullReads++;
...@@ -181,8 +191,10 @@ await Effect.runPromise(...@@ -181,8 +191,10 @@ await Effect.runPromise(
181 NodeAssert.equal(yield* controls.shouldFinalizeRun(), false);191 NodeAssert.equal(yield* controls.shouldFinalizeRun(), false);
182 NodeAssert.deepEqual(yield* controls.loadInheritedBackgroundTurnItems(), []);192 NodeAssert.deepEqual(yield* controls.loadInheritedBackgroundTurnItems(), []);
183 NodeAssert.equal(fullReads, i + 1);193 NodeAssert.equal(fullReads, i + 1);
194 NodeAssert.equal(threadReads, i + 1);
184 NodeAssert.equal(yield* controls.hasUnpairedRunInterruptRequest(), false);195 NodeAssert.equal(yield* controls.hasUnpairedRunInterruptRequest(), false);
185 NodeAssert.equal(fullReads, i + 1);196 NodeAssert.equal(fullReads, i + 1);
197 NodeAssert.equal(threadReads, i + 1);
186 current = null;198 current = null;
187 if (i === Math.floor(count / 2) - 1 || i === count - 1) {199 if (i === Math.floor(count / 2) - 1 || i === count - 1) {
188 yield* Effect.promise(async () => {200 yield* Effect.promise(async () => {
apps/server/src/serverSettings.test.ts+23-1
...@@ -580,7 +580,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {...@@ -580,7 +580,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
580 ).pipe(Effect.provide(layerServerSettings())),580 ).pipe(Effect.provide(layerServerSettings())),
581 );581 );
582582
583 it.effect("persists custom usage prices and removes them from the settings file", () =>583 it.effect("preserves saved usage settings during cleanup edits and removes custom prices", () =>
584 Effect.scoped(584 Effect.scoped(
585 Effect.gen(function* () {585 Effect.gen(function* () {
586 const serverConfig = yield* ServerConfig.ServerConfig;586 const serverConfig = yield* ServerConfig.ServerConfig;
...@@ -596,8 +596,30 @@ it.layer(NodeServices.layer)("server settings", (it) => {...@@ -596,8 +596,30 @@ it.layer(NodeServices.layer)("server settings", (it) => {
596 .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings))));596 .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings))));
597597
598 yield* serverSettings.updateSettings({ usagePriceOverrides: { "example-model": prices } });598 yield* serverSettings.updateSettings({ usagePriceOverrides: { "example-model": prices } });
599 const usageModelAliases = { "example-preview": "example-model" };
600 const sourceId = UsageLimitSourceId.make("hub");
601 const usageLimitSources = {
602 [sourceId]: {
603 kind: "cliproxy" as const,
604 url: "https://hub.example.test",
605 managementKey: "",
606 enabled: true,
607 },
608 };
609 const saved = yield* serverSettings.updateSettings({
610 usageModelAliases,
611 usageLimitSources,
612 });
613 const updated = yield* serverSettings.updateSettings({
614 storageCleanup: { worktreeSettledAfterDays: 0, worktreeKeepWhen: "any-local-files" },
615 });
616 assert.deepStrictEqual(updated.usageLimitSources, saved.usageLimitSources);
599 const persisted = yield* readPersisted;617 const persisted = yield* readPersisted;
600 assert.deepStrictEqual(persisted.usagePriceOverrides, { "example-model": prices });618 assert.deepStrictEqual(persisted.usagePriceOverrides, { "example-model": prices });
619 assert.deepStrictEqual(persisted.usageModelAliases, usageModelAliases);
620 assert.deepStrictEqual(persisted.usageLimitSources, usageLimitSources);
621 assert.equal(persisted.storageCleanup.worktreeSettledAfterDays, 0);
622 assert.equal(persisted.storageCleanup.worktreeKeepWhen, "any-local-files");
601623
602 yield* serverSettings.updateSettings({ usagePriceOverrides: { "example-model": null } });624 yield* serverSettings.updateSettings({ usagePriceOverrides: { "example-model": null } });
603 const restored = yield* readPersisted;625 const restored = yield* readPersisted;
apps/server/src/storageCleanup.test.ts+410-187
...@@ -9,11 +9,14 @@ import type * as Path from "effect/Path";...@@ -9,11 +9,14 @@ import type * as Path from "effect/Path";
9import type * as Scope from "effect/Scope";9import type * as Scope from "effect/Scope";
10import * as Stream from "effect/Stream";10import * as Stream from "effect/Stream";
11import * as Schema from "effect/Schema";11import * as Schema from "effect/Schema";
12import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts";
13import { TestClock } from "effect/testing";
12import * as SqlClient from "effect/sql/SqlClient";14import * as SqlClient from "effect/sql/SqlClient";
13import {15import {
14 DEFAULT_SERVER_SETTINGS,16 DEFAULT_SERVER_SETTINGS,
15 StorageCleanupReport,17 StorageCleanupReport,
16 type WorktreeKeepWhen,18 type WorktreeKeepWhen,
19 type StorageCleanupSettings,
17} from "@t3tools/contracts";20} from "@t3tools/contracts";
18import * as ServerConfig from "./config.ts";21import * as ServerConfig from "./config.ts";
19import * as StorageCleanup from "./storageCleanup.ts";22import * as StorageCleanup from "./storageCleanup.ts";
...@@ -21,6 +24,7 @@ import * as GitVcsDriver from "./vcs/GitVcsDriver.ts";...@@ -21,6 +24,7 @@ import * as GitVcsDriver from "./vcs/GitVcsDriver.ts";
21import * as GitManager from "./git/GitManager.ts";24import * as GitManager from "./git/GitManager.ts";
22import * as Settings from "./serverSettings.ts";25import * as Settings from "./serverSettings.ts";
23import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";26import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";
27import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
24import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";28import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
25import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";29import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
26import * as TerminalManager from "./terminal/Manager.ts";30import * as TerminalManager from "./terminal/Manager.ts";
...@@ -231,178 +235,197 @@ describe("merged pull request cleanup", () => {...@@ -231,178 +235,197 @@ describe("merged pull request cleanup", () => {
231 });235 });
232});236});
233237
234const cleanupFixture = Effect.gen(function* () {238const cleanupFixture = (
235 const fs = yield* FileSystem.FileSystem;239 storageCleanup: StorageCleanupSettings = {
236 const config = yield* ServerConfig.ServerConfig;240 ...DEFAULT_SERVER_SETTINGS.storageCleanup,
237 const git = yield* GitVcsDriver.GitVcsDriver;241 worktreeAfterDays: 8,
238 const repo = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-repo-" });242 worktreeOnDelete: true,
239 const command = (cwd: string, args: string[]) =>243 },
240 git.execute({ operation: "cleanup-test", cwd, args });244) =>
241 yield* command(repo, ["init", "-b", "main"]);245 Effect.gen(function* () {
242 yield* command(repo, ["config", "user.email", "cleanup@example.test"]);246 const fs = yield* FileSystem.FileSystem;
243 yield* command(repo, ["config", "user.name", "Cleanup test"]);247 const config = yield* ServerConfig.ServerConfig;
244 yield* fs.writeFileString(`${repo}/tracked.txt`, "original\n");248 const git = yield* GitVcsDriver.GitVcsDriver;
245 yield* fs.writeFileString(`${repo}/.gitignore`, ".env\ndist/\nnode_modules/\n");249 const repo = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-repo-" });
246 yield* command(repo, ["add", "."]);250 const command = (cwd: string, args: string[]) =>
247 yield* command(repo, ["commit", "-m", "initial"]);251 git.execute({ operation: "cleanup-test", cwd, args });
248 yield* fs.makeDirectory(config.worktreesDir, { recursive: true });252 yield* command(repo, ["init", "-b", "main"]);
249 const worktree = `${config.worktreesDir}/feature`;253 yield* command(repo, ["config", "user.email", "cleanup@example.test"]);
250 yield* command(repo, ["worktree", "add", "-b", "feature", worktree]);254 yield* command(repo, ["config", "user.name", "Cleanup test"]);
251 // worktreeOnDelete lets domain events request sweeps once a test calls activate.255 yield* fs.writeFileString(`${repo}/tracked.txt`, "original\n");
252 let settings: import("@t3tools/contracts").ServerSettings = {256 yield* fs.writeFileString(`${repo}/.gitignore`, ".env\ndist/\nnode_modules/\n");
253 ...DEFAULT_SERVER_SETTINGS,257 yield* command(repo, ["add", "."]);
254 storageCleanup: {258 yield* command(repo, ["commit", "-m", "initial"]);
255 ...DEFAULT_SERVER_SETTINGS.storageCleanup,259 yield* fs.makeDirectory(config.worktreesDir, { recursive: true });
256 worktreeAfterDays: 8,260 const worktree = `${config.worktreesDir}/feature`;
257 worktreeOnDelete: true,261 yield* command(repo, ["worktree", "add", "-b", "feature", worktree]);
258 },262 // worktreeOnDelete lets domain events request sweeps once a test calls activate.
259 };263 let settings: import("@t3tools/contracts").ServerSettings = {
260 let threads = [shell({ branch: "feature", worktreePath: worktree })];264 ...DEFAULT_SERVER_SETTINGS,
261 let editBeforeRemoval = false;265 storageCleanup,
262 let openSessionCwd: string | null = null;266 };
263 let fullStatusCalls = 0;267 let threads = [shell({ branch: "feature", worktreePath: worktree })];
264 let snapshotReads = 0;268 let editBeforeRemoval = false;
265 let events: Stream.Stream<OrchestrationV2DomainEvent> = Stream.empty;269 let openSessionCwd: string | null = null;
266 const activated = yield* Deferred.make<void>();270 let sessionLive = true;
267 const context = yield* Layer.build(StorageCleanup.layer).pipe(271 let fullStatusCalls = 0;
268 Effect.provideService(GitVcsDriver.GitVcsDriver, {272 let snapshotReads = 0;
269 ...git,273 let events: Stream.Stream<OrchestrationV2DomainEvent> = Stream.empty;
270 execute: (input) =>274 const activated = yield* Deferred.make<void>();
271 Effect.gen(function* () {275 const context = yield* Layer.build(StorageCleanup.layer).pipe(
272 if (editBeforeRemoval && input.args.includes("remove"))276 Effect.provideService(GitVcsDriver.GitVcsDriver, {
273 yield* fs.writeFileString(`${worktree}/tracked.txt`, "late edit\n");277 ...git,
274 return yield* git.execute(input);278 execute: (input) =>
275 }),279 Effect.gen(function* () {
276 statusDetailsLocal: (...args) => {280 if (editBeforeRemoval && input.args.includes("remove"))
277 fullStatusCalls++;281 yield* fs.writeFileString(`${worktree}/tracked.txt`, "late edit\n");
278 return git.statusDetailsLocal(...args);282 return yield* git.execute(input);
279 },283 }),
280 }),284 statusDetailsLocal: (...args) => {
281 Effect.provideService(Settings.ServerSettingsService, {285 fullStatusCalls++;
282 getSettings: Effect.sync(() => settings),286 return git.statusDetailsLocal(...args);
283 subscribeChanges: Effect.succeed(Stream.empty),
284 } as unknown as Settings.ServerSettingsService["Service"]),
285 Effect.provideService(ProjectStore.ProjectStoreV2, {
286 listShells: () => Effect.succeed([{ id: ProjectId.make("project-1"), workspaceRoot: repo }]),
287 } as unknown as ProjectStore.ProjectStoreV2["Service"]),
288 Effect.provideService(ProjectionStore.ProjectionStoreV2, {
289 getShellSnapshot: (input?: { location?: string }) =>
290 Effect.sync(() => {
291 snapshotReads++;
292 return { threads: input?.location === "archive" ? [] : threads };
293 }),
294 getThreadWorktreePaths: () =>
295 Effect.sync(() =>
296 threads.flatMap((thread) =>
297 thread.worktreePath === null
298 ? []
299 : [{ threadId: thread.id, worktreePath: thread.worktreePath }],
300 ),
301 ),
302 getThreadShell: (threadId: ThreadId) =>
303 Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null),
304 } as unknown as ProjectionStore.ProjectionStoreV2["Service"]),
305 Effect.provideService(Orchestrator.OrchestratorV2, {
306 streamDomainEvents: Stream.suspend(() => events),
307 } as unknown as Orchestrator.OrchestratorV2["Service"]),
308 Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) =>
309 Effect.sync(() =>
310 openSessionCwd !== null && strings.join("").includes("projection_provider_sessions")
311 ? [
312 {
313 payload_json: JSON.stringify({
314 id: "session-1",
315 driver: "codex",
316 providerInstanceId: "codex",
317 status: "ready",
318 cwd: openSessionCwd,
319 model: null,
320 capabilities: CodexProviderCapabilitiesV2,
321 createdAt: "2026-06-01T12:00:00.000Z",
322 updatedAt: "2026-06-01T12:00:00.000Z",
323 lastError: null,
324 }),
325 },
326 ]
327 : [],
328 )) as unknown as SqlClient.SqlClient),
329 Effect.provideService(GitManager.GitManager, {
330 invalidateStatus: () => Effect.void,
331 } as unknown as GitManager.GitManager["Service"]),
332 Effect.provideService(TerminalManager.TerminalManager, {
333 subscribeMetadata: () => Effect.succeed(() => {}),
334 } as unknown as TerminalManager.TerminalManager["Service"]),
335 Effect.provideService(ServerActivation, Deferred.await(activated)),
336 );
337 return {
338 service: Context.get(context, StorageCleanup.StorageCleanup),
339 editBeforeRemoval: () => {
340 editBeforeRemoval = true;
341 },
342 fs,
343 worktree,
344 config,
345 command,
346 enableFiles: () => {
347 settings = {
348 ...settings,
349 storageCleanup: {
350 ...settings.storageCleanup,
351 browserArtifactsAfterDays: 8,
352 logsAfterDays: 8,
353 },
354 };
355 },
356 setPolicy: (worktreeKeepWhen: WorktreeKeepWhen) => {
357 settings = { ...settings, storageCleanup: { ...settings.storageCleanup, worktreeKeepWhen } };
358 },
359 disable: () => {
360 settings = {
361 ...settings,
362 storageCleanup: {
363 ...settings.storageCleanup,
364 worktreeAfterDays: null,
365 worktreeOnDelete: false,
366 },287 },
367 };288 }),
368 },289 Effect.provideService(Settings.ServerSettingsService, {
369 setThreads: (next: typeof threads) => {290 getSettings: Effect.sync(() => settings),
370 threads = next;291 subscribeChanges: Effect.succeed(Stream.empty),
371 },292 } as unknown as Settings.ServerSettingsService["Service"]),
372 setOpenSessionCwd: (cwd: string | null) => {293 Effect.provideService(ProjectStore.ProjectStoreV2, {
373 openSessionCwd = cwd;294 listShells: () =>
374 },295 Effect.succeed([{ id: ProjectId.make("project-1"), workspaceRoot: repo }]),
375 fullStatusCalls: () => fullStatusCalls,296 } as unknown as ProjectStore.ProjectStoreV2["Service"]),
376 snapshotReads: () => snapshotReads,297 Effect.provideService(ProjectionStore.ProjectionStoreV2, {
377 /** Starts the hourly sweep and the subscription to `domainEvents`, which wait until now. */298 getShellSnapshot: (input?: { location?: string }) =>
378 activate: (domainEvents: Stream.Stream<OrchestrationV2DomainEvent>) => {299 Effect.sync(() => {
379 events = domainEvents;300 snapshotReads++;
380 return Deferred.succeed(activated, undefined);301 return { threads: input?.location === "archive" ? [] : threads };
381 },302 }),
382 };303 getThreadWorktreePaths: () =>
383});304 Effect.sync(() =>
305 threads.flatMap((thread) =>
306 thread.worktreePath === null
307 ? []
308 : [{ threadId: thread.id, worktreePath: thread.worktreePath }],
309 ),
310 ),
311 getThreadShell: (threadId: ThreadId) =>
312 Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null),
313 } as unknown as ProjectionStore.ProjectionStoreV2["Service"]),
314 Effect.provideService(ProviderSessionManager.ProviderSessionManagerV2, {
315 isLive: () => Effect.sync(() => sessionLive),
316 } as unknown as ProviderSessionManager.ProviderSessionManagerV2["Service"]),
317 Effect.provideService(Orchestrator.OrchestratorV2, {
318 streamDomainEvents: Stream.suspend(() => events),
319 } as unknown as Orchestrator.OrchestratorV2["Service"]),
320 Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) =>
321 Effect.sync(() =>
322 openSessionCwd !== null && strings.join("").includes("projection_provider_sessions")
323 ? [
324 {
325 payload_json: JSON.stringify({
326 id: "session-1",
327 driver: "codex",
328 providerInstanceId: "codex",
329 status: "ready",
330 cwd: openSessionCwd,
331 model: null,
332 capabilities: CodexProviderCapabilitiesV2,
333 createdAt: "2026-06-01T12:00:00.000Z",
334 updatedAt: "2026-06-01T12:00:00.000Z",
335 lastError: null,
336 }),
337 },
338 ]
339 : [],
340 )) as unknown as SqlClient.SqlClient),
341 Effect.provideService(GitManager.GitManager, {
342 invalidateStatus: () => Effect.void,
343 } as unknown as GitManager.GitManager["Service"]),
344 Effect.provideService(TerminalManager.TerminalManager, {
345 subscribeMetadata: () => Effect.succeed(() => {}),
346 } as unknown as TerminalManager.TerminalManager["Service"]),
347 Effect.provideService(ServerActivation, Deferred.await(activated)),
348 );
349 return {
350 service: Context.get(context, StorageCleanup.StorageCleanup),
351 repo,
352 editBeforeRemoval: () => {
353 editBeforeRemoval = true;
354 },
355 fs,
356 worktree,
357 config,
358 command,
359 enableFiles: () => {
360 settings = {
361 ...settings,
362 storageCleanup: {
363 ...settings.storageCleanup,
364 browserArtifactsAfterDays: 8,
365 logsAfterDays: 8,
366 },
367 };
368 },
369 setSettings: (next: Partial<typeof settings>) => {
370 settings = { ...settings, ...next };
371 },
372 setPolicy: (worktreeKeepWhen: WorktreeKeepWhen) => {
373 settings = {
374 ...settings,
375 storageCleanup: { ...settings.storageCleanup, worktreeKeepWhen },
376 };
377 },
378 disable: () => {
379 settings = {
380 ...settings,
381 storageCleanup: {
382 ...settings.storageCleanup,
383 worktreeAfterDays: null,
384 worktreeOnDelete: false,
385 },
386 };
387 },
388 setThreads: (next: typeof threads) => {
389 threads = next;
390 },
391 setSessionLive: (live: boolean) => {
392 sessionLive = live;
393 },
394 setOpenSessionCwd: (cwd: string | null) => {
395 openSessionCwd = cwd;
396 },
397 fullStatusCalls: () => fullStatusCalls,
398 snapshotReads: () => snapshotReads,
399 /** Starts the hourly sweep and the subscription to `domainEvents`, which wait until now. */
400 activate: (domainEvents: Stream.Stream<OrchestrationV2DomainEvent>) => {
401 events = domainEvents;
402 return Deferred.succeed(activated, undefined);
403 },
404 };
405 });
384/** The fixture, with the size walk of its worktree held until `resume` succeeds. */406/** The fixture, with the size walk of its worktree held until `resume` succeeds. */
385const pausedMeasurementFixture = Effect.gen(function* () {407const pausedMeasurementFixture = (storageCleanup?: StorageCleanupSettings) =>
386 const fs = yield* FileSystem.FileSystem;408 Effect.gen(function* () {
387 const measuring = yield* Deferred.make<void>();409 const fs = yield* FileSystem.FileSystem;
388 const resume = yield* Deferred.make<void>();410 const measuring = yield* Deferred.make<void>();
389 let measuredPath: string | undefined;411 const resume = yield* Deferred.make<void>();
390 const fixture = yield* cleanupFixture.pipe(412 let measuredPath: string | undefined;
391 Effect.provideService(FileSystem.FileSystem, {413 const fixture = yield* cleanupFixture(storageCleanup).pipe(
392 ...fs,414 Effect.provideService(FileSystem.FileSystem, {
393 readDirectory: (directory, options) =>415 ...fs,
394 Effect.gen(function* () {416 readDirectory: (directory, options) =>
395 if (directory === measuredPath) {417 Effect.gen(function* () {
396 yield* Deferred.succeed(measuring, undefined);418 if (directory === measuredPath) {
397 yield* Deferred.await(resume);419 yield* Deferred.succeed(measuring, undefined);
398 }420 yield* Deferred.await(resume);
399 return yield* fs.readDirectory(directory, options);421 }
400 }),422 return yield* fs.readDirectory(directory, options);
401 }),423 }),
402 );424 }),
403 measuredPath = yield* fs.realPath(fixture.worktree);425 );
404 return { ...fixture, measuring, resume };426 measuredPath = yield* fs.realPath(fixture.worktree);
405});427 return { ...fixture, measuring, resume };
428 });
406const sessionUpdated = (429const sessionUpdated = (
407 index: number,430 index: number,
408 status: OrchestrationV2ProviderSession["status"],431 status: OrchestrationV2ProviderSession["status"],
...@@ -444,7 +467,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -444,7 +467,7 @@ describe("storage cleanup reports and local file policies", () => {
444 it.live("measures nested regular files without following symlinks", () =>467 it.live("measures nested regular files without following symlinks", () =>
445 runCleanupTest(468 runCleanupTest(
446 Effect.gen(function* () {469 Effect.gen(function* () {
447 const { service, fs, worktree, setPolicy } = yield* cleanupFixture;470 const { service, fs, worktree, setPolicy } = yield* cleanupFixture();
448 setPolicy("tracked-changes");471 setPolicy("tracked-changes");
449 const sizes = yield* Effect.forEach(yield* fs.readDirectory(worktree), (name) =>472 const sizes = yield* Effect.forEach(yield* fs.readDirectory(worktree), (name) =>
450 fs.stat(`${worktree}/${name}`).pipe(Effect.map((stat) => Number(stat.size))),473 fs.stat(`${worktree}/${name}`).pipe(Effect.map((stat) => Number(stat.size))),
...@@ -468,7 +491,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -468,7 +491,7 @@ describe("storage cleanup reports and local file policies", () => {
468 runCleanupTest(491 runCleanupTest(
469 Effect.gen(function* () {492 Effect.gen(function* () {
470 const fs = yield* FileSystem.FileSystem;493 const fs = yield* FileSystem.FileSystem;
471 const { service, worktree } = yield* cleanupFixture.pipe(494 const { service, worktree } = yield* cleanupFixture().pipe(
472 Effect.provideService(FileSystem.FileSystem, {495 Effect.provideService(FileSystem.FileSystem, {
473 ...fs,496 ...fs,
474 readDirectory: (directory) => fs.readDirectory(`${directory}/missing-directory`),497 readDirectory: (directory) => fs.readDirectory(`${directory}/missing-directory`),
...@@ -498,7 +521,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -498,7 +521,7 @@ describe("storage cleanup reports and local file policies", () => {
498 ] as const)("keeps a worktree when %s during size measurement", ([, threadsWhileMeasuring]) =>521 ] as const)("keeps a worktree when %s during size measurement", ([, threadsWhileMeasuring]) =>
499 runCleanupTest(522 runCleanupTest(
500 Effect.gen(function* () {523 Effect.gen(function* () {
501 const fixture = yield* pausedMeasurementFixture;524 const fixture = yield* pausedMeasurementFixture();
502 const { fs, worktree } = fixture;525 const { fs, worktree } = fixture;
503 fixture.setPolicy("tracked-changes");526 fixture.setPolicy("tracked-changes");
504 yield* fs.writeFileString(`${worktree}/notes.txt`, "untracked\n");527 yield* fs.writeFileString(`${worktree}/notes.txt`, "untracked\n");
...@@ -521,7 +544,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -521,7 +544,7 @@ describe("storage cleanup reports and local file policies", () => {
521 it.live("reads the thread snapshot once per sweep, not per worktree", () =>544 it.live("reads the thread snapshot once per sweep, not per worktree", () =>
522 runCleanupTest(545 runCleanupTest(
523 Effect.gen(function* () {546 Effect.gen(function* () {
524 const { service, snapshotReads } = yield* cleanupFixture;547 const { service, snapshotReads } = yield* cleanupFixture();
525 expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" });548 expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" });
526 expect(snapshotReads()).toBe(1);549 expect(snapshotReads()).toBe(1);
527 }),550 }),
...@@ -535,7 +558,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -535,7 +558,7 @@ describe("storage cleanup reports and local file policies", () => {
535 ([first, second, sweeps]) =>558 ([first, second, sweeps]) =>
536 runCleanupTest(559 runCleanupTest(
537 Effect.gen(function* () {560 Effect.gen(function* () {
538 const fixture = yield* pausedMeasurementFixture;561 const fixture = yield* pausedMeasurementFixture();
539 const handled = yield* Deferred.make<void>();562 const handled = yield* Deferred.make<void>();
540 // 20 updates arrive while the first automatic sweep measures the worktree.563 // 20 updates arrive while the first automatic sweep measures the worktree.
541 yield* fixture.activate(564 yield* fixture.activate(
...@@ -560,7 +583,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -560,7 +583,7 @@ describe("storage cleanup reports and local file policies", () => {
560 Effect.gen(function* () {583 Effect.gen(function* () {
561 const fs = yield* FileSystem.FileSystem;584 const fs = yield* FileSystem.FileSystem;
562 const walked: Array<string> = [];585 const walked: Array<string> = [];
563 const fixture = yield* cleanupFixture.pipe(586 const fixture = yield* cleanupFixture().pipe(
564 Effect.provideService(FileSystem.FileSystem, {587 Effect.provideService(FileSystem.FileSystem, {
565 ...fs,588 ...fs,
566 readDirectory: (directory, options) => {589 readDirectory: (directory, options) => {
...@@ -578,7 +601,8 @@ describe("storage cleanup reports and local file policies", () => {...@@ -578,7 +601,8 @@ describe("storage cleanup reports and local file policies", () => {
578 bytes: null,601 bytes: null,
579 });602 });
580 expect(walked.filter((directory) => directory.startsWith(root))).toEqual([]);603 expect(walked.filter((directory) => directory.startsWith(root))).toEqual([]);
581 fixture.setOpenSessionCwd(null);604 // The stored session stays ready after its runtime was released.
605 fixture.setSessionLive(false);
582 const removed = yield* fixture.service.runNow;606 const removed = yield* fixture.service.runNow;
583 expect(removed.entries[0]).toMatchObject({ outcome: "removed" });607 expect(removed.entries[0]).toMatchObject({ outcome: "removed" });
584 expect(removed.entries[0]?.bytes).toBeGreaterThan(0);608 expect(removed.entries[0]?.bytes).toBeGreaterThan(0);
...@@ -590,7 +614,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -590,7 +614,7 @@ describe("storage cleanup reports and local file policies", () => {
590 runCleanupTest(614 runCleanupTest(
591 Effect.gen(function* () {615 Effect.gen(function* () {
592 const { service, fs, worktree, command, setThreads, fullStatusCalls } =616 const { service, fs, worktree, command, setThreads, fullStatusCalls } =
593 yield* cleanupFixture;617 yield* cleanupFixture();
594 const changed = { outcome: "kept", reason: "Repository or branch changed" };618 const changed = { outcome: "kept", reason: "Repository or branch changed" };
595 yield* command(worktree, ["checkout", "--detach"]);619 yield* command(worktree, ["checkout", "--detach"]);
596 expect((yield* service.runNow).entries[0]).toMatchObject(changed);620 expect((yield* service.runNow).entries[0]).toMatchObject(changed);
...@@ -656,7 +680,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -656,7 +680,7 @@ describe("storage cleanup reports and local file policies", () => {
656 ] as const)("applies %j", ([policy, file, outcome]) =>680 ] as const)("applies %j", ([policy, file, outcome]) =>
657 runCleanupTest(681 runCleanupTest(
658 Effect.gen(function* () {682 Effect.gen(function* () {
659 const { service, fs, worktree, setPolicy } = yield* cleanupFixture;683 const { service, fs, worktree, setPolicy } = yield* cleanupFixture();
660 setPolicy(policy);684 setPolicy(policy);
661 expect(yield* service.latestReport).toBeNull();685 expect(yield* service.latestReport).toBeNull();
662 yield* fs.writeFileString(`${worktree}/${file}`, "local data\n");686 yield* fs.writeFileString(`${worktree}/${file}`, "local data\n");
...@@ -690,7 +714,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -690,7 +714,7 @@ describe("storage cleanup reports and local file policies", () => {
690 it.live("preserves tracked edits written after the final status check", () =>714 it.live("preserves tracked edits written after the final status check", () =>
691 runCleanupTest(715 runCleanupTest(
692 Effect.gen(function* () {716 Effect.gen(function* () {
693 const fixture = yield* cleanupFixture;717 const fixture = yield* cleanupFixture();
694 fixture.setPolicy("tracked-changes");718 fixture.setPolicy("tracked-changes");
695 fixture.editBeforeRemoval();719 fixture.editBeforeRemoval();
696 yield* fixture.fs.writeFileString(`${fixture.worktree}/notes.txt`, "untracked\n");720 yield* fixture.fs.writeFileString(`${fixture.worktree}/notes.txt`, "untracked\n");
...@@ -708,7 +732,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -708,7 +732,7 @@ describe("storage cleanup reports and local file policies", () => {
708 it.live("streams the current report and subsequent runs", () =>732 it.live("streams the current report and subsequent runs", () =>
709 runCleanupTest(733 runCleanupTest(
710 Effect.gen(function* () {734 Effect.gen(function* () {
711 const { service } = yield* cleanupFixture;735 const { service } = yield* cleanupFixture();
712 const reports = yield* service.reports.pipe(736 const reports = yield* service.reports.pipe(
713 Stream.take(2),737 Stream.take(2),
714 Stream.tap((report) => (report === null ? service.runNow : Effect.void)),738 Stream.tap((report) => (report === null ? service.runNow : Effect.void)),
...@@ -723,7 +747,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -723,7 +747,7 @@ describe("storage cleanup reports and local file policies", () => {
723 it.live("caps entries while preserving totals and prioritizing removals", () =>747 it.live("caps entries while preserving totals and prioritizing removals", () =>
724 runCleanupTest(748 runCleanupTest(
725 Effect.gen(function* () {749 Effect.gen(function* () {
726 const { service, fs, worktree, setThreads } = yield* cleanupFixture;750 const { service, fs, worktree, setThreads } = yield* cleanupFixture();
727 const threads = [];751 const threads = [];
728 for (let i = 0; i < 205; i++) {752 for (let i = 0; i < 205; i++) {
729 const worktreePath = `${worktree}-${i}`;753 const worktreePath = `${worktree}-${i}`;
...@@ -752,7 +776,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -752,7 +776,7 @@ describe("storage cleanup reports and local file policies", () => {
752 it.live("keeps staged edits in tracked-only mode", () =>776 it.live("keeps staged edits in tracked-only mode", () =>
753 runCleanupTest(777 runCleanupTest(
754 Effect.gen(function* () {778 Effect.gen(function* () {
755 const { service, fs, worktree, setPolicy, command } = yield* cleanupFixture;779 const { service, fs, worktree, setPolicy, command } = yield* cleanupFixture();
756 setPolicy("tracked-changes");780 setPolicy("tracked-changes");
757 yield* fs.writeFileString(`${worktree}/tracked.txt`, "staged\n");781 yield* fs.writeFileString(`${worktree}/tracked.txt`, "staged\n");
758 yield* command(worktree, ["add", "tracked.txt"]);782 yield* command(worktree, ["add", "tracked.txt"]);
...@@ -769,7 +793,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -769,7 +793,7 @@ describe("storage cleanup reports and local file policies", () => {
769 () =>793 () =>
770 runCleanupTest(794 runCleanupTest(
771 Effect.gen(function* () {795 Effect.gen(function* () {
772 const fixture = yield* cleanupFixture;796 const fixture = yield* cleanupFixture();
773 const { service, fs, worktree, setThreads } = fixture;797 const { service, fs, worktree, setThreads } = fixture;
774 setThreads([shell({ branch: "feature", worktreePath: worktree, status: "running" })]);798 setThreads([shell({ branch: "feature", worktreePath: worktree, status: "running" })]);
775 expect((yield* service.runNow).entries[0]?.reason).toContain("Thread is running");799 expect((yield* service.runNow).entries[0]?.reason).toContain("Thread is running");
...@@ -790,7 +814,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -790,7 +814,7 @@ describe("storage cleanup reports and local file policies", () => {
790 it.live("serializes concurrent requests and returns each run's report", () =>814 it.live("serializes concurrent requests and returns each run's report", () =>
791 runCleanupTest(815 runCleanupTest(
792 Effect.gen(function* () {816 Effect.gen(function* () {
793 const { service } = yield* cleanupFixture;817 const { service } = yield* cleanupFixture();
794 const reports = yield* Effect.all([service.runNow, service.runNow], {818 const reports = yield* Effect.all([service.runNow, service.runNow], {
795 concurrency: "unbounded",819 concurrency: "unbounded",
796 });820 });
...@@ -805,7 +829,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -805,7 +829,7 @@ describe("storage cleanup reports and local file policies", () => {
805 it.live("aggregates expired artifacts and rotated logs while retaining current logs", () =>829 it.live("aggregates expired artifacts and rotated logs while retaining current logs", () =>
806 runCleanupTest(830 runCleanupTest(
807 Effect.gen(function* () {831 Effect.gen(function* () {
808 const { service, fs, config, enableFiles, disable } = yield* cleanupFixture;832 const { service, fs, config, enableFiles, disable } = yield* cleanupFixture();
809 disable();833 disable();
810 enableFiles();834 enableFiles();
811 yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true });835 yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true });
...@@ -848,7 +872,7 @@ describe("storage cleanup reports and local file policies", () => {...@@ -848,7 +872,7 @@ describe("storage cleanup reports and local file policies", () => {
848 it.live("reports git failures without aborting the remaining categories", () =>872 it.live("reports git failures without aborting the remaining categories", () =>
849 runCleanupTest(873 runCleanupTest(
850 Effect.gen(function* () {874 Effect.gen(function* () {
851 const { service, worktree, command, enableFiles } = yield* cleanupFixture;875 const { service, worktree, command, enableFiles } = yield* cleanupFixture();
852 enableFiles();876 enableFiles();
853 yield* command(worktree, ["worktree", "lock", worktree]);877 yield* command(worktree, ["worktree", "lock", worktree]);
854 const report = yield* service.runNow;878 const report = yield* service.runNow;
...@@ -863,3 +887,202 @@ describe("storage cleanup reports and local file policies", () => {...@@ -863,3 +887,202 @@ describe("storage cleanup reports and local file policies", () => {
863 ),887 ),
864 );888 );
865});889});
890
891describe("settled worktree retention", () => {
892 it.effect.each([
893 "immediate",
894 "settle-script",
895 "expired",
896 "boundary",
897 "wait",
898 "recent-update",
899 "active",
900 "dirty",
901 "project-off",
902 "project-custom",
903 "inactive-settling",
904 ] as const)("applies retention and safety checks (%s)", (scenario) =>
905 Effect.gen(function* () {
906 yield* TestClock.setTime(NOW_MS);
907 const fixture = yield* cleanupFixture();
908 const days = ["wait", "boundary", "expired", "recent-update"].includes(scenario) ? 8 : 0;
909 const rules = {
910 ...DEFAULT_SERVER_SETTINGS.storageCleanup,
911 worktreeSettledAfterDays:
912 scenario === "project-custom" || scenario === "inactive-settling" ? null : days,
913 // Another rule must not bypass the settle-script grace either.
914 worktreeAfterDays: scenario === "inactive-settling" ? 1 : null,
915 };
916 fixture.setSettings({
917 storageCleanup: rules,
918 projectSettingsOverrides: {
919 [ProjectId.make("project-1")]:
920 scenario === "project-off"
921 ? { worktreeCleanup: { mode: "off" } }
922 : scenario === "project-custom"
923 ? {
924 worktreeCleanup: {
925 mode: "custom",
926 rules: { ...rules, worktreeSettledAfterDays: 0 },
927 },
928 }
929 : {},
930 },
931 });
932 const thread = shell({
933 branch: "feature",
934 worktreePath: fixture.worktree,
935 settledAt:
936 scenario === "active"
937 ? null
938 : at(
939 scenario === "expired" ||
940 scenario === "recent-update" ||
941 scenario === "inactive-settling"
942 ? -20 * DAY_MS
943 : scenario === "boundary"
944 ? -8 * DAY_MS
945 : scenario === "wait"
946 ? -8 * DAY_MS + 1
947 : scenario === "settle-script"
948 ? 0
949 : -60_000,
950 ),
951 updatedAt: at(
952 scenario === "recent-update" ||
953 scenario === "settle-script" ||
954 scenario === "inactive-settling"
955 ? 0
956 : -10 * DAY_MS,
957 ),
958 });
959 fixture.setThreads([thread]);
960 if (scenario === "dirty")
961 yield* fixture.fs.writeFileString(`${fixture.worktree}/tracked.txt`, "edit\n");
962 const report = yield* fixture.service.runNow;
963 const removed = ["immediate", "expired", "boundary", "project-custom"].includes(scenario);
964 expect(yield* fixture.fs.exists(fixture.worktree)).toBe(!removed);
965 expect(report.counts.removed).toBe(removed ? 1 : 0);
966 if (scenario !== "project-off") {
967 const reason =
968 scenario === "active"
969 ? "Not settled"
970 : scenario === "wait"
971 ? "Settled for 7 of 8 days"
972 : ["settle-script", "recent-update", "inactive-settling"].includes(scenario)
973 ? "Settled moments ago"
974 : scenario === "dirty"
975 ? "Has uncommitted changes (1 file)"
976 : `Settled for ${scenario === "expired" ? 20 : scenario === "boundary" ? 8 : 0} days`;
977 expect(report.entries[0]).toMatchObject({ outcome: removed ? "removed" : "kept", reason });
978 }
979 expect(thread.branch).toBe("feature");
980 expect(
981 (yield* fixture.command(fixture.repo, ["rev-parse", "--verify", "refs/heads/feature"]))
982 .exitCode,
983 ).toBe(0);
984 }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped),
985 );
986});
987
988describe("settled worktree rechecks", () => {
989 it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])(
990 "keeps a checkout changed during measurement (%j)",
991 (change) =>
992 runCleanupTest(
993 Effect.gen(function* () {
994 const fixture = yield* pausedMeasurementFixture();
995 const thread = shell({
996 branch: "feature",
997 worktreePath: fixture.worktree,
998 settledAt: at(0),
999 });
1000 fixture.setSettings({
1001 storageCleanup: {
1002 ...DEFAULT_SERVER_SETTINGS.storageCleanup,
1003 worktreeSettledAfterDays: 0,
1004 },
1005 });
1006 fixture.setThreads([thread]);
1007 const run = yield* fixture.service.runNow.pipe(Effect.forkChild);
1008 yield* Deferred.await(fixture.measuring);
1009 fixture.setThreads([{ ...thread, ...change }]);
1010 yield* Deferred.succeed(fixture.resume, undefined);
1011 const report = yield* Fiber.join(run);
1012 expect(report.entries[0]?.reason).toBe(
1013 "Thread activity or shared worktree changed since check",
1014 );
1015 expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true);
1016 }),
1017 ),
1018 );
1019});
1020
1021it.live("keeps saved usage data when removing a settled checkout", () =>
1022 runCleanupTest(
1023 Effect.gen(function* () {
1024 const fixture = yield* cleanupFixture();
1025 fixture.setSettings({
1026 storageCleanup: { ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeSettledAfterDays: 0 },
1027 });
1028 fixture.setThreads([
1029 shell({ branch: "feature", worktreePath: fixture.worktree, settledAt: at(0) }),
1030 ]);
1031 yield* fixture.fs.makeDirectory(fixture.config.stateDir, { recursive: true });
1032 const cachePath = `${fixture.config.stateDir}/${SCAN_CACHE_FILE_NAME}`;
1033 const savedUsage = JSON.stringify({ tokens: 12345, cost: 1.25 });
1034 yield* fixture.fs.writeFileString(cachePath, savedUsage);
1035 expect((yield* fixture.service.runNow).counts.removed).toBe(1);
1036 expect(yield* fixture.fs.readFileString(cachePath)).toBe(savedUsage);
1037 }),
1038 ),
1039);
1040
1041it.live.each(["thread.settled", "stopped", "ready"] as const)(
1042 "leaves settled cleanup to the regular sweeps after %s",
1043 (trigger) =>
1044 runCleanupTest(
1045 Effect.gen(function* () {
1046 const fixture = yield* pausedMeasurementFixture({
1047 ...DEFAULT_SERVER_SETTINGS.storageCleanup,
1048 worktreeSettledAfterDays: 0,
1049 });
1050 fixture.setSettings({
1051 storageCleanup: {
1052 ...DEFAULT_SERVER_SETTINGS.storageCleanup,
1053 worktreeSettledAfterDays: 0,
1054 },
1055 });
1056 const thread = shell({
1057 branch: "feature",
1058 worktreePath: fixture.worktree,
1059 settledAt: at(0),
1060 });
1061 fixture.setThreads([thread]);
1062 const handled = yield* Deferred.make<void>();
1063 yield* fixture.activate(
1064 Stream.fromEffect(Deferred.await(fixture.measuring)).pipe(
1065 Stream.flatMap(() => Stream.range(0, 19)),
1066 Stream.map((index): OrchestrationV2DomainEvent =>
1067 trigger === "thread.settled"
1068 ? {
1069 id: EventId.make(`settled-${index}`),
1070 type: "thread.settled",
1071 threadId: thread.id,
1072 occurredAt: at(0),
1073 payload: { ...thread, lastVisitedAt: null },
1074 }
1075 : sessionUpdated(index, trigger),
1076 ),
1077 Stream.concat(
1078 Stream.fromEffect(Deferred.succeed(handled, undefined)).pipe(Stream.drain),
1079 ),
1080 ),
1081 );
1082 yield* Deferred.await(handled);
1083 yield* Deferred.succeed(fixture.resume, undefined);
1084 yield* fixture.service.drain;
1085 expect(fixture.snapshotReads()).toBe(1);
1086 }),
1087 ),
1088);
apps/server/src/storageCleanup.ts+47-13
...@@ -38,6 +38,7 @@ import * as GitManager from "./git/GitManager.ts";...@@ -38,6 +38,7 @@ import * as GitManager from "./git/GitManager.ts";
38import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";38import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
39import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";39import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
40import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";40import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";
41import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
41import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts";42import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts";
42import { forkParked } from "./serverActivation.ts";43import { forkParked } from "./serverActivation.ts";
43import * as Settings from "./serverSettings.ts";44import * as Settings from "./serverSettings.ts";
...@@ -56,6 +57,9 @@ const decodeCleanupSession = Schema.decodeUnknownEffect(...@@ -56,6 +57,9 @@ const decodeCleanupSession = Schema.decodeUnknownEffect(
56);57);
5758
58const DAY_MS = 86_400_000;59const DAY_MS = 86_400_000;
60// After a settle, the project's settle script needs time to open its
61// terminal, which then protects the worktree.
62const SETTLE_SCRIPT_GRACE_MS = 60_000;
5963
60const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1);64const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1);
61const REPORT_ENTRY_LIMIT = 200;65const REPORT_ENTRY_LIMIT = 200;
...@@ -69,6 +73,7 @@ function cleanupFailureReason(error: { readonly message: string }) {...@@ -69,6 +73,7 @@ function cleanupFailureReason(error: { readonly message: string }) {
6973
70const worktreeCleanupEnabled = (rules: WorktreeCleanupRules) =>74const worktreeCleanupEnabled = (rules: WorktreeCleanupRules) =>
71 rules.worktreeAfterDays !== null ||75 rules.worktreeAfterDays !== null ||
76 rules.worktreeSettledAfterDays !== null ||
72 rules.worktreeOnMerge ||77 rules.worktreeOnMerge ||
73 rules.worktreeOnDelete ||78 rules.worktreeOnDelete ||
74 rules.worktreeUnchanged;79 rules.worktreeUnchanged;
...@@ -172,6 +177,7 @@ const make = Effect.gen(function* () {...@@ -172,6 +177,7 @@ const make = Effect.gen(function* () {
172 const projectStore = yield* ProjectStore.ProjectStoreV2;177 const projectStore = yield* ProjectStore.ProjectStoreV2;
173 const engine = yield* Orchestrator.OrchestratorV2;178 const engine = yield* Orchestrator.OrchestratorV2;
174 const projections = yield* ProjectionStore.ProjectionStoreV2;179 const projections = yield* ProjectionStore.ProjectionStoreV2;
180 const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
175 const sql = yield* SqlClient.SqlClient;181 const sql = yield* SqlClient.SqlClient;
176 const git = yield* GitVcsDriver.GitVcsDriver;182 const git = yield* GitVcsDriver.GitVcsDriver;
177 const gitManager = yield* GitManager.GitManager;183 const gitManager = yield* GitManager.GitManager;
...@@ -483,15 +489,35 @@ const make = Effect.gen(function* () {...@@ -483,15 +489,35 @@ const make = Effect.gen(function* () {
483 return keep("repository or branch changed");489 return keep("repository or branch changed");
484 const changes = yield* localChangesReason(worktreePath, settings);490 const changes = yield* localChangesReason(worktreePath, settings);
485 if (changes !== null) return keep(changes);491 if (changes !== null) return keep(changes);
492 // The project's settle script starts in the worktree right after a
493 // settle. Settling stamps updatedAt now, even when automatic
494 // settlement backdates settledAt to the thread's last activity.
495 if (
496 !deleted &&
497 thread.settledAt !== null &&
498 now - DateTime.toEpochMillis(thread.updatedAt) < SETTLE_SCRIPT_GRACE_MS
499 )
500 return keep("settled moments ago");
486 const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" });501 const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" });
487 const old =502 const old =
488 !deleted &&503 !deleted &&
489 settings.worktreeAfterDays !== null &&504 settings.worktreeAfterDays !== null &&
490 storageCleanupActivityAt(thread) < now - settings.worktreeAfterDays * DAY_MS;505 storageCleanupActivityAt(thread) < now - settings.worktreeAfterDays * DAY_MS;
491 let eligible = deleted || old;506 const settledFor =
507 deleted || thread.settledAt === null
508 ? null
509 : now - DateTime.toEpochMillis(thread.settledAt);
510 const settledDays = settledFor === null ? null : Math.floor(settledFor / DAY_MS);
511 const settled =
512 settledFor !== null &&
513 settings.worktreeSettledAfterDays !== null &&
514 settledFor >= settings.worktreeSettledAfterDays * DAY_MS;
515 let eligible = deleted || old || settled;
492 let removalReason = deleted516 let removalReason = deleted
493 ? "thread was deleted"517 ? "thread was deleted"
494 : `inactive for ${Math.floor((now - storageCleanupActivityAt(thread)) / DAY_MS)} days`;518 : old
519 ? `inactive for ${Math.floor((now - storageCleanupActivityAt(thread)) / DAY_MS)} days`
520 : `settled for ${settledDays} days`;
495 if (!eligible && (settings.worktreeUnchanged || settings.worktreeOnMerge)) {521 if (!eligible && (settings.worktreeUnchanged || settings.worktreeOnMerge)) {
496 const repositoryCwd = path.resolve(project.workspaceRoot);522 const repositoryCwd = path.resolve(project.workspaceRoot);
497 const remote = yield* git.resolvePrimaryRemoteName(repositoryCwd);523 const remote = yield* git.resolvePrimaryRemoteName(repositoryCwd);
...@@ -539,9 +565,13 @@ const make = Effect.gen(function* () {...@@ -539,9 +565,13 @@ const make = Effect.gen(function* () {
539 return keep(565 return keep(
540 !deleted && settings.worktreeAfterDays !== null566 !deleted && settings.worktreeAfterDays !== null
541 ? `inactive for ${Math.floor((now - storageCleanupActivityAt(thread)) / DAY_MS)} of ${settings.worktreeAfterDays} days`567 ? `inactive for ${Math.floor((now - storageCleanupActivityAt(thread)) / DAY_MS)} of ${settings.worktreeAfterDays} days`
542 : settings.worktreeOnMerge568 : settings.worktreeSettledAfterDays !== null
543 ? "not merged"569 ? settledFor === null
544 : "has commits beyond the default branch",570 ? "not settled"
571 : `settled for ${settledDays} of ${settings.worktreeSettledAfterDays} days`
572 : settings.worktreeOnMerge
573 ? "not merged"
574 : "has commits beyond the default branch",
545 );575 );
546 // Threads, queued deletion work and provider sessions as they are now, so a576 // Threads, queued deletion work and provider sessions as they are now, so a
547 // queued turn, resumed session or new thread sharing this path cancels577 // queued turn, resumed session or new thread sharing this path cancels
...@@ -577,7 +607,8 @@ const make = Effect.gen(function* () {...@@ -577,7 +607,8 @@ const make = Effect.gen(function* () {
577 if (607 if (
578 latestShell === null ||608 latestShell === null ||
579 !storageCleanupThreadIdle(latestShell, now) ||609 !storageCleanupThreadIdle(latestShell, now) ||
580 storageCleanupActivityAt(latestShell) !== storageCleanupActivityAt(thread)610 storageCleanupActivityAt(latestShell) !== storageCleanupActivityAt(thread) ||
611 !Equal.equals(latestShell.settledAt, thread.settledAt)
581 )612 )
582 return "thread activity or shared worktree changed since check";613 return "thread activity or shared worktree changed since check";
583 }614 }
...@@ -589,13 +620,16 @@ const make = Effect.gen(function* () {...@@ -589,13 +620,16 @@ const make = Effect.gen(function* () {
589 const sessions = yield* Effect.forEach(sessionRows, (row) =>620 const sessions = yield* Effect.forEach(sessionRows, (row) =>
590 decodeCleanupSession(row.payload_json),621 decodeCleanupSession(row.payload_json),
591 );622 );
592 if (623 // A stored session can stay ready after its runtime was released,
593 sessions.some((session) => {624 // for example once every thread detached; only a live one holds it.
594 const cwd = path.resolve(session.cwd);625 for (const session of sessions) {
595 return cwd === worktreePath || inside(worktreePath, cwd);626 const cwd = path.resolve(session.cwd);
596 })627 if (
597 )628 (cwd === worktreePath || inside(worktreePath, cwd)) &&
598 return "provider session is still open";629 (yield* providerSessions.isLive(session.id))
630 )
631 return "provider session is still open";
632 }
599 return null;633 return null;
600 });634 });
601 // Most cancelled removals are caught here, before the worktree is walked.635 // Most cancelled removals are caught here, before the worktree is walked.
apps/web/src/components/settings/StorageSettings.test.tsx created+168
...@@ -0,0 +1,168 @@
1// @vitest-environment jsdom
2import { RegistryProvider } from "@effect/atom-react";
3import { DEFAULT_SERVER_SETTINGS, ProjectId, type StorageCleanupReport } from "@t3tools/contracts";
4import { act, type ReactNode } from "react";
5import { createRoot, type Root } from "react-dom/client";
6import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
7
8const state = vi.hoisted(() => ({
9 settings: {} as typeof DEFAULT_SERVER_SETTINGS,
10 run: vi.fn(),
11}));
12
13vi.mock("../../state/server", async () => {
14 const { Atom } = await import("effect/reactivity");
15 const report = Atom.make({ _tag: "Success", value: null, waiting: false });
16 const allowed = Atom.make(true);
17 return {
18 serverEnvironment: {
19 storageCleanupReport: () => report,
20 runStorageCleanup: { permissionAtom: () => allowed },
21 },
22 };
23});
24vi.mock("../../state/use-atom-command", () => ({ useAtomCommand: () => state.run }));
25vi.mock("./SettingsScopeContext", () => ({
26 useSettingsScope: () => {
27 const environment = {
28 environmentId: "env",
29 label: "Machine",
30 serverConfig: {
31 settings: state.settings,
32 environment: {
33 capabilities: {
34 storageCleanup: true,
35 storageCleanupRun: true,
36 settledWorktreeCleanup: true,
37 },
38 },
39 },
40 };
41 return {
42 scope: { kind: "environment" },
43 environments: [environment],
44 connectedEnvironments: [environment],
45 targets: [],
46 target: null,
47 };
48 },
49}));
50vi.mock("./useScopedSettings", () => ({
51 useScopedSettings: () => state.settings,
52 useScopedSettingsMixed: () => false,
53 useUpdateScopedSettings: () => vi.fn(),
54 useClearScopedSettings: () => vi.fn(),
55}));
56vi.mock("./settingsLayout", () => ({
57 SettingsPageContainer: ({ children }: { children: ReactNode }) => children,
58 SettingsSection: ({ children }: { children: ReactNode }) => children,
59 SettingsRow: ({ description, control }: { description: ReactNode; control: ReactNode }) => (
60 <div>
61 {description}
62 {control}
63 </div>
64 ),
65 SettingResetButton: () => null,
66 useRelativeTimeTick: () => undefined,
67}));
68vi.mock("../ui/toast", () => ({ toastManager: { add: vi.fn() } }));
69
70import { StorageSettingsPanel } from "./StorageSettings";
71
72let root: Root;
73let container: HTMLDivElement;
74
75beforeEach(() => {
76 vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
77 state.settings = {
78 ...DEFAULT_SERVER_SETTINGS,
79 storageCleanup: {
80 worktreeKeepWhen: "uncommitted-changes",
81 worktreeAfterDays: null,
82 worktreeSettledAfterDays: null,
83 worktreeOnMerge: false,
84 worktreeOnDelete: false,
85 worktreeUnchanged: false,
86 browserArtifactsAfterDays: null,
87 logsAfterDays: null,
88 },
89 projectSettingsOverrides: {},
90 };
91 state.run.mockReset();
92 container = document.createElement("div");
93 document.body.append(container);
94 root = createRoot(container);
95});
96
97afterEach(async () => {
98 await act(async () => root.unmount());
99 container.remove();
100 vi.unstubAllGlobals();
101});
102
103async function render() {
104 await act(async () =>
105 root.render(
106 <RegistryProvider>
107 <StorageSettingsPanel />
108 </RegistryProvider>,
109 ),
110 );
111 return Array.from(container.querySelectorAll("button")).find(
112 (button) => button.textContent === "Delete now",
113 )!;
114}
115
116it.each(["machine-wide immediate", "project custom"])(
117 "runs cleanup when only %s settled retention is enabled",
118 async (scope) => {
119 if (scope === "machine-wide immediate") {
120 state.settings = {
121 ...state.settings,
122 storageCleanup: { ...state.settings.storageCleanup, worktreeSettledAfterDays: 0 },
123 };
124 } else {
125 state.settings = {
126 ...state.settings,
127 projectSettingsOverrides: {
128 [ProjectId.make("project")]: {
129 worktreeCleanup: {
130 mode: "custom",
131 rules: { ...state.settings.storageCleanup, worktreeSettledAfterDays: 7 },
132 },
133 },
134 },
135 };
136 }
137 const report: StorageCleanupReport = {
138 trigger: "manual",
139 startedAt: "2026-10-14T12:00:00.000Z",
140 finishedAt: "2026-10-14T12:00:01.000Z",
141 entries: [],
142 counts: { removed: 0, kept: 0, failed: 0 },
143 omittedCount: 0,
144 bytesFreed: 0,
145 };
146 let complete!: (result: { _tag: "Success"; value: StorageCleanupReport }) => void;
147 state.run.mockImplementation(() => new Promise((resolve) => (complete = resolve)));
148 const button = await render();
149
150 expect(button.disabled).toBe(false);
151 await act(async () => button.click());
152 expect(state.run).toHaveBeenCalledExactlyOnceWith({ environmentId: "env", input: {} });
153 expect(button.disabled).toBe(true);
154 expect(button.textContent).toBe("Deleting…");
155
156 await act(async () => complete({ _tag: "Success", value: report }));
157 expect(button.disabled).toBe(false);
158 expect(button.textContent).toBe("Delete now");
159 },
160);
161
162it("keeps cleanup disabled when settled retention and all other rules are off", async () => {
163 const button = await render();
164
165 expect(button.disabled).toBe(true);
166 await act(async () => button.click());
167 expect(state.run).not.toHaveBeenCalled();
168});
apps/web/src/components/settings/StorageSettings.tsx+36-3
...@@ -106,10 +106,12 @@ function RetentionControl({...@@ -106,10 +106,12 @@ function RetentionControl({
106 label,106 label,
107 value,107 value,
108 onChange,108 onChange,
109 minimum = 1,
109}: {110}: {
110 label: string;111 label: string;
111 value: number | null;112 value: number | null;
112 onChange: (value: number | null) => void;113 onChange: (value: number | null) => void;
114 minimum?: number;
113}) {115}) {
114 const [draft, setDraft] = useState(value);116 const [draft, setDraft] = useState(value);
115 const [savedValue, setSavedValue] = useState(value);117 const [savedValue, setSavedValue] = useState(value);
...@@ -123,7 +125,7 @@ function RetentionControl({...@@ -123,7 +125,7 @@ function RetentionControl({
123 {value !== null ? (125 {value !== null ? (
124 <NumberField126 <NumberField
125 value={draft}127 value={draft}
126 min={1}128 min={minimum}
127 max={3650}129 max={3650}
128 step={1}130 step={1}
129 size="sm"131 size="sm"
...@@ -132,7 +134,7 @@ function RetentionControl({...@@ -132,7 +134,7 @@ function RetentionControl({
132 onValueCommitted={(next) => {134 onValueCommitted={(next) => {
133 if (next === null) setDraft(value);135 if (next === null) setDraft(value);
134 else {136 else {
135 const days = Math.min(3650, Math.max(1, Math.round(next)));137 const days = Math.min(3650, Math.max(minimum, Math.round(next)));
136 setDraft(days);138 setDraft(days);
137 onChange(days);139 onChange(days);
138 }140 }
...@@ -271,6 +273,7 @@ function CleanupSection() {...@@ -271,6 +273,7 @@ function CleanupSection() {
271 if (!settings) return false;273 if (!settings) return false;
272 const enabled = (rules: WorktreeCleanupRules) =>274 const enabled = (rules: WorktreeCleanupRules) =>
273 rules.worktreeAfterDays !== null ||275 rules.worktreeAfterDays !== null ||
276 rules.worktreeSettledAfterDays !== null ||
274 rules.worktreeOnMerge ||277 rules.worktreeOnMerge ||
275 rules.worktreeOnDelete ||278 rules.worktreeOnDelete ||
276 rules.worktreeUnchanged;279 rules.worktreeUnchanged;
...@@ -397,7 +400,7 @@ function CleanupSection() {...@@ -397,7 +400,7 @@ function CleanupSection() {
397}400}
398401
399export function StorageSettingsPanel() {402export function StorageSettingsPanel() {
400 const { scope, connectedEnvironments, targets, target } = useSettingsScope();403 const { scope, environments, connectedEnvironments, targets, target } = useSettingsScope();
401 const scopedSettings = useScopedSettings();404 const scopedSettings = useScopedSettings();
402 const isProjectScope = scope.kind === "project" || scope.kind === "checkout";405 const isProjectScope = scope.kind === "project" || scope.kind === "checkout";
403 const settings = {406 const settings = {
...@@ -538,6 +541,36 @@ export function StorageSettingsPanel() {...@@ -538,6 +541,36 @@ export function StorageSettingsPanel() {
538 />541 />
539 }542 }
540 />543 />
544 {connectedEnvironments.length > 0 &&
545 connectedEnvironments.every(
546 (environment) =>
547 environment.serverConfig?.environment.capabilities.settledWorktreeCleanup ===
548 true,
549 ) && (
550 <SettingsRow
551 title="Delete settled worktrees"
552 status={[
553 ruleStatus("worktreeSettledAfterDays"),
554 environments.length > connectedEnvironments.length
555 ? "Offline machines will not get this setting"
556 : undefined,
557 ]
558 .filter(Boolean)
559 .join(". ")}
560 description="Remove worktrees this many days after their thread settles. Set 0 days to remove them at the next cleanup. Branches and thread history are kept."
561 serverScoped={!isProjectScope}
562 control={
563 <RetentionControl
564 label="Delete settled worktrees"
565 value={settings.worktreeSettledAfterDays}
566 minimum={0}
567 onChange={(worktreeSettledAfterDays) =>
568 updateWorktree({ worktreeSettledAfterDays })
569 }
570 />
571 }
572 />
573 )}
541 <SettingsRow574 <SettingsRow
542 title="Delete merged worktrees"575 title="Delete merged worktrees"
543 status={ruleStatus("worktreeOnMerge")}576 status={ruleStatus("worktreeOnMerge")}
apps/web/src/components/settings/scopedSettings.test.ts+15
...@@ -158,6 +158,7 @@ describe("scoped settings writes", () => {...@@ -158,6 +158,7 @@ describe("scoped settings writes", () => {
158 mode: "custom",158 mode: "custom",
159 rules: {159 rules: {
160 worktreeAfterDays: 12,160 worktreeAfterDays: 12,
161 worktreeSettledAfterDays: null,
161 worktreeOnDelete: true,162 worktreeOnDelete: true,
162 worktreeOnMerge: true,163 worktreeOnMerge: true,
163 worktreeUnchanged: false,164 worktreeUnchanged: false,
...@@ -182,6 +183,7 @@ describe("scoped settings writes", () => {...@@ -182,6 +183,7 @@ describe("scoped settings writes", () => {
182 expect(policies).toEqual([183 expect(policies).toEqual([
183 {184 {
184 worktreeAfterDays: 12,185 worktreeAfterDays: 12,
186 worktreeSettledAfterDays: null,
185 worktreeOnDelete: false,187 worktreeOnDelete: false,
186 worktreeOnMerge: true,188 worktreeOnMerge: true,
187 worktreeUnchanged: false,189 worktreeUnchanged: false,
...@@ -189,6 +191,7 @@ describe("scoped settings writes", () => {...@@ -189,6 +191,7 @@ describe("scoped settings writes", () => {
189 },191 },
190 {192 {
191 worktreeAfterDays: null,193 worktreeAfterDays: null,
194 worktreeSettledAfterDays: null,
192 worktreeOnDelete: false,195 worktreeOnDelete: false,
193 worktreeOnMerge: false,196 worktreeOnMerge: false,
194 worktreeUnchanged: false,197 worktreeUnchanged: false,
...@@ -381,6 +384,7 @@ describe("scoped settings writes", () => {...@@ -381,6 +384,7 @@ describe("scoped settings writes", () => {
381 mode: "custom",384 mode: "custom",
382 rules: {385 rules: {
383 worktreeAfterDays: 8,386 worktreeAfterDays: 8,
387 worktreeSettledAfterDays: null,
384 worktreeOnDelete: false,388 worktreeOnDelete: false,
385 worktreeOnMerge: true,389 worktreeOnMerge: true,
386 worktreeUnchanged: false,390 worktreeUnchanged: false,
...@@ -394,6 +398,15 @@ describe("scoped settings writes", () => {...@@ -394,6 +398,15 @@ describe("scoped settings writes", () => {
394 const plan = planScopedSettingsPatch(project, [machine, customized], {398 const plan = planScopedSettingsPatch(project, [machine, customized], {
395 worktreeCleanup: { mode: "custom", rules: { worktreeOnDelete: true } },399 worktreeCleanup: { mode: "custom", rules: { worktreeOnDelete: true } },
396 });400 });
401 const settledPlan = planScopedSettingsPatch(project, [machine, customized], {
402 worktreeCleanup: { mode: "custom", rules: { worktreeSettledAfterDays: 0 } },
403 });
404 expect(
405 settledPlan.serverWrites.map((write) => write.patch.projectSettingsOverrides),
406 ).toMatchObject([
407 { [projectId]: { worktreeCleanup: { rules: { worktreeSettledAfterDays: 0 } } } },
408 { [laptopProjectId]: { worktreeCleanup: { rules: { worktreeSettledAfterDays: 0 } } } },
409 ]);
397 expect(plan.serverWrites.map((write) => write.patch.projectSettingsOverrides)).toEqual([410 expect(plan.serverWrites.map((write) => write.patch.projectSettingsOverrides)).toEqual([
398 {411 {
399 [projectId]: {412 [projectId]: {
...@@ -402,6 +415,7 @@ describe("scoped settings writes", () => {...@@ -402,6 +415,7 @@ describe("scoped settings writes", () => {
402 mode: "custom",415 mode: "custom",
403 rules: {416 rules: {
404 worktreeAfterDays: 8,417 worktreeAfterDays: 8,
418 worktreeSettledAfterDays: null,
405 worktreeOnDelete: true,419 worktreeOnDelete: true,
406 worktreeOnMerge: true,420 worktreeOnMerge: true,
407 worktreeUnchanged: false,421 worktreeUnchanged: false,
...@@ -416,6 +430,7 @@ describe("scoped settings writes", () => {...@@ -416,6 +430,7 @@ describe("scoped settings writes", () => {
416 mode: "custom",430 mode: "custom",
417 rules: {431 rules: {
418 worktreeAfterDays: 30,432 worktreeAfterDays: 30,
433 worktreeSettledAfterDays: null,
419 worktreeOnDelete: true,434 worktreeOnDelete: true,
420 worktreeOnMerge: false,435 worktreeOnMerge: false,
421 worktreeUnchanged: false,436 worktreeUnchanged: false,
docs/user/project-settings.md+4-3
...@@ -149,9 +149,10 @@ Inherit follows each machine's rules; Off keeps that project's worktrees until y...@@ -149,9 +149,10 @@ Inherit follows each machine's rules; Off keeps that project's worktrees until y
149manually. Custom applies separate worktree rules to the selected project or checkout. Browser149manually. Custom applies separate worktree rules to the selected project or checkout. Browser
150captures and log retention remain machine-wide.150captures and log retention remain machine-wide.
151151
152Worktrees can be removed after a chosen number of inactive days, after merging, or when they152Worktrees can be removed after a chosen number of inactive days, after a thread settles, after
153have no commits beyond the default branch. Only T3-managed worktrees are eligible. Active153merging, or when they have no commits beyond the default branch. Set settled retention to 0 days
154sessions and shared worktrees prevent removal. **Keep worktrees with local changes** defaults154to remove a worktree at the next cleanup after settlement. Only T3-managed worktrees are eligible.
155Active sessions and shared worktrees prevent removal. **Keep worktrees with local changes** defaults
155to **Uncommitted changes**, which protects tracked edits and untracked files but deletes ignored156to **Uncommitted changes**, which protects tracked edits and untracked files but deletes ignored
156files such as `.env` and build output. **Any local files** also protects ignored files other than157files such as `.env` and build output. **Any local files** also protects ignored files other than
157`node_modules`. **Edited tracked files** protects only tracked edits and allows untracked and158`node_modules`. **Edited tracked files** protects only tracked edits and allows untracked and
packages/contracts/src/environment.test.ts+14
...@@ -14,6 +14,20 @@ const descriptor = {...@@ -14,6 +14,20 @@ const descriptor = {
14} as const;14} as const;
1515
16describe("ExecutionEnvironmentDescriptor", () => {16describe("ExecutionEnvironmentDescriptor", () => {
17 it("keeps settled cleanup optional alongside current worktree capabilities", () => {
18 expect(decodeDescriptor(descriptor).capabilities.settledWorktreeCleanup).toBeUndefined();
19 const capabilities = {
20 settledWorktreeCleanup: true,
21 storageCleanupRun: true,
22 projectWorktreeCleanup: true,
23 worktreesDirectory: true,
24 threadAutoSettlementScope: true,
25 };
26 expect(decodeDescriptor({ ...descriptor, capabilities }).capabilities).toMatchObject(
27 capabilities,
28 );
29 });
30
17 it("decodes old, recognized and future manual installation descriptors", () => {31 it("decodes old, recognized and future manual installation descriptors", () => {
18 expect(decodeDescriptor(descriptor).capabilities.serverInstallation).toBeUndefined();32 expect(decodeDescriptor(descriptor).capabilities.serverInstallation).toBeUndefined();
19 for (const installation of [{ kind: "npx" }, { kind: "npm-global", prefix: "/opt/node" }]) {33 for (const installation of [{ kind: "npx" }, { kind: "npm-global", prefix: "/opt/node" }]) {
packages/contracts/src/environment.ts+1
...@@ -129,6 +129,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({...@@ -129,6 +129,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
129 /** Server checks OTLP endpoints for the telemetry export settings. */129 /** Server checks OTLP endpoints for the telemetry export settings. */
130 otlpEndpointCheck: Schema.optionalKey(Schema.Boolean),130 otlpEndpointCheck: Schema.optionalKey(Schema.Boolean),
131 projectWorktreeCleanup: Schema.optionalKey(Schema.Boolean),131 projectWorktreeCleanup: Schema.optionalKey(Schema.Boolean),
132 settledWorktreeCleanup: Schema.optionalKey(Schema.Boolean),
132 /** Server honors the `worktreesDirectory` setting. */133 /** Server honors the `worktreesDirectory` setting. */
133 worktreesDirectory: Schema.optionalKey(Schema.Boolean),134 worktreesDirectory: Schema.optionalKey(Schema.Boolean),
134 /** Server can exclude PR-linked threads from inactivity settlement. */135 /** Server can exclude PR-linked threads from inactivity settlement. */
packages/contracts/src/settings.test.ts+31-1
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1import { describe, expect, it } from "vite-plus/test";1import { describe, expect, it } from "vite-plus/test";
2import * as Schema from "effect/Schema";2import * as Schema from "effect/Schema";
33
4import type { CommandId, ProjectId } from "./baseSchemas.ts";4import { type CommandId, ProjectId } from "./baseSchemas.ts";
5import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts";5import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts";
6import {6import {
7 ClientSettingsSchema,7 ClientSettingsSchema,
...@@ -56,6 +56,7 @@ describe("storage cleanup settings", () => {...@@ -56,6 +56,7 @@ describe("storage cleanup settings", () => {
56 expect(decodeServerSettings({}).worktreeCleanup).toBeNull();56 expect(decodeServerSettings({}).worktreeCleanup).toBeNull();
57 expect(decodeServerSettings({}).storageCleanup).toEqual({57 expect(decodeServerSettings({}).storageCleanup).toEqual({
58 worktreeAfterDays: null,58 worktreeAfterDays: null,
59 worktreeSettledAfterDays: null,
59 worktreeOnMerge: false,60 worktreeOnMerge: false,
60 worktreeOnDelete: false,61 worktreeOnDelete: false,
61 worktreeUnchanged: false,62 worktreeUnchanged: false,
...@@ -72,6 +73,9 @@ describe("storage cleanup settings", () => {...@@ -72,6 +73,9 @@ describe("storage cleanup settings", () => {
72 expect(decodeServerSettingsPatch({ storageCleanup: { worktreeAfterDays: null } })).toEqual({73 expect(decodeServerSettingsPatch({ storageCleanup: { worktreeAfterDays: null } })).toEqual({
73 storageCleanup: { worktreeAfterDays: null },74 storageCleanup: { worktreeAfterDays: null },
74 });75 });
76 expect(decodeServerSettingsPatch({ storageCleanup: { worktreeSettledAfterDays: 0 } })).toEqual({
77 storageCleanup: { worktreeSettledAfterDays: 0 },
78 });
75 });79 });
7680
77 it("accepts partial custom patches but requires complete stored project rules", () => {81 it("accepts partial custom patches but requires complete stored project rules", () => {
...@@ -87,6 +91,26 @@ describe("storage cleanup settings", () => {...@@ -87,6 +91,26 @@ describe("storage cleanup settings", () => {
87 },91 },
88 }),92 }),
89 ).toThrow();93 ).toThrow();
94 expect(
95 decodeServerSettings({
96 projectSettingsOverrides: {
97 project: {
98 worktreeCleanup: {
99 mode: "custom",
100 rules: {
101 worktreeAfterDays: null,
102 worktreeOnMerge: false,
103 worktreeOnDelete: false,
104 worktreeUnchanged: false,
105 },
106 },
107 },
108 },
109 }).projectSettingsOverrides[ProjectId.make("project")]?.worktreeCleanup,
110 ).toMatchObject({
111 mode: "custom",
112 rules: { worktreeSettledAfterDays: null, worktreeKeepWhen: "uncommitted-changes" },
113 });
90 });114 });
91115
92 it("decodes older custom rules with the default local-file policy", () => {116 it("decodes older custom rules with the default local-file policy", () => {
...@@ -117,6 +141,12 @@ describe("storage cleanup settings", () => {...@@ -117,6 +141,12 @@ describe("storage cleanup settings", () => {
117 decodeServerSettingsPatch({ storageCleanup: { browserArtifactsAfterDays: days } }),141 decodeServerSettingsPatch({ storageCleanup: { browserArtifactsAfterDays: days } }),
118 ).toThrow();142 ).toThrow();
119 });143 });
144
145 it.each([-1, 1.5, 3651])("rejects invalid settled retention %s", (days) => {
146 expect(() =>
147 decodeServerSettingsPatch({ storageCleanup: { worktreeSettledAfterDays: days } }),
148 ).toThrow();
149 });
120});150});
121151
122describe("ClientSettings rich text composer", () => {152describe("ClientSettings rich text composer", () => {
packages/contracts/src/settings.ts+11
...@@ -934,6 +934,9 @@ export type ResponseStreamingMode = typeof ResponseStreamingMode.Type;...@@ -934,6 +934,9 @@ export type ResponseStreamingMode = typeof ResponseStreamingMode.Type;
934const StorageRetentionDays = Schema.NullOr(934const StorageRetentionDays = Schema.NullOr(
935 Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 3650 })),935 Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 3650 })),
936);936);
937const SettledWorktreeRetentionDays = Schema.NullOr(
938 Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 3650 })),
939);
937940
938export const WorktreeKeepWhen = Schema.Literals([941export const WorktreeKeepWhen = Schema.Literals([
939 "any-local-files",942 "any-local-files",
...@@ -947,6 +950,9 @@ export const WorktreeCleanupRules = Schema.Struct({...@@ -947,6 +950,9 @@ export const WorktreeCleanupRules = Schema.Struct({
947 Schema.withDecodingDefault(Effect.succeed("uncommitted-changes")),950 Schema.withDecodingDefault(Effect.succeed("uncommitted-changes")),
948 ),951 ),
949 worktreeAfterDays: StorageRetentionDays,952 worktreeAfterDays: StorageRetentionDays,
953 worktreeSettledAfterDays: SettledWorktreeRetentionDays.pipe(
954 Schema.withDecodingDefault(Effect.succeed(null)),
955 ),
950 worktreeOnMerge: Schema.Boolean,956 worktreeOnMerge: Schema.Boolean,
951 worktreeOnDelete: Schema.Boolean,957 worktreeOnDelete: Schema.Boolean,
952 worktreeUnchanged: Schema.Boolean,958 worktreeUnchanged: Schema.Boolean,
...@@ -1089,6 +1095,9 @@ export const StorageCleanupSettings = Schema.Struct({...@@ -1089,6 +1095,9 @@ export const StorageCleanupSettings = Schema.Struct({
1089 Schema.withDecodingDefault(Effect.succeed("uncommitted-changes")),1095 Schema.withDecodingDefault(Effect.succeed("uncommitted-changes")),
1090 ),1096 ),
1091 worktreeAfterDays: StorageRetentionDays.pipe(Schema.withDecodingDefault(Effect.succeed(null))),1097 worktreeAfterDays: StorageRetentionDays.pipe(Schema.withDecodingDefault(Effect.succeed(null))),
1098 worktreeSettledAfterDays: SettledWorktreeRetentionDays.pipe(
1099 Schema.withDecodingDefault(Effect.succeed(null)),
1100 ),
1092 worktreeOnMerge: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),1101 worktreeOnMerge: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
1093 worktreeOnDelete: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),1102 worktreeOnDelete: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
1094 worktreeUnchanged: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),1103 worktreeUnchanged: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
...@@ -1519,6 +1528,7 @@ export const ServerSettingsPatch = Schema.Struct({...@@ -1519,6 +1528,7 @@ export const ServerSettingsPatch = Schema.Struct({
1519 rules: Schema.Struct({1528 rules: Schema.Struct({
1520 worktreeKeepWhen: Schema.optionalKey(WorktreeKeepWhen),1529 worktreeKeepWhen: Schema.optionalKey(WorktreeKeepWhen),
1521 worktreeAfterDays: Schema.optionalKey(StorageRetentionDays),1530 worktreeAfterDays: Schema.optionalKey(StorageRetentionDays),
1531 worktreeSettledAfterDays: Schema.optionalKey(SettledWorktreeRetentionDays),
1522 worktreeOnMerge: Schema.optionalKey(Schema.Boolean),1532 worktreeOnMerge: Schema.optionalKey(Schema.Boolean),
1523 worktreeOnDelete: Schema.optionalKey(Schema.Boolean),1533 worktreeOnDelete: Schema.optionalKey(Schema.Boolean),
1524 worktreeUnchanged: Schema.optionalKey(Schema.Boolean),1534 worktreeUnchanged: Schema.optionalKey(Schema.Boolean),
...@@ -1531,6 +1541,7 @@ export const ServerSettingsPatch = Schema.Struct({...@@ -1531,6 +1541,7 @@ export const ServerSettingsPatch = Schema.Struct({
1531 Schema.Struct({1541 Schema.Struct({
1532 worktreeKeepWhen: Schema.optionalKey(WorktreeKeepWhen),1542 worktreeKeepWhen: Schema.optionalKey(WorktreeKeepWhen),
1533 worktreeAfterDays: Schema.optionalKey(StorageRetentionDays),1543 worktreeAfterDays: Schema.optionalKey(StorageRetentionDays),
1544 worktreeSettledAfterDays: Schema.optionalKey(SettledWorktreeRetentionDays),
1534 worktreeOnMerge: Schema.optionalKey(Schema.Boolean),1545 worktreeOnMerge: Schema.optionalKey(Schema.Boolean),
1535 worktreeOnDelete: Schema.optionalKey(Schema.Boolean),1546 worktreeOnDelete: Schema.optionalKey(Schema.Boolean),
1536 worktreeUnchanged: Schema.optionalKey(Schema.Boolean),1547 worktreeUnchanged: Schema.optionalKey(Schema.Boolean),
packages/shared/src/projectSettings.test.ts+21-4
...@@ -422,9 +422,19 @@ describe("projectSettingsOverrides patches", () => {...@@ -422,9 +422,19 @@ describe("projectSettingsOverrides patches", () => {
422describe("resolveWorktreeCleanup", () => {422describe("resolveWorktreeCleanup", () => {
423 it("inherits machine rules, disables one project and keeps custom rules isolated", () => {423 it("inherits machine rules, disables one project and keeps custom rules isolated", () => {
424 const machine = applyServerSettingsPatch(DEFAULT_SERVER_SETTINGS, {424 const machine = applyServerSettingsPatch(DEFAULT_SERVER_SETTINGS, {
425 storageCleanup: { worktreeAfterDays: 8, worktreeOnDelete: true, logsAfterDays: 3 },425 storageCleanup: {
426 worktreeAfterDays: 8,
427 worktreeSettledAfterDays: 2,
428 worktreeKeepWhen: "any-local-files",
429 worktreeOnDelete: true,
430 logsAfterDays: 3,
431 },
426 });432 });
427 const inherited = resolveWorktreeCleanup(machine, projectId);433 const inherited = resolveWorktreeCleanup(machine, projectId);
434 expect(inherited).toMatchObject({
435 worktreeSettledAfterDays: 2,
436 worktreeKeepWhen: "any-local-files",
437 });
428 const off = applyServerSettingsPatch(machine, {438 const off = applyServerSettingsPatch(machine, {
429 projectSettingsOverrides: {439 projectSettingsOverrides: {
430 [projectId]: { worktreeCleanup: { mode: "off" } },440 [projectId]: { worktreeCleanup: { mode: "off" } },
...@@ -432,20 +442,26 @@ describe("resolveWorktreeCleanup", () => {...@@ -432,20 +442,26 @@ describe("resolveWorktreeCleanup", () => {
432 });442 });
433 expect(resolveWorktreeCleanup(off, projectId)).toEqual({443 expect(resolveWorktreeCleanup(off, projectId)).toEqual({
434 worktreeAfterDays: null,444 worktreeAfterDays: null,
445 worktreeSettledAfterDays: null,
435 worktreeOnDelete: false,446 worktreeOnDelete: false,
436 worktreeOnMerge: false,447 worktreeOnMerge: false,
437 worktreeUnchanged: false,448 worktreeUnchanged: false,
438 worktreeKeepWhen: "uncommitted-changes",449 worktreeKeepWhen: "any-local-files",
439 });450 });
440 expect(resolveWorktreeCleanup(off, otherProjectId)).toEqual(inherited);451 expect(resolveWorktreeCleanup(off, otherProjectId)).toEqual(inherited);
441 const custom = applyServerSettingsPatch(off, {452 const custom = applyServerSettingsPatch(off, {
442 projectSettingsOverrides: {453 projectSettingsOverrides: {
443 [projectId]: {454 [projectId]: {
444 worktreeCleanup: { mode: "custom", rules: { ...inherited, worktreeAfterDays: 15 } },455 worktreeCleanup: {
456 mode: "custom",
457 rules: { ...inherited, worktreeAfterDays: 15, worktreeSettledAfterDays: 0 },
458 },
445 },459 },
446 },460 },
447 });461 });
448 expect(resolveWorktreeCleanup(custom, projectId).worktreeAfterDays).toBe(15);462 expect(resolveWorktreeCleanup(custom, projectId).worktreeAfterDays).toBe(15);
463 expect(resolveWorktreeCleanup(custom, projectId).worktreeSettledAfterDays).toBe(0);
464 expect(resolveWorktreeCleanup(custom, otherProjectId)).toEqual(inherited);
449 expect(custom.storageCleanup.logsAfterDays).toBe(3);465 expect(custom.storageCleanup.logsAfterDays).toBe(3);
450 const reset = applyServerSettingsPatch(custom, {466 const reset = applyServerSettingsPatch(custom, {
451 projectSettingsOverrides: {467 projectSettingsOverrides: {
...@@ -456,7 +472,7 @@ describe("resolveWorktreeCleanup", () => {...@@ -456,7 +472,7 @@ describe("resolveWorktreeCleanup", () => {
456 });472 });
457 it("completes partial machine custom rules and preserves them across edits", () => {473 it("completes partial machine custom rules and preserves them across edits", () => {
458 const initial = applyServerSettingsPatch(DEFAULT_SERVER_SETTINGS, {474 const initial = applyServerSettingsPatch(DEFAULT_SERVER_SETTINGS, {
459 storageCleanup: { worktreeAfterDays: 8, worktreeOnDelete: true },475 storageCleanup: { worktreeAfterDays: 8, worktreeSettledAfterDays: 0, worktreeOnDelete: true },
460 });476 });
461 const custom = applyServerSettingsPatch(initial, {477 const custom = applyServerSettingsPatch(initial, {
462 worktreeCleanup: { mode: "custom", rules: { worktreeOnMerge: true } },478 worktreeCleanup: { mode: "custom", rules: { worktreeOnMerge: true } },
...@@ -466,6 +482,7 @@ describe("resolveWorktreeCleanup", () => {...@@ -466,6 +482,7 @@ describe("resolveWorktreeCleanup", () => {
466 });482 });
467 expect(resolveWorktreeCleanup(edited, null)).toEqual({483 expect(resolveWorktreeCleanup(edited, null)).toEqual({
468 worktreeAfterDays: 15,484 worktreeAfterDays: 15,
485 worktreeSettledAfterDays: 0,
469 worktreeOnDelete: true,486 worktreeOnDelete: true,
470 worktreeOnMerge: true,487 worktreeOnMerge: true,
471 worktreeUnchanged: false,488 worktreeUnchanged: false,
packages/shared/src/projectSettings.ts+3
...@@ -279,6 +279,7 @@ export function resolveWorktreeCleanup(...@@ -279,6 +279,7 @@ export function resolveWorktreeCleanup(
279 return {279 return {
280 worktreeKeepWhen: settings.storageCleanup.worktreeKeepWhen,280 worktreeKeepWhen: settings.storageCleanup.worktreeKeepWhen,
281 worktreeAfterDays: null,281 worktreeAfterDays: null,
282 worktreeSettledAfterDays: null,
282 worktreeOnMerge: false,283 worktreeOnMerge: false,
283 worktreeOnDelete: false,284 worktreeOnDelete: false,
284 worktreeUnchanged: false,285 worktreeUnchanged: false,
...@@ -286,6 +287,7 @@ export function resolveWorktreeCleanup(...@@ -286,6 +287,7 @@ export function resolveWorktreeCleanup(
286 const {287 const {
287 worktreeKeepWhen,288 worktreeKeepWhen,
288 worktreeAfterDays,289 worktreeAfterDays,
290 worktreeSettledAfterDays,
289 worktreeOnMerge,291 worktreeOnMerge,
290 worktreeOnDelete,292 worktreeOnDelete,
291 worktreeUnchanged,293 worktreeUnchanged,
...@@ -293,6 +295,7 @@ export function resolveWorktreeCleanup(...@@ -293,6 +295,7 @@ export function resolveWorktreeCleanup(
293 return {295 return {
294 worktreeKeepWhen,296 worktreeKeepWhen,
295 worktreeAfterDays,297 worktreeAfterDays,
298 worktreeSettledAfterDays,
296 worktreeOnMerge,299 worktreeOnMerge,
297 worktreeOnDelete,300 worktreeOnDelete,
298 worktreeUnchanged,301 worktreeUnchanged,
packages/shared/src/serverSettings.test.ts+1
...@@ -63,6 +63,7 @@ describe("serverSettings helpers", () => {...@@ -63,6 +63,7 @@ describe("serverSettings helpers", () => {
63 }).storageCleanup,63 }).storageCleanup,
64 ).toEqual({64 ).toEqual({
65 worktreeAfterDays: null,65 worktreeAfterDays: null,
66 worktreeSettledAfterDays: null,
66 worktreeOnMerge: true,67 worktreeOnMerge: true,
67 worktreeOnDelete: false,68 worktreeOnDelete: false,
68 worktreeUnchanged: false,69 worktreeUnchanged: false,
packages/shared/src/serverSettings.ts+1
...@@ -523,6 +523,7 @@ export function applyServerSettingsPatch(...@@ -523,6 +523,7 @@ export function applyServerSettingsPatch(
523 rules: {523 rules: {
524 worktreeKeepWhen: next.storageCleanup.worktreeKeepWhen,524 worktreeKeepWhen: next.storageCleanup.worktreeKeepWhen,
525 worktreeAfterDays: next.storageCleanup.worktreeAfterDays,525 worktreeAfterDays: next.storageCleanup.worktreeAfterDays,
526 worktreeSettledAfterDays: next.storageCleanup.worktreeSettledAfterDays,
526 worktreeOnMerge: next.storageCleanup.worktreeOnMerge,527 worktreeOnMerge: next.storageCleanup.worktreeOnMerge,
527 worktreeOnDelete: next.storageCleanup.worktreeOnDelete,528 worktreeOnDelete: next.storageCleanup.worktreeOnDelete,
528 worktreeUnchanged: next.storageCleanup.worktreeUnchanged,529 worktreeUnchanged: next.storageCleanup.worktreeUnchanged,