Commit eeaac49a9d20d94970b26fe7775fc8ddb4672f4f | the-knife

apps/desktop/src/backend/DesktopBackendManager.test.ts+1-1

...	...	@@ -264,7 +264,7 @@ describe("DesktopBackendManager", () => {
264	264	assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output");
265	265	assert.equal(
266	266	Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)),
267		2_000,
	267	5_000,
268	268	);
269	269
270	270	assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability);

apps/desktop/src/backend/DesktopBackendManager.ts+3-1

...	...	@@ -65,7 +65,9 @@ const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
65	65	const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1);
66	66	const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100);
67	67	const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1);
68		const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(2);
	68	// Same as the service launcher. A killed backend leaves its agents running,
	69	// and a busy one needs a few seconds to stop every provider session.
	70	const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5);
69	71	const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5);
70	72	const BACKEND_READINESS_PATH = "/.well-known/t3/environment";
71	73	const { logWarning: logBackendProcessWarning } =

apps/desktop/src/updates/DesktopUpdates.test.ts+63

...	...	@@ -652,6 +652,67 @@ describe("DesktopUpdates", () => {
652	652	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
653	653	});
654	654
	655	it.effect("records the version an install hands to the installer", () => {
	656	let recordsAtStop: ReadonlyArray<string> = [];
	657	const harness = makeHarness({
	658	stopBackend: Effect.sync(() => {
	659	recordsAtStop = [...harness.installRecords.values()];
	660	}),
	661	});
	662
	663	return Effect.scoped(
	664	Effect.gen(function* () {
	665	const updates = yield* DesktopUpdates.DesktopUpdates;
	666	yield* updates.configure;
	667	harness.emit("update-downloaded", { version: "1.2.4" });
	668	yield* flushCallbacks;
	669
	670	assert.isTrue((yield* updates.install).accepted);
	671	assert.deepEqual(
	672	recordsAtStop.map((record) => JSON.parse(record)),
	673	[{ fromVersion: "1.2.3", targetVersion: "1.2.4" }],
	674	);
	675	}),
	676	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
	677	});
	678
	679	it.effect("flags an install that did not apply and keeps the flag through checks", () => {
	680	const harness = makeHarness({
	681	installRecord: JSON.stringify({ fromVersion: "1.2.3", targetVersion: "1.2.4" }),
	682	});
	683
	684	return Effect.scoped(
	685	Effect.gen(function* () {
	686	const updates = yield* DesktopUpdates.DesktopUpdates;
	687	yield* updates.configure;
	688	assert.equal((yield* updates.getState).notAppliedVersion, "1.2.4");
	689	// Kept, so the next start on 1.2.3 still knows 1.2.4 did not apply.
	690	assert.isDefined(harness.previousInstallRecord());
	691
	692	harness.emit("update-available", { version: "1.2.4" });
	693	yield* flushCallbacks;
	694	const state = yield* updates.getState;
	695	assert.equal(state.status, "available");
	696	assert.equal(state.notAppliedVersion, "1.2.4");
	697	}),
	698	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
	699	});
	700
	701	it.effect("does not flag an install that applied", () => {
	702	const harness = makeHarness({
	703	installRecord: JSON.stringify({ fromVersion: "1.2.2", targetVersion: "1.2.3" }),
	704	});
	705
	706	return Effect.scoped(
	707	Effect.gen(function* () {
	708	const updates = yield* DesktopUpdates.DesktopUpdates;
	709	yield* updates.configure;
	710	assert.isUndefined((yield* updates.getState).notAppliedVersion);
	711	assert.isUndefined(harness.previousInstallRecord());
	712	}),
	713	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
	714	});
	715
655	716	it.effect("drops the update restart marker when an install is interrupted", () =>
656	717	Effect.gen(function* () {
657	718	const stopping = yield* Deferred.make<void>();
...	...	@@ -703,6 +764,8 @@ describe("DesktopUpdates", () => {
703	764	assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]);
704	765	// The restarted old backend must release its tunnel on a later quit.
705	766	assert.equal(harness.updateRestartMarkers.size, 0);
	767	// The app never quit, so the next start must not report an unapplied install.
	768	assert.equal(harness.installRecords.size, 0);
706	769	}),
707	770	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
708	771	});

apps/desktop/src/updates/DesktopUpdates.ts+70-2

...	...	@@ -46,6 +46,7 @@ import {
46	46	reduceDesktopUpdateStateOnInstallFailure,
47	47	reduceDesktopUpdateStateOnNoUpdate,
48	48	reduceDesktopUpdateStateOnUpdateAvailable,
	49	resolveNotAppliedInstallVersion,
49	50	} from "./updateMachine.ts";
50	51
51	52	const AUTO_UPDATE_STARTUP_DELAY = "15 seconds";
...	...	@@ -72,6 +73,14 @@ const UpdateInfo = Schema.Struct({
72	73	const DownloadProgressInfo = Schema.Struct({
73	74	percent: Schema.Number,
74	75	});
	76	/** Saved in the runtime folder before an install quits the app, read on the next start. */
	77	export const DESKTOP_UPDATE_INSTALL_RECORD_FILE = "desktop-update-install.json";
	78	const DesktopUpdateInstallRecord = Schema.fromJsonString(
	79	Schema.Struct({ fromVersion: Schema.String, targetVersion: Schema.String }),
	80	);
	81	const decodeInstallRecord = Schema.decodeUnknownEffect(DesktopUpdateInstallRecord);
	82	const encodeInstallRecord = Schema.encodeEffect(DesktopUpdateInstallRecord);
	83
75	84	const decodeAppUpdateYmlConfig = Schema.decodeUnknownEffect(AppUpdateYmlConfig);
76	85	const decodeUpdateInfo = Schema.decodeUnknownEffect(UpdateInfo);
77	86	const decodeDownloadProgressInfo = Schema.decodeUnknownEffect(DownloadProgressInfo);
...	...	@@ -347,6 +356,7 @@ export const make = Effect.gen(function* () {
347	356	const finishedUpdateActions = yield* PubSub.unbounded<UpdateAction>();
348	357	const updaterConfiguredRef = yield* Ref.make(false);
349	358	const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1);
	359	const notAppliedVersionRef = yield* Ref.make(Option.none<string>());
350	360	const updateStateRef = yield* Ref.make<DesktopUpdateState>(
351	361	createInitialDesktopUpdateState(
352	362	environment.appVersion,
...	...	@@ -600,11 +610,57 @@ export const make = Effect.gen(function* () {
600	610	.remove(updateRestartMarkerPath, { force: true })
601	611	.pipe(Effect.ignore);
602	612
	613	// Records the version an install hands to the OS installer, so the next
	614	// start can tell whether it applied. Best effort, like the restart marker.
	615	const installRecordPath = environment.path.join(
	616	updateRestartMarkerDir,
	617	DESKTOP_UPDATE_INSTALL_RECORD_FILE,
	618	);
	619	const writeInstallRecord = (targetVersion: string) =>
	620	encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion }).pipe(
	621	Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)),
	622	Effect.catch((error) =>
	623	logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }),
	624	),
	625	);
	626	const removeInstallRecord = fileSystem
	627	.remove(installRecordPath, { force: true })
	628	.pipe(Effect.ignore);
	629
	630	// Reads the record of the last install. Returns the version it did not
	631	// apply, when the app came back on the version it installed from. That
	632	// record stays through later starts, until the version changes or a new
	633	// install replaces it, so no restart forgets the failed version.
	634	const readNotAppliedInstallVersion = fileSystem.readFileString(installRecordPath).pipe(
	635	Effect.flatMap(decodeInstallRecord),
	636	Effect.option,
	637	Effect.map((record) =>
	638	Option.flatMap(record, (value) =>
	639	Option.fromNullishOr(resolveNotAppliedInstallVersion(value, environment.appVersion)),
	640	),
	641	),
	642	Effect.tap((notAppliedVersion) =>
	643	Option.isNone(notAppliedVersion) ? removeInstallRecord : Effect.void,
	644	),
	645	);
	646
	647	const baseUpdateState = (channel: DesktopUpdateChannel, enabled: boolean) =>
	648	Ref.get(notAppliedVersionRef).pipe(
	649	Effect.map((notAppliedVersion) => ({
	650	...createBaseUpdateState(channel, enabled, environment),
	651	...Option.match(notAppliedVersion, {
	652	onNone: () => ({}),
	653	onSome: (version) => ({ notAppliedVersion: version }),
	654	}),
	655	})),
	656	);
	657
603	658	const resetInstallAction = Effect.all(
604	659	[
605	660	finishUpdateAction("install"),
606	661	Ref.set(desktopState.quitting, false),
607	662	removeUpdateRestartMarker,
	663	removeInstallRecord,
608	664	],
609	665	{ discard: true },
610	666	);
...	...	@@ -621,6 +677,7 @@ export const make = Effect.gen(function* () {
621	677
622	678	yield* Ref.set(desktopState.quitting, false);
623	679	yield* removeUpdateRestartMarker;
	680	yield* removeInstallRecord;
624	681	yield* Effect.gen(function* () {
625	682	const instances = yield* pool.list;
626	683	const restartExit = yield* Effect.forEach(instances, (instance) => instance.start, {
...	...	@@ -691,6 +748,8 @@ export const make = Effect.gen(function* () {
691	748
692	749	return yield* Effect.gen(function* () {
693	750	yield* writeUpdateRestartMarker;
	751	const { downloadedVersion } = yield* Ref.get(updateStateRef);
	752	if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion);
694	753	// Stop every backend in the pool, not just the primary. With
695	754	// parallel WSL + Windows backends, leaving the WSL instance up
696	755	// means quitAndInstall's app.quit() exits before the pool's
...	...	@@ -985,6 +1044,15 @@ export const make = Effect.gen(function* () {
985	1044	const appUpdateYmlConfig = yield* readAppUpdateYml;
986	1045	yield* Ref.set(appUpdateYmlConfigRef, appUpdateYmlConfig);
987	1046
	1047	const notAppliedVersion = yield* readNotAppliedInstallVersion;
	1048	yield* Ref.set(notAppliedVersionRef, notAppliedVersion);
	1049	if (Option.isSome(notAppliedVersion)) {
	1050	yield* logUpdaterWarning("the last update install did not apply", {
	1051	version: environment.appVersion,
	1052	targetVersion: notAppliedVersion.value,
	1053	});
	1054	}
	1055
988	1056	if (config.mockUpdates) {
989	1057	yield* electronUpdater.setFeedURL({
990	1058	provider: "generic",
...	...	@@ -994,7 +1062,7 @@ export const make = Effect.gen(function* () {
994	1062
995	1063	const settings = yield* desktopSettings.get;
996	1064	const enabled = yield* shouldEnableAutoUpdates;
997		yield* setState(createBaseUpdateState(settings.updateChannel, enabled, environment));
	1065	yield* setState(yield* baseUpdateState(settings.updateChannel, enabled));
998	1066	if (!enabled) {
999	1067	return;
1000	1068	}
...	...	@@ -1065,7 +1133,7 @@ export const make = Effect.gen(function* () {
1065	1133	);
1066	1134
1067	1135	const enabled = yield* shouldEnableAutoUpdates;
1068		yield* setState(createBaseUpdateState(nextChannel, enabled, environment));
	1136	yield* setState(yield* baseUpdateState(nextChannel, enabled));
1069	1137
1070	1138	if (!enabled \|\| !(yield* Ref.get(updaterConfiguredRef))) {
1071	1139	return yield* Ref.get(updateStateRef);

apps/desktop/src/updates/remoteUpdateFlow.test.ts+18

...	...	@@ -34,6 +34,24 @@ function makeState(overrides: Partial<DesktopUpdateState> = {}): DesktopUpdateSt
34	34	}
35	35
36	36	describe("nextRemoteDesktopUpdateStep", () => {
	37	it("does not install again a version that did not apply on this machine", () => {
	38	const step = nextRemoteDesktopUpdateStep(
	39	makeState({ status: "available", availableVersion: "1.2.4", notAppliedVersion: "1.2.4" }),
	40	NO_ATTEMPTS,
	41	null,
	42	);
	43	assert.equal(step.action, "done");
	44	assert.include(step.action === "done" ? step.reason : "", "1.2.4 did not apply");
	45	assert.deepEqual(
	46	nextRemoteDesktopUpdateStep(
	47	makeState({ status: "available", availableVersion: "1.2.5", notAppliedVersion: "1.2.4" }),
	48	NO_ATTEMPTS,
	49	null,
	50	),
	51	{ action: "download" },
	52	);
	53	});
	54
37	55	it("fails immediately when updates are disabled, preferring the known reason", () => {
38	56	const state = makeState({ enabled: false, status: "disabled" });
39	57	assert.deepEqual(nextRemoteDesktopUpdateStep(state, NO_ATTEMPTS, "dev build"), {

apps/desktop/src/updates/remoteUpdateFlow.ts+13

...	...	@@ -52,6 +52,19 @@ export function nextRemoteDesktopUpdateStep(
52	52	reason: disabledReason ?? "Automatic updates are disabled on this machine.",
53	53	};
54	54	}
	55	// An install of this version already failed to apply here. Installing it
	56	// again from afar would stop the app's threads for nothing, so only the user
	57	// on this machine retries it.
	58	if (
	59	state.notAppliedVersion !== undefined &&
	60	(state.downloadedVersion ?? state.availableVersion) === state.notAppliedVersion
	61	) {
	62	return {
	63	action: "done",
	64	outcome: "failed",
	65	reason: `Update ${state.notAppliedVersion} did not apply on this machine. Retry it from the desktop app there.`,
	66	};
	67	}
55	68	// Mirror installDownloadedUpdate's own admission rule exactly, so the run
56	69	// never reports an install that the updater then refuses. A download
57	70	// survives an unrelated background updater error (errorContext null) and

apps/desktop/src/updates/updateMachine.test.ts+14

...	...	@@ -11,6 +11,7 @@ import {
11	11	reduceDesktopUpdateStateOnInstallFailure,
12	12	reduceDesktopUpdateStateOnNoUpdate,
13	13	reduceDesktopUpdateStateOnUpdateAvailable,
	14	resolveNotAppliedInstallVersion,
14	15	} from "./updateMachine.ts";
15	16
16	17	const runtimeInfo = {
...	...	@@ -255,3 +256,16 @@ describe("updateMachine", () => {
255	256	expect(state.omittedReleaseCount).toBe(0);
256	257	});
257	258	});
	259
	260	describe("resolveNotAppliedInstallVersion", () => {
	261	const record = { fromVersion: "1.2.3", targetVersion: "1.2.4" };
	262
	263	it("flags the target when the app came back on the version it installed from", () => {
	264	expect(resolveNotAppliedInstallVersion(record, "1.2.3")).toBe("1.2.4");
	265	});
	266
	267	it("stays quiet when the install applied or the app runs another version", () => {
	268	expect(resolveNotAppliedInstallVersion(record, "1.2.4")).toBeNull();
	269	expect(resolveNotAppliedInstallVersion(record, "1.2.5")).toBeNull();
	270	});
	271	});

apps/desktop/src/updates/updateMachine.ts+20

...	...	@@ -211,3 +211,23 @@ export function reduceDesktopUpdateStateOnInstallFailure(
211	211	canRetry: true,
212	212	};
213	213	}
	214
	215	/** The install this app handed to the OS installer, saved right before it quits. */
	216	export interface DesktopUpdateInstallRecord {
	217	readonly fromVersion: string;
	218	readonly targetVersion: string;
	219	}
	220
	221	/**
	222	* Read on the next start: the version an install did not apply, because the
	223	* app came back on the version it installed from. Null when the install
	224	* applied, or when the app now runs some other version.
	225	*/
	226	export function resolveNotAppliedInstallVersion(
	227	record: DesktopUpdateInstallRecord,
	228	appVersion: string,
	229	): string \| null {
	230	return record.fromVersion === appVersion && record.targetVersion !== appVersion
	231	? record.targetVersion
	232	: null;
	233	}

apps/desktop/src/updates/updatesTestHarness.ts+25-7

...	...	@@ -37,6 +37,8 @@ export interface UpdatesHarnessOptions {
37	37	readonly platform?: NodeJS.Platform;
38	38	/** Contents of the resources/package-type marker a Linux package ships. */
39	39	readonly packageType?: string \| undefined;
	40	/** Contents of the install record a previous run left behind. */
	41	readonly installRecord?: string \| undefined;
40	42	}
41	43
42	44	export function makeHarness(options: UpdatesHarnessOptions = {}) {
...	...	@@ -208,12 +210,21 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
208	210	} satisfies DesktopAppSettings.DesktopAppSettings["Service"])
209	211	: DesktopAppSettings.layer;
210	212
211		// Tracks the restart markers installs leave, so installs stay free of real
212		// disk I/O that would outrun the tests' settle loops.
	213	// Tracks the restart markers and install records installs leave, so
	214	// installs stay free of real disk I/O that would outrun the tests' settle loops.
213	215	const updateRestartMarkers = new Set<string>();
	216	// Contents of the install record, by path once an install writes one.
	217	const installRecords = new Map<string, string>();
	218	let previousInstallRecord = options.installRecord;
	219	const isInstallRecord = (path: string) =>
	220	path.endsWith(DesktopUpdates.DESKTOP_UPDATE_INSTALL_RECORD_FILE);
214	221	const layerFileSystem = FileSystem.layerNoop({
215		readFileString: (path) =>
216		path === "/missing/resources/package-type" && options.packageType !== undefined
	222	readFileString: (path) => {
	223	const installRecord = isInstallRecord(path)
	224	? (installRecords.get(path) ?? previousInstallRecord)
	225	: undefined;
	226	if (installRecord !== undefined) return Effect.succeed(installRecord);
	227	return path === "/missing/resources/package-type" && options.packageType !== undefined
217	228	? Effect.succeed(options.packageType)
218	229	: Effect.fail(
219	230	PlatformError.systemError({
...	...	@@ -222,15 +233,19 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
222	233	_tag: "NotFound",
223	234	pathOrDescriptor: path,
224	235	}),
225		),
	236	);
	237	},
226	238	makeDirectory: () => Effect.void,
227		writeFileString: (path) =>
	239	writeFileString: (path, data) =>
228	240	Effect.sync(() => {
229		updateRestartMarkers.add(path);
	241	if (isInstallRecord(path)) installRecords.set(path, data);
	242	else updateRestartMarkers.add(path);
230	243	}),
231	244	remove: (path) =>
232	245	Effect.sync(() => {
233	246	updateRestartMarkers.delete(path);
	247	installRecords.delete(path);
	248	if (isInstallRecord(path)) previousInstallRecord = undefined;
234	249	}),
235	250	});
236	251
...	...	@@ -259,6 +274,9 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
259	274	quitAndInstalls: () => quitAndInstallCount,
260	275	installSteps,
261	276	updateRestartMarkers,
	277	installRecords,
	278	/** The record the test started with, until the app removes it. */
	279	previousInstallRecord: () => previousInstallRecord,
262	280	downloadCount: () => downloadCount,
263	281	feedUrls: (): ElectronUpdater.ElectronUpdaterFeedUrl[] => feedUrls,
264	282	fullChangelog: () => fullChangelog,

apps/server/src/mcp/OrchestratorMcpService.ts+4-3

...	...	@@ -81,6 +81,7 @@ import {
81	81	DispatchModeLimit,
82	82	type DispatchModeRefusal,
83	83	} from "../orchestration-v2/DispatchModeLimit.ts";
	84	import { runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts";
84	85	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
85	86	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
86	87	import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
...	...	@@ -330,9 +331,9 @@ function invalidOptionSelections(
330	331	}
331	332
332	333	function taskStatusForRun(
333		run: Pick<OrchestrationV2Run, "status"> \| undefined,
	334	run: Pick<OrchestrationV2Run, "status" \| "restartCancelledBackgroundWork"> \| undefined,
334	335	): OrchestratorMcpDelegateTaskResult["status"] {
335		switch (run?.status) {
	336	switch (run === undefined ? undefined : runOutcome(run)) {
336	337	case "queued":
337	338	return "queued";
338	339	case "waiting":
...	...	@@ -2642,7 +2643,7 @@ const make = Effect.gen(function* () {
2642	2643	return {
2643	2644	threadId: input.threadId,
2644	2645	runId: result.run?.id ?? null,
2645		status: result.run?.status ?? "idle",
	2646	status: result.run === null ? "idle" : runOutcome(result.run),
2646	2647	timedOut: result.timedOut,
2647	2648	} satisfies OrchestratorMcpThreadWaitResult;
2648	2649	}),

apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts+65-7

...	...	@@ -803,6 +803,8 @@ const seedRestartCancelledChild = (input: {
803	803	readonly continuationPending: boolean;
804	804	/** "completed" seeds a settled turn whose background work the restart cancelled. */
805	805	readonly runStatus?: "cancelled" \| "completed";
	806	/** Background work the restart cancelled on the child's run. */
	807	readonly lostWork?: OrchestrationV2Run["restartCancelledBackgroundWork"];
806	808	readonly now: DateTime.Utc;
807	809	}) =>
808	810	Effect.gen(function* () {
...	...	@@ -880,16 +882,22 @@ const seedRestartCancelledChild = (input: {
880	882	},
881	883	],
882	884	});
	885	const childRun = runEvent({
	886	threadId: childThreadId,
	887	runId: childRunId,
	888	ordinal: 1,
	889	status: input.runStatus ?? "cancelled",
	890	now: input.now,
	891	});
883	892	yield* eventSink.writeWithEffects({
884	893	commandId: reconcileCommandId(input.name),
885	894	events: [
886		runEvent({
887		threadId: childThreadId,
888		runId: childRunId,
889		ordinal: 1,
890		status: input.runStatus ?? "cancelled",
891		now: input.now,
892		}),
	895	input.lostWork === undefined
	896	? childRun
	897	: {
	898	...childRun,
	899	payload: { ...childRun.payload, restartCancelledBackgroundWork: input.lostWork },
	900	},
893	901	],
894	902	effects: input.continuationPending
895	903	? [
...	...	@@ -1427,6 +1435,56 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
1427	1435	}),
1428	1436	);
1429	1437
	1438	it.effect("reports a child whose monitor the restart cut as interrupted, not completed", () =>
	1439	Effect.gen(function* () {
	1440	const orchestrator = yield* Orchestrator.OrchestratorV2;
	1441	const now = yield* DateTime.now;
	1442	const threadId = ThreadId.make("thread:restart-monitor-parent");
	1443	const projectId = ProjectId.make("project:restart-monitor-parent");
	1444	const runId = RunId.make("run:restart-monitor-parent");
	1445	const rootNodeId = NodeId.make("node:restart-monitor-parent-root");
	1446	yield* seedParentWithTerminalTask({
	1447	threadId,
	1448	projectId,
	1449	runId,
	1450	rootNodeId,
	1451	taskId: NodeId.make("node:restart-monitor-parent-settled"),
	1452	deliveryState: "delivered",
	1453	now,
	1454	});
	1455	// The child ended its turn to wait on a gate monitor; the restart cut it.
	1456	const child = yield* seedRestartCancelledChild({
	1457	parentThreadId: threadId,
	1458	projectId,
	1459	parentRunId: runId,
	1460	rootNodeId,
	1461	name: "restart-monitor-child",
	1462	completionWake: "always",
	1463	continuationPending: true,
	1464	runStatus: "completed",
	1465	lostWork: [{ kind: "monitor", label: "Gate attempt 9" }],
	1466	now,
	1467	});
	1468	yield* orchestrator.recoverDelegatedTasks;
	1469	const held = yield* orchestrator.getThreadProjection(threadId);
	1470	assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running");
	1471
	1472	// Continuation is off, so the cut run is the outcome. It still goes to the parent.
	1473	yield* continueRestartedRun({
	1474	threadId: child.childThreadId,
	1475	sourceRunId: child.childRunId,
	1476	}).pipe(
	1477	Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })),
	1478	);
	1479
	1480	const settled = yield* orchestrator.getThreadProjection(threadId);
	1481	const task = settled.subagents.find((row) => row.id === child.taskId);
	1482	assert.equal(task?.status, "interrupted");
	1483	assert.equal(task?.result, "Child task ended with status interrupted.");
	1484	assert.equal(task?.completionDelivery?.state, "claimed");
	1485	}),
	1486	);
	1487
1430	1488	// A running provider turn does not prove the provider consumed the delivery
1431	1489	// (Claude marks the turn running before it reads the prompt), so a cut
1432	1490	// delivery is offered again even when a continuation resumes that turn.

apps/server/src/orchestration-v2/EffectOutbox.ts+29

...	...	@@ -224,6 +224,8 @@ export interface EffectOutboxV2Shape {
224	224	readonly signalCancellations: (effectIds: ReadonlyArray<string>) => Effect.Effect<void>;
225	225	readonly awaitCancellation: (effectId: string) => Effect.Effect<void>;
226	226	readonly clearCancellation: (effectId: string) => Effect.Effect<void>;
	227	/** Resolves once the effect is neither pending nor running, or right away when it does not exist. */
	228	readonly awaitSettled: (effectId: string) => Effect.Effect<void, EffectOutboxError>;
227	229	readonly reconcileAfterProcessLoss: Effect.Effect<
228	230	{ readonly requeued: number; readonly cancelled: number },
229	231	EffectOutboxError
...	...	@@ -307,6 +309,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
307	309	// for distinct threads without allowing notifications to grow unbounded.
308	310	const available = yield* Queue.dropping<void>(64);
309	311	const cancellationSignals = new Map<string, Deferred.Deferred<void>>();
	312	const settledSignals = new Map<string, Deferred.Deferred<void>>();
	313	const signalSettled = (effectIds: ReadonlyArray<string>) =>
	314	Effect.forEach(
	315	effectIds,
	316	(effectId) => {
	317	const signal = settledSignals.get(effectId);
	318	if (signal === undefined) return Effect.void;
	319	settledSignals.delete(effectId);
	320	return Deferred.succeed(signal, undefined);
	321	},
	322	{ discard: true },
	323	);
310	324	const notifyAvailable = (count = 1) =>
311	325	Queue.offerAll(
312	326	available,
...	...	@@ -477,6 +491,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
477	491	},
478	492	{ discard: true },
479	493	);
	494	yield* signalSettled(effectIds);
480	495	// Cancellation can unblock other pending work on the same thread.
481	496	// This method is deliberately post-commit, so it is also the safe
482	497	// place to wake claimers after the durable status change.
...	...	@@ -487,6 +502,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
487	502	Effect.sync(() => {
488	503	cancellationSignals.delete(effectId);
489	504	}),
	505	awaitSettled: (effectId) =>
	506	Effect.gen(function* () {
	507	// Registered before the read, so a settle right after it still resolves.
	508	const signal = settledSignals.get(effectId) ?? Deferred.makeUnsafe<void>();
	509	settledSignals.set(effectId, signal);
	510	const effect = yield* service.get(effectId);
	511	if (Option.exists(effect, ({ status }) => status === "pending" \|\| status === "running")) {
	512	return yield* Deferred.await(signal);
	513	}
	514	// Settled already: release every waiter that shares this signal.
	515	yield* signalSettled([effectId]);
	516	}),
490	517	reconcileAfterProcessLoss: Effect.gen(function* () {
491	518	const now = DateTime.formatIso(yield* DateTime.now);
492	519	const cancelledRows = yield* sql<{ readonly effect_id: string }>`
...	...	@@ -599,6 +626,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
599	626	`;
600	627	if (rows.length === 1) {
601	628	cancellationSignals.delete(effectId);
	629	yield* signalSettled([effectId]);
602	630	yield* notifyAvailable();
603	631	}
604	632	return rows.length === 1;
...	...	@@ -657,6 +685,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
657	685	`;
658	686	if (rows.length === 1) {
659	687	cancellationSignals.delete(effectId);
	688	yield* signalSettled([effectId]);
660	689	yield* notifyAvailable();
661	690	}
662	691	return rows.length === 1;

apps/server/src/orchestration-v2/FoundationPersistence.test.ts+31

...	...	@@ -2084,6 +2084,37 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2084	2084	}),
2085	2085	);
2086	2086
	2087	it.effect("wakes a settle waiter when its effect finishes", () =>
	2088	Effect.gen(function* () {
	2089	const outbox = yield* EffectOutbox.EffectOutboxV2;
	2090	const effectId = "effect:foundation-await-settled";
	2091	yield* outbox.enqueue([
	2092	{
	2093	id: effectId,
	2094	commandId: CommandId.make("command:foundation-await-settled"),
	2095	threadId: ThreadId.make("thread:foundation-await-settled"),
	2096	request: {
	2097	type: "provider-runtime.continue",
	2098	sourceRunId: RunId.make("run:foundation-await-settled"),
	2099	},
	2100	},
	2101	]);
	2102	const waiter = yield* outbox.awaitSettled(effectId).pipe(Effect.forkChild);
	2103	const claimed = yield* outbox.claimNext({
	2104	workerId: "settle-worker",
	2105	leaseDurationMs: 30_000,
	2106	});
	2107	assert.isTrue(Option.isSome(claimed));
	2108	yield* Effect.yieldNow;
	2109	assert.isUndefined(waiter.pollUnsafe());
	2110
	2111	yield* outbox.succeed({ effectId, workerId: "settle-worker" });
	2112	yield* Fiber.join(waiter);
	2113	// A settled effect does not wait.
	2114	yield* outbox.awaitSettled(effectId);
	2115	}),
	2116	);
	2117
2087	2118	it.effect("runs title generation beside critical work while serializing each lane", () =>
2088	2119	Effect.gen(function* () {
2089	2120	const outbox = yield* EffectOutbox.EffectOutboxV2;

apps/server/src/orchestration-v2/Orchestrator.ts+21-5

...	...	@@ -88,7 +88,7 @@ import { CommandPolicyV2, resolveMessageDispatchIntent } from "./CommandPolicy.t
88	88	import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
89	89	import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
90	90	import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
91		import { isRestartNoteSource } from "./RestartBackgroundNote.ts";
	91	import { isRestartContinuationSource, runOutcome } from "./RestartBackgroundNote.ts";
92	92	import {
93	93	isBufferedProviderNotification,
94	94	isUndeliveredMailboxSteer,
...	...	@@ -293,6 +293,16 @@ export interface OrchestratorV2Shape {
293	293	childThreadId: ThreadId,
294	294	taskId?: NodeId,
295	295	) => Effect.Effect<boolean, OrchestratorProjectionError>;
	296	/**
	297	* Whether a restart continuation of this cut run has not started yet, so the
	298	* run's status is not its outcome. A failed read counts as not pending.
	299	*/
	300	readonly restartContinuationPending: (run: OrchestrationV2Run) => Effect.Effect<boolean>;
	301	/**
	302	* Resolves once that continuation has run or was declined. A declined one
	303	* records no thread event, so a waiter needs this to wake.
	304	*/
	305	readonly awaitRestartContinuation: (run: OrchestrationV2Run) => Effect.Effect<void>;
296	306	readonly dispatch: (
297	307	command: OrchestrationV2ServerCommand,
298	308	) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
...	...	@@ -4624,8 +4634,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
4624	4634	const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
4625	4635	if (
4626	4636	!source \|\|
4627		source.status !== "cancelled" \|\|
4628		isRestartNoteSource(source, projection.providerTurns) \|\|
	4637	!isRestartContinuationSource(source, projection.providerTurns) \|\|
4629	4638	projection.thread.archivedAt !== null \|\|
4630	4639	projection.thread.deletedAt !== null \|\|
4631	4640	projection.thread.providerInstanceId !== source.providerInstanceId \|\|
...	...	@@ -10147,7 +10156,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10147	10156	: undefined
10148	10157	: taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0];
10149	10158	if (childRun === undefined) return;
10150		const terminalStatus = delegatedTaskTerminalStatus(childRun.status);
	10159	const outcome = runOutcome(childRun);
	10160	const terminalStatus = delegatedTaskTerminalStatus(outcome);
10151	10161	if (terminalStatus === null) {
10152	10162	return;
10153	10163	}
...	...	@@ -10213,7 +10223,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10213	10223	}
10214	10224
10215	10225	const now = yield* DateTime.now;
10216		const result = subagentResultForRun(childProjection, childRun);
	10226	const result = subagentResultForRun(childProjection, { ...childRun, status: outcome });
10217	10227	const parentRun =
10218	10228	task.runId === null
10219	10229	? undefined
...	...	@@ -11459,6 +11469,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11459	11469	recoverDelegatedTasks,
11460	11470	recoverDelegatedTask,
11461	11471	delegatedTaskResultPending,
	11472	restartContinuationPending: (run) =>
	11473	awaitsRestartContinuation(run).pipe(Effect.orElseSucceed(() => false)),
	11474	awaitRestartContinuation: (run) =>
	11475	effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
11462	11476	dispatch: dispatchWithReceipt,
11463	11477	searchThreadStream: (input) =>
11464	11478	projectionStore
...	...	@@ -11607,6 +11621,8 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
11607	11621	recoverDelegatedTasks: Effect.void,
11608	11622	recoverDelegatedTask: () => Effect.void,
11609	11623	delegatedTaskResultPending: () => Effect.succeed(false),
	11624	restartContinuationPending: () => Effect.succeed(false),
	11625	awaitRestartContinuation: () => Effect.void,
11610	11626	dispatch: (command) =>
11611	11627	Effect.fail(
11612	11628	new OrchestratorDispatchError({

apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+52

...	...	@@ -209,6 +209,58 @@ it.effect.each(["startup", "shutdown"] as const)(
209	209	}).pipe(Effect.provide(layerRecoveryTest)),
210	210	);
211	211
	212	it.effect("stops a session its thread detached, which kept its live status", () =>
	213	Effect.gen(function* () {
	214	const projections = yield* ProjectionStore.ProjectionStoreV2;
	215	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
	216	const now = yield* DateTime.now;
	217	const threadId = yield* createThread("detached-session");
	218	const providerSessionId = ProviderSessionId.make("session:recovery:detached");
	219	yield* projections.apply({
	220	id: EventId.make("event:recovery:detached-session:attached"),
	221	type: "provider-session.attached",
	222	threadId,
	223	driver,
	224	providerInstanceId,
	225	occurredAt: now,
	226	payload: {
	227	id: providerSessionId,
	228	driver,
	229	providerInstanceId,
	230	status: "ready",
	231	cwd: "/workspace",
	232	model: modelSelection.model,
	233	capabilities: CodexProviderCapabilitiesV2,
	234	createdAt: now,
	235	updatedAt: now,
	236	lastError: null,
	237	},
	238	});
	239	// The server died before the release that should follow the detach.
	240	yield* projections.apply({
	241	id: EventId.make("event:recovery:detached-session:detached"),
	242	type: "provider-session.detached",
	243	threadId,
	244	driver,
	245	providerInstanceId,
	246	occurredAt: now,
	247	payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
	248	});
	249	assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
	250
	251	const summary = yield* recovery.reconcile("startup");
	252
	253	assert.equal(summary.stoppedSessions, 1);
	254	const sql = yield* SqlClient.SqlClient;
	255	const rows = yield* sql<{ readonly status: string }>`
	256	SELECT status FROM orchestration_v2_projection_provider_sessions
	257	WHERE provider_session_id = ${providerSessionId}
	258	`;
	259	assert.deepEqual(rows, [{ status: "stopped" }]);
	260	assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
	261	}).pipe(Effect.provide(layerRecoveryTest)),
	262	);
	263
212	264	it.effect("selects unfinished recovery work without reading settled thread histories", () =>
213	265	Effect.gen(function* () {
214	266	const projections = yield* ProjectionStore.ProjectionStoreV2;

apps/server/src/orchestration-v2/ProjectionStore.ts+38-14

...	...	@@ -3666,6 +3666,16 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3666	3666	ON bindings.provider_session_id = sessions.provider_session_id
3667	3667	WHERE sessions.status NOT IN ('stopped', 'error')
3668	3668	UNION
	3669	-- A detach drops the binding but keeps the session's status.
	3670	SELECT sessions.thread_id
	3671	FROM orchestration_v2_projection_provider_sessions AS sessions
	3672	WHERE sessions.status NOT IN ('stopped', 'error')
	3673	AND sessions.thread_id IS NOT NULL
	3674	AND NOT EXISTS (
	3675	SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings
	3676	WHERE bindings.provider_session_id = sessions.provider_session_id
	3677	)
	3678	UNION
3669	3679	SELECT thread_id FROM pending_provider_threads
3670	3680	UNION
3671	3681	SELECT nodes.thread_id FROM pending_provider_threads
...	...	@@ -4123,24 +4133,38 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4123	4133	ORDER BY COALESCE(subagent.started_at, ''), subagent.subagent_id ASC
4124	4134	`,
4125	4135	sql<PayloadRow>`
4126		SELECT DISTINCT session.payload_json
	4136	SELECT session.payload_json
4127	4137	FROM orchestration_v2_projection_provider_sessions AS session
4128		JOIN orchestration_v2_projection_provider_session_bindings AS binding
4129		ON binding.provider_session_id = session.provider_session_id
4130		WHERE binding.thread_id = ${threadId}
4131		AND (
4132		session.status NOT IN ('stopped', 'error')
4133		OR session.provider_session_id IN (
4134		SELECT provider_thread.provider_session_id
4135		FROM orchestration_v2_projection_provider_threads AS provider_thread
4136		WHERE provider_thread.provider_thread_id IN (
4137		SELECT run.provider_thread_id
4138		FROM orchestration_v2_projection_runs AS run
4139		WHERE run.thread_id = ${threadId}
4140		AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
	4138	WHERE (
	4139	session.provider_session_id IN (
	4140	SELECT binding.provider_session_id
	4141	FROM orchestration_v2_projection_provider_session_bindings AS binding
	4142	WHERE binding.thread_id = ${threadId}
	4143	)
	4144	AND (
	4145	session.status NOT IN ('stopped', 'error')
	4146	OR session.provider_session_id IN (
	4147	SELECT provider_thread.provider_session_id
	4148	FROM orchestration_v2_projection_provider_threads AS provider_thread
	4149	WHERE provider_thread.provider_thread_id IN (
	4150	SELECT run.provider_thread_id
	4151	FROM orchestration_v2_projection_runs AS run
	4152	WHERE run.thread_id = ${threadId}
	4153	AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
	4154	)
4141	4155	)
4142	4156	)
4143	4157	)
	4158	-- This thread detached it, which keeps its status. Its process
	4159	-- did not outlive the server either, so recovery stops it.
	4160	OR (
	4161	session.thread_id = ${threadId}
	4162	AND session.status NOT IN ('stopped', 'error')
	4163	AND NOT EXISTS (
	4164	SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS binding
	4165	WHERE binding.provider_session_id = session.provider_session_id
	4166	)
	4167	)
4144	4168	ORDER BY session.updated_at ASC, session.provider_session_id ASC
4145	4169	`,
4146	4170	sql<PayloadRow>`

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+26

...	...	@@ -7,6 +7,7 @@ import {
7	7	type OrchestrationV2RestartCancelledBackgroundWork,
8	8	type OrchestrationV2Subagent,
9	9	type OrchestrationV2ThreadProjection,
	10	isOrchestrationV2WorkActive,
10	11	ThreadId,
11	12	} from "@t3tools/contracts";
12	13	import * as Cause from "effect/Cause";
...	...	@@ -445,6 +446,31 @@ export const make = Effect.gen(function* () {
445	446	});
446	447	}
447	448	}
	449	// A blocking delegate_task wait dies with the turn that made it, and its
	450	// continuation never got the tool result. Let the child's result wake
	451	// the parent like an async task, so it reaches the continued turn.
	452	const cutRunIds = new Set(runs.map((run) => run.id));
	453	for (const task of projection.subagents ?? []) {
	454	if (
	455	task.origin !== "app_owned" \|\|
	456	task.runId === null \|\|
	457	!cutRunIds.has(task.runId) \|\|
	458	task.completionWake !== "settled_only" \|\|
	459	!isOrchestrationV2WorkActive(task.status)
	460	)
	461	continue;
	462	events.push({
	463	id: yield* allocateEventId(),
	464	type: "subagent.updated",
	465	threadId: projection.thread.id,
	466	runId: task.runId,
	467	nodeId: task.id,
	468	driver: task.driver,
	469	providerInstanceId: task.providerInstanceId,
	470	occurredAt: now,
	471	payload: { ...task, completionWake: "always", updatedAt: now },
	472	});
	473	}
448	474	// Process loss also orphans background-capable turn items on already-
449	475	// settled runs (e.g. post-settle Waiting work). Skip items already
450	476	// cancelled above for recovered nonterminal runs to avoid duplicate

apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+36

...	...	@@ -1681,6 +1681,42 @@ it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts
1681	1681	}),
1682	1682	);
1683	1683
	1684	it.effect("ProviderSessionManagerV2 releases sessions in parallel on server shutdown", () =>
	1685	Effect.gen(function* () {
	1686	const state = yield* Ref.make(emptyState);
	1687	yield* Effect.gen(function* () {
	1688	const eventSink = yield* EventSink.EventSinkV2;
	1689	const idAllocator = yield* IdAllocator.IdAllocatorV2;
	1690	const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
	1691	const now = yield* DateTime.now;
	1692	for (const name of ["a", "b", "c"]) {
	1693	const threadId = ThreadId.make(`thread-provider-session-manager-parallel-shutdown-${name}`);
	1694	yield* eventSink.write({
	1695	events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
	1696	});
	1697	yield* manager.open({
	1698	threadId,
	1699	providerSessionId: yield* idAllocator.allocate.providerSession({
	1700	providerInstanceId: modelSelection.instanceId,
	1701	threadId,
	1702	}),
	1703	modelSelection,
	1704	runtimePolicy,
	1705	});
	1706	}
	1707
	1708	// Every close hangs until its time box ends. One time box must cover all
	1709	// three sessions, or a busy server outlives the launcher's kill timer.
	1710	const stopping = yield* manager.shutdown.pipe(Effect.forkChild);
	1711	yield* TestClock.adjust("30 seconds");
	1712	yield* Effect.yieldNow;
	1713	assert.isDefined(stopping.pollUnsafe());
	1714	}).pipe(
	1715	Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, hangSessionScopeClose: true })),
	1716	);
	1717	}),
	1718	);
	1719
1684	1720	it.effect("ProviderSessionManagerV2 closes event subscriptions normally on server shutdown", () =>
1685	1721	Effect.gen(function* () {
1686	1722	const state = yield* Ref.make(emptyState);

apps/server/src/orchestration-v2/ProviderSessionManager.ts+3-3

...	...	@@ -1999,8 +1999,8 @@ export const layerWithOptions = (
1999	1999	// Parent of every session scope. On layer close, shutdown releases the
2000	2000	// live sessions first, then closes any session whose open is still in
2001	2001	// flight, time-boxed so a stuck adapter cannot hold up server shutdown.
2002		// Parallel, so one session whose close hangs does not stop the rest from
2003		// closing within the time box.
	2002	// Both steps run in parallel. The service launcher kills the server
	2003	// 5 s after it asks it to stop, and a busy server can hold 15 sessions.
2004	2004	const sessionScopes = yield* Scope.make("parallel");
2005	2005	const shutdown = Effect.gen(function* () {
2006	2006	const activeSessions = [...(yield* Ref.get(sessions)).values()];
...	...	@@ -2018,7 +2018,7 @@ export const layerWithOptions = (
2018	2018	}),
2019	2019	),
2020	2020	),
2021		{ discard: true },
	2021	{ concurrency: "unbounded", discard: true },
2022	2022	);
2023	2023	});
2024	2024	yield* Effect.addFinalizer(() =>

apps/server/src/orchestration-v2/RestartBackgroundNote.ts+38

...	...	@@ -123,6 +123,44 @@ export function isRestartNoteSource(
123	123	);
124	124	}
125	125
	126	/**
	127	* Work an agent ends its turn to wait for: a monitor, a subagent, or a
	128	* background task. A restart that cancels it wakes the settled turn, because
	129	* nothing else will. A shell does not wake it: most are dev servers that
	130	* nobody waits on, so their note waits for the next user message.
	131	*/
	132	export function isAwaitedWork(work: Pick<Work, "kind">): boolean {
	133	return work.kind !== "shell";
	134	}
	135
	136	/**
	137	* A run's status as its waiter should read it: a delegating parent,
	138	* `task_status`, or `t3_thread_wait`. A turn that ended to wait on work the
	139	* restart then cancelled did not finish, so it is "interrupted", not
	140	* "completed", and its last message is not a final result.
	141	*/
	142	export function runOutcome(
	143	run: Pick<OrchestrationV2Run, "status" \| "restartCancelledBackgroundWork">,
	144	): OrchestrationV2Run["status"] {
	145	return (run.status === "completed" \|\| run.status === "cancelled") &&
	146	(run.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
	147	? "interrupted"
	148	: run.status;
	149	}
	150
	151	/**
	152	* Whether a restart continuation may prompt from `source`: a turn the restart
	153	* cut mid-way, or a settled turn that lost awaited work in the restart.
	154	*/
	155	export function isRestartContinuationSource(
	156	source: Pick<OrchestrationV2Run, "status" \| "activeAttemptId" \| "restartCancelledBackgroundWork">,
	157	providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" \| "status">>,
	158	): boolean {
	159	return isRestartNoteSource(source, providerTurns)
	160	? (source.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
	161	: source.status === "cancelled";
	162	}
	163
126	164	/**
127	165	* The note a restart continuation of `source` carries, and whether the turn it
128	166	* continues had settled (the note is then the whole prompt). A continuation cut

apps/server/src/orchestration-v2/RestartContinuation.test.ts+191-40

...	...	@@ -223,6 +223,48 @@ it.effect.each(["completed", "waiting", "cancelled"] as const)(
223	223	}),
224	224	);
225	225
	226	it.effect.each(["completed", "waiting", "cancelled"] as const)(
	227	"wakes a %s run whose monitor the restart cancelled, with the note as its prompt",
	228	(status) =>
	229	Effect.gen(function* () {
	230	const base = makeProjection();
	231	const work = [
	232	{ kind: "monitor" as const, label: "Watch CI run 42" },
	233	{ kind: "shell" as const, label: "vp run dev" },
	234	];
	235	const projection = {
	236	...base,
	237	runs: [{ ...base.runs[0]!, status, restartCancelledBackgroundWork: work }],
	238	providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
	239	} as unknown as OrchestrationV2ThreadProjection;
	240	const commands: Parameters<
	241	ThreadManagementService.ThreadManagementService["Service"]["dispatch"]
	242	>[0][] = [];
	243	yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
	244	Effect.provide(
	245	Layer.merge(
	246	Layer.mock(ThreadManagementService.ThreadManagementService)({
	247	getThreadRecords: () => Effect.succeed(projection),
	248	recoverDelegatedTask: () => Effect.void,
	249	dispatch: (command) => {
	250	commands.push(command);
	251	return Effect.succeed({} as never);
	252	},
	253	}),
	254	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
	255	),
	256	),
	257	);
	258	assert.lengthOf(commands, 1);
	259	const command = commands[0]!;
	260	assert.equal(command.type, "message.dispatch");
	261	if (command.type !== "message.dispatch") return;
	262	assert.equal(command.restartContinuationOfRunId, runId);
	263	assert.include(command.text, "- monitor: Watch CI run 42");
	264	assert.include(command.text, "- shell: vp run dev");
	265	}),
	266	);
	267
226	268	it.effect("does not continue a failed run that lost background work", () =>
227	269	Effect.gen(function* () {
228	270	const base = makeProjection();
...	...	@@ -319,6 +361,62 @@ it.effect.each([
319	361	}),
320	362	);
321	363
	364	it.effect("lets a cut blocking delegate_task wake its parent like an async task", () =>
	365	Effect.gen(function* () {
	366	const base = makeProjection();
	367	const task = (id: string, completionWake: "settled_only" \| "always", status: string) => ({
	368	id: NodeId.make(id),
	369	threadId,
	370	runId,
	371	origin: "app_owned",
	372	childThreadId: ThreadId.make(`thread:${id}`),
	373	driver,
	374	providerInstanceId: instanceId,
	375	completionWake,
	376	status,
	377	});
	378	const projection = {
	379	...base,
	380	subagents: [
	381	task("task:waited", "settled_only", "running"),
	382	task("task:async", "always", "running"),
	383	task("task:finished", "settled_only", "completed"),
	384	],
	385	} as unknown as OrchestrationV2ThreadProjection;
	386	let committed: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] \| undefined;
	387	const recovery = yield* ProviderRuntimeRecovery.make.pipe(
	388	Effect.provide(
	389	Layer.mergeAll(
	390	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
	391	Layer.mock(ProjectionStore.ProjectionStoreV2)({
	392	getRecoveryThreadIds: () => Effect.succeed([threadId]),
	393	getRuntimeRecoveryProjection: () => Effect.succeed(projection),
	394	}),
	395	Layer.mock(EventSink.EventSinkV2)({
	396	commitCommand: (input) => {
	397	committed = input;
	398	return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
	399	},
	400	}),
	401	IdAllocator.layer,
	402	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
	403	Layer.mock(EffectOutbox.EffectOutboxV2)({
	404	reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
	405	}),
	406	),
	407	),
	408	);
	409	yield* recovery.reconcile("startup");
	410	const upgraded = (committed?.events ?? []).flatMap((event) =>
	411	event.type === "subagent.updated" ? [event.payload] : [],
	412	);
	413	assert.deepEqual(
	414	upgraded.map((row) => ({ id: row.id, completionWake: row.completionWake })),
	415	[{ id: NodeId.make("task:waited"), completionWake: "always" }],
	416	);
	417	}),
	418	);
	419
322	420	it.effect("does not duplicate delivery and yields to newer user work or opt-out", () =>
323	421	Effect.gen(function* () {
324	422	let projection = makeProjection();
...	...	@@ -424,7 +522,7 @@ it.effect("prepares no continuation for background work another provider thread
424	522	);
425	523	yield* recovery.prepareForShutdown;
426	524	assert.lengthOf(writes, 0);
427		// Work on the settled run's own provider thread must not wake it either.
	525	// The same work on the settled run's own provider thread does wake it.
428	526	const ownWork = {
429	527	...projection,
430	528	turnItems: [{ ...projection.turnItems[0]!, providerThreadId }],
...	...	@@ -451,18 +549,54 @@ it.effect("prepares no continuation for background work another provider thread
451	549	),
452	550	);
453	551	yield* ownRecovery.prepareForShutdown;
454		assert.lengthOf(writes, 0);
	552	assert.lengthOf(writes, 1);
	553	assert.deepEqual(writes[0]!.effects[0]?.request, {
	554	type: "provider-runtime.continue",
	555	sourceRunId: runId,
	556	});
455	557	}),
456	558	);
457	559
	560	const backgroundSubagent = {
	561	turnItem: {
	562	id: "turn-item:background-subagent",
	563	runId,
	564	nodeId: null,
	565	providerThreadId,
	566	providerInstanceId: instanceId,
	567	type: "subagent",
	568	subagentId: "subagent:background",
	569	title: "Background reviewer",
	570	status: "running",
	571	},
	572	work: {
	573	kind: "subagent" as const,
	574	label: "Background reviewer",
	575	id: "turn-item:background-subagent",
	576	},
	577	};
	578	const devServer = {
	579	turnItem: {
	580	id: "turn-item:dev-server",
	581	runId,
	582	nodeId: null,
	583	providerThreadId,
	584	providerInstanceId: instanceId,
	585	type: "command_execution",
	586	input: "vp run dev",
	587	status: "running",
	588	},
	589	work: { kind: "shell" as const, label: "vp run dev", id: "turn-item:dev-server" },
	590	};
	591
458	592	it.effect.each([
459		["completed", scratchProjectId],
460		["completed", ProjectId.make("restart-project")],
461		["waiting", scratchProjectId],
462		["waiting", ProjectId.make("restart-project")],
	593	["completed", scratchProjectId, backgroundSubagent, true],
	594	["waiting", ProjectId.make("restart-project"), backgroundSubagent, true],
	595	["completed", ProjectId.make("restart-project"), devServer, false],
	596	["waiting", scratchProjectId, devServer, false],
463	597	] as const)(
464		"cleans up background work without waking a %s run in project %s",
465		([status, projectId]) =>
	598	"cleans up background work of a %s run in project %s and wakes it only for awaited work",
	599	([status, projectId, background, wakes]) =>
466	600	Effect.gen(function* () {
467	601	const base = makeProjection();
468	602	const projection = {
...	...	@@ -470,29 +604,20 @@ it.effect.each([
470	604	thread: { ...base.thread, projectId },
471	605	runs: [{ ...base.runs[0]!, status }],
472	606	providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
473		turnItems: [
474		{
475		id: "turn-item:background-subagent",
476		runId,
477		nodeId: null,
478		providerThreadId,
479		providerInstanceId: instanceId,
480		type: "subagent",
481		subagentId: "subagent:background",
482		title: "Background reviewer",
483		status: "running",
484		},
485		],
486		subagents: [
487		{
488		id: "subagent:background",
489		runId,
490		origin: "provider_native",
491		driver,
492		providerInstanceId: instanceId,
493		status: "running",
494		},
495		],
	607	turnItems: [background.turnItem],
	608	subagents:
	609	background === backgroundSubagent
	610	? [
	611	{
	612	id: "subagent:background",
	613	runId,
	614	origin: "provider_native",
	615	driver,
	616	providerInstanceId: instanceId,
	617	status: "running",
	618	},
	619	]
	620	: [],
496	621	} as unknown as OrchestrationV2ThreadProjection;
497	622	for (const trigger of ["startup", "shutdown"] as const) {
498	623	const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
...	...	@@ -530,25 +655,51 @@ it.effect.each([
530	655	);
531	656	if (trigger === "shutdown") yield* recovery.prepareForShutdown;
532	657	yield* recovery.reconcile(trigger);
533		assert.lengthOf(writes, 0);
	658	// Shutdown records the wake before it cancels the work; a startup
	659	// after a kill records it with the cancellation.
	660	const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects];
	661	assert.deepEqual(
	662	recorded.map((effect) => effect.request),
	663	wakes ? [{ type: "provider-runtime.continue", sourceRunId: runId }] : [],
	664	);
534	665	assert.lengthOf(commits, 1);
535		assert.lengthOf(commits[0]!.effects, 0);
536	666	const events = commits[0]!.events;
537		for (const type of ["turn-item.updated", "subagent.updated"] as const)
538		assert.isTrue(
539		events.some((event) => event.type === type && event.payload.status === "cancelled"),
540		);
	667	assert.isTrue(
	668	events.some(
	669	(event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
	670	),
	671	);
541	672	const note = events.find((event) => event.type === "run.background-work-cancelled");
542	673	assert.equal(note?.runId, runId);
543		assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [
544		{ kind: "subagent", label: "Background reviewer", id: "turn-item:background-subagent" },
545		]);
	674	assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [background.work]);
546	675	if (status === "completed")
547	676	assert.isFalse(events.some((event) => event.type === "run.updated"));
548	677	}
549	678	}),
550	679	);
551	680
	681	it("does not wake a settled run for work left by a session that already stopped", () => {
	682	const base = makeProjection();
	683	const projection = {
	684	...base,
	685	runs: [{ ...base.runs[0]!, status: "completed" }],
	686	providerThreads: [{ ...base.providerThreads[0]!, status: "idle" }],
	687	turnItems: [backgroundSubagent.turnItem],
	688	} as unknown as OrchestrationV2ThreadProjection;
	689	assert.equal(restartContinuationRun(projection, now)?.id, runId);
	690	for (const status of ["stopped", "error"])
	691	assert.isUndefined(
	692	restartContinuationRun(
	693	{
	694	...projection,
	695	providerSessions: [{ ...projection.providerSessions[0]!, status }],
	696	} as OrchestrationV2ThreadProjection,
	697	now,
	698	),
	699	status,
	700	);
	701	});
	702
552	703	it.effect("does not cancel or resume a run that completes while shutdown intent commits", () =>
553	704	Effect.gen(function* () {
554	705	let projection = makeProjection();

apps/server/src/orchestration-v2/RestartContinuation.ts+56-12

...	...	@@ -16,6 +16,10 @@ import * as ServerSettings from "../serverSettings.ts";
16	16	import { isNativeMaintenanceCommand } from "./Orchestrator.ts";
17	17	import * as ThreadManagementService from "./ThreadManagementService.ts";
18	18	import {
	19	cancelledRosterTaskWork,
	20	cancelledTurnItemWork,
	21	isAwaitedWork,
	22	isRestartContinuationSource,
19	23	isRestartNoteSource,
20	24	restartCancelledBackgroundWorkNote,
21	25	restartContinuationNote,
...	...	@@ -29,7 +33,9 @@ const MAX_CONTINUATION_CHAIN = 3;
29	33	/**
30	34	* Whether a restart must leave this run for the user: the thread is put away,
31	35	* the provider turn already ended, or the user owes an answer. Checked when
32		* intent is recorded and again when the continuation is delivered.
	36	* intent is recorded and again when the continuation is delivered. A
	37	* `settled` run is woken for the awaited work it lost, so its ended provider
	38	* turn does not block it.
33	39	*/
34	40	function restartContinuationBlocked(
35	41	projection: Pick<
...	...	@@ -38,6 +44,7 @@ function restartContinuationBlocked(
38	44	>,
39	45	source: OrchestrationV2Run,
40	46	now: DateTime.Utc,
	47	settled: boolean,
41	48	): boolean {
42	49	const { thread } = projection;
43	50	if (
...	...	@@ -49,6 +56,7 @@ function restartContinuationBlocked(
49	56	return true;
50	57	const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId);
51	58	if (
	59	!settled &&
52	60	turn !== undefined &&
53	61	(turn.status === "completed" \|\| turn.status === "failed" \|\| turn.status === "interrupted")
54	62	)
...	...	@@ -82,8 +90,38 @@ function restartContinuationBlocked(
82	90	}
83	91
84	92	/**
85		* Resume only an unfinished root turn. Leftover background work is cleaned up
86		* separately and reported on the next user turn; it must not wake a settled run.
	93	* Open awaited work (see `isAwaitedWork`) on a provider thread. App-owned
	94	* delegations are not counted: their own threads continue and report back.
	95	*/
	96	function hasOpenAwaitedWork(
	97	projection: Pick<ProjectionRuntimeRecoveryState, "runs" \| "turnItems">,
	98	providerThread: ProjectionRuntimeRecoveryState["providerThreads"][number],
	99	): boolean {
	100	const openItem = projection.turnItems.some((item) => {
	101	if (item.status !== "pending" && item.status !== "running" && item.status !== "waiting")
	102	return false;
	103	if (item.type === "subagent" && item.origin === "app_owned" && item.childThreadId !== null)
	104	return false;
	105	const work = cancelledTurnItemWork(item);
	106	const providerThreadId =
	107	item.providerThreadId ??
	108	projection.runs.find((run) => run.id === item.runId)?.providerThreadId;
	109	return work !== undefined && isAwaitedWork(work) && providerThreadId === providerThread.id;
	110	});
	111	return (
	112	openItem \|\|
	113	(providerThread.ownerNodeId === null &&
	114	(providerThread.pendingBackgroundTasks ?? []).some((task) =>
	115	isAwaitedWork(cancelledRosterTaskWork(task)),
	116	))
	117	);
	118	}
	119
	120	/**
	121	* The run a restart continues: an unfinished root turn, or a settled one
	122	* waiting on awaited work that the restart cancels. A settled run needs a
	123	* live session, so stale records from an older crash never wake it. Other
	124	* leftover work is cleaned up and reported on the next user turn.
87	125	*/
88	126	export function restartContinuationRun(
89	127	projection: Pick<
...	...	@@ -109,9 +147,10 @@ export function restartContinuationRun(
109	147	if (!run) return;
110	148	const preparedContinuation =
111	149	run.status === "starting" && run.restartContinuationOfRunId !== undefined;
112		if (run.status !== "running" && !preparedContinuation) return;
113		if (restartContinuationBlocked(projection, run, now)) return;
114		const liveTurnRequired = !preparedContinuation;
	150	const settled = run.status === "completed" \|\| run.status === "waiting";
	151	if (run.status !== "running" && !preparedContinuation && !settled) return;
	152	if (restartContinuationBlocked(projection, run, now, settled)) return;
	153	const liveTurnRequired = !preparedContinuation && !settled;
115	154	if (projection.thread.providerInstanceId !== run.providerInstanceId) return;
116	155	const providerThread = projection.providerThreads.find(
117	156	(thread) => thread.id === run.providerThreadId,
...	...	@@ -138,9 +177,10 @@ export function restartContinuationRun(
138	177	session === undefined \|\|
139	178	session.providerInstanceId !== run.providerInstanceId \|\|
140	179	session.driver !== providerThread.driver \|\|
141		(liveTurnRequired && (session.status === "stopped" \|\| session.status === "error"))
	180	(!preparedContinuation && (session.status === "stopped" \|\| session.status === "error"))
142	181	)
143	182	return;
	183	if (settled) return hasOpenAwaitedWork(projection, providerThread) ? run : undefined;
144	184	if (
145	185	liveTurnRequired &&
146	186	!projection.providerTurns.some(
...	...	@@ -173,13 +213,17 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
173	213	return;
174	214	if (projection.messages.some((message) => message.id === messageId)) return;
175	215	const source = projection.runs.find((run) => run.id === input.sourceRunId);
176		// Pending effects from older versions may target settled background work,
177		// including waiting runs that reconciliation subsequently cancelled.
	216	// Reconciliation cancels a waiting run it cannot checkpoint, so a settled
	217	// source can be cancelled too. It continues only for awaited work.
178	218	if (
179	219	!source \|\|
180		source.status !== "cancelled" \|\|
181		isRestartNoteSource(source, projection.providerTurns) \|\|
182		restartContinuationBlocked(projection, source, yield* DateTime.now)
	220	!isRestartContinuationSource(source, projection.providerTurns) \|\|
	221	restartContinuationBlocked(
	222	projection,
	223	source,
	224	yield* DateTime.now,
	225	isRestartNoteSource(source, projection.providerTurns),
	226	)
183	227	)
184	228	return;
185	229	// A user submission after reconciliation takes precedence over an automatic

apps/server/src/orchestration-v2/ThreadManagementService.test.ts+123

...	...	@@ -597,6 +597,7 @@ it.effect.each([
597	597	Layer.provide(
598	598	Layer.mock(Orchestrator.OrchestratorV2)({
599	599	getThreadEventSequence: () => Effect.succeed(0),
	600	restartContinuationPending: () => Effect.succeed(false),
600	601	// No run update arrives, so the timeout path runs while a final
601	602	// projection read can still observe a terminal run.
602	603	streamStoredEventsFrom: () =>
...	...	@@ -666,6 +667,7 @@ it.effect("waitForThread reads the run again only when the run updates", () =>
666	667	Layer.provide(
667	668	Layer.mock(Orchestrator.OrchestratorV2)({
668	669	getThreadEventSequence: () => Effect.succeed(0),
	670	restartContinuationPending: () => Effect.succeed(false),
669	671	// Only the run.updated stream carries events in this test.
670	672	streamStoredEventsFrom: (input) =>
671	673	input?.eventType === "run.updated"
...	...	@@ -705,6 +707,126 @@ it.effect("waitForThread reads the run again only when the run updates", () =>
705	707	}),
706	708	);
707	709
	710	it.effect("waitForThread follows a restart continuation instead of the cut run", () =>
	711	Effect.gen(function* () {
	712	const projectId = ProjectId.make("project:thread-management:wait-restart");
	713	const threadId = ThreadId.make("thread:thread-management:wait-restart");
	714	const runId = RunId.make("run:thread-management:wait-restart");
	715	const continuationId = RunId.make("run:thread-management:wait-restart:2");
	716	const subscribed = yield* Deferred.make<void>();
	717	const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
	718	// The restart cut the run; its continuation effect has not started yet.
	719	let runs: Array<{
	720	id: RunId;
	721	status: OrchestrationV2Run["status"];
	722	restartContinuationOfRunId?: RunId;
	723	}> = [{ id: runId, status: "cancelled" }];
	724	let pending = true;
	725	const stored = (sequence: number, event: object) =>
	726	({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent;
	727	const layerTest = ThreadManagementService.layer.pipe(
	728	Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
	729	Layer.provide(
	730	Layer.mock(Orchestrator.OrchestratorV2)({
	731	getThreadEventSequence: () => Effect.succeed(0),
	732	restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId),
	733	awaitRestartContinuation: () => Effect.never,
	734	streamStoredEventsFrom: (input) =>
	735	input?.eventType === "run.updated"
	736	? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
	737	Stream.drain,
	738	Stream.concat(Stream.fromQueue(events)),
	739	)
	740	: Stream.never,
	741	getThreadRecords: () =>
	742	Effect.sync(
	743	() =>
	744	({
	745	thread: { id: threadId, projectId, deletedAt: null },
	746	runs,
	747	}) as unknown as OrchestrationV2ThreadProjection,
	748	),
	749	}),
	750	),
	751	Layer.provide(ServerSettings.layerTest()),
	752	Layer.provide(layerReceiptsTest),
	753	);
	754	const service = yield* ThreadManagementService.ThreadManagementService.pipe(
	755	Effect.provide(layerTest),
	756	);
	757	const fiber = yield* service
	758	.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
	759	.pipe(Effect.forkChild);
	760	yield* Deferred.await(subscribed);
	761	const continuation = { id: continuationId, restartContinuationOfRunId: runId };
	762	runs = [...runs, { ...continuation, status: "running" }];
	763	pending = false;
	764	yield* Queue.offer(
	765	events,
	766	stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }),
	767	);
	768	runs = [runs[0]!, { ...continuation, status: "completed" }];
	769	yield* Queue.offer(
	770	events,
	771	stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }),
	772	);
	773	const result = yield* Fiber.join(fiber);
	774
	775	expect(result).toMatchObject({
	776	timedOut: false,
	777	run: { id: continuationId, status: "completed" },
	778	});
	779	}),
	780	);
	781
	782	it.effect("waitForThread returns the cut run once its continuation is declined", () =>
	783	Effect.gen(function* () {
	784	const projectId = ProjectId.make("project:thread-management:wait-declined");
	785	const threadId = ThreadId.make("thread:thread-management:wait-declined");
	786	const runId = RunId.make("run:thread-management:wait-declined");
	787	const subscribed = yield* Deferred.make<void>();
	788	const declined = yield* Deferred.make<void>();
	789	let pending = true;
	790	const layerTest = ThreadManagementService.layer.pipe(
	791	Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
	792	Layer.provide(
	793	Layer.mock(Orchestrator.OrchestratorV2)({
	794	getThreadEventSequence: () => Effect.succeed(0),
	795	restartContinuationPending: () => Effect.succeed(pending),
	796	awaitRestartContinuation: () => Deferred.await(declined),
	797	// Declining records no thread event.
	798	streamStoredEventsFrom: (input) =>
	799	input?.eventType === "run.updated"
	800	? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
	801	Stream.drain,
	802	Stream.concat(Stream.never),
	803	)
	804	: Stream.never,
	805	getThreadRecords: () =>
	806	Effect.succeed({
	807	thread: { id: threadId, projectId, deletedAt: null },
	808	runs: [{ id: runId, status: "cancelled" }],
	809	} as unknown as OrchestrationV2ThreadProjection),
	810	}),
	811	),
	812	Layer.provide(ServerSettings.layerTest()),
	813	Layer.provide(layerReceiptsTest),
	814	);
	815	const service = yield* ThreadManagementService.ThreadManagementService.pipe(
	816	Effect.provide(layerTest),
	817	);
	818	const fiber = yield* service
	819	.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
	820	.pipe(Effect.forkChild);
	821	yield* Deferred.await(subscribed);
	822	pending = false;
	823	yield* Deferred.succeed(declined, undefined);
	824	const result = yield* Fiber.join(fiber);
	825
	826	expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "cancelled" } });
	827	}),
	828	);
	829
708	830	it.effect.each([
709	831	"running",
710	832	"missing",
...	...	@@ -938,6 +1060,7 @@ it.effect.each([
938	1060	Layer.provide(
939	1061	Layer.mock(Orchestrator.OrchestratorV2)({
940	1062	getThreadEventSequence: () => Effect.succeed(0),
	1063	restartContinuationPending: () => Effect.succeed(false),
941	1064	getThreadRecords: () =>
942	1065	Effect.succeed({
943	1066	thread: { id: threadId, projectId, deletedAt: null },

apps/server/src/orchestration-v2/ThreadManagementService.ts+68-43

...	...	@@ -486,6 +486,20 @@ export function latestRun(
486	486	return projection.runs.toSorted((left, right) => right.ordinal - left.ordinal)[0];
487	487	}
488	488
	489	/** The newest run in `run`'s restart continuation chain, or `run` itself. */
	490	function latestRestartContinuation(
	491	runs: ReadonlyArray<OrchestrationV2Run>,
	492	run: OrchestrationV2Run,
	493	): OrchestrationV2Run {
	494	let current = run;
	495	for (let hops = 0; hops < runs.length; hops++) {
	496	const next = runs.find((candidate) => candidate.restartContinuationOfRunId === current.id);
	497	if (next === undefined) break;
	498	current = next;
	499	}
	500	return current;
	501	}
	502
489	503	export function latestActiveRun(
490	504	projection: Pick<OrchestrationV2ThreadProjection, "runs">,
491	505	): OrchestrationV2Run \| undefined {
...	...	@@ -844,65 +858,76 @@ const make = Effect.gen(function* () {
844	858	if (selectedRun === undefined) {
845	859	return { threadId: input.threadId, run: null, timedOut: false };
846	860	}
847		if (isTerminalRunStatus(selectedRun.status)) {
848		return { threadId: input.threadId, run: selectedRun, timedOut: false };
	861	// A restart cuts a run and continues it as a new run. Follow that chain,
	862	// so a cut run is not the answer while its continuation is pending.
	863	const outcome = (runs: ReadonlyArray<OrchestrationV2Run>) =>
	864	Effect.gen(function* () {
	865	const current = runs.find((candidate) => candidate.id === selectedRun.id);
	866	if (current === undefined) {
	867	return yield* new ThreadManagementRunNotFoundError({
	868	threadId: input.threadId,
	869	runId: selectedRun.id,
	870	});
	871	}
	872	const run = latestRestartContinuation(runs, current);
	873	const pending =
	874	isTerminalRunStatus(run.status) &&
	875	(yield* orchestrator.restartContinuationPending(run));
	876	return { run, pending, settled: isTerminalRunStatus(run.status) && !pending };
	877	});
	878	const initial = yield* outcome(target.runs);
	879	if (initial.settled) {
	880	return { threadId: input.threadId, run: initial.run, timedOut: false };
849	881	}
850	882
851		// Re-read the run only when the thread records a run update or its
852		// deletion, instead of polling the projection for up to an hour. Each
853		// stream keeps one event type, so transcript events never fill its buffer.
	883	// Re-read the runs only when the thread records an update to the waited
	884	// run or a continuation, or its deletion, instead of polling the
	885	// projection for up to an hour. Each stream keeps one event type, so
	886	// transcript events never fill its buffer. A declined continuation
	887	// records no event, so its effect settling wakes the wait too.
854	888	const wait = Stream.merge(
855		orchestrator.streamStoredEventsFrom({
856		threadId: input.threadId,
857		afterSequence,
858		eventType: "run.updated",
859		}),
860		orchestrator.streamStoredEventsFrom({
861		threadId: input.threadId,
862		afterSequence,
863		eventType: "thread.deleted",
864		}),
865		).pipe(
866		Stream.mapError(loadError),
867		Stream.filter(
868		(stored) =>
869		stored.event.type !== "run.updated" \|\| stored.event.payload.id === selectedRun.id,
	889	Stream.merge(
	890	orchestrator.streamStoredEventsFrom({
	891	threadId: input.threadId,
	892	afterSequence,
	893	eventType: "run.updated",
	894	}),
	895	orchestrator.streamStoredEventsFrom({
	896	threadId: input.threadId,
	897	afterSequence,
	898	eventType: "thread.deleted",
	899	}),
	900	).pipe(
	901	Stream.mapError(loadError),
	902	Stream.filter(
	903	(stored) =>
	904	stored.event.type !== "run.updated" \|\|
	905	stored.event.payload.id === selectedRun.id \|\|
	906	stored.event.payload.restartContinuationOfRunId !== undefined,
	907	),
870	908	),
	909	initial.pending
	910	? Stream.fromEffect(orchestrator.awaitRestartContinuation(initial.run))
	911	: Stream.empty,
	912	).pipe(
871	913	Stream.mapEffect(() =>
872		getProjectThreadRecords(input, ["runs"], { runIds: [selectedRun.id] }).pipe(
873		Effect.flatMap((current) => {
874		const run = current.runs.find((candidate) => candidate.id === selectedRun.id);
875		return run === undefined
876		? Effect.fail(
877		new ThreadManagementRunNotFoundError({
878		threadId: input.threadId,
879		runId: selectedRun.id,
880		}),
881		)
882		: Effect.succeed(run);
883		}),
	914	getProjectThreadRecords(input, ["runs"]).pipe(
	915	Effect.flatMap((current) => outcome(current.runs)),
884	916	),
885	917	),
886		Stream.filter((run) => isTerminalRunStatus(run.status)),
	918	Stream.filter((current) => current.settled),
887	919	Stream.runHead,
888	920	Effect.timeoutOption(Duration.millis(Math.max(1, input.timeoutMs))),
889	921	);
890	922	const waited = Option.flatten(yield* wait);
891	923	if (Option.isSome(waited)) {
892		return { threadId: input.threadId, run: waited.value, timedOut: false };
893		}
894		const current = yield* getProjectThreadRecords(input, ["runs"], { runIds: [selectedRun.id] });
895		const run = current.runs.find((candidate) => candidate.id === selectedRun.id);
896		if (run === undefined) {
897		return yield* new ThreadManagementRunNotFoundError({
898		threadId: input.threadId,
899		runId: selectedRun.id,
900		});
	924	return { threadId: input.threadId, run: waited.value.run, timedOut: false };
901	925	}
902	926	// The run may have reached a terminal status while the timeout was
903	927	// winning the race; the final projection read decides what actually
904	928	// happened, so only a still-active run counts as timed out.
905		return { threadId: input.threadId, run, timedOut: !isTerminalRunStatus(run.status) };
	929	const final = yield* outcome((yield* getProjectThreadRecords(input, ["runs"])).runs);
	930	return { threadId: input.threadId, run: final.run, timedOut: !final.settled };
906	931	});
907	932
908	933	const settleAfterRun: ThreadManagementServiceShape["settleAfterRun"] = Effect.fn(

apps/server/src/orchestration-v2/runtimeLayer.test.ts+27-14

...	...	@@ -3251,18 +3251,28 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3251	3251	}),
3252	3252	);
3253	3253
3254		it.effect.each(["failed", "completed", "waiting", "cancelled"] as const)(
3255		"does not admit a restart continuation of a %s run with only lost background work",
3256		(status) =>
	3254	it.effect.each([
	3255	["failed", "shell", false],
	3256	["completed", "shell", false],
	3257	["waiting", "shell", false],
	3258	["cancelled", "shell", false],
	3259	// A settled run lost work it was waiting on, so nothing else wakes it.
	3260	["failed", "monitor", false],
	3261	["completed", "monitor", true],
	3262	["waiting", "monitor", true],
	3263	["cancelled", "monitor", true],
	3264	] as const)(
	3265	"admits a restart continuation of a %s run that lost a %s: %s",
	3266	([status, kind, admitted]) =>
3257	3267	Effect.gen(function* () {
3258	3268	const orchestrator = yield* Orchestrator.OrchestratorV2;
3259	3269	const eventSink = yield* EventSink.EventSinkV2;
3260		const threadId = ThreadId.make(`runtime-layer-restart-${status}-source`);
	3270	const threadId = ThreadId.make(`runtime-layer-restart-${status}-${kind}-source`);
3261	3271	yield* orchestrator.dispatch({
3262	3272	type: "thread.create",
3263	3273	createdBy: "user",
3264	3274	creationSource: "web",
3265		commandId: CommandId.make(`restart-${status}-create`),
	3275	commandId: CommandId.make(`restart-${status}-${kind}-create`),
3266	3276	threadId,
3267	3277	projectId: ProjectId.make("restart-project"),
3268	3278	title: "Restart",
...	...	@@ -3276,9 +3286,9 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3276	3286	type: "message.dispatch",
3277	3287	createdBy: "user",
3278	3288	creationSource: "web",
3279		commandId: CommandId.make(`restart-${status}-user-message`),
	3289	commandId: CommandId.make(`restart-${status}-${kind}-user-message`),
3280	3290	threadId,
3281		messageId: MessageId.make(`restart-${status}-user-message`),
	3291	messageId: MessageId.make(`restart-${status}-${kind}-user-message`),
3282	3292	text: "Original work",
3283	3293	attachments: [],
3284	3294	modelSelection,
...	...	@@ -3287,13 +3297,13 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3287	3297	const original = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
3288	3298	const now = yield* DateTime.now;
3289	3299	yield* eventSink.commitCommand({
3290		commandId: CommandId.make(`restart-${status}-reconcile`),
	3300	commandId: CommandId.make(`restart-${status}-${kind}-reconcile`),
3291	3301	threadId,
3292	3302	commandType: "provider-runtime.reconcile",
3293	3303	acceptedAt: now,
3294	3304	events: [
3295	3305	{
3296		id: EventId.make(`restart-${status}-run`),
	3306	id: EventId.make(`restart-${status}-${kind}-run`),
3297	3307	type: "run.updated",
3298	3308	threadId,
3299	3309	runId: original.id,
...	...	@@ -3301,14 +3311,14 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3301	3311	payload: { ...original, status, completedAt: now },
3302	3312	},
3303	3313	{
3304		id: EventId.make(`restart-${status}-work`),
	3314	id: EventId.make(`restart-${status}-${kind}-work`),
3305	3315	type: "run.background-work-cancelled",
3306	3316	threadId,
3307	3317	runId: original.id,
3308	3318	occurredAt: now,
3309	3319	payload: {
3310	3320	runId: original.id,
3311		restartCancelledBackgroundWork: [{ kind: "shell", label: "sleep 25" }],
	3321	restartCancelledBackgroundWork: [{ kind, label: "sleep 25" }],
3312	3322	},
3313	3323	},
3314	3324	],
...	...	@@ -3318,9 +3328,9 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3318	3328	type: "message.dispatch",
3319	3329	createdBy: "agent",
3320	3330	creationSource: "server",
3321		commandId: CommandId.make(`restart-${status}-continuation`),
	3331	commandId: CommandId.make(`restart-${status}-${kind}-continuation`),
3322	3332	threadId,
3323		messageId: MessageId.make(`restart-${status}-continuation`),
	3333	messageId: MessageId.make(`restart-${status}-${kind}-continuation`),
3324	3334	text: "Note: the T3 server restarted.",
3325	3335	attachments: [],
3326	3336	modelSelection,
...	...	@@ -3328,8 +3338,11 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3328	3338	restartContinuationOfRunId: original.id,
3329	3339	});
3330	3340	const projection = yield* orchestrator.getThreadProjection(threadId);
3331		assert.lengthOf(projection.runs, 1);
3332	3341	assert.equal(projection.runs[0]?.status, status);
	3342	assert.deepEqual(
	3343	projection.runs.slice(1).map((run) => run.restartContinuationOfRunId),
	3344	admitted ? [original.id] : [],
	3345	);
3333	3346	}),
3334	3347	);
3335	3348

apps/server/src/process/agentScope.test.ts+61-4

...	...	@@ -7,10 +7,18 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
7	7	import { describe, expect, it } from "@effect/vitest";
8	8	import * as HostProcess from "@t3tools/shared/HostProcess";
9	9	import * as Effect from "effect/Effect";
	10	import * as Layer from "effect/Layer";
10	11	import { ChildProcessSpawner } from "effect/process";
11	12
	13	import * as ServerConfig from "../config.ts";
12	14	import * as ProcessRunner from "../processRunner.ts";
13		import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts";
	15	import {
	16	agentSliceMemoryLimits,
	17	classifyScope,
	18	make,
	19	ownedAgentSlice,
	20	type ScopeState,
	21	} from "./agentScope.ts";
14	22
15	23	const GiB = 1024 ** 3;
16	24
...	...	@@ -55,11 +63,14 @@ describe("agentSliceMemoryLimits", () => {
55	63	});
56	64
57	65	// Runs the real service against a fake user manager. `units` holds the
58		// ActiveState and Result that `systemctl show` reports for each scope.
	66	// ActiveState and Result that `systemctl show` reports for each scope or
	67	// slice, and `calls` records every command in order.
59	68	const withFakeSystemd = <A, E>(
60	69	body: (input: {
61	70	readonly scope: Effect.Success<typeof make>;
62	71	readonly units: Map<string, { activeState: string; result: string }>;
	72	readonly calls: ReadonlyArray<ReadonlyArray<string>>;
	73	readonly stateDir: string;
63	74	}) => Effect.Effect<A, E>,
64	75	) =>
65	76	Effect.gen(function* () {
...	...	@@ -68,6 +79,7 @@ const withFakeSystemd = <A, E>(
68	79	NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
69	80	}
70	81	const units = new Map<string, { activeState: string; result: string }>();
	82	const calls: Array<ReadonlyArray<string>> = [];
71	83	const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({
72	84	stdout,
73	85	stderr: "",
...	...	@@ -81,6 +93,7 @@ const withFakeSystemd = <A, E>(
81	93	const runner = ProcessRunner.ProcessRunner.of({
82	94	run: ({ args }) =>
83	95	Effect.sync(() => {
	96	calls.push(args);
84	97	const [, verb, unit = ""] = args;
85	98	if (verb === "reset-failed") units.delete(unit);
86	99	if (verb !== "show") return output("");
...	...	@@ -97,14 +110,25 @@ const withFakeSystemd = <A, E>(
97	110	Effect.provideService(HostProcess.Platform, "linux"),
98	111	Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
99	112	);
100		return yield* body({ scope, units }).pipe(
	113	const { stateDir } = yield* ServerConfig.ServerConfig;
	114	return yield* body({ scope, units, calls, stateDir }).pipe(
101	115	Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
102	116	);
103		}).pipe(Effect.scoped, Effect.provide(NodeServices.layer));
	117	}).pipe(
	118	Effect.scoped,
	119	Effect.provide(
	120	ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-scope-" }).pipe(
	121	Layer.provideMerge(NodeServices.layer),
	122	),
	123	),
	124	);
104	125
105	126	const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>
106	127	launch.args.find((arg) => arg.startsWith("--unit="))?.slice("--unit=".length) ?? "";
107	128
	129	const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
	130	launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
	131
108	132	describe("AgentScope service", () => {
109	133	it.effect("labels a thread from its newest scope until its next session clears it", () =>
110	134	withFakeSystemd(({ scope, units }) =>
...	...	@@ -142,4 +166,37 @@ describe("AgentScope service", () => {
142	166	}),
143	167	),
144	168	);
	169
	170	it.effect("stops agents a killed server left before the first launch", () =>
	171	withFakeSystemd(({ scope, units, calls, stateDir }) =>
	172	Effect.gen(function* () {
	173	units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
	174	const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
	175	const stops = calls.filter(([, verb]) => verb === "stop");
	176	expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
	177	// Agents run in this server's own slice; terminals stay outside it.
	178	expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
	179	// An agent that ignores SIGTERM is killed well inside the stop's wait.
	180	expect(agent.args).toContain("--property=TimeoutStopSec=10s");
	181	const terminal = yield* scope.wrap({
	182	command: "agent",
	183	args: [],
	184	name: "terminal",
	185	survivesRestart: true,
	186	});
	187	expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
	188	// Only the first launch looks for leftovers.
	189	expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1);
	190	}),
	191	),
	192	);
	193
	194	it.effect("leaves a quiet slice alone", () =>
	195	withFakeSystemd(({ scope, calls }) =>
	196	Effect.gen(function* () {
	197	yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
	198	expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
	199	}),
	200	),
	201	);
145	202	});

apps/server/src/process/agentScope.ts+73-14

...	...	@@ -15,6 +15,7 @@ import * as FileSystem from "effect/FileSystem";
15	15	import * as Layer from "effect/Layer";
16	16	import * as Path from "effect/Path";
17	17
	18	import * as ServerConfig from "../config.ts";
18	19	import * as ProcessRunner from "../processRunner.ts";
19	20
20	21	/**
...	...	@@ -24,6 +25,18 @@ import * as ProcessRunner from "../processRunner.ts";
24	25	*/
25	26	const AGENT_SLICE = "app-t3code-agents.slice";
26	27
	28	/**
	29	* The child slice for one server's agents, named after its state directory.
	30	* Only one server uses a state directory at a time, so an agent in this slice
	31	* when the server starts was left by an earlier server on the same state,
	32	* for example one the launcher killed during an update. Other servers on the
	33	* machine, such as dev servers, have their own slice.
	34	*/
	35	export function ownedAgentSlice(stateDir: string): string {
	36	const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
	37	return `app-t3code-agents-${hash}.slice`;
	38	}
	39
27	40	const GiB = 1024 ** 3;
28	41
29	42	/**
...	...	@@ -84,9 +97,12 @@ function parseCgroupCounter(text: string, key: string): number \| undefined {
84	97	return match === null ? undefined : Number(match[1]);
85	98	}
86	99
	100	// A stop sends SIGTERM, then SIGKILL after this long. systemd's 90 s default
	101	// would outlast the wait for a previous server's agents below.
	102	const STOP_TIMEOUT_OPTION = "--property=TimeoutStopSec=10s";
87	103	const STOPPING_POLL_INTERVAL = "100 millis";
88	104	const STOPPING_POLL_ATTEMPTS = 20;
89		// A stopping scope can wait out TimeoutStopSec (90 s by default) for a stuck process.
	105	// A stopping scope can wait out its TimeoutStopSec for a stuck process.
90	106	const CLEAR_FAILED_INTERVAL = "1 second";
91	107	const CLEAR_FAILED_ATTEMPTS = 120;
92	108	const MAX_TRACKED_THREADS = 500;
...	...	@@ -99,6 +115,8 @@ export const make = Effect.gen(function* () {
99	115	const fileSystem = yield* FileSystem.FileSystem;
100	116	const path = yield* Path.Path;
101	117	const hostEnvironment = yield* HostProcess.Environment;
	118	const { stateDir } = yield* ServerConfig.ServerConfig;
	119	const ownedSlice = ownedAgentSlice(stateDir);
102	120	const layerScope = yield* Effect.scope;
103	121
104	122	const resolve = (command: string, env: NodeJS.ProcessEnv) =>
...	...	@@ -111,8 +129,45 @@ export const make = Effect.gen(function* () {
111	129	const run = (command: string, args: ReadonlyArray<string>) =>
112	130	runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option);
113	131
114		// Runs once, on the first agent launch. It proves this process can reach a
115		// systemd user manager, creates the agents slice, and sets its limits.
	132	const readCgroupFile = (controlGroup: string, file: string) =>
	133	fileSystem
	134	.readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
	135	.pipe(Effect.orElseSucceed(() => ""));
	136
	137	// Old agents keep their scopes when the server is killed, and the restarted
	138	// server resumes their sessions. Stop them first, so two copies of one
	139	// session never work in the same worktree.
	140	const stopLeftoverAgents = (systemctl: string) =>
	141	Effect.gen(function* () {
	142	const shown = yield* run(systemctl, [
	143	"--user",
	144	"show",
	145	ownedSlice,
	146	"--property=ActiveState,ControlGroup",
	147	]);
	148	if (shown._tag === "None") return;
	149	const values = parseKeyValues(shown.value.stdout);
	150	if (values.get("ActiveState") !== "active") return;
	151	// An active slice can be empty once its last agent exited.
	152	const controlGroup = values.get("ControlGroup") ?? "";
	153	const events = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
	154	if (parseCgroupCounter(events, "populated") === 0) return;
	155	yield* Effect.logWarning("Stopping agents left by a previous server", { slice: ownedSlice });
	156	// Agents get SIGTERM, then SIGKILL after their stop timeout.
	157	const stopped = yield* runner
	158	.run({ command: systemctl, args: ["--user", "stop", ownedSlice], timeout: "30 seconds" })
	159	.pipe(Effect.option);
	160	if (stopped._tag === "None" \|\| stopped.value.code !== 0) {
	161	yield* Effect.logWarning("Could not stop agents left by a previous server", {
	162	slice: ownedSlice,
	163	stderr: stopped._tag === "Some" ? stopped.value.stderr.trim() : undefined,
	164	});
	165	}
	166	});
	167
	168	// Runs once, on the first agent or terminal launch. It proves this process
	169	// can reach a systemd user manager, stops agents a previous server left,
	170	// creates the agents slice, and sets its limits.
116	171	const systemd = yield* Effect.cached(
117	172	Effect.gen(function* () {
118	173	const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;
...	...	@@ -120,6 +175,7 @@ export const make = Effect.gen(function* () {
120	175	const systemdRun = yield* resolve("systemd-run", hostEnvironment);
121	176	const systemctl = yield* resolve("systemctl", hostEnvironment);
122	177	if (systemdRun === null \|\| systemctl === null) return undefined;
	178	yield* stopLeftoverAgents(systemctl);
123	179	const probe = (options: ReadonlyArray<string>) =>
124	180	run(systemdRun, [
125	181	"--user",
...	...	@@ -133,12 +189,13 @@ export const make = Effect.gen(function* () {
133	189	]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0));
134	190	// Stop the whole scope when the kernel kills one of its processes, and
135	191	// record the result as oom-kill. Scopes take OOMPolicy from systemd 253;
136		// older user managers still get plain scopes, without the OOM label.
137		const oomOptions = ["--property=OOMPolicy=stop"];
	192	// older user managers still get scopes, without the OOM label.
	193	const stopOptions = [STOP_TIMEOUT_OPTION];
	194	const oomOptions = [...stopOptions, "--property=OOMPolicy=stop"];
138	195	const scopeOptions = (yield* probe(oomOptions))
139	196	? oomOptions
140		: (yield* probe([]))
141		? []
	197	: (yield* probe(stopOptions))
	198	? stopOptions
142	199	: undefined;
143	200	if (scopeOptions === undefined) {
144	201	yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
...	...	@@ -176,11 +233,6 @@ export const make = Effect.gen(function* () {
176	233	// at a time, so only its latest scope can explain a failure.
177	234	const scopeByThread = new Map<string, { unit: string; oomKilled: boolean }>();
178	235
179		const readCgroupFile = (controlGroup: string, file: string) =>
180		fileSystem
181		.readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
182		.pipe(Effect.orElseSucceed(() => ""));
183
184	236	const readScope = (systemctl: string, unit: string) =>
185	237	Effect.gen(function* () {
186	238	const shown = yield* run(systemctl, [
...	...	@@ -290,7 +342,14 @@ export const make = Effect.gen(function* () {
290	342	});
291	343
292	344	return {
293		wrap: Effect.fn("AgentScope.wrap")(function* ({ command, args, name, threadId, env }) {
	345	wrap: Effect.fn("AgentScope.wrap")(function* ({
	346	command,
	347	args,
	348	name,
	349	threadId,
	350	env,
	351	survivesRestart,
	352	}) {
294	353	const resolved = yield* resolve(command, env ?? hostEnvironment);
295	354	// Leave unresolved commands alone so the spawn reports the missing binary.
296	355	if (resolved === null) return { command, args };
...	...	@@ -301,7 +360,7 @@ export const make = Effect.gen(function* () {
301	360	const systemdScope: AgentSystemdScope = {
302	361	systemdRun: scope.systemdRun,
303	362	runtimeDir: scope.runtimeDir,
304		slice: AGENT_SLICE,
	363	slice: survivesRestart === true ? AGENT_SLICE : ownedSlice,
305	364	unit,
306	365	// Without a thread nothing reads the result, so let systemd unload
307	366	// the unit even when it fails.

apps/server/src/terminal/Manager.ts+1

...	...	@@ -2196,6 +2196,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
2196	2196	args: candidate.args ?? [],
2197	2197	name: "terminal",
2198	2198	env: spawnEnv,
	2199	survivesRestart: true,
2199	2200	});
2200	2201	const attempt = yield* Effect.result(
2201	2202	options.ptyAdapter.spawn({

apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts+2-1

...	...	@@ -1,6 +1,6 @@
1	1	import { describe, expect, it } from "vite-plus/test";
2	2	import {
3		type EnvironmentId,
	3	EnvironmentId,
4	4	ProviderDriverKind,
5	5	ProviderInstanceId,
6	6	type ServerProvider,
...	...	@@ -1076,6 +1076,7 @@ describe("getProviderUpdateRunToastView", () => {
1076	1076	providerDriver: string,
1077	1077	result: Parameters<typeof getProviderUpdateRunToastView>[0][number]["result"],
1078	1078	) => ({
	1079	environmentId: EnvironmentId.make(machineLabel),
1079	1080	machineLabel,
1080	1081	driver: driver(providerDriver),
1081	1082	instanceId: instanceId(providerDriver),

apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts+36-14

...	...	@@ -333,6 +333,7 @@ export function getProviderUpdateProgressToastView(input: {
333	333
334	334	/** One provider update sent by the cross-machine "Update all", with its result. */
335	335	export interface ProviderUpdateRun {
	336	readonly environmentId: EnvironmentId;
336	337	readonly machineLabel: string;
337	338	readonly driver: ProviderDriverKind;
338	339	readonly instanceId: ProviderInstanceId;
...	...	@@ -342,6 +343,35 @@ export interface ProviderUpdateRun {
342	343	>;
343	344	}
344	345
	346	/**
	347	* What one provider update in a cross-machine run did. A failure carries a
	348	* line that names the machine and provider, such as "alvin · Codex: npm exited".
	349	*/
	350	export function getProviderUpdateRunOutcome(
	351	run: ProviderUpdateRun,
	352	):
	353	\| { readonly status: "interrupted" \| "updated" }
	354	\| { readonly status: "failed"; readonly line: string } {
	355	if (isAtomCommandInterrupted(run.result)) return { status: "interrupted" };
	356	const label = `${run.machineLabel} · ${PROVIDER_DISPLAY_NAMES[run.driver] ?? run.driver}`;
	357	if (run.result._tag === "Failure") {
	358	const error = squashAtomCommandFailure(run.result);
	359	return {
	360	status: "failed",
	361	line: `${label}: ${error instanceof Error ? error.message : "Provider update failed."}`,
	362	};
	363	}
	364	const updateState = run.result.value.providers.find(
	365	(provider) => provider.instanceId === run.instanceId,
	366	)?.updateState;
	367	return updateState?.status === "succeeded"
	368	? { status: "updated" }
	369	: {
	370	status: "failed",
	371	line: `${label}: ${updateState?.message ?? "Provider update did not finish."}`,
	372	};
	373	}
	374
345	375	/**
346	376	* Summarize a cross-machine "Update all" as one toast, or null when every
347	377	* request was interrupted. Each update that did not succeed gets its own line,
...	...	@@ -350,23 +380,15 @@ export interface ProviderUpdateRun {
350	380	export function getProviderUpdateRunToastView(
351	381	runs: ReadonlyArray<ProviderUpdateRun>,
352	382	): Pick<ProviderUpdateToastView, "type" \| "title" \| "description"> \| null {
353		const settled = runs.filter((run) => !isAtomCommandInterrupted(run.result));
	383	const settled = runs
	384	.map(getProviderUpdateRunOutcome)
	385	.filter((outcome) => outcome.status !== "interrupted");
354	386	if (settled.length === 0) {
355	387	return null;
356	388	}
357		const failureLines = settled.flatMap((run) => {
358		const label = `${run.machineLabel} · ${PROVIDER_DISPLAY_NAMES[run.driver] ?? run.driver}`;
359		if (run.result._tag === "Failure") {
360		const error = squashAtomCommandFailure(run.result);
361		return [`${label}: ${error instanceof Error ? error.message : "Provider update failed."}`];
362		}
363		const updateState = run.result.value.providers.find(
364		(provider) => provider.instanceId === run.instanceId,
365		)?.updateState;
366		return updateState?.status === "succeeded"
367		? []
368		: [`${label}: ${updateState?.message ?? "Provider update did not finish."}`];
369		});
	389	const failureLines = settled.flatMap((outcome) =>
	390	outcome.status === "failed" ? [outcome.line] : [],
	391	);
370	392	if (failureLines.length === 0) {
371	393	return {
372	394	type: "success",

apps/web/src/components/ProviderUpdatesAction.tsx+6-3

...	...	@@ -20,7 +20,8 @@ import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";
20	20	* Each server queues updates that share an installer, so sending them all
21	21	* together is safe. The server still checks permissions: a session that
22	22	* cannot operate a machine gets a failure line for it. The returned promise
23		* settles when every update has finished.
	23	* settles with every result when every update has finished. `quiet` skips the
	24	* toast, for a caller that reports the results itself.
24	25	*/
25	26	export function useProviderUpdates() {
26	27	const { environments } = useEnvironments();
...	...	@@ -44,10 +45,11 @@ export function useProviderUpdates() {
44	45	[environments],
45	46	);
46	47
47		const updateAll = async () => {
	48	const updateAll = async ({ quiet = false }: { readonly quiet?: boolean } = {}) => {
48	49	const runs = await Promise.all(
49	50	machines.flatMap(({ environmentId, label, candidates }) =>
50	51	candidates.map(async (candidate): Promise<ProviderUpdateRun> => ({
	52	environmentId,
51	53	machineLabel: label,
52	54	driver: candidate.driver,
53	55	instanceId: candidate.instanceId,
...	...	@@ -58,7 +60,7 @@ export function useProviderUpdates() {
58	60	})),
59	61	),
60	62	);
61		const view = getProviderUpdateRunToastView(runs);
	63	const view = quiet ? null : getProviderUpdateRunToastView(runs);
62	64	if (view) {
63	65	toastManager.add(
64	66	stackedThreadToast({
...	...	@@ -67,6 +69,7 @@ export function useProviderUpdates() {
67	69	}),
68	70	);
69	71	}
	72	return runs;
70	73	};
71	74
72	75	return { machines, updateAll };

apps/web/src/components/ServerUpdateAction.test.tsx+2-3

...	...	@@ -498,13 +498,12 @@ describe("ServerUpdateProgress", () => {
498	498
499	499	expect(markup).toContain("Restarting…");
500	500	// The wait state is monochrome and calm: no versions, no step rail, no
501		// success/warning colors, one duty-cycled pulse on the dot.
	501	// success/warning colors, and a static dot that never repaints.
502	502	expect(markup).not.toContain("0.0.30");
503	503	expect(markup).not.toContain("Resum");
504	504	expect(markup).not.toContain("text-success");
505	505	expect(markup).not.toContain("text-primary");
506		expect(markup).toContain("animate-status-pulse");
507		expect(markup).not.toContain("animate-spin");
	506	expect(markup).not.toContain("animate-");
508	507	});
509	508
510	509	it("folds the sub-second installing handoff into the download phase", () => {

apps/web/src/components/ServerUpdateAction.tsx+37-26

...	...	@@ -65,17 +65,26 @@ type UpdateButtonProps = Pick<ComponentProps<typeof Button>, "variant" \| "size"
65	65	readonly appearance?: "button" \| "icon";
66	66	};
67	67
	68	/** What {@link useServerUpdate} did. "skipped" covers a missing permission, an
	69	update already in flight, and an interrupted request. */
	70	export type ServerUpdateOutcome =
	71	\| { readonly status: "updated" \| "skipped" }
	72	\| { readonly status: "failed"; readonly message: string };
	73
68	74	/** Updates one server and waits until it is back on the target version.
69		Reports its own success or failure toast; never rejects. */
	75	Toasts its success or failure unless `quiet`; never rejects. */
70	76	export function useServerUpdate() {
71	77	const updateServer = useAtomCommand(serverEnvironment.updateServer, { reportFailure: false });
72		return async (target: ServerUpdateTarget, failureTitle = "Server update failed") => {
	78	return async (
	79	target: ServerUpdateTarget,
	80	{ quiet = false }: { readonly quiet?: boolean } = {},
	81	): Promise<ServerUpdateOutcome> => {
73	82	const { environmentId, serverLabel, selfUpdate, targetVersion } = target;
74	83	if (
75	84	!canUpdateServer(appAtomRegistry.get(environmentSession.sessionStateAtom(environmentId))) \|\|
76	85	pendingUpdateEnvironmentIds.has(environmentId)
77	86	)
78		return;
	87	return { status: "skipped" };
79	88	pendingUpdateEnvironmentIds.add(environmentId);
80	89	try {
81	90	const result = await updateServer({
...	...	@@ -88,23 +97,30 @@ export function useServerUpdate() {
88	97	},
89	98	});
90	99	if (result._tag === "Failure") {
91		if (isAtomCommandInterrupted(result)) return;
	100	if (isAtomCommandInterrupted(result)) return { status: "skipped" };
92	101	throw squashAtomCommandFailure(result);
93	102	}
94		toastManager.add({
95		type: "success",
96		title: `${serverLabel} updated`,
97		description:
98		selfUpdate === "desktop-managed"
99		? `Desktop app relaunched on ${result.value.targetVersion}.`
100		: `Reconnected on t3@${result.value.targetVersion}.`,
101		});
	103	if (!quiet) {
	104	toastManager.add({
	105	type: "success",
	106	title: `${serverLabel} updated`,
	107	description:
	108	selfUpdate === "desktop-managed"
	109	? `Desktop app relaunched on ${result.value.targetVersion}.`
	110	: `Reconnected on t3@${result.value.targetVersion}.`,
	111	});
	112	}
	113	return { status: "updated" };
102	114	} catch (error) {
103		toastManager.add({
104		type: "error",
105		title: failureTitle,
106		description: updateFailureMessage(error),
107		});
	115	const message = updateFailureMessage(error);
	116	if (!quiet) {
	117	toastManager.add({
	118	type: "error",
	119	title: `${serverLabel} update failed`,
	120	description: message,
	121	});
	122	}
	123	return { status: "failed", message };
108	124	} finally {
109	125	pendingUpdateEnvironmentIds.delete(environmentId);
110	126	}
...	...	@@ -145,9 +161,7 @@ export function ServerUpdatesAction({
145	161	)) ?? true;
146	162	if (!confirmed) return;
147	163	}
148		await Promise.all(
149		available.map((target) => update(target, `${target.serverLabel} update failed`)),
150		);
	164	await Promise.all(available.map((target) => update(target)));
151	165	} finally {
152	166	pending.current = false;
153	167	setIsPending(false);
...	...	@@ -175,7 +189,7 @@ export function canUpdateServer(
175	189
176	190	/**
177	191	* One-row status for an in-flight server update: "Downloading…" then
178		* "Restarting…". The update is a wait, not a warning: a single pulsing dot
	192	* "Restarting…". The update is a wait, not a warning: a single static dot
179	193	* and label, no step rail, no versions. Failure turns the row red with the
180	194	* rollback reason.
181	195	*/
...	...	@@ -197,10 +211,7 @@ export function ServerUpdateProgress({
197	211	}
198	212	return (
199	213	<div className="mt-1 flex items-center gap-2 text-xs font-medium text-foreground">
200		<span
201		className="size-1.5 shrink-0 animate-status-pulse rounded-full bg-foreground"
202		aria-hidden="true"
203		/>
	214	<span className="size-1.5 shrink-0 rounded-full bg-foreground" aria-hidden="true" />
204	215	<span>{serverUpdateStageLabel(state.stage)}</span>
205	216	</div>
206	217	);
...	...	@@ -381,7 +392,7 @@ export function OutdatedServerUpdateAction({
381	392	} catch (error) {
382	393	toastManager.add({
383	394	type: "error",
384		title: "Server update failed",
	395	title: `${serverLabel} update failed`,
385	396	description: updateFailureMessage(error),
386	397	});
387	398	} finally {

apps/web/src/components/desktopUpdate.logic.test.ts+20

...	...	@@ -57,6 +57,26 @@ describe("desktop update button state", () => {
57	57	expect(getDesktopUpdateButtonTooltip(state)).toContain("Click to retry");
58	58	});
59	59
	60	it("says when the same version did not apply before", () => {
	61	const state: DesktopUpdateState = {
	62	...baseState,
	63	status: "downloaded",
	64	availableVersion: "1.1.0",
	65	downloadedVersion: "1.1.0",
	66	notAppliedVersion: "1.1.0",
	67	};
	68	expect(getDesktopUpdateButtonTooltip(state)).toBe(
	69	"Update 1.1.0 did not apply. Click to retry.",
	70	);
	71	expect(
	72	getDesktopUpdateButtonTooltip({
	73	...state,
	74	availableVersion: "1.2.0",
	75	downloadedVersion: "1.2.0",
	76	}),
	77	).toContain("1.2.0 downloaded");
	78	});
	79
60	80	it("keeps install action available after an install error", () => {
61	81	const state: DesktopUpdateState = {
62	82	...baseState,

apps/web/src/components/desktopUpdate.logic.ts+7

...	...	@@ -71,6 +71,13 @@ export function getArm64IntelBuildWarningDescription(state: DesktopUpdateState):
71	71	}
72	72
73	73	export function getDesktopUpdateButtonTooltip(state: DesktopUpdateState): string {
	74	if (
	75	state.notAppliedVersion !== undefined &&
	76	(state.status === "available" \|\| state.status === "downloaded") &&
	77	getDesktopUpdateDownloadedVersion(state) === state.notAppliedVersion
	78	) {
	79	return `Update ${state.notAppliedVersion} did not apply. Click to retry.`;
	80	}
74	81	if (state.status === "available") {
75	82	return `Update ${state.availableVersion ?? "available"} ready to download`;
76	83	}

apps/web/src/components/sidebar/SidebarUpdatePill.tsx+131-71

...	...	@@ -1,12 +1,23 @@
	1	import { useAtomValue } from "@effect/atom-react";
1	2	import type { DesktopBridge, DesktopUpdateState } from "@t3tools/contracts";
	3	import { Atom } from "effect/reactivity";
2	4	import { TriangleAlertIcon } from "lucide-react";
3		import { type ComponentProps, useCallback, useEffect, useId, useRef, useState } from "react";
	5	import {
	6	type ComponentProps,
	7	useCallback,
	8	useEffect,
	9	useId,
	10	useMemo,
	11	useRef,
	12	useState,
	13	} from "react";
4	14	import { flushSync } from "react-dom";
5	15	import { isElectron } from "../../env";
6	16	import { useMediaQuery } from "../../hooks/useMediaQuery";
7	17	import { cn } from "../../lib/utils";
8	18	import { ensureLocalApi } from "../../localApi";
9	19	import { useDesktopUpdateState } from "../../state/desktopUpdate";
	20	import { serverEnvironment } from "../../state/server";
10	21	import { stackedThreadToast, toastManager } from "../ui/toast";
11	22	import {
12	23	canCheckForUpdate,
...	...	@@ -33,7 +44,13 @@ import {
33	44	} from "./DesktopUpdateStatusIcon";
34	45	import { SidebarUpdateReleaseNotes } from "./SidebarUpdateReleaseNotes";
35	46	import { UpdateEverythingDialog } from "./UpdateEverythingDialog";
36		import { useUpdateEverything } from "./useUpdateEverything";
	47	import {
	48	describeUpdateRun,
	49	resolveUpdateRunMachineStatus,
	50	type UpdateEverythingRun,
	51	} from "./updateEverything.logic";
	52	import { useUpdateEverythingRun } from "./updateEverythingRun";
	53	import { type ReportLocalUpdateError, useUpdateEverything } from "./useUpdateEverything";
37	54
38	55	type SidebarUpdatePopoverChangeDetails = Parameters<
39	56	NonNullable<ComponentProps<typeof Popover>["onOpenChange"]>
...	...	@@ -68,27 +85,34 @@ export function openSidebarUpdateReleaseNotesPopoverOnForwardTab(
68	85	function resolveSidebarUpdatePresentation({
69	86	action,
70	87	showsEverything,
	88	run,
71	89	isDownloading,
72	90	showCheckIcon,
73	91	}: {
74	92	readonly action: ReturnType<typeof resolveDesktopUpdateButtonAction>;
75		/** Remote work is waiting or running, and this app is not downloading. */
	93	/** Remote work is waiting, and this app is not downloading. */
76	94	readonly showsEverything: boolean;
	95	/** The running "update everything" pass. Its state sets the icon. */
	96	readonly run: UpdateEverythingRun \| null;
77	97	readonly isDownloading: boolean;
78	98	readonly showCheckIcon: boolean;
79	99	}) {
80		const showUpdateDetails = action !== "none" \|\| showsEverything \|\| isDownloading;
	100	const showUpdateDetails = action !== "none" \|\| showsEverything \|\| run !== null \|\| isDownloading;
81	101	const iconStatus: DesktopUpdateStatusIconState = showCheckIcon
82	102	? "checking"
83	103	: isDownloading
84	104	? "downloading"
85		: showsEverything
86		? "everything"
87		: action === "install"
	105	: run !== null
	106	? run.localVersion !== null && action === "install"
88	107	? "downloaded"
89		: action === "download"
90		? "available"
91		: "idle";
	108	: "everything"
	109	: showsEverything
	110	? "everything"
	111	: action === "install"
	112	? "downloaded"
	113	: action === "download"
	114	? "available"
	115	: "idle";
92	116
93	117	return {
94	118	iconStatus,
...	...	@@ -117,11 +141,18 @@ function SidebarUpdateArchitectureWarningContent() {
117	141	);
118	142	}
119	143
	144	const toastUpdateError: ReportLocalUpdateError = (title, description) => {
	145	toastManager.add(stackedThreadToast({ type: "error", title, description }));
	146	};
	147
120	148	/** Downloads this app's update. Resolves true when it is ready to install;
121		failures toast and resolve false. */
	149	failures go to `reportError` (a toast by default) and resolve false. */
122	150	async function downloadLocalUpdate(
123	151	bridge: DesktopBridge,
124		{ notifyDownloaded }: { readonly notifyDownloaded: boolean },
	152	{
	153	notifyDownloaded,
	154	reportError = toastUpdateError,
	155	}: { readonly notifyDownloaded: boolean; readonly reportError?: ReportLocalUpdateError },
125	156	): Promise<boolean> {
126	157	try {
127	158	const result = await bridge.downloadUpdate();
...	...	@@ -131,60 +162,83 @@ async function downloadLocalUpdate(
131	162	const actionError = shouldToastDesktopUpdateActionResult(result)
132	163	? getDesktopUpdateActionError(result)
133	164	: null;
134		if (actionError) {
135		toastManager.add(
136		stackedThreadToast({
137		type: "error",
138		title: "Could not download update",
139		description: actionError,
140		}),
141		);
142		}
	165	if (actionError) reportError("Could not download update", actionError);
143	166	return result.completed;
144	167	} catch (error) {
145		toastManager.add(
146		stackedThreadToast({
147		type: "error",
148		title: "Could not start update download",
149		description: error instanceof Error ? error.message : "An unexpected error occurred.",
150		}),
	168	reportError(
	169	"Could not start update download",
	170	error instanceof Error ? error.message : "An unexpected error occurred.",
151	171	);
152	172	return false;
153	173	}
154	174	}
155	175
156		/** Installs this app's downloaded update, which relaunches the app.
157		Failures toast. A refused install (the download is not marked ready yet)
	176	/** Installs this app's downloaded update, which relaunches the app. Resolves
	177	true when the relaunch is under way. Failures go to `reportError` (a toast
	178	by default). A refused install (the download is not marked ready yet)
158	179	leaves the restart to the user. */
159		async function installLocalUpdate(bridge: DesktopBridge): Promise<void> {
	180	async function installLocalUpdate(
	181	bridge: DesktopBridge,
	182	reportError: ReportLocalUpdateError = toastUpdateError,
	183	): Promise<boolean> {
160	184	try {
161	185	const result = await bridge.installUpdate();
162	186	if (!result.accepted) {
163	187	showDesktopUpdateDownloadedToast(bridge, result.state);
164		return;
	188	return false;
165	189	}
166	190	const actionError = shouldToastDesktopUpdateActionResult(result)
167	191	? getDesktopUpdateActionError(result)
168	192	: null;
169		if (!actionError) return;
170		toastManager.add(
171		stackedThreadToast({
172		type: "error",
173		title: "Could not install update",
174		description: actionError,
175		}),
176		);
	193	if (!actionError) return true;
	194	reportError("Could not install update", actionError);
	195	return false;
177	196	} catch (error) {
178		toastManager.add(
179		stackedThreadToast({
180		type: "error",
181		title: "Could not install update",
182		description: error instanceof Error ? error.message : "An unexpected error occurred.",
183		}),
	197	reportError(
	198	"Could not install update",
	199	error instanceof Error ? error.message : "An unexpected error occurred.",
184	200	);
	201	return false;
185	202	}
186	203	}
187	204
	205	/** Tooltip lines for a running pass: this app first, then one per machine. */
	206	function useUpdateRunTooltip(
	207	run: UpdateEverythingRun \| null,
	208	state: DesktopUpdateState \| null,
	209	localReady: boolean,
	210	): string \| null {
	211	const statusesAtom = useMemo(
	212	() =>
	213	Atom.make((get) =>
	214	run === null
	215	? []
	216	: run.machines.map((machine) => ({
	217	label: machine.label,
	218	status: resolveUpdateRunMachineStatus(
	219	machine,
	220	run.phase,
	221	get(serverEnvironment.updateStateAtom(machine.environmentId)),
	222	),
	223	})),
	224	),
	225	[run],
	226	);
	227	const statuses = useAtomValue(statusesAtom);
	228	if (run === null) return null;
	229	const local =
	230	run.localVersion === null
	231	? null
	232	: state?.status === "downloading"
	233	? `downloading${state.downloadPercent === null ? "" : ` ${Math.floor(state.downloadPercent)}%`}`
	234	: run.phase === "installing"
	235	? "restarting"
	236	: localReady
	237	? "restarts last"
	238	: "waiting";
	239	return describeUpdateRun(statuses, local).join("\n");
	240	}
	241
188	242	/**
189	243	* The sidebar update button. With only a local app update it downloads, then
190	244	* installs. When other machines' servers or any providers are behind, one
...	...	@@ -198,7 +252,6 @@ export function SidebarUpdatePill() {
198	252	function SidebarUpdateControl() {
199	253	const state = useDesktopUpdateState();
200	254	const [isActionPending, setIsActionPending] = useState(false);
201		const [isUpdatingEverything, setIsUpdatingEverything] = useState(false);
202	255	const [isConfirmingEverything, setIsConfirmingEverything] = useState(false);
203	256	const [checkAnimationKey, setCheckAnimationKey] = useState(0);
204	257	const [isCheckAnimationLatched, setIsCheckAnimationLatched] = useState(false);
...	...	@@ -224,7 +277,9 @@ function SidebarUpdateControl() {
224	277	const everything = useUpdateEverything(localUpdateVersion);
225	278	const runsEverything = everything.hasUpdates && !isDownloading;
226	279	// Providers leave the list once queued, so the run keeps its own state.
227		const isRunningEverything = isUpdatingEverything && !isDownloading;
	280	const run = useUpdateEverythingRun();
	281	const isRunningEverything = run !== null;
	282	const runTooltip = useUpdateRunTooltip(run, state, action === "install");
228	283	const showCheckIcon = shouldShowDesktopUpdateCheckIcon({
229	284	isAnimationLatched: isCheckAnimationLatched,
230	285	isChecking: state?.status === "checking",
...	...	@@ -232,21 +287,23 @@ function SidebarUpdateControl() {
232	287	});
233	288	const { iconStatus, showUpdateDetails, showUpdateIconState } = resolveSidebarUpdatePresentation({
234	289	action,
235		showsEverything: runsEverything \|\| isRunningEverything,
	290	showsEverything: runsEverything,
	291	run,
236	292	isDownloading,
237	293	showCheckIcon,
238	294	});
239		const tooltip = isRunningEverything
240		? "Updating…"
241		: runsEverything
242		? everything.summary
243		: showUpdateDetails
244		? state
245		? getDesktopUpdateButtonTooltip(state)
246		: "Update available"
247		: showCheckIcon
248		? "Checking for updates…"
249		: "Check for updates";
	295	const tooltip =
	296	runTooltip !== null
	297	? runTooltip
	298	: runsEverything
	299	? everything.summary
	300	: showUpdateDetails
	301	? state
	302	? getDesktopUpdateButtonTooltip(state)
	303	: "Update available"
	304	: showCheckIcon
	305	? "Checking for updates…"
	306	: "Check for updates";
250	307	const disabled = showCheckIcon
251	308	? true
252	309	: runsEverything
...	...	@@ -254,11 +311,10 @@ function SidebarUpdateControl() {
254	311	: showUpdateDetails
255	312	? isDesktopUpdateButtonDisabled(state)
256	313	: !canCheckForUpdate(state);
257		const isInteractionDisabled = disabled \|\| isActionPending;
258		const showReleaseNotesPopover = shouldUseSidebarUpdateReleaseNotesPopover(
259		showUpdateDetails,
260		state,
261		);
	314	const isInteractionDisabled = disabled \|\| isActionPending \|\| isRunningEverything;
	315	// During a run the tooltip shows its progress instead of release notes.
	316	const showReleaseNotesPopover =
	317	!isRunningEverything && shouldUseSidebarUpdateReleaseNotesPopover(showUpdateDetails, state);
262	318
263	319	useEffect(() => {
264	320	if (!showReleaseNotesPopover) {
...	...	@@ -349,13 +405,17 @@ function SidebarUpdateControl() {
349	405	const bridge = window.desktopBridge;
350	406	setIsConfirmingEverything(false);
351	407	setIsActionPending(true);
352		setIsUpdatingEverything(true);
353	408	try {
354	409	await everything.run({
355	410	...(bridge && action === "download"
356		? { downloadLocal: () => downloadLocalUpdate(bridge, { notifyDownloaded: false }) }
	411	? {
	412	downloadLocal: (reportError) =>
	413	downloadLocalUpdate(bridge, { notifyDownloaded: false, reportError }),
	414	}
	415	: {}),
	416	...(bridge && action !== "none"
	417	? { installLocal: (reportError) => installLocalUpdate(bridge, reportError) }
357	418	: {}),
358		...(bridge && action !== "none" ? { installLocal: () => installLocalUpdate(bridge) } : {}),
359	419	});
360	420	} catch (error) {
361	421	toastManager.add(
...	...	@@ -366,7 +426,6 @@ function SidebarUpdateControl() {
366	426	}),
367	427	);
368	428	} finally {
369		setIsUpdatingEverything(false);
370	429	setIsActionPending(false);
371	430	}
372	431	}, [action, everything]);
...	...	@@ -381,7 +440,7 @@ function SidebarUpdateControl() {
381	440	}, [prefersReducedMotion, state?.status]);
382	441
383	442	// Web has no local app to check, so it only shows the button for remote work.
384		if (!isElectron && !runsEverything && !isUpdatingEverything) return null;
	443	if (!isElectron && !runsEverything && !isRunningEverything) return null;
385	444
386	445	const updateButton = (
387	446	<button
...	...	@@ -400,7 +459,7 @@ function SidebarUpdateControl() {
400	459	"text-(--sidebar-icon-color)",
401	460	!isInteractionDisabled && "hover:bg-sidebar-row-hover hover:text-sidebar-foreground",
402	461	),
403		((disabled && !showUpdateIconState) \|\| isRunningEverything) && "opacity-60",
	462	disabled && !showUpdateIconState && "opacity-60",
404	463	)}
405	464	onClick={handleAction}
406	465	onBlur={() => {
...	...	@@ -471,7 +530,7 @@ function SidebarUpdateControl() {
471	530	side="top"
472	531	variant={showUpdateDetails ? "glass" : "default"}
473	532	>
474		{tooltip}
	533	<span className="whitespace-pre-line">{tooltip}</span>
475	534	</TooltipPopup>
476	535	) : null}
477	536	</Tooltip>
...	...	@@ -506,6 +565,7 @@ function SidebarUpdateControl() {
506	565	summary={everything.summary}
507	566	lines={everything.lines}
508	567	restarts={everything.restarts}
	568	skipped={everything.skipped}
509	569	onConfirm={() => void runEverything()}
510	570	/>
511	571	</SidebarMenuItem>

apps/web/src/components/sidebar/UpdateEverythingDialog.tsx+10-1

...	...	@@ -13,7 +13,7 @@ import {
13	13	AlertDialogTitle,
14	14	} from "../ui/alert-dialog";
15	15	import { Button } from "../ui/button";
16		import type { RestartingMachine } from "./updateEverything.logic";
	16	import type { RestartingMachine, SkippedMachine } from "./updateEverything.logic";
17	17
18	18	/** A thread an agent is working on now. A restart stops these; queued threads wait. */
19	19	const isRunning = (status: string) =>
...	...	@@ -42,6 +42,8 @@ interface UpdateEverythingDialogProps {
42	42	readonly summary: string;
43	43	readonly lines: ReadonlyArray<string>;
44	44	readonly restarts: ReadonlyArray<RestartingMachine>;
	45	/** Behind machines the run leaves alone. */
	46	readonly skipped: ReadonlyArray<SkippedMachine>;
45	47	readonly onConfirm: () => void;
46	48	}
47	49
...	...	@@ -50,6 +52,7 @@ interface UpdateEverythingDialogProps {
50	52	* restarts stop, and how many can resume afterwards because "Continue threads
51	53	* after restarts" is on for their server. Recovery still skips some, such as a
52	54	* thread waiting on the user, so the text says "can resume", not "will".
	55	* Machines the run skips are listed with the reason, so none drop silently.
53	56	*/
54	57	export function UpdateEverythingDialog({
55	58	open,
...	...	@@ -70,6 +73,7 @@ function UpdateEverythingDialogContent({
70	73	summary,
71	74	lines,
72	75	restarts,
	76	skipped,
73	77	onConfirm,
74	78	}: Omit<UpdateEverythingDialogProps, "open" \| "onOpenChange">) {
75	79	const stopped = useAtomValue(
...	...	@@ -107,6 +111,11 @@ function UpdateEverythingDialogContent({
107	111	: `${threads} will be interrupted. ${continuing} can resume after the restart.`}
108	112	</p>
109	113	) : null}
	114	{skipped.length > 0 ? (
	115	<p className="whitespace-pre-line pt-2 text-sm">
	116	{skipped.map((machine) => `Skips ${machine.label}: ${machine.reason}`).join("\n")}
	117	</p>
	118	) : null}
110	119	</AlertDialogHeader>
111	120	<AlertDialogFooter>
112	121	<AlertDialogClose render={<Button variant="outline" />}>Cancel</AlertDialogClose>

apps/web/src/components/sidebar/updateEverything.logic.test.ts+123-15

...	...	@@ -4,9 +4,13 @@ import { describe, expect, it } from "vite-plus/test";
4	4	import {
5	5	collectRestartingMachines,
6	6	collectServerUpdateTargets,
	7	collectUpdateEverythingResult,
7	8	describeUpdateEverything,
	9	getUpdateEverythingToastView,
	10	resolveUpdateRunMachineStatus,
8	11	runUpdateEverything,
9	12	type UpdateEverythingMachine,
	13	type UpdateRunMachine,
10	14	} from "./updateEverything.logic";
11	15
12	16	function machine(
...	...	@@ -41,16 +45,26 @@ function machine(
41	45	}
42	46
43	47	describe("collectServerUpdateTargets", () => {
44		it("updates only remote servers that can move to the target on their channel", () => {
45		const targets = collectServerUpdateTargets(
	48	it("updates remote servers that can move to the target and names every skipped one", () => {
	49	const { targets, skipped } = collectServerUpdateTargets(
46	50	[
47	51	machine("this-app", "0.0.40", "desktop-managed", { hostedByThisApp: true }),
48	52	machine("service", "0.0.40", "boot-service"),
49	53	machine("nightly-service", "0.0.49-nightly.20261001.3", "boot-service"),
50	54	machine("other-desktop", "0.0.49-nightly.20261001.3", "desktop-managed"),
51	55	machine("offline", "0.0.40", "boot-service", { connected: false }),
	56	machine("never-connected", "0.0.40", "boot-service", { serverConfig: null }),
52	57	machine("no-scope", "0.0.40", "boot-service", { canMaintain: false }),
	58	machine("busy", "0.0.40", "boot-service", {
	59	updateState: {
	60	status: "running",
	61	stage: "downloading",
	62	fromVersion: "0.0.40",
	63	targetVersion: "0.0.50",
	64	},
	65	}),
53	66	machine("current", "0.0.50", "boot-service"),
	67	machine("current-offline", "0.0.50", "boot-service", { connected: false }),
54	68	machine("manual", "0.0.40", undefined),
55	69	],
56	70	"0.0.50",
...	...	@@ -60,6 +74,14 @@ describe("collectServerUpdateTargets", () => {
60	74	["service", "0.0.50"],
61	75	["other-desktop", "0.0.50"],
62	76	]);
	77	expect(skipped.map((entry) => [entry.label, entry.reason])).toEqual([
	78	["nightly-service", "on the nightly channel"],
	79	["offline", "not connected"],
	80	["never-connected", "not connected"],
	81	["no-scope", "no permission"],
	82	["busy", "already updating"],
	83	["manual", "needs a manual update"],
	84	]);
63	85	});
64	86	});
65	87
...	...	@@ -109,37 +131,35 @@ describe("collectRestartingMachines", () => {
109	131	});
110	132
111	133	describe("runUpdateEverything", () => {
112		it("restarts servers after providers finish and installs this app last", async () => {
	134	it("restarts servers after providers finish and returns every result", async () => {
113	135	const calls: string[] = [];
114		let finishProviders = () => {};
	136	let finishProviders = (_value: string) => {};
115	137	const run = runUpdateEverything({
116	138	downloadLocal: async () => {
117	139	calls.push("download");
118	140	return true;
119	141	},
120	142	updateProviders: () =>
121		new Promise<void>((resolve) => {
	143	new Promise<string>((resolve) => {
122	144	calls.push("providers");
123	145	finishProviders = resolve;
124	146	}),
125	147	updateServers: async () => {
126	148	calls.push("servers");
127		},
128		installLocal: async () => {
129		calls.push("install");
	149	return 2;
130	150	},
131	151	});
132	152
133	153	await new Promise((resolve) => setTimeout(resolve, 0));
134	154	expect(calls).toEqual(["download", "providers"]);
135		finishProviders();
136		await run;
137		expect(calls).toEqual(["download", "providers", "servers", "install"]);
	155	finishProviders("provider results");
	156	expect(await run).toEqual({ localReady: true, providers: "provider results", servers: 2 });
	157	expect(calls).toEqual(["download", "providers", "servers"]);
138	158	});
139	159
140	160	it("still updates remote machines when the local download fails", async () => {
141	161	const calls: string[] = [];
142		await runUpdateEverything({
	162	const result = await runUpdateEverything({
143	163	downloadLocal: async () => false,
144	164	updateProviders: async () => {
145	165	calls.push("providers");
...	...	@@ -147,12 +167,100 @@ describe("runUpdateEverything", () => {
147	167	updateServers: async () => {
148	168	calls.push("servers");
149	169	},
150		installLocal: async () => {
151		calls.push("install");
152		},
153	170	});
154	171
155	172	expect(calls).toEqual(["providers", "servers"]);
	173	expect(result.localReady).toBe(false);
	174	});
	175	});
	176
	177	describe("resolveUpdateRunMachineStatus", () => {
	178	const runMachine = (overrides: Partial<UpdateRunMachine> = {}): UpdateRunMachine => ({
	179	environmentId: EnvironmentId.make("alvin"),
	180	label: "alvin",
	181	updatesProviders: true,
	182	updatesServer: true,
	183	result: null,
	184	...overrides,
	185	});
	186	const running = (stage: "downloading" \| "installing" \| "resuming") =>
	187	({ status: "running", stage, fromVersion: "0.0.40", targetVersion: "0.0.50" }) as const;
	188
	189	it("follows the run phase, then the server's own update stage", () => {
	190	const idle = { status: "idle" } as const;
	191	expect(resolveUpdateRunMachineStatus(runMachine(), "providers", idle)).toBe("providers");
	192	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", idle)).toBe("waiting");
	193	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", running("installing"))).toBe(
	194	"downloading",
	195	);
	196	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", running("resuming"))).toBe(
	197	"restarting",
	198	);
	199	expect(
	200	resolveUpdateRunMachineStatus(
	201	runMachine({ updatesProviders: false }),
	202	"providers",
	203	running("downloading"),
	204	),
	205	).toBe("downloading");
	206	expect(resolveUpdateRunMachineStatus(runMachine({ result: "failed" }), "servers", idle)).toBe(
	207	"failed",
	208	);
	209	});
	210	});
	211
	212	describe("update everything summary", () => {
	213	const result = collectUpdateEverythingResult({
	214	localVersion: "0.0.50",
	215	localFailures: ["Could not download update: Network error."],
	216	servers: [
	217	{ label: "alvin", outcome: { status: "failed", message: "Download failed." } },
	218	{ label: "box", outcome: { status: "updated" } },
	219	{ label: "studio", outcome: { status: "updated" } },
	220	{ label: "busy", outcome: { status: "skipped" } },
	221	],
	222	providers: [
	223	{ status: "updated" },
	224	{ status: "interrupted" },
	225	{ status: "failed", line: "box · Codex: npm exited with code 1." },
	226	],
	227	});
	228
	229	it("counts updates and names the machine of each failure", () => {
	230	expect(result).toEqual({
	231	localVersion: "0.0.50",
	232	updatedServers: 2,
	233	updatedProviders: 1,
	234	failures: [
	235	"This app: Could not download update: Network error.",
	236	"alvin: Download failed.",
	237	"box · Codex: npm exited with code 1.",
	238	],
	239	});
	240	});
	241
	242	it("includes this app only when the relaunch landed on its version", () => {
	243	expect(getUpdateEverythingToastView(result, "0.0.50")).toEqual({
	244	type: "error",
	245	title: "Updated this app, 2 servers, and 1 provider",
	246	description: [
	247	"This app: Could not download update: Network error.",
	248	"alvin: Download failed.",
	249	"box · Codex: npm exited with code 1.",
	250	].join("\n"),
	251	});
	252	expect(getUpdateEverythingToastView({ ...result, failures: [] }, "0.0.40")).toEqual({
	253	type: "success",
	254	title: "Updated 2 servers and 1 provider",
	255	});
	256	});
	257
	258	it("reports a failure when nothing updated, and nothing when nothing finished", () => {
	259	const empty = { localVersion: null, updatedServers: 0, updatedProviders: 0 };
	260	expect(
	261	getUpdateEverythingToastView({ ...empty, failures: ["alvin: offline"] }, "0.0.50"),
	262	).toEqual({ type: "error", title: "Update failed", description: "alvin: offline" });
	263	expect(getUpdateEverythingToastView({ ...empty, failures: [] }, "0.0.50")).toBeNull();
156	264	});
157	265	});
158	266

apps/web/src/components/sidebar/updateEverything.logic.ts+215-45

...	...	@@ -8,7 +8,7 @@ import {
8	8	supportsDesktopAppUpdate,
9	9	supportsServerUpdateThreadContinuation,
10	10	} from "~/versionSkew";
11		import type { ServerUpdateTarget } from "../ServerUpdateAction";
	11	import type { ServerUpdateOutcome, ServerUpdateTarget } from "../ServerUpdateAction";
12	12
13	13	export interface UpdateEverythingMachine {
14	14	readonly environmentId: EnvironmentId;
...	...	@@ -63,78 +63,248 @@ export function collectRestartingMachines(
63	63	});
64	64	}
65	65
	66	/** A machine the run leaves alone, with a short reason such as "not connected". */
	67	export interface SkippedMachine {
	68	readonly environmentId: EnvironmentId;
	69	readonly label: string;
	70	readonly reason: string;
	71	}
	72
66	73	/**
67	74	* The servers that "update everything" moves to `targetVersion`: connected,
68		* permitted, able to update over RPC, and behind the target. Two kinds stay
69		* out. Servers this app hosts would relaunch the app mid-run, and the app's
70		* own install updates them anyway. A service on another release channel
71		* stays there; a desktop-managed server follows its own app's update feed, so
72		* the channel rule does not apply to it.
	75	* permitted, able to update over RPC, behind the target, and on its release
	76	* channel. Every other behind server comes back in `skipped` with a reason for
	77	* the confirmation; a machine that never connected has an unknown version, so
	78	* it counts as skipped. Two kinds are in neither list: servers already on the
	79	* target, and servers this app hosts, which would relaunch the app mid-run
	80	* and update with the app's own install anyway. A desktop-managed server
	81	* follows its own app's update feed, so the channel rule does not apply to it.
73	82	*/
74	83	export function collectServerUpdateTargets(
75	84	machines: ReadonlyArray<UpdateEverythingMachine>,
76	85	targetVersion: string,
77		): ServerUpdateTarget[] {
78		return machines.flatMap((machine) => {
	86	): { readonly targets: ServerUpdateTarget[]; readonly skipped: SkippedMachine[] } {
	87	const targets: ServerUpdateTarget[] = [];
	88	const skipped: SkippedMachine[] = [];
	89	for (const machine of machines) {
79	90	const config = machine.serverConfig;
	91	if (machine.hostedByThisApp) continue;
80	92	if (
81		config === null \|\|
82		!machine.connected \|\|
83		machine.hostedByThisApp \|\|
84		!machine.canMaintain \|\|
85		machine.updateState.status === "running"
	93	config !== null &&
	94	resolveVersionMismatch(config.environment.serverVersion, targetVersion) === null
86	95	) {
87		return [];
	96	continue;
	97	}
	98	const skip = (reason: string) =>
	99	skipped.push({ environmentId: machine.environmentId, label: machine.label, reason });
	100	if (config === null \|\| !machine.connected) {
	101	skip("not connected");
	102	continue;
	103	}
	104	if (!machine.canMaintain) {
	105	skip("no permission");
	106	continue;
	107	}
	108	if (machine.updateState.status === "running") {
	109	skip("already updating");
	110	continue;
88	111	}
89	112	const selfUpdate = resolveServerSelfUpdateCapability(config);
90	113	const desktopAppUpdate = supportsDesktopAppUpdate(config);
91		if (selfUpdate === null \|\| (selfUpdate === "desktop-managed" && !desktopAppUpdate)) {
92		return [];
	114	if (selfUpdate === null) {
	115	skip("needs a manual update");
	116	continue;
	117	}
	118	if (selfUpdate === "desktop-managed" && !desktopAppUpdate) {
	119	skip("needs a newer desktop app");
	120	continue;
93	121	}
94	122	const serverVersion = config.environment.serverVersion;
95		if (resolveVersionMismatch(serverVersion, targetVersion) === null) return [];
96	123	if (
97	124	selfUpdate !== "desktop-managed" &&
98	125	cliReleaseChannelOf(serverVersion) !== cliReleaseChannelOf(targetVersion)
99	126	) {
100		return [];
	127	skip(`on the ${cliReleaseChannelOf(serverVersion)} channel`);
	128	continue;
101	129	}
102		return [
103		{
104		environmentId: machine.environmentId,
105		serverLabel: machine.label,
106		selfUpdate,
107		installation: config.environment.capabilities.serverInstallation,
108		desktopAppUpdate,
109		threadContinuation: supportsServerUpdateThreadContinuation(config),
110		continueThreadsAfterServerUpdate: config.settings.continueThreadsAfterServerUpdate ?? false,
111		targetVersion,
112		},
113		];
114		});
	130	targets.push({
	131	environmentId: machine.environmentId,
	132	serverLabel: machine.label,
	133	selfUpdate,
	134	installation: config.environment.capabilities.serverInstallation,
	135	desktopAppUpdate,
	136	threadContinuation: supportsServerUpdateThreadContinuation(config),
	137	continueThreadsAfterServerUpdate: config.settings.continueThreadsAfterServerUpdate ?? false,
	138	targetVersion,
	139	});
	140	}
	141	return { targets, skipped };
115	142	}
116	143
117		export interface UpdateEverythingSteps {
118		/** Downloads this app's update. Resolves true when it is ready to install. */
	144	export interface UpdateEverythingSteps<Providers, Servers> {
	145	/** Downloads this app's update. Resolves true when it is ready to install.
	146	Without it, `localReady` is true: there is nothing to download. */
119	147	readonly downloadLocal?: () => Promise<boolean>;
120		readonly updateProviders: () => Promise<void>;
121		readonly updateServers: () => Promise<void>;
122		/** Installs this app's update. The relaunch ends the run. */
123		readonly installLocal?: () => Promise<void>;
	148	readonly updateProviders: () => Promise<Providers>;
	149	readonly updateServers: () => Promise<Servers>;
124	150	}
125	151
126	152	/**
127		* Runs one "update everything" pass. Providers
128		* finish before any server restarts, so no install is cut off. The local download runs alongside the
129		* remote work. This app installs last because its relaunch ends the run; a
130		* failed download skips only that install. Steps report their own results.
	153	* Runs the remote half of one "update everything" pass and returns each
	154	* step's results. Providers finish before any server restarts, so no install
	155	* is cut off. The local download runs alongside the remote work. The caller
	156	* installs this app afterwards when `localReady`, because the relaunch ends
	157	* the run; a failed download skips only that install.
131	158	*/
132		export async function runUpdateEverything(steps: UpdateEverythingSteps): Promise<void> {
133		const [localReady] = await Promise.all([
	159	export async function runUpdateEverything<Providers, Servers>(
	160	steps: UpdateEverythingSteps<Providers, Servers>,
	161	): Promise<{
	162	readonly localReady: boolean;
	163	readonly providers: Providers;
	164	readonly servers: Servers;
	165	}> {
	166	const [localReady, remote] = await Promise.all([
134	167	steps.downloadLocal?.() ?? Promise.resolve(true),
135		steps.updateProviders().then(steps.updateServers),
	168	steps
	169	.updateProviders()
	170	.then(async (providers) => ({ providers, servers: await steps.updateServers() })),
136	171	]);
137		if (localReady) await steps.installLocal?.();
	172	return { localReady, ...remote };
	173	}
	174
	175	/** The current state of one machine in a running pass. */
	176	export type UpdateRunMachineStatus =
	177	\| "providers"
	178	\| "waiting"
	179	\| "downloading"
	180	\| "restarting"
	181	\| "done"
	182	\| "failed";
	183
	184	/** One machine in a running "update everything" pass. */
	185	export interface UpdateRunMachine {
	186	readonly environmentId: EnvironmentId;
	187	readonly label: string;
	188	readonly updatesProviders: boolean;
	189	readonly updatesServer: boolean;
	190	/** Set when every update for this machine has finished. */
	191	readonly result: "done" \| "failed" \| null;
	192	}
	193
	194	/** A running "update everything" pass, kept while it runs. */
	195	export interface UpdateEverythingRun {
	196	/** The version this app installs last, or null when it does not install. */
	197	readonly localVersion: string \| null;
	198	/** Providers update first. Servers update after every provider finished. */
	199	readonly phase: "providers" \| "servers" \| "installing";
	200	readonly machines: ReadonlyArray<UpdateRunMachine>;
	201	}
	202
	203	/** Status of one machine, from the run and that server's own update state. */
	204	export function resolveUpdateRunMachineStatus(
	205	machine: UpdateRunMachine,
	206	phase: UpdateEverythingRun["phase"],
	207	serverState: ServerUpdateState,
	208	): UpdateRunMachineStatus {
	209	if (machine.result !== null) return machine.result;
	210	if (phase === "providers" && machine.updatesProviders) return "providers";
	211	if (machine.updatesServer && serverState.status === "running") {
	212	return serverState.stage === "resuming" ? "restarting" : "downloading";
	213	}
	214	return "waiting";
	215	}
	216
	217	const RUN_STATUS_LABELS: Record<UpdateRunMachineStatus, string> = {
	218	providers: "updating providers",
	219	waiting: "waiting",
	220	downloading: "downloading",
	221	restarting: "restarting",
	222	done: "done",
	223	failed: "failed",
	224	};
	225
	226	/**
	227	* Tooltip lines for a running pass, such as "alvin: restarting". `local` is
	228	* this app's own state, such as "downloading 40%", or null when it does not
	229	* install.
	230	*/
	231	export function describeUpdateRun(
	232	machines: ReadonlyArray<{ readonly label: string; readonly status: UpdateRunMachineStatus }>,
	233	local: string \| null,
	234	): ReadonlyArray<string> {
	235	return [
	236	"Updating…",
	237	...(local === null ? [] : [`This app: ${local}`]),
	238	...machines.map((machine) => `${machine.label}: ${RUN_STATUS_LABELS[machine.status]}`),
	239	];
	240	}
	241
	242	/** What one pass did. It is saved before this app installs, so it outlives the relaunch. */
	243	export interface UpdateEverythingResult {
	244	/** The version this app installed last, or null when it did not install. */
	245	readonly localVersion: string \| null;
	246	readonly updatedServers: number;
	247	readonly updatedProviders: number;
	248	/** One line per failure, each naming its machine. */
	249	readonly failures: ReadonlyArray<string>;
	250	}
	251
	252	/** Counts what a pass updated and lists each failure with its machine. */
	253	export function collectUpdateEverythingResult(input: {
	254	readonly localVersion: string \| null;
	255	/** This app's own download or install failures. */
	256	readonly localFailures: ReadonlyArray<string>;
	257	readonly servers: ReadonlyArray<{
	258	readonly label: string;
	259	readonly outcome: ServerUpdateOutcome;
	260	}>;
	261	readonly providers: ReadonlyArray<
	262	\| { readonly status: "interrupted" \| "updated" }
	263	\| { readonly status: "failed"; readonly line: string }
	264	>;
	265	}): UpdateEverythingResult {
	266	return {
	267	localVersion: input.localVersion,
	268	updatedServers: input.servers.filter(({ outcome }) => outcome.status === "updated").length,
	269	updatedProviders: input.providers.filter((outcome) => outcome.status === "updated").length,
	270	failures: [
	271	...input.localFailures.map((failure) => `This app: ${failure}`),
	272	...input.servers.flatMap(({ label, outcome }) =>
	273	outcome.status === "failed" ? [`${label}: ${outcome.message}`] : [],
	274	),
	275	...input.providers.flatMap((outcome) => (outcome.status === "failed" ? [outcome.line] : [])),
	276	],
	277	};
	278	}
	279
	280	/**
	281	* The one toast for a finished pass, such as "Updated this app, 8 servers,
	282	* and 3 providers" with a line per failure. `appVersion` is the version this
	283	* app runs now: after the relaunch it tells whether the local install
	284	* applied. Null when nothing finished, for example every request was
	285	* interrupted.
	286	*/
	287	export function getUpdateEverythingToastView(
	288	result: UpdateEverythingResult,
	289	appVersion: string,
	290	): {
	291	readonly type: "success" \| "error";
	292	readonly title: string;
	293	readonly description?: string;
	294	} \| null {
	295	const parts = [
	296	result.localVersion !== null && result.localVersion === appVersion ? "this app" : null,
	297	countLabel(result.updatedServers, "server"),
	298	countLabel(result.updatedProviders, "provider"),
	299	].filter((part) => part !== null);
	300	if (result.failures.length === 0) {
	301	return parts.length === 0 ? null : { type: "success", title: `Updated ${joinList(parts)}` };
	302	}
	303	return {
	304	type: "error",
	305	title: parts.length === 0 ? "Update failed" : `Updated ${joinList(parts)}`,
	306	description: result.failures.join("\n"),
	307	};
138	308	}
139	309
140	310	export interface UpdateEverythingPlan {

apps/web/src/components/sidebar/updateEverythingRun.tsx created+142

...	...	@@ -0,0 +1,142 @@
	1	import { useAtomValue } from "@effect/atom-react";
	2	import type { EnvironmentId } from "@t3tools/contracts";
	3	import * as Option from "effect/Option";
	4	import * as Schema from "effect/Schema";
	5	import { Atom } from "effect/reactivity";
	6	import { useEffect } from "react";
	7
	8	import { APP_VERSION } from "~/branding";
	9	import { appAtomRegistry } from "~/rpc/atomRegistry";
	10	import { useDesktopUpdateState } from "~/state/desktopUpdate";
	11	import { stackedThreadToast, toastManager } from "../ui/toast";
	12	import {
	13	getUpdateEverythingToastView,
	14	type UpdateEverythingResult,
	15	type UpdateEverythingRun,
	16	} from "./updateEverything.logic";
	17
	18	/** The running "update everything" pass, or null. Outlives the sidebar button. */
	19	const updateEverythingRunAtom = Atom.make<UpdateEverythingRun \| null>(null).pipe(
	20	Atom.keepAlive,
	21	Atom.withLabel("update-everything-run"),
	22	);
	23
	24	export function useUpdateEverythingRun(): UpdateEverythingRun \| null {
	25	return useAtomValue(updateEverythingRunAtom);
	26	}
	27
	28	/** Starts, changes, or ends (null) the running pass. */
	29	export function setUpdateEverythingRun(
	30	update: (run: UpdateEverythingRun \| null) => UpdateEverythingRun \| null,
	31	): void {
	32	appAtomRegistry.update(updateEverythingRunAtom, update);
	33	}
	34
	35	/** Records the result of every update for one machine in the running pass. */
	36	export function settleUpdateRunMachine(environmentId: EnvironmentId, failed: boolean): void {
	37	setUpdateEverythingRun(
	38	(run) =>
	39	run && {
	40	...run,
	41	machines: run.machines.map((machine) =>
	42	machine.environmentId === environmentId
	43	? { ...machine, result: failed ? "failed" : "done" }
	44	: machine,
	45	),
	46	},
	47	);
	48	}
	49
	50	const SAVED_RESULT_KEY = "t3code:update-everything-result:v1";
	51	// A result older than this belongs to an install that never relaunched the app.
	52	const SAVED_RESULT_MAX_AGE_MS = 10 * 60 * 1000;
	53
	54	const SavedResult = Schema.Struct({
	55	savedAt: Schema.Number,
	56	localVersion: Schema.NullOr(Schema.String),
	57	updatedServers: Schema.Number,
	58	updatedProviders: Schema.Number,
	59	failures: Schema.Array(Schema.String),
	60	});
	61	const decodeSavedResult = Schema.decodeUnknownOption(Schema.fromJsonString(SavedResult));
	62
	63	/** Keeps a result through this app's relaunch. Call it right before the install. */
	64	export function saveUpdateEverythingResult(result: UpdateEverythingResult): void {
	65	try {
	66	window.localStorage.setItem(
	67	SAVED_RESULT_KEY,
	68	JSON.stringify({ ...result, savedAt: Date.now() } satisfies typeof SavedResult.Type),
	69	);
	70	} catch {
	71	// Without storage the summary is lost with the relaunch; the update still runs.
	72	}
	73	}
	74
	75	export function clearSavedUpdateEverythingResult(): void {
	76	try {
	77	window.localStorage.removeItem(SAVED_RESULT_KEY);
	78	} catch {
	79	// Nothing to clear.
	80	}
	81	}
	82
	83	function takeSavedUpdateEverythingResult(): UpdateEverythingResult \| null {
	84	try {
	85	const raw = window.localStorage.getItem(SAVED_RESULT_KEY);
	86	if (raw === null) return null;
	87	window.localStorage.removeItem(SAVED_RESULT_KEY);
	88	return Option.getOrNull(
	89	Option.filter(
	90	decodeSavedResult(raw),
	91	(saved) => Date.now() - saved.savedAt < SAVED_RESULT_MAX_AGE_MS,
	92	),
	93	);
	94	} catch {
	95	return null;
	96	}
	97	}
	98
	99	/** Shows the one summary toast for a finished pass. */
	100	export function showUpdateEverythingResult(result: UpdateEverythingResult): void {
	101	const view = getUpdateEverythingToastView(result, APP_VERSION);
	102	if (!view) return;
	103	toastManager.add(
	104	stackedThreadToast({
	105	type: view.type,
	106	title: view.title,
	107	...(view.description
	108	? { description: <span className="whitespace-pre-line">{view.description}</span> }
	109	: {}),
	110	}),
	111	);
	112	}
	113
	114	let notifiedNotAppliedVersion: string \| null = null;
	115
	116	/**
	117	* Desktop launch notices for updates: the summary of a pass that ended with
	118	* this app's relaunch, and a warning when the last install did not apply.
	119	* Mount once at the app root.
	120	*/
	121	export function DesktopUpdateLaunchNotice() {
	122	const notAppliedVersion = useDesktopUpdateState()?.notAppliedVersion ?? null;
	123
	124	useEffect(() => {
	125	const result = takeSavedUpdateEverythingResult();
	126	if (result) showUpdateEverythingResult(result);
	127	}, []);
	128
	129	useEffect(() => {
	130	if (notAppliedVersion === null \|\| notifiedNotAppliedVersion === notAppliedVersion) return;
	131	notifiedNotAppliedVersion = notAppliedVersion;
	132	toastManager.add(
	133	stackedThreadToast({
	134	type: "error",
	135	title: "Update did not apply",
	136	description: `T3 Code is still on ${APP_VERSION}. Install ${notAppliedVersion} again from the update button.`,
	137	}),
	138	);
	139	}, [notAppliedVersion]);
	140
	141	return null;
	142	}

apps/web/src/components/sidebar/useUpdateEverything.ts+126-10

...	...	@@ -10,21 +10,37 @@ import { useEnvironments } from "~/state/environments";
10	10	import { useNpmReleasePublished } from "~/state/npmRelease";
11	11	import { serverEnvironment } from "~/state/server";
12	12	import { environmentSession } from "~/state/session";
	13	import { getProviderUpdateRunOutcome } from "../ProviderUpdateLaunchNotification.logic";
13	14	import { useProviderUpdates } from "../ProviderUpdatesAction";
14	15	import { canUpdateServer, useServerUpdate } from "../ServerUpdateAction";
15	16	import {
16	17	collectRestartingMachines,
17	18	collectServerUpdateTargets,
	19	collectUpdateEverythingResult,
18	20	describeUpdateEverything,
19	21	runUpdateEverything,
	22	type SkippedMachine,
	23	type UpdateRunMachine,
20	24	} from "./updateEverything.logic";
	25	import {
	26	clearSavedUpdateEverythingResult,
	27	saveUpdateEverythingResult,
	28	setUpdateEverythingRun,
	29	settleUpdateRunMachine,
	30	showUpdateEverythingResult,
	31	} from "./updateEverythingRun";
	32
	33	/** Where this app's own download or install reports a failure. */
	34	export type ReportLocalUpdateError = (title: string, description: string) => void;
21	35
22	36	/**
23	37	* The remote half of the sidebar update button: outdated providers on every
24	38	* connected machine, and servers behind the version this client runs or is
25	39	* about to install (`localUpdateVersion`). Servers that install from npm appear
26	40	* only once npm confirms that version. `run` performs the whole pass, including
27		* this app's own download and install when given.
	41	* this app's own download and install when given. It updates quietly, tracks
	42	* each machine in the run state, and reports one summary toast at the end.
	43	* When this app installs, the summary waits for the relaunch.
28	44	*/
29	45	export function useUpdateEverything(localUpdateVersion: string \| null) {
30	46	const targetVersion = localUpdateVersion ?? APP_VERSION;
...	...	@@ -50,7 +66,10 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
50	66	[environments],
51	67	);
52	68	const machines = useAtomValue(machinesAtom);
53		const behindServers = collectServerUpdateTargets(machines, targetVersion);
	69	const { targets: behindServers, skipped: unreachable } = collectServerUpdateTargets(
	70	machines,
	71	targetVersion,
	72	);
54	73	// A desktop-managed server installs from its own app's update feed, so only
55	74	// servers that download t3 from npm wait for npm to publish the version.
56	75	const needsNpm = behindServers.some((server) => server.selfUpdate !== "desktop-managed");
...	...	@@ -58,6 +77,16 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
58	77	const servers = published
59	78	? behindServers
60	79	: behindServers.filter((server) => server.selfUpdate === "desktop-managed");
	80	const skipped: ReadonlyArray<SkippedMachine> = [
	81	...unreachable,
	82	...behindServers
	83	.filter((server) => !servers.includes(server))
	84	.map((server) => ({
	85	environmentId: server.environmentId,
	86	label: server.serverLabel,
	87	reason: `t3@${targetVersion} is not on npm yet`,
	88	})),
	89	];
61	90	const restarts = collectRestartingMachines(machines, servers, localUpdateVersion !== null);
62	91	const providers = useProviderUpdates();
63	92	const updateServer = useServerUpdate();
...	...	@@ -82,13 +111,100 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
82	111	lines,
83	112	/** The servers this run restarts, which stops their running threads. */
84	113	restarts,
85		run: (local: { downloadLocal?: () => Promise<boolean>; installLocal?: () => Promise<void> }) =>
86		runUpdateEverything({
87		...local,
88		updateProviders: providers.updateAll,
89		updateServers: async () => {
90		await Promise.all(servers.map((server) => updateServer(server)));
91		},
92		}),
	114	/** Behind machines the run leaves alone, with the reason. */
	115	skipped,
	116	/** Runs the pass. `installLocal` resolves true when this app is relaunching.
	117	Both report failures to the pass, which lists them in its summary. */
	118	run: async ({
	119	downloadLocal,
	120	installLocal,
	121	}: {
	122	downloadLocal?: (reportError: ReportLocalUpdateError) => Promise<boolean>;
	123	installLocal?: (reportError: ReportLocalUpdateError) => Promise<boolean>;
	124	}) => {
	125	const localFailures: Array<string> = [];
	126	const reportLocalError: ReportLocalUpdateError = (title, description) => {
	127	localFailures.push(`${title}: ${description}`);
	128	};
	129	const machinesById = new Map<UpdateRunMachine["environmentId"], UpdateRunMachine>();
	130	for (const machine of providers.machines) {
	131	machinesById.set(machine.environmentId, {
	132	environmentId: machine.environmentId,
	133	label: machine.label,
	134	updatesProviders: true,
	135	updatesServer: false,
	136	result: null,
	137	});
	138	}
	139	for (const server of servers) {
	140	const existing = machinesById.get(server.environmentId);
	141	machinesById.set(server.environmentId, {
	142	environmentId: server.environmentId,
	143	label: server.serverLabel,
	144	updatesProviders: existing?.updatesProviders ?? false,
	145	updatesServer: true,
	146	result: null,
	147	});
	148	}
	149	const localVersion = installLocal ? localUpdateVersion : null;
	150	setUpdateEverythingRun(() => ({
	151	localVersion,
	152	phase: "providers",
	153	machines: Array.from(machinesById.values()),
	154	}));
	155	const failedProviderMachines = new Set<UpdateRunMachine["environmentId"]>();
	156	try {
	157	const outcome = await runUpdateEverything({
	158	...(downloadLocal ? { downloadLocal: () => downloadLocal(reportLocalError) } : {}),
	159	updateProviders: async () => {
	160	const runs = await providers.updateAll({ quiet: true });
	161	const outcomes = runs.map((run) => {
	162	const runOutcome = getProviderUpdateRunOutcome(run);
	163	// An interrupted update did not finish either, so its row is not "done".
	164	if (runOutcome.status !== "updated") failedProviderMachines.add(run.environmentId);
	165	return runOutcome;
	166	});
	167	for (const machine of machinesById.values()) {
	168	if (!machine.updatesServer) {
	169	settleUpdateRunMachine(
	170	machine.environmentId,
	171	failedProviderMachines.has(machine.environmentId),
	172	);
	173	}
	174	}
	175	setUpdateEverythingRun((run) => run && { ...run, phase: "servers" });
	176	return outcomes;
	177	},
	178	updateServers: () =>
	179	Promise.all(
	180	servers.map(async (server) => {
	181	const serverOutcome = await updateServer(server, { quiet: true });
	182	settleUpdateRunMachine(
	183	server.environmentId,
	184	serverOutcome.status === "failed" \|\|
	185	failedProviderMachines.has(server.environmentId),
	186	);
	187	return { label: server.serverLabel, outcome: serverOutcome };
	188	}),
	189	),
	190	});
	191	const result = () =>
	192	collectUpdateEverythingResult({
	193	localVersion: null,
	194	localFailures,
	195	servers: outcome.servers,
	196	providers: outcome.providers,
	197	});
	198	if (outcome.localReady && installLocal) {
	199	setUpdateEverythingRun((run) => run && { ...run, phase: "installing" });
	200	saveUpdateEverythingResult({ ...result(), localVersion });
	201	if (await installLocal(reportLocalError)) return;
	202	clearSavedUpdateEverythingResult();
	203	}
	204	showUpdateEverythingResult(result());
	205	} finally {
	206	setUpdateEverythingRun(() => null);
	207	}
	208	},
93	209	};
94	210	}

apps/web/src/routes/__root.tsx+2

...	...	@@ -30,6 +30,7 @@ import { DesktopAppActivationCoordinator } from "../components/desktop/DesktopAp
30	30	import { DesktopWebLinkCoordinator } from "../components/desktop/DesktopWebLinkCoordinator";
31	31	import { RunningThreadKeepAlive } from "../components/desktop/RunningThreadKeepAlive";
32	32	import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification";
	33	import { DesktopUpdateLaunchNotice } from "../components/sidebar/updateEverythingRun";
33	34	import { NightlyMobileBetaNotice } from "../components/NightlyMobileBeta";
34	35	import { LegacyThreadMigrationToast } from "../components/LegacyThreadMigrationToast";
35	36	import { ThreadNotificationCoordinator } from "../components/ThreadNotificationCoordinator";
...	...	@@ -265,6 +266,7 @@ function RootRouteView() {
265	266	<EventRouter skipInitialBootstrapNavigation={returningFromWelcomeRef.current} />
266	267	) : null}
267	268	{primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null}
	269	{isElectron ? <DesktopUpdateLaunchNotice /> : null}
268	270	{/* Hosted Nightly is "hosted-static", not authenticated, and needs it too. */}
269	271	<NightlyMobileBetaNotice />
270	272	{appShell}

docs/internals/server-updates.md+7-4

...	...	@@ -64,8 +64,11 @@ opt-out. The
64	64	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)
65	65	requires matching durable run, provider thread, session, and native resume identity.
66	66	Queued runs never started, so recovery holds them and continues the run they wait
67		behind. A finished run qualifies only when the restart cancelled its background work;
68		its continuation tells the provider what will not report back.
	67	behind. A settled run qualifies only when the restart cancels work the agent waits on
	68	(a monitor, subagent, or background task) and its session was live, so stale records
	69	from an older crash never wake it. Its continuation prompt is the note of what will not
	70	report back. A shell alone does not wake a settled run: most are dev servers nobody
	71	waits on, so that note waits for the next user message.
69	72
70	73	Recovery retires effects tied to the lost process and records continuation intent
71	74	in the durable outbox. That intent survives another restart before provider startup.
...	...	@@ -77,8 +80,8 @@ cannot be overwritten by a stale cancellation.
77	80	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)
78	81	rechecks the preference, provider selection, newer user work, a stop the user requested,
79	82	and maintenance turns such as `/compact` before dispatching. Intent and delivery both
80		refuse a thread that is archived, settled, or still snoozed, a provider turn that already
81		ended, and a run waiting on the user: an approval, question, or secret that was pending,
	83	refuse a thread that is archived, settled, or still snoozed, a cut turn whose provider
	84	turn already ended, and a run waiting on the user: an approval, question, or secret that was pending,
82	85	or that recovery closed on the cut turn. Message-capable questions stay answerable, so
83	86	they do not block. Delivery also stops after three consecutive continuations in one
84	87	`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable

docs/user/updating.md+10-3

...	...	@@ -16,9 +16,13 @@ installs the latest release from its own channel. Hover the button to see what
16	16	it will update.
17	17
18	18	Server updates appear only after that version is published to npm. Remote
19		desktop apps do not wait for npm. Servers that need
20		a manual update are not included. Use the notice or Settings → Connections
21		for those.
	19	desktop apps do not wait for npm. The confirmation lists each machine it skips
	20	and why, such as a machine that is offline or needs a manual update. Use the
	21	notice or Settings → Connections for those.
	22
	23	If a desktop app restarts on its old version, T3 Code tells you the update did
	24	not apply. It does not install that version again on its own, even when another
	25	device asks. Use the update button on that machine to retry.
22	26
23	27	The confirmation counts the running threads that the restarts will interrupt,
24	28	and how many can resume afterwards. See Continue threads after restarts
...	...	@@ -37,6 +41,9 @@ different value, use Apply to all in Settings after it connects.
37	41	T3 Code must start again on that machine;
38	42	the setting does not enable automatic startup. Terminal commands may still be
39	43	interrupted, and threads without saved provider resume state need a new message.
	44	A thread that finished its turn to wait on a monitor, subagent, or background task
	45	is woken and told that this work stopped. A background shell, such as a dev server,
	46	does not wake it; the thread learns about it with your next message.
40	47	A thread is not resumed while it waits for your approval, answer, or secret, or
41	48	while it is archived, snoozed, or settled. After three automatic resumes in a row,
42	49	the thread waits for your next message.

packages/contracts/src/ipc.ts+7

...	...	@@ -284,6 +284,12 @@ export interface DesktopUpdateState {
284	284	message: string \| null;
285	285	errorContext: "check" \| "download" \| "install" \| null;
286	286	canRetry: boolean;
	287	/**
	288	* The version the last install handed to the OS installer that did not
	289	* apply: the app came back on its old version. Server-triggered updates do
	290	* not install it again; the user can retry it from the update button.
	291	*/
	292	notAppliedVersion?: string;
287	293	}
288	294
289	295	export interface DesktopUpdateReleaseNote {
...	...	@@ -315,6 +321,7 @@ export const DesktopUpdateStateSchema = Schema.Struct({
315	321	message: Schema.NullOr(Schema.String),
316	322	errorContext: Schema.NullOr(Schema.Literals(["check", "download", "install"])),
317	323	canRetry: Schema.Boolean,
	324	notAppliedVersion: Schema.optionalKey(Schema.String),
318	325	});
319	326
320	327	/** The desktop app's `t3` command on PATH, managed from Settings. */

packages/shared/src/AgentScope.ts+6

...	...	@@ -25,6 +25,12 @@ export interface AgentScopeShape {
25	25	readonly name: string;
26	26	readonly threadId?: string \| undefined;
27	27	readonly env?: NodeJS.ProcessEnv \| undefined;
	28	/**
	29	* Set for terminals, so a job the user detached (tmux, nohup) outlives a
	30	* server restart. Agents leave it unset: the next server stops agents a
	31	* killed server left running before it resumes their sessions.
	32	*/
	33	readonly survivesRestart?: boolean \| undefined;
28	34	}) => Effect.Effect<AgentScopeCommand>;
29	35	/**
30	36	* Whether the latest agent scope of this thread was killed because the

author	Theo Browne <me@t3.gg> 2026-10-14 00:38:17-07:00
committer	GitHub <noreply@github.com> 2026-10-14 00:38:17-07:00
log	eeaac49a9d20d94970b26fe7775fc8ddb4672f4f
tree	c36ecbafcf2f63527bcb37c0d96c1b9097de3299
parent	3bbfdc7755db099081ad546ff9e8263fc07b82e9
signature	Signed by PGP key B5690EEEBB952194

fix: update all no longer runs agents twice or loses background work (#19291)