Commit eeaac49a9d20d94970b26fe7775fc8ddb4672f4f | the-knife

apps/desktop/src/backend/DesktopBackendManager.test.ts+1-1

...	@@ -264,7 +264,7 @@ describe("DesktopBackendManager", () => {	...	@@ -264,7 +264,7 @@ describe("DesktopBackendManager", () => {
264	assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output");	264	assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output");
265	assert.equal(	265	assert.equal(
266	Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)),	266	Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)),
267	2_000,	267	5_000,
268	);	268	);
269		269
270	assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability);	270	assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability);

apps/desktop/src/backend/DesktopBackendManager.ts+3-1

 @@ -65,7 +65,9 @@ const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
 const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1);
 const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100);
 const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1);
-const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(2);
+// Same as the service launcher. A killed backend leaves its agents running,
+// and a busy one needs a few seconds to stop every provider session.
+const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5);
 const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5);
 const BACKEND_READINESS_PATH = "/.well-known/t3/environment";
 const { logWarning: logBackendProcessWarning } =

apps/desktop/src/updates/DesktopUpdates.test.ts+63

...	@@ -652,6 +652,67 @@ describe("DesktopUpdates", () => {	...	@@ -652,6 +652,67 @@ describe("DesktopUpdates", () => {
652	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));	652	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
653	});	653	});
654		654
		655	it.effect("records the version an install hands to the installer", () => {
		656	let recordsAtStop: ReadonlyArray<string> = [];
		657	const harness = makeHarness({
		658	stopBackend: Effect.sync(() => {
		659	recordsAtStop = [...harness.installRecords.values()];
		660	}),
		661	});
		662
		663	return Effect.scoped(
		664	Effect.gen(function* () {
		665	const updates = yield* DesktopUpdates.DesktopUpdates;
		666	yield* updates.configure;
		667	harness.emit("update-downloaded", { version: "1.2.4" });
		668	yield* flushCallbacks;
		669
		670	assert.isTrue((yield* updates.install).accepted);
		671	assert.deepEqual(
		672	recordsAtStop.map((record) => JSON.parse(record)),
		673	[{ fromVersion: "1.2.3", targetVersion: "1.2.4" }],
		674	);
		675	}),
		676	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
		677	});
		678
		679	it.effect("flags an install that did not apply and keeps the flag through checks", () => {
		680	const harness = makeHarness({
		681	installRecord: JSON.stringify({ fromVersion: "1.2.3", targetVersion: "1.2.4" }),
		682	});
		683
		684	return Effect.scoped(
		685	Effect.gen(function* () {
		686	const updates = yield* DesktopUpdates.DesktopUpdates;
		687	yield* updates.configure;
		688	assert.equal((yield* updates.getState).notAppliedVersion, "1.2.4");
		689	// Kept, so the next start on 1.2.3 still knows 1.2.4 did not apply.
		690	assert.isDefined(harness.previousInstallRecord());
		691
		692	harness.emit("update-available", { version: "1.2.4" });
		693	yield* flushCallbacks;
		694	const state = yield* updates.getState;
		695	assert.equal(state.status, "available");
		696	assert.equal(state.notAppliedVersion, "1.2.4");
		697	}),
		698	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
		699	});
		700
		701	it.effect("does not flag an install that applied", () => {
		702	const harness = makeHarness({
		703	installRecord: JSON.stringify({ fromVersion: "1.2.2", targetVersion: "1.2.3" }),
		704	});
		705
		706	return Effect.scoped(
		707	Effect.gen(function* () {
		708	const updates = yield* DesktopUpdates.DesktopUpdates;
		709	yield* updates.configure;
		710	assert.isUndefined((yield* updates.getState).notAppliedVersion);
		711	assert.isUndefined(harness.previousInstallRecord());
		712	}),
		713	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
		714	});
		715
655	it.effect("drops the update restart marker when an install is interrupted", () =>	716	it.effect("drops the update restart marker when an install is interrupted", () =>
656	Effect.gen(function* () {	717	Effect.gen(function* () {
657	const stopping = yield* Deferred.make<void>();	718	const stopping = yield* Deferred.make<void>();
...	@@ -703,6 +764,8 @@ describe("DesktopUpdates", () => {	...	@@ -703,6 +764,8 @@ describe("DesktopUpdates", () => {
703	assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]);	764	assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]);
704	// The restarted old backend must release its tunnel on a later quit.	765	// The restarted old backend must release its tunnel on a later quit.
705	assert.equal(harness.updateRestartMarkers.size, 0);	766	assert.equal(harness.updateRestartMarkers.size, 0);
		767	// The app never quit, so the next start must not report an unapplied install.
		768	assert.equal(harness.installRecords.size, 0);
706	}),	769	}),
707	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));	770	).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer)));
708	});	771	});

apps/desktop/src/updates/DesktopUpdates.ts+70-2

...	@@ -46,6 +46,7 @@ import {	...	@@ -46,6 +46,7 @@ import {
46	reduceDesktopUpdateStateOnInstallFailure,	46	reduceDesktopUpdateStateOnInstallFailure,
47	reduceDesktopUpdateStateOnNoUpdate,	47	reduceDesktopUpdateStateOnNoUpdate,
48	reduceDesktopUpdateStateOnUpdateAvailable,	48	reduceDesktopUpdateStateOnUpdateAvailable,
		49	resolveNotAppliedInstallVersion,
49	} from "./updateMachine.ts";	50	} from "./updateMachine.ts";
50		51
51	const AUTO_UPDATE_STARTUP_DELAY = "15 seconds";	52	const AUTO_UPDATE_STARTUP_DELAY = "15 seconds";
...	@@ -72,6 +73,14 @@ const UpdateInfo = Schema.Struct({	...	@@ -72,6 +73,14 @@ const UpdateInfo = Schema.Struct({
72	const DownloadProgressInfo = Schema.Struct({	73	const DownloadProgressInfo = Schema.Struct({
73	percent: Schema.Number,	74	percent: Schema.Number,
74	});	75	});
		76	/** Saved in the runtime folder before an install quits the app, read on the next start. */
		77	export const DESKTOP_UPDATE_INSTALL_RECORD_FILE = "desktop-update-install.json";
		78	const DesktopUpdateInstallRecord = Schema.fromJsonString(
		79	Schema.Struct({ fromVersion: Schema.String, targetVersion: Schema.String }),
		80	);
		81	const decodeInstallRecord = Schema.decodeUnknownEffect(DesktopUpdateInstallRecord);
		82	const encodeInstallRecord = Schema.encodeEffect(DesktopUpdateInstallRecord);
		83
75	const decodeAppUpdateYmlConfig = Schema.decodeUnknownEffect(AppUpdateYmlConfig);	84	const decodeAppUpdateYmlConfig = Schema.decodeUnknownEffect(AppUpdateYmlConfig);
76	const decodeUpdateInfo = Schema.decodeUnknownEffect(UpdateInfo);	85	const decodeUpdateInfo = Schema.decodeUnknownEffect(UpdateInfo);
77	const decodeDownloadProgressInfo = Schema.decodeUnknownEffect(DownloadProgressInfo);	86	const decodeDownloadProgressInfo = Schema.decodeUnknownEffect(DownloadProgressInfo);
...	@@ -347,6 +356,7 @@ export const make = Effect.gen(function* () {	...	@@ -347,6 +356,7 @@ export const make = Effect.gen(function* () {
347	const finishedUpdateActions = yield* PubSub.unbounded<UpdateAction>();	356	const finishedUpdateActions = yield* PubSub.unbounded<UpdateAction>();
348	const updaterConfiguredRef = yield* Ref.make(false);	357	const updaterConfiguredRef = yield* Ref.make(false);
349	const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1);	358	const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1);
		359	const notAppliedVersionRef = yield* Ref.make(Option.none<string>());
350	const updateStateRef = yield* Ref.make<DesktopUpdateState>(	360	const updateStateRef = yield* Ref.make<DesktopUpdateState>(
351	createInitialDesktopUpdateState(	361	createInitialDesktopUpdateState(
352	environment.appVersion,	362	environment.appVersion,
...	@@ -600,11 +610,57 @@ export const make = Effect.gen(function* () {	...	@@ -600,11 +610,57 @@ export const make = Effect.gen(function* () {
600	.remove(updateRestartMarkerPath, { force: true })	610	.remove(updateRestartMarkerPath, { force: true })
601	.pipe(Effect.ignore);	611	.pipe(Effect.ignore);
602		612
		613	// Records the version an install hands to the OS installer, so the next
		614	// start can tell whether it applied. Best effort, like the restart marker.
		615	const installRecordPath = environment.path.join(
		616	updateRestartMarkerDir,
		617	DESKTOP_UPDATE_INSTALL_RECORD_FILE,
		618	);
		619	const writeInstallRecord = (targetVersion: string) =>
		620	encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion }).pipe(
		621	Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)),
		622	Effect.catch((error) =>
		623	logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }),
		624	),
		625	);
		626	const removeInstallRecord = fileSystem
		627	.remove(installRecordPath, { force: true })
		628	.pipe(Effect.ignore);
		629
		630	// Reads the record of the last install. Returns the version it did not
		631	// apply, when the app came back on the version it installed from. That
		632	// record stays through later starts, until the version changes or a new
		633	// install replaces it, so no restart forgets the failed version.
		634	const readNotAppliedInstallVersion = fileSystem.readFileString(installRecordPath).pipe(
		635	Effect.flatMap(decodeInstallRecord),
		636	Effect.option,
		637	Effect.map((record) =>
		638	Option.flatMap(record, (value) =>
		639	Option.fromNullishOr(resolveNotAppliedInstallVersion(value, environment.appVersion)),
		640	),
		641	),
		642	Effect.tap((notAppliedVersion) =>
		643	Option.isNone(notAppliedVersion) ? removeInstallRecord : Effect.void,
		644	),
		645	);
		646
		647	const baseUpdateState = (channel: DesktopUpdateChannel, enabled: boolean) =>
		648	Ref.get(notAppliedVersionRef).pipe(
		649	Effect.map((notAppliedVersion) => ({
		650	...createBaseUpdateState(channel, enabled, environment),
		651	...Option.match(notAppliedVersion, {
		652	onNone: () => ({}),
		653	onSome: (version) => ({ notAppliedVersion: version }),
		654	}),
		655	})),
		656	);
		657
603	const resetInstallAction = Effect.all(	658	const resetInstallAction = Effect.all(
604	[	659	[
605	finishUpdateAction("install"),	660	finishUpdateAction("install"),
606	Ref.set(desktopState.quitting, false),	661	Ref.set(desktopState.quitting, false),
607	removeUpdateRestartMarker,	662	removeUpdateRestartMarker,
		663	removeInstallRecord,
608	],	664	],
609	{ discard: true },	665	{ discard: true },
610	);	666	);
...	@@ -621,6 +677,7 @@ export const make = Effect.gen(function* () {	...	@@ -621,6 +677,7 @@ export const make = Effect.gen(function* () {
621		677
622	yield* Ref.set(desktopState.quitting, false);	678	yield* Ref.set(desktopState.quitting, false);
623	yield* removeUpdateRestartMarker;	679	yield* removeUpdateRestartMarker;
		680	yield* removeInstallRecord;
624	yield* Effect.gen(function* () {	681	yield* Effect.gen(function* () {
625	const instances = yield* pool.list;	682	const instances = yield* pool.list;
626	const restartExit = yield* Effect.forEach(instances, (instance) => instance.start, {	683	const restartExit = yield* Effect.forEach(instances, (instance) => instance.start, {
...	@@ -691,6 +748,8 @@ export const make = Effect.gen(function* () {	...	@@ -691,6 +748,8 @@ export const make = Effect.gen(function* () {
691		748
692	return yield* Effect.gen(function* () {	749	return yield* Effect.gen(function* () {
693	yield* writeUpdateRestartMarker;	750	yield* writeUpdateRestartMarker;
		751	const { downloadedVersion } = yield* Ref.get(updateStateRef);
		752	if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion);
694	// Stop every backend in the pool, not just the primary. With	753	// Stop every backend in the pool, not just the primary. With
695	// parallel WSL + Windows backends, leaving the WSL instance up	754	// parallel WSL + Windows backends, leaving the WSL instance up
696	// means quitAndInstall's app.quit() exits before the pool's	755	// means quitAndInstall's app.quit() exits before the pool's
...	@@ -985,6 +1044,15 @@ export const make = Effect.gen(function* () {	...	@@ -985,6 +1044,15 @@ export const make = Effect.gen(function* () {
985	const appUpdateYmlConfig = yield* readAppUpdateYml;	1044	const appUpdateYmlConfig = yield* readAppUpdateYml;
986	yield* Ref.set(appUpdateYmlConfigRef, appUpdateYmlConfig);	1045	yield* Ref.set(appUpdateYmlConfigRef, appUpdateYmlConfig);
987		1046
		1047	const notAppliedVersion = yield* readNotAppliedInstallVersion;
		1048	yield* Ref.set(notAppliedVersionRef, notAppliedVersion);
		1049	if (Option.isSome(notAppliedVersion)) {
		1050	yield* logUpdaterWarning("the last update install did not apply", {
		1051	version: environment.appVersion,
		1052	targetVersion: notAppliedVersion.value,
		1053	});
		1054	}
		1055
988	if (config.mockUpdates) {	1056	if (config.mockUpdates) {
989	yield* electronUpdater.setFeedURL({	1057	yield* electronUpdater.setFeedURL({
990	provider: "generic",	1058	provider: "generic",
...	@@ -994,7 +1062,7 @@ export const make = Effect.gen(function* () {	...	@@ -994,7 +1062,7 @@ export const make = Effect.gen(function* () {
994		1062
995	const settings = yield* desktopSettings.get;	1063	const settings = yield* desktopSettings.get;
996	const enabled = yield* shouldEnableAutoUpdates;	1064	const enabled = yield* shouldEnableAutoUpdates;
997	yield* setState(createBaseUpdateState(settings.updateChannel, enabled, environment));	1065	yield* setState(yield* baseUpdateState(settings.updateChannel, enabled));
998	if (!enabled) {	1066	if (!enabled) {
999	return;	1067	return;
1000	}	1068	}
...	@@ -1065,7 +1133,7 @@ export const make = Effect.gen(function* () {	...	@@ -1065,7 +1133,7 @@ export const make = Effect.gen(function* () {
1065	);	1133	);
1066		1134
1067	const enabled = yield* shouldEnableAutoUpdates;	1135	const enabled = yield* shouldEnableAutoUpdates;
1068	yield* setState(createBaseUpdateState(nextChannel, enabled, environment));	1136	yield* setState(yield* baseUpdateState(nextChannel, enabled));
1069		1137
1070	if (!enabled \|\| !(yield* Ref.get(updaterConfiguredRef))) {	1138	if (!enabled \|\| !(yield* Ref.get(updaterConfiguredRef))) {
1071	return yield* Ref.get(updateStateRef);	1139	return yield* Ref.get(updateStateRef);

apps/desktop/src/updates/remoteUpdateFlow.test.ts+18

...	@@ -34,6 +34,24 @@ function makeState(overrides: Partial<DesktopUpdateState> = {}): DesktopUpdateSt	...	@@ -34,6 +34,24 @@ function makeState(overrides: Partial<DesktopUpdateState> = {}): DesktopUpdateSt
34	}	34	}
35		35
36	describe("nextRemoteDesktopUpdateStep", () => {	36	describe("nextRemoteDesktopUpdateStep", () => {
		37	it("does not install again a version that did not apply on this machine", () => {
		38	const step = nextRemoteDesktopUpdateStep(
		39	makeState({ status: "available", availableVersion: "1.2.4", notAppliedVersion: "1.2.4" }),
		40	NO_ATTEMPTS,
		41	null,
		42	);
		43	assert.equal(step.action, "done");
		44	assert.include(step.action === "done" ? step.reason : "", "1.2.4 did not apply");
		45	assert.deepEqual(
		46	nextRemoteDesktopUpdateStep(
		47	makeState({ status: "available", availableVersion: "1.2.5", notAppliedVersion: "1.2.4" }),
		48	NO_ATTEMPTS,
		49	null,
		50	),
		51	{ action: "download" },
		52	);
		53	});
		54
37	it("fails immediately when updates are disabled, preferring the known reason", () => {	55	it("fails immediately when updates are disabled, preferring the known reason", () => {
38	const state = makeState({ enabled: false, status: "disabled" });	56	const state = makeState({ enabled: false, status: "disabled" });
39	assert.deepEqual(nextRemoteDesktopUpdateStep(state, NO_ATTEMPTS, "dev build"), {	57	assert.deepEqual(nextRemoteDesktopUpdateStep(state, NO_ATTEMPTS, "dev build"), {

apps/desktop/src/updates/remoteUpdateFlow.ts+13

...	@@ -52,6 +52,19 @@ export function nextRemoteDesktopUpdateStep(	...	@@ -52,6 +52,19 @@ export function nextRemoteDesktopUpdateStep(
52	reason: disabledReason ?? "Automatic updates are disabled on this machine.",	52	reason: disabledReason ?? "Automatic updates are disabled on this machine.",
53	};	53	};
54	}	54	}
		55	// An install of this version already failed to apply here. Installing it
		56	// again from afar would stop the app's threads for nothing, so only the user
		57	// on this machine retries it.
		58	if (
		59	state.notAppliedVersion !== undefined &&
		60	(state.downloadedVersion ?? state.availableVersion) === state.notAppliedVersion
		61	) {
		62	return {
		63	action: "done",
		64	outcome: "failed",
		65	reason: `Update ${state.notAppliedVersion} did not apply on this machine. Retry it from the desktop app there.`,
		66	};
		67	}
55	// Mirror installDownloadedUpdate's own admission rule exactly, so the run	68	// Mirror installDownloadedUpdate's own admission rule exactly, so the run
56	// never reports an install that the updater then refuses. A download	69	// never reports an install that the updater then refuses. A download
57	// survives an unrelated background updater error (errorContext null) and	70	// survives an unrelated background updater error (errorContext null) and

apps/desktop/src/updates/updateMachine.test.ts+14

...	@@ -11,6 +11,7 @@ import {	...	@@ -11,6 +11,7 @@ import {
11	reduceDesktopUpdateStateOnInstallFailure,	11	reduceDesktopUpdateStateOnInstallFailure,
12	reduceDesktopUpdateStateOnNoUpdate,	12	reduceDesktopUpdateStateOnNoUpdate,
13	reduceDesktopUpdateStateOnUpdateAvailable,	13	reduceDesktopUpdateStateOnUpdateAvailable,
		14	resolveNotAppliedInstallVersion,
14	} from "./updateMachine.ts";	15	} from "./updateMachine.ts";
15		16
16	const runtimeInfo = {	17	const runtimeInfo = {
...	@@ -255,3 +256,16 @@ describe("updateMachine", () => {	...	@@ -255,3 +256,16 @@ describe("updateMachine", () => {
255	expect(state.omittedReleaseCount).toBe(0);	256	expect(state.omittedReleaseCount).toBe(0);
256	});	257	});
257	});	258	});
		259
		260	describe("resolveNotAppliedInstallVersion", () => {
		261	const record = { fromVersion: "1.2.3", targetVersion: "1.2.4" };
		262
		263	it("flags the target when the app came back on the version it installed from", () => {
		264	expect(resolveNotAppliedInstallVersion(record, "1.2.3")).toBe("1.2.4");
		265	});
		266
		267	it("stays quiet when the install applied or the app runs another version", () => {
		268	expect(resolveNotAppliedInstallVersion(record, "1.2.4")).toBeNull();
		269	expect(resolveNotAppliedInstallVersion(record, "1.2.5")).toBeNull();
		270	});
		271	});

apps/desktop/src/updates/updateMachine.ts+20

...	@@ -211,3 +211,23 @@ export function reduceDesktopUpdateStateOnInstallFailure(	...	@@ -211,3 +211,23 @@ export function reduceDesktopUpdateStateOnInstallFailure(
211	canRetry: true,	211	canRetry: true,
212	};	212	};
213	}	213	}
		214
		215	/** The install this app handed to the OS installer, saved right before it quits. */
		216	export interface DesktopUpdateInstallRecord {
		217	readonly fromVersion: string;
		218	readonly targetVersion: string;
		219	}
		220
		221	/**
		222	* Read on the next start: the version an install did not apply, because the
		223	* app came back on the version it installed from. Null when the install
		224	* applied, or when the app now runs some other version.
		225	*/
		226	export function resolveNotAppliedInstallVersion(
		227	record: DesktopUpdateInstallRecord,
		228	appVersion: string,
		229	): string \| null {
		230	return record.fromVersion === appVersion && record.targetVersion !== appVersion
		231	? record.targetVersion
		232	: null;
		233	}

apps/desktop/src/updates/updatesTestHarness.ts+25-7

 @@ -37,6 +37,8 @@ export interface UpdatesHarnessOptions {
   readonly platform?: NodeJS.Platform;
   /** Contents of the resources/package-type marker a Linux package ships. */
   readonly packageType?: string | undefined;
+  /** Contents of the install record a previous run left behind. */
+  readonly installRecord?: string | undefined;
 }
 export function makeHarness(options: UpdatesHarnessOptions = {}) {
 @@ -208,12 +210,21 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
         } satisfies DesktopAppSettings.DesktopAppSettings["Service"])
       : DesktopAppSettings.layer;
-  // Tracks the restart markers installs leave, so installs stay free of real
+  // Tracks the restart markers and install records installs leave, so
-  // disk I/O that would outrun the tests' settle loops.
+  // installs stay free of real disk I/O that would outrun the tests' settle loops.
   const updateRestartMarkers = new Set<string>();
+  // Contents of the install record, by path once an install writes one.
+  const installRecords = new Map<string, string>();
+  let previousInstallRecord = options.installRecord;
+  const isInstallRecord = (path: string) =>
+    path.endsWith(DesktopUpdates.DESKTOP_UPDATE_INSTALL_RECORD_FILE);
   const layerFileSystem = FileSystem.layerNoop({
-    readFileString: (path) =>
+    readFileString: (path) => {
-      path === "/missing/resources/package-type" && options.packageType !== undefined
+      const installRecord = isInstallRecord(path)
+        ? (installRecords.get(path) ?? previousInstallRecord)
+        : undefined;
+      if (installRecord !== undefined) return Effect.succeed(installRecord);
+      return path === "/missing/resources/package-type" && options.packageType !== undefined
         ? Effect.succeed(options.packageType)
         : Effect.fail(
             PlatformError.systemError({
 @@ -222,15 +233,19 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
               _tag: "NotFound",
               pathOrDescriptor: path,
             }),
-          ),
+          );
+    },
     makeDirectory: () => Effect.void,
-    writeFileString: (path) =>
+    writeFileString: (path, data) =>
       Effect.sync(() => {
-        updateRestartMarkers.add(path);
+        if (isInstallRecord(path)) installRecords.set(path, data);
+        else updateRestartMarkers.add(path);
       }),
     remove: (path) =>
       Effect.sync(() => {
         updateRestartMarkers.delete(path);
+        installRecords.delete(path);
+        if (isInstallRecord(path)) previousInstallRecord = undefined;
       }),
   });
 @@ -259,6 +274,9 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) {
     quitAndInstalls: () => quitAndInstallCount,
     installSteps,
     updateRestartMarkers,
+    installRecords,
+    /** The record the test started with, until the app removes it. */
+    previousInstallRecord: () => previousInstallRecord,
     downloadCount: () => downloadCount,
     feedUrls: (): ElectronUpdater.ElectronUpdaterFeedUrl[] => feedUrls,
     fullChangelog: () => fullChangelog,

apps/server/src/mcp/OrchestratorMcpService.ts+4-3

...	@@ -81,6 +81,7 @@ import {	...	@@ -81,6 +81,7 @@ import {
81	DispatchModeLimit,	81	DispatchModeLimit,
82	type DispatchModeRefusal,	82	type DispatchModeRefusal,
83	} from "../orchestration-v2/DispatchModeLimit.ts";	83	} from "../orchestration-v2/DispatchModeLimit.ts";
		84	import { runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts";
84	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";	85	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
85	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";	86	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
86	import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";	87	import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
...	@@ -330,9 +331,9 @@ function invalidOptionSelections(	...	@@ -330,9 +331,9 @@ function invalidOptionSelections(
330	}	331	}
331		332
332	function taskStatusForRun(	333	function taskStatusForRun(
333	run: Pick<OrchestrationV2Run, "status"> \| undefined,	334	run: Pick<OrchestrationV2Run, "status" \| "restartCancelledBackgroundWork"> \| undefined,
334	): OrchestratorMcpDelegateTaskResult["status"] {	335	): OrchestratorMcpDelegateTaskResult["status"] {
335	switch (run?.status) {	336	switch (run === undefined ? undefined : runOutcome(run)) {
336	case "queued":	337	case "queued":
337	return "queued";	338	return "queued";
338	case "waiting":	339	case "waiting":
...	@@ -2642,7 +2643,7 @@ const make = Effect.gen(function* () {	...	@@ -2642,7 +2643,7 @@ const make = Effect.gen(function* () {
2642	return {	2643	return {
2643	threadId: input.threadId,	2644	threadId: input.threadId,
2644	runId: result.run?.id ?? null,	2645	runId: result.run?.id ?? null,
2645	status: result.run?.status ?? "idle",	2646	status: result.run === null ? "idle" : runOutcome(result.run),
2646	timedOut: result.timedOut,	2647	timedOut: result.timedOut,
2647	} satisfies OrchestratorMcpThreadWaitResult;	2648	} satisfies OrchestratorMcpThreadWaitResult;
2648	}),	2649	}),

apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts+65-7

...	@@ -803,6 +803,8 @@ const seedRestartCancelledChild = (input: {	...	@@ -803,6 +803,8 @@ const seedRestartCancelledChild = (input: {
803	readonly continuationPending: boolean;	803	readonly continuationPending: boolean;
804	/** "completed" seeds a settled turn whose background work the restart cancelled. */	804	/** "completed" seeds a settled turn whose background work the restart cancelled. */
805	readonly runStatus?: "cancelled" \| "completed";	805	readonly runStatus?: "cancelled" \| "completed";
		806	/** Background work the restart cancelled on the child's run. */
		807	readonly lostWork?: OrchestrationV2Run["restartCancelledBackgroundWork"];
806	readonly now: DateTime.Utc;	808	readonly now: DateTime.Utc;
807	}) =>	809	}) =>
808	Effect.gen(function* () {	810	Effect.gen(function* () {
...	@@ -880,16 +882,22 @@ const seedRestartCancelledChild = (input: {	...	@@ -880,16 +882,22 @@ const seedRestartCancelledChild = (input: {
880	},	882	},
881	],	883	],
882	});	884	});
		885	const childRun = runEvent({
		886	threadId: childThreadId,
		887	runId: childRunId,
		888	ordinal: 1,
		889	status: input.runStatus ?? "cancelled",
		890	now: input.now,
		891	});
883	yield* eventSink.writeWithEffects({	892	yield* eventSink.writeWithEffects({
884	commandId: reconcileCommandId(input.name),	893	commandId: reconcileCommandId(input.name),
885	events: [	894	events: [
886	runEvent({	895	input.lostWork === undefined
887	threadId: childThreadId,	896	? childRun
888	runId: childRunId,	897	: {
889	ordinal: 1,	898	...childRun,
890	status: input.runStatus ?? "cancelled",	899	payload: { ...childRun.payload, restartCancelledBackgroundWork: input.lostWork },
891	now: input.now,	900	},
892	}),
893	],	901	],
894	effects: input.continuationPending	902	effects: input.continuationPending
895	? [	903	? [
...	@@ -1427,6 +1435,56 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {	...	@@ -1427,6 +1435,56 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
1427	}),	1435	}),
1428	);	1436	);
1429		1437
		1438	it.effect("reports a child whose monitor the restart cut as interrupted, not completed", () =>
		1439	Effect.gen(function* () {
		1440	const orchestrator = yield* Orchestrator.OrchestratorV2;
		1441	const now = yield* DateTime.now;
		1442	const threadId = ThreadId.make("thread:restart-monitor-parent");
		1443	const projectId = ProjectId.make("project:restart-monitor-parent");
		1444	const runId = RunId.make("run:restart-monitor-parent");
		1445	const rootNodeId = NodeId.make("node:restart-monitor-parent-root");
		1446	yield* seedParentWithTerminalTask({
		1447	threadId,
		1448	projectId,
		1449	runId,
		1450	rootNodeId,
		1451	taskId: NodeId.make("node:restart-monitor-parent-settled"),
		1452	deliveryState: "delivered",
		1453	now,
		1454	});
		1455	// The child ended its turn to wait on a gate monitor; the restart cut it.
		1456	const child = yield* seedRestartCancelledChild({
		1457	parentThreadId: threadId,
		1458	projectId,
		1459	parentRunId: runId,
		1460	rootNodeId,
		1461	name: "restart-monitor-child",
		1462	completionWake: "always",
		1463	continuationPending: true,
		1464	runStatus: "completed",
		1465	lostWork: [{ kind: "monitor", label: "Gate attempt 9" }],
		1466	now,
		1467	});
		1468	yield* orchestrator.recoverDelegatedTasks;
		1469	const held = yield* orchestrator.getThreadProjection(threadId);
		1470	assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running");
		1471
		1472	// Continuation is off, so the cut run is the outcome. It still goes to the parent.
		1473	yield* continueRestartedRun({
		1474	threadId: child.childThreadId,
		1475	sourceRunId: child.childRunId,
		1476	}).pipe(
		1477	Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })),
		1478	);
		1479
		1480	const settled = yield* orchestrator.getThreadProjection(threadId);
		1481	const task = settled.subagents.find((row) => row.id === child.taskId);
		1482	assert.equal(task?.status, "interrupted");
		1483	assert.equal(task?.result, "Child task ended with status interrupted.");
		1484	assert.equal(task?.completionDelivery?.state, "claimed");
		1485	}),
		1486	);
		1487
1430	// A running provider turn does not prove the provider consumed the delivery	1488	// A running provider turn does not prove the provider consumed the delivery
1431	// (Claude marks the turn running before it reads the prompt), so a cut	1489	// (Claude marks the turn running before it reads the prompt), so a cut
1432	// delivery is offered again even when a continuation resumes that turn.	1490	// delivery is offered again even when a continuation resumes that turn.

apps/server/src/orchestration-v2/EffectOutbox.ts+29

 @@ -224,6 +224,8 @@ export interface EffectOutboxV2Shape {
   readonly signalCancellations: (effectIds: ReadonlyArray<string>) => Effect.Effect<void>;
   readonly awaitCancellation: (effectId: string) => Effect.Effect<void>;
   readonly clearCancellation: (effectId: string) => Effect.Effect<void>;
+  /** Resolves once the effect is neither pending nor running, or right away when it does not exist. */
+  readonly awaitSettled: (effectId: string) => Effect.Effect<void, EffectOutboxError>;
   readonly reconcileAfterProcessLoss: Effect.Effect<
     { readonly requeued: number; readonly cancelled: number },
     EffectOutboxError
 @@ -307,6 +309,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
     // for distinct threads without allowing notifications to grow unbounded.
     const available = yield* Queue.dropping<void>(64);
     const cancellationSignals = new Map<string, Deferred.Deferred<void>>();
+    const settledSignals = new Map<string, Deferred.Deferred<void>>();
+    const signalSettled = (effectIds: ReadonlyArray<string>) =>
+      Effect.forEach(
+        effectIds,
+        (effectId) => {
+          const signal = settledSignals.get(effectId);
+          if (signal === undefined) return Effect.void;
+          settledSignals.delete(effectId);
+          return Deferred.succeed(signal, undefined);
+        },
+        { discard: true },
+      );
     const notifyAvailable = (count = 1) =>
       Queue.offerAll(
         available,
 @@ -477,6 +491,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
             },
             { discard: true },
           );
+          yield* signalSettled(effectIds);
           // Cancellation can unblock other pending work on the same thread.
           // This method is deliberately post-commit, so it is also the safe
           // place to wake claimers after the durable status change.
 @@ -487,6 +502,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
         Effect.sync(() => {
           cancellationSignals.delete(effectId);
         }),
+      awaitSettled: (effectId) =>
+        Effect.gen(function* () {
+          // Registered before the read, so a settle right after it still resolves.
+          const signal = settledSignals.get(effectId) ?? Deferred.makeUnsafe<void>();
+          settledSignals.set(effectId, signal);
+          const effect = yield* service.get(effectId);
+          if (Option.exists(effect, ({ status }) => status === "pending" || status === "running")) {
+            return yield* Deferred.await(signal);
+          }
+          // Settled already: release every waiter that shares this signal.
+          yield* signalSettled([effectId]);
+        }),
       reconcileAfterProcessLoss: Effect.gen(function* () {
         const now = DateTime.formatIso(yield* DateTime.now);
         const cancelledRows = yield* sql<{ readonly effect_id: string }>`
 @@ -599,6 +626,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
           `;
           if (rows.length === 1) {
             cancellationSignals.delete(effectId);
+            yield* signalSettled([effectId]);
             yield* notifyAvailable();
           }
           return rows.length === 1;
 @@ -657,6 +685,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
           `;
           if (rows.length === 1) {
             cancellationSignals.delete(effectId);
+            yield* signalSettled([effectId]);
             yield* notifyAvailable();
           }
           return rows.length === 1;

apps/server/src/orchestration-v2/FoundationPersistence.test.ts+31

...	@@ -2084,6 +2084,37 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {	...	@@ -2084,6 +2084,37 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2084	}),	2084	}),
2085	);	2085	);
2086		2086
		2087	it.effect("wakes a settle waiter when its effect finishes", () =>
		2088	Effect.gen(function* () {
		2089	const outbox = yield* EffectOutbox.EffectOutboxV2;
		2090	const effectId = "effect:foundation-await-settled";
		2091	yield* outbox.enqueue([
		2092	{
		2093	id: effectId,
		2094	commandId: CommandId.make("command:foundation-await-settled"),
		2095	threadId: ThreadId.make("thread:foundation-await-settled"),
		2096	request: {
		2097	type: "provider-runtime.continue",
		2098	sourceRunId: RunId.make("run:foundation-await-settled"),
		2099	},
		2100	},
		2101	]);
		2102	const waiter = yield* outbox.awaitSettled(effectId).pipe(Effect.forkChild);
		2103	const claimed = yield* outbox.claimNext({
		2104	workerId: "settle-worker",
		2105	leaseDurationMs: 30_000,
		2106	});
		2107	assert.isTrue(Option.isSome(claimed));
		2108	yield* Effect.yieldNow;
		2109	assert.isUndefined(waiter.pollUnsafe());
		2110
		2111	yield* outbox.succeed({ effectId, workerId: "settle-worker" });
		2112	yield* Fiber.join(waiter);
		2113	// A settled effect does not wait.
		2114	yield* outbox.awaitSettled(effectId);
		2115	}),
		2116	);
		2117
2087	it.effect("runs title generation beside critical work while serializing each lane", () =>	2118	it.effect("runs title generation beside critical work while serializing each lane", () =>
2088	Effect.gen(function* () {	2119	Effect.gen(function* () {
2089	const outbox = yield* EffectOutbox.EffectOutboxV2;	2120	const outbox = yield* EffectOutbox.EffectOutboxV2;

apps/server/src/orchestration-v2/Orchestrator.ts+21-5

 @@ -88,7 +88,7 @@ import { CommandPolicyV2, resolveMessageDispatchIntent } from "./CommandPolicy.t
 import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
 import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
 import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
-import { isRestartNoteSource } from "./RestartBackgroundNote.ts";
+import { isRestartContinuationSource, runOutcome } from "./RestartBackgroundNote.ts";
 import {
   isBufferedProviderNotification,
   isUndeliveredMailboxSteer,
 @@ -293,6 +293,16 @@ export interface OrchestratorV2Shape {
     childThreadId: ThreadId,
     taskId?: NodeId,
   ) => Effect.Effect<boolean, OrchestratorProjectionError>;
+  /**
+   * Whether a restart continuation of this cut run has not started yet, so the
+   * run's status is not its outcome. A failed read counts as not pending.
+   */
+  readonly restartContinuationPending: (run: OrchestrationV2Run) => Effect.Effect<boolean>;
+  /**
+   * Resolves once that continuation has run or was declined. A declined one
+   * records no thread event, so a waiter needs this to wake.
+   */
+  readonly awaitRestartContinuation: (run: OrchestrationV2Run) => Effect.Effect<void>;
   readonly dispatch: (
     command: OrchestrationV2ServerCommand,
   ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
 @@ -4624,8 +4634,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
         const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
         if (
           !source ||
-          source.status !== "cancelled" ||
+          !isRestartContinuationSource(source, projection.providerTurns) ||
-          isRestartNoteSource(source, projection.providerTurns) ||
           projection.thread.archivedAt !== null ||
           projection.thread.deletedAt !== null ||
           projection.thread.providerInstanceId !== source.providerInstanceId ||
 @@ -10147,7 +10156,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
             : undefined
           : taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0];
       if (childRun === undefined) return;
-      const terminalStatus = delegatedTaskTerminalStatus(childRun.status);
+      const outcome = runOutcome(childRun);
+      const terminalStatus = delegatedTaskTerminalStatus(outcome);
       if (terminalStatus === null) {
         return;
       }
 @@ -10213,7 +10223,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
       }
       const now = yield* DateTime.now;
-      const result = subagentResultForRun(childProjection, childRun);
+      const result = subagentResultForRun(childProjection, { ...childRun, status: outcome });
       const parentRun =
         task.runId === null
           ? undefined
 @@ -11459,6 +11469,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
     recoverDelegatedTasks,
     recoverDelegatedTask,
     delegatedTaskResultPending,
+    restartContinuationPending: (run) =>
+      awaitsRestartContinuation(run).pipe(Effect.orElseSucceed(() => false)),
+    awaitRestartContinuation: (run) =>
+      effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
     dispatch: dispatchWithReceipt,
     searchThreadStream: (input) =>
       projectionStore
 @@ -11607,6 +11621,8 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
     recoverDelegatedTasks: Effect.void,
     recoverDelegatedTask: () => Effect.void,
     delegatedTaskResultPending: () => Effect.succeed(false),
+    restartContinuationPending: () => Effect.succeed(false),
+    awaitRestartContinuation: () => Effect.void,
     dispatch: (command) =>
       Effect.fail(
         new OrchestratorDispatchError({

apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+52

...	@@ -209,6 +209,58 @@ it.effect.each(["startup", "shutdown"] as const)(	...	@@ -209,6 +209,58 @@ it.effect.each(["startup", "shutdown"] as const)(
209	}).pipe(Effect.provide(layerRecoveryTest)),	209	}).pipe(Effect.provide(layerRecoveryTest)),
210	);	210	);
211		211
		212	it.effect("stops a session its thread detached, which kept its live status", () =>
		213	Effect.gen(function* () {
		214	const projections = yield* ProjectionStore.ProjectionStoreV2;
		215	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
		216	const now = yield* DateTime.now;
		217	const threadId = yield* createThread("detached-session");
		218	const providerSessionId = ProviderSessionId.make("session:recovery:detached");
		219	yield* projections.apply({
		220	id: EventId.make("event:recovery:detached-session:attached"),
		221	type: "provider-session.attached",
		222	threadId,
		223	driver,
		224	providerInstanceId,
		225	occurredAt: now,
		226	payload: {
		227	id: providerSessionId,
		228	driver,
		229	providerInstanceId,
		230	status: "ready",
		231	cwd: "/workspace",
		232	model: modelSelection.model,
		233	capabilities: CodexProviderCapabilitiesV2,
		234	createdAt: now,
		235	updatedAt: now,
		236	lastError: null,
		237	},
		238	});
		239	// The server died before the release that should follow the detach.
		240	yield* projections.apply({
		241	id: EventId.make("event:recovery:detached-session:detached"),
		242	type: "provider-session.detached",
		243	threadId,
		244	driver,
		245	providerInstanceId,
		246	occurredAt: now,
		247	payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
		248	});
		249	assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
		250
		251	const summary = yield* recovery.reconcile("startup");
		252
		253	assert.equal(summary.stoppedSessions, 1);
		254	const sql = yield* SqlClient.SqlClient;
		255	const rows = yield* sql<{ readonly status: string }>`
		256	SELECT status FROM orchestration_v2_projection_provider_sessions
		257	WHERE provider_session_id = ${providerSessionId}
		258	`;
		259	assert.deepEqual(rows, [{ status: "stopped" }]);
		260	assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
		261	}).pipe(Effect.provide(layerRecoveryTest)),
		262	);
		263
212	it.effect("selects unfinished recovery work without reading settled thread histories", () =>	264	it.effect("selects unfinished recovery work without reading settled thread histories", () =>
213	Effect.gen(function* () {	265	Effect.gen(function* () {
214	const projections = yield* ProjectionStore.ProjectionStoreV2;	266	const projections = yield* ProjectionStore.ProjectionStoreV2;

apps/server/src/orchestration-v2/ProjectionStore.ts+38-14

 @@ -3666,6 +3666,16 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
                   ON bindings.provider_session_id = sessions.provider_session_id
                 WHERE sessions.status NOT IN ('stopped', 'error')
                 UNION
+                -- A detach drops the binding but keeps the session's status.
+                SELECT sessions.thread_id
+                FROM orchestration_v2_projection_provider_sessions AS sessions
+                WHERE sessions.status NOT IN ('stopped', 'error')
+                  AND sessions.thread_id IS NOT NULL
+                  AND NOT EXISTS (
+                    SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings
+                    WHERE bindings.provider_session_id = sessions.provider_session_id
+                  )
+                UNION
                 SELECT thread_id FROM pending_provider_threads
                 UNION
                 SELECT nodes.thread_id FROM pending_provider_threads
 @@ -4123,24 +4133,38 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
               ORDER BY COALESCE(subagent.started_at, ''), subagent.subagent_id ASC
             `,
           sql<PayloadRow>`
-              SELECT DISTINCT session.payload_json
+              SELECT session.payload_json
               FROM orchestration_v2_projection_provider_sessions AS session
-              JOIN orchestration_v2_projection_provider_session_bindings AS binding
+              WHERE (
-                ON binding.provider_session_id = session.provider_session_id
+                  session.provider_session_id IN (
-              WHERE binding.thread_id = ${threadId}
+                    SELECT binding.provider_session_id
-                AND (
+                    FROM orchestration_v2_projection_provider_session_bindings AS binding
-                  session.status NOT IN ('stopped', 'error')
+                    WHERE binding.thread_id = ${threadId}
-                  OR session.provider_session_id IN (
+                  )
-                    SELECT provider_thread.provider_session_id
+                  AND (
-                    FROM orchestration_v2_projection_provider_threads AS provider_thread
+                    session.status NOT IN ('stopped', 'error')
-                    WHERE provider_thread.provider_thread_id IN (
+                    OR session.provider_session_id IN (
-                      SELECT run.provider_thread_id
+                      SELECT provider_thread.provider_session_id
-                      FROM orchestration_v2_projection_runs AS run
+                      FROM orchestration_v2_projection_provider_threads AS provider_thread
-                      WHERE run.thread_id = ${threadId}
+                      WHERE provider_thread.provider_thread_id IN (
-                        AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
+                        SELECT run.provider_thread_id
+                        FROM orchestration_v2_projection_runs AS run
+                        WHERE run.thread_id = ${threadId}
+                          AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
+                      )
                     )
                   )
                 )
+                -- This thread detached it, which keeps its status. Its process
+                -- did not outlive the server either, so recovery stops it.
+                OR (
+                  session.thread_id = ${threadId}
+                  AND session.status NOT IN ('stopped', 'error')
+                  AND NOT EXISTS (
+                    SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS binding
+                    WHERE binding.provider_session_id = session.provider_session_id
+                  )
+                )
               ORDER BY session.updated_at ASC, session.provider_session_id ASC
             `,
           sql<PayloadRow>`

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+26

...	@@ -7,6 +7,7 @@ import {	...	@@ -7,6 +7,7 @@ import {
7	type OrchestrationV2RestartCancelledBackgroundWork,	7	type OrchestrationV2RestartCancelledBackgroundWork,
8	type OrchestrationV2Subagent,	8	type OrchestrationV2Subagent,
9	type OrchestrationV2ThreadProjection,	9	type OrchestrationV2ThreadProjection,
		10	isOrchestrationV2WorkActive,
10	ThreadId,	11	ThreadId,
11	} from "@t3tools/contracts";	12	} from "@t3tools/contracts";
12	import * as Cause from "effect/Cause";	13	import * as Cause from "effect/Cause";
...	@@ -445,6 +446,31 @@ export const make = Effect.gen(function* () {	...	@@ -445,6 +446,31 @@ export const make = Effect.gen(function* () {
445	});	446	});
446	}	447	}
447	}	448	}
		449	// A blocking delegate_task wait dies with the turn that made it, and its
		450	// continuation never got the tool result. Let the child's result wake
		451	// the parent like an async task, so it reaches the continued turn.
		452	const cutRunIds = new Set(runs.map((run) => run.id));
		453	for (const task of projection.subagents ?? []) {
		454	if (
		455	task.origin !== "app_owned" \|\|
		456	task.runId === null \|\|
		457	!cutRunIds.has(task.runId) \|\|
		458	task.completionWake !== "settled_only" \|\|
		459	!isOrchestrationV2WorkActive(task.status)
		460	)
		461	continue;
		462	events.push({
		463	id: yield* allocateEventId(),
		464	type: "subagent.updated",
		465	threadId: projection.thread.id,
		466	runId: task.runId,
		467	nodeId: task.id,
		468	driver: task.driver,
		469	providerInstanceId: task.providerInstanceId,
		470	occurredAt: now,
		471	payload: { ...task, completionWake: "always", updatedAt: now },
		472	});
		473	}
448	// Process loss also orphans background-capable turn items on already-	474	// Process loss also orphans background-capable turn items on already-
449	// settled runs (e.g. post-settle Waiting work). Skip items already	475	// settled runs (e.g. post-settle Waiting work). Skip items already
450	// cancelled above for recovered nonterminal runs to avoid duplicate	476	// cancelled above for recovered nonterminal runs to avoid duplicate

apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+36

...	@@ -1681,6 +1681,42 @@ it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts	...	@@ -1681,6 +1681,42 @@ it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts
1681	}),	1681	}),
1682	);	1682	);
1683		1683
		1684	it.effect("ProviderSessionManagerV2 releases sessions in parallel on server shutdown", () =>
		1685	Effect.gen(function* () {
		1686	const state = yield* Ref.make(emptyState);
		1687	yield* Effect.gen(function* () {
		1688	const eventSink = yield* EventSink.EventSinkV2;
		1689	const idAllocator = yield* IdAllocator.IdAllocatorV2;
		1690	const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
		1691	const now = yield* DateTime.now;
		1692	for (const name of ["a", "b", "c"]) {
		1693	const threadId = ThreadId.make(`thread-provider-session-manager-parallel-shutdown-${name}`);
		1694	yield* eventSink.write({
		1695	events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
		1696	});
		1697	yield* manager.open({
		1698	threadId,
		1699	providerSessionId: yield* idAllocator.allocate.providerSession({
		1700	providerInstanceId: modelSelection.instanceId,
		1701	threadId,
		1702	}),
		1703	modelSelection,
		1704	runtimePolicy,
		1705	});
		1706	}
		1707
		1708	// Every close hangs until its time box ends. One time box must cover all
		1709	// three sessions, or a busy server outlives the launcher's kill timer.
		1710	const stopping = yield* manager.shutdown.pipe(Effect.forkChild);
		1711	yield* TestClock.adjust("30 seconds");
		1712	yield* Effect.yieldNow;
		1713	assert.isDefined(stopping.pollUnsafe());
		1714	}).pipe(
		1715	Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, hangSessionScopeClose: true })),
		1716	);
		1717	}),
		1718	);
		1719
1684	it.effect("ProviderSessionManagerV2 closes event subscriptions normally on server shutdown", () =>	1720	it.effect("ProviderSessionManagerV2 closes event subscriptions normally on server shutdown", () =>
1685	Effect.gen(function* () {	1721	Effect.gen(function* () {
1686	const state = yield* Ref.make(emptyState);	1722	const state = yield* Ref.make(emptyState);

apps/server/src/orchestration-v2/ProviderSessionManager.ts+3-3

...	@@ -1999,8 +1999,8 @@ export const layerWithOptions = (	...	@@ -1999,8 +1999,8 @@ export const layerWithOptions = (
1999	// Parent of every session scope. On layer close, shutdown releases the	1999	// Parent of every session scope. On layer close, shutdown releases the
2000	// live sessions first, then closes any session whose open is still in	2000	// live sessions first, then closes any session whose open is still in
2001	// flight, time-boxed so a stuck adapter cannot hold up server shutdown.	2001	// flight, time-boxed so a stuck adapter cannot hold up server shutdown.
2002	// Parallel, so one session whose close hangs does not stop the rest from	2002	// Both steps run in parallel. The service launcher kills the server
2003	// closing within the time box.	2003	// 5 s after it asks it to stop, and a busy server can hold 15 sessions.
2004	const sessionScopes = yield* Scope.make("parallel");	2004	const sessionScopes = yield* Scope.make("parallel");
2005	const shutdown = Effect.gen(function* () {	2005	const shutdown = Effect.gen(function* () {
2006	const activeSessions = [...(yield* Ref.get(sessions)).values()];	2006	const activeSessions = [...(yield* Ref.get(sessions)).values()];
...	@@ -2018,7 +2018,7 @@ export const layerWithOptions = (	...	@@ -2018,7 +2018,7 @@ export const layerWithOptions = (
2018	}),	2018	}),
2019	),	2019	),
2020	),	2020	),
2021	{ discard: true },	2021	{ concurrency: "unbounded", discard: true },
2022	);	2022	);
2023	});	2023	});
2024	yield* Effect.addFinalizer(() =>	2024	yield* Effect.addFinalizer(() =>

apps/server/src/orchestration-v2/RestartBackgroundNote.ts+38

...	@@ -123,6 +123,44 @@ export function isRestartNoteSource(	...	@@ -123,6 +123,44 @@ export function isRestartNoteSource(
123	);	123	);
124	}	124	}
125		125
		126	/**
		127	* Work an agent ends its turn to wait for: a monitor, a subagent, or a
		128	* background task. A restart that cancels it wakes the settled turn, because
		129	* nothing else will. A shell does not wake it: most are dev servers that
		130	* nobody waits on, so their note waits for the next user message.
		131	*/
		132	export function isAwaitedWork(work: Pick<Work, "kind">): boolean {
		133	return work.kind !== "shell";
		134	}
		135
		136	/**
		137	* A run's status as its waiter should read it: a delegating parent,
		138	* `task_status`, or `t3_thread_wait`. A turn that ended to wait on work the
		139	* restart then cancelled did not finish, so it is "interrupted", not
		140	* "completed", and its last message is not a final result.
		141	*/
		142	export function runOutcome(
		143	run: Pick<OrchestrationV2Run, "status" \| "restartCancelledBackgroundWork">,
		144	): OrchestrationV2Run["status"] {
		145	return (run.status === "completed" \|\| run.status === "cancelled") &&
		146	(run.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
		147	? "interrupted"
		148	: run.status;
		149	}
		150
		151	/**
		152	* Whether a restart continuation may prompt from `source`: a turn the restart
		153	* cut mid-way, or a settled turn that lost awaited work in the restart.
		154	*/
		155	export function isRestartContinuationSource(
		156	source: Pick<OrchestrationV2Run, "status" \| "activeAttemptId" \| "restartCancelledBackgroundWork">,
		157	providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" \| "status">>,
		158	): boolean {
		159	return isRestartNoteSource(source, providerTurns)
		160	? (source.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
		161	: source.status === "cancelled";
		162	}
		163
126	/**	164	/**
127	* The note a restart continuation of `source` carries, and whether the turn it	165	* The note a restart continuation of `source` carries, and whether the turn it
128	* continues had settled (the note is then the whole prompt). A continuation cut	166	* continues had settled (the note is then the whole prompt). A continuation cut

apps/server/src/orchestration-v2/RestartContinuation.test.ts+191-40

...	@@ -223,6 +223,48 @@ it.effect.each(["completed", "waiting", "cancelled"] as const)(	...	@@ -223,6 +223,48 @@ it.effect.each(["completed", "waiting", "cancelled"] as const)(
223	}),	223	}),
224	);	224	);
225		225
		226	it.effect.each(["completed", "waiting", "cancelled"] as const)(
		227	"wakes a %s run whose monitor the restart cancelled, with the note as its prompt",
		228	(status) =>
		229	Effect.gen(function* () {
		230	const base = makeProjection();
		231	const work = [
		232	{ kind: "monitor" as const, label: "Watch CI run 42" },
		233	{ kind: "shell" as const, label: "vp run dev" },
		234	];
		235	const projection = {
		236	...base,
		237	runs: [{ ...base.runs[0]!, status, restartCancelledBackgroundWork: work }],
		238	providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
		239	} as unknown as OrchestrationV2ThreadProjection;
		240	const commands: Parameters<
		241	ThreadManagementService.ThreadManagementService["Service"]["dispatch"]
		242	>[0][] = [];
		243	yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
		244	Effect.provide(
		245	Layer.merge(
		246	Layer.mock(ThreadManagementService.ThreadManagementService)({
		247	getThreadRecords: () => Effect.succeed(projection),
		248	recoverDelegatedTask: () => Effect.void,
		249	dispatch: (command) => {
		250	commands.push(command);
		251	return Effect.succeed({} as never);
		252	},
		253	}),
		254	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
		255	),
		256	),
		257	);
		258	assert.lengthOf(commands, 1);
		259	const command = commands[0]!;
		260	assert.equal(command.type, "message.dispatch");
		261	if (command.type !== "message.dispatch") return;
		262	assert.equal(command.restartContinuationOfRunId, runId);
		263	assert.include(command.text, "- monitor: Watch CI run 42");
		264	assert.include(command.text, "- shell: vp run dev");
		265	}),
		266	);
		267
226	it.effect("does not continue a failed run that lost background work", () =>	268	it.effect("does not continue a failed run that lost background work", () =>
227	Effect.gen(function* () {	269	Effect.gen(function* () {
228	const base = makeProjection();	270	const base = makeProjection();
...	@@ -319,6 +361,62 @@ it.effect.each([	...	@@ -319,6 +361,62 @@ it.effect.each([
319	}),	361	}),
320	);	362	);
321		363
		364	it.effect("lets a cut blocking delegate_task wake its parent like an async task", () =>
		365	Effect.gen(function* () {
		366	const base = makeProjection();
		367	const task = (id: string, completionWake: "settled_only" \| "always", status: string) => ({
		368	id: NodeId.make(id),
		369	threadId,
		370	runId,
		371	origin: "app_owned",
		372	childThreadId: ThreadId.make(`thread:${id}`),
		373	driver,
		374	providerInstanceId: instanceId,
		375	completionWake,
		376	status,
		377	});
		378	const projection = {
		379	...base,
		380	subagents: [
		381	task("task:waited", "settled_only", "running"),
		382	task("task:async", "always", "running"),
		383	task("task:finished", "settled_only", "completed"),
		384	],
		385	} as unknown as OrchestrationV2ThreadProjection;
		386	let committed: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] \| undefined;
		387	const recovery = yield* ProviderRuntimeRecovery.make.pipe(
		388	Effect.provide(
		389	Layer.mergeAll(
		390	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
		391	Layer.mock(ProjectionStore.ProjectionStoreV2)({
		392	getRecoveryThreadIds: () => Effect.succeed([threadId]),
		393	getRuntimeRecoveryProjection: () => Effect.succeed(projection),
		394	}),
		395	Layer.mock(EventSink.EventSinkV2)({
		396	commitCommand: (input) => {
		397	committed = input;
		398	return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
		399	},
		400	}),
		401	IdAllocator.layer,
		402	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
		403	Layer.mock(EffectOutbox.EffectOutboxV2)({
		404	reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
		405	}),
		406	),
		407	),
		408	);
		409	yield* recovery.reconcile("startup");
		410	const upgraded = (committed?.events ?? []).flatMap((event) =>
		411	event.type === "subagent.updated" ? [event.payload] : [],
		412	);
		413	assert.deepEqual(
		414	upgraded.map((row) => ({ id: row.id, completionWake: row.completionWake })),
		415	[{ id: NodeId.make("task:waited"), completionWake: "always" }],
		416	);
		417	}),
		418	);
		419
322	it.effect("does not duplicate delivery and yields to newer user work or opt-out", () =>	420	it.effect("does not duplicate delivery and yields to newer user work or opt-out", () =>
323	Effect.gen(function* () {	421	Effect.gen(function* () {
324	let projection = makeProjection();	422	let projection = makeProjection();
...	@@ -424,7 +522,7 @@ it.effect("prepares no continuation for background work another provider thread	...	@@ -424,7 +522,7 @@ it.effect("prepares no continuation for background work another provider thread
424	);	522	);
425	yield* recovery.prepareForShutdown;	523	yield* recovery.prepareForShutdown;
426	assert.lengthOf(writes, 0);	524	assert.lengthOf(writes, 0);
427	// Work on the settled run's own provider thread must not wake it either.	525	// The same work on the settled run's own provider thread does wake it.
428	const ownWork = {	526	const ownWork = {
429	...projection,	527	...projection,
430	turnItems: [{ ...projection.turnItems[0]!, providerThreadId }],	528	turnItems: [{ ...projection.turnItems[0]!, providerThreadId }],
...	@@ -451,18 +549,54 @@ it.effect("prepares no continuation for background work another provider thread	...	@@ -451,18 +549,54 @@ it.effect("prepares no continuation for background work another provider thread
451	),	549	),
452	);	550	);
453	yield* ownRecovery.prepareForShutdown;	551	yield* ownRecovery.prepareForShutdown;
454	assert.lengthOf(writes, 0);	552	assert.lengthOf(writes, 1);
		553	assert.deepEqual(writes[0]!.effects[0]?.request, {
		554	type: "provider-runtime.continue",
		555	sourceRunId: runId,
		556	});
455	}),	557	}),
456	);	558	);
457		559
		560	const backgroundSubagent = {
		561	turnItem: {
		562	id: "turn-item:background-subagent",
		563	runId,
		564	nodeId: null,
		565	providerThreadId,
		566	providerInstanceId: instanceId,
		567	type: "subagent",
		568	subagentId: "subagent:background",
		569	title: "Background reviewer",
		570	status: "running",
		571	},
		572	work: {
		573	kind: "subagent" as const,
		574	label: "Background reviewer",
		575	id: "turn-item:background-subagent",
		576	},
		577	};
		578	const devServer = {
		579	turnItem: {
		580	id: "turn-item:dev-server",
		581	runId,
		582	nodeId: null,
		583	providerThreadId,
		584	providerInstanceId: instanceId,
		585	type: "command_execution",
		586	input: "vp run dev",
		587	status: "running",
		588	},
		589	work: { kind: "shell" as const, label: "vp run dev", id: "turn-item:dev-server" },
		590	};
		591
458	it.effect.each([	592	it.effect.each([
459	["completed", scratchProjectId],	593	["completed", scratchProjectId, backgroundSubagent, true],
460	["completed", ProjectId.make("restart-project")],	594	["waiting", ProjectId.make("restart-project"), backgroundSubagent, true],
461	["waiting", scratchProjectId],	595	["completed", ProjectId.make("restart-project"), devServer, false],
462	["waiting", ProjectId.make("restart-project")],	596	["waiting", scratchProjectId, devServer, false],
463	] as const)(	597	] as const)(
464	"cleans up background work without waking a %s run in project %s",	598	"cleans up background work of a %s run in project %s and wakes it only for awaited work",
465	([status, projectId]) =>	599	([status, projectId, background, wakes]) =>
466	Effect.gen(function* () {	600	Effect.gen(function* () {
467	const base = makeProjection();	601	const base = makeProjection();
468	const projection = {	602	const projection = {
...	@@ -470,29 +604,20 @@ it.effect.each([	...	@@ -470,29 +604,20 @@ it.effect.each([
470	thread: { ...base.thread, projectId },	604	thread: { ...base.thread, projectId },
471	runs: [{ ...base.runs[0]!, status }],	605	runs: [{ ...base.runs[0]!, status }],
472	providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],	606	providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
473	turnItems: [	607	turnItems: [background.turnItem],
474	{	608	subagents:
475	id: "turn-item:background-subagent",	609	background === backgroundSubagent
476	runId,	610	? [
477	nodeId: null,	611	{
478	providerThreadId,	612	id: "subagent:background",
479	providerInstanceId: instanceId,	613	runId,
480	type: "subagent",	614	origin: "provider_native",
481	subagentId: "subagent:background",	615	driver,
482	title: "Background reviewer",	616	providerInstanceId: instanceId,
483	status: "running",	617	status: "running",
484	},	618	},
485	],	619	]
486	subagents: [	620	: [],
487	{
488	id: "subagent:background",
489	runId,
490	origin: "provider_native",
491	driver,
492	providerInstanceId: instanceId,
493	status: "running",
494	},
495	],
496	} as unknown as OrchestrationV2ThreadProjection;	621	} as unknown as OrchestrationV2ThreadProjection;
497	for (const trigger of ["startup", "shutdown"] as const) {	622	for (const trigger of ["startup", "shutdown"] as const) {
498	const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];	623	const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
...	@@ -530,25 +655,51 @@ it.effect.each([	...	@@ -530,25 +655,51 @@ it.effect.each([
530	);	655	);
531	if (trigger === "shutdown") yield* recovery.prepareForShutdown;	656	if (trigger === "shutdown") yield* recovery.prepareForShutdown;
532	yield* recovery.reconcile(trigger);	657	yield* recovery.reconcile(trigger);
533	assert.lengthOf(writes, 0);	658	// Shutdown records the wake before it cancels the work; a startup
		659	// after a kill records it with the cancellation.
		660	const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects];
		661	assert.deepEqual(
		662	recorded.map((effect) => effect.request),
		663	wakes ? [{ type: "provider-runtime.continue", sourceRunId: runId }] : [],
		664	);
534	assert.lengthOf(commits, 1);	665	assert.lengthOf(commits, 1);
535	assert.lengthOf(commits[0]!.effects, 0);
536	const events = commits[0]!.events;	666	const events = commits[0]!.events;
537	for (const type of ["turn-item.updated", "subagent.updated"] as const)	667	assert.isTrue(
538	assert.isTrue(	668	events.some(
539	events.some((event) => event.type === type && event.payload.status === "cancelled"),	669	(event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
540	);	670	),
		671	);
541	const note = events.find((event) => event.type === "run.background-work-cancelled");	672	const note = events.find((event) => event.type === "run.background-work-cancelled");
542	assert.equal(note?.runId, runId);	673	assert.equal(note?.runId, runId);
543	assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [	674	assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [background.work]);
544	{ kind: "subagent", label: "Background reviewer", id: "turn-item:background-subagent" },
545	]);
546	if (status === "completed")	675	if (status === "completed")
547	assert.isFalse(events.some((event) => event.type === "run.updated"));	676	assert.isFalse(events.some((event) => event.type === "run.updated"));
548	}	677	}
549	}),	678	}),
550	);	679	);
551		680
		681	it("does not wake a settled run for work left by a session that already stopped", () => {
		682	const base = makeProjection();
		683	const projection = {
		684	...base,
		685	runs: [{ ...base.runs[0]!, status: "completed" }],
		686	providerThreads: [{ ...base.providerThreads[0]!, status: "idle" }],
		687	turnItems: [backgroundSubagent.turnItem],
		688	} as unknown as OrchestrationV2ThreadProjection;
		689	assert.equal(restartContinuationRun(projection, now)?.id, runId);
		690	for (const status of ["stopped", "error"])
		691	assert.isUndefined(
		692	restartContinuationRun(
		693	{
		694	...projection,
		695	providerSessions: [{ ...projection.providerSessions[0]!, status }],
		696	} as OrchestrationV2ThreadProjection,
		697	now,
		698	),
		699	status,
		700	);
		701	});
		702
552	it.effect("does not cancel or resume a run that completes while shutdown intent commits", () =>	703	it.effect("does not cancel or resume a run that completes while shutdown intent commits", () =>
553	Effect.gen(function* () {	704	Effect.gen(function* () {
554	let projection = makeProjection();	705	let projection = makeProjection();

apps/server/src/orchestration-v2/RestartContinuation.ts+56-12

 @@ -16,6 +16,10 @@ import * as ServerSettings from "../serverSettings.ts";
 import { isNativeMaintenanceCommand } from "./Orchestrator.ts";
 import * as ThreadManagementService from "./ThreadManagementService.ts";
 import {
+  cancelledRosterTaskWork,
+  cancelledTurnItemWork,
+  isAwaitedWork,
+  isRestartContinuationSource,
   isRestartNoteSource,
   restartCancelledBackgroundWorkNote,
   restartContinuationNote,
 @@ -29,7 +33,9 @@ const MAX_CONTINUATION_CHAIN = 3;
 /**
  * Whether a restart must leave this run for the user: the thread is put away,
  * the provider turn already ended, or the user owes an answer. Checked when
- * intent is recorded and again when the continuation is delivered.
+ * intent is recorded and again when the continuation is delivered. A
+ * `settled` run is woken for the awaited work it lost, so its ended provider
+ * turn does not block it.
  */
 function restartContinuationBlocked(
   projection: Pick<
 @@ -38,6 +44,7 @@ function restartContinuationBlocked(
   >,
   source: OrchestrationV2Run,
   now: DateTime.Utc,
+  settled: boolean,
 ): boolean {
   const { thread } = projection;
   if (
 @@ -49,6 +56,7 @@ function restartContinuationBlocked(
     return true;
   const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId);
   if (
+    !settled &&
     turn !== undefined &&
     (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")
   )
 @@ -82,8 +90,38 @@ function restartContinuationBlocked(
 }
 /**
- * Resume only an unfinished root turn. Leftover background work is cleaned up
+ * Open awaited work (see `isAwaitedWork`) on a provider thread. App-owned
- * separately and reported on the next user turn; it must not wake a settled run.
+ * delegations are not counted: their own threads continue and report back.
+ */
+function hasOpenAwaitedWork(
+  projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">,
+  providerThread: ProjectionRuntimeRecoveryState["providerThreads"][number],
+): boolean {
+  const openItem = projection.turnItems.some((item) => {
+    if (item.status !== "pending" && item.status !== "running" && item.status !== "waiting")
+      return false;
+    if (item.type === "subagent" && item.origin === "app_owned" && item.childThreadId !== null)
+      return false;
+    const work = cancelledTurnItemWork(item);
+    const providerThreadId =
+      item.providerThreadId ??
+      projection.runs.find((run) => run.id === item.runId)?.providerThreadId;
+    return work !== undefined && isAwaitedWork(work) && providerThreadId === providerThread.id;
+  });
+  return (
+    openItem ||
+    (providerThread.ownerNodeId === null &&
+      (providerThread.pendingBackgroundTasks ?? []).some((task) =>
+        isAwaitedWork(cancelledRosterTaskWork(task)),
+      ))
+  );
+}
+/**
+ * The run a restart continues: an unfinished root turn, or a settled one
+ * waiting on awaited work that the restart cancels. A settled run needs a
+ * live session, so stale records from an older crash never wake it. Other
+ * leftover work is cleaned up and reported on the next user turn.
  */
 export function restartContinuationRun(
   projection: Pick<
 @@ -109,9 +147,10 @@ export function restartContinuationRun(
   if (!run) return;
   const preparedContinuation =
     run.status === "starting" && run.restartContinuationOfRunId !== undefined;
-  if (run.status !== "running" && !preparedContinuation) return;
+  const settled = run.status === "completed" || run.status === "waiting";
-  if (restartContinuationBlocked(projection, run, now)) return;
+  if (run.status !== "running" && !preparedContinuation && !settled) return;
-  const liveTurnRequired = !preparedContinuation;
+  if (restartContinuationBlocked(projection, run, now, settled)) return;
+  const liveTurnRequired = !preparedContinuation && !settled;
   if (projection.thread.providerInstanceId !== run.providerInstanceId) return;
   const providerThread = projection.providerThreads.find(
     (thread) => thread.id === run.providerThreadId,
 @@ -138,9 +177,10 @@ export function restartContinuationRun(
     session === undefined ||
     session.providerInstanceId !== run.providerInstanceId ||
     session.driver !== providerThread.driver ||
-    (liveTurnRequired && (session.status === "stopped" || session.status === "error"))
+    (!preparedContinuation && (session.status === "stopped" || session.status === "error"))
   )
     return;
+  if (settled) return hasOpenAwaitedWork(projection, providerThread) ? run : undefined;
   if (
     liveTurnRequired &&
     !projection.providerTurns.some(
 @@ -173,13 +213,17 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
       return;
     if (projection.messages.some((message) => message.id === messageId)) return;
     const source = projection.runs.find((run) => run.id === input.sourceRunId);
-    // Pending effects from older versions may target settled background work,
+    // Reconciliation cancels a waiting run it cannot checkpoint, so a settled
-    // including waiting runs that reconciliation subsequently cancelled.
+    // source can be cancelled too. It continues only for awaited work.
     if (
       !source ||
-      source.status !== "cancelled" ||
+      !isRestartContinuationSource(source, projection.providerTurns) ||
-      isRestartNoteSource(source, projection.providerTurns) ||
+      restartContinuationBlocked(
-      restartContinuationBlocked(projection, source, yield* DateTime.now)
+        projection,
+        source,
+        yield* DateTime.now,
+        isRestartNoteSource(source, projection.providerTurns),
+      )
     )
       return;
     // A user submission after reconciliation takes precedence over an automatic

apps/server/src/orchestration-v2/ThreadManagementService.test.ts+123

...	@@ -597,6 +597,7 @@ it.effect.each([	...	@@ -597,6 +597,7 @@ it.effect.each([
597	Layer.provide(	597	Layer.provide(
598	Layer.mock(Orchestrator.OrchestratorV2)({	598	Layer.mock(Orchestrator.OrchestratorV2)({
599	getThreadEventSequence: () => Effect.succeed(0),	599	getThreadEventSequence: () => Effect.succeed(0),
		600	restartContinuationPending: () => Effect.succeed(false),
600	// No run update arrives, so the timeout path runs while a final	601	// No run update arrives, so the timeout path runs while a final
601	// projection read can still observe a terminal run.	602	// projection read can still observe a terminal run.
602	streamStoredEventsFrom: () =>	603	streamStoredEventsFrom: () =>
...	@@ -666,6 +667,7 @@ it.effect("waitForThread reads the run again only when the run updates", () =>	...	@@ -666,6 +667,7 @@ it.effect("waitForThread reads the run again only when the run updates", () =>
666	Layer.provide(	667	Layer.provide(
667	Layer.mock(Orchestrator.OrchestratorV2)({	668	Layer.mock(Orchestrator.OrchestratorV2)({
668	getThreadEventSequence: () => Effect.succeed(0),	669	getThreadEventSequence: () => Effect.succeed(0),
		670	restartContinuationPending: () => Effect.succeed(false),
669	// Only the run.updated stream carries events in this test.	671	// Only the run.updated stream carries events in this test.
670	streamStoredEventsFrom: (input) =>	672	streamStoredEventsFrom: (input) =>
671	input?.eventType === "run.updated"	673	input?.eventType === "run.updated"
...	@@ -705,6 +707,126 @@ it.effect("waitForThread reads the run again only when the run updates", () =>	...	@@ -705,6 +707,126 @@ it.effect("waitForThread reads the run again only when the run updates", () =>
705	}),	707	}),
706	);	708	);
707		709
		710	it.effect("waitForThread follows a restart continuation instead of the cut run", () =>
		711	Effect.gen(function* () {
		712	const projectId = ProjectId.make("project:thread-management:wait-restart");
		713	const threadId = ThreadId.make("thread:thread-management:wait-restart");
		714	const runId = RunId.make("run:thread-management:wait-restart");
		715	const continuationId = RunId.make("run:thread-management:wait-restart:2");
		716	const subscribed = yield* Deferred.make<void>();
		717	const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
		718	// The restart cut the run; its continuation effect has not started yet.
		719	let runs: Array<{
		720	id: RunId;
		721	status: OrchestrationV2Run["status"];
		722	restartContinuationOfRunId?: RunId;
		723	}> = [{ id: runId, status: "cancelled" }];
		724	let pending = true;
		725	const stored = (sequence: number, event: object) =>
		726	({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent;
		727	const layerTest = ThreadManagementService.layer.pipe(
		728	Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
		729	Layer.provide(
		730	Layer.mock(Orchestrator.OrchestratorV2)({
		731	getThreadEventSequence: () => Effect.succeed(0),
		732	restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId),
		733	awaitRestartContinuation: () => Effect.never,
		734	streamStoredEventsFrom: (input) =>
		735	input?.eventType === "run.updated"
		736	? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
		737	Stream.drain,
		738	Stream.concat(Stream.fromQueue(events)),
		739	)
		740	: Stream.never,
		741	getThreadRecords: () =>
		742	Effect.sync(
		743	() =>
		744	({
		745	thread: { id: threadId, projectId, deletedAt: null },
		746	runs,
		747	}) as unknown as OrchestrationV2ThreadProjection,
		748	),
		749	}),
		750	),
		751	Layer.provide(ServerSettings.layerTest()),
		752	Layer.provide(layerReceiptsTest),
		753	);
		754	const service = yield* ThreadManagementService.ThreadManagementService.pipe(
		755	Effect.provide(layerTest),
		756	);
		757	const fiber = yield* service
		758	.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
		759	.pipe(Effect.forkChild);
		760	yield* Deferred.await(subscribed);
		761	const continuation = { id: continuationId, restartContinuationOfRunId: runId };
		762	runs = [...runs, { ...continuation, status: "running" }];
		763	pending = false;
		764	yield* Queue.offer(
		765	events,
		766	stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }),
		767	);
		768	runs = [runs[0]!, { ...continuation, status: "completed" }];
		769	yield* Queue.offer(
		770	events,
		771	stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }),
		772	);
		773	const result = yield* Fiber.join(fiber);
		774
		775	expect(result).toMatchObject({
		776	timedOut: false,
		777	run: { id: continuationId, status: "completed" },
		778	});
		779	}),
		780	);
		781
		782	it.effect("waitForThread returns the cut run once its continuation is declined", () =>
		783	Effect.gen(function* () {
		784	const projectId = ProjectId.make("project:thread-management:wait-declined");
		785	const threadId = ThreadId.make("thread:thread-management:wait-declined");
		786	const runId = RunId.make("run:thread-management:wait-declined");
		787	const subscribed = yield* Deferred.make<void>();
		788	const declined = yield* Deferred.make<void>();
		789	let pending = true;
		790	const layerTest = ThreadManagementService.layer.pipe(
		791	Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
		792	Layer.provide(
		793	Layer.mock(Orchestrator.OrchestratorV2)({
		794	getThreadEventSequence: () => Effect.succeed(0),
		795	restartContinuationPending: () => Effect.succeed(pending),
		796	awaitRestartContinuation: () => Deferred.await(declined),
		797	// Declining records no thread event.
		798	streamStoredEventsFrom: (input) =>
		799	input?.eventType === "run.updated"
		800	? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
		801	Stream.drain,
		802	Stream.concat(Stream.never),
		803	)
		804	: Stream.never,
		805	getThreadRecords: () =>
		806	Effect.succeed({
		807	thread: { id: threadId, projectId, deletedAt: null },
		808	runs: [{ id: runId, status: "cancelled" }],
		809	} as unknown as OrchestrationV2ThreadProjection),
		810	}),
		811	),
		812	Layer.provide(ServerSettings.layerTest()),
		813	Layer.provide(layerReceiptsTest),
		814	);
		815	const service = yield* ThreadManagementService.ThreadManagementService.pipe(
		816	Effect.provide(layerTest),
		817	);
		818	const fiber = yield* service
		819	.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 })
		820	.pipe(Effect.forkChild);
		821	yield* Deferred.await(subscribed);
		822	pending = false;
		823	yield* Deferred.succeed(declined, undefined);
		824	const result = yield* Fiber.join(fiber);
		825
		826	expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "cancelled" } });
		827	}),
		828	);
		829
708	it.effect.each([	830	it.effect.each([
709	"running",	831	"running",
710	"missing",	832	"missing",
...	@@ -938,6 +1060,7 @@ it.effect.each([	...	@@ -938,6 +1060,7 @@ it.effect.each([
938	Layer.provide(	1060	Layer.provide(
939	Layer.mock(Orchestrator.OrchestratorV2)({	1061	Layer.mock(Orchestrator.OrchestratorV2)({
940	getThreadEventSequence: () => Effect.succeed(0),	1062	getThreadEventSequence: () => Effect.succeed(0),
		1063	restartContinuationPending: () => Effect.succeed(false),
941	getThreadRecords: () =>	1064	getThreadRecords: () =>
942	Effect.succeed({	1065	Effect.succeed({
943	thread: { id: threadId, projectId, deletedAt: null },	1066	thread: { id: threadId, projectId, deletedAt: null },

apps/server/src/orchestration-v2/ThreadManagementService.ts+68-43

 @@ -486,6 +486,20 @@ export function latestRun(
   return projection.runs.toSorted((left, right) => right.ordinal - left.ordinal)[0];
 }
+/** The newest run in `run`'s restart continuation chain, or `run` itself. */
+function latestRestartContinuation(
+  runs: ReadonlyArray<OrchestrationV2Run>,
+  run: OrchestrationV2Run,
+): OrchestrationV2Run {
+  let current = run;
+  for (let hops = 0; hops < runs.length; hops++) {
+    const next = runs.find((candidate) => candidate.restartContinuationOfRunId === current.id);
+    if (next === undefined) break;
+    current = next;
+  }
+  return current;
+}
 export function latestActiveRun(
   projection: Pick<OrchestrationV2ThreadProjection, "runs">,
 ): OrchestrationV2Run | undefined {
 @@ -844,65 +858,76 @@ const make = Effect.gen(function* () {
       if (selectedRun === undefined) {
         return { threadId: input.threadId, run: null, timedOut: false };
       }
-      if (isTerminalRunStatus(selectedRun.status)) {
+      // A restart cuts a run and continues it as a new run. Follow that chain,
-        return { threadId: input.threadId, run: selectedRun, timedOut: false };
+      // so a cut run is not the answer while its continuation is pending.
+      const outcome = (runs: ReadonlyArray<OrchestrationV2Run>) =>
+        Effect.gen(function* () {
+          const current = runs.find((candidate) => candidate.id === selectedRun.id);
+          if (current === undefined) {
+            return yield* new ThreadManagementRunNotFoundError({
+              threadId: input.threadId,
+              runId: selectedRun.id,
+            });
+          }
+          const run = latestRestartContinuation(runs, current);
+          const pending =
+            isTerminalRunStatus(run.status) &&
+            (yield* orchestrator.restartContinuationPending(run));
+          return { run, pending, settled: isTerminalRunStatus(run.status) && !pending };
+        });
+      const initial = yield* outcome(target.runs);
+      if (initial.settled) {
+        return { threadId: input.threadId, run: initial.run, timedOut: false };
       }
-      // Re-read the run only when the thread records a run update or its
+      // Re-read the runs only when the thread records an update to the waited
-      // deletion, instead of polling the projection for up to an hour. Each
+      // run or a continuation, or its deletion, instead of polling the
-      // stream keeps one event type, so transcript events never fill its buffer.
+      // projection for up to an hour. Each stream keeps one event type, so
+      // transcript events never fill its buffer. A declined continuation
+      // records no event, so its effect settling wakes the wait too.
       const wait = Stream.merge(
-        orchestrator.streamStoredEventsFrom({
+        Stream.merge(
-          threadId: input.threadId,
+          orchestrator.streamStoredEventsFrom({
-          afterSequence,
+            threadId: input.threadId,
-          eventType: "run.updated",
+            afterSequence,
-        }),
+            eventType: "run.updated",
-        orchestrator.streamStoredEventsFrom({
+          }),
-          threadId: input.threadId,
+          orchestrator.streamStoredEventsFrom({
-          afterSequence,
+            threadId: input.threadId,
-          eventType: "thread.deleted",
+            afterSequence,
-        }),
+            eventType: "thread.deleted",
-      ).pipe(
+          }),
-        Stream.mapError(loadError),
+        ).pipe(
-        Stream.filter(
+          Stream.mapError(loadError),
-          (stored) =>
+          Stream.filter(
-            stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id,
+            (stored) =>
+              stored.event.type !== "run.updated" ||
+              stored.event.payload.id === selectedRun.id ||
+              stored.event.payload.restartContinuationOfRunId !== undefined,
+          ),
         ),
+        initial.pending
+          ? Stream.fromEffect(orchestrator.awaitRestartContinuation(initial.run))
+          : Stream.empty,
+      ).pipe(
         Stream.mapEffect(() =>
-          getProjectThreadRecords(input, ["runs"], { runIds: [selectedRun.id] }).pipe(
+          getProjectThreadRecords(input, ["runs"]).pipe(
-            Effect.flatMap((current) => {
+            Effect.flatMap((current) => outcome(current.runs)),
-              const run = current.runs.find((candidate) => candidate.id === selectedRun.id);
-              return run === undefined
-                ? Effect.fail(
-                    new ThreadManagementRunNotFoundError({
-                      threadId: input.threadId,
-                      runId: selectedRun.id,
-                    }),
-                  )
-                : Effect.succeed(run);
-            }),
           ),
         ),
-        Stream.filter((run) => isTerminalRunStatus(run.status)),
+        Stream.filter((current) => current.settled),
         Stream.runHead,
         Effect.timeoutOption(Duration.millis(Math.max(1, input.timeoutMs))),
       );
       const waited = Option.flatten(yield* wait);
       if (Option.isSome(waited)) {
-        return { threadId: input.threadId, run: waited.value, timedOut: false };
+        return { threadId: input.threadId, run: waited.value.run, timedOut: false };
-      }
-      const current = yield* getProjectThreadRecords(input, ["runs"], { runIds: [selectedRun.id] });
-      const run = current.runs.find((candidate) => candidate.id === selectedRun.id);
-      if (run === undefined) {
-        return yield* new ThreadManagementRunNotFoundError({
-          threadId: input.threadId,
-          runId: selectedRun.id,
-        });
       }
       // The run may have reached a terminal status while the timeout was
       // winning the race; the final projection read decides what actually
       // happened, so only a still-active run counts as timed out.
-      return { threadId: input.threadId, run, timedOut: !isTerminalRunStatus(run.status) };
+      const final = yield* outcome((yield* getProjectThreadRecords(input, ["runs"])).runs);
+      return { threadId: input.threadId, run: final.run, timedOut: !final.settled };
     });
   const settleAfterRun: ThreadManagementServiceShape["settleAfterRun"] = Effect.fn(

apps/server/src/orchestration-v2/runtimeLayer.test.ts+27-14

 @@ -3251,18 +3251,28 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
       }),
   );
-  it.effect.each(["failed", "completed", "waiting", "cancelled"] as const)(
+  it.effect.each([
-    "does not admit a restart continuation of a %s run with only lost background work",
+    ["failed", "shell", false],
-    (status) =>
+    ["completed", "shell", false],
+    ["waiting", "shell", false],
+    ["cancelled", "shell", false],
+    // A settled run lost work it was waiting on, so nothing else wakes it.
+    ["failed", "monitor", false],
+    ["completed", "monitor", true],
+    ["waiting", "monitor", true],
+    ["cancelled", "monitor", true],
+  ] as const)(
+    "admits a restart continuation of a %s run that lost a %s: %s",
+    ([status, kind, admitted]) =>
       Effect.gen(function* () {
         const orchestrator = yield* Orchestrator.OrchestratorV2;
         const eventSink = yield* EventSink.EventSinkV2;
-        const threadId = ThreadId.make(`runtime-layer-restart-${status}-source`);
+        const threadId = ThreadId.make(`runtime-layer-restart-${status}-${kind}-source`);
         yield* orchestrator.dispatch({
           type: "thread.create",
           createdBy: "user",
           creationSource: "web",
-          commandId: CommandId.make(`restart-${status}-create`),
+          commandId: CommandId.make(`restart-${status}-${kind}-create`),
           threadId,
           projectId: ProjectId.make("restart-project"),
           title: "Restart",
 @@ -3276,9 +3286,9 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
           type: "message.dispatch",
           createdBy: "user",
           creationSource: "web",
-          commandId: CommandId.make(`restart-${status}-user-message`),
+          commandId: CommandId.make(`restart-${status}-${kind}-user-message`),
           threadId,
-          messageId: MessageId.make(`restart-${status}-user-message`),
+          messageId: MessageId.make(`restart-${status}-${kind}-user-message`),
           text: "Original work",
           attachments: [],
           modelSelection,
 @@ -3287,13 +3297,13 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
         const original = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
         const now = yield* DateTime.now;
         yield* eventSink.commitCommand({
-          commandId: CommandId.make(`restart-${status}-reconcile`),
+          commandId: CommandId.make(`restart-${status}-${kind}-reconcile`),
           threadId,
           commandType: "provider-runtime.reconcile",
           acceptedAt: now,
           events: [
             {
-              id: EventId.make(`restart-${status}-run`),
+              id: EventId.make(`restart-${status}-${kind}-run`),
               type: "run.updated",
               threadId,
               runId: original.id,
 @@ -3301,14 +3311,14 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
               payload: { ...original, status, completedAt: now },
             },
             {
-              id: EventId.make(`restart-${status}-work`),
+              id: EventId.make(`restart-${status}-${kind}-work`),
               type: "run.background-work-cancelled",
               threadId,
               runId: original.id,
               occurredAt: now,
               payload: {
                 runId: original.id,
-                restartCancelledBackgroundWork: [{ kind: "shell", label: "sleep 25" }],
+                restartCancelledBackgroundWork: [{ kind, label: "sleep 25" }],
               },
             },
           ],
 @@ -3318,9 +3328,9 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
           type: "message.dispatch",
           createdBy: "agent",
           creationSource: "server",
-          commandId: CommandId.make(`restart-${status}-continuation`),
+          commandId: CommandId.make(`restart-${status}-${kind}-continuation`),
           threadId,
-          messageId: MessageId.make(`restart-${status}-continuation`),
+          messageId: MessageId.make(`restart-${status}-${kind}-continuation`),
           text: "Note: the T3 server restarted.",
           attachments: [],
           modelSelection,
 @@ -3328,8 +3338,11 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
           restartContinuationOfRunId: original.id,
         });
         const projection = yield* orchestrator.getThreadProjection(threadId);
-        assert.lengthOf(projection.runs, 1);
         assert.equal(projection.runs[0]?.status, status);
+        assert.deepEqual(
+          projection.runs.slice(1).map((run) => run.restartContinuationOfRunId),
+          admitted ? [original.id] : [],
+        );
       }),
   );

apps/server/src/process/agentScope.test.ts+61-4

...	@@ -7,10 +7,18 @@ import * as NodeServices from "@effect/platform-node/NodeServices";	...	@@ -7,10 +7,18 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
7	import { describe, expect, it } from "@effect/vitest";	7	import { describe, expect, it } from "@effect/vitest";
8	import * as HostProcess from "@t3tools/shared/HostProcess";	8	import * as HostProcess from "@t3tools/shared/HostProcess";
9	import * as Effect from "effect/Effect";	9	import * as Effect from "effect/Effect";
		10	import * as Layer from "effect/Layer";
10	import { ChildProcessSpawner } from "effect/process";	11	import { ChildProcessSpawner } from "effect/process";
11		12
		13	import * as ServerConfig from "../config.ts";
12	import * as ProcessRunner from "../processRunner.ts";	14	import * as ProcessRunner from "../processRunner.ts";
13	import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts";	15	import {
		16	agentSliceMemoryLimits,
		17	classifyScope,
		18	make,
		19	ownedAgentSlice,
		20	type ScopeState,
		21	} from "./agentScope.ts";
14		22
15	const GiB = 1024 ** 3;	23	const GiB = 1024 ** 3;
16		24
...	@@ -55,11 +63,14 @@ describe("agentSliceMemoryLimits", () => {	...	@@ -55,11 +63,14 @@ describe("agentSliceMemoryLimits", () => {
55	});	63	});
56		64
57	// Runs the real service against a fake user manager. `units` holds the	65	// Runs the real service against a fake user manager. `units` holds the
58	// ActiveState and Result that `systemctl show` reports for each scope.	66	// ActiveState and Result that `systemctl show` reports for each scope or
		67	// slice, and `calls` records every command in order.
59	const withFakeSystemd = <A, E>(	68	const withFakeSystemd = <A, E>(
60	body: (input: {	69	body: (input: {
61	readonly scope: Effect.Success<typeof make>;	70	readonly scope: Effect.Success<typeof make>;
62	readonly units: Map<string, { activeState: string; result: string }>;	71	readonly units: Map<string, { activeState: string; result: string }>;
		72	readonly calls: ReadonlyArray<ReadonlyArray<string>>;
		73	readonly stateDir: string;
63	}) => Effect.Effect<A, E>,	74	}) => Effect.Effect<A, E>,
64	) =>	75	) =>
65	Effect.gen(function* () {	76	Effect.gen(function* () {
...	@@ -68,6 +79,7 @@ const withFakeSystemd = <A, E>(	...	@@ -68,6 +79,7 @@ const withFakeSystemd = <A, E>(
68	NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });	79	NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
69	}	80	}
70	const units = new Map<string, { activeState: string; result: string }>();	81	const units = new Map<string, { activeState: string; result: string }>();
		82	const calls: Array<ReadonlyArray<string>> = [];
71	const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({	83	const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({
72	stdout,	84	stdout,
73	stderr: "",	85	stderr: "",
...	@@ -81,6 +93,7 @@ const withFakeSystemd = <A, E>(	...	@@ -81,6 +93,7 @@ const withFakeSystemd = <A, E>(
81	const runner = ProcessRunner.ProcessRunner.of({	93	const runner = ProcessRunner.ProcessRunner.of({
82	run: ({ args }) =>	94	run: ({ args }) =>
83	Effect.sync(() => {	95	Effect.sync(() => {
		96	calls.push(args);
84	const [, verb, unit = ""] = args;	97	const [, verb, unit = ""] = args;
85	if (verb === "reset-failed") units.delete(unit);	98	if (verb === "reset-failed") units.delete(unit);
86	if (verb !== "show") return output("");	99	if (verb !== "show") return output("");
...	@@ -97,14 +110,25 @@ const withFakeSystemd = <A, E>(	...	@@ -97,14 +110,25 @@ const withFakeSystemd = <A, E>(
97	Effect.provideService(HostProcess.Platform, "linux"),	110	Effect.provideService(HostProcess.Platform, "linux"),
98	Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),	111	Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
99	);	112	);
100	return yield* body({ scope, units }).pipe(	113	const { stateDir } = yield* ServerConfig.ServerConfig;
		114	return yield* body({ scope, units, calls, stateDir }).pipe(
101	Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),	115	Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
102	);	116	);
103	}).pipe(Effect.scoped, Effect.provide(NodeServices.layer));	117	}).pipe(
		118	Effect.scoped,
		119	Effect.provide(
		120	ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-scope-" }).pipe(
		121	Layer.provideMerge(NodeServices.layer),
		122	),
		123	),
		124	);
104		125
105	const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>	126	const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>
106	launch.args.find((arg) => arg.startsWith("--unit="))?.slice("--unit=".length) ?? "";	127	launch.args.find((arg) => arg.startsWith("--unit="))?.slice("--unit=".length) ?? "";
107		128
		129	const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
		130	launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
		131
108	describe("AgentScope service", () => {	132	describe("AgentScope service", () => {
109	it.effect("labels a thread from its newest scope until its next session clears it", () =>	133	it.effect("labels a thread from its newest scope until its next session clears it", () =>
110	withFakeSystemd(({ scope, units }) =>	134	withFakeSystemd(({ scope, units }) =>
...	@@ -142,4 +166,37 @@ describe("AgentScope service", () => {	...	@@ -142,4 +166,37 @@ describe("AgentScope service", () => {
142	}),	166	}),
143	),	167	),
144	);	168	);
		169
		170	it.effect("stops agents a killed server left before the first launch", () =>
		171	withFakeSystemd(({ scope, units, calls, stateDir }) =>
		172	Effect.gen(function* () {
		173	units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
		174	const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
		175	const stops = calls.filter(([, verb]) => verb === "stop");
		176	expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
		177	// Agents run in this server's own slice; terminals stay outside it.
		178	expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
		179	// An agent that ignores SIGTERM is killed well inside the stop's wait.
		180	expect(agent.args).toContain("--property=TimeoutStopSec=10s");
		181	const terminal = yield* scope.wrap({
		182	command: "agent",
		183	args: [],
		184	name: "terminal",
		185	survivesRestart: true,
		186	});
		187	expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
		188	// Only the first launch looks for leftovers.
		189	expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1);
		190	}),
		191	),
		192	);
		193
		194	it.effect("leaves a quiet slice alone", () =>
		195	withFakeSystemd(({ scope, calls }) =>
		196	Effect.gen(function* () {
		197	yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
		198	expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
		199	}),
		200	),
		201	);
145	});	202	});

apps/server/src/process/agentScope.ts+73-14

 @@ -15,6 +15,7 @@ import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
+import * as ServerConfig from "../config.ts";
 import * as ProcessRunner from "../processRunner.ts";
 /**
 @@ -24,6 +25,18 @@ import * as ProcessRunner from "../processRunner.ts";
  */
 const AGENT_SLICE = "app-t3code-agents.slice";
+/**
+ * The child slice for one server's agents, named after its state directory.
+ * Only one server uses a state directory at a time, so an agent in this slice
+ * when the server starts was left by an earlier server on the same state,
+ * for example one the launcher killed during an update. Other servers on the
+ * machine, such as dev servers, have their own slice.
+ */
+export function ownedAgentSlice(stateDir: string): string {
+  const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
+  return `app-t3code-agents-${hash}.slice`;
+}
 const GiB = 1024 ** 3;
 /**
 @@ -84,9 +97,12 @@ function parseCgroupCounter(text: string, key: string): number | undefined {
   return match === null ? undefined : Number(match[1]);
 }
+// A stop sends SIGTERM, then SIGKILL after this long. systemd's 90 s default
+// would outlast the wait for a previous server's agents below.
+const STOP_TIMEOUT_OPTION = "--property=TimeoutStopSec=10s";
 const STOPPING_POLL_INTERVAL = "100 millis";
 const STOPPING_POLL_ATTEMPTS = 20;
-// A stopping scope can wait out TimeoutStopSec (90 s by default) for a stuck process.
+// A stopping scope can wait out its TimeoutStopSec for a stuck process.
 const CLEAR_FAILED_INTERVAL = "1 second";
 const CLEAR_FAILED_ATTEMPTS = 120;
 const MAX_TRACKED_THREADS = 500;
 @@ -99,6 +115,8 @@ export const make = Effect.gen(function* () {
   const fileSystem = yield* FileSystem.FileSystem;
   const path = yield* Path.Path;
   const hostEnvironment = yield* HostProcess.Environment;
+  const { stateDir } = yield* ServerConfig.ServerConfig;
+  const ownedSlice = ownedAgentSlice(stateDir);
   const layerScope = yield* Effect.scope;
   const resolve = (command: string, env: NodeJS.ProcessEnv) =>
 @@ -111,8 +129,45 @@ export const make = Effect.gen(function* () {
   const run = (command: string, args: ReadonlyArray<string>) =>
     runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option);
-  // Runs once, on the first agent launch. It proves this process can reach a
+  const readCgroupFile = (controlGroup: string, file: string) =>
-  // systemd user manager, creates the agents slice, and sets its limits.
+    fileSystem
+      .readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
+      .pipe(Effect.orElseSucceed(() => ""));
+  // Old agents keep their scopes when the server is killed, and the restarted
+  // server resumes their sessions. Stop them first, so two copies of one
+  // session never work in the same worktree.
+  const stopLeftoverAgents = (systemctl: string) =>
+    Effect.gen(function* () {
+      const shown = yield* run(systemctl, [
+        "--user",
+        "show",
+        ownedSlice,
+        "--property=ActiveState,ControlGroup",
+      ]);
+      if (shown._tag === "None") return;
+      const values = parseKeyValues(shown.value.stdout);
+      if (values.get("ActiveState") !== "active") return;
+      // An active slice can be empty once its last agent exited.
+      const controlGroup = values.get("ControlGroup") ?? "";
+      const events = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
+      if (parseCgroupCounter(events, "populated") === 0) return;
+      yield* Effect.logWarning("Stopping agents left by a previous server", { slice: ownedSlice });
+      // Agents get SIGTERM, then SIGKILL after their stop timeout.
+      const stopped = yield* runner
+        .run({ command: systemctl, args: ["--user", "stop", ownedSlice], timeout: "30 seconds" })
+        .pipe(Effect.option);
+      if (stopped._tag === "None" || stopped.value.code !== 0) {
+        yield* Effect.logWarning("Could not stop agents left by a previous server", {
+          slice: ownedSlice,
+          stderr: stopped._tag === "Some" ? stopped.value.stderr.trim() : undefined,
+        });
+      }
+    });
+  // Runs once, on the first agent or terminal launch. It proves this process
+  // can reach a systemd user manager, stops agents a previous server left,
+  // creates the agents slice, and sets its limits.
   const systemd = yield* Effect.cached(
     Effect.gen(function* () {
       const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;
 @@ -120,6 +175,7 @@ export const make = Effect.gen(function* () {
       const systemdRun = yield* resolve("systemd-run", hostEnvironment);
       const systemctl = yield* resolve("systemctl", hostEnvironment);
       if (systemdRun === null || systemctl === null) return undefined;
+      yield* stopLeftoverAgents(systemctl);
       const probe = (options: ReadonlyArray<string>) =>
         run(systemdRun, [
           "--user",
 @@ -133,12 +189,13 @@ export const make = Effect.gen(function* () {
         ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0));
       // Stop the whole scope when the kernel kills one of its processes, and
       // record the result as oom-kill. Scopes take OOMPolicy from systemd 253;
-      // older user managers still get plain scopes, without the OOM label.
+      // older user managers still get scopes, without the OOM label.
-      const oomOptions = ["--property=OOMPolicy=stop"];
+      const stopOptions = [STOP_TIMEOUT_OPTION];
+      const oomOptions = [...stopOptions, "--property=OOMPolicy=stop"];
       const scopeOptions = (yield* probe(oomOptions))
         ? oomOptions
-        : (yield* probe([]))
+        : (yield* probe(stopOptions))
-          ? []
+          ? stopOptions
           : undefined;
       if (scopeOptions === undefined) {
         yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
 @@ -176,11 +233,6 @@ export const make = Effect.gen(function* () {
   // at a time, so only its latest scope can explain a failure.
   const scopeByThread = new Map<string, { unit: string; oomKilled: boolean }>();
-  const readCgroupFile = (controlGroup: string, file: string) =>
-    fileSystem
-      .readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
-      .pipe(Effect.orElseSucceed(() => ""));
   const readScope = (systemctl: string, unit: string) =>
     Effect.gen(function* () {
       const shown = yield* run(systemctl, [
 @@ -290,7 +342,14 @@ export const make = Effect.gen(function* () {
     });
   return {
-    wrap: Effect.fn("AgentScope.wrap")(function* ({ command, args, name, threadId, env }) {
+    wrap: Effect.fn("AgentScope.wrap")(function* ({
+      command,
+      args,
+      name,
+      threadId,
+      env,
+      survivesRestart,
+    }) {
       const resolved = yield* resolve(command, env ?? hostEnvironment);
       // Leave unresolved commands alone so the spawn reports the missing binary.
       if (resolved === null) return { command, args };
 @@ -301,7 +360,7 @@ export const make = Effect.gen(function* () {
       const systemdScope: AgentSystemdScope = {
         systemdRun: scope.systemdRun,
         runtimeDir: scope.runtimeDir,
-        slice: AGENT_SLICE,
+        slice: survivesRestart === true ? AGENT_SLICE : ownedSlice,
         unit,
         // Without a thread nothing reads the result, so let systemd unload
         // the unit even when it fails.

apps/server/src/terminal/Manager.ts+1

...	@@ -2196,6 +2196,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func	...	@@ -2196,6 +2196,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
2196	args: candidate.args ?? [],	2196	args: candidate.args ?? [],
2197	name: "terminal",	2197	name: "terminal",
2198	env: spawnEnv,	2198	env: spawnEnv,
		2199	survivesRestart: true,
2199	});	2200	});
2200	const attempt = yield* Effect.result(	2201	const attempt = yield* Effect.result(
2201	options.ptyAdapter.spawn({	2202	options.ptyAdapter.spawn({

apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts+2-1

...	@@ -1,6 +1,6 @@	...	@@ -1,6 +1,6 @@
1	import { describe, expect, it } from "vite-plus/test";	1	import { describe, expect, it } from "vite-plus/test";
2	import {	2	import {
3	type EnvironmentId,	3	EnvironmentId,
4	ProviderDriverKind,	4	ProviderDriverKind,
5	ProviderInstanceId,	5	ProviderInstanceId,
6	type ServerProvider,	6	type ServerProvider,
...	@@ -1076,6 +1076,7 @@ describe("getProviderUpdateRunToastView", () => {	...	@@ -1076,6 +1076,7 @@ describe("getProviderUpdateRunToastView", () => {
1076	providerDriver: string,	1076	providerDriver: string,
1077	result: Parameters<typeof getProviderUpdateRunToastView>[0][number]["result"],	1077	result: Parameters<typeof getProviderUpdateRunToastView>[0][number]["result"],
1078	) => ({	1078	) => ({
		1079	environmentId: EnvironmentId.make(machineLabel),
1079	machineLabel,	1080	machineLabel,
1080	driver: driver(providerDriver),	1081	driver: driver(providerDriver),
1081	instanceId: instanceId(providerDriver),	1082	instanceId: instanceId(providerDriver),

apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts+36-14

 @@ -333,6 +333,7 @@ export function getProviderUpdateProgressToastView(input: {
 /** One provider update sent by the cross-machine "Update all", with its result. */
 export interface ProviderUpdateRun {
+  readonly environmentId: EnvironmentId;
   readonly machineLabel: string;
   readonly driver: ProviderDriverKind;
   readonly instanceId: ProviderInstanceId;
 @@ -342,6 +343,35 @@ export interface ProviderUpdateRun {
   >;
 }
+/**
+ * What one provider update in a cross-machine run did. A failure carries a
+ * line that names the machine and provider, such as "alvin · Codex: npm exited".
+ */
+export function getProviderUpdateRunOutcome(
+  run: ProviderUpdateRun,
+):
+  | { readonly status: "interrupted" | "updated" }
+  | { readonly status: "failed"; readonly line: string } {
+  if (isAtomCommandInterrupted(run.result)) return { status: "interrupted" };
+  const label = `${run.machineLabel} · ${PROVIDER_DISPLAY_NAMES[run.driver] ?? run.driver}`;
+  if (run.result._tag === "Failure") {
+    const error = squashAtomCommandFailure(run.result);
+    return {
+      status: "failed",
+      line: `${label}: ${error instanceof Error ? error.message : "Provider update failed."}`,
+    };
+  }
+  const updateState = run.result.value.providers.find(
+    (provider) => provider.instanceId === run.instanceId,
+  )?.updateState;
+  return updateState?.status === "succeeded"
+    ? { status: "updated" }
+    : {
+        status: "failed",
+        line: `${label}: ${updateState?.message ?? "Provider update did not finish."}`,
+      };
+}
 /**
  * Summarize a cross-machine "Update all" as one toast, or null when every
  * request was interrupted. Each update that did not succeed gets its own line,
 @@ -350,23 +380,15 @@ export interface ProviderUpdateRun {
 export function getProviderUpdateRunToastView(
   runs: ReadonlyArray<ProviderUpdateRun>,
 ): Pick<ProviderUpdateToastView, "type" | "title" | "description"> | null {
-  const settled = runs.filter((run) => !isAtomCommandInterrupted(run.result));
+  const settled = runs
+    .map(getProviderUpdateRunOutcome)
+    .filter((outcome) => outcome.status !== "interrupted");
   if (settled.length === 0) {
     return null;
   }
-  const failureLines = settled.flatMap((run) => {
+  const failureLines = settled.flatMap((outcome) =>
-    const label = `${run.machineLabel} · ${PROVIDER_DISPLAY_NAMES[run.driver] ?? run.driver}`;
+    outcome.status === "failed" ? [outcome.line] : [],
-    if (run.result._tag === "Failure") {
+  );
-      const error = squashAtomCommandFailure(run.result);
-      return [`${label}: ${error instanceof Error ? error.message : "Provider update failed."}`];
-    }
-    const updateState = run.result.value.providers.find(
-      (provider) => provider.instanceId === run.instanceId,
-    )?.updateState;
-    return updateState?.status === "succeeded"
-      ? []
-      : [`${label}: ${updateState?.message ?? "Provider update did not finish."}`];
-  });
   if (failureLines.length === 0) {
     return {
       type: "success",

apps/web/src/components/ProviderUpdatesAction.tsx+6-3

...	@@ -20,7 +20,8 @@ import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";	...	@@ -20,7 +20,8 @@ import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";
20	* Each server queues updates that share an installer, so sending them all	20	* Each server queues updates that share an installer, so sending them all
21	* together is safe. The server still checks permissions: a session that	21	* together is safe. The server still checks permissions: a session that
22	* cannot operate a machine gets a failure line for it. The returned promise	22	* cannot operate a machine gets a failure line for it. The returned promise
23	* settles when every update has finished.	23	* settles with every result when every update has finished. `quiet` skips the
		24	* toast, for a caller that reports the results itself.
24	*/	25	*/
25	export function useProviderUpdates() {	26	export function useProviderUpdates() {
26	const { environments } = useEnvironments();	27	const { environments } = useEnvironments();
...	@@ -44,10 +45,11 @@ export function useProviderUpdates() {	...	@@ -44,10 +45,11 @@ export function useProviderUpdates() {
44	[environments],	45	[environments],
45	);	46	);
46		47
47	const updateAll = async () => {	48	const updateAll = async ({ quiet = false }: { readonly quiet?: boolean } = {}) => {
48	const runs = await Promise.all(	49	const runs = await Promise.all(
49	machines.flatMap(({ environmentId, label, candidates }) =>	50	machines.flatMap(({ environmentId, label, candidates }) =>
50	candidates.map(async (candidate): Promise<ProviderUpdateRun> => ({	51	candidates.map(async (candidate): Promise<ProviderUpdateRun> => ({
		52	environmentId,
51	machineLabel: label,	53	machineLabel: label,
52	driver: candidate.driver,	54	driver: candidate.driver,
53	instanceId: candidate.instanceId,	55	instanceId: candidate.instanceId,
...	@@ -58,7 +60,7 @@ export function useProviderUpdates() {	...	@@ -58,7 +60,7 @@ export function useProviderUpdates() {
58	})),	60	})),
59	),	61	),
60	);	62	);
61	const view = getProviderUpdateRunToastView(runs);	63	const view = quiet ? null : getProviderUpdateRunToastView(runs);
62	if (view) {	64	if (view) {
63	toastManager.add(	65	toastManager.add(
64	stackedThreadToast({	66	stackedThreadToast({
...	@@ -67,6 +69,7 @@ export function useProviderUpdates() {	...	@@ -67,6 +69,7 @@ export function useProviderUpdates() {
67	}),	69	}),
68	);	70	);
69	}	71	}
		72	return runs;
70	};	73	};
71		74
72	return { machines, updateAll };	75	return { machines, updateAll };

apps/web/src/components/ServerUpdateAction.test.tsx+2-3

 @@ -498,13 +498,12 @@ describe("ServerUpdateProgress", () => {
     expect(markup).toContain("Restarting…");
     // The wait state is monochrome and calm: no versions, no step rail, no
-    // success/warning colors, one duty-cycled pulse on the dot.
+    // success/warning colors, and a static dot that never repaints.
     expect(markup).not.toContain("0.0.30");
     expect(markup).not.toContain("Resum");
     expect(markup).not.toContain("text-success");
     expect(markup).not.toContain("text-primary");
-    expect(markup).toContain("animate-status-pulse");
+    expect(markup).not.toContain("animate-");
-    expect(markup).not.toContain("animate-spin");
   });
   it("folds the sub-second installing handoff into the download phase", () => {

apps/web/src/components/ServerUpdateAction.tsx+37-26

 @@ -65,17 +65,26 @@ type UpdateButtonProps = Pick<ComponentProps<typeof Button>, "variant" | "size"
   readonly appearance?: "button" | "icon";
 };
+/** What {@link useServerUpdate} did. "skipped" covers a missing permission, an
+    update already in flight, and an interrupted request. */
+export type ServerUpdateOutcome =
+  | { readonly status: "updated" | "skipped" }
+  | { readonly status: "failed"; readonly message: string };
 /** Updates one server and waits until it is back on the target version.
-    Reports its own success or failure toast; never rejects. */
+    Toasts its success or failure unless `quiet`; never rejects. */
 export function useServerUpdate() {
   const updateServer = useAtomCommand(serverEnvironment.updateServer, { reportFailure: false });
-  return async (target: ServerUpdateTarget, failureTitle = "Server update failed") => {
+  return async (
+    target: ServerUpdateTarget,
+    { quiet = false }: { readonly quiet?: boolean } = {},
+  ): Promise<ServerUpdateOutcome> => {
     const { environmentId, serverLabel, selfUpdate, targetVersion } = target;
     if (
       !canUpdateServer(appAtomRegistry.get(environmentSession.sessionStateAtom(environmentId))) ||
       pendingUpdateEnvironmentIds.has(environmentId)
     )
-      return;
+      return { status: "skipped" };
     pendingUpdateEnvironmentIds.add(environmentId);
     try {
       const result = await updateServer({
 @@ -88,23 +97,30 @@ export function useServerUpdate() {
         },
       });
       if (result._tag === "Failure") {
-        if (isAtomCommandInterrupted(result)) return;
+        if (isAtomCommandInterrupted(result)) return { status: "skipped" };
         throw squashAtomCommandFailure(result);
       }
-      toastManager.add({
+      if (!quiet) {
-        type: "success",
+        toastManager.add({
-        title: `${serverLabel} updated`,
+          type: "success",
-        description:
+          title: `${serverLabel} updated`,
-          selfUpdate === "desktop-managed"
+          description:
-            ? `Desktop app relaunched on ${result.value.targetVersion}.`
+            selfUpdate === "desktop-managed"
-            : `Reconnected on t3@${result.value.targetVersion}.`,
+              ? `Desktop app relaunched on ${result.value.targetVersion}.`
-      });
+              : `Reconnected on t3@${result.value.targetVersion}.`,
+        });
+      }
+      return { status: "updated" };
     } catch (error) {
-      toastManager.add({
+      const message = updateFailureMessage(error);
-        type: "error",
+      if (!quiet) {
-        title: failureTitle,
+        toastManager.add({
-        description: updateFailureMessage(error),
+          type: "error",
-      });
+          title: `${serverLabel} update failed`,
+          description: message,
+        });
+      }
+      return { status: "failed", message };
     } finally {
       pendingUpdateEnvironmentIds.delete(environmentId);
     }
 @@ -145,9 +161,7 @@ export function ServerUpdatesAction({
           )) ?? true;
         if (!confirmed) return;
       }
-      await Promise.all(
+      await Promise.all(available.map((target) => update(target)));
-        available.map((target) => update(target, `${target.serverLabel} update failed`)),
-      );
     } finally {
       pending.current = false;
       setIsPending(false);
 @@ -175,7 +189,7 @@ export function canUpdateServer(
 /**
  * One-row status for an in-flight server update: "Downloading…" then
- * "Restarting…". The update is a wait, not a warning: a single pulsing dot
+ * "Restarting…". The update is a wait, not a warning: a single static dot
  * and label, no step rail, no versions. Failure turns the row red with the
  * rollback reason.
  */
 @@ -197,10 +211,7 @@ export function ServerUpdateProgress({
   }
   return (
     <div className="mt-1 flex items-center gap-2 text-xs font-medium text-foreground">
-      <span
+      <span className="size-1.5 shrink-0 rounded-full bg-foreground" aria-hidden="true" />
-        className="size-1.5 shrink-0 animate-status-pulse rounded-full bg-foreground"
-        aria-hidden="true"
-      />
       <span>{serverUpdateStageLabel(state.stage)}</span>
     </div>
   );
 @@ -381,7 +392,7 @@ export function OutdatedServerUpdateAction({
     } catch (error) {
       toastManager.add({
         type: "error",
-        title: "Server update failed",
+        title: `${serverLabel} update failed`,
         description: updateFailureMessage(error),
       });
     } finally {

apps/web/src/components/desktopUpdate.logic.test.ts+20

...	@@ -57,6 +57,26 @@ describe("desktop update button state", () => {	...	@@ -57,6 +57,26 @@ describe("desktop update button state", () => {
57	expect(getDesktopUpdateButtonTooltip(state)).toContain("Click to retry");	57	expect(getDesktopUpdateButtonTooltip(state)).toContain("Click to retry");
58	});	58	});
59		59
		60	it("says when the same version did not apply before", () => {
		61	const state: DesktopUpdateState = {
		62	...baseState,
		63	status: "downloaded",
		64	availableVersion: "1.1.0",
		65	downloadedVersion: "1.1.0",
		66	notAppliedVersion: "1.1.0",
		67	};
		68	expect(getDesktopUpdateButtonTooltip(state)).toBe(
		69	"Update 1.1.0 did not apply. Click to retry.",
		70	);
		71	expect(
		72	getDesktopUpdateButtonTooltip({
		73	...state,
		74	availableVersion: "1.2.0",
		75	downloadedVersion: "1.2.0",
		76	}),
		77	).toContain("1.2.0 downloaded");
		78	});
		79
60	it("keeps install action available after an install error", () => {	80	it("keeps install action available after an install error", () => {
61	const state: DesktopUpdateState = {	81	const state: DesktopUpdateState = {
62	...baseState,	82	...baseState,

apps/web/src/components/desktopUpdate.logic.ts+7

...	@@ -71,6 +71,13 @@ export function getArm64IntelBuildWarningDescription(state: DesktopUpdateState):	...	@@ -71,6 +71,13 @@ export function getArm64IntelBuildWarningDescription(state: DesktopUpdateState):
71	}	71	}
72		72
73	export function getDesktopUpdateButtonTooltip(state: DesktopUpdateState): string {	73	export function getDesktopUpdateButtonTooltip(state: DesktopUpdateState): string {
		74	if (
		75	state.notAppliedVersion !== undefined &&
		76	(state.status === "available" \|\| state.status === "downloaded") &&
		77	getDesktopUpdateDownloadedVersion(state) === state.notAppliedVersion
		78	) {
		79	return `Update ${state.notAppliedVersion} did not apply. Click to retry.`;
		80	}
74	if (state.status === "available") {	81	if (state.status === "available") {
75	return `Update ${state.availableVersion ?? "available"} ready to download`;	82	return `Update ${state.availableVersion ?? "available"} ready to download`;
76	}	83	}

apps/web/src/components/sidebar/SidebarUpdatePill.tsx+131-71

 @@ -1,12 +1,23 @@
+import { useAtomValue } from "@effect/atom-react";
 import type { DesktopBridge, DesktopUpdateState } from "@t3tools/contracts";
+import { Atom } from "effect/reactivity";
 import { TriangleAlertIcon } from "lucide-react";
-import { type ComponentProps, useCallback, useEffect, useId, useRef, useState } from "react";
+import {
+  type ComponentProps,
+  useCallback,
+  useEffect,
+  useId,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 import { flushSync } from "react-dom";
 import { isElectron } from "../../env";
 import { useMediaQuery } from "../../hooks/useMediaQuery";
 import { cn } from "../../lib/utils";
 import { ensureLocalApi } from "../../localApi";
 import { useDesktopUpdateState } from "../../state/desktopUpdate";
+import { serverEnvironment } from "../../state/server";
 import { stackedThreadToast, toastManager } from "../ui/toast";
 import {
   canCheckForUpdate,
 @@ -33,7 +44,13 @@ import {
 } from "./DesktopUpdateStatusIcon";
 import { SidebarUpdateReleaseNotes } from "./SidebarUpdateReleaseNotes";
 import { UpdateEverythingDialog } from "./UpdateEverythingDialog";
-import { useUpdateEverything } from "./useUpdateEverything";
+import {
+  describeUpdateRun,
+  resolveUpdateRunMachineStatus,
+  type UpdateEverythingRun,
+} from "./updateEverything.logic";
+import { useUpdateEverythingRun } from "./updateEverythingRun";
+import { type ReportLocalUpdateError, useUpdateEverything } from "./useUpdateEverything";
 type SidebarUpdatePopoverChangeDetails = Parameters<
   NonNullable<ComponentProps<typeof Popover>["onOpenChange"]>
 @@ -68,27 +85,34 @@ export function openSidebarUpdateReleaseNotesPopoverOnForwardTab(
 function resolveSidebarUpdatePresentation({
   action,
   showsEverything,
+  run,
   isDownloading,
   showCheckIcon,
 }: {
   readonly action: ReturnType<typeof resolveDesktopUpdateButtonAction>;
-  /** Remote work is waiting or running, and this app is not downloading. */
+  /** Remote work is waiting, and this app is not downloading. */
   readonly showsEverything: boolean;
+  /** The running "update everything" pass. Its state sets the icon. */
+  readonly run: UpdateEverythingRun | null;
   readonly isDownloading: boolean;
   readonly showCheckIcon: boolean;
 }) {
-  const showUpdateDetails = action !== "none" || showsEverything || isDownloading;
+  const showUpdateDetails = action !== "none" || showsEverything || run !== null || isDownloading;
   const iconStatus: DesktopUpdateStatusIconState = showCheckIcon
     ? "checking"
     : isDownloading
       ? "downloading"
-      : showsEverything
+      : run !== null
-        ? "everything"
+        ? run.localVersion !== null && action === "install"
-        : action === "install"
           ? "downloaded"
-          : action === "download"
+          : "everything"
-            ? "available"
+        : showsEverything
-            : "idle";
+          ? "everything"
+          : action === "install"
+            ? "downloaded"
+            : action === "download"
+              ? "available"
+              : "idle";
   return {
     iconStatus,
 @@ -117,11 +141,18 @@ function SidebarUpdateArchitectureWarningContent() {
   );
 }
+const toastUpdateError: ReportLocalUpdateError = (title, description) => {
+  toastManager.add(stackedThreadToast({ type: "error", title, description }));
+};
 /** Downloads this app's update. Resolves true when it is ready to install;
-    failures toast and resolve false. */
+    failures go to `reportError` (a toast by default) and resolve false. */
 async function downloadLocalUpdate(
   bridge: DesktopBridge,
-  { notifyDownloaded }: { readonly notifyDownloaded: boolean },
+  {
+    notifyDownloaded,
+    reportError = toastUpdateError,
+  }: { readonly notifyDownloaded: boolean; readonly reportError?: ReportLocalUpdateError },
 ): Promise<boolean> {
   try {
     const result = await bridge.downloadUpdate();
 @@ -131,60 +162,83 @@ async function downloadLocalUpdate(
     const actionError = shouldToastDesktopUpdateActionResult(result)
       ? getDesktopUpdateActionError(result)
       : null;
-    if (actionError) {
+    if (actionError) reportError("Could not download update", actionError);
-      toastManager.add(
-        stackedThreadToast({
-          type: "error",
-          title: "Could not download update",
-          description: actionError,
-        }),
-      );
-    }
     return result.completed;
   } catch (error) {
-    toastManager.add(
+    reportError(
-      stackedThreadToast({
+      "Could not start update download",
-        type: "error",
+      error instanceof Error ? error.message : "An unexpected error occurred.",
-        title: "Could not start update download",
-        description: error instanceof Error ? error.message : "An unexpected error occurred.",
-      }),
     );
     return false;
   }
 }
-/** Installs this app's downloaded update, which relaunches the app.
+/** Installs this app's downloaded update, which relaunches the app. Resolves
-    Failures toast. A refused install (the download is not marked ready yet)
+    true when the relaunch is under way. Failures go to `reportError` (a toast
+    by default). A refused install (the download is not marked ready yet)
     leaves the restart to the user. */
-async function installLocalUpdate(bridge: DesktopBridge): Promise<void> {
+async function installLocalUpdate(
+  bridge: DesktopBridge,
+  reportError: ReportLocalUpdateError = toastUpdateError,
+): Promise<boolean> {
   try {
     const result = await bridge.installUpdate();
     if (!result.accepted) {
       showDesktopUpdateDownloadedToast(bridge, result.state);
-      return;
+      return false;
     }
     const actionError = shouldToastDesktopUpdateActionResult(result)
       ? getDesktopUpdateActionError(result)
       : null;
-    if (!actionError) return;
+    if (!actionError) return true;
-    toastManager.add(
+    reportError("Could not install update", actionError);
-      stackedThreadToast({
+    return false;
-        type: "error",
-        title: "Could not install update",
-        description: actionError,
-      }),
-    );
   } catch (error) {
-    toastManager.add(
+    reportError(
-      stackedThreadToast({
+      "Could not install update",
-        type: "error",
+      error instanceof Error ? error.message : "An unexpected error occurred.",
-        title: "Could not install update",
-        description: error instanceof Error ? error.message : "An unexpected error occurred.",
-      }),
     );
+    return false;
   }
 }
+/** Tooltip lines for a running pass: this app first, then one per machine. */
+function useUpdateRunTooltip(
+  run: UpdateEverythingRun | null,
+  state: DesktopUpdateState | null,
+  localReady: boolean,
+): string | null {
+  const statusesAtom = useMemo(
+    () =>
+      Atom.make((get) =>
+        run === null
+          ? []
+          : run.machines.map((machine) => ({
+              label: machine.label,
+              status: resolveUpdateRunMachineStatus(
+                machine,
+                run.phase,
+                get(serverEnvironment.updateStateAtom(machine.environmentId)),
+              ),
+            })),
+      ),
+    [run],
+  );
+  const statuses = useAtomValue(statusesAtom);
+  if (run === null) return null;
+  const local =
+    run.localVersion === null
+      ? null
+      : state?.status === "downloading"
+        ? `downloading${state.downloadPercent === null ? "" : ` ${Math.floor(state.downloadPercent)}%`}`
+        : run.phase === "installing"
+          ? "restarting"
+          : localReady
+            ? "restarts last"
+            : "waiting";
+  return describeUpdateRun(statuses, local).join("\n");
+}
 /**
  * The sidebar update button. With only a local app update it downloads, then
  * installs. When other machines' servers or any providers are behind, one
 @@ -198,7 +252,6 @@ export function SidebarUpdatePill() {
 function SidebarUpdateControl() {
   const state = useDesktopUpdateState();
   const [isActionPending, setIsActionPending] = useState(false);
-  const [isUpdatingEverything, setIsUpdatingEverything] = useState(false);
   const [isConfirmingEverything, setIsConfirmingEverything] = useState(false);
   const [checkAnimationKey, setCheckAnimationKey] = useState(0);
   const [isCheckAnimationLatched, setIsCheckAnimationLatched] = useState(false);
 @@ -224,7 +277,9 @@ function SidebarUpdateControl() {
   const everything = useUpdateEverything(localUpdateVersion);
   const runsEverything = everything.hasUpdates && !isDownloading;
   // Providers leave the list once queued, so the run keeps its own state.
-  const isRunningEverything = isUpdatingEverything && !isDownloading;
+  const run = useUpdateEverythingRun();
+  const isRunningEverything = run !== null;
+  const runTooltip = useUpdateRunTooltip(run, state, action === "install");
   const showCheckIcon = shouldShowDesktopUpdateCheckIcon({
     isAnimationLatched: isCheckAnimationLatched,
     isChecking: state?.status === "checking",
 @@ -232,21 +287,23 @@ function SidebarUpdateControl() {
   });
   const { iconStatus, showUpdateDetails, showUpdateIconState } = resolveSidebarUpdatePresentation({
     action,
-    showsEverything: runsEverything || isRunningEverything,
+    showsEverything: runsEverything,
+    run,
     isDownloading,
     showCheckIcon,
   });
-  const tooltip = isRunningEverything
+  const tooltip =
-    ? "Updating…"
+    runTooltip !== null
-    : runsEverything
+      ? runTooltip
-      ? everything.summary
+      : runsEverything
-      : showUpdateDetails
+        ? everything.summary
-        ? state
+        : showUpdateDetails
-          ? getDesktopUpdateButtonTooltip(state)
+          ? state
-          : "Update available"
+            ? getDesktopUpdateButtonTooltip(state)
-        : showCheckIcon
+            : "Update available"
-          ? "Checking for updates…"
+          : showCheckIcon
-          : "Check for updates";
+            ? "Checking for updates…"
+            : "Check for updates";
   const disabled = showCheckIcon
     ? true
     : runsEverything
 @@ -254,11 +311,10 @@ function SidebarUpdateControl() {
       : showUpdateDetails
         ? isDesktopUpdateButtonDisabled(state)
         : !canCheckForUpdate(state);
-  const isInteractionDisabled = disabled || isActionPending;
+  const isInteractionDisabled = disabled || isActionPending || isRunningEverything;
-  const showReleaseNotesPopover = shouldUseSidebarUpdateReleaseNotesPopover(
+  // During a run the tooltip shows its progress instead of release notes.
-    showUpdateDetails,
+  const showReleaseNotesPopover =
-    state,
+    !isRunningEverything && shouldUseSidebarUpdateReleaseNotesPopover(showUpdateDetails, state);
-  );
   useEffect(() => {
     if (!showReleaseNotesPopover) {
 @@ -349,13 +405,17 @@ function SidebarUpdateControl() {
     const bridge = window.desktopBridge;
     setIsConfirmingEverything(false);
     setIsActionPending(true);
-    setIsUpdatingEverything(true);
     try {
       await everything.run({
         ...(bridge && action === "download"
-          ? { downloadLocal: () => downloadLocalUpdate(bridge, { notifyDownloaded: false }) }
+          ? {
+              downloadLocal: (reportError) =>
+                downloadLocalUpdate(bridge, { notifyDownloaded: false, reportError }),
+            }
+          : {}),
+        ...(bridge && action !== "none"
+          ? { installLocal: (reportError) => installLocalUpdate(bridge, reportError) }
           : {}),
-        ...(bridge && action !== "none" ? { installLocal: () => installLocalUpdate(bridge) } : {}),
       });
     } catch (error) {
       toastManager.add(
 @@ -366,7 +426,6 @@ function SidebarUpdateControl() {
         }),
       );
     } finally {
-      setIsUpdatingEverything(false);
       setIsActionPending(false);
     }
   }, [action, everything]);
 @@ -381,7 +440,7 @@ function SidebarUpdateControl() {
   }, [prefersReducedMotion, state?.status]);
   // Web has no local app to check, so it only shows the button for remote work.
-  if (!isElectron && !runsEverything && !isUpdatingEverything) return null;
+  if (!isElectron && !runsEverything && !isRunningEverything) return null;
   const updateButton = (
     <button
 @@ -400,7 +459,7 @@ function SidebarUpdateControl() {
               "text-(--sidebar-icon-color)",
               !isInteractionDisabled && "hover:bg-sidebar-row-hover hover:text-sidebar-foreground",
             ),
-        ((disabled && !showUpdateIconState) || isRunningEverything) && "opacity-60",
+        disabled && !showUpdateIconState && "opacity-60",
       )}
       onClick={handleAction}
       onBlur={() => {
 @@ -471,7 +530,7 @@ function SidebarUpdateControl() {
               side="top"
               variant={showUpdateDetails ? "glass" : "default"}
             >
-              {tooltip}
+              <span className="whitespace-pre-line">{tooltip}</span>
             </TooltipPopup>
           ) : null}
         </Tooltip>
 @@ -506,6 +565,7 @@ function SidebarUpdateControl() {
         summary={everything.summary}
         lines={everything.lines}
         restarts={everything.restarts}
+        skipped={everything.skipped}
         onConfirm={() => void runEverything()}
       />
     </SidebarMenuItem>

apps/web/src/components/sidebar/UpdateEverythingDialog.tsx+10-1

...	@@ -13,7 +13,7 @@ import {	...	@@ -13,7 +13,7 @@ import {
13	AlertDialogTitle,	13	AlertDialogTitle,
14	} from "../ui/alert-dialog";	14	} from "../ui/alert-dialog";
15	import { Button } from "../ui/button";	15	import { Button } from "../ui/button";
16	import type { RestartingMachine } from "./updateEverything.logic";	16	import type { RestartingMachine, SkippedMachine } from "./updateEverything.logic";
17		17
18	/** A thread an agent is working on now. A restart stops these; queued threads wait. */	18	/** A thread an agent is working on now. A restart stops these; queued threads wait. */
19	const isRunning = (status: string) =>	19	const isRunning = (status: string) =>
...	@@ -42,6 +42,8 @@ interface UpdateEverythingDialogProps {	...	@@ -42,6 +42,8 @@ interface UpdateEverythingDialogProps {
42	readonly summary: string;	42	readonly summary: string;
43	readonly lines: ReadonlyArray<string>;	43	readonly lines: ReadonlyArray<string>;
44	readonly restarts: ReadonlyArray<RestartingMachine>;	44	readonly restarts: ReadonlyArray<RestartingMachine>;
		45	/** Behind machines the run leaves alone. */
		46	readonly skipped: ReadonlyArray<SkippedMachine>;
45	readonly onConfirm: () => void;	47	readonly onConfirm: () => void;
46	}	48	}
47		49
...	@@ -50,6 +52,7 @@ interface UpdateEverythingDialogProps {	...	@@ -50,6 +52,7 @@ interface UpdateEverythingDialogProps {
50	* restarts stop, and how many can resume afterwards because "Continue threads	52	* restarts stop, and how many can resume afterwards because "Continue threads
51	* after restarts" is on for their server. Recovery still skips some, such as a	53	* after restarts" is on for their server. Recovery still skips some, such as a
52	* thread waiting on the user, so the text says "can resume", not "will".	54	* thread waiting on the user, so the text says "can resume", not "will".
		55	* Machines the run skips are listed with the reason, so none drop silently.
53	*/	56	*/
54	export function UpdateEverythingDialog({	57	export function UpdateEverythingDialog({
55	open,	58	open,
...	@@ -70,6 +73,7 @@ function UpdateEverythingDialogContent({	...	@@ -70,6 +73,7 @@ function UpdateEverythingDialogContent({
70	summary,	73	summary,
71	lines,	74	lines,
72	restarts,	75	restarts,
		76	skipped,
73	onConfirm,	77	onConfirm,
74	}: Omit<UpdateEverythingDialogProps, "open" \| "onOpenChange">) {	78	}: Omit<UpdateEverythingDialogProps, "open" \| "onOpenChange">) {
75	const stopped = useAtomValue(	79	const stopped = useAtomValue(
...	@@ -107,6 +111,11 @@ function UpdateEverythingDialogContent({	...	@@ -107,6 +111,11 @@ function UpdateEverythingDialogContent({
107	: `${threads} will be interrupted. ${continuing} can resume after the restart.`}	111	: `${threads} will be interrupted. ${continuing} can resume after the restart.`}
108	</p>	112	</p>
109	) : null}	113	) : null}
		114	{skipped.length > 0 ? (
		115	<p className="whitespace-pre-line pt-2 text-sm">
		116	{skipped.map((machine) => `Skips ${machine.label}: ${machine.reason}`).join("\n")}
		117	</p>
		118	) : null}
110	</AlertDialogHeader>	119	</AlertDialogHeader>
111	<AlertDialogFooter>	120	<AlertDialogFooter>
112	<AlertDialogClose render={<Button variant="outline" />}>Cancel</AlertDialogClose>	121	<AlertDialogClose render={<Button variant="outline" />}>Cancel</AlertDialogClose>

apps/web/src/components/sidebar/updateEverything.logic.test.ts+123-15

...	@@ -4,9 +4,13 @@ import { describe, expect, it } from "vite-plus/test";	...	@@ -4,9 +4,13 @@ import { describe, expect, it } from "vite-plus/test";
4	import {	4	import {
5	collectRestartingMachines,	5	collectRestartingMachines,
6	collectServerUpdateTargets,	6	collectServerUpdateTargets,
		7	collectUpdateEverythingResult,
7	describeUpdateEverything,	8	describeUpdateEverything,
		9	getUpdateEverythingToastView,
		10	resolveUpdateRunMachineStatus,
8	runUpdateEverything,	11	runUpdateEverything,
9	type UpdateEverythingMachine,	12	type UpdateEverythingMachine,
		13	type UpdateRunMachine,
10	} from "./updateEverything.logic";	14	} from "./updateEverything.logic";
11		15
12	function machine(	16	function machine(
...	@@ -41,16 +45,26 @@ function machine(	...	@@ -41,16 +45,26 @@ function machine(
41	}	45	}
42		46
43	describe("collectServerUpdateTargets", () => {	47	describe("collectServerUpdateTargets", () => {
44	it("updates only remote servers that can move to the target on their channel", () => {	48	it("updates remote servers that can move to the target and names every skipped one", () => {
45	const targets = collectServerUpdateTargets(	49	const { targets, skipped } = collectServerUpdateTargets(
46	[	50	[
47	machine("this-app", "0.0.40", "desktop-managed", { hostedByThisApp: true }),	51	machine("this-app", "0.0.40", "desktop-managed", { hostedByThisApp: true }),
48	machine("service", "0.0.40", "boot-service"),	52	machine("service", "0.0.40", "boot-service"),
49	machine("nightly-service", "0.0.49-nightly.20261001.3", "boot-service"),	53	machine("nightly-service", "0.0.49-nightly.20261001.3", "boot-service"),
50	machine("other-desktop", "0.0.49-nightly.20261001.3", "desktop-managed"),	54	machine("other-desktop", "0.0.49-nightly.20261001.3", "desktop-managed"),
51	machine("offline", "0.0.40", "boot-service", { connected: false }),	55	machine("offline", "0.0.40", "boot-service", { connected: false }),
		56	machine("never-connected", "0.0.40", "boot-service", { serverConfig: null }),
52	machine("no-scope", "0.0.40", "boot-service", { canMaintain: false }),	57	machine("no-scope", "0.0.40", "boot-service", { canMaintain: false }),
		58	machine("busy", "0.0.40", "boot-service", {
		59	updateState: {
		60	status: "running",
		61	stage: "downloading",
		62	fromVersion: "0.0.40",
		63	targetVersion: "0.0.50",
		64	},
		65	}),
53	machine("current", "0.0.50", "boot-service"),	66	machine("current", "0.0.50", "boot-service"),
		67	machine("current-offline", "0.0.50", "boot-service", { connected: false }),
54	machine("manual", "0.0.40", undefined),	68	machine("manual", "0.0.40", undefined),
55	],	69	],
56	"0.0.50",	70	"0.0.50",
...	@@ -60,6 +74,14 @@ describe("collectServerUpdateTargets", () => {	...	@@ -60,6 +74,14 @@ describe("collectServerUpdateTargets", () => {
60	["service", "0.0.50"],	74	["service", "0.0.50"],
61	["other-desktop", "0.0.50"],	75	["other-desktop", "0.0.50"],
62	]);	76	]);
		77	expect(skipped.map((entry) => [entry.label, entry.reason])).toEqual([
		78	["nightly-service", "on the nightly channel"],
		79	["offline", "not connected"],
		80	["never-connected", "not connected"],
		81	["no-scope", "no permission"],
		82	["busy", "already updating"],
		83	["manual", "needs a manual update"],
		84	]);
63	});	85	});
64	});	86	});
65		87
...	@@ -109,37 +131,35 @@ describe("collectRestartingMachines", () => {	...	@@ -109,37 +131,35 @@ describe("collectRestartingMachines", () => {
109	});	131	});
110		132
111	describe("runUpdateEverything", () => {	133	describe("runUpdateEverything", () => {
112	it("restarts servers after providers finish and installs this app last", async () => {	134	it("restarts servers after providers finish and returns every result", async () => {
113	const calls: string[] = [];	135	const calls: string[] = [];
114	let finishProviders = () => {};	136	let finishProviders = (_value: string) => {};
115	const run = runUpdateEverything({	137	const run = runUpdateEverything({
116	downloadLocal: async () => {	138	downloadLocal: async () => {
117	calls.push("download");	139	calls.push("download");
118	return true;	140	return true;
119	},	141	},
120	updateProviders: () =>	142	updateProviders: () =>
121	new Promise<void>((resolve) => {	143	new Promise<string>((resolve) => {
122	calls.push("providers");	144	calls.push("providers");
123	finishProviders = resolve;	145	finishProviders = resolve;
124	}),	146	}),
125	updateServers: async () => {	147	updateServers: async () => {
126	calls.push("servers");	148	calls.push("servers");
127	},	149	return 2;
128	installLocal: async () => {
129	calls.push("install");
130	},	150	},
131	});	151	});
132		152
133	await new Promise((resolve) => setTimeout(resolve, 0));	153	await new Promise((resolve) => setTimeout(resolve, 0));
134	expect(calls).toEqual(["download", "providers"]);	154	expect(calls).toEqual(["download", "providers"]);
135	finishProviders();	155	finishProviders("provider results");
136	await run;	156	expect(await run).toEqual({ localReady: true, providers: "provider results", servers: 2 });
137	expect(calls).toEqual(["download", "providers", "servers", "install"]);	157	expect(calls).toEqual(["download", "providers", "servers"]);
138	});	158	});
139		159
140	it("still updates remote machines when the local download fails", async () => {	160	it("still updates remote machines when the local download fails", async () => {
141	const calls: string[] = [];	161	const calls: string[] = [];
142	await runUpdateEverything({	162	const result = await runUpdateEverything({
143	downloadLocal: async () => false,	163	downloadLocal: async () => false,
144	updateProviders: async () => {	164	updateProviders: async () => {
145	calls.push("providers");	165	calls.push("providers");
...	@@ -147,12 +167,100 @@ describe("runUpdateEverything", () => {	...	@@ -147,12 +167,100 @@ describe("runUpdateEverything", () => {
147	updateServers: async () => {	167	updateServers: async () => {
148	calls.push("servers");	168	calls.push("servers");
149	},	169	},
150	installLocal: async () => {
151	calls.push("install");
152	},
153	});	170	});
154		171
155	expect(calls).toEqual(["providers", "servers"]);	172	expect(calls).toEqual(["providers", "servers"]);
		173	expect(result.localReady).toBe(false);
		174	});
		175	});
		176
		177	describe("resolveUpdateRunMachineStatus", () => {
		178	const runMachine = (overrides: Partial<UpdateRunMachine> = {}): UpdateRunMachine => ({
		179	environmentId: EnvironmentId.make("alvin"),
		180	label: "alvin",
		181	updatesProviders: true,
		182	updatesServer: true,
		183	result: null,
		184	...overrides,
		185	});
		186	const running = (stage: "downloading" \| "installing" \| "resuming") =>
		187	({ status: "running", stage, fromVersion: "0.0.40", targetVersion: "0.0.50" }) as const;
		188
		189	it("follows the run phase, then the server's own update stage", () => {
		190	const idle = { status: "idle" } as const;
		191	expect(resolveUpdateRunMachineStatus(runMachine(), "providers", idle)).toBe("providers");
		192	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", idle)).toBe("waiting");
		193	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", running("installing"))).toBe(
		194	"downloading",
		195	);
		196	expect(resolveUpdateRunMachineStatus(runMachine(), "servers", running("resuming"))).toBe(
		197	"restarting",
		198	);
		199	expect(
		200	resolveUpdateRunMachineStatus(
		201	runMachine({ updatesProviders: false }),
		202	"providers",
		203	running("downloading"),
		204	),
		205	).toBe("downloading");
		206	expect(resolveUpdateRunMachineStatus(runMachine({ result: "failed" }), "servers", idle)).toBe(
		207	"failed",
		208	);
		209	});
		210	});
		211
		212	describe("update everything summary", () => {
		213	const result = collectUpdateEverythingResult({
		214	localVersion: "0.0.50",
		215	localFailures: ["Could not download update: Network error."],
		216	servers: [
		217	{ label: "alvin", outcome: { status: "failed", message: "Download failed." } },
		218	{ label: "box", outcome: { status: "updated" } },
		219	{ label: "studio", outcome: { status: "updated" } },
		220	{ label: "busy", outcome: { status: "skipped" } },
		221	],
		222	providers: [
		223	{ status: "updated" },
		224	{ status: "interrupted" },
		225	{ status: "failed", line: "box · Codex: npm exited with code 1." },
		226	],
		227	});
		228
		229	it("counts updates and names the machine of each failure", () => {
		230	expect(result).toEqual({
		231	localVersion: "0.0.50",
		232	updatedServers: 2,
		233	updatedProviders: 1,
		234	failures: [
		235	"This app: Could not download update: Network error.",
		236	"alvin: Download failed.",
		237	"box · Codex: npm exited with code 1.",
		238	],
		239	});
		240	});
		241
		242	it("includes this app only when the relaunch landed on its version", () => {
		243	expect(getUpdateEverythingToastView(result, "0.0.50")).toEqual({
		244	type: "error",
		245	title: "Updated this app, 2 servers, and 1 provider",
		246	description: [
		247	"This app: Could not download update: Network error.",
		248	"alvin: Download failed.",
		249	"box · Codex: npm exited with code 1.",
		250	].join("\n"),
		251	});
		252	expect(getUpdateEverythingToastView({ ...result, failures: [] }, "0.0.40")).toEqual({
		253	type: "success",
		254	title: "Updated 2 servers and 1 provider",
		255	});
		256	});
		257
		258	it("reports a failure when nothing updated, and nothing when nothing finished", () => {
		259	const empty = { localVersion: null, updatedServers: 0, updatedProviders: 0 };
		260	expect(
		261	getUpdateEverythingToastView({ ...empty, failures: ["alvin: offline"] }, "0.0.50"),
		262	).toEqual({ type: "error", title: "Update failed", description: "alvin: offline" });
		263	expect(getUpdateEverythingToastView({ ...empty, failures: [] }, "0.0.50")).toBeNull();
156	});	264	});
157	});	265	});
158		266

apps/web/src/components/sidebar/updateEverything.logic.ts+215-45

...	@@ -8,7 +8,7 @@ import {	...	@@ -8,7 +8,7 @@ import {
8	supportsDesktopAppUpdate,	8	supportsDesktopAppUpdate,
9	supportsServerUpdateThreadContinuation,	9	supportsServerUpdateThreadContinuation,
10	} from "~/versionSkew";	10	} from "~/versionSkew";
11	import type { ServerUpdateTarget } from "../ServerUpdateAction";	11	import type { ServerUpdateOutcome, ServerUpdateTarget } from "../ServerUpdateAction";
12		12
13	export interface UpdateEverythingMachine {	13	export interface UpdateEverythingMachine {
14	readonly environmentId: EnvironmentId;	14	readonly environmentId: EnvironmentId;
...	@@ -63,78 +63,248 @@ export function collectRestartingMachines(	...	@@ -63,78 +63,248 @@ export function collectRestartingMachines(
63	});	63	});
64	}	64	}
65		65
		66	/** A machine the run leaves alone, with a short reason such as "not connected". */
		67	export interface SkippedMachine {
		68	readonly environmentId: EnvironmentId;
		69	readonly label: string;
		70	readonly reason: string;
		71	}
		72
66	/**	73	/**
67	* The servers that "update everything" moves to `targetVersion`: connected,	74	* The servers that "update everything" moves to `targetVersion`: connected,
68	* permitted, able to update over RPC, and behind the target. Two kinds stay	75	* permitted, able to update over RPC, behind the target, and on its release
69	* out. Servers this app hosts would relaunch the app mid-run, and the app's	76	* channel. Every other behind server comes back in `skipped` with a reason for
70	* own install updates them anyway. A service on another release channel	77	* the confirmation; a machine that never connected has an unknown version, so
71	* stays there; a desktop-managed server follows its own app's update feed, so	78	* it counts as skipped. Two kinds are in neither list: servers already on the
72	* the channel rule does not apply to it.	79	* target, and servers this app hosts, which would relaunch the app mid-run
		80	* and update with the app's own install anyway. A desktop-managed server
		81	* follows its own app's update feed, so the channel rule does not apply to it.
73	*/	82	*/
74	export function collectServerUpdateTargets(	83	export function collectServerUpdateTargets(
75	machines: ReadonlyArray<UpdateEverythingMachine>,	84	machines: ReadonlyArray<UpdateEverythingMachine>,
76	targetVersion: string,	85	targetVersion: string,
77	): ServerUpdateTarget[] {	86	): { readonly targets: ServerUpdateTarget[]; readonly skipped: SkippedMachine[] } {
78	return machines.flatMap((machine) => {	87	const targets: ServerUpdateTarget[] = [];
		88	const skipped: SkippedMachine[] = [];
		89	for (const machine of machines) {
79	const config = machine.serverConfig;	90	const config = machine.serverConfig;
		91	if (machine.hostedByThisApp) continue;
80	if (	92	if (
81	config === null \|\|	93	config !== null &&
82	!machine.connected \|\|	94	resolveVersionMismatch(config.environment.serverVersion, targetVersion) === null
83	machine.hostedByThisApp \|\|
84	!machine.canMaintain \|\|
85	machine.updateState.status === "running"
86	) {	95	) {
87	return [];	96	continue;
		97	}
		98	const skip = (reason: string) =>
		99	skipped.push({ environmentId: machine.environmentId, label: machine.label, reason });
		100	if (config === null \|\| !machine.connected) {
		101	skip("not connected");
		102	continue;
		103	}
		104	if (!machine.canMaintain) {
		105	skip("no permission");
		106	continue;
		107	}
		108	if (machine.updateState.status === "running") {
		109	skip("already updating");
		110	continue;
88	}	111	}
89	const selfUpdate = resolveServerSelfUpdateCapability(config);	112	const selfUpdate = resolveServerSelfUpdateCapability(config);
90	const desktopAppUpdate = supportsDesktopAppUpdate(config);	113	const desktopAppUpdate = supportsDesktopAppUpdate(config);
91	if (selfUpdate === null \|\| (selfUpdate === "desktop-managed" && !desktopAppUpdate)) {	114	if (selfUpdate === null) {
92	return [];	115	skip("needs a manual update");
		116	continue;
		117	}
		118	if (selfUpdate === "desktop-managed" && !desktopAppUpdate) {
		119	skip("needs a newer desktop app");
		120	continue;
93	}	121	}
94	const serverVersion = config.environment.serverVersion;	122	const serverVersion = config.environment.serverVersion;
95	if (resolveVersionMismatch(serverVersion, targetVersion) === null) return [];
96	if (	123	if (
97	selfUpdate !== "desktop-managed" &&	124	selfUpdate !== "desktop-managed" &&
98	cliReleaseChannelOf(serverVersion) !== cliReleaseChannelOf(targetVersion)	125	cliReleaseChannelOf(serverVersion) !== cliReleaseChannelOf(targetVersion)
99	) {	126	) {
100	return [];	127	skip(`on the ${cliReleaseChannelOf(serverVersion)} channel`);
		128	continue;
101	}	129	}
102	return [	130	targets.push({
103	{	131	environmentId: machine.environmentId,
104	environmentId: machine.environmentId,	132	serverLabel: machine.label,
105	serverLabel: machine.label,	133	selfUpdate,
106	selfUpdate,	134	installation: config.environment.capabilities.serverInstallation,
107	installation: config.environment.capabilities.serverInstallation,	135	desktopAppUpdate,
108	desktopAppUpdate,	136	threadContinuation: supportsServerUpdateThreadContinuation(config),
109	threadContinuation: supportsServerUpdateThreadContinuation(config),	137	continueThreadsAfterServerUpdate: config.settings.continueThreadsAfterServerUpdate ?? false,
110	continueThreadsAfterServerUpdate: config.settings.continueThreadsAfterServerUpdate ?? false,	138	targetVersion,
111	targetVersion,	139	});
112	},	140	}
113	];	141	return { targets, skipped };
114	});
115	}	142	}
116		143
117	export interface UpdateEverythingSteps {	144	export interface UpdateEverythingSteps<Providers, Servers> {
118	/** Downloads this app's update. Resolves true when it is ready to install. */	145	/** Downloads this app's update. Resolves true when it is ready to install.
		146	Without it, `localReady` is true: there is nothing to download. */
119	readonly downloadLocal?: () => Promise<boolean>;	147	readonly downloadLocal?: () => Promise<boolean>;
120	readonly updateProviders: () => Promise<void>;	148	readonly updateProviders: () => Promise<Providers>;
121	readonly updateServers: () => Promise<void>;	149	readonly updateServers: () => Promise<Servers>;
122	/** Installs this app's update. The relaunch ends the run. */
123	readonly installLocal?: () => Promise<void>;
124	}	150	}
125		151
126	/**	152	/**
127	* Runs one "update everything" pass. Providers	153	* Runs the remote half of one "update everything" pass and returns each
128	* finish before any server restarts, so no install is cut off. The local download runs alongside the	154	* step's results. Providers finish before any server restarts, so no install
129	* remote work. This app installs last because its relaunch ends the run; a	155	* is cut off. The local download runs alongside the remote work. The caller
130	* failed download skips only that install. Steps report their own results.	156	* installs this app afterwards when `localReady`, because the relaunch ends
		157	* the run; a failed download skips only that install.
131	*/	158	*/
132	export async function runUpdateEverything(steps: UpdateEverythingSteps): Promise<void> {	159	export async function runUpdateEverything<Providers, Servers>(
133	const [localReady] = await Promise.all([	160	steps: UpdateEverythingSteps<Providers, Servers>,
		161	): Promise<{
		162	readonly localReady: boolean;
		163	readonly providers: Providers;
		164	readonly servers: Servers;
		165	}> {
		166	const [localReady, remote] = await Promise.all([
134	steps.downloadLocal?.() ?? Promise.resolve(true),	167	steps.downloadLocal?.() ?? Promise.resolve(true),
135	steps.updateProviders().then(steps.updateServers),	168	steps
		169	.updateProviders()
		170	.then(async (providers) => ({ providers, servers: await steps.updateServers() })),
136	]);	171	]);
137	if (localReady) await steps.installLocal?.();	172	return { localReady, ...remote };
		173	}
		174
		175	/** The current state of one machine in a running pass. */
		176	export type UpdateRunMachineStatus =
		177	\| "providers"
		178	\| "waiting"
		179	\| "downloading"
		180	\| "restarting"
		181	\| "done"
		182	\| "failed";
		183
		184	/** One machine in a running "update everything" pass. */
		185	export interface UpdateRunMachine {
		186	readonly environmentId: EnvironmentId;
		187	readonly label: string;
		188	readonly updatesProviders: boolean;
		189	readonly updatesServer: boolean;
		190	/** Set when every update for this machine has finished. */
		191	readonly result: "done" \| "failed" \| null;
		192	}
		193
		194	/** A running "update everything" pass, kept while it runs. */
		195	export interface UpdateEverythingRun {
		196	/** The version this app installs last, or null when it does not install. */
		197	readonly localVersion: string \| null;
		198	/** Providers update first. Servers update after every provider finished. */
		199	readonly phase: "providers" \| "servers" \| "installing";
		200	readonly machines: ReadonlyArray<UpdateRunMachine>;
		201	}
		202
		203	/** Status of one machine, from the run and that server's own update state. */
		204	export function resolveUpdateRunMachineStatus(
		205	machine: UpdateRunMachine,
		206	phase: UpdateEverythingRun["phase"],
		207	serverState: ServerUpdateState,
		208	): UpdateRunMachineStatus {
		209	if (machine.result !== null) return machine.result;
		210	if (phase === "providers" && machine.updatesProviders) return "providers";
		211	if (machine.updatesServer && serverState.status === "running") {
		212	return serverState.stage === "resuming" ? "restarting" : "downloading";
		213	}
		214	return "waiting";
		215	}
		216
		217	const RUN_STATUS_LABELS: Record<UpdateRunMachineStatus, string> = {
		218	providers: "updating providers",
		219	waiting: "waiting",
		220	downloading: "downloading",
		221	restarting: "restarting",
		222	done: "done",
		223	failed: "failed",
		224	};
		225
		226	/**
		227	* Tooltip lines for a running pass, such as "alvin: restarting". `local` is
		228	* this app's own state, such as "downloading 40%", or null when it does not
		229	* install.
		230	*/
		231	export function describeUpdateRun(
		232	machines: ReadonlyArray<{ readonly label: string; readonly status: UpdateRunMachineStatus }>,
		233	local: string \| null,
		234	): ReadonlyArray<string> {
		235	return [
		236	"Updating…",
		237	...(local === null ? [] : [`This app: ${local}`]),
		238	...machines.map((machine) => `${machine.label}: ${RUN_STATUS_LABELS[machine.status]}`),
		239	];
		240	}
		241
		242	/** What one pass did. It is saved before this app installs, so it outlives the relaunch. */
		243	export interface UpdateEverythingResult {
		244	/** The version this app installed last, or null when it did not install. */
		245	readonly localVersion: string \| null;
		246	readonly updatedServers: number;
		247	readonly updatedProviders: number;
		248	/** One line per failure, each naming its machine. */
		249	readonly failures: ReadonlyArray<string>;
		250	}
		251
		252	/** Counts what a pass updated and lists each failure with its machine. */
		253	export function collectUpdateEverythingResult(input: {
		254	readonly localVersion: string \| null;
		255	/** This app's own download or install failures. */
		256	readonly localFailures: ReadonlyArray<string>;
		257	readonly servers: ReadonlyArray<{
		258	readonly label: string;
		259	readonly outcome: ServerUpdateOutcome;
		260	}>;
		261	readonly providers: ReadonlyArray<
		262	\| { readonly status: "interrupted" \| "updated" }
		263	\| { readonly status: "failed"; readonly line: string }
		264	>;
		265	}): UpdateEverythingResult {
		266	return {
		267	localVersion: input.localVersion,
		268	updatedServers: input.servers.filter(({ outcome }) => outcome.status === "updated").length,
		269	updatedProviders: input.providers.filter((outcome) => outcome.status === "updated").length,
		270	failures: [
		271	...input.localFailures.map((failure) => `This app: ${failure}`),
		272	...input.servers.flatMap(({ label, outcome }) =>
		273	outcome.status === "failed" ? [`${label}: ${outcome.message}`] : [],
		274	),
		275	...input.providers.flatMap((outcome) => (outcome.status === "failed" ? [outcome.line] : [])),
		276	],
		277	};
		278	}
		279
		280	/**
		281	* The one toast for a finished pass, such as "Updated this app, 8 servers,
		282	* and 3 providers" with a line per failure. `appVersion` is the version this
		283	* app runs now: after the relaunch it tells whether the local install
		284	* applied. Null when nothing finished, for example every request was
		285	* interrupted.
		286	*/
		287	export function getUpdateEverythingToastView(
		288	result: UpdateEverythingResult,
		289	appVersion: string,
		290	): {
		291	readonly type: "success" \| "error";
		292	readonly title: string;
		293	readonly description?: string;
		294	} \| null {
		295	const parts = [
		296	result.localVersion !== null && result.localVersion === appVersion ? "this app" : null,
		297	countLabel(result.updatedServers, "server"),
		298	countLabel(result.updatedProviders, "provider"),
		299	].filter((part) => part !== null);
		300	if (result.failures.length === 0) {
		301	return parts.length === 0 ? null : { type: "success", title: `Updated ${joinList(parts)}` };
		302	}
		303	return {
		304	type: "error",
		305	title: parts.length === 0 ? "Update failed" : `Updated ${joinList(parts)}`,
		306	description: result.failures.join("\n"),
		307	};
138	}	308	}
139		309
140	export interface UpdateEverythingPlan {	310	export interface UpdateEverythingPlan {

apps/web/src/components/sidebar/updateEverythingRun.tsx created+142

...	@@ -0,0 +1,142 @@
1	import { useAtomValue } from "@effect/atom-react";
2	import type { EnvironmentId } from "@t3tools/contracts";
3	import * as Option from "effect/Option";
4	import * as Schema from "effect/Schema";
5	import { Atom } from "effect/reactivity";
6	import { useEffect } from "react";
7
8	import { APP_VERSION } from "~/branding";
9	import { appAtomRegistry } from "~/rpc/atomRegistry";
10	import { useDesktopUpdateState } from "~/state/desktopUpdate";
11	import { stackedThreadToast, toastManager } from "../ui/toast";
12	import {
13	getUpdateEverythingToastView,
14	type UpdateEverythingResult,
15	type UpdateEverythingRun,
16	} from "./updateEverything.logic";
17
18	/** The running "update everything" pass, or null. Outlives the sidebar button. */
19	const updateEverythingRunAtom = Atom.make<UpdateEverythingRun \| null>(null).pipe(
20	Atom.keepAlive,
21	Atom.withLabel("update-everything-run"),
22	);
23
24	export function useUpdateEverythingRun(): UpdateEverythingRun \| null {
25	return useAtomValue(updateEverythingRunAtom);
26	}
27
28	/** Starts, changes, or ends (null) the running pass. */
29	export function setUpdateEverythingRun(
30	update: (run: UpdateEverythingRun \| null) => UpdateEverythingRun \| null,
31	): void {
32	appAtomRegistry.update(updateEverythingRunAtom, update);
33	}
34
35	/** Records the result of every update for one machine in the running pass. */
36	export function settleUpdateRunMachine(environmentId: EnvironmentId, failed: boolean): void {
37	setUpdateEverythingRun(
38	(run) =>
39	run && {
40	...run,
41	machines: run.machines.map((machine) =>
42	machine.environmentId === environmentId
43	? { ...machine, result: failed ? "failed" : "done" }
44	: machine,
45	),
46	},
47	);
48	}
49
50	const SAVED_RESULT_KEY = "t3code:update-everything-result:v1";
51	// A result older than this belongs to an install that never relaunched the app.
52	const SAVED_RESULT_MAX_AGE_MS = 10 * 60 * 1000;
53
54	const SavedResult = Schema.Struct({
55	savedAt: Schema.Number,
56	localVersion: Schema.NullOr(Schema.String),
57	updatedServers: Schema.Number,
58	updatedProviders: Schema.Number,
59	failures: Schema.Array(Schema.String),
60	});
61	const decodeSavedResult = Schema.decodeUnknownOption(Schema.fromJsonString(SavedResult));
62
63	/** Keeps a result through this app's relaunch. Call it right before the install. */
64	export function saveUpdateEverythingResult(result: UpdateEverythingResult): void {
65	try {
66	window.localStorage.setItem(
67	SAVED_RESULT_KEY,
68	JSON.stringify({ ...result, savedAt: Date.now() } satisfies typeof SavedResult.Type),
69	);
70	} catch {
71	// Without storage the summary is lost with the relaunch; the update still runs.
72	}
73	}
74
75	export function clearSavedUpdateEverythingResult(): void {
76	try {
77	window.localStorage.removeItem(SAVED_RESULT_KEY);
78	} catch {
79	// Nothing to clear.
80	}
81	}
82
83	function takeSavedUpdateEverythingResult(): UpdateEverythingResult \| null {
84	try {
85	const raw = window.localStorage.getItem(SAVED_RESULT_KEY);
86	if (raw === null) return null;
87	window.localStorage.removeItem(SAVED_RESULT_KEY);
88	return Option.getOrNull(
89	Option.filter(
90	decodeSavedResult(raw),
91	(saved) => Date.now() - saved.savedAt < SAVED_RESULT_MAX_AGE_MS,
92	),
93	);
94	} catch {
95	return null;
96	}
97	}
98
99	/** Shows the one summary toast for a finished pass. */
100	export function showUpdateEverythingResult(result: UpdateEverythingResult): void {
101	const view = getUpdateEverythingToastView(result, APP_VERSION);
102	if (!view) return;
103	toastManager.add(
104	stackedThreadToast({
105	type: view.type,
106	title: view.title,
107	...(view.description
108	? { description: <span className="whitespace-pre-line">{view.description}</span> }
109	: {}),
110	}),
111	);
112	}
113
114	let notifiedNotAppliedVersion: string \| null = null;
115
116	/**
117	* Desktop launch notices for updates: the summary of a pass that ended with
118	* this app's relaunch, and a warning when the last install did not apply.
119	* Mount once at the app root.
120	*/
121	export function DesktopUpdateLaunchNotice() {
122	const notAppliedVersion = useDesktopUpdateState()?.notAppliedVersion ?? null;
123
124	useEffect(() => {
125	const result = takeSavedUpdateEverythingResult();
126	if (result) showUpdateEverythingResult(result);
127	}, []);
128
129	useEffect(() => {
130	if (notAppliedVersion === null \|\| notifiedNotAppliedVersion === notAppliedVersion) return;
131	notifiedNotAppliedVersion = notAppliedVersion;
132	toastManager.add(
133	stackedThreadToast({
134	type: "error",
135	title: "Update did not apply",
136	description: `T3 Code is still on ${APP_VERSION}. Install ${notAppliedVersion} again from the update button.`,
137	}),
138	);
139	}, [notAppliedVersion]);
140
141	return null;
142	}

apps/web/src/components/sidebar/useUpdateEverything.ts+126-10

...	@@ -10,21 +10,37 @@ import { useEnvironments } from "~/state/environments";	...	@@ -10,21 +10,37 @@ import { useEnvironments } from "~/state/environments";
10	import { useNpmReleasePublished } from "~/state/npmRelease";	10	import { useNpmReleasePublished } from "~/state/npmRelease";
11	import { serverEnvironment } from "~/state/server";	11	import { serverEnvironment } from "~/state/server";
12	import { environmentSession } from "~/state/session";	12	import { environmentSession } from "~/state/session";
		13	import { getProviderUpdateRunOutcome } from "../ProviderUpdateLaunchNotification.logic";
13	import { useProviderUpdates } from "../ProviderUpdatesAction";	14	import { useProviderUpdates } from "../ProviderUpdatesAction";
14	import { canUpdateServer, useServerUpdate } from "../ServerUpdateAction";	15	import { canUpdateServer, useServerUpdate } from "../ServerUpdateAction";
15	import {	16	import {
16	collectRestartingMachines,	17	collectRestartingMachines,
17	collectServerUpdateTargets,	18	collectServerUpdateTargets,
		19	collectUpdateEverythingResult,
18	describeUpdateEverything,	20	describeUpdateEverything,
19	runUpdateEverything,	21	runUpdateEverything,
		22	type SkippedMachine,
		23	type UpdateRunMachine,
20	} from "./updateEverything.logic";	24	} from "./updateEverything.logic";
		25	import {
		26	clearSavedUpdateEverythingResult,
		27	saveUpdateEverythingResult,
		28	setUpdateEverythingRun,
		29	settleUpdateRunMachine,
		30	showUpdateEverythingResult,
		31	} from "./updateEverythingRun";
		32
		33	/** Where this app's own download or install reports a failure. */
		34	export type ReportLocalUpdateError = (title: string, description: string) => void;
21		35
22	/**	36	/**
23	* The remote half of the sidebar update button: outdated providers on every	37	* The remote half of the sidebar update button: outdated providers on every
24	* connected machine, and servers behind the version this client runs or is	38	* connected machine, and servers behind the version this client runs or is
25	* about to install (`localUpdateVersion`). Servers that install from npm appear	39	* about to install (`localUpdateVersion`). Servers that install from npm appear
26	* only once npm confirms that version. `run` performs the whole pass, including	40	* only once npm confirms that version. `run` performs the whole pass, including
27	* this app's own download and install when given.	41	* this app's own download and install when given. It updates quietly, tracks
		42	* each machine in the run state, and reports one summary toast at the end.
		43	* When this app installs, the summary waits for the relaunch.
28	*/	44	*/
29	export function useUpdateEverything(localUpdateVersion: string \| null) {	45	export function useUpdateEverything(localUpdateVersion: string \| null) {
30	const targetVersion = localUpdateVersion ?? APP_VERSION;	46	const targetVersion = localUpdateVersion ?? APP_VERSION;
...	@@ -50,7 +66,10 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {	...	@@ -50,7 +66,10 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
50	[environments],	66	[environments],
51	);	67	);
52	const machines = useAtomValue(machinesAtom);	68	const machines = useAtomValue(machinesAtom);
53	const behindServers = collectServerUpdateTargets(machines, targetVersion);	69	const { targets: behindServers, skipped: unreachable } = collectServerUpdateTargets(
		70	machines,
		71	targetVersion,
		72	);
54	// A desktop-managed server installs from its own app's update feed, so only	73	// A desktop-managed server installs from its own app's update feed, so only
55	// servers that download t3 from npm wait for npm to publish the version.	74	// servers that download t3 from npm wait for npm to publish the version.
56	const needsNpm = behindServers.some((server) => server.selfUpdate !== "desktop-managed");	75	const needsNpm = behindServers.some((server) => server.selfUpdate !== "desktop-managed");
...	@@ -58,6 +77,16 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {	...	@@ -58,6 +77,16 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
58	const servers = published	77	const servers = published
59	? behindServers	78	? behindServers
60	: behindServers.filter((server) => server.selfUpdate === "desktop-managed");	79	: behindServers.filter((server) => server.selfUpdate === "desktop-managed");
		80	const skipped: ReadonlyArray<SkippedMachine> = [
		81	...unreachable,
		82	...behindServers
		83	.filter((server) => !servers.includes(server))
		84	.map((server) => ({
		85	environmentId: server.environmentId,
		86	label: server.serverLabel,
		87	reason: `t3@${targetVersion} is not on npm yet`,
		88	})),
		89	];
61	const restarts = collectRestartingMachines(machines, servers, localUpdateVersion !== null);	90	const restarts = collectRestartingMachines(machines, servers, localUpdateVersion !== null);
62	const providers = useProviderUpdates();	91	const providers = useProviderUpdates();
63	const updateServer = useServerUpdate();	92	const updateServer = useServerUpdate();
...	@@ -82,13 +111,100 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {	...	@@ -82,13 +111,100 @@ export function useUpdateEverything(localUpdateVersion: string \| null) {
82	lines,	111	lines,
83	/** The servers this run restarts, which stops their running threads. */	112	/** The servers this run restarts, which stops their running threads. */
84	restarts,	113	restarts,
85	run: (local: { downloadLocal?: () => Promise<boolean>; installLocal?: () => Promise<void> }) =>	114	/** Behind machines the run leaves alone, with the reason. */
86	runUpdateEverything({	115	skipped,
87	...local,	116	/** Runs the pass. `installLocal` resolves true when this app is relaunching.
88	updateProviders: providers.updateAll,	117	Both report failures to the pass, which lists them in its summary. */
89	updateServers: async () => {	118	run: async ({
90	await Promise.all(servers.map((server) => updateServer(server)));	119	downloadLocal,
91	},	120	installLocal,
92	}),	121	}: {
		122	downloadLocal?: (reportError: ReportLocalUpdateError) => Promise<boolean>;
		123	installLocal?: (reportError: ReportLocalUpdateError) => Promise<boolean>;
		124	}) => {
		125	const localFailures: Array<string> = [];
		126	const reportLocalError: ReportLocalUpdateError = (title, description) => {
		127	localFailures.push(`${title}: ${description}`);
		128	};
		129	const machinesById = new Map<UpdateRunMachine["environmentId"], UpdateRunMachine>();
		130	for (const machine of providers.machines) {
		131	machinesById.set(machine.environmentId, {
		132	environmentId: machine.environmentId,
		133	label: machine.label,
		134	updatesProviders: true,
		135	updatesServer: false,
		136	result: null,
		137	});
		138	}
		139	for (const server of servers) {
		140	const existing = machinesById.get(server.environmentId);
		141	machinesById.set(server.environmentId, {
		142	environmentId: server.environmentId,
		143	label: server.serverLabel,
		144	updatesProviders: existing?.updatesProviders ?? false,
		145	updatesServer: true,
		146	result: null,
		147	});
		148	}
		149	const localVersion = installLocal ? localUpdateVersion : null;
		150	setUpdateEverythingRun(() => ({
		151	localVersion,
		152	phase: "providers",
		153	machines: Array.from(machinesById.values()),
		154	}));
		155	const failedProviderMachines = new Set<UpdateRunMachine["environmentId"]>();
		156	try {
		157	const outcome = await runUpdateEverything({
		158	...(downloadLocal ? { downloadLocal: () => downloadLocal(reportLocalError) } : {}),
		159	updateProviders: async () => {
		160	const runs = await providers.updateAll({ quiet: true });
		161	const outcomes = runs.map((run) => {
		162	const runOutcome = getProviderUpdateRunOutcome(run);
		163	// An interrupted update did not finish either, so its row is not "done".
		164	if (runOutcome.status !== "updated") failedProviderMachines.add(run.environmentId);
		165	return runOutcome;
		166	});
		167	for (const machine of machinesById.values()) {
		168	if (!machine.updatesServer) {
		169	settleUpdateRunMachine(
		170	machine.environmentId,
		171	failedProviderMachines.has(machine.environmentId),
		172	);
		173	}
		174	}
		175	setUpdateEverythingRun((run) => run && { ...run, phase: "servers" });
		176	return outcomes;
		177	},
		178	updateServers: () =>
		179	Promise.all(
		180	servers.map(async (server) => {
		181	const serverOutcome = await updateServer(server, { quiet: true });
		182	settleUpdateRunMachine(
		183	server.environmentId,
		184	serverOutcome.status === "failed" \|\|
		185	failedProviderMachines.has(server.environmentId),
		186	);
		187	return { label: server.serverLabel, outcome: serverOutcome };
		188	}),
		189	),
		190	});
		191	const result = () =>
		192	collectUpdateEverythingResult({
		193	localVersion: null,
		194	localFailures,
		195	servers: outcome.servers,
		196	providers: outcome.providers,
		197	});
		198	if (outcome.localReady && installLocal) {
		199	setUpdateEverythingRun((run) => run && { ...run, phase: "installing" });
		200	saveUpdateEverythingResult({ ...result(), localVersion });
		201	if (await installLocal(reportLocalError)) return;
		202	clearSavedUpdateEverythingResult();
		203	}
		204	showUpdateEverythingResult(result());
		205	} finally {
		206	setUpdateEverythingRun(() => null);
		207	}
		208	},
93	};	209	};
94	}	210	}

apps/web/src/routes/__root.tsx+2

docs/internals/server-updates.md+7-4

docs/user/updating.md+10-3

...	@@ -16,9 +16,13 @@ installs the latest release from its own channel. Hover the button to see what	...	@@ -16,9 +16,13 @@ installs the latest release from its own channel. Hover the button to see what
16	it will update.	16	it will update.
17		17
18	Server updates appear only after that version is published to npm. Remote	18	Server updates appear only after that version is published to npm. Remote
19	desktop apps do not wait for npm. Servers that need	19	desktop apps do not wait for npm. The confirmation lists each machine it skips
20	a manual update are not included. Use the notice or Settings → Connections	20	and why, such as a machine that is offline or needs a manual update. Use the
21	for those.	21	notice or Settings → Connections for those.
		22
		23	If a desktop app restarts on its old version, T3 Code tells you the update did
		24	not apply. It does not install that version again on its own, even when another
		25	device asks. Use the update button on that machine to retry.
22		26
23	The confirmation counts the running threads that the restarts will interrupt,	27	The confirmation counts the running threads that the restarts will interrupt,
24	and how many can resume afterwards. See Continue threads after restarts	28	and how many can resume afterwards. See Continue threads after restarts
...	@@ -37,6 +41,9 @@ different value, use Apply to all in Settings after it connects.	...	@@ -37,6 +41,9 @@ different value, use Apply to all in Settings after it connects.
37	T3 Code must start again on that machine;	41	T3 Code must start again on that machine;
38	the setting does not enable automatic startup. Terminal commands may still be	42	the setting does not enable automatic startup. Terminal commands may still be
39	interrupted, and threads without saved provider resume state need a new message.	43	interrupted, and threads without saved provider resume state need a new message.
		44	A thread that finished its turn to wait on a monitor, subagent, or background task
		45	is woken and told that this work stopped. A background shell, such as a dev server,
		46	does not wake it; the thread learns about it with your next message.
40	A thread is not resumed while it waits for your approval, answer, or secret, or	47	A thread is not resumed while it waits for your approval, answer, or secret, or
41	while it is archived, snoozed, or settled. After three automatic resumes in a row,	48	while it is archived, snoozed, or settled. After three automatic resumes in a row,
42	the thread waits for your next message.	49	the thread waits for your next message.

packages/contracts/src/ipc.ts+7

 @@ -284,6 +284,12 @@ export interface DesktopUpdateState {
   message: string | null;
   errorContext: "check" | "download" | "install" | null;
   canRetry: boolean;
+  /**
+   * The version the last install handed to the OS installer that did not
+   * apply: the app came back on its old version. Server-triggered updates do
+   * not install it again; the user can retry it from the update button.
+   */
+  notAppliedVersion?: string;
 }
 export interface DesktopUpdateReleaseNote {
 @@ -315,6 +321,7 @@ export const DesktopUpdateStateSchema = Schema.Struct({
   message: Schema.NullOr(Schema.String),
   errorContext: Schema.NullOr(Schema.Literals(["check", "download", "install"])),
   canRetry: Schema.Boolean,
+  notAppliedVersion: Schema.optionalKey(Schema.String),
 });
 /** The desktop app's `t3` command on PATH, managed from Settings. */

packages/shared/src/AgentScope.ts+6

...	@@ -25,6 +25,12 @@ export interface AgentScopeShape {	...	@@ -25,6 +25,12 @@ export interface AgentScopeShape {
25	readonly name: string;	25	readonly name: string;
26	readonly threadId?: string \| undefined;	26	readonly threadId?: string \| undefined;
27	readonly env?: NodeJS.ProcessEnv \| undefined;	27	readonly env?: NodeJS.ProcessEnv \| undefined;
		28	/**
		29	* Set for terminals, so a job the user detached (tmux, nohup) outlives a
		30	* server restart. Agents leave it unset: the next server stops agents a
		31	* killed server left running before it resumes their sessions.
		32	*/
		33	readonly survivesRestart?: boolean \| undefined;
28	}) => Effect.Effect<AgentScopeCommand>;	34	}) => Effect.Effect<AgentScopeCommand>;
29	/**	35	/**
30	* Whether the latest agent scope of this thread was killed because the	36	* Whether the latest agent scope of this thread was killed because the

author	Theo Browne <me@t3.gg> 2026-10-14 00:38:17-07:00
committer	GitHub <noreply@github.com> 2026-10-14 00:38:17-07:00
log	eeaac49a9d20d94970b26fe7775fc8ddb4672f4f
tree	c36ecbafcf2f63527bcb37c0d96c1b9097de3299
parent	3bbfdc7755db099081ad546ff9e8263fc07b82e9
signature	Signed by PGP key B5690EEEBB952194

...	@@ -30,6 +30,7 @@ import { DesktopAppActivationCoordinator } from "../components/desktop/DesktopAp	...	@@ -30,6 +30,7 @@ import { DesktopAppActivationCoordinator } from "../components/desktop/DesktopAp
30	import { DesktopWebLinkCoordinator } from "../components/desktop/DesktopWebLinkCoordinator";	30	import { DesktopWebLinkCoordinator } from "../components/desktop/DesktopWebLinkCoordinator";
31	import { RunningThreadKeepAlive } from "../components/desktop/RunningThreadKeepAlive";	31	import { RunningThreadKeepAlive } from "../components/desktop/RunningThreadKeepAlive";
32	import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification";	32	import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification";
		33	import { DesktopUpdateLaunchNotice } from "../components/sidebar/updateEverythingRun";
33	import { NightlyMobileBetaNotice } from "../components/NightlyMobileBeta";	34	import { NightlyMobileBetaNotice } from "../components/NightlyMobileBeta";
34	import { LegacyThreadMigrationToast } from "../components/LegacyThreadMigrationToast";	35	import { LegacyThreadMigrationToast } from "../components/LegacyThreadMigrationToast";
35	import { ThreadNotificationCoordinator } from "../components/ThreadNotificationCoordinator";	36	import { ThreadNotificationCoordinator } from "../components/ThreadNotificationCoordinator";
...	@@ -265,6 +266,7 @@ function RootRouteView() {	...	@@ -265,6 +266,7 @@ function RootRouteView() {
265	<EventRouter skipInitialBootstrapNavigation={returningFromWelcomeRef.current} />	266	<EventRouter skipInitialBootstrapNavigation={returningFromWelcomeRef.current} />
266	) : null}	267	) : null}
267	{primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null}	268	{primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null}
		269	{isElectron ? <DesktopUpdateLaunchNotice /> : null}
268	{/* Hosted Nightly is "hosted-static", not authenticated, and needs it too. */}	270	{/* Hosted Nightly is "hosted-static", not authenticated, and needs it too. */}
269	<NightlyMobileBetaNotice />	271	<NightlyMobileBetaNotice />
270	{appShell}	272	{appShell}

...	@@ -64,8 +64,11 @@ opt-out. The	...	@@ -64,8 +64,11 @@ opt-out. The
64	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)	64	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)
65	requires matching durable run, provider thread, session, and native resume identity.	65	requires matching durable run, provider thread, session, and native resume identity.
66	Queued runs never started, so recovery holds them and continues the run they wait	66	Queued runs never started, so recovery holds them and continues the run they wait
67	behind. A finished run qualifies only when the restart cancelled its background work;	67	behind. A settled run qualifies only when the restart cancels work the agent waits on
68	its continuation tells the provider what will not report back.	68	(a monitor, subagent, or background task) and its session was live, so stale records
		69	from an older crash never wake it. Its continuation prompt is the note of what will not
		70	report back. A shell alone does not wake a settled run: most are dev servers nobody
		71	waits on, so that note waits for the next user message.
69		72
70	Recovery retires effects tied to the lost process and records continuation intent	73	Recovery retires effects tied to the lost process and records continuation intent
71	in the durable outbox. That intent survives another restart before provider startup.	74	in the durable outbox. That intent survives another restart before provider startup.
...	@@ -77,8 +80,8 @@ cannot be overwritten by a stale cancellation.	...	@@ -77,8 +80,8 @@ cannot be overwritten by a stale cancellation.
77	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)	80	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)
78	rechecks the preference, provider selection, newer user work, a stop the user requested,	81	rechecks the preference, provider selection, newer user work, a stop the user requested,
79	and maintenance turns such as `/compact` before dispatching. Intent and delivery both	82	and maintenance turns such as `/compact` before dispatching. Intent and delivery both
80	refuse a thread that is archived, settled, or still snoozed, a provider turn that already	83	refuse a thread that is archived, settled, or still snoozed, a cut turn whose provider
81	ended, and a run waiting on the user: an approval, question, or secret that was pending,	84	turn already ended, and a run waiting on the user: an approval, question, or secret that was pending,
82	or that recovery closed on the cut turn. Message-capable questions stay answerable, so	85	or that recovery closed on the cut turn. Message-capable questions stay answerable, so
83	they do not block. Delivery also stops after three consecutive continuations in one	86	they do not block. Delivery also stops after three consecutive continuations in one
84	`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable	87	`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable

fix: update all no longer runs agents twice or loses background work (#19291)