Commit 5108c978b1d9a8bd029e5ea96781d8cbc90b3877 | the-knife

apps/server/src/mcp/OrchestratorMcpService.test.ts+101-1

...	@@ -11,10 +11,12 @@ import {	...	@@ -11,10 +11,12 @@ import {
11	type OrchestrationV2ThreadProjection,	11	type OrchestrationV2ThreadProjection,
12	type ServerProvider,	12	type ServerProvider,
13	} from "@t3tools/contracts";	13	} from "@t3tools/contracts";
		14	import * as DateTime from "effect/DateTime";
14	import * as Effect from "effect/Effect";	15	import * as Effect from "effect/Effect";
15	import * as Layer from "effect/Layer";	16	import * as Layer from "effect/Layer";
16	import * as Ref from "effect/Ref";	17	import * as Ref from "effect/Ref";
17		18
		19	import { OrchestratorProjectionError } from "../orchestration-v2/Orchestrator.ts";
18	import type { ProviderAdapterV2Shape } from "../orchestration-v2/ProviderAdapter.ts";	20	import type { ProviderAdapterV2Shape } from "../orchestration-v2/ProviderAdapter.ts";
19	import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";	21	import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
20	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";	22	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
...	@@ -52,7 +54,22 @@ describe("OrchestratorMcpService", () => {	...	@@ -52,7 +54,22 @@ describe("OrchestratorMcpService", () => {
52	} as unknown as OrchestrationV2ThreadProjection;	54	} as unknown as OrchestrationV2ThreadProjection;
53	const childProjection = {	55	const childProjection = {
54	thread: { id: childThreadId },	56	thread: { id: childThreadId },
55	runs: [{ id: childRunId, ordinal: 1, status: "completed" }],	57	runs: [
		58	{
		59	id: childRunId,
		60	ordinal: 1,
		61	status: "completed",
		62	startedAt: DateTime.makeUnsafe("2026-10-03T10:00:00Z"),
		63	completedAt: DateTime.makeUnsafe("2026-10-03T10:10:00Z"),
		64	},
		65	{
		66	id: RunId.make("run:mcp-ack-continuation"),
		67	ordinal: 2,
		68	status: "failed",
		69	startedAt: DateTime.makeUnsafe("2026-10-03T10:02:00Z"),
		70	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00Z"),
		71	},
		72	],
56	contextTransfers: [],	73	contextTransfers: [],
57	messages: [],	74	messages: [],
58	subagents: [],	75	subagents: [],
...	@@ -126,6 +143,8 @@ describe("OrchestratorMcpService", () => {	...	@@ -126,6 +143,8 @@ describe("OrchestratorMcpService", () => {
126	const result = yield* service.taskStatus(scope, taskId);	143	const result = yield* service.taskStatus(scope, taskId);
127	assert.equal(result.status, "completed");	144	assert.equal(result.status, "completed");
128	assert.equal(result.summary, "terminal result");	145	assert.equal(result.summary, "terminal result");
		146	assert.equal(result.latestTerminalRunId, childRunId);
		147	assert.equal(result.latestTerminalStatus, "completed");
129	const commandIds = yield* Ref.get(acknowledgementCommandIds);	148	const commandIds = yield* Ref.get(acknowledgementCommandIds);
130	assert.equal(commandIds.length, 2);	149	assert.equal(commandIds.length, 2);
131	assert.notEqual(commandIds[0], commandIds[1]);	150	assert.notEqual(commandIds[0], commandIds[1]);
...	@@ -133,6 +152,87 @@ describe("OrchestratorMcpService", () => {	...	@@ -133,6 +152,87 @@ describe("OrchestratorMcpService", () => {
133	}),	152	}),
134	);	153	);
135		154
		155	it.effect("reports a restart-cut child as working until its continuation settles", () =>
		156	Effect.gen(function* () {
		157	const parentThreadId = ThreadId.make("thread:mcp-restart-parent");
		158	const childThreadId = ThreadId.make("thread:mcp-restart-child");
		159	const taskId = NodeId.make("node:mcp-restart-task");
		160	const dispatched = yield* Ref.make(0);
		161	let awaitsRestart = true;
		162	let readFails = true;
		163	const parentProjection = {
		164	thread: { id: parentThreadId },
		165	runs: [],
		166	contextTransfers: [],
		167	subagents: [
		168	{
		169	id: taskId,
		170	threadId: parentThreadId,
		171	origin: "app_owned",
		172	childThreadId,
		173	driver: "codex",
		174	model: "gpt-5.6-terra",
		175	status: "running",
		176	result: null,
		177	completionDelivery: { state: "pending" },
		178	},
		179	],
		180	} as unknown as OrchestrationV2ThreadProjection;
		181	const childProjection = {
		182	thread: { id: childThreadId },
		183	runs: [{ id: RunId.make("run:mcp-restart-child"), ordinal: 1, status: "cancelled" }],
		184	contextTransfers: [],
		185	messages: [],
		186	subagents: [],
		187	providerThreads: [],
		188	turnItems: [],
		189	} as unknown as OrchestrationV2ThreadProjection;
		190	const dependencies = Layer.mergeAll(
		191	NodeServices.layer,
		192	Layer.mock(ThreadManagementService.ThreadManagementService)({
		193	getThreadRecords: (threadId) =>
		194	Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection),
		195	delegatedTaskResultPending: () =>
		196	readFails
		197	? Effect.fail(new OrchestratorProjectionError({ threadId: childThreadId }))
		198	: Effect.succeed(awaitsRestart),
		199	dispatch: () => Ref.update(dispatched, (count) => count + 1).pipe(Effect.as({} as never)),
		200	}),
		201	Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
		202	Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({
		203	list: () => Effect.succeed([]),
		204	}),
		205	Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
		206	);
		207	const scope: McpInvocationScope = {
		208	environmentId: EnvironmentId.make("environment:mcp-restart"),
		209	threadId: parentThreadId,
		210	providerSessionId: "provider-session:mcp-restart",
		211	providerInstanceId: ProviderInstanceId.make("codex"),
		212	capabilities: new Set(["orchestration"]),
		213	issuedAt: 1,
		214	};
		215
		216	yield* Effect.gen(function* () {
		217	const service = yield* OrchestratorMcpService.OrchestratorMcpService;
		218	const failed = yield* service.taskStatus(scope, taskId).pipe(Effect.flip);
		219	assert.equal(failed.code, "orchestration_error");
		220	assert.equal(yield* Ref.get(dispatched), 0);
		221	readFails = false;
		222	const held = yield* service.taskStatus(scope, taskId);
		223	assert.equal(held.status, "running");
		224	assert.equal(held.workState, "working");
		225	assert.isNull(held.summary);
		226	// Acknowledging the cut run would suppress the real result's wake.
		227	assert.equal(yield* Ref.get(dispatched), 0);
		228	awaitsRestart = false;
		229	const settled = yield* service.taskStatus(scope, taskId);
		230	assert.equal(settled.status, "cancelled");
		231	assert.equal(yield* Ref.get(dispatched), 1);
		232	}).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies))));
		233	}),
		234	);
		235
136	it.effect("does not dispose delivery when a nonterminal task has no active child run", () =>	236	it.effect("does not dispose delivery when a nonterminal task has no active child run", () =>
137	Effect.gen(function* () {	237	Effect.gen(function* () {
138	const parentThreadId = ThreadId.make("thread:mcp-cancel-parent");	238	const parentThreadId = ThreadId.make("thread:mcp-cancel-parent");

apps/server/src/mcp/OrchestratorMcpService.ts+15-2

 @@ -52,6 +52,7 @@ import {
   type ServerProvider,
   ThreadId,
 } from "@t3tools/contracts";
+import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
 import * as Context from "effect/Context";
 import * as Crypto from "effect/Crypto";
 import * as DateTime from "effect/DateTime";
 @@ -412,7 +413,10 @@ function latestTerminalResultRun(
         run.status !== "rolled_back" &&
         (run.id === delegatedRun?.id || run.startedAt !== null),
     )
-    .toSorted((left, right) => right.ordinal - left.ordinal)[0];
+    .reduce<OrchestrationV2Run | undefined>(
+      (latest, run) => (latest === undefined || runRanAfter(run, latest) ? run : latest),
+      undefined,
+    );
 }
 function canExposeTaskRunResult(run: OrchestrationV2Run | undefined): run is OrchestrationV2Run {
 @@ -1055,7 +1059,16 @@ const make = Effect.gen(function* () {
         messages: [...childControls.messages, ...resultRecords.messages],
         turnItems: resultRecords.turnItems,
       };
-      const workState = task.result !== null ? "result_available" : progress.state;
+      // A restart cut the child's run and its continuation has not settled, or
+      // the child started working again after this read.
+      const heldForRestart =
+        task.result === null &&
+        progress.state === "result_available" &&
+        (yield* threadManagement
+          .delegatedTaskResultPending(task.childThreadId)
+          .pipe(Effect.mapError(threadManagementFailure)));
+      const workState =
+        task.result !== null ? "result_available" : heldForRestart ? "working" : progress.state;
       const status =
         task.result !== null
           ? taskStatusForRun(

apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts+624

...	@@ -7,6 +7,7 @@ import {	...	@@ -7,6 +7,7 @@ import {
7	MessageId,	7	MessageId,
8	type ModelSelection,	8	type ModelSelection,
9	NodeId,	9	NodeId,
		10	type OrchestrationV2Run,
10	ProjectId,	11	ProjectId,
11	ProviderDriverKind,	12	ProviderDriverKind,
12	ProviderInstanceId,	13	ProviderInstanceId,
...	@@ -35,6 +36,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";	...	@@ -35,6 +36,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
35	import * as EventSink from "./EventSink.ts";	36	import * as EventSink from "./EventSink.ts";
36	import * as Orchestrator from "./Orchestrator.ts";	37	import * as Orchestrator from "./Orchestrator.ts";
37	import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts";	38	import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts";
		39	import { continueRestartedRun } from "./RestartContinuation.ts";
38	import {	40	import {
39	OrchestrationV2EventSinkLayerLive,	41	OrchestrationV2EventSinkLayerLive,
40	OrchestrationV2LayerLive,	42	OrchestrationV2LayerLive,
...	@@ -384,6 +386,98 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {	...	@@ -384,6 +386,98 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
384	}),	386	}),
385	);	387	);
386		388
		389	it.effect("acceptance batches a settled_only sibling once its spawning run ended", () =>
		390	Effect.gen(function* () {
		391	const orchestrator = yield* Orchestrator.OrchestratorV2;
		392	const sink = yield* EventSink.EventSinkV2;
		393	const now = yield* DateTime.now;
		394	const threadId = ThreadId.make("mailbox-mixed");
		395	const runId = RunId.make("mailbox-mixed-parent");
		396	const taskId = NodeId.make("mailbox-mixed-always");
		397	const siblingId = NodeId.make("mailbox-mixed-settled-only");
		398	const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
		399	yield* seedParentWithTerminalTask({
		400	threadId,
		401	runId,
		402	projectId: ProjectId.make("mailbox-mixed-project"),
		403	rootNodeId: NodeId.make("mailbox-mixed-root"),
		404	taskId,
		405	deliveryState: "claimed",
		406	completionWake: "always",
		407	deliveryTaskIds: [taskId],
		408	now,
		409	});
		410	const projection = yield* orchestrator.getThreadProjection(threadId);
		411	const task = projection.subagents[0]!;
		412	const spawningRun = projection.runs.find((row) => row.id === runId)!;
		413	// A restart cut the spawning run; its continuation is the live run now.
		414	yield* sink.write({
		415	events: [
		416	{
		417	...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }),
		418	payload: { ...spawningRun, status: "cancelled" as const, completedAt: now },
		419	},
		420	runEvent({
		421	threadId,
		422	runId: RunId.make("mailbox-mixed-continuation"),
		423	ordinal: 2,
		424	status: "running",
		425	now,
		426	}),
		427	{
		428	id: EventId.make("mailbox-mixed-message"),
		429	type: "message.updated",
		430	threadId,
		431	runId,
		432	occurredAt: now,
		433	payload: {
		434	id: messageId,
		435	threadId,
		436	runId,
		437	nodeId: task.parentNodeId,
		438	role: "user",
		439	text: "Background task finished",
		440	attachments: [],
		441	streaming: false,
		442	createdBy: "agent",
		443	creationSource: "server",
		444	createdAt: now,
		445	updatedAt: now,
		446	delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
		447	},
		448	},
		449	{
		450	id: EventId.make(`event:${siblingId}`),
		451	type: "subagent.updated" as const,
		452	threadId,
		453	runId,
		454	nodeId: siblingId,
		455	occurredAt: now,
		456	payload: {
		457	...task,
		458	id: siblingId,
		459	completionWake: "settled_only" as const,
		460	completionDelivery: { state: "pending" as const, observedByRunId: null },
		461	},
		462	},
		463	],
		464	});
		465	yield* orchestrator.dispatch({
		466	type: "notification.delivery.accept",
		467	commandId: CommandId.make("accept-mixed"),
		468	threadId,
		469	messageId,
		470	});
		471	const accepted = yield* orchestrator.getThreadProjection(threadId);
		472	const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
		473	assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]);
		474	assert.equal(
		475	accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state,
		476	"claimed",
		477	);
		478	}),
		479	);
		480
387	it.effect("builds completion text and metadata from the same live cohort", () =>	481	it.effect("builds completion text and metadata from the same live cohort", () =>
388	Effect.gen(function* () {	482	Effect.gen(function* () {
389	const orchestrator = yield* Orchestrator.OrchestratorV2;	483	const orchestrator = yield* Orchestrator.OrchestratorV2;
...	@@ -652,3 +746,533 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {	...	@@ -652,3 +746,533 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
652	}),	746	}),
653	);	747	);
654	});	748	});
		749
		750	// Runtime reconciliation writes restart cancellations under this command
		751	// prefix, which the live terminal-run listener skips.
		752	const reconcileCommandId = (name: string) => CommandId.make(`command:runtime-reconcile:${name}`);
		753
		754	const parentProviderThreadId = (threadId: ThreadId) =>
		755	ProviderThreadId.make(`provider-thread:${String(threadId).replace("thread:", "")}`);
		756
		757	const runEvent = (input: {
		758	readonly threadId: ThreadId;
		759	readonly runId: RunId;
		760	readonly ordinal: number;
		761	readonly status: OrchestrationV2Run["status"];
		762	readonly now: DateTime.Utc;
		763	readonly providerThreadId?: ProviderThreadId;
		764	readonly userMessageId?: MessageId;
		765	readonly delegatedCompletion?: OrchestrationV2Run["delegatedCompletion"];
		766	}) => ({
		767	id: EventId.make(`event:${input.runId}:${input.status}`),
		768	type: "run.updated" as const,
		769	threadId: input.threadId,
		770	runId: input.runId,
		771	providerInstanceId: modelSelection.instanceId,
		772	occurredAt: input.now,
		773	payload: {
		774	id: input.runId,
		775	threadId: input.threadId,
		776	ordinal: input.ordinal,
		777	providerInstanceId: modelSelection.instanceId,
		778	modelSelection,
		779	providerThreadId: input.providerThreadId ?? null,
		780	userMessageId: input.userMessageId ?? MessageId.make(`message:${input.runId}`),
		781	rootNodeId: null,
		782	activeAttemptId: null,
		783	status: input.status,
		784	requestedAt: input.now,
		785	startedAt: input.now,
		786	completedAt: input.status === "running" ? null : input.now,
		787	checkpointId: null,
		788	contextHandoffId: null,
		789	...(input.delegatedCompletion === undefined
		790	? {}
		791	: { delegatedCompletion: input.delegatedCompletion }),
		792	},
		793	});
		794
		795	/** A running app-owned task whose child thread's first run the restart cancelled. */
		796	const seedRestartCancelledChild = (input: {
		797	readonly parentThreadId: ThreadId;
		798	readonly projectId: ProjectId;
		799	readonly parentRunId: RunId;
		800	readonly rootNodeId: NodeId;
		801	readonly name: string;
		802	readonly completionWake: "always" \| "settled_only";
		803	readonly continuationPending: boolean;
		804	/** "completed" seeds a settled turn whose background work the restart cancelled. */
		805	readonly runStatus?: "cancelled" \| "completed";
		806	readonly now: DateTime.Utc;
		807	}) =>
		808	Effect.gen(function* () {
		809	const eventSink = yield* EventSink.EventSinkV2;
		810	const taskId = NodeId.make(`node:${input.name}`);
		811	const childThreadId = ThreadId.make(`thread:${input.name}`);
		812	const childRunId = RunId.make(`run:${input.name}:1`);
		813	yield* eventSink.write({
		814	commandId: CommandId.make(`command:seed-child:${input.name}`),
		815	events: [
		816	{
		817	id: EventId.make(`event:${input.name}:thread`),
		818	type: "thread.created",
		819	threadId: childThreadId,
		820	occurredAt: input.now,
		821	payload: {
		822	createdBy: "agent",
		823	creationSource: "server",
		824	id: childThreadId,
		825	projectId: input.projectId,
		826	title: input.name,
		827	providerInstanceId: modelSelection.instanceId,
		828	modelSelection,
		829	runtimeMode: "full-access",
		830	interactionMode: "default",
		831	branch: null,
		832	worktreePath: null,
		833	activeProviderThreadId: null,
		834	lineage: {
		835	parentThreadId: input.parentThreadId,
		836	relationshipToParent: "subagent",
		837	rootThreadId: input.parentThreadId,
		838	},
		839	forkedFrom: { type: "node", nodeId: taskId },
		840	createdAt: input.now,
		841	updatedAt: input.now,
		842	archivedAt: null,
		843	settledOverride: null,
		844	settledAt: null,
		845	lastVisitedAt: null,
		846	deletedAt: null,
		847	},
		848	},
		849	{
		850	id: EventId.make(`event:${input.name}:task`),
		851	type: "subagent.updated",
		852	threadId: input.parentThreadId,
		853	runId: input.parentRunId,
		854	nodeId: taskId,
		855	driver,
		856	providerInstanceId: modelSelection.instanceId,
		857	occurredAt: input.now,
		858	payload: {
		859	id: taskId,
		860	threadId: input.parentThreadId,
		861	runId: input.parentRunId,
		862	parentNodeId: input.rootNodeId,
		863	origin: "app_owned",
		864	createdBy: "agent",
		865	driver,
		866	providerInstanceId: modelSelection.instanceId,
		867	providerThreadId: null,
		868	childThreadId,
		869	nativeTaskRef: null,
		870	prompt: `Run ${input.name}.`,
		871	title: null,
		872	model: null,
		873	completionWake: input.completionWake,
		874	status: "running",
		875	result: null,
		876	startedAt: input.now,
		877	completedAt: null,
		878	updatedAt: input.now,
		879	},
		880	},
		881	],
		882	});
		883	yield* eventSink.writeWithEffects({
		884	commandId: reconcileCommandId(input.name),
		885	events: [
		886	runEvent({
		887	threadId: childThreadId,
		888	runId: childRunId,
		889	ordinal: 1,
		890	status: input.runStatus ?? "cancelled",
		891	now: input.now,
		892	}),
		893	],
		894	effects: input.continuationPending
		895	? [
		896	{
		897	id: `effect:restart-continuation:${childRunId}`,
		898	commandId: reconcileCommandId(input.name),
		899	threadId: childThreadId,
		900	request: { type: "provider-runtime.continue", sourceRunId: childRunId },
		901	},
		902	]
		903	: [],
		904	});
		905	return { taskId, childThreadId, childRunId };
		906	});
		907
		908	it.layer(TestLayer)("delegated tasks across a server restart", (it) => {
		909	it.effect("holds a restart-cancelled child for its continuation's result", () =>
		910	Effect.gen(function* () {
		911	const orchestrator = yield* Orchestrator.OrchestratorV2;
		912	const eventSink = yield* EventSink.EventSinkV2;
		913	const now = yield* DateTime.now;
		914	const threadId = ThreadId.make("thread:restart-parent");
		915	const projectId = ProjectId.make("project:restart-parent");
		916	const runId = RunId.make("run:restart-parent");
		917	const rootNodeId = NodeId.make("node:restart-parent-root");
		918	yield* seedParentWithTerminalTask({
		919	threadId,
		920	projectId,
		921	runId,
		922	rootNodeId,
		923	taskId: NodeId.make("node:restart-parent-settled"),
		924	deliveryState: "delivered",
		925	now,
		926	});
		927	const child = (
		928	name: string,
		929	continuationPending: boolean,
		930	runStatus?: "cancelled" \| "completed",
		931	) =>
		932	seedRestartCancelledChild({
		933	parentThreadId: threadId,
		934	projectId,
		935	parentRunId: runId,
		936	rootNodeId,
		937	name,
		938	completionWake: "always",
		939	continuationPending,
		940	...(runStatus === undefined ? {} : { runStatus }),
		941	now,
		942	});
		943	const resumed = yield* child("restart-resumed-child", true);
		944	const stopped = yield* child("restart-stopped-child", false);
		945	// Settled with only background work left: its interim reply is not the result.
		946	const backgrounded = yield* child("restart-backgrounded-child", true, "completed");
		947	// A second restart cut the first continuation before it started.
		948	const recut = yield* child("restart-recut-child", false);
		949	const recutContinuationId = RunId.make("run:restart-recut-child:2");
		950	const recutCommandId = CommandId.make("command:restart-recut-child:reconcile");
		951	const recutRun = runEvent({
		952	threadId: recut.childThreadId,
		953	runId: recutContinuationId,
		954	ordinal: 2,
		955	status: "cancelled",
		956	now,
		957	});
		958	yield* eventSink.writeWithEffects({
		959	commandId: recutCommandId,
		960	events: [
		961	{
		962	...recutRun,
		963	payload: {
		964	...recutRun.payload,
		965	startedAt: null,
		966	restartContinuationOfRunId: recut.childRunId,
		967	},
		968	},
		969	],
		970	effects: [
		971	{
		972	id: `effect:restart-continuation:${recutContinuationId}`,
		973	commandId: recutCommandId,
		974	threadId: recut.childThreadId,
		975	request: { type: "provider-runtime.continue", sourceRunId: recutContinuationId },
		976	},
		977	],
		978	});
		979
		980	yield* orchestrator.recoverDelegatedTasks;
		981
		982	const recovered = yield* orchestrator.getThreadProjection(threadId);
		983	const task = (id: NodeId) => recovered.subagents.find((row) => row.id === id);
		984	assert.equal(task(stopped.taskId)?.status, "cancelled");
		985	assert.equal(task(stopped.taskId)?.completionDelivery?.state, "claimed");
		986	assert.equal(task(resumed.taskId)?.status, "running");
		987	assert.isNull(task(resumed.taskId)?.result ?? null);
		988	assert.equal(task(backgrounded.taskId)?.status, "running");
		989	assert.isNull(task(backgrounded.taskId)?.result ?? null);
		990	assert.equal(task(recut.taskId)?.status, "running");
		991	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId));
		992	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId));
		993	assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
		994	// A replayed first continuation settling must not release the second one's hold.
		995	yield* orchestrator.recoverDelegatedTask(recut.childThreadId, recut.childRunId);
		996	const replayed = yield* orchestrator.getThreadProjection(threadId);
		997	assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running");
		998	// A caller that read the cancelled run before the child resumed sees it as pending.
		999	yield* eventSink.write({
		1000	commandId: CommandId.make("command:restart-stopped-child:resumed"),
		1001	events: [
		1002	runEvent({
		1003	threadId: stopped.childThreadId,
		1004	runId: RunId.make("run:restart-stopped-child:2"),
		1005	ordinal: 2,
		1006	status: "running",
		1007	now,
		1008	}),
		1009	],
		1010	});
		1011	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
		1012	assert.isFalse(
		1013	recovered.contextTransfers.some(
		1014	(transfer) => transfer.sourceThreadId === resumed.childThreadId,
		1015	),
		1016	);
		1017
		1018	// The continuation's own run finishing settles the task with its result.
		1019	const afterSequence = yield* eventSink.latestSequence();
		1020	const continuationRunId = RunId.make("run:restart-resumed-child:2");
		1021	yield* eventSink.write({
		1022	commandId: CommandId.make("command:restart-resumed-child:completed"),
		1023	events: [
		1024	{
		1025	id: EventId.make("event:restart-resumed-child:result"),
		1026	type: "message.updated",
		1027	threadId: resumed.childThreadId,
		1028	runId: continuationRunId,
		1029	occurredAt: now,
		1030	payload: {
		1031	id: MessageId.make("message:restart-resumed-child:result"),
		1032	threadId: resumed.childThreadId,
		1033	runId: continuationRunId,
		1034	nodeId: null,
		1035	role: "assistant",
		1036	text: "Finished after the restart.",
		1037	attachments: [],
		1038	streaming: false,
		1039	createdBy: "agent",
		1040	creationSource: "server",
		1041	createdAt: now,
		1042	updatedAt: now,
		1043	},
		1044	},
		1045	runEvent({
		1046	threadId: resumed.childThreadId,
		1047	runId: continuationRunId,
		1048	ordinal: 2,
		1049	status: "completed",
		1050	now,
		1051	}),
		1052	],
		1053	});
		1054	const settled = yield* eventSink
		1055	.stream({ afterSequence, eventType: "subagent.updated" })
		1056	.pipe(
		1057	Stream.filter(
		1058	(stored) =>
		1059	stored.event.type === "subagent.updated" &&
		1060	stored.event.payload.id === resumed.taskId,
		1061	),
		1062	Stream.take(1),
		1063	Stream.runHead,
		1064	);
		1065	assert.isTrue(settled._tag === "Some");
		1066	const finished = yield* orchestrator.getThreadProjection(threadId);
		1067	const finishedTask = finished.subagents.find((row) => row.id === resumed.taskId);
		1068	assert.equal(finishedTask?.status, "completed");
		1069	assert.equal(finishedTask?.result, "Finished after the restart.");
		1070	}),
		1071	);
		1072
		1073	it.effect("settles a restart-cancelled child whose continuation declines to start", () =>
		1074	Effect.gen(function* () {
		1075	const orchestrator = yield* Orchestrator.OrchestratorV2;
		1076	const now = yield* DateTime.now;
		1077	const threadId = ThreadId.make("thread:restart-declined-parent");
		1078	const projectId = ProjectId.make("project:restart-declined-parent");
		1079	const runId = RunId.make("run:restart-declined-parent");
		1080	const rootNodeId = NodeId.make("node:restart-declined-parent-root");
		1081	yield* seedParentWithTerminalTask({
		1082	threadId,
		1083	projectId,
		1084	runId,
		1085	rootNodeId,
		1086	taskId: NodeId.make("node:restart-declined-parent-settled"),
		1087	deliveryState: "delivered",
		1088	now,
		1089	});
		1090	const child = yield* seedRestartCancelledChild({
		1091	parentThreadId: threadId,
		1092	projectId,
		1093	parentRunId: runId,
		1094	rootNodeId,
		1095	name: "restart-declined-child",
		1096	completionWake: "always",
		1097	continuationPending: true,
		1098	now,
		1099	});
		1100	yield* orchestrator.recoverDelegatedTasks;
		1101	const held = yield* orchestrator.getThreadProjection(threadId);
		1102	assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running");
		1103
		1104	yield* continueRestartedRun({
		1105	threadId: child.childThreadId,
		1106	sourceRunId: child.childRunId,
		1107	}).pipe(
		1108	Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })),
		1109	);
		1110
		1111	const settled = yield* orchestrator.getThreadProjection(threadId);
		1112	const task = settled.subagents.find((row) => row.id === child.taskId);
		1113	assert.equal(task?.status, "cancelled");
		1114	assert.equal(task?.completionDelivery?.state, "claimed");
		1115	}),
		1116	);
		1117
		1118	// A running provider turn does not prove the provider consumed the delivery
		1119	// (Claude marks the turn running before it reads the prompt), so a cut
		1120	// delivery is offered again even when a continuation resumes that turn.
		1121	it.effect("re-offers a cut delivery even when a continuation resumes its turn", () =>
		1122	Effect.gen(function* () {
		1123	const orchestrator = yield* Orchestrator.OrchestratorV2;
		1124	const eventSink = yield* EventSink.EventSinkV2;
		1125	const now = yield* DateTime.now;
		1126	const seed = (name: string, continuationPending: boolean) =>
		1127	Effect.gen(function* () {
		1128	const threadId = ThreadId.make(`thread:${name}`);
		1129	const runId = RunId.make(`run:${name}`);
		1130	const taskId = NodeId.make(`node:${name}-task`);
		1131	const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
		1132	const deliveryRunId = RunId.make(`run:${name}:delivery`);
		1133	yield* seedParentWithTerminalTask({
		1134	threadId,
		1135	projectId: ProjectId.make(`project:${name}`),
		1136	runId,
		1137	rootNodeId: NodeId.make(`node:${name}-root`),
		1138	taskId,
		1139	deliveryState: "claimed",
		1140	completionWake: "always",
		1141	deliveryTaskIds: [taskId],
		1142	now,
		1143	});
		1144	yield* eventSink.writeWithEffects({
		1145	commandId: reconcileCommandId(name),
		1146	events: [
		1147	{
		1148	id: EventId.make(`event:${name}:delivery-message`),
		1149	type: "message.updated",
		1150	threadId,
		1151	runId: deliveryRunId,
		1152	occurredAt: now,
		1153	payload: {
		1154	id: messageId,
		1155	threadId,
		1156	runId: deliveryRunId,
		1157	nodeId: null,
		1158	role: "user",
		1159	text: `Delegated task ${taskId} reached a terminal state.`,
		1160	attachments: [],
		1161	streaming: false,
		1162	createdBy: "agent",
		1163	creationSource: "server",
		1164	createdAt: now,
		1165	updatedAt: now,
		1166	delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
		1167	},
		1168	},
		1169	runEvent({
		1170	threadId,
		1171	runId: deliveryRunId,
		1172	ordinal: 2,
		1173	status: "cancelled",
		1174	now,
		1175	providerThreadId: parentProviderThreadId(threadId),
		1176	userMessageId: messageId,
		1177	}),
		1178	],
		1179	effects: continuationPending
		1180	? [
		1181	{
		1182	id: `effect:restart-continuation:${deliveryRunId}`,
		1183	commandId: reconcileCommandId(name),
		1184	threadId,
		1185	request: { type: "provider-runtime.continue", sourceRunId: deliveryRunId },
		1186	},
		1187	]
		1188	: [],
		1189	});
		1190	return { threadId, runId, taskId };
		1191	});
		1192	const cut = yield* seed("delivery-cut", false);
		1193	const resumed = yield* seed("delivery-resumed", true);
		1194
		1195	yield* orchestrator.recoverDelegatedTasks;
		1196
		1197	for (const seeded of [cut, resumed]) {
		1198	const projection = yield* orchestrator.getThreadProjection(seeded.threadId);
		1199	assert.deepEqual(
		1200	projection.subagents.find((row) => row.id === seeded.taskId)?.completionDelivery?.state,
		1201	"claimed",
		1202	);
		1203	const delivery = projection.runs.find((row) => row.id === seeded.runId)?.delegatedCompletion
		1204	?.delivery;
		1205	assert.equal(delivery?.generation, 2);
		1206	assert.deepEqual(delivery?.taskIds, [seeded.taskId]);
		1207	}
		1208	}),
		1209	);
		1210
		1211	it.effect("wakes the parent for a settled_only task once its spawning turn ended", () =>
		1212	Effect.gen(function* () {
		1213	const orchestrator = yield* Orchestrator.OrchestratorV2;
		1214	const eventSink = yield* EventSink.EventSinkV2;
		1215	const now = yield* DateTime.now;
		1216	const threadId = ThreadId.make("thread:settled-only-restart");
		1217	const projectId = ProjectId.make("project:settled-only-restart");
		1218	const runId = RunId.make("run:settled-only-restart");
		1219	const rootNodeId = NodeId.make("node:settled-only-restart-root");
		1220	yield* seedParentWithTerminalTask({
		1221	threadId,
		1222	projectId,
		1223	runId,
		1224	rootNodeId,
		1225	taskId: NodeId.make("node:settled-only-restart-settled"),
		1226	deliveryState: "delivered",
		1227	now,
		1228	});
		1229	// The restart cut the turn blocked in delegate_task(wait); its
		1230	// continuation is a new, live run.
		1231	yield* eventSink.write({
		1232	commandId: reconcileCommandId("settled-only-restart"),
		1233	events: [
		1234	runEvent({
		1235	threadId,
		1236	runId,
		1237	ordinal: 1,
		1238	status: "cancelled",
		1239	now,
		1240	providerThreadId: parentProviderThreadId(threadId),
		1241	userMessageId: MessageId.make(`message:seed-user:${threadId}`),
		1242	delegatedCompletion: { disposition: "open", nextGeneration: 2, delivery: null },
		1243	}),
		1244	runEvent({
		1245	threadId,
		1246	runId: RunId.make("run:settled-only-restart:continuation"),
		1247	ordinal: 2,
		1248	status: "running",
		1249	now,
		1250	providerThreadId: parentProviderThreadId(threadId),
		1251	}),
		1252	],
		1253	});
		1254	const child = yield* seedRestartCancelledChild({
		1255	parentThreadId: threadId,
		1256	projectId,
		1257	parentRunId: runId,
		1258	rootNodeId,
		1259	name: "settled-only-restart-child",
		1260	completionWake: "settled_only",
		1261	continuationPending: false,
		1262	now,
		1263	});
		1264
		1265	yield* orchestrator.recoverDelegatedTasks;
		1266
		1267	const projection = yield* orchestrator.getThreadProjection(threadId);
		1268	assert.equal(
		1269	projection.subagents.find((row) => row.id === child.taskId)?.completionDelivery?.state,
		1270	"claimed",
		1271	);
		1272	assert.deepEqual(
		1273	projection.runs.find((row) => row.id === runId)?.delegatedCompletion?.delivery?.taskIds,
		1274	[child.taskId],
		1275	);
		1276	}),
		1277	);
		1278	});

apps/server/src/orchestration-v2/EffectWorker.test.ts+49-2

...	@@ -79,6 +79,8 @@ function restartEffect(	...	@@ -79,6 +79,8 @@ function restartEffect(
79	function makeExecutorLayer(input: {	79	function makeExecutorLayer(input: {
80	readonly events: Ref.Ref<ReadonlyArray<string>>;	80	readonly events: Ref.Ref<ReadonlyArray<string>>;
81	readonly failFirstStart?: Ref.Ref<boolean>;	81	readonly failFirstStart?: Ref.Ref<boolean>;
		82	readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
		83	readonly continueAfterRestart?: boolean;
82	}) {	84	}) {
83	const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);	85	const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
84	const dependencies = Layer.mergeAll(	86	const dependencies = Layer.mergeAll(
...	@@ -148,8 +150,10 @@ function makeExecutorLayer(input: {	...	@@ -148,8 +150,10 @@ function makeExecutorLayer(input: {
148	Layer.provide(	150	Layer.provide(
149	Layer.mergeAll(	151	Layer.mergeAll(
150	dependencies,	152	dependencies,
151	Layer.mock(ThreadManagementService.ThreadManagementService)({}),	153	Layer.mock(ThreadManagementService.ThreadManagementService)(input.threads ?? {}),
152	ServerSettings.layerTest(),	154	ServerSettings.layerTest(
		155	input.continueAfterRestart === true ? { continueThreadsAfterServerUpdate: true } : {},
		156	),
153	),	157	),
154	),	158	),
155	);	159	);
...	@@ -753,3 +757,46 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (	...	@@ -753,3 +757,46 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (
753	]);	757	]);
754	}),	758	}),
755	);	759	);
		760
		761	it.effect("settles a delegated child once its restart continuation fails for good", () =>
		762	Effect.gen(function* () {
		763	const timestamp = DateTime.formatIso(yield* DateTime.now);
		764	const events = yield* Ref.make<ReadonlyArray<string>>([]);
		765	const recovered = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
		766	const layer = makeExecutorLayer({
		767	events,
		768	continueAfterRestart: true,
		769	threads: {
		770	getThreadRecords: () => Effect.fail(new Error("provider instance removed") as never),
		771	recoverDelegatedTask: (childThreadId) =>
		772	Ref.update(recovered, (ids) => [...ids, childThreadId]),
		773	},
		774	});
		775	const effect: EffectOutbox.OrchestrationEffectV2 = {
		776	id: `effect:restart-continuation:${runId}`,
		777	commandId: CommandId.make("command:restart-continuation-failure"),
		778	threadId,
		779	request: { type: "provider-runtime.continue", sourceRunId: runId },
		780	status: "running",
		781	attemptCount: 1,
		782	availableAt: timestamp,
		783	leaseOwner: "test-worker",
		784	leaseExpiresAt: timestamp,
		785	createdAt: timestamp,
		786	updatedAt: timestamp,
		787	completedAt: null,
		788	lastError: null,
		789	};
		790	yield* Effect.gen(function* () {
		791	const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
		792	assert.isTrue(
		793	Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: true }))),
		794	);
		795	assert.deepEqual(yield* Ref.get(recovered), []);
		796	assert.isTrue(
		797	Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: false }))),
		798	);
		799	assert.deepEqual(yield* Ref.get(recovered), [threadId]);
		800	}).pipe(Effect.provide(layer));
		801	}),
		802	);

apps/server/src/orchestration-v2/EffectWorker.ts+10-5

 @@ -108,13 +108,17 @@ export const executorLayer: Layer.Layer<
       execute: (effect, options) => {
         const willRetry = options?.willRetry ?? false;
         switch (effect.request.type) {
-          case "provider-runtime.continue":
+          case "provider-runtime.continue": {
-            return continueRestartedRun({
+            const sourceRunId = effect.request.sourceRunId;
-              threadId: effect.threadId,
+            return continueRestartedRun({ threadId: effect.threadId, sourceRunId }).pipe(
-              sourceRunId: effect.request.sourceRunId,
-            }).pipe(
               Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
               Effect.provideService(ServerSettings.ServerSettingsService, settings),
+              // A continuation that will never run still owes a delegated parent a result.
+              Effect.tapError(() =>
+                willRetry
+                  ? Effect.void
+                  : threads.recoverDelegatedTask(effect.threadId, sourceRunId),
+              ),
               Effect.mapError(
                 (cause) =>
                   new OrchestrationEffectExecutionError({
 @@ -124,6 +128,7 @@ export const executorLayer: Layer.Layer<
                   }),
               ),
             );
+          }
           case "provider-session.detach":
             return providerSessions
               .detach({

apps/server/src/orchestration-v2/Orchestrator.ts+209-94

 @@ -1,6 +1,7 @@
 import {
   latestExecutedRun,
   latestRootProviderFailure,
+  runRanAfter,
   usageLimitBlockedRun,
 } from "@t3tools/shared/orchestrationV2ThreadError";
 import { threadPullRequestsOf } from "@t3tools/shared/threadPullRequests";
 @@ -77,6 +78,7 @@ import { notificationTurnItem } from "./Notification.ts";
 import { isRestartNoteSource } from "./RestartBackgroundNote.ts";
 import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts";
 import { EventSinkV2 } from "./EventSink.ts";
+import * as EffectOutbox from "./EffectOutbox.ts";
 import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
 import { IdAllocatorV2 } from "./IdAllocator.ts";
 import {
 @@ -242,6 +244,17 @@ export interface OrchestratorV2DispatchResult {
 export interface OrchestratorV2Shape {
   readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
+  /** Startup pass that settles delegated-task results and deliveries runs left behind. */
+  readonly recoverDelegatedTasks: Effect.Effect<void>;
+  /** Settles a delegated child whose restart continuation of `sourceRunId` declined or failed. */
+  readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
+  /**
+   * Whether a delegated child's apparent result is not final yet: a restart
+   * continuation is pending, or the child is working again.
+   */
+  readonly delegatedTaskResultPending: (
+    childThreadId: ThreadId,
+  ) => Effect.Effect<boolean, OrchestratorProjectionError>;
   readonly dispatch: (
     command: OrchestrationV2ServerCommand,
   ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
 @@ -338,7 +351,8 @@ function wakeWorkStartedAt(
   return previous === undefined ? {} : { workStartedAt: orchestrationV2RunWorkStartedAt(previous) };
 }
-function isNativeMaintenanceCommand(message: {
+/** A native /compact or /logout turn: provider maintenance, not agent work. */
+export function isNativeMaintenanceCommand(message: {
   readonly text: string;
   readonly attachments: ReadonlyArray<ChatAttachment>;
   readonly context?: import("@t3tools/contracts").OrchestrationMessageContext | undefined;
 @@ -718,6 +732,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
   const idAllocator = yield* IdAllocatorV2;
   const projects = yield* ProjectStore.ProjectStoreV2;
   const projectionStore = yield* ProjectionStoreV2;
+  const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
   const nextTurnItemOrdinal = (
     projection: Pick<OrchestrationV2ThreadProjection, "thread"> &
       Partial<Pick<OrchestrationV2ThreadProjection, "turnItems">>,
 @@ -4357,7 +4372,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
           projection.thread.archivedAt !== null ||
           projection.thread.deletedAt !== null ||
           projection.thread.providerInstanceId !== source.providerInstanceId ||
-          projection.runs.some((run) => run.ordinal > source.ordinal)
+          // Held queued runs never started; they wait behind the continuation.
+          projection.runs.some(
+            (run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
+          )
         ) {
           // Preserve the current row so stale automatic deliveries receive an
           // accepted receipt without changing work or repeatedly retrying.
 @@ -6594,12 +6612,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
     // policy when the child terminalizes. Both writers hold this parent lock,
     // so a terminal task means finalize already committed the terminal row and
     // already made its offer decision under the pre-upgrade policy: under
-    // settled_only it offered iff the parent had no live run. Plan a delivery
+    // settled_only it offered iff the spawning run was not live. Plan a
-    // only when the parent has a live run now, which is precisely the case
+    // delivery only when that run is live now, which is precisely the case
     // where finalize skipped. The mailbox steers a capable active session or
-    // queues behind that run. When the parent is not live, finalize already
+    // queues behind that run. When that run is not live, finalize already
-    // offered and a second
+    // offered and a second offer would wake the parent twice. (If the parent settled in between,
-    // offer would wake the parent twice. (If the parent settled in between,
     // this skips a wake that finalize also skipped; a missed wake is cheaper
     // than a duplicate one, and the result is already in the projection.)
     const parentRun =
 @@ -6609,7 +6626,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
     const completionPlan =
       command.completionWake === "always" &&
       isTerminalDelegatedTaskStatus(task.status) &&
-      hasLiveRun(parentProjection)
+      parentRun !== undefined &&
+      hasLiveRun({ runs: [parentRun] })
         ? yield* planDelegatedCompletionDelivery({
             parentProjection,
             parentRun,
 @@ -8494,6 +8512,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
         : undefined;
     });
+  /**
+   * A run with a pending restart continuation has no outcome yet: restart
+   * reconciliation cancelled it mid-turn, or cancelled the background work a
+   * settled turn was waiting on. The continuation's own run settles it, or
+   * RestartContinuation recovers the thread when it declines to continue.
+   */
+  const awaitsRestartContinuation = (run: OrchestrationV2Run) =>
+    effectOutbox
+      .get(`effect:restart-continuation:${run.id}`)
+      .pipe(
+        Effect.map(
+          Option.exists((effect) => effect.status === "pending" || effect.status === "running"),
+        ),
+      );
+  /**
+   * A delegated child's result is held while its result run, or a run after it
+   * that a second restart cut before it started, still has a continuation pending.
+   */
+  const childAwaitsRestartContinuation = (
+    runs: ReadonlyArray<OrchestrationV2Run>,
+    resultRun: OrchestrationV2Run,
+    settledContinuationOf?: RunId,
+  ) =>
+    Effect.forEach(
+      runs.filter(
+        (run) =>
+          run.id !== settledContinuationOf &&
+          (run.id === resultRun.id || runRanAfter(run, resultRun)),
+      ),
+      awaitsRestartContinuation,
+    ).pipe(Effect.map((pending) => pending.includes(true)));
   const planDelegatedCompletionDelivery = Effect.fn(
     "orchestrationV2.planDelegatedCompletionDelivery",
   )(function* (input: {
 @@ -8541,9 +8592,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
         offer: false,
       };
     }
+    // settled_only defers to a blocking delegate_task wait, which lives only as
+    // long as the turn that spawned the task. Once that turn is over (a restart
+    // continues it as a new run), nothing else delivers the result.
     if (
       (input.task.completionWake ?? "settled_only") === "settled_only" &&
-      hasLiveRun(input.parentProjection)
+      hasLiveRun({ runs: [input.parentRun] })
     ) {
       return {
         task: input.updatedTask,
 @@ -8694,9 +8748,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
    * delegated_task.wake-policy handler rewrites the same subagent row under
    * that lock with a full-row payload, and unserialized writers clobber each
    * other (stale policy on the terminal row, or a terminal row regressed to
-   * running).
+   * running). `settledContinuationOf` names the restart continuation that just
+   * declined or failed, so its own still-running effect does not hold the child.
    */
-  const finalizeAppOwnedSubagent = (childThreadId: ThreadId) =>
+  const finalizeAppOwnedSubagent = (
+    childThreadId: ThreadId,
+    options?: { readonly settledContinuationOf?: RunId },
+  ) =>
     Effect.gen(function* () {
       const childControls = yield* projectionStore.getThreadRecords(
         childThreadId,
 @@ -8719,6 +8777,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
       if (terminalStatus === null) {
         return;
       }
+      if (
+        yield* childAwaitsRestartContinuation(
+          childControls.runs,
+          childRun,
+          options?.settledContinuationOf,
+        )
+      ) {
+        return;
+      }
       const childResult = yield* projectionStore.getThreadRecords(
         childThreadId,
 @@ -9147,7 +9214,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
       });
       return;
     }
-    const parentIsLive = hasLiveRun(projection);
+    // settled_only tasks defer only to a blocking wait in their spawning run.
+    const parentIsLive = hasLiveRun({ runs: [parentRun] });
     const pendingTaskIds =
       projection.thread.archivedAt === null && projection.thread.deletedAt === null
         ? projection.subagents
 @@ -9749,97 +9817,140 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
       Effect.forkDetach,
     );
-  // Recover child results from projections. Queue recovery instead holds
+  // Settles child results and completion deliveries whose runs ended without
-  // unstarted runs until an explicit queue.resume command arrives.
+  // the listener above: before this boot, or in runtime reconciliation, which
-  yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(
+  // it skips. Startup runs this after reconciliation and before the effect
-    Effect.flatMap((threadIds) =>
+  // worker. Queue recovery instead holds unstarted runs until an explicit
-      Effect.forEach(
+  // queue.resume command arrives.
-        threadIds,
+  const recoverDelegatedTasks = Effect.gen(function* () {
-        (threadId) =>
+    yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(
-          Effect.gen(function* () {
+      Effect.flatMap((threadIds) =>
-            const thread = yield* projectionStore.getThreadShell(threadId);
+        Effect.forEach(
-            const parentThreadId = thread?.lineage.parentThreadId;
+          threadIds,
-            if (parentThreadId === undefined || parentThreadId === null) return;
+          (threadId) =>
-            yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
+            Effect.gen(function* () {
-          }).pipe(
+              const thread = yield* projectionStore.getThreadShell(threadId);
-            Effect.catchCause((cause) =>
+              const parentThreadId = thread?.lineage.parentThreadId;
-              Effect.logWarning("Failed to recover terminal app-owned subagent", {
+              if (parentThreadId === undefined || parentThreadId === null) return;
-                childThreadId: threadId,
+              yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
-                cause,
+            }).pipe(
-              }),
-            ),
-          ),
-        { concurrency: 8, discard: true },
-      ),
-    ),
-    Effect.catchCause((cause) =>
-      Effect.logWarning("Failed to inspect app-owned subagents during recovery", {
-        cause,
-      }),
-    ),
-  );
-  yield* projectionStore.getRecoveryThreadIds("delegated-completions").pipe(
-    Effect.flatMap((threadIds) =>
-      Effect.forEach(
-        threadIds,
-        (threadId) =>
-          threadDispatch
-            .withLock(
-              threadId,
-              Effect.gen(function* () {
-                const projection = yield* projectionStore.getThreadRecords(threadId, [
-                  "runs",
-                  "messages",
-                ]);
-                const terminalDeliveryRunIds = projection.runs
-                  .filter((run) => delegatedTaskTerminalStatus(run.status) !== null)
-                  .filter((run) =>
-                    projection.messages.some(
-                      (message) =>
-                        message.id === run.userMessageId &&
-                        message.delegatedCompletion !== undefined,
-                    ),
-                  )
-                  .map((run) => run.id);
-                for (const runId of terminalDeliveryRunIds) {
-                  yield* finalizeDelegatedCompletionDelivery(threadId, runId);
-                }
-                const refreshed =
-                  terminalDeliveryRunIds.length === 0
-                    ? projection
-                    : yield* projectionStore.getThreadRecords(threadId, ["runs", "messages"], {
-                        messageRoles: ["user"],
-                      });
-                for (const run of refreshed.runs) {
-                  if (
-                    run.delegatedCompletion?.delivery !== null &&
-                    run.delegatedCompletion !== undefined
-                  ) {
-                    yield* offerDelegatedCompletionDelivery(threadId, run.id);
-                  }
-                }
-              }),
-            )
-            .pipe(
               Effect.catchCause((cause) =>
-                Effect.logWarning("Failed to recover delegated completion delivery", {
+                Effect.logWarning("Failed to recover terminal app-owned subagent", {
-                  threadId,
+                  childThreadId: threadId,
                   cause,
                 }),
               ),
             ),
-        { concurrency: 8, discard: true },
+          { concurrency: 8, discard: true },
+        ),
       ),
-    ),
+      Effect.catchCause((cause) =>
-    Effect.catchCause((cause) =>
+        Effect.logWarning("Failed to inspect app-owned subagents during recovery", {
-      Effect.logWarning("Failed to inspect delegated completion delivery during recovery", {
+          cause,
-        cause,
+        }),
-      }),
+      ),
-    ),
+    );
-  );
+    yield* projectionStore.getRecoveryThreadIds("delegated-completions").pipe(
+      Effect.flatMap((threadIds) =>
+        Effect.forEach(
+          threadIds,
+          (threadId) =>
+            threadDispatch
+              .withLock(
+                threadId,
+                Effect.gen(function* () {
+                  const projection = yield* projectionStore.getThreadRecords(threadId, [
+                    "runs",
+                    "messages",
+                  ]);
+                  const terminalDeliveryRunIds = projection.runs
+                    .filter((run) => delegatedTaskTerminalStatus(run.status) !== null)
+                    .filter((run) =>
+                      projection.messages.some(
+                        (message) =>
+                          message.id === run.userMessageId &&
+                          message.delegatedCompletion !== undefined,
+                      ),
+                    )
+                    .map((run) => run.id);
+                  for (const runId of terminalDeliveryRunIds) {
+                    yield* finalizeDelegatedCompletionDelivery(threadId, runId);
+                  }
+                  const refreshed =
+                    terminalDeliveryRunIds.length === 0
+                      ? projection
+                      : yield* projectionStore.getThreadRecords(threadId, ["runs", "messages"], {
+                          messageRoles: ["user"],
+                        });
+                  for (const run of refreshed.runs) {
+                    if (
+                      run.delegatedCompletion?.delivery !== null &&
+                      run.delegatedCompletion !== undefined
+                    ) {
+                      yield* offerDelegatedCompletionDelivery(threadId, run.id);
+                    }
+                  }
+                }),
+              )
+              .pipe(
+                Effect.catchCause((cause) =>
+                  Effect.logWarning("Failed to recover delegated completion delivery", {
+                    threadId,
+                    cause,
+                  }),
+                ),
+              ),
+          { concurrency: 8, discard: true },
+        ),
+      ),
+      Effect.catchCause((cause) =>
+        Effect.logWarning("Failed to inspect delegated completion delivery during recovery", {
+          cause,
+        }),
+      ),
+    );
+  });
+  const delegatedTaskResultPending = (childThreadId: ThreadId) =>
+    Effect.gen(function* () {
+      const child = yield* projectionStore.getThreadRecords(
+        childThreadId,
+        ["runs", "messages", "subagents", "providerThreads", "providerTurns", "attempts"],
+        { messageRoles: ["user"] },
+      );
+      const progress = delegatedTaskProgress(child);
+      // A caller's older read saw a result; newer work since then means it is not final.
+      if (progress.state !== "result_available") return true;
+      if (progress.resultRun === undefined) return false;
+      return yield* childAwaitsRestartContinuation(child.runs, progress.resultRun);
+    }).pipe(
+      Effect.mapError(
+        (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
+      ),
+    );
+  const recoverDelegatedTask = (threadId: ThreadId, sourceRunId: RunId) =>
+    Effect.gen(function* () {
+      const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
+      if (parentThreadId === undefined) return;
+      yield* threadDispatch.withLock(
+        parentThreadId,
+        finalizeAppOwnedSubagent(threadId, { settledContinuationOf: sourceRunId }),
+      );
+    }).pipe(
+      Effect.catchCause((cause) =>
+        Effect.logWarning("Failed to recover delegated task after restart", {
+          childThreadId: threadId,
+          cause,
+        }),
+      ),
+    );
   return OrchestratorV2.of({
     resumeQueuedRuns,
+    recoverDelegatedTasks,
+    recoverDelegatedTask,
+    delegatedTaskResultPending,
     dispatch: dispatchWithReceipt,
     getTimelinePage: (threadId, options) =>
       projectionStore
 @@ -9933,6 +10044,7 @@ export const layer: Layer.Layer<
   | CommandPolicyV2
   | CommandReceiptStoreV2
   | ContextHandoffServiceV2
+  | EffectOutbox.EffectOutboxV2
   | EventSinkV2
   | IdAllocatorV2
   | ProjectStore.ProjectStoreV2
 @@ -9956,6 +10068,9 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
         cause: "Orchestration V2 live runtime is not configured.",
       }),
     ),
+    recoverDelegatedTasks: Effect.void,
+    recoverDelegatedTask: () => Effect.void,
+    delegatedTaskResultPending: () => Effect.succeed(false),
     dispatch: (command) =>
       Effect.fail(
         new OrchestratorDispatchError({

apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+67

...	@@ -420,6 +420,73 @@ it.effect("includes shared sessions and provider-owned background rosters in rec	...	@@ -420,6 +420,73 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
420	}).pipe(Effect.provide(TestLayer)),	420	}).pipe(Effect.provide(TestLayer)),
421	);	421	);
422		422
		423	it.effect("reads the run that owns a background roster, not a queued or resumed one", () =>
		424	Effect.gen(function* () {
		425	const projections = yield* ProjectionStore.ProjectionStoreV2;
		426	const now = yield* DateTime.now;
		427	const withRoster = Effect.fn(function* (threadId: ThreadId) {
		428	yield* projections.apply({
		429	id: EventId.make(`event:${threadId}:roster`),
		430	type: "provider-thread.updated",
		431	threadId,
		432	driver,
		433	providerInstanceId,
		434	occurredAt: now,
		435	payload: {
		436	id: ProviderThreadId.make(`provider-thread:${threadId}`),
		437	appThreadId: threadId,
		438	ownerNodeId: null,
		439	driver,
		440	providerInstanceId,
		441	providerSessionId: null,
		442	nativeThreadRef: null,
		443	nativeConversationHeadRef: null,
		444	status: "idle",
		445	firstRunOrdinal: null,
		446	lastRunOrdinal: null,
		447	handoffIds: [],
		448	forkedFrom: null,
		449	createdAt: now,
		450	updatedAt: now,
		451	pendingBackgroundTasks: [
		452	{ taskId: "background", description: "Still running", kind: "command" },
		453	],
		454	},
		455	});
		456	});
		457	const runIds = (threadId: ThreadId) =>
		458	projections
		459	.getRuntimeRecoveryProjection(threadId)
		460	.pipe(Effect.map((state) => state.runs.map((run) => run.id)));
		461	// Settled with background work left, then a queued follow-up.
		462	const settled = yield* createThread("roster-before-queue");
		463	const settledRun = yield* createRun(settled, "completed", {
		464	providerThreadId: ProviderThreadId.make(`provider-thread:${settled}`),
		465	});
		466	const queuedRun = yield* createRun(settled, "queued", { ordinal: 2, completedAt: null });
		467	yield* withRoster(settled);
		468	assert.deepEqual(yield* runIds(settled), [settledRun, queuedRun]);
		469	// A resumed queued run (ordinal 1) ended after a continuation (ordinal 2).
		470	const resumed = yield* createThread("roster-after-resume");
		471	const resumedRun = yield* createRun(resumed, "completed", {
		472	providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
		473	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
		474	});
		475	const continuationRun = yield* createRun(resumed, "completed", {
		476	providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
		477	ordinal: 2,
		478	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
		479	});
		480	yield* withRoster(resumed);
		481	assert.deepEqual(yield* runIds(resumed), [resumedRun, continuationRun]);
		482	// Without a roster, settled history is not read.
		483	const quiet = yield* createThread("no-roster-before-queue");
		484	yield* createRun(quiet, "completed");
		485	const quietQueued = yield* createRun(quiet, "queued", { ordinal: 2, completedAt: null });
		486	assert.deepEqual(yield* runIds(quiet), [quietQueued]);
		487	}).pipe(Effect.provide(TestLayer)),
		488	);
		489
423	it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>	490	it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>
424	Effect.gen(function* () {	491	Effect.gen(function* () {
425	const projections = yield* ProjectionStore.ProjectionStoreV2;	492	const projections = yield* ProjectionStore.ProjectionStoreV2;

apps/server/src/orchestration-v2/ProjectionStore.test.ts+55

...	@@ -2276,8 +2276,33 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {	...	@@ -2276,8 +2276,33 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {
2276	},	2276	},
2277	});	2277	});
2278	yield* assertSummary("Plan limit reached.", "usage_limit");	2278	yield* assertSummary("Plan limit reached.", "usage_limit");
		2279	// A restart continuation ran ahead of the held queue and ended before the
		2280	// resumed failed run: the failure is still the latest executed run.
		2281	const aheadRunId = RunId.make("run:limit-shell:ran-ahead");
		2282	yield* store.apply({
		2283	id: EventId.make("event:limit-shell:ran-ahead"),
		2284	type: "run.created",
		2285	threadId,
		2286	runId: aheadRunId,
		2287	nodeId: NodeId.make("node:limit-shell:ran-ahead"),
		2288	driver,
		2289	providerInstanceId,
		2290	occurredAt: now,
		2291	payload: {
		2292	...original,
		2293	id: aheadRunId,
		2294	ordinal: original.ordinal + 3,
		2295	rootNodeId: NodeId.make("node:limit-shell:ran-ahead"),
		2296	userMessageId: MessageId.make("message:limit-shell:ran-ahead"),
		2297	status: "completed",
		2298	startedAt: DateTime.subtract(now, { minutes: 10 }),
		2299	completedAt: DateTime.subtract(now, { minutes: 5 }),
		2300	},
		2301	});
		2302	yield* assertSummary("Plan limit reached.", "usage_limit");
2279	const sql = yield* SqlClient.SqlClient;	2303	const sql = yield* SqlClient.SqlClient;
2280	// The rest of this case treats the failed run as the latest run.	2304	// The rest of this case treats the failed run as the latest run.
		2305	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`;
2281	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`;	2306	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`;
2282	yield* assertSummary("Plan limit reached.", "usage_limit");	2307	yield* assertSummary("Plan limit reached.", "usage_limit");
2283	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${cancelledRunId}`;	2308	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${cancelledRunId}`;
...	@@ -4372,6 +4397,36 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {	...	@@ -4372,6 +4397,36 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {
4372	released && projectThreadAwarenessV2({ environmentId, project, thread: released });	4397	released && projectThreadAwarenessV2({ environmentId, project, thread: released });
4373	assert.equal(state?.phase, "completed");	4398	assert.equal(state?.phase, "completed");
4374	assert.equal(state?.updatedAt, DateTime.formatIso(releasedAt));	4399	assert.equal(state?.updatedAt, DateTime.formatIso(releasedAt));
		4400
		4401	// A later provider must not hide this owner's roster from restart recovery.
		4402	yield* store.apply({
		4403	id: EventId.make("event:held-completion:monitor-restarted"),
		4404	type: "provider-thread.updated",
		4405	threadId,
		4406	occurredAt: releasedAt,
		4407	payload: { ...providerThread, updatedAt: releasedAt },
		4408	});
		4409	const newerRunId = RunId.make("run:held-completion:new-provider");
		4410	yield* store.apply({
		4411	id: EventId.make("event:held-completion:new-provider"),
		4412	type: "run.created",
		4413	threadId,
		4414	runId: newerRunId,
		4415	occurredAt: releasedAt,
		4416	payload: {
		4417	...(yield* store.getThreadProjection(threadId)).runs[0]!,
		4418	id: newerRunId,
		4419	ordinal: 2,
		4420	providerThreadId: ProviderThreadId.make("provider-thread:held-completion:new-provider"),
		4421	requestedAt: releasedAt,
		4422	startedAt: releasedAt,
		4423	completedAt: releasedAt,
		4424	},
		4425	});
		4426	assert.deepEqual(
		4427	(yield* store.getRuntimeRecoveryProjection(threadId)).runs.map((run) => run.id),
		4428	[runId, newerRunId],
		4429	);
4375	}),	4430	}),
4376	);	4431	);
4377	});	4432	});

apps/server/src/orchestration-v2/ProjectionStore.ts+39-5

...	@@ -522,8 +522,10 @@ function needsRecovery(	...	@@ -522,8 +522,10 @@ function needsRecovery(
522	}	522	}
523	case "runtime":	523	case "runtime":
524	return (	524	return (
525	projection.runs.some((run) =>	525	projection.runs.some(
526	["queued", "preparing", "starting", "running", "waiting"].includes(run.status),	526	(run) =>
		527	["preparing", "starting", "running", "waiting"].includes(run.status) \|\|
		528	(run.status === "queued" && run.queueHeld !== true),
527	) \|\|	529	) \|\|
528	projection.runtimeRequests.some((request) => request.status === "pending") \|\|	530	projection.runtimeRequests.some((request) => request.status === "pending") \|\|
529	projection.providerSessions.some(	531	projection.providerSessions.some(
...	@@ -3307,7 +3309,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =	...	@@ -3307,7 +3309,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3307	latest.status = 'cancelled'	3309	latest.status = 'cancelled'
3308	AND json_extract(latest.payload_json, '$.startedAt') IS NULL	3310	AND json_extract(latest.payload_json, '$.startedAt') IS NULL
3309	)	3311	)
3310	ORDER BY latest.ordinal DESC, latest.run_id DESC LIMIT 1	3312	-- latestExecutedRun: the run that ended last (runRanAfter).
		3313	ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC,
		3314	latest.ordinal DESC, latest.run_id DESC
		3315	LIMIT 1
3311	) AND r.status = 'failed'	3316	) AND r.status = 'failed'
3312	INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = (	3317	INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = (
3313	SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error	3318	SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error
...	@@ -3433,7 +3438,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =	...	@@ -3433,7 +3438,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3433	ELSE 0 END	3438	ELSE 0 END
3434	)	3439	)
3435	SELECT thread_id FROM orchestration_v2_projection_runs	3440	SELECT thread_id FROM orchestration_v2_projection_runs
3436	WHERE status IN ('queued', 'preparing', 'starting', 'running', 'waiting')	3441	WHERE status IN ('preparing', 'starting', 'running', 'waiting')
		3442	UNION
		3443	-- A held queue already went through recovery; rereading it on
		3444	-- every boot costs a projection read per held thread.
		3445	SELECT thread_id FROM orchestration_v2_projection_runs
		3446	WHERE status = 'queued'
		3447	AND CASE WHEN json_valid(payload_json)
		3448	THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
		3449	ELSE 1 END
3437	UNION	3450	UNION
3438	SELECT thread_id FROM orchestration_v2_projection_runtime_requests	3451	SELECT thread_id FROM orchestration_v2_projection_runtime_requests
3439	WHERE status = 'pending'	3452	WHERE status = 'pending'
...	@@ -3805,6 +3818,24 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =	...	@@ -3805,6 +3818,24 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3805	WHERE latest.thread_id = ${threadId}	3818	WHERE latest.thread_id = ${threadId}
3806	ORDER BY latest.ordinal DESC LIMIT 1	3819	ORDER BY latest.ordinal DESC LIMIT 1
3807	)	3820	)
		3821	-- Retain the latest run for each provider thread with lost
		3822	-- background work, including owners used before a handoff.
		3823	OR run.run_id IN (
		3824	SELECT (
		3825	SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended
		3826	WHERE ended.thread_id = ${threadId}
		3827	AND ended.provider_thread_id = roster.provider_thread_id
		3828	AND ended.status NOT IN ('queued', 'rolled_back')
		3829	ORDER BY ended.completed_at IS NULL DESC, ended.completed_at DESC,
		3830	ended.ordinal DESC
		3831	LIMIT 1
		3832	)
		3833	FROM orchestration_v2_projection_provider_threads AS roster
		3834	WHERE roster.thread_id = ${threadId}
		3835	AND CASE WHEN json_valid(roster.payload_json)
		3836	THEN json_array_length(roster.payload_json, '$.pendingBackgroundTasks') > 0
		3837	ELSE 0 END
		3838	)
3808	OR run.run_id IN (	3839	OR run.run_id IN (
3809	SELECT item.run_id FROM orchestration_v2_projection_turn_items AS item	3840	SELECT item.run_id FROM orchestration_v2_projection_turn_items AS item
3810	WHERE item.thread_id = ${threadId}	3841	WHERE item.thread_id = ${threadId}
...	@@ -4899,7 +4930,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =	...	@@ -4899,7 +4930,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4899	candidate.status = 'cancelled'	4930	candidate.status = 'cancelled'
4900	AND json_extract(candidate.payload_json, '$.startedAt') IS NULL	4931	AND json_extract(candidate.payload_json, '$.startedAt') IS NULL
4901	)	4932	)
4902	ORDER BY candidate.ordinal DESC, candidate.run_id DESC LIMIT 1	4933	-- latestExecutedRun: the run that ended last (runRanAfter).
		4934	ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC,
		4935	candidate.ordinal DESC, candidate.run_id DESC
		4936	LIMIT 1
4903	) AND blocked.status = 'failed'	4937	) AND blocked.status = 'failed'
4904	WHERE t.deleted_at IS NULL${threadId === undefined ? sql`` : sql` AND t.thread_id = ${threadId}`}${	4938	WHERE t.deleted_at IS NULL${threadId === undefined ? sql`` : sql` AND t.thread_id = ${threadId}`}${
4905	location === "active"	4939	location === "active"

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryPerformance.test.ts created+510

...	@@ -0,0 +1,510 @@
1	import { assert, it } from "@effect/vitest";
2	import {
3	CheckpointScopeId,
4	CommandId,
5	EventId,
6	MessageId,
7	NodeId,
8	type OrchestrationV2AppThread,
9	type OrchestrationV2DomainEvent,
10	type OrchestrationV2Run,
11	ProjectId,
12	ProviderDriverKind,
13	ProviderInstanceId,
14	ProviderSessionId,
15	ProviderThreadId,
16	ProviderTurnId,
17	RunAttemptId,
18	RunId,
19	ThreadId,
20	TurnItemId,
21	} from "@t3tools/contracts";
22	import * as Console from "effect/Console";
23	import * as DateTime from "effect/DateTime";
24	import * as Effect from "effect/Effect";
25	import * as Layer from "effect/Layer";
26	import * as SqlClient from "effect/unstable/sql/SqlClient";
27
28	import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts";
29	import * as ServerSettings from "../serverSettings.ts";
30	import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
31	import * as EffectOutbox from "./EffectOutbox.ts";
32	import * as EventSink from "./EventSink.ts";
33	import * as EventStore from "./EventStore.ts";
34	import * as IdAllocator from "./IdAllocator.ts";
35	import * as ProjectionStore from "./ProjectionStore.ts";
36	import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
37
38	// Startup recovery cost for restart continuation. The default case is small
39	// enough for CI and asserts correctness; T3_BENCH_RECOVERY=1 adds the matrix.
40	// `it.live` keeps a real clock: each reconcile gets a fresh command id, so the
41	// second recover cannot hide behind command receipt dedup.
42
43	const stores = Layer.mergeAll(
44	EventStore.layer,
45	ProjectionStore.layer,
46	EffectOutbox.layer,
47	IdAllocator.layer,
48	).pipe(Layer.provideMerge(SqlitePersistenceMemory));
49	const TestLayer = ProviderRuntimeRecovery.layer.pipe(
50	Layer.provideMerge(EventSink.layer.pipe(Layer.provideMerge(stores))),
51	Layer.provideMerge(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })),
52	);
53
54	const providerInstanceId = ProviderInstanceId.make("codex");
55	const driver = ProviderDriverKind.make("codex");
56	const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
57	const projectId = ProjectId.make("project:recovery-bench");
58	const ITEMS_PER_SETTLED_THREAD = 10;
59	const OPEN_ITEMS_PER_ACTIVE_THREAD = 3;
60	const WAITING_THREADS = 3;
61	const SEED_BATCH = 500;
62
63	type UnstampedEvent = OrchestrationV2DomainEvent extends infer Event
64	? Event extends OrchestrationV2DomainEvent
65	? Omit<Event, "id" \| "occurredAt">
66	: never
67	: never;
68
69	interface Scenario {
70	readonly settled: number;
71	readonly active: number;
72	}
73
74	interface Seeded {
75	readonly unheldQueuedThreads: number;
76	readonly waitingThreads: number;
77	}
78
79	const seedScenario = Effect.fn(function* (scenario: Scenario) {
80	const projections = yield* ProjectionStore.ProjectionStoreV2;
81	const outbox = yield* EffectOutbox.EffectOutboxV2;
82	const sql = yield* SqlClient.SqlClient;
83	const now = yield* DateTime.now;
84	let eventIndex = 0;
85	const apply = (event: UnstampedEvent) =>
86	projections.apply({
87	...event,
88	id: EventId.make(`event:bench:${eventIndex++}`),
89	occurredAt: now,
90	} as OrchestrationV2DomainEvent);
91
92	const thread = (id: ThreadId, overrides: Partial<OrchestrationV2AppThread> = {}) =>
93	apply({
94	type: "thread.created",
95	threadId: id,
96	payload: {
97	createdBy: "user",
98	creationSource: "web",
99	id,
100	projectId,
101	title: id,
102	providerInstanceId,
103	modelSelection,
104	runtimeMode: "full-access",
105	interactionMode: "default",
106	branch: null,
107	worktreePath: null,
108	activeProviderThreadId: null,
109	lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: id },
110	forkedFrom: null,
111	createdAt: now,
112	updatedAt: now,
113	archivedAt: null,
114	settledOverride: null,
115	settledAt: null,
116	lastVisitedAt: null,
117	deletedAt: null,
118	...overrides,
119	},
120	});
121
122	const run = (
123	threadId: ThreadId,
124	ordinal: number,
125	status: OrchestrationV2Run["status"],
126	overrides: Partial<OrchestrationV2Run> = {},
127	) => {
128	const runId = RunId.make(`run:${threadId}:${ordinal}`);
129	return apply({
130	type: "run.created",
131	threadId,
132	runId,
133	payload: {
134	id: runId,
135	threadId,
136	ordinal,
137	providerInstanceId,
138	modelSelection,
139	providerThreadId: null,
140	userMessageId: MessageId.make(`message:${runId}`),
141	rootNodeId: null,
142	activeAttemptId: null,
143	status,
144	requestedAt: now,
145	startedAt: status === "queued" ? null : now,
146	completedAt: status === "completed" ? now : null,
147	checkpointId: null,
148	contextHandoffId: null,
149	...overrides,
150	},
151	}).pipe(Effect.as(runId));
152	};
153
154	const commandItem = (input: {
155	readonly threadId: ThreadId;
156	readonly runId: RunId;
157	readonly ordinal: number;
158	readonly status: "running" \| "completed";
159	readonly nodeId?: NodeId;
160	readonly providerThreadId?: ProviderThreadId;
161	readonly providerTurnId?: ProviderTurnId;
162	}) =>
163	apply({
164	type: "turn-item.updated",
165	threadId: input.threadId,
166	runId: input.runId,
167	payload: {
168	id: TurnItemId.make(`item:${input.runId}:${input.ordinal}`),
169	threadId: input.threadId,
170	runId: input.runId,
171	nodeId: input.nodeId ?? null,
172	providerThreadId: input.providerThreadId ?? null,
173	providerTurnId: input.providerTurnId ?? null,
174	nativeItemRef: null,
175	parentItemId: null,
176	ordinal: input.ordinal,
177	type: "command_execution",
178	status: input.status,
179	title: `Command ${input.ordinal}`,
180	input: `echo ${input.ordinal}`,
181	output: input.status === "completed" ? `${input.ordinal}\n` : "",
182	...(input.status === "completed" ? { exitCode: 0 } : {}),
183	startedAt: now,
184	completedAt: input.status === "completed" ? now : null,
185	updatedAt: now,
186	},
187	});
188
189	const settledThread = Effect.fn(function* (index: number) {
190	const threadId = ThreadId.make(`thread:bench:settled:${index}`);
191	yield* thread(threadId);
192	const runId = yield* run(threadId, 1, "completed");
193	for (let ordinal = 1; ordinal <= ITEMS_PER_SETTLED_THREAD; ordinal += 1) {
194	yield* commandItem({ threadId, runId, ordinal, status: "completed" });
195	}
196	// ~5% of threads carry a queued follow-up; half are already held, and an
197	// earlier recovery already handled those, so only unheld ones are candidates.
198	if (index % 20 === 0) {
199	const queueHeld = index % 40 === 0;
200	yield* run(threadId, 2, "queued", { queueHeld });
201	return !queueHeld;
202	}
203	return false;
204	});
205
206	const activeThread = Effect.fn(function* (index: number) {
207	const threadId = ThreadId.make(`thread:bench:active:${index}`);
208	const providerSessionId = ProviderSessionId.make(`session:bench:${index}`);
209	const providerThreadId = ProviderThreadId.make(`provider-thread:bench:${index}`);
210	const runId = RunId.make(`run:${threadId}:1`);
211	const attemptId = RunAttemptId.make(`attempt:bench:${index}`);
212	const nodeId = NodeId.make(`node:bench:${index}`);
213	const providerTurnId = ProviderTurnId.make(`provider-turn:bench:${index}`);
214	yield* thread(threadId, { activeProviderThreadId: providerThreadId });
215	yield* apply({
216	type: "provider-session.attached",
217	threadId,
218	driver,
219	providerInstanceId,
220	payload: {
221	id: providerSessionId,
222	driver,
223	providerInstanceId,
224	status: "ready",
225	cwd: "/workspace",
226	model: modelSelection.model,
227	capabilities: CodexProviderCapabilitiesV2,
228	createdAt: now,
229	updatedAt: now,
230	lastError: null,
231	},
232	});
233	yield* apply({
234	type: "provider-thread.updated",
235	threadId,
236	driver,
237	providerInstanceId,
238	payload: {
239	id: providerThreadId,
240	appThreadId: threadId,
241	ownerNodeId: null,
242	driver,
243	providerInstanceId,
244	providerSessionId,
245	nativeThreadRef: { driver, nativeId: `native:bench:${index}`, strength: "strong" },
246	nativeConversationHeadRef: null,
247	status: "active",
248	firstRunOrdinal: 1,
249	lastRunOrdinal: 1,
250	handoffIds: [],
251	forkedFrom: null,
252	createdAt: now,
253	updatedAt: now,
254	pendingBackgroundTasks: [],
255	},
256	});
257	yield* run(threadId, 1, "running", {
258	providerThreadId,
259	rootNodeId: nodeId,
260	activeAttemptId: attemptId,
261	});
262	yield* apply({
263	type: "run-attempt.created",
264	threadId,
265	runId,
266	payload: {
267	id: attemptId,
268	runId,
269	attemptOrdinal: 1,
270	rootNodeId: nodeId,
271	providerInstanceId,
272	providerThreadId,
273	providerTurnId,
274	reason: "initial",
275	status: "running",
276	startedAt: now,
277	completedAt: null,
278	},
279	});
280	yield* apply({
281	type: "node.updated",
282	threadId,
283	runId,
284	nodeId,
285	providerInstanceId,
286	payload: {
287	id: nodeId,
288	threadId,
289	runId,
290	parentNodeId: null,
291	rootNodeId: nodeId,
292	kind: "root_turn",
293	status: "running",
294	countsForRun: true,
295	providerThreadId,
296	providerTurnId,
297	nativeItemRef: null,
298	runtimeRequestId: null,
299	checkpointScopeId: null,
300	startedAt: now,
301	completedAt: null,
302	},
303	});
304	yield* apply({
305	type: "provider-turn.updated",
306	threadId,
307	runId,
308	nodeId,
309	providerInstanceId,
310	payload: {
311	id: providerTurnId,
312	providerThreadId,
313	nodeId,
314	runAttemptId: attemptId,
315	nativeTurnRef: { driver, nativeId: `native-turn:bench:${index}`, strength: "strong" },
316	ordinal: 1,
317	status: "running",
318	startedAt: now,
319	completedAt: null,
320	},
321	});
322	for (let ordinal = 1; ordinal <= OPEN_ITEMS_PER_ACTIVE_THREAD; ordinal += 1) {
323	yield* commandItem({
324	threadId,
325	runId,
326	ordinal,
327	status: "running",
328	nodeId,
329	providerThreadId,
330	providerTurnId,
331	});
332	}
333	});
334
335	const waitingThread = Effect.fn(function* (index: number) {
336	const threadId = ThreadId.make(`thread:bench:waiting:${index}`);
337	yield* thread(threadId);
338	yield* run(threadId, 1, "completed");
339	const runId = yield* run(threadId, 2, "waiting");
340	yield* outbox.enqueue([
341	{
342	id: `effect:bench:checkpoint:${runId}`,
343	commandId: CommandId.make(`command:effect:checkpoint.capture:${runId}`),
344	threadId,
345	request: {
346	type: "checkpoint.capture",
347	runId,
348	scopeId: CheckpointScopeId.make(`scope:bench:${index}`),
349	},
350	},
351	]);
352	});
353
354	let unheldQueuedThreads = 0;
355	for (let start = 0; start < scenario.settled; start += SEED_BATCH) {
356	const end = Math.min(scenario.settled, start + SEED_BATCH);
357	yield* sql.withTransaction(
358	Effect.gen(function* () {
359	for (let index = start; index < end; index += 1) {
360	if (yield* settledThread(index)) unheldQueuedThreads += 1;
361	}
362	}),
363	);
364	}
365	yield* sql.withTransaction(
366	Effect.gen(function* () {
367	for (let index = 0; index < scenario.active; index += 1) yield* activeThread(index);
368	for (let index = 0; index < WAITING_THREADS; index += 1) yield* waitingThread(index);
369	}),
370	);
371	return { unheldQueuedThreads, waitingThreads: WAITING_THREADS } satisfies Seeded;
372	});
373
374	const timed = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
375	Effect.gen(function* () {
376	const start = performance.now();
377	const value = yield* effect;
378	return [value, performance.now() - start] as const;
379	});
380
381	const continuationEffectCount = Effect.gen(function* () {
382	const sql = yield* SqlClient.SqlClient;
383	const rows = yield* sql<{ readonly count: number }>`
384	SELECT COUNT(*) AS count FROM orchestration_v2_effect_outbox
385	WHERE effect_id LIKE 'effect:restart-continuation:%' AND status = 'pending'
386	`;
387	return Number(rows[0]?.count ?? 0);
388	});
389
390	interface Measurement {
391	readonly settled: number;
392	readonly active: number;
393	readonly seedMs: number;
394	readonly candidates: number;
395	readonly selectMs: number;
396	readonly prepareMs: number;
397	readonly shutdownMs: number;
398	readonly startupAfterShutdownMs: number;
399	readonly crashRecoverMs: number;
400	readonly secondRecoverMs: number;
401	}
402
403	/** A graceful restart: prepare intent, reconcile on shutdown, recover on boot. */
404	const measureGraceful = Effect.fn(function* (scenario: Scenario) {
405	const [seeded, seedMs] = yield* timed(seedScenario(scenario));
406	const projections = yield* ProjectionStore.ProjectionStoreV2;
407	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
408	const [candidates, selectMs] = yield* timed(projections.getRecoveryThreadIds("runtime"));
409	assert.equal(
410	candidates.length,
411	scenario.active + seeded.unheldQueuedThreads + seeded.waitingThreads,
412	"runtime recovery candidates",
413	);
414	const [, prepareMs] = yield* timed(recovery.prepareForShutdown);
415	assert.equal(yield* continuationEffectCount, scenario.active, "prepared continuations");
416	const [, shutdownMs] = yield* timed(recovery.reconcile("shutdown"));
417	const [, startupAfterShutdownMs] = yield* timed(recovery.recover);
418	assert.equal(yield* continuationEffectCount, scenario.active, "continuations after boot");
419	return {
420	seedMs,
421	candidates: candidates.length,
422	selectMs,
423	prepareMs,
424	shutdownMs,
425	startupAfterShutdownMs,
426	};
427	});
428
429	/** A crash: no shutdown hook ran, so boot recovery records the continuations. */
430	const measureCrash = Effect.fn(function* (scenario: Scenario) {
431	yield* seedScenario(scenario);
432	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
433	const [summary, crashRecoverMs] = yield* timed(recovery.recover);
434	assert.equal(summary.terminalizedRuns, scenario.active, "terminalized active runs");
435	assert.equal(summary.stoppedSessions, scenario.active, "stopped active sessions");
436	assert.equal(yield* continuationEffectCount, scenario.active, "recorded continuations");
437	const [, secondRecoverMs] = yield* timed(recovery.recover);
438	assert.equal(yield* continuationEffectCount, scenario.active, "second recover is idempotent");
439	return { crashRecoverMs, secondRecoverMs };
440	});
441
442	const measure = Effect.fn(function* (scenario: Scenario) {
443	const graceful = yield* measureGraceful(scenario).pipe(Effect.provide(Layer.fresh(TestLayer)));
444	const crash = yield* measureCrash(scenario).pipe(Effect.provide(Layer.fresh(TestLayer)));
445	return { ...scenario, ...graceful, ...crash } satisfies Measurement;
446	});
447
448	const formatTable = (rows: ReadonlyArray<Measurement>) => {
449	const ms = (value: number) => value.toFixed(1);
450	const perActive = (value: number, active: number) => (value / Math.max(active, 1)).toFixed(2);
451	const header = [
452	"N",
453	"A",
454	"seed",
455	"cand",
456	"select",
457	"prepare",
458	"prep/A",
459	"shutdown",
460	"boot-after",
461	"crash-recover",
462	"recover/A",
463	"recover-2",
464	];
465	const lines = rows.map((row) => [
466	String(row.settled),
467	String(row.active),
468	ms(row.seedMs),
469	String(row.candidates),
470	ms(row.selectMs),
471	ms(row.prepareMs),
472	perActive(row.prepareMs, row.active),
473	ms(row.shutdownMs),
474	ms(row.startupAfterShutdownMs),
475	ms(row.crashRecoverMs),
476	perActive(row.crashRecoverMs, row.active),
477	ms(row.secondRecoverMs),
478	]);
479	const widths = header.map((cell, column) =>
480	Math.max(cell.length, ...lines.map((line) => line[column]!.length)),
481	);
482	return [header, ...lines]
483	.map((line) => line.map((cell, column) => cell.padStart(widths[column]!)).join(" "))
484	.join("\n");
485	};
486
487	it.live(
488	"restart recovery records one continuation per active thread and is idempotent",
489	() =>
490	Effect.gen(function* () {
491	const row = yield* measure({ settled: 500, active: 20 });
492	yield* Console.log(`provider runtime recovery (ms)\n${formatTable([row])}`);
493	}),
494	60_000,
495	);
496
497	it.live.skipIf(process.env.T3_BENCH_RECOVERY !== "1")(
498	"restart recovery scales with active threads, not settled history",
499	() =>
500	Effect.gen(function* () {
501	const rows: Array<Measurement> = [];
502	for (const settled of [5_000, 50_000]) {
503	for (const active of [10, 100, 1_000]) {
504	rows.push(yield* measure({ settled, active }));
505	yield* Console.log(`provider runtime recovery (ms)\n${formatTable(rows)}`);
506	}
507	}
508	}),
509	3_600_000,
510	);

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts+148

...	@@ -1272,3 +1272,151 @@ it.effect(	...	@@ -1272,3 +1272,151 @@ it.effect(
1272	}).pipe(Effect.provide(layer));	1272	}).pipe(Effect.provide(layer));
1273	},	1273	},
1274	);	1274	);
		1275
		1276	it.effect("leaves delegated tasks to their own child threads after process loss", () => {
		1277	const threadId = ThreadId.make("thread_recovery_delegation");
		1278	const settledRunId = RunId.make("run_recovery_delegation_settled");
		1279	const runningRunId = RunId.make("run_recovery_delegation_running");
		1280	const providerThreadId = ProviderThreadId.make("provider_thread_recovery_delegation");
		1281	const claudeInstanceId = ProviderInstanceId.make("claude");
		1282	const driver = ProviderDriverKind.make("claude");
		1283	const settledTaskId = NodeId.make("node_recovery_delegation_settled");
		1284	const runningTaskId = NodeId.make("node_recovery_delegation_running");
		1285	const nativeSubagentId = NodeId.make("node_recovery_native_subagent");
		1286	const delegatedTask = (id: NodeId, runId: RunId) => ({
		1287	id,
		1288	runId,
		1289	origin: "app_owned",
		1290	childThreadId: ThreadId.make(`thread:delegated-task:${id}`),
		1291	driver,
		1292	providerInstanceId: claudeInstanceId,
		1293	status: "running",
		1294	});
		1295	const subagentItem = (subagentId: NodeId, runId: RunId, origin: string) => ({
		1296	id: TurnItemId.make(`turn_item:${subagentId}`),
		1297	runId,
		1298	nodeId: subagentId,
		1299	providerThreadId,
		1300	type: "subagent",
		1301	status: "running",
		1302	subagentId,
		1303	origin,
		1304	childThreadId:
		1305	origin === "app_owned" ? ThreadId.make(`thread:delegated-task:${subagentId}`) : null,
		1306	providerInstanceId: claudeInstanceId,
		1307	title: `subagent ${subagentId}`,
		1308	});
		1309	let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] \| null =
		1310	null;
		1311	const projection = {
		1312	thread: { id: threadId, providerInstanceId: claudeInstanceId },
		1313	runtimeRequests: [],
		1314	providerSessions: [],
		1315	providerThreads: [
		1316	{
		1317	id: providerThreadId,
		1318	driver,
		1319	providerInstanceId: claudeInstanceId,
		1320	ownerNodeId: null,
		1321	status: "idle",
		1322	pendingBackgroundTasks: [],
		1323	},
		1324	],
		1325	providerTurns: [],
		1326	runs: [
		1327	{
		1328	id: settledRunId,
		1329	ordinal: 1,
		1330	status: "completed",
		1331	providerThreadId,
		1332	providerInstanceId: claudeInstanceId,
		1333	},
		1334	{
		1335	id: runningRunId,
		1336	ordinal: 2,
		1337	status: "running",
		1338	providerThreadId,
		1339	providerInstanceId: claudeInstanceId,
		1340	},
		1341	],
		1342	attempts: [],
		1343	nodes: [
		1344	{ id: settledTaskId, runId: settledRunId, status: "running", kind: "subagent" },
		1345	{ id: runningTaskId, runId: runningRunId, status: "running", kind: "subagent" },
		1346	{ id: nativeSubagentId, runId: settledRunId, status: "running", kind: "subagent" },
		1347	],
		1348	subagents: [
		1349	delegatedTask(settledTaskId, settledRunId),
		1350	delegatedTask(runningTaskId, runningRunId),
		1351	{
		1352	...delegatedTask(nativeSubagentId, settledRunId),
		1353	origin: "provider_native",
		1354	childThreadId: null,
		1355	},
		1356	],
		1357	messages: [],
		1358	turnItems: [
		1359	subagentItem(settledTaskId, settledRunId, "app_owned"),
		1360	subagentItem(runningTaskId, runningRunId, "app_owned"),
		1361	subagentItem(nativeSubagentId, settledRunId, "provider_native"),
		1362	],
		1363	} as unknown as OrchestrationV2ThreadProjection;
		1364	const layer = ProviderRuntimeRecovery.layer.pipe(
		1365	Layer.provide(ServerSettings.layerTest()),
		1366	Layer.provide(
		1367	Layer.mergeAll(
		1368	Layer.mock(ProjectionStore.ProjectionStoreV2)({
		1369	getRecoveryThreadIds: () => Effect.succeed([threadId]),
		1370	getRuntimeRecoveryProjection: () => Effect.succeed(projection),
		1371	}),
		1372	Layer.mock(EventSink.EventSinkV2)({
		1373	commitCommand: (input) => {
		1374	committedInput = input;
		1375	return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
		1376	},
		1377	}),
		1378	IdAllocator.layer,
		1379	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
		1380	runRecoveryOnce: Effect.succeed(false),
		1381	}),
		1382	Layer.mock(EffectOutbox.EffectOutboxV2)({
		1383	listByCommandId: () => Effect.succeed([]),
		1384	reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
		1385	}),
		1386	),
		1387	),
		1388	);
		1389
		1390	return Effect.gen(function* () {
		1391	yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
		1392	const events = committedInput?.events ?? [];
		1393	const touched = events.flatMap((event) =>
		1394	event.type === "turn-item.updated"
		1395	? event.payload.type === "subagent"
		1396	? [event.payload.subagentId]
		1397	: []
		1398	: event.type === "subagent.updated" \|\| event.type === "node.updated"
		1399	? [event.payload.id]
		1400	: [],
		1401	);
		1402	// The cut run itself is cancelled; only the provider-native subagent dies with it.
		1403	assert.isTrue(
		1404	events.some(
		1405	(event) =>
		1406	event.type === "run.updated" &&
		1407	event.payload.id === runningRunId &&
		1408	event.payload.status === "cancelled",
		1409	),
		1410	);
		1411	assert.notInclude(touched, settledTaskId);
		1412	assert.notInclude(touched, runningTaskId);
		1413	assert.include(touched, nativeSubagentId);
		1414	// The note lists only provider-native work: delegated tasks report back on their own.
		1415	const noted = events.flatMap((event) =>
		1416	event.type === "run.background-work-cancelled"
		1417	? event.payload.restartCancelledBackgroundWork.map((work) => work.label)
		1418	: [],
		1419	);
		1420	assert.deepEqual(noted, [`subagent ${nativeSubagentId}`]);
		1421	}).pipe(Effect.provide(layer));
		1422	});

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+71-28

 @@ -1,12 +1,15 @@
+import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
 import {
   CommandId,
   type OrchestrationV2DomainEvent,
   type ProviderThreadId,
   type OrchestrationV2RestartCancelledBackgroundWork,
+  type OrchestrationV2Subagent,
   type OrchestrationV2ThreadProjection,
   ThreadId,
 } from "@t3tools/contracts";
+import * as Cause from "effect/Cause";
 import * as Context from "effect/Context";
 import * as DateTime from "effect/DateTime";
 import * as Effect from "effect/Effect";
 @@ -93,6 +96,23 @@ function isNonterminalNodeStatus(status: string): boolean {
   return status === "pending" || status === "running" || status === "waiting";
 }
+/**
+ * A delegate_task child. Its own thread is reconciled and continued on its own
+ * and reports back through the app, so it is not provider background work.
+ */
+function isAppOwnedDelegation(task: {
+  readonly origin: OrchestrationV2Subagent["origin"];
+  readonly childThreadId: ThreadId | null;
+}): boolean {
+  return task.origin === "app_owned" && task.childThreadId !== null;
+}
+function isAppOwnedDelegationItem(
+  item: OrchestrationV2ThreadProjection["turnItems"][number],
+): boolean {
+  return item.type === "subagent" && isAppOwnedDelegation(item);
+}
 function providerThreadHasPendingBackgroundTasks(
   providerThread: OrchestrationV2ThreadProjection["providerThreads"][number],
 ): boolean {
 @@ -138,7 +158,11 @@ function providerThreadsWithOpenBackgroundWork(
 ): ReadonlySet<ProviderThreadId> {
   const ids = new Set<ProviderThreadId>();
   for (const item of projection.turnItems ?? []) {
-    if (!isBackgroundCapableTurnItemType(item.type) || !isNonterminalTurnItemStatus(item.status))
+    if (
+      !isBackgroundCapableTurnItemType(item.type) ||
+      !isNonterminalTurnItemStatus(item.status) ||
+      isAppOwnedDelegationItem(item)
+    )
       continue;
     const providerThreadId =
       item.providerThreadId ??
 @@ -166,7 +190,7 @@ function latestStartedRun(
       run.providerThreadId === providerThreadId &&
       run.status !== "queued" &&
       run.status !== "rolled_back" &&
-      (latest === undefined || run.ordinal > latest.ordinal)
+      (latest === undefined || runRanAfter(run, latest))
         ? run
         : latest,
     undefined,
 @@ -222,6 +246,13 @@ export const make = Effect.gen(function* () {
       const requests = projection.runtimeRequests.filter(
         (request) => request.status === "pending" && request.responseCapability.type !== "message",
       );
+      // Delegated task rows, items and nodes stay open: the child settles them.
+      const delegatedTaskNodeIds = new Set<string>([
+        ...(projection.subagents ?? []).filter(isAppOwnedDelegation).map((subagent) => subagent.id),
+        ...(projection.turnItems ?? []).flatMap((item) =>
+          item.type === "subagent" && isAppOwnedDelegation(item) ? [item.subagentId] : [],
+        ),
+      ]);
       const detail = `Cancelled because the server ${trigger === "startup" ? "restarted" : "shut down"} before the provider work completed.`;
       const commandId = CommandId.make(
         `command:runtime-reconcile:${trigger}:${projection.thread.id}:${DateTime.formatIso(now)}`,
 @@ -333,6 +364,7 @@ export const make = Effect.gen(function* () {
           (candidate) =>
             candidate.runId === run.id &&
             !messageRequestNodeIds.has(candidate.id) &&
+            !delegatedTaskNodeIds.has(candidate.id) &&
             (candidate.status === "pending" ||
               candidate.status === "running" ||
               candidate.status === "waiting"),
 @@ -351,6 +383,7 @@ export const make = Effect.gen(function* () {
         for (const subagent of projection.subagents.filter(
           (candidate) =>
             candidate.runId === run.id &&
+            !isAppOwnedDelegation(candidate) &&
             (candidate.status === "pending" ||
               candidate.status === "running" ||
               candidate.status === "waiting"),
 @@ -404,6 +437,7 @@ export const make = Effect.gen(function* () {
           (candidate) =>
             candidate.runId === run.id &&
             (candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
+            !isAppOwnedDelegationItem(candidate) &&
             (candidate.status === "pending" ||
               candidate.status === "running" ||
               candidate.status === "waiting"),
 @@ -436,7 +470,7 @@ export const make = Effect.gen(function* () {
         if (!isBackgroundCapableTurnItemType(item.type)) {
           continue;
         }
-        if (!isNonterminalTurnItemStatus(item.status)) {
+        if (!isNonterminalTurnItemStatus(item.status) || isAppOwnedDelegationItem(item)) {
           continue;
         }
         const providerInstanceId = resolveStaleBackgroundItemProviderInstanceId(item, projection);
 @@ -763,32 +797,41 @@ export const make = Effect.gen(function* () {
     if (!enabled) return;
     const threadIds = yield* projections.getRecoveryThreadIds("runtime");
     for (const threadId of threadIds) {
-      const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
+      yield* Effect.gen(function* () {
-      if (
+        const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
-        !resolveProjectSettings(enabled, projection.thread.projectId).settings
+        if (
-          .continueThreadsAfterServerUpdate
+          !resolveProjectSettings(enabled, projection.thread.projectId).settings
-      )
+            .continueThreadsAfterServerUpdate
-        continue;
+        )
-      // Shutdown reconciliation cancels the background work below, so a
+          return;
-      // settled thread's continuation must be captured while it is still open.
+        // Shutdown reconciliation cancels the background work below, so a
-      const run = restartContinuationRun(
+        // settled thread's continuation must be captured while it is still open.
-        projection,
+        const run = restartContinuationRun(
-        providerThreadsWithOpenBackgroundWork(projection),
+          projection,
+          providerThreadsWithOpenBackgroundWork(projection),
+        );
+        if (!run) return;
+        const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
+        yield* eventSink.writeWithEffects({
+          commandId,
+          events: [],
+          effects: [
+            {
+              id: `effect:restart-continuation:${run.id}`,
+              commandId,
+              threadId,
+              request: { type: "provider-runtime.continue", sourceRunId: run.id },
+            },
+          ],
+        });
+      }).pipe(
+        // One failing thread must not cost the threads after it their continuation.
+        Effect.catchCauseIf(
+          (cause) => !Cause.hasInterruptsOnly(cause),
+          (cause) =>
+            Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
+        ),
       );
-      if (!run) continue;
-      const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
-      yield* eventSink.writeWithEffects({
-        commandId,
-        events: [],
-        effects: [
-          {
-            id: `effect:restart-continuation:${run.id}`,
-            commandId,
-            threadId,
-            request: { type: "provider-runtime.continue", sourceRunId: run.id },
-          },
-        ],
-      });
     }
   }).pipe(
     Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),

apps/server/src/orchestration-v2/ProviderSessionManager.ts+22

...	@@ -362,6 +362,25 @@ export const layerWithOptions = (	...	@@ -362,6 +362,25 @@ export const layerWithOptions = (
362	},	362	},
363	);	363	);
364	const layerScope = yield* Effect.scope;	364	const layerScope = yield* Effect.scope;
		365	// Ctrl+C, or a stop that signals the whole process group, reaches the
		366	// provider CLIs with the server. They report their own background work
		367	// stopped before shutdown captures restart continuations, so provider
		368	// events after the signal are dropped; restart recovery owns that state.
		369	const shutdownSignal = { received: false };
		370	const onShutdownSignal = () => {
		371	shutdownSignal.received = true;
		372	};
		373	yield* Effect.acquireRelease(
		374	Effect.sync(() => {
		375	process.on("SIGINT", onShutdownSignal);
		376	process.on("SIGTERM", onShutdownSignal);
		377	}),
		378	() =>
		379	Effect.sync(() => {
		380	process.off("SIGINT", onShutdownSignal);
		381	process.off("SIGTERM", onShutdownSignal);
		382	}),
		383	);
365	const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());	384	const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
366	// One retry per released entry, so a later release with the same id	385	// One retry per released entry, so a later release with the same id
367	// cannot drop cleanup for threads only the earlier session served.	386	// cannot drop cleanup for threads only the earlier session served.
...	@@ -1542,6 +1561,7 @@ export const layerWithOptions = (	...	@@ -1542,6 +1561,7 @@ export const layerWithOptions = (
1542	let stoppedByProvider = false;	1561	let stoppedByProvider = false;
1543	return entry.runtime.events.pipe(	1562	return entry.runtime.events.pipe(
1544	Stream.runForEach((event) => {	1563	Stream.runForEach((event) => {
		1564	if (shutdownSignal.received) return Effect.void;
1545	if (	1565	if (
1546	event.type === "provider_session.updated" &&	1566	event.type === "provider_session.updated" &&
1547	event.providerSession.status === "stopped"	1567	event.providerSession.status === "stopped"
...	@@ -1600,6 +1620,8 @@ export const layerWithOptions = (	...	@@ -1600,6 +1620,8 @@ export const layerWithOptions = (
1600	Effect.exit,	1620	Effect.exit,
1601	Effect.flatMap((exit) =>	1621	Effect.flatMap((exit) =>
1602	Effect.gen(function* () {	1622	Effect.gen(function* () {
		1623	// A provider that exits on the shutdown signal is released by shutdown.
		1624	if (shutdownSignal.received) return;
1603	const current = (yield* Ref.get(sessions)).get(	1625	const current = (yield* Ref.get(sessions)).get(
1604	sessionKey(entry.runtime.providerSessionId),	1626	sessionKey(entry.runtime.providerSessionId),
1605	);	1627	);

apps/server/src/orchestration-v2/ProviderTurnStartService.ts+2-3

...	@@ -953,6 +953,7 @@ export const layer: Layer.Layer<	...	@@ -953,6 +953,7 @@ export const layer: Layer.Layer<
953	run,	953	run,
954	projection.runs,	954	projection.runs,
955	projection.providerTurns,	955	projection.providerTurns,
		956	projection.attempts,
956	);	957	);
957	const restartCancelledWork = pendingRestartCancelledBackgroundWork({	958	const restartCancelledWork = pendingRestartCancelledBackgroundWork({
958	runs: projection.runs,	959	runs: projection.runs,
...	@@ -967,9 +968,7 @@ export const layer: Layer.Layer<	...	@@ -967,9 +968,7 @@ export const layer: Layer.Layer<
967	.map((candidate) => candidate.id),	968	.map((candidate) => candidate.id),
968	),	969	),
969	run,	970	run,
970	runAttemptIds: projection.attempts	971	attempts: projection.attempts,
971	.filter((candidate) => candidate.runId === run.id)
972	.map((candidate) => candidate.id),
973	});	972	});
974	const restartNote =	973	const restartNote =
975	restartCancelledWork.length === 0	974	restartCancelledWork.length === 0

apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts+106-2

...	@@ -6,12 +6,15 @@ import {	...	@@ -6,12 +6,15 @@ import {
6	RunId,	6	RunId,
7	type OrchestrationV2Run,	7	type OrchestrationV2Run,
8	} from "@t3tools/contracts";	8	} from "@t3tools/contracts";
		9	import * as DateTime from "effect/DateTime";
9		10
10	import {	11	import {
11	cancelledRosterTaskWork,	12	cancelledRosterTaskWork,
12	cancelledTurnItemWork,	13	cancelledTurnItemWork,
		14	isRestartNoteContinuation,
13	pendingRestartCancelledBackgroundWork,	15	pendingRestartCancelledBackgroundWork,
14	restartCancelledBackgroundWorkNote,	16	restartCancelledBackgroundWorkNote,
		17	restartContinuationNote,
15	mergeRestartCancelledBackgroundWork,	18	mergeRestartCancelledBackgroundWork,
16	} from "./RestartBackgroundNote.ts";	19	} from "./RestartBackgroundNote.ts";
17		20
...	@@ -51,7 +54,7 @@ it("keeps the note for the provider thread that lost the work across a provider	...	@@ -51,7 +54,7 @@ it("keeps the note for the provider thread that lost the work across a provider
51	providerTurns: runs.filter((source) => source.id !== target.id).map(turnFor),	54	providerTurns: runs.filter((source) => source.id !== target.id).map(turnFor),
52	compactionMessageIds: new Set(),	55	compactionMessageIds: new Set(),
53	run: target,	56	run: target,
54	runAttemptIds: target.activeAttemptId === null ? [] : [target.activeAttemptId],	57	attempts: [{ id: target.activeAttemptId!, runId: target.id }],
55	});	58	});
56		59
57	// Codex never lost the work, so it neither receives nor consumes the note.	60	// Codex never lost the work, so it neither receives nor consumes the note.
...	@@ -62,6 +65,30 @@ it("keeps the note for the provider thread that lost the work across a provider	...	@@ -62,6 +65,30 @@ it("keeps the note for the provider thread that lost the work across a provider
62	assert.deepEqual(pending(later, [root, onCodex, backOnClaude, later]), []);	65	assert.deepEqual(pending(later, [root, onCodex, backOnClaude, later]), []);
63	});	66	});
64		67
		68	it("delivers a resumed queued run's note even after a higher-ordinal run", () => {
		69	// Run 3 ran ahead of held run 2; run 2 then resumed, lost background work in
		70	// a second restart, and its continuation was superseded by a new message.
		71	const ranFirst = run(3, claudeThread, {
		72	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
		73	});
		74	const resumed = run(2, claudeThread, {
		75	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
		76	restartCancelledBackgroundWork: lost,
		77	});
		78	const next = run(4, claudeThread);
		79	const runs = [resumed, ranFirst, next];
		80	assert.deepEqual(
		81	pendingRestartCancelledBackgroundWork({
		82	runs,
		83	providerTurns: [resumed, ranFirst].map(turnFor),
		84	compactionMessageIds: new Set(),
		85	run: next,
		86	attempts: [{ id: next.activeAttemptId!, runId: next.id }],
		87	}),
		88	lost,
		89	);
		90	});
		91
65	it("bounds the note so it cannot crowd out the turn's context", () => {	92	it("bounds the note so it cannot crowd out the turn's context", () => {
66	const work = Array.from({ length: 25 }, (_, index) => ({	93	const work = Array.from({ length: 25 }, (_, index) => ({
67	kind: "shell" as const,	94	kind: "shell" as const,
...	@@ -123,7 +150,7 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()	...	@@ -123,7 +150,7 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()
123	],	150	],
124	compactionMessageIds: new Set(),	151	compactionMessageIds: new Set(),
125	run: steered,	152	run: steered,
126	runAttemptIds,	153	attempts: runAttemptIds.map((id) => ({ id, runId: steered.id })),
127	});	154	});
128		155
129	// The first attempt reached the provider with the note; its replacement must not repeat it.	156	// The first attempt reached the provider with the note; its replacement must not repeat it.
...	@@ -131,3 +158,80 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()	...	@@ -131,3 +158,80 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()
131	// A first attempt that never reached the provider did not deliver it.	158	// A first attempt that never reached the provider did not deliver it.
132	assert.deepEqual(pending([firstAttempt, RunAttemptId.make("attempt:2b")], false), lost);	159	assert.deepEqual(pending([firstAttempt, RunAttemptId.make("attempt:2b")], false), lost);
133	});	160	});
		161
		162	it("counts a prompted mid-turn or chained continuation as delivering the note", () => {
		163	const cut = run(1, claudeThread, { status: "cancelled", restartCancelledBackgroundWork: lost });
		164	const cutTurn = { ...turnFor(cut), status: "cancelled" as const };
		165	const continuation = run(2, claudeThread, { restartContinuationOfRunId: cut.id });
		166	// A turn cut mid-way that lost work is prompted with the note, not resumed natively.
		167	assert.isTrue(isRestartNoteContinuation(continuation, [cut, continuation], [cutTurn], []));
		168	assert.isFalse(
		169	isRestartNoteContinuation(
		170	continuation,
		171	[{ ...cut, restartCancelledBackgroundWork: [] }, continuation],
		172	[cutTurn],
		173	[],
		174	),
		175	);
		176	const later = run(3, claudeThread);
		177	const pending = (delivered: boolean) =>
		178	pendingRestartCancelledBackgroundWork({
		179	runs: [cut, continuation, later],
		180	providerTurns: [cutTurn, ...(delivered ? [turnFor(continuation)] : [])],
		181	compactionMessageIds: new Set(),
		182	run: later,
		183	attempts: [{ id: later.activeAttemptId!, runId: later.id }],
		184	});
		185	assert.deepEqual(pending(true), []);
		186	// A continuation that never reached the provider delivered nothing.
		187	assert.deepEqual(pending(false), lost);
		188
		189	// Its own continuation carries the original note forward.
		190	const unstarted = { ...continuation, status: "cancelled" as const };
		191	const chained = run(3, claudeThread, { restartContinuationOfRunId: unstarted.id });
		192	assert.deepEqual(restartContinuationNote(unstarted, [cut, unstarted], [cutTurn], []), {
		193	work: lost,
		194	settled: false,
		195	});
		196	assert.isTrue(isRestartNoteContinuation(chained, [cut, unstarted, chained], [cutTurn], []));
		197	// Claude announces running before accepting the prompt. A second restart
		198	// can cancel that turn without delivering anything to the provider.
		199	for (const status of ["running", "cancelled"] as const) {
		200	const turns = [cutTurn, { ...turnFor(unstarted), status }];
		201	assert.deepEqual(restartContinuationNote(unstarted, [cut, unstarted], turns, []), {
		202	work: lost,
		203	settled: false,
		204	});
		205	assert.deepEqual(
		206	pendingRestartCancelledBackgroundWork({
		207	runs: [cut, unstarted, later],
		208	providerTurns: turns,
		209	compactionMessageIds: new Set(),
		210	run: later,
		211	attempts: [{ id: later.activeAttemptId!, runId: later.id }],
		212	}),
		213	lost,
		214	);
		215	}
		216
		217	// A steer replaced the attempt after its completed turn delivered the note.
		218	const steered = { ...unstarted, activeAttemptId: RunAttemptId.make("attempt:replacement") };
		219	const attempts = [{ id: unstarted.activeAttemptId!, runId: steered.id }];
		220	const turns = [
		221	cutTurn,
		222	turnFor(unstarted),
		223	{ ...turnFor(steered), status: "cancelled" as const },
		224	];
		225	assert.deepEqual(restartContinuationNote(steered, [cut, steered], turns, attempts).work, []);
		226	assert.isFalse(isRestartNoteContinuation(chained, [cut, steered, chained], turns, attempts));
		227	assert.deepEqual(
		228	pendingRestartCancelledBackgroundWork({
		229	runs: [cut, steered, later],
		230	providerTurns: turns,
		231	compactionMessageIds: new Set(),
		232	run: later,
		233	attempts,
		234	}),
		235	[],
		236	);
		237	});

apps/server/src/orchestration-v2/RestartBackgroundNote.ts+67-13

 @@ -3,10 +3,13 @@ import type {
   OrchestrationV2ProviderTurn,
   OrchestrationV2RestartCancelledBackgroundWork,
   OrchestrationV2Run,
+  OrchestrationV2RunAttempt,
   OrchestrationV2TurnItem,
 } from "@t3tools/contracts";
+import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
 type Work = OrchestrationV2RestartCancelledBackgroundWork;
+type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
 const MAX_LABEL_LENGTH = 160;
 @@ -120,25 +123,69 @@ export function isRestartNoteSource(
   );
 }
-/** A restart continuation whose prompt is the note rather than a resume. */
+/**
+ * The note a restart continuation of `source` carries, and whether the turn it
+ * continues had settled (the note is then the whole prompt). A continuation cut
+ * before its provider turn completed may not have delivered its own note:
+ * adapters can announce a running turn before accepting the prompt. Carry the
+ * note forward until a completed turn proves delivery.
+ */
+export function restartContinuationNote(
+  source: OrchestrationV2Run,
+  runs: ReadonlyArray<OrchestrationV2Run>,
+  providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
+  attempts: ReadonlyArray<Attempt>,
+): { readonly work: ReadonlyArray<Work>; readonly settled: boolean } {
+  const completedAttempts = new Set(
+    providerTurns.filter((turn) => turn.status === "completed").map((turn) => turn.runAttemptId),
+  );
+  const completedRuns = new Set(
+    attempts.filter((attempt) => completedAttempts.has(attempt.id)).map((attempt) => attempt.runId),
+  );
+  let current = source;
+  let work = current.restartCancelledBackgroundWork ?? [];
+  const visited = new Set([current.id]);
+  while (
+    current.restartContinuationOfRunId !== undefined &&
+    !completedRuns.has(current.id) &&
+    !(current.activeAttemptId !== null && completedAttempts.has(current.activeAttemptId))
+  ) {
+    const previous = runs.find((candidate) => candidate.id === current.restartContinuationOfRunId);
+    if (previous === undefined || visited.has(previous.id)) break;
+    visited.add(previous.id);
+    current = previous;
+    work = mergeRestartCancelledBackgroundWork(current.restartCancelledBackgroundWork ?? [], work);
+  }
+  return { work, settled: isRestartNoteSource(current, providerTurns) };
+}
+/**
+ * A restart continuation prompted with the note rather than a native resume.
+ * A turn cut mid-way that lost background work is prompted too, so the note
+ * is delivered with it; only a continuation without a note resumes natively.
+ */
 export function isRestartNoteContinuation(
   run: Pick<OrchestrationV2Run, "restartContinuationOfRunId">,
   runs: ReadonlyArray<OrchestrationV2Run>,
   providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
+  attempts: ReadonlyArray<Attempt>,
 ): boolean {
   const source =
     run.restartContinuationOfRunId === undefined
       ? undefined
       : runs.find((candidate) => candidate.id === run.restartContinuationOfRunId);
-  return source !== undefined && isRestartNoteSource(source, providerTurns);
+  return (
+    source !== undefined &&
+    restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0
+  );
 }
 /**
  * Work cancelled by a restart that the run's provider thread has not been told
  * about yet. The note belongs to the provider thread that lost the work: turns
  * on another provider (after a switch) neither owe it nor deliver it. A later
- * run on the same provider thread delivers it once its attempt reaches the
+ * completed turn on the same provider thread proves delivery, so the pending
- * provider, so the pending set is derived rather than cleared. Compactions and
+ * set is derived rather than cleared. Compactions and
  * resumed turns carry no note, and a rolled-back run left native history, so
  * none of them counts as delivery. A note continuation's own prompt is the note.
  */
 @@ -150,13 +197,14 @@ export function pendingRestartCancelledBackgroundWork(input: {
     OrchestrationV2Run,
     | "id"
     | "ordinal"
+    | "completedAt"
     | "userMessageId"
     | "providerThreadId"
     | "restartContinuationOfRunId"
     | "activeAttemptId"
   >;
-  /** Every attempt id of `run`; a steer replaces the attempt but not the run. */
+  /** Include earlier attempts: a steer replaces the attempt but not the run. */
-  readonly runAttemptIds: ReadonlyArray<OrchestrationV2Run["activeAttemptId"] & string>;
+  readonly attempts: ReadonlyArray<Attempt>;
 }): ReadonlyArray<Work> {
   const isCompaction = (run: typeof input.run) => input.compactionMessageIds.has(run.userMessageId);
   // The current run prepends the note unless it is a compaction or a
 @@ -170,7 +218,7 @@ export function pendingRestartCancelledBackgroundWork(input: {
   const providerThreadId = input.run.providerThreadId;
   const deliveredAttemptIds = new Set(
     input.providerTurns
-      .filter((turn) => turn.providerThreadId === providerThreadId)
+      .filter((turn) => turn.providerThreadId === providerThreadId && turn.status === "completed")
       .map((turn) => turn.runAttemptId),
   );
   const sameThread = input.runs.filter((run) => run.providerThreadId === providerThreadId);
 @@ -179,23 +227,29 @@ export function pendingRestartCancelledBackgroundWork(input: {
       candidate.id !== input.run.id &&
       candidate.activeAttemptId !== null &&
       candidate.status !== "rolled_back" &&
-      deliveredAttemptIds.has(candidate.activeAttemptId) &&
+      (deliveredAttemptIds.has(candidate.activeAttemptId) ||
+        input.attempts.some(
+          (attempt) => attempt.runId === candidate.id && deliveredAttemptIds.has(attempt.id),
+        )) &&
       !isCompaction(candidate) &&
       (candidate.restartContinuationOfRunId === undefined ||
-        isRestartNoteContinuation(candidate, input.runs, input.providerTurns)),
+        isRestartNoteContinuation(candidate, input.runs, input.providerTurns, input.attempts)),
   );
   // A steer restarts this run on a new attempt: an earlier attempt that already
   // reached the provider delivered the note, so the replacement must not repeat it.
-  const alreadyDelivered = input.runAttemptIds.some(
+  const alreadyDelivered = input.attempts.some(
-    (attemptId) => attemptId !== input.run.activeAttemptId && deliveredAttemptIds.has(attemptId),
+    (attempt) =>
+      attempt.runId === input.run.id &&
+      attempt.id !== input.run.activeAttemptId &&
+      deliveredAttemptIds.has(attempt.id),
   );
   if (alreadyDelivered) return [];
   return sameThread
     .filter(
       (source) =>
-        source.ordinal < input.run.ordinal &&
+        runRanAfter(input.run, source) &&
         (source.restartCancelledBackgroundWork?.length ?? 0) > 0 &&
-        !prompted.some((later) => later.ordinal > source.ordinal),
+        !prompted.some((later) => runRanAfter(later, source)),
     )
     .reduce<ReadonlyArray<Work>>(
       (work, source) =>

apps/server/src/orchestration-v2/RestartContinuation.test.ts+217

...	@@ -1,5 +1,6 @@	...	@@ -1,5 +1,6 @@
1	import { assert, it } from "@effect/vitest";	1	import { assert, it } from "@effect/vitest";
2	import {	2	import {
		3	MessageId,
3	ProjectId,	4	ProjectId,
4	ProviderDriverKind,	5	ProviderDriverKind,
5	ProviderInstanceId,	6	ProviderInstanceId,
...	@@ -11,6 +12,7 @@ import {	...	@@ -11,6 +12,7 @@ import {
11	ThreadId,	12	ThreadId,
12	type OrchestrationV2ThreadProjection,	13	type OrchestrationV2ThreadProjection,
13	} from "@t3tools/contracts";	14	} from "@t3tools/contracts";
		15	import * as DateTime from "effect/DateTime";
14	import * as Effect from "effect/Effect";	16	import * as Effect from "effect/Effect";
15	import * as Layer from "effect/Layer";	17	import * as Layer from "effect/Layer";
16	import * as ServerSettings from "../serverSettings.ts";	18	import * as ServerSettings from "../serverSettings.ts";
...	@@ -201,6 +203,7 @@ it.effect("prompts a settled thread's continuation with the note of its lost wor	...	@@ -201,6 +203,7 @@ it.effect("prompts a settled thread's continuation with the note of its lost wor
201	Layer.merge(	203	Layer.merge(
202	Layer.mock(ThreadManagementService.ThreadManagementService)({	204	Layer.mock(ThreadManagementService.ThreadManagementService)({
203	getThreadRecords: () => Effect.succeed(projection),	205	getThreadRecords: () => Effect.succeed(projection),
		206	recoverDelegatedTask: () => Effect.void,
204	dispatch: (command) => {	207	dispatch: (command) => {
205	commands.push(command);	208	commands.push(command);
206	return Effect.succeed({} as never);	209	return Effect.succeed({} as never);
...	@@ -246,6 +249,7 @@ it.effect("does not continue a failed run that lost background work", () =>	...	@@ -246,6 +249,7 @@ it.effect("does not continue a failed run that lost background work", () =>
246	Layer.merge(	249	Layer.merge(
247	Layer.mock(ThreadManagementService.ThreadManagementService)({	250	Layer.mock(ThreadManagementService.ThreadManagementService)({
248	getThreadRecords: () => Effect.succeed(projection),	251	getThreadRecords: () => Effect.succeed(projection),
		252	recoverDelegatedTask: () => Effect.void,
249	dispatch: (command) => {	253	dispatch: (command) => {
250	commands.push(command);	254	commands.push(command);
251	return Effect.succeed({} as never);	255	return Effect.succeed({} as never);
...	@@ -328,6 +332,7 @@ it.effect("does not duplicate delivery and yields to newer user work or opt-out"	...	@@ -328,6 +332,7 @@ it.effect("does not duplicate delivery and yields to newer user work or opt-out"
328	>[0][] = [];	332	>[0][] = [];
329	const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({	333	const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({
330	getThreadRecords: () => Effect.succeed(projection),	334	getThreadRecords: () => Effect.succeed(projection),
		335	recoverDelegatedTask: () => Effect.void,
331	dispatch: (command) => {	336	dispatch: (command) => {
332	commands.push(command);	337	commands.push(command);
333	if (command.type === "message.dispatch")	338	if (command.type === "message.dispatch")
...	@@ -499,6 +504,7 @@ it.effect("does not cancel or resume a run that completes while shutdown intent	...	@@ -499,6 +504,7 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
499	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),	504	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
500	Layer.mock(ThreadManagementService.ThreadManagementService)({	505	Layer.mock(ThreadManagementService.ThreadManagementService)({
501	getThreadRecords: () => Effect.succeed(projection),	506	getThreadRecords: () => Effect.succeed(projection),
		507	recoverDelegatedTask: () => Effect.void,
502	dispatch: () =>	508	dispatch: () =>
503	Effect.sync(() => {	509	Effect.sync(() => {
504	dispatched = true;	510	dispatched = true;
...	@@ -511,3 +517,214 @@ it.effect("does not cancel or resume a run that completes while shutdown intent	...	@@ -511,3 +517,214 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
511	assert.isFalse(dispatched);	517	assert.isFalse(dispatched);
512	}),	518	}),
513	);	519	);
		520
		521	const continuationTexts = (projection: OrchestrationV2ThreadProjection) =>
		522	Effect.gen(function* () {
		523	const texts: Array<string> = [];
		524	yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
		525	Effect.provide(
		526	Layer.merge(
		527	Layer.mock(ThreadManagementService.ThreadManagementService)({
		528	getThreadRecords: () => Effect.succeed(projection),
		529	recoverDelegatedTask: () => Effect.void,
		530	dispatch: (command) => {
		531	if (command.type === "message.dispatch") texts.push(command.text);
		532	return Effect.succeed({} as never);
		533	},
		534	}),
		535	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
		536	),
		537	),
		538	);
		539	return texts;
		540	});
		541
		542	const cutMidTurn = (extra: Record<string, unknown> = {}) => {
		543	const base = makeProjection();
		544	return {
		545	...base,
		546	runs: [
		547	{
		548	...base.runs[0]!,
		549	status: "cancelled",
		550	userMessageId: MessageId.make("message:user"),
		551	...extra,
		552	},
		553	],
		554	providerTurns: [{ ...base.providerTurns[0]!, status: "cancelled" }],
		555	} as unknown as OrchestrationV2ThreadProjection;
		556	};
		557
		558	const queuedFollowUp = {
		559	id: RunId.make("run:queued"),
		560	ordinal: 2,
		561	providerInstanceId: instanceId,
		562	providerThreadId,
		563	status: "queued",
		564	queueHeld: true,
		565	};
		566
		567	it.effect("continues a cut run past queued follow-ups, which stay held", () =>
		568	Effect.gen(function* () {
		569	const live = makeProjection();
		570	assert.equal(
		571	restartContinuationRun({
		572	...live,
		573	runs: [...live.runs, queuedFollowUp],
		574	} as unknown as OrchestrationV2ThreadProjection)?.id,
		575	runId,
		576	);
		577	const cut = cutMidTurn();
		578	const texts = yield* continuationTexts({
		579	...cut,
		580	runs: [...cut.runs, queuedFollowUp],
		581	} as unknown as OrchestrationV2ThreadProjection);
		582	assert.deepEqual(texts, ["Continue where you left off."]);
		583	}),
		584	);
		585
		586	it.effect("continues a resumed queued run that ran after an earlier continuation", () =>
		587	Effect.gen(function* () {
		588	// The continuation (ordinal 3) ran ahead of the held queue, then the user
		589	// resumed the queued run (ordinal 1 here) and the server restarted again.
		590	const finishedContinuation = {
		591	id: RunId.make("run:earlier-continuation"),
		592	ordinal: 3,
		593	providerInstanceId: instanceId,
		594	providerThreadId,
		595	status: "completed",
		596	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
		597	};
		598	const live = makeProjection();
		599	assert.equal(
		600	restartContinuationRun({
		601	...live,
		602	runs: [...live.runs, finishedContinuation],
		603	} as unknown as OrchestrationV2ThreadProjection)?.id,
		604	runId,
		605	);
		606	const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });
		607	const texts = yield* continuationTexts({
		608	...cut,
		609	runs: [...cut.runs, finishedContinuation],
		610	} as unknown as OrchestrationV2ThreadProjection);
		611	assert.deepEqual(texts, ["Continue where you left off."]);
		612	}),
		613	);
		614
		615	it.effect("does not continue a run the user asked to stop before the restart", () =>
		616	Effect.gen(function* () {
		617	const texts = yield* continuationTexts({
		618	...cutMidTurn(),
		619	turnItems: [{ id: "turn-item:interrupt", runId, type: "run_interrupt_request" }],
		620	} as unknown as OrchestrationV2ThreadProjection);
		621	assert.deepEqual(texts, []);
		622	}),
		623	);
		624
		625	it.effect("does not continue a cut /compact or /logout turn", () =>
		626	Effect.gen(function* () {
		627	for (const text of ["/compact", " /LOGOUT "]) {
		628	const texts = yield* continuationTexts({
		629	...cutMidTurn(),
		630	messages: [{ id: MessageId.make("message:user"), text, attachments: [] }],
		631	} as unknown as OrchestrationV2ThreadProjection);
		632	assert.deepEqual(texts, [], text);
		633	}
		634	const texts = yield* continuationTexts({
		635	...cutMidTurn(),
		636	messages: [{ id: MessageId.make("message:user"), text: "/compact later", attachments: [] }],
		637	} as unknown as OrchestrationV2ThreadProjection);
		638	assert.lengthOf(texts, 1);
		639	}),
		640	);
		641
		642	it.effect("tells a turn cut mid-way about the background work it lost", () =>
		643	Effect.gen(function* () {
		644	const texts = yield* continuationTexts(
		645	cutMidTurn({
		646	restartCancelledBackgroundWork: [{ kind: "subagent", label: "Background reviewer" }],
		647	}),
		648	);
		649	assert.lengthOf(texts, 1);
		650	assert.include(texts[0]!, "Background reviewer");
		651	assert.isTrue(texts[0]!.endsWith("Continue where you left off."));
		652	}),
		653	);
		654
		655	it.effect(
		656	"carries the note forward when its continuation was cut before reaching the provider",
		657	() =>
		658	Effect.gen(function* () {
		659	const base = makeProjection();
		660	const original = {
		661	...base.runs[0]!,
		662	id: RunId.make("run:original"),
		663	status: "completed",
		664	activeAttemptId: RunAttemptId.make("attempt:original"),
		665	restartCancelledBackgroundWork: [{ kind: "shell", label: "sleep 25 && echo DONE" }],
		666	};
		667	const projection = {
		668	...base,
		669	runs: [
		670	original,
		671	{
		672	...base.runs[0]!,
		673	ordinal: 2,
		674	status: "cancelled",
		675	restartContinuationOfRunId: original.id,
		676	},
		677	],
		678	// The original turn settled; the continuation's attempt never started one.
		679	providerTurns: [
		680	{
		681	...base.providerTurns[0]!,
		682	runAttemptId: original.activeAttemptId,
		683	status: "completed",
		684	},
		685	],
		686	} as unknown as OrchestrationV2ThreadProjection;
		687	const texts = yield* continuationTexts(projection);
		688	assert.lengthOf(texts, 1);
		689	assert.include(texts[0]!, "sleep 25 && echo DONE");
		690	assert.notInclude(texts[0]!, "Continue where");
		691	}),
		692	);
		693
		694	it.effect("prepares later threads' continuations when one thread fails", () =>
		695	Effect.gen(function* () {
		696	const brokenThreadId = ThreadId.make("thread:broken");
		697	const writes: Parameters<EventSink.EventSinkV2["Service"]["writeWithEffects"]>[0][] = [];
		698	const recovery = yield* ProviderRuntimeRecovery.make.pipe(
		699	Effect.provide(
		700	Layer.mergeAll(
		701	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
		702	Layer.mock(ProjectionStore.ProjectionStoreV2)({
		703	getRecoveryThreadIds: () => Effect.succeed([brokenThreadId, threadId]),
		704	getRuntimeRecoveryProjection: (id) =>
		705	id === brokenThreadId
		706	? Effect.fail(
		707	new ProjectionStore.ProjectionStoreThreadNotFoundError({ threadId: id }),
		708	)
		709	: Effect.succeed(makeProjection()),
		710	}),
		711	Layer.mock(EventSink.EventSinkV2)({
		712	writeWithEffects: (input) =>
		713	Effect.sync(() => {
		714	writes.push(input);
		715	return [];
		716	}),
		717	}),
		718	IdAllocator.layer,
		719	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
		720	Layer.mock(EffectOutbox.EffectOutboxV2)({}),
		721	),
		722	),
		723	);
		724	yield* recovery.prepareForShutdown;
		725	assert.deepEqual(
		726	writes.map((write) => write.effects[0]?.request),
		727	[{ type: "provider-runtime.continue", sourceRunId: runId }],
		728	);
		729	}),
		730	);

apps/server/src/orchestration-v2/RestartContinuation.ts+64-7

...	@@ -1,3 +1,4 @@	...	@@ -1,3 +1,4 @@
		1	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1	import { resolveProjectSettings } from "@t3tools/shared/projectSettings";	2	import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
2	import {	3	import {
3	CommandId,	4	CommandId,
...	@@ -11,12 +12,16 @@ import * as Effect from "effect/Effect";	...	@@ -11,12 +12,16 @@ import * as Effect from "effect/Effect";
11	import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";	12	import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
12		13
13	import * as ServerSettings from "../serverSettings.ts";	14	import * as ServerSettings from "../serverSettings.ts";
		15	import { isNativeMaintenanceCommand } from "./Orchestrator.ts";
14	import * as ThreadManagementService from "./ThreadManagementService.ts";	16	import * as ThreadManagementService from "./ThreadManagementService.ts";
15	import {	17	import {
16	isRestartNoteSource,	18	isRestartNoteSource,
17	restartCancelledBackgroundWorkNote,	19	restartCancelledBackgroundWorkNote,
		20	restartContinuationNote,
18	} from "./RestartBackgroundNote.ts";	21	} from "./RestartBackgroundNote.ts";
19		22
		23	const CONTINUE_PROMPT = "Continue where you left off.";
		24
20	/**	25	/**
21	* The run a restart continuation resumes, if any: an unfinished root run, or a	26	* The run a restart continuation resumes, if any: an unfinished root run, or a
22	* settled one whose own provider thread lost background work in the restart	27	* settled one whose own provider thread lost background work in the restart
...	@@ -30,8 +35,12 @@ export function restartContinuationRun(	...	@@ -30,8 +35,12 @@ export function restartContinuationRun(
30	cancelledWorkProviderThreadIds: ReadonlySet<ProviderThreadId> = new Set(),	35	cancelledWorkProviderThreadIds: ReadonlySet<ProviderThreadId> = new Set(),
31	): OrchestrationV2Run \| undefined {	36	): OrchestrationV2Run \| undefined {
32	if (projection.thread.archivedAt !== null \|\| projection.thread.deletedAt !== null) return;	37	if (projection.thread.archivedAt !== null \|\| projection.thread.deletedAt !== null) return;
		38	// Queued runs never started; recovery holds them behind the cut run.
33	const run = projection.runs.reduce<OrchestrationV2Run \| undefined>(	39	const run = projection.runs.reduce<OrchestrationV2Run \| undefined>(
34	(latest, candidate) => (!latest \|\| candidate.ordinal > latest.ordinal ? candidate : latest),	40	(latest, candidate) =>
		41	candidate.status !== "queued" && (!latest \|\| runRanAfter(candidate, latest))
		42	? candidate
		43	: latest,
35	undefined,	44	undefined,
36	);	45	);
37	if (!run) return;	46	if (!run) return;
...	@@ -98,7 +107,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta	...	@@ -98,7 +107,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
98	const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);	107	const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);
99	const projection = yield* threads.getThreadRecords(	108	const projection = yield* threads.getThreadRecords(
100	input.threadId,	109	input.threadId,
101	["messages", "runs", "providerTurns"],	110	["messages", "runs", "providerTurns", "attempts"],
102	{ messageIds: [messageId] },	111	{ messageIds: [messageId] },
103	);	112	);
104	if (	113	if (
...	@@ -114,17 +123,54 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta	...	@@ -114,17 +123,54 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
114	const noteSource =	123	const noteSource =
115	source !== undefined && isRestartNoteSource(source, projection.providerTurns);	124	source !== undefined && isRestartNoteSource(source, projection.providerTurns);
116	if (!source \|\| (source.status !== "cancelled" && !noteSource)) return;	125	if (!source \|\| (source.status !== "cancelled" && !noteSource)) return;
117	// A user submission after reconciliation takes precedence over an automatic prompt.	126	// A user submission after reconciliation takes precedence over an automatic
118	if (projection.runs.some((run) => run.ordinal > source.ordinal)) return;	127	// prompt. Queued runs never started and stay held behind this one.
		128	if (
		129	projection.runs.some(
		130	(run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
		131	)
		132	)
		133	return;
119	if (projection.thread.providerInstanceId !== source.providerInstanceId) return;	134	if (projection.thread.providerInstanceId !== source.providerInstanceId) return;
		135	const sourceRecords = yield* threads.getThreadRecords(
		136	input.threadId,
		137	["messages", "turnItems"],
		138	{
		139	messageIds: [source.userMessageId],
		140	turnItemRunIds: [source.id],
		141	turnItemTypes: ["run_interrupt_request"],
		142	},
		143	);
		144	// The user asked this run to stop before the restart cut it.
		145	if (
		146	sourceRecords.turnItems.some(
		147	(item) => item.runId === source.id && item.type === "run_interrupt_request",
		148	)
		149	)
		150	return;
		151	const sourceMessage = sourceRecords.messages.find(
		152	(message) => message.id === source.userMessageId,
		153	);
		154	if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
		155	const note = restartContinuationNote(
		156	source,
		157	projection.runs,
		158	projection.providerTurns,
		159	projection.attempts,
		160	);
		161	const noteText =
		162	note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work);
120	yield* threads.dispatch({	163	yield* threads.dispatch({
121	type: "message.dispatch",	164	type: "message.dispatch",
122	commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`),	165	commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`),
123	threadId: input.threadId,	166	threadId: input.threadId,
124	messageId,	167	messageId,
125	text: noteSource	168	text:
126	? restartCancelledBackgroundWorkNote(source.restartCancelledBackgroundWork ?? [])	169	noteText === undefined
127	: "Continue where you left off.",	170	? CONTINUE_PROMPT
		171	: note.settled
		172	? noteText
		173	: `${noteText}\n\n${CONTINUE_PROMPT}`,
128	attachments: [],	174	attachments: [],
129	modelSelection: source.modelSelection,	175	modelSelection: source.modelSelection,
130	dispatchMode: { type: "start_immediately" },	176	dispatchMode: { type: "start_immediately" },
...	@@ -133,4 +179,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta	...	@@ -133,4 +179,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
133	restartContinuationOfRunId: input.sourceRunId,	179	restartContinuationOfRunId: input.sourceRunId,
134	});	180	});
135	},	181	},
		182	// A delegated child this declined to continue still owes its parent a
		183	// result. Once a continuation run exists this is a no-op; that run settles it.
		184	(effect, input) =>
		185	effect.pipe(
		186	Effect.andThen(
		187	Effect.gen(function* () {
		188	const threads = yield* ThreadManagementService.ThreadManagementService;
		189	yield* threads.recoverDelegatedTask(input.threadId, input.sourceRunId);
		190	}),
		191	),
		192	),
136	);	193	);

apps/server/src/orchestration-v2/SubagentProjection.test.ts+22

...	@@ -166,6 +166,28 @@ function taskFixture() {	...	@@ -166,6 +166,28 @@ function taskFixture() {
166	return { projection: { ...projection, runs: [run] }, run };	166	return { projection: { ...projection, runs: [run] }, run };
167	}	167	}
168		168
		169	it("reports the run that ended last, not the highest ordinal", () => {
		170	const { projection, run } = taskFixture();
		171	// A restart continuation (ordinal 4) ran ahead of held queued runs 2 and 3.
		172	const ended = (ordinal: number, completedAt: string): OrchestrationV2Run => ({
		173	...run,
		174	id: RunId.make(`run:${ordinal}`),
		175	ordinal,
		176	completedAt: DateTime.makeUnsafe(completedAt),
		177	});
		178	const progress = delegatedTaskProgress({
		179	...projection,
		180	runs: [
		181	{ ...run, status: "cancelled" },
		182	ended(4, "2026-07-24T10:00:00.000Z"),
		183	ended(2, "2026-07-24T10:05:00.000Z"),
		184	ended(3, "2026-07-24T10:10:00.000Z"),
		185	],
		186	});
		187	assert.equal(progress.state, "result_available");
		188	assert.equal(progress.resultRun?.ordinal, 3);
		189	});
		190
169	it("waits for nested work and retains the report across monitor acknowledgements", () => {	191	it("waits for nested work and retains the report across monitor acknowledgements", () => {
170	const { projection, run } = taskFixture();	192	const { projection, run } = taskFixture();
171	assert.equal(	193	assert.equal(

apps/server/src/orchestration-v2/SubagentProjection.ts+2-1

...	@@ -16,6 +16,7 @@ import type {	...	@@ -16,6 +16,7 @@ import type {
16	ThreadId,	16	ThreadId,
17	TurnItemId,	17	TurnItemId,
18	} from "@t3tools/contracts";	18	} from "@t3tools/contracts";
		19	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
19	import * as DateTime from "effect/DateTime";	20	import * as DateTime from "effect/DateTime";
20	import { isOrchestrationV2WorkActive } from "@t3tools/contracts";	21	import { isOrchestrationV2WorkActive } from "@t3tools/contracts";
21		22
...	@@ -239,7 +240,7 @@ export function delegatedTaskProgress(projection: {	...	@@ -239,7 +240,7 @@ export function delegatedTaskProgress(projection: {
239	projection.providerThreads.some((thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0);	240	projection.providerThreads.some((thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0);
240	const resultRun = workRuns	241	const resultRun = workRuns
241	.filter((run) => terminal(run.status) && (run.startedAt !== null \|\| run.ordinal === 1))	242	.filter((run) => terminal(run.status) && (run.startedAt !== null \|\| run.ordinal === 1))
242	.toSorted((a, b) => b.ordinal - a.ordinal)[0];	243	.toSorted((a, b) => (runRanAfter(a, b) ? -1 : runRanAfter(b, a) ? 1 : 0))[0];
243	return {	244	return {
244	state:	245	state:
245	active \|\| resultRun === undefined	246	active \|\| resultRun === undefined

apps/server/src/orchestration-v2/ThreadManagementService.ts+4

apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts+4

...	@@ -433,6 +433,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(	...	@@ -433,6 +433,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
433	dispatch: orchestrator.dispatch,	433	dispatch: orchestrator.dispatch,
434	getThreadRecords: orchestrator.getThreadRecords,	434	getThreadRecords: orchestrator.getThreadRecords,
435	getThreadProjection: orchestrator.getThreadProjection,	435	getThreadProjection: orchestrator.getThreadProjection,
		436	recoverDelegatedTask: orchestrator.recoverDelegatedTask,
		437	delegatedTaskResultPending: orchestrator.delegatedTaskResultPending,
436	});	438	});
437	}),	439	}),
438	).pipe(Layer.provide(orchestratorProvided));	440	).pipe(Layer.provide(orchestratorProvided));
...	@@ -497,6 +499,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(	...	@@ -497,6 +499,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
497	Orchestrator.OrchestratorV2,	499	Orchestrator.OrchestratorV2,
498	Effect.gen(function* () {	500	Effect.gen(function* () {
499	const orchestrator = yield* Orchestrator.OrchestratorV2;	501	const orchestrator = yield* Orchestrator.OrchestratorV2;
		502	// As in serverRuntimeStartup: after runtime recovery, before the worker.
		503	yield* orchestrator.recoverDelegatedTasks;
500	yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);	504	yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
501	return orchestrator;	505	return orchestrator;
502	}),	506	}),

apps/server/src/relay/AgentAwarenessRelay.test.ts+2

...	@@ -206,6 +206,8 @@ const makeTestRelay = Effect.fnUntraced(function* (	...	@@ -206,6 +206,8 @@ const makeTestRelay = Effect.fnUntraced(function* (
206	waitForThread: unused,	206	waitForThread: unused,
207	interruptThread: unused,	207	interruptThread: unused,
208	getThreadEventSequence: unused,	208	getThreadEventSequence: unused,
		209	recoverDelegatedTask: unused,
		210	delegatedTaskResultPending: unused,
209	streamStoredEvents: Stream.empty,	211	streamStoredEvents: Stream.empty,
210	streamStoredEventsFrom: () => Stream.empty,	212	streamStoredEventsFrom: () => Stream.empty,
211	streamDomainEvents: options.domainEvents ?? Stream.empty,	213	streamDomainEvents: options.domainEvents ?? Stream.empty,

apps/server/src/scheduling/Scheduler.test.ts+24

...	@@ -6,6 +6,7 @@ import * as Queue from "effect/Queue";	...	@@ -6,6 +6,7 @@ import * as Queue from "effect/Queue";
6	import * as Ref from "effect/Ref";	6	import * as Ref from "effect/Ref";
7	import * as TestClock from "effect/testing/TestClock";	7	import * as TestClock from "effect/testing/TestClock";
8		8
		9	import * as ServerActivation from "../serverActivation.ts";
9	import * as Scheduler from "./Scheduler.ts";	10	import * as Scheduler from "./Scheduler.ts";
10		11
11	it.effect("keeps other sources running after a source defects", () =>	12	it.effect("keeps other sources running after a source defects", () =>
...	@@ -84,3 +85,26 @@ it.effect("unregisters closed sources and interrupts their in-flight work", () =	...	@@ -84,3 +85,26 @@ it.effect("unregisters closed sources and interrupts their in-flight work", () =
84	assert.equal(yield* Ref.get(runs), 1);	85	assert.equal(yield* Ref.get(runs), 1);
85	}).pipe(Effect.provide(Scheduler.layer)),	86	}).pipe(Effect.provide(Scheduler.layer)),
86	);	87	);
		88
		89	it.effect("holds due work and the clock until startup recovery activates the server", () =>
		90	Effect.gen(function* () {
		91	const activation = yield* Deferred.make<void>();
		92	const runs = yield* Ref.make(0);
		93	const ran = yield* Deferred.make<void>();
		94	yield* Effect.gen(function* () {
		95	const scheduler = yield* Scheduler.Scheduler;
		96	yield* scheduler.register(
		97	"due",
		98	Ref.update(runs, (n) => n + 1).pipe(Effect.andThen(Deferred.succeed(ran, undefined))),
		99	);
		100	yield* TestClock.adjust("10 seconds");
		101	assert.equal(yield* Ref.get(runs), 0);
		102	yield* Deferred.succeed(activation, undefined);
		103	yield* Deferred.await(ran);
		104	}).pipe(
		105	Effect.provide(Scheduler.layer),
		106	Effect.provideService(ServerActivation.ServerActivation, Deferred.await(activation)),
		107	);
		108	assert.isAtLeast(yield* Ref.get(runs), 1);
		109	}),
		110	);

apps/server/src/scheduling/Scheduler.ts+5-2

 @@ -6,6 +6,8 @@ import * as Ref from "effect/Ref";
 import * as Scope from "effect/Scope";
 import * as Semaphore from "effect/Semaphore";
+import { forkParked } from "../serverActivation.ts";
 /** Sources load due work from their durable state; registration owns its execution lifetime. */
 export class Scheduler extends Context.Service<
   Scheduler,
 @@ -47,14 +49,15 @@ const make = Effect.gen(function* () {
           return next;
         }),
     );
-    yield* run;
+    // Due work waits for startup recovery, which would cancel runs it started.
+    yield* forkParked(run);
   });
   const tick = Ref.get(sources).pipe(
     Effect.flatMap((current) => Effect.forEach(current.values(), (run) => run, { discard: true })),
   );
   // One clock for all due-work sources. A slow source cannot block another
   // source or overlap itself, and no extra missed-tick backlog is queued.
-  yield* Effect.sleep("5 seconds").pipe(Effect.andThen(tick), Effect.forever, Effect.forkScoped);
+  yield* forkParked(Effect.sleep("5 seconds").pipe(Effect.andThen(tick), Effect.forever));
   return Scheduler.of({ register });
 });

apps/server/src/serverRuntimeStartup.test.ts+10-1

 @@ -31,11 +31,20 @@ it.effect("starts without scanning or rebuilding projection history", () =>
     const result = yield* ServerRuntimeStartup.runOrderedV2StartupPhases({
       importLegacyShells: record("import"),
       recover: record("recover").pipe(Effect.as({ closedRequests: 2 })),
+      recoverDelegatedTasks: record("delegated"),
       startEffectWorker: record("worker"),
       autoBootstrap: record("bootstrap").pipe(Effect.as({ projectId: "project-1" })),
     });
-    assert.deepEqual(yield* Ref.get(calls), ["import", "recover", "worker", "bootstrap"]);
+    // Delegated recovery reads the runs recovery terminalizes, and settles them
+    // before the worker runs restart continuations that would otherwise race it.
+    assert.deepEqual(yield* Ref.get(calls), [
+      "import",
+      "recover",
+      "delegated",
+      "worker",
+      "bootstrap",
+    ]);
     assert.deepEqual(result, {
       recovery: { closedRequests: 2 },
       bootstrap: { projectId: "project-1" },

apps/server/src/serverRuntimeStartup.ts+11

 @@ -32,6 +32,7 @@ import * as Keybindings from "./keybindings.ts";
 import * as ExternalLauncher from "./process/externalLauncher.ts";
 import * as EffectWorker from "./orchestration-v2/EffectWorker.ts";
 import * as LegacyV1ThreadImporter from "./orchestration-v2/legacy/LegacyV1ThreadImporter.ts";
+import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
 import * as ProviderRuntimeRecovery from "./orchestration-v2/ProviderRuntimeRecoveryService.ts";
 import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
 import * as ThreadLaunch from "./orchestration-v2/ThreadLaunchService.ts";
 @@ -386,21 +387,26 @@ export function runOrderedV2StartupPhases<
   Bootstrap,
   ImportError,
   RecoveryError,
+  DelegationError,
   WorkerError,
   BootstrapError,
   ImportContext,
   RecoveryContext,
+  DelegationContext,
   WorkerContext,
   BootstrapContext,
 >(input: {
   readonly importLegacyShells: Effect.Effect<Import, ImportError, ImportContext>;
   readonly recover: Effect.Effect<Recovery, RecoveryError, RecoveryContext>;
+  /** Settles delegated tasks whose runs recovery just terminalized. */
+  readonly recoverDelegatedTasks: Effect.Effect<void, DelegationError, DelegationContext>;
   readonly startEffectWorker: Effect.Effect<void, WorkerError, WorkerContext>;
   readonly autoBootstrap: Effect.Effect<Bootstrap, BootstrapError, BootstrapContext>;
 }) {
   return Effect.gen(function* () {
     yield* input.importLegacyShells;
     const recovery = yield* input.recover;
+    yield* input.recoverDelegatedTasks;
     yield* input.startEffectWorker;
     const bootstrap = yield* input.autoBootstrap;
     return { recovery, bootstrap } as const;
 @@ -413,6 +419,7 @@ const make = (options?: StartupOptions) =>
     const keybindings = yield* Keybindings.Keybindings;
     const legacyV1ThreadImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;
     const providerRuntimeRecovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
+    const orchestrator = yield* Orchestrator.OrchestratorV2;
     const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
     const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
     const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents;
 @@ -509,6 +516,10 @@ const make = (options?: StartupOptions) =>
           ),
         ),
         recover: runStartupPhase("orchestration-v2.recovery", providerRuntimeRecovery.recover),
+        recoverDelegatedTasks: runStartupPhase(
+          "orchestration-v2.delegated-tasks.recover",
+          orchestrator.recoverDelegatedTasks,
+        ),
         startEffectWorker: runStartupPhase(
           "orchestration-v2.effect-worker.start",
           startEffectWorkerWithRelay({

docs/internals/server-updates.md+16-5

...	@@ -61,7 +61,9 @@ stopped backends and replays the failure for the same token.	...	@@ -61,7 +61,9 @@ stopped backends and replays the failure for the same token.
61	Restart continuation is an environment-owned preference, off by default. The	61	Restart continuation is an environment-owned preference, off by default. The
62	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)	62	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)
63	requires matching durable run, provider thread, session, and native resume identity.	63	requires matching durable run, provider thread, session, and native resume identity.
64	Ordinary queued work, finished runs, and background-only work do not qualify.	64	Queued runs never started, so recovery holds them and continues the run they wait
		65	behind. A finished run qualifies only when the restart cancelled its background work;
		66	its continuation tells the provider what will not report back.
65		67
66	Recovery retires effects tied to the lost process and records continuation intent	68	Recovery retires effects tied to the lost process and records continuation intent
67	in the durable outbox. That intent survives another restart before provider startup.	69	in the durable outbox. That intent survives another restart before provider startup.
...	@@ -71,7 +73,16 @@ closing providers, then reconciles after ingestion has stopped so a late complet	...	@@ -71,7 +73,16 @@ closing providers, then reconciles after ingestion has stopped so a late complet
71	cannot be overwritten by a stale cancellation.	73	cannot be overwritten by a stale cancellation.
72		74
73	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)	75	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)
74	rechecks the preference, archive state, provider selection, and newer user work before	76	rechecks the preference, archive state, provider selection, newer user work, a stop
75	dispatching. Stable command and message IDs prevent duplicate submissions after an	77	the user requested, and maintenance turns such as `/compact` before dispatching. Stable
76	outbox retry. Codex resumes without adding provider prompt text; other adapters receive	78	command and message IDs prevent duplicate submissions after an outbox retry. Codex
77	the continuation message through their normal turn path.	79	resumes without adding provider prompt text unless the turn lost background work;
		80	other adapters receive the continuation message through their normal turn path.
		81
		82	Delegated tasks (`delegate_task` child threads) are reconciled as their own threads,
		83	never as the parent's background work. The orchestrator settles child results and
		84	completion deliveries in a startup pass after reconciliation, because the terminal-run
		85	listener ignores reconciliation's cancellations. A cancelled child whose restart
		86	continuation is still pending in the outbox is not a result yet; the continuation's run
		87	settles it, or the handler settles it when it declines to continue. Schedulers wait for
		88	activation so they cannot start runs that reconciliation would then cancel.

packages/shared/src/orchestrationV2ThreadError.ts+18-1

...	@@ -55,11 +55,28 @@ export function latestExecutedRun(	...	@@ -55,11 +55,28 @@ export function latestExecutedRun(
55	for (const run of runs) {	55	for (const run of runs) {
56	if (run.status === "queued") continue;	56	if (run.status === "queued") continue;
57	if (run.status === "cancelled" && run.startedAt === null) continue;	57	if (run.status === "cancelled" && run.startedAt === null) continue;
58	if (latest === null \|\| run.ordinal > latest.ordinal) latest = run;	58	if (latest === null \|\| runRanAfter(run, latest)) latest = run;
59	}	59	}
60	return latest;	60	return latest;
61	}	61	}
62		62
		63	/**
		64	* Whether started `run` ran after `other`. Ordinals follow submission, but a
		65	* run can start ahead of a held queue (a restart continuation, or a message
		66	* sent while the queue is held), so a queued run resumed later can have a
		67	* lower ordinal than one that already ended. An unfinished run is the latest.
		68	*/
		69	export function runRanAfter(
		70	run: Pick<OrchestrationV2Run, "ordinal" \| "completedAt">,
		71	other: Pick<OrchestrationV2Run, "ordinal" \| "completedAt">,
		72	): boolean {
		73	const end = (candidate: typeof run) =>
		74	!candidate.completedAt
		75	? Number.POSITIVE_INFINITY
		76	: DateTime.toEpochMillis(candidate.completedAt);
		77	return end(run) === end(other) ? run.ordinal > other.ordinal : end(run) > end(other);
		78	}
		79
63	/**	80	/**
64	* The latest run that actually started, when it stopped because the	81	* The latest run that actually started, when it stopped because the
65	* subscription limit was reached. Queued messages after that run must stay	82	* subscription limit was reached. Queued messages after that run must stay

author	maria <maria@kuuro.net> 2026-10-03 20:46:32-03:00
committer	GitHub <noreply@github.com> 2026-10-03 20:46:32-03:00
log	5108c978b1d9a8bd029e5ea96781d8cbc90b3877
tree	c6258fd10d79c90283ce0103e03a966d34e60ec8
parent	f786ff3548aef904b7bdd47f263e8704ba20ca4b
signature	Signed by PGP key B5690EEEBB952194

...	@@ -313,6 +313,8 @@ export interface ThreadManagementServiceShape {	...	@@ -313,6 +313,8 @@ export interface ThreadManagementServiceShape {
313	input: ThreadManagementInterruptInput,	313	input: ThreadManagementInterruptInput,
314	) => Effect.Effect<ThreadManagementInterruptResult, ThreadManagementFailure>;	314	) => Effect.Effect<ThreadManagementInterruptResult, ThreadManagementFailure>;
315	readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"];	315	readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"];
		316	readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"];
		317	readonly delegatedTaskResultPending: Orchestrator.OrchestratorV2["Service"]["delegatedTaskResultPending"];
316	readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"];	318	readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"];
317	readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"];	319	readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"];
318	readonly streamDomainEvents: Orchestrator.OrchestratorV2["Service"]["streamDomainEvents"];	320	readonly streamDomainEvents: Orchestrator.OrchestratorV2["Service"]["streamDomainEvents"];
...	@@ -736,6 +738,8 @@ const make = Effect.gen(function* () {	...	@@ -736,6 +738,8 @@ const make = Effect.gen(function* () {
736	waitForThread,	738	waitForThread,
737	interruptThread,	739	interruptThread,
738	getThreadEventSequence: orchestrator.getThreadEventSequence,	740	getThreadEventSequence: orchestrator.getThreadEventSequence,
		741	recoverDelegatedTask: orchestrator.recoverDelegatedTask,
		742	delegatedTaskResultPending: orchestrator.delegatedTaskResultPending,
739	streamStoredEvents: orchestrator.streamStoredEvents,	743	streamStoredEvents: orchestrator.streamStoredEvents,
740	streamStoredEventsFrom: orchestrator.streamStoredEventsFrom,	744	streamStoredEventsFrom: orchestrator.streamStoredEventsFrom,
741	streamDomainEvents: orchestrator.streamDomainEvents,	745	streamDomainEvents: orchestrator.streamDomainEvents,

fix(server): restarts keep delegated tasks, queued threads, and stops intact (#15323)