Commit 5108c978b1d9a8bd029e5ea96781d8cbc90b3877 | the-knife

author	maria <maria@kuuro.net> 2026-10-03 20:46:32-03:00
committer	GitHub <noreply@github.com> 2026-10-03 20:46:32-03:00
log	5108c978b1d9a8bd029e5ea96781d8cbc90b3877
tree	c6258fd10d79c90283ce0103e03a966d34e60ec8
parent	f786ff3548aef904b7bdd47f263e8704ba20ca4b
signature	Signed by PGP key B5690EEEBB952194

fix(server): restarts keep delegated tasks, queued threads, and stops intact (#15323)

29 files changed, 2494 insertions(+), 172 deletions(-)

apps/server/src/mcp/OrchestratorMcpService.test.ts+101-1

...	...	@@ -11,10 +11,12 @@ import {
11	11	type OrchestrationV2ThreadProjection,
12	12	type ServerProvider,
13	13	} from "@t3tools/contracts";
	14	import * as DateTime from "effect/DateTime";
14	15	import * as Effect from "effect/Effect";
15	16	import * as Layer from "effect/Layer";
16	17	import * as Ref from "effect/Ref";
17	18
	19	import { OrchestratorProjectionError } from "../orchestration-v2/Orchestrator.ts";
18	20	import type { ProviderAdapterV2Shape } from "../orchestration-v2/ProviderAdapter.ts";
19	21	import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
20	22	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
...	...	@@ -52,7 +54,22 @@ describe("OrchestratorMcpService", () => {
52	54	} as unknown as OrchestrationV2ThreadProjection;
53	55	const childProjection = {
54	56	thread: { id: childThreadId },
55		runs: [{ id: childRunId, ordinal: 1, status: "completed" }],
	57	runs: [
	58	{
	59	id: childRunId,
	60	ordinal: 1,
	61	status: "completed",
	62	startedAt: DateTime.makeUnsafe("2026-10-03T10:00:00Z"),
	63	completedAt: DateTime.makeUnsafe("2026-10-03T10:10:00Z"),
	64	},
	65	{
	66	id: RunId.make("run:mcp-ack-continuation"),
	67	ordinal: 2,
	68	status: "failed",
	69	startedAt: DateTime.makeUnsafe("2026-10-03T10:02:00Z"),
	70	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00Z"),
	71	},
	72	],
56	73	contextTransfers: [],
57	74	messages: [],
58	75	subagents: [],
...	...	@@ -126,6 +143,8 @@ describe("OrchestratorMcpService", () => {
126	143	const result = yield* service.taskStatus(scope, taskId);
127	144	assert.equal(result.status, "completed");
128	145	assert.equal(result.summary, "terminal result");
	146	assert.equal(result.latestTerminalRunId, childRunId);
	147	assert.equal(result.latestTerminalStatus, "completed");
129	148	const commandIds = yield* Ref.get(acknowledgementCommandIds);
130	149	assert.equal(commandIds.length, 2);
131	150	assert.notEqual(commandIds[0], commandIds[1]);
...	...	@@ -133,6 +152,87 @@ describe("OrchestratorMcpService", () => {
133	152	}),
134	153	);
135	154
	155	it.effect("reports a restart-cut child as working until its continuation settles", () =>
	156	Effect.gen(function* () {
	157	const parentThreadId = ThreadId.make("thread:mcp-restart-parent");
	158	const childThreadId = ThreadId.make("thread:mcp-restart-child");
	159	const taskId = NodeId.make("node:mcp-restart-task");
	160	const dispatched = yield* Ref.make(0);
	161	let awaitsRestart = true;
	162	let readFails = true;
	163	const parentProjection = {
	164	thread: { id: parentThreadId },
	165	runs: [],
	166	contextTransfers: [],
	167	subagents: [
	168	{
	169	id: taskId,
	170	threadId: parentThreadId,
	171	origin: "app_owned",
	172	childThreadId,
	173	driver: "codex",
	174	model: "gpt-5.6-terra",
	175	status: "running",
	176	result: null,
	177	completionDelivery: { state: "pending" },
	178	},
	179	],
	180	} as unknown as OrchestrationV2ThreadProjection;
	181	const childProjection = {
	182	thread: { id: childThreadId },
	183	runs: [{ id: RunId.make("run:mcp-restart-child"), ordinal: 1, status: "cancelled" }],
	184	contextTransfers: [],
	185	messages: [],
	186	subagents: [],
	187	providerThreads: [],
	188	turnItems: [],
	189	} as unknown as OrchestrationV2ThreadProjection;
	190	const dependencies = Layer.mergeAll(
	191	NodeServices.layer,
	192	Layer.mock(ThreadManagementService.ThreadManagementService)({
	193	getThreadRecords: (threadId) =>
	194	Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection),
	195	delegatedTaskResultPending: () =>
	196	readFails
	197	? Effect.fail(new OrchestratorProjectionError({ threadId: childThreadId }))
	198	: Effect.succeed(awaitsRestart),
	199	dispatch: () => Ref.update(dispatched, (count) => count + 1).pipe(Effect.as({} as never)),
	200	}),
	201	Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
	202	Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({
	203	list: () => Effect.succeed([]),
	204	}),
	205	Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
	206	);
	207	const scope: McpInvocationScope = {
	208	environmentId: EnvironmentId.make("environment:mcp-restart"),
	209	threadId: parentThreadId,
	210	providerSessionId: "provider-session:mcp-restart",
	211	providerInstanceId: ProviderInstanceId.make("codex"),
	212	capabilities: new Set(["orchestration"]),
	213	issuedAt: 1,
	214	};
	215
	216	yield* Effect.gen(function* () {
	217	const service = yield* OrchestratorMcpService.OrchestratorMcpService;
	218	const failed = yield* service.taskStatus(scope, taskId).pipe(Effect.flip);
	219	assert.equal(failed.code, "orchestration_error");
	220	assert.equal(yield* Ref.get(dispatched), 0);
	221	readFails = false;
	222	const held = yield* service.taskStatus(scope, taskId);
	223	assert.equal(held.status, "running");
	224	assert.equal(held.workState, "working");
	225	assert.isNull(held.summary);
	226	// Acknowledging the cut run would suppress the real result's wake.
	227	assert.equal(yield* Ref.get(dispatched), 0);
	228	awaitsRestart = false;
	229	const settled = yield* service.taskStatus(scope, taskId);
	230	assert.equal(settled.status, "cancelled");
	231	assert.equal(yield* Ref.get(dispatched), 1);
	232	}).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies))));
	233	}),
	234	);
	235
136	236	it.effect("does not dispose delivery when a nonterminal task has no active child run", () =>
137	237	Effect.gen(function* () {
138	238	const parentThreadId = ThreadId.make("thread:mcp-cancel-parent");

apps/server/src/mcp/OrchestratorMcpService.ts+15-2

...	...	@@ -52,6 +52,7 @@ import {
52	52	type ServerProvider,
53	53	ThreadId,
54	54	} from "@t3tools/contracts";
	55	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
55	56	import * as Context from "effect/Context";
56	57	import * as Crypto from "effect/Crypto";
57	58	import * as DateTime from "effect/DateTime";
...	...	@@ -412,7 +413,10 @@ function latestTerminalResultRun(
412	413	run.status !== "rolled_back" &&
413	414	(run.id === delegatedRun?.id \|\| run.startedAt !== null),
414	415	)
415		.toSorted((left, right) => right.ordinal - left.ordinal)[0];
	416	.reduce<OrchestrationV2Run \| undefined>(
	417	(latest, run) => (latest === undefined \|\| runRanAfter(run, latest) ? run : latest),
	418	undefined,
	419	);
416	420	}
417	421
418	422	function canExposeTaskRunResult(run: OrchestrationV2Run \| undefined): run is OrchestrationV2Run {
...	...	@@ -1055,7 +1059,16 @@ const make = Effect.gen(function* () {
1055	1059	messages: [...childControls.messages, ...resultRecords.messages],
1056	1060	turnItems: resultRecords.turnItems,
1057	1061	};
1058		const workState = task.result !== null ? "result_available" : progress.state;
	1062	// A restart cut the child's run and its continuation has not settled, or
	1063	// the child started working again after this read.
	1064	const heldForRestart =
	1065	task.result === null &&
	1066	progress.state === "result_available" &&
	1067	(yield* threadManagement
	1068	.delegatedTaskResultPending(task.childThreadId)
	1069	.pipe(Effect.mapError(threadManagementFailure)));
	1070	const workState =
	1071	task.result !== null ? "result_available" : heldForRestart ? "working" : progress.state;
1059	1072	const status =
1060	1073	task.result !== null
1061	1074	? taskStatusForRun(

apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts+624

...	...	@@ -7,6 +7,7 @@ import {
7	7	MessageId,
8	8	type ModelSelection,
9	9	NodeId,
	10	type OrchestrationV2Run,
10	11	ProjectId,
11	12	ProviderDriverKind,
12	13	ProviderInstanceId,
...	...	@@ -35,6 +36,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
35	36	import * as EventSink from "./EventSink.ts";
36	37	import * as Orchestrator from "./Orchestrator.ts";
37	38	import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts";
	39	import { continueRestartedRun } from "./RestartContinuation.ts";
38	40	import {
39	41	OrchestrationV2EventSinkLayerLive,
40	42	OrchestrationV2LayerLive,
...	...	@@ -384,6 +386,98 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
384	386	}),
385	387	);
386	388
	389	it.effect("acceptance batches a settled_only sibling once its spawning run ended", () =>
	390	Effect.gen(function* () {
	391	const orchestrator = yield* Orchestrator.OrchestratorV2;
	392	const sink = yield* EventSink.EventSinkV2;
	393	const now = yield* DateTime.now;
	394	const threadId = ThreadId.make("mailbox-mixed");
	395	const runId = RunId.make("mailbox-mixed-parent");
	396	const taskId = NodeId.make("mailbox-mixed-always");
	397	const siblingId = NodeId.make("mailbox-mixed-settled-only");
	398	const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
	399	yield* seedParentWithTerminalTask({
	400	threadId,
	401	runId,
	402	projectId: ProjectId.make("mailbox-mixed-project"),
	403	rootNodeId: NodeId.make("mailbox-mixed-root"),
	404	taskId,
	405	deliveryState: "claimed",
	406	completionWake: "always",
	407	deliveryTaskIds: [taskId],
	408	now,
	409	});
	410	const projection = yield* orchestrator.getThreadProjection(threadId);
	411	const task = projection.subagents[0]!;
	412	const spawningRun = projection.runs.find((row) => row.id === runId)!;
	413	// A restart cut the spawning run; its continuation is the live run now.
	414	yield* sink.write({
	415	events: [
	416	{
	417	...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }),
	418	payload: { ...spawningRun, status: "cancelled" as const, completedAt: now },
	419	},
	420	runEvent({
	421	threadId,
	422	runId: RunId.make("mailbox-mixed-continuation"),
	423	ordinal: 2,
	424	status: "running",
	425	now,
	426	}),
	427	{
	428	id: EventId.make("mailbox-mixed-message"),
	429	type: "message.updated",
	430	threadId,
	431	runId,
	432	occurredAt: now,
	433	payload: {
	434	id: messageId,
	435	threadId,
	436	runId,
	437	nodeId: task.parentNodeId,
	438	role: "user",
	439	text: "Background task finished",
	440	attachments: [],
	441	streaming: false,
	442	createdBy: "agent",
	443	creationSource: "server",
	444	createdAt: now,
	445	updatedAt: now,
	446	delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
	447	},
	448	},
	449	{
	450	id: EventId.make(`event:${siblingId}`),
	451	type: "subagent.updated" as const,
	452	threadId,
	453	runId,
	454	nodeId: siblingId,
	455	occurredAt: now,
	456	payload: {
	457	...task,
	458	id: siblingId,
	459	completionWake: "settled_only" as const,
	460	completionDelivery: { state: "pending" as const, observedByRunId: null },
	461	},
	462	},
	463	],
	464	});
	465	yield* orchestrator.dispatch({
	466	type: "notification.delivery.accept",
	467	commandId: CommandId.make("accept-mixed"),
	468	threadId,
	469	messageId,
	470	});
	471	const accepted = yield* orchestrator.getThreadProjection(threadId);
	472	const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
	473	assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]);
	474	assert.equal(
	475	accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state,
	476	"claimed",
	477	);
	478	}),
	479	);
	480
387	481	it.effect("builds completion text and metadata from the same live cohort", () =>
388	482	Effect.gen(function* () {
389	483	const orchestrator = yield* Orchestrator.OrchestratorV2;
...	...	@@ -652,3 +746,533 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
652	746	}),
653	747	);
654	748	});
	749
	750	// Runtime reconciliation writes restart cancellations under this command
	751	// prefix, which the live terminal-run listener skips.
	752	const reconcileCommandId = (name: string) => CommandId.make(`command:runtime-reconcile:${name}`);
	753
	754	const parentProviderThreadId = (threadId: ThreadId) =>
	755	ProviderThreadId.make(`provider-thread:${String(threadId).replace("thread:", "")}`);
	756
	757	const runEvent = (input: {
	758	readonly threadId: ThreadId;
	759	readonly runId: RunId;
	760	readonly ordinal: number;
	761	readonly status: OrchestrationV2Run["status"];
	762	readonly now: DateTime.Utc;
	763	readonly providerThreadId?: ProviderThreadId;
	764	readonly userMessageId?: MessageId;
	765	readonly delegatedCompletion?: OrchestrationV2Run["delegatedCompletion"];
	766	}) => ({
	767	id: EventId.make(`event:${input.runId}:${input.status}`),
	768	type: "run.updated" as const,
	769	threadId: input.threadId,
	770	runId: input.runId,
	771	providerInstanceId: modelSelection.instanceId,
	772	occurredAt: input.now,
	773	payload: {
	774	id: input.runId,
	775	threadId: input.threadId,
	776	ordinal: input.ordinal,
	777	providerInstanceId: modelSelection.instanceId,
	778	modelSelection,
	779	providerThreadId: input.providerThreadId ?? null,
	780	userMessageId: input.userMessageId ?? MessageId.make(`message:${input.runId}`),
	781	rootNodeId: null,
	782	activeAttemptId: null,
	783	status: input.status,
	784	requestedAt: input.now,
	785	startedAt: input.now,
	786	completedAt: input.status === "running" ? null : input.now,
	787	checkpointId: null,
	788	contextHandoffId: null,
	789	...(input.delegatedCompletion === undefined
	790	? {}
	791	: { delegatedCompletion: input.delegatedCompletion }),
	792	},
	793	});
	794
	795	/** A running app-owned task whose child thread's first run the restart cancelled. */
	796	const seedRestartCancelledChild = (input: {
	797	readonly parentThreadId: ThreadId;
	798	readonly projectId: ProjectId;
	799	readonly parentRunId: RunId;
	800	readonly rootNodeId: NodeId;
	801	readonly name: string;
	802	readonly completionWake: "always" \| "settled_only";
	803	readonly continuationPending: boolean;
	804	/** "completed" seeds a settled turn whose background work the restart cancelled. */
	805	readonly runStatus?: "cancelled" \| "completed";
	806	readonly now: DateTime.Utc;
	807	}) =>
	808	Effect.gen(function* () {
	809	const eventSink = yield* EventSink.EventSinkV2;
	810	const taskId = NodeId.make(`node:${input.name}`);
	811	const childThreadId = ThreadId.make(`thread:${input.name}`);
	812	const childRunId = RunId.make(`run:${input.name}:1`);
	813	yield* eventSink.write({
	814	commandId: CommandId.make(`command:seed-child:${input.name}`),
	815	events: [
	816	{
	817	id: EventId.make(`event:${input.name}:thread`),
	818	type: "thread.created",
	819	threadId: childThreadId,
	820	occurredAt: input.now,
	821	payload: {
	822	createdBy: "agent",
	823	creationSource: "server",
	824	id: childThreadId,
	825	projectId: input.projectId,
	826	title: input.name,
	827	providerInstanceId: modelSelection.instanceId,
	828	modelSelection,
	829	runtimeMode: "full-access",
	830	interactionMode: "default",
	831	branch: null,
	832	worktreePath: null,
	833	activeProviderThreadId: null,
	834	lineage: {
	835	parentThreadId: input.parentThreadId,
	836	relationshipToParent: "subagent",
	837	rootThreadId: input.parentThreadId,
	838	},
	839	forkedFrom: { type: "node", nodeId: taskId },
	840	createdAt: input.now,
	841	updatedAt: input.now,
	842	archivedAt: null,
	843	settledOverride: null,
	844	settledAt: null,
	845	lastVisitedAt: null,
	846	deletedAt: null,
	847	},
	848	},
	849	{
	850	id: EventId.make(`event:${input.name}:task`),
	851	type: "subagent.updated",
	852	threadId: input.parentThreadId,
	853	runId: input.parentRunId,
	854	nodeId: taskId,
	855	driver,
	856	providerInstanceId: modelSelection.instanceId,
	857	occurredAt: input.now,
	858	payload: {
	859	id: taskId,
	860	threadId: input.parentThreadId,
	861	runId: input.parentRunId,
	862	parentNodeId: input.rootNodeId,
	863	origin: "app_owned",
	864	createdBy: "agent",
	865	driver,
	866	providerInstanceId: modelSelection.instanceId,
	867	providerThreadId: null,
	868	childThreadId,
	869	nativeTaskRef: null,
	870	prompt: `Run ${input.name}.`,
	871	title: null,
	872	model: null,
	873	completionWake: input.completionWake,
	874	status: "running",
	875	result: null,
	876	startedAt: input.now,
	877	completedAt: null,
	878	updatedAt: input.now,
	879	},
	880	},
	881	],
	882	});
	883	yield* eventSink.writeWithEffects({
	884	commandId: reconcileCommandId(input.name),
	885	events: [
	886	runEvent({
	887	threadId: childThreadId,
	888	runId: childRunId,
	889	ordinal: 1,
	890	status: input.runStatus ?? "cancelled",
	891	now: input.now,
	892	}),
	893	],
	894	effects: input.continuationPending
	895	? [
	896	{
	897	id: `effect:restart-continuation:${childRunId}`,
	898	commandId: reconcileCommandId(input.name),
	899	threadId: childThreadId,
	900	request: { type: "provider-runtime.continue", sourceRunId: childRunId },
	901	},
	902	]
	903	: [],
	904	});
	905	return { taskId, childThreadId, childRunId };
	906	});
	907
	908	it.layer(TestLayer)("delegated tasks across a server restart", (it) => {
	909	it.effect("holds a restart-cancelled child for its continuation's result", () =>
	910	Effect.gen(function* () {
	911	const orchestrator = yield* Orchestrator.OrchestratorV2;
	912	const eventSink = yield* EventSink.EventSinkV2;
	913	const now = yield* DateTime.now;
	914	const threadId = ThreadId.make("thread:restart-parent");
	915	const projectId = ProjectId.make("project:restart-parent");
	916	const runId = RunId.make("run:restart-parent");
	917	const rootNodeId = NodeId.make("node:restart-parent-root");
	918	yield* seedParentWithTerminalTask({
	919	threadId,
	920	projectId,
	921	runId,
	922	rootNodeId,
	923	taskId: NodeId.make("node:restart-parent-settled"),
	924	deliveryState: "delivered",
	925	now,
	926	});
	927	const child = (
	928	name: string,
	929	continuationPending: boolean,
	930	runStatus?: "cancelled" \| "completed",
	931	) =>
	932	seedRestartCancelledChild({
	933	parentThreadId: threadId,
	934	projectId,
	935	parentRunId: runId,
	936	rootNodeId,
	937	name,
	938	completionWake: "always",
	939	continuationPending,
	940	...(runStatus === undefined ? {} : { runStatus }),
	941	now,
	942	});
	943	const resumed = yield* child("restart-resumed-child", true);
	944	const stopped = yield* child("restart-stopped-child", false);
	945	// Settled with only background work left: its interim reply is not the result.
	946	const backgrounded = yield* child("restart-backgrounded-child", true, "completed");
	947	// A second restart cut the first continuation before it started.
	948	const recut = yield* child("restart-recut-child", false);
	949	const recutContinuationId = RunId.make("run:restart-recut-child:2");
	950	const recutCommandId = CommandId.make("command:restart-recut-child:reconcile");
	951	const recutRun = runEvent({
	952	threadId: recut.childThreadId,
	953	runId: recutContinuationId,
	954	ordinal: 2,
	955	status: "cancelled",
	956	now,
	957	});
	958	yield* eventSink.writeWithEffects({
	959	commandId: recutCommandId,
	960	events: [
	961	{
	962	...recutRun,
	963	payload: {
	964	...recutRun.payload,
	965	startedAt: null,
	966	restartContinuationOfRunId: recut.childRunId,
	967	},
	968	},
	969	],
	970	effects: [
	971	{
	972	id: `effect:restart-continuation:${recutContinuationId}`,
	973	commandId: recutCommandId,
	974	threadId: recut.childThreadId,
	975	request: { type: "provider-runtime.continue", sourceRunId: recutContinuationId },
	976	},
	977	],
	978	});
	979
	980	yield* orchestrator.recoverDelegatedTasks;
	981
	982	const recovered = yield* orchestrator.getThreadProjection(threadId);
	983	const task = (id: NodeId) => recovered.subagents.find((row) => row.id === id);
	984	assert.equal(task(stopped.taskId)?.status, "cancelled");
	985	assert.equal(task(stopped.taskId)?.completionDelivery?.state, "claimed");
	986	assert.equal(task(resumed.taskId)?.status, "running");
	987	assert.isNull(task(resumed.taskId)?.result ?? null);
	988	assert.equal(task(backgrounded.taskId)?.status, "running");
	989	assert.isNull(task(backgrounded.taskId)?.result ?? null);
	990	assert.equal(task(recut.taskId)?.status, "running");
	991	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId));
	992	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId));
	993	assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
	994	// A replayed first continuation settling must not release the second one's hold.
	995	yield* orchestrator.recoverDelegatedTask(recut.childThreadId, recut.childRunId);
	996	const replayed = yield* orchestrator.getThreadProjection(threadId);
	997	assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running");
	998	// A caller that read the cancelled run before the child resumed sees it as pending.
	999	yield* eventSink.write({
	1000	commandId: CommandId.make("command:restart-stopped-child:resumed"),
	1001	events: [
	1002	runEvent({
	1003	threadId: stopped.childThreadId,
	1004	runId: RunId.make("run:restart-stopped-child:2"),
	1005	ordinal: 2,
	1006	status: "running",
	1007	now,
	1008	}),
	1009	],
	1010	});
	1011	assert.isTrue(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
	1012	assert.isFalse(
	1013	recovered.contextTransfers.some(
	1014	(transfer) => transfer.sourceThreadId === resumed.childThreadId,
	1015	),
	1016	);
	1017
	1018	// The continuation's own run finishing settles the task with its result.
	1019	const afterSequence = yield* eventSink.latestSequence();
	1020	const continuationRunId = RunId.make("run:restart-resumed-child:2");
	1021	yield* eventSink.write({
	1022	commandId: CommandId.make("command:restart-resumed-child:completed"),
	1023	events: [
	1024	{
	1025	id: EventId.make("event:restart-resumed-child:result"),
	1026	type: "message.updated",
	1027	threadId: resumed.childThreadId,
	1028	runId: continuationRunId,
	1029	occurredAt: now,
	1030	payload: {
	1031	id: MessageId.make("message:restart-resumed-child:result"),
	1032	threadId: resumed.childThreadId,
	1033	runId: continuationRunId,
	1034	nodeId: null,
	1035	role: "assistant",
	1036	text: "Finished after the restart.",
	1037	attachments: [],
	1038	streaming: false,
	1039	createdBy: "agent",
	1040	creationSource: "server",
	1041	createdAt: now,
	1042	updatedAt: now,
	1043	},
	1044	},
	1045	runEvent({
	1046	threadId: resumed.childThreadId,
	1047	runId: continuationRunId,
	1048	ordinal: 2,
	1049	status: "completed",
	1050	now,
	1051	}),
	1052	],
	1053	});
	1054	const settled = yield* eventSink
	1055	.stream({ afterSequence, eventType: "subagent.updated" })
	1056	.pipe(
	1057	Stream.filter(
	1058	(stored) =>
	1059	stored.event.type === "subagent.updated" &&
	1060	stored.event.payload.id === resumed.taskId,
	1061	),
	1062	Stream.take(1),
	1063	Stream.runHead,
	1064	);
	1065	assert.isTrue(settled._tag === "Some");
	1066	const finished = yield* orchestrator.getThreadProjection(threadId);
	1067	const finishedTask = finished.subagents.find((row) => row.id === resumed.taskId);
	1068	assert.equal(finishedTask?.status, "completed");
	1069	assert.equal(finishedTask?.result, "Finished after the restart.");
	1070	}),
	1071	);
	1072
	1073	it.effect("settles a restart-cancelled child whose continuation declines to start", () =>
	1074	Effect.gen(function* () {
	1075	const orchestrator = yield* Orchestrator.OrchestratorV2;
	1076	const now = yield* DateTime.now;
	1077	const threadId = ThreadId.make("thread:restart-declined-parent");
	1078	const projectId = ProjectId.make("project:restart-declined-parent");
	1079	const runId = RunId.make("run:restart-declined-parent");
	1080	const rootNodeId = NodeId.make("node:restart-declined-parent-root");
	1081	yield* seedParentWithTerminalTask({
	1082	threadId,
	1083	projectId,
	1084	runId,
	1085	rootNodeId,
	1086	taskId: NodeId.make("node:restart-declined-parent-settled"),
	1087	deliveryState: "delivered",
	1088	now,
	1089	});
	1090	const child = yield* seedRestartCancelledChild({
	1091	parentThreadId: threadId,
	1092	projectId,
	1093	parentRunId: runId,
	1094	rootNodeId,
	1095	name: "restart-declined-child",
	1096	completionWake: "always",
	1097	continuationPending: true,
	1098	now,
	1099	});
	1100	yield* orchestrator.recoverDelegatedTasks;
	1101	const held = yield* orchestrator.getThreadProjection(threadId);
	1102	assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running");
	1103
	1104	yield* continueRestartedRun({
	1105	threadId: child.childThreadId,
	1106	sourceRunId: child.childRunId,
	1107	}).pipe(
	1108	Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })),
	1109	);
	1110
	1111	const settled = yield* orchestrator.getThreadProjection(threadId);
	1112	const task = settled.subagents.find((row) => row.id === child.taskId);
	1113	assert.equal(task?.status, "cancelled");
	1114	assert.equal(task?.completionDelivery?.state, "claimed");
	1115	}),
	1116	);
	1117
	1118	// A running provider turn does not prove the provider consumed the delivery
	1119	// (Claude marks the turn running before it reads the prompt), so a cut
	1120	// delivery is offered again even when a continuation resumes that turn.
	1121	it.effect("re-offers a cut delivery even when a continuation resumes its turn", () =>
	1122	Effect.gen(function* () {
	1123	const orchestrator = yield* Orchestrator.OrchestratorV2;
	1124	const eventSink = yield* EventSink.EventSinkV2;
	1125	const now = yield* DateTime.now;
	1126	const seed = (name: string, continuationPending: boolean) =>
	1127	Effect.gen(function* () {
	1128	const threadId = ThreadId.make(`thread:${name}`);
	1129	const runId = RunId.make(`run:${name}`);
	1130	const taskId = NodeId.make(`node:${name}-task`);
	1131	const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
	1132	const deliveryRunId = RunId.make(`run:${name}:delivery`);
	1133	yield* seedParentWithTerminalTask({
	1134	threadId,
	1135	projectId: ProjectId.make(`project:${name}`),
	1136	runId,
	1137	rootNodeId: NodeId.make(`node:${name}-root`),
	1138	taskId,
	1139	deliveryState: "claimed",
	1140	completionWake: "always",
	1141	deliveryTaskIds: [taskId],
	1142	now,
	1143	});
	1144	yield* eventSink.writeWithEffects({
	1145	commandId: reconcileCommandId(name),
	1146	events: [
	1147	{
	1148	id: EventId.make(`event:${name}:delivery-message`),
	1149	type: "message.updated",
	1150	threadId,
	1151	runId: deliveryRunId,
	1152	occurredAt: now,
	1153	payload: {
	1154	id: messageId,
	1155	threadId,
	1156	runId: deliveryRunId,
	1157	nodeId: null,
	1158	role: "user",
	1159	text: `Delegated task ${taskId} reached a terminal state.`,
	1160	attachments: [],
	1161	streaming: false,
	1162	createdBy: "agent",
	1163	creationSource: "server",
	1164	createdAt: now,
	1165	updatedAt: now,
	1166	delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
	1167	},
	1168	},
	1169	runEvent({
	1170	threadId,
	1171	runId: deliveryRunId,
	1172	ordinal: 2,
	1173	status: "cancelled",
	1174	now,
	1175	providerThreadId: parentProviderThreadId(threadId),
	1176	userMessageId: messageId,
	1177	}),
	1178	],
	1179	effects: continuationPending
	1180	? [
	1181	{
	1182	id: `effect:restart-continuation:${deliveryRunId}`,
	1183	commandId: reconcileCommandId(name),
	1184	threadId,
	1185	request: { type: "provider-runtime.continue", sourceRunId: deliveryRunId },
	1186	},
	1187	]
	1188	: [],
	1189	});
	1190	return { threadId, runId, taskId };
	1191	});
	1192	const cut = yield* seed("delivery-cut", false);
	1193	const resumed = yield* seed("delivery-resumed", true);
	1194
	1195	yield* orchestrator.recoverDelegatedTasks;
	1196
	1197	for (const seeded of [cut, resumed]) {
	1198	const projection = yield* orchestrator.getThreadProjection(seeded.threadId);
	1199	assert.deepEqual(
	1200	projection.subagents.find((row) => row.id === seeded.taskId)?.completionDelivery?.state,
	1201	"claimed",
	1202	);
	1203	const delivery = projection.runs.find((row) => row.id === seeded.runId)?.delegatedCompletion
	1204	?.delivery;
	1205	assert.equal(delivery?.generation, 2);
	1206	assert.deepEqual(delivery?.taskIds, [seeded.taskId]);
	1207	}
	1208	}),
	1209	);
	1210
	1211	it.effect("wakes the parent for a settled_only task once its spawning turn ended", () =>
	1212	Effect.gen(function* () {
	1213	const orchestrator = yield* Orchestrator.OrchestratorV2;
	1214	const eventSink = yield* EventSink.EventSinkV2;
	1215	const now = yield* DateTime.now;
	1216	const threadId = ThreadId.make("thread:settled-only-restart");
	1217	const projectId = ProjectId.make("project:settled-only-restart");
	1218	const runId = RunId.make("run:settled-only-restart");
	1219	const rootNodeId = NodeId.make("node:settled-only-restart-root");
	1220	yield* seedParentWithTerminalTask({
	1221	threadId,
	1222	projectId,
	1223	runId,
	1224	rootNodeId,
	1225	taskId: NodeId.make("node:settled-only-restart-settled"),
	1226	deliveryState: "delivered",
	1227	now,
	1228	});
	1229	// The restart cut the turn blocked in delegate_task(wait); its
	1230	// continuation is a new, live run.
	1231	yield* eventSink.write({
	1232	commandId: reconcileCommandId("settled-only-restart"),
	1233	events: [
	1234	runEvent({
	1235	threadId,
	1236	runId,
	1237	ordinal: 1,
	1238	status: "cancelled",
	1239	now,
	1240	providerThreadId: parentProviderThreadId(threadId),
	1241	userMessageId: MessageId.make(`message:seed-user:${threadId}`),
	1242	delegatedCompletion: { disposition: "open", nextGeneration: 2, delivery: null },
	1243	}),
	1244	runEvent({
	1245	threadId,
	1246	runId: RunId.make("run:settled-only-restart:continuation"),
	1247	ordinal: 2,
	1248	status: "running",
	1249	now,
	1250	providerThreadId: parentProviderThreadId(threadId),
	1251	}),
	1252	],
	1253	});
	1254	const child = yield* seedRestartCancelledChild({
	1255	parentThreadId: threadId,
	1256	projectId,
	1257	parentRunId: runId,
	1258	rootNodeId,
	1259	name: "settled-only-restart-child",
	1260	completionWake: "settled_only",
	1261	continuationPending: false,
	1262	now,
	1263	});
	1264
	1265	yield* orchestrator.recoverDelegatedTasks;
	1266
	1267	const projection = yield* orchestrator.getThreadProjection(threadId);
	1268	assert.equal(
	1269	projection.subagents.find((row) => row.id === child.taskId)?.completionDelivery?.state,
	1270	"claimed",
	1271	);
	1272	assert.deepEqual(
	1273	projection.runs.find((row) => row.id === runId)?.delegatedCompletion?.delivery?.taskIds,
	1274	[child.taskId],
	1275	);
	1276	}),
	1277	);
	1278	});

apps/server/src/orchestration-v2/EffectWorker.test.ts+49-2

...	...	@@ -79,6 +79,8 @@ function restartEffect(
79	79	function makeExecutorLayer(input: {
80	80	readonly events: Ref.Ref<ReadonlyArray<string>>;
81	81	readonly failFirstStart?: Ref.Ref<boolean>;
	82	readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
	83	readonly continueAfterRestart?: boolean;
82	84	}) {
83	85	const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
84	86	const dependencies = Layer.mergeAll(
...	...	@@ -148,8 +150,10 @@ function makeExecutorLayer(input: {
148	150	Layer.provide(
149	151	Layer.mergeAll(
150	152	dependencies,
151		Layer.mock(ThreadManagementService.ThreadManagementService)({}),
152		ServerSettings.layerTest(),
	153	Layer.mock(ThreadManagementService.ThreadManagementService)(input.threads ?? {}),
	154	ServerSettings.layerTest(
	155	input.continueAfterRestart === true ? { continueThreadsAfterServerUpdate: true } : {},
	156	),
153	157	),
154	158	),
155	159	);
...	...	@@ -753,3 +757,46 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (
753	757	]);
754	758	}),
755	759	);
	760
	761	it.effect("settles a delegated child once its restart continuation fails for good", () =>
	762	Effect.gen(function* () {
	763	const timestamp = DateTime.formatIso(yield* DateTime.now);
	764	const events = yield* Ref.make<ReadonlyArray<string>>([]);
	765	const recovered = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
	766	const layer = makeExecutorLayer({
	767	events,
	768	continueAfterRestart: true,
	769	threads: {
	770	getThreadRecords: () => Effect.fail(new Error("provider instance removed") as never),
	771	recoverDelegatedTask: (childThreadId) =>
	772	Ref.update(recovered, (ids) => [...ids, childThreadId]),
	773	},
	774	});
	775	const effect: EffectOutbox.OrchestrationEffectV2 = {
	776	id: `effect:restart-continuation:${runId}`,
	777	commandId: CommandId.make("command:restart-continuation-failure"),
	778	threadId,
	779	request: { type: "provider-runtime.continue", sourceRunId: runId },
	780	status: "running",
	781	attemptCount: 1,
	782	availableAt: timestamp,
	783	leaseOwner: "test-worker",
	784	leaseExpiresAt: timestamp,
	785	createdAt: timestamp,
	786	updatedAt: timestamp,
	787	completedAt: null,
	788	lastError: null,
	789	};
	790	yield* Effect.gen(function* () {
	791	const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
	792	assert.isTrue(
	793	Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: true }))),
	794	);
	795	assert.deepEqual(yield* Ref.get(recovered), []);
	796	assert.isTrue(
	797	Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: false }))),
	798	);
	799	assert.deepEqual(yield* Ref.get(recovered), [threadId]);
	800	}).pipe(Effect.provide(layer));
	801	}),
	802	);

apps/server/src/orchestration-v2/EffectWorker.ts+10-5

...	...	@@ -108,13 +108,17 @@ export const executorLayer: Layer.Layer<
108	108	execute: (effect, options) => {
109	109	const willRetry = options?.willRetry ?? false;
110	110	switch (effect.request.type) {
111		case "provider-runtime.continue":
112		return continueRestartedRun({
113		threadId: effect.threadId,
114		sourceRunId: effect.request.sourceRunId,
115		}).pipe(
	111	case "provider-runtime.continue": {
	112	const sourceRunId = effect.request.sourceRunId;
	113	return continueRestartedRun({ threadId: effect.threadId, sourceRunId }).pipe(
116	114	Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
117	115	Effect.provideService(ServerSettings.ServerSettingsService, settings),
	116	// A continuation that will never run still owes a delegated parent a result.
	117	Effect.tapError(() =>
	118	willRetry
	119	? Effect.void
	120	: threads.recoverDelegatedTask(effect.threadId, sourceRunId),
	121	),
118	122	Effect.mapError(
119	123	(cause) =>
120	124	new OrchestrationEffectExecutionError({
...	...	@@ -124,6 +128,7 @@ export const executorLayer: Layer.Layer<
124	128	}),
125	129	),
126	130	);
	131	}
127	132	case "provider-session.detach":
128	133	return providerSessions
129	134	.detach({

apps/server/src/orchestration-v2/Orchestrator.ts+209-94

...	...	@@ -1,6 +1,7 @@
1	1	import {
2	2	latestExecutedRun,
3	3	latestRootProviderFailure,
	4	runRanAfter,
4	5	usageLimitBlockedRun,
5	6	} from "@t3tools/shared/orchestrationV2ThreadError";
6	7	import { threadPullRequestsOf } from "@t3tools/shared/threadPullRequests";
...	...	@@ -77,6 +78,7 @@ import { notificationTurnItem } from "./Notification.ts";
77	78	import { isRestartNoteSource } from "./RestartBackgroundNote.ts";
78	79	import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts";
79	80	import { EventSinkV2 } from "./EventSink.ts";
	81	import * as EffectOutbox from "./EffectOutbox.ts";
80	82	import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
81	83	import { IdAllocatorV2 } from "./IdAllocator.ts";
82	84	import {
...	...	@@ -242,6 +244,17 @@ export interface OrchestratorV2DispatchResult {
242	244
243	245	export interface OrchestratorV2Shape {
244	246	readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
	247	/** Startup pass that settles delegated-task results and deliveries runs left behind. */
	248	readonly recoverDelegatedTasks: Effect.Effect<void>;
	249	/** Settles a delegated child whose restart continuation of `sourceRunId` declined or failed. */
	250	readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
	251	/**
	252	* Whether a delegated child's apparent result is not final yet: a restart
	253	* continuation is pending, or the child is working again.
	254	*/
	255	readonly delegatedTaskResultPending: (
	256	childThreadId: ThreadId,
	257	) => Effect.Effect<boolean, OrchestratorProjectionError>;
245	258	readonly dispatch: (
246	259	command: OrchestrationV2ServerCommand,
247	260	) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
...	...	@@ -338,7 +351,8 @@ function wakeWorkStartedAt(
338	351	return previous === undefined ? {} : { workStartedAt: orchestrationV2RunWorkStartedAt(previous) };
339	352	}
340	353
341		function isNativeMaintenanceCommand(message: {
	354	/** A native /compact or /logout turn: provider maintenance, not agent work. */
	355	export function isNativeMaintenanceCommand(message: {
342	356	readonly text: string;
343	357	readonly attachments: ReadonlyArray<ChatAttachment>;
344	358	readonly context?: import("@t3tools/contracts").OrchestrationMessageContext \| undefined;
...	...	@@ -718,6 +732,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
718	732	const idAllocator = yield* IdAllocatorV2;
719	733	const projects = yield* ProjectStore.ProjectStoreV2;
720	734	const projectionStore = yield* ProjectionStoreV2;
	735	const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
721	736	const nextTurnItemOrdinal = (
722	737	projection: Pick<OrchestrationV2ThreadProjection, "thread"> &
723	738	Partial<Pick<OrchestrationV2ThreadProjection, "turnItems">>,
...	...	@@ -4357,7 +4372,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
4357	4372	projection.thread.archivedAt !== null \|\|
4358	4373	projection.thread.deletedAt !== null \|\|
4359	4374	projection.thread.providerInstanceId !== source.providerInstanceId \|\|
4360		projection.runs.some((run) => run.ordinal > source.ordinal)
	4375	// Held queued runs never started; they wait behind the continuation.
	4376	projection.runs.some(
	4377	(run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
	4378	)
4361	4379	) {
4362	4380	// Preserve the current row so stale automatic deliveries receive an
4363	4381	// accepted receipt without changing work or repeatedly retrying.
...	...	@@ -6594,12 +6612,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6594	6612	// policy when the child terminalizes. Both writers hold this parent lock,
6595	6613	// so a terminal task means finalize already committed the terminal row and
6596	6614	// already made its offer decision under the pre-upgrade policy: under
6597		// settled_only it offered iff the parent had no live run. Plan a delivery
6598		// only when the parent has a live run now, which is precisely the case
	6615	// settled_only it offered iff the spawning run was not live. Plan a
	6616	// delivery only when that run is live now, which is precisely the case
6599	6617	// where finalize skipped. The mailbox steers a capable active session or
6600		// queues behind that run. When the parent is not live, finalize already
6601		// offered and a second
6602		// offer would wake the parent twice. (If the parent settled in between,
	6618	// queues behind that run. When that run is not live, finalize already
	6619	// offered and a second offer would wake the parent twice. (If the parent settled in between,
6603	6620	// this skips a wake that finalize also skipped; a missed wake is cheaper
6604	6621	// than a duplicate one, and the result is already in the projection.)
6605	6622	const parentRun =
...	...	@@ -6609,7 +6626,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6609	6626	const completionPlan =
6610	6627	command.completionWake === "always" &&
6611	6628	isTerminalDelegatedTaskStatus(task.status) &&
6612		hasLiveRun(parentProjection)
	6629	parentRun !== undefined &&
	6630	hasLiveRun({ runs: [parentRun] })
6613	6631	? yield* planDelegatedCompletionDelivery({
6614	6632	parentProjection,
6615	6633	parentRun,
...	...	@@ -8494,6 +8512,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8494	8512	: undefined;
8495	8513	});
8496	8514
	8515	/**
	8516	* A run with a pending restart continuation has no outcome yet: restart
	8517	* reconciliation cancelled it mid-turn, or cancelled the background work a
	8518	* settled turn was waiting on. The continuation's own run settles it, or
	8519	* RestartContinuation recovers the thread when it declines to continue.
	8520	*/
	8521	const awaitsRestartContinuation = (run: OrchestrationV2Run) =>
	8522	effectOutbox
	8523	.get(`effect:restart-continuation:${run.id}`)
	8524	.pipe(
	8525	Effect.map(
	8526	Option.exists((effect) => effect.status === "pending" \|\| effect.status === "running"),
	8527	),
	8528	);
	8529
	8530	/**
	8531	* A delegated child's result is held while its result run, or a run after it
	8532	* that a second restart cut before it started, still has a continuation pending.
	8533	*/
	8534	const childAwaitsRestartContinuation = (
	8535	runs: ReadonlyArray<OrchestrationV2Run>,
	8536	resultRun: OrchestrationV2Run,
	8537	settledContinuationOf?: RunId,
	8538	) =>
	8539	Effect.forEach(
	8540	runs.filter(
	8541	(run) =>
	8542	run.id !== settledContinuationOf &&
	8543	(run.id === resultRun.id \|\| runRanAfter(run, resultRun)),
	8544	),
	8545	awaitsRestartContinuation,
	8546	).pipe(Effect.map((pending) => pending.includes(true)));
	8547
8497	8548	const planDelegatedCompletionDelivery = Effect.fn(
8498	8549	"orchestrationV2.planDelegatedCompletionDelivery",
8499	8550	)(function* (input: {
...	...	@@ -8541,9 +8592,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8541	8592	offer: false,
8542	8593	};
8543	8594	}
	8595	// settled_only defers to a blocking delegate_task wait, which lives only as
	8596	// long as the turn that spawned the task. Once that turn is over (a restart
	8597	// continues it as a new run), nothing else delivers the result.
8544	8598	if (
8545	8599	(input.task.completionWake ?? "settled_only") === "settled_only" &&
8546		hasLiveRun(input.parentProjection)
	8600	hasLiveRun({ runs: [input.parentRun] })
8547	8601	) {
8548	8602	return {
8549	8603	task: input.updatedTask,
...	...	@@ -8694,9 +8748,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8694	8748	* delegated_task.wake-policy handler rewrites the same subagent row under
8695	8749	* that lock with a full-row payload, and unserialized writers clobber each
8696	8750	* other (stale policy on the terminal row, or a terminal row regressed to
8697		* running).
	8751	* running). `settledContinuationOf` names the restart continuation that just
	8752	* declined or failed, so its own still-running effect does not hold the child.
8698	8753	*/
8699		const finalizeAppOwnedSubagent = (childThreadId: ThreadId) =>
	8754	const finalizeAppOwnedSubagent = (
	8755	childThreadId: ThreadId,
	8756	options?: { readonly settledContinuationOf?: RunId },
	8757	) =>
8700	8758	Effect.gen(function* () {
8701	8759	const childControls = yield* projectionStore.getThreadRecords(
8702	8760	childThreadId,
...	...	@@ -8719,6 +8777,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8719	8777	if (terminalStatus === null) {
8720	8778	return;
8721	8779	}
	8780	if (
	8781	yield* childAwaitsRestartContinuation(
	8782	childControls.runs,
	8783	childRun,
	8784	options?.settledContinuationOf,
	8785	)
	8786	) {
	8787	return;
	8788	}
8722	8789
8723	8790	const childResult = yield* projectionStore.getThreadRecords(
8724	8791	childThreadId,
...	...	@@ -9147,7 +9214,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9147	9214	});
9148	9215	return;
9149	9216	}
9150		const parentIsLive = hasLiveRun(projection);
	9217	// settled_only tasks defer only to a blocking wait in their spawning run.
	9218	const parentIsLive = hasLiveRun({ runs: [parentRun] });
9151	9219	const pendingTaskIds =
9152	9220	projection.thread.archivedAt === null && projection.thread.deletedAt === null
9153	9221	? projection.subagents
...	...	@@ -9749,97 +9817,140 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9749	9817	Effect.forkDetach,
9750	9818	);
9751	9819
9752		// Recover child results from projections. Queue recovery instead holds
9753		// unstarted runs until an explicit queue.resume command arrives.
9754		yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(
9755		Effect.flatMap((threadIds) =>
9756		Effect.forEach(
9757		threadIds,
9758		(threadId) =>
9759		Effect.gen(function* () {
9760		const thread = yield* projectionStore.getThreadShell(threadId);
9761		const parentThreadId = thread?.lineage.parentThreadId;
9762		if (parentThreadId === undefined \|\| parentThreadId === null) return;
9763		yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
9764		}).pipe(
9765		Effect.catchCause((cause) =>
9766		Effect.logWarning("Failed to recover terminal app-owned subagent", {
9767		childThreadId: threadId,
9768		cause,
9769		}),
9770		),
9771		),
9772		{ concurrency: 8, discard: true },
9773		),
9774		),
9775		Effect.catchCause((cause) =>
9776		Effect.logWarning("Failed to inspect app-owned subagents during recovery", {
9777		cause,
9778		}),
9779		),
9780		);
9781		yield* projectionStore.getRecoveryThreadIds("delegated-completions").pipe(
9782		Effect.flatMap((threadIds) =>
9783		Effect.forEach(
9784		threadIds,
9785		(threadId) =>
9786		threadDispatch
9787		.withLock(
9788		threadId,
9789		Effect.gen(function* () {
9790		const projection = yield* projectionStore.getThreadRecords(threadId, [
9791		"runs",
9792		"messages",
9793		]);
9794		const terminalDeliveryRunIds = projection.runs
9795		.filter((run) => delegatedTaskTerminalStatus(run.status) !== null)
9796		.filter((run) =>
9797		projection.messages.some(
9798		(message) =>
9799		message.id === run.userMessageId &&
9800		message.delegatedCompletion !== undefined,
9801		),
9802		)
9803		.map((run) => run.id);
9804		for (const runId of terminalDeliveryRunIds) {
9805		yield* finalizeDelegatedCompletionDelivery(threadId, runId);
9806		}
9807		const refreshed =
9808		terminalDeliveryRunIds.length === 0
9809		? projection
9810		: yield* projectionStore.getThreadRecords(threadId, ["runs", "messages"], {
9811		messageRoles: ["user"],
9812		});
9813		for (const run of refreshed.runs) {
9814		if (
9815		run.delegatedCompletion?.delivery !== null &&
9816		run.delegatedCompletion !== undefined
9817		) {
9818		yield* offerDelegatedCompletionDelivery(threadId, run.id);
9819		}
9820		}
9821		}),
9822		)
9823		.pipe(
	9820	// Settles child results and completion deliveries whose runs ended without
	9821	// the listener above: before this boot, or in runtime reconciliation, which
	9822	// it skips. Startup runs this after reconciliation and before the effect
	9823	// worker. Queue recovery instead holds unstarted runs until an explicit
	9824	// queue.resume command arrives.
	9825	const recoverDelegatedTasks = Effect.gen(function* () {
	9826	yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(
	9827	Effect.flatMap((threadIds) =>
	9828	Effect.forEach(
	9829	threadIds,
	9830	(threadId) =>
	9831	Effect.gen(function* () {
	9832	const thread = yield* projectionStore.getThreadShell(threadId);
	9833	const parentThreadId = thread?.lineage.parentThreadId;
	9834	if (parentThreadId === undefined \|\| parentThreadId === null) return;
	9835	yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
	9836	}).pipe(
9824	9837	Effect.catchCause((cause) =>
9825		Effect.logWarning("Failed to recover delegated completion delivery", {
9826		threadId,
	9838	Effect.logWarning("Failed to recover terminal app-owned subagent", {
	9839	childThreadId: threadId,
9827	9840	cause,
9828	9841	}),
9829	9842	),
9830	9843	),
9831		{ concurrency: 8, discard: true },
	9844	{ concurrency: 8, discard: true },
	9845	),
9832	9846	),
9833		),
9834		Effect.catchCause((cause) =>
9835		Effect.logWarning("Failed to inspect delegated completion delivery during recovery", {
9836		cause,
9837		}),
9838		),
9839		);
	9847	Effect.catchCause((cause) =>
	9848	Effect.logWarning("Failed to inspect app-owned subagents during recovery", {
	9849	cause,
	9850	}),
	9851	),
	9852	);
	9853	yield* projectionStore.getRecoveryThreadIds("delegated-completions").pipe(
	9854	Effect.flatMap((threadIds) =>
	9855	Effect.forEach(
	9856	threadIds,
	9857	(threadId) =>
	9858	threadDispatch
	9859	.withLock(
	9860	threadId,
	9861	Effect.gen(function* () {
	9862	const projection = yield* projectionStore.getThreadRecords(threadId, [
	9863	"runs",
	9864	"messages",
	9865	]);
	9866	const terminalDeliveryRunIds = projection.runs
	9867	.filter((run) => delegatedTaskTerminalStatus(run.status) !== null)
	9868	.filter((run) =>
	9869	projection.messages.some(
	9870	(message) =>
	9871	message.id === run.userMessageId &&
	9872	message.delegatedCompletion !== undefined,
	9873	),
	9874	)
	9875	.map((run) => run.id);
	9876	for (const runId of terminalDeliveryRunIds) {
	9877	yield* finalizeDelegatedCompletionDelivery(threadId, runId);
	9878	}
	9879	const refreshed =
	9880	terminalDeliveryRunIds.length === 0
	9881	? projection
	9882	: yield* projectionStore.getThreadRecords(threadId, ["runs", "messages"], {
	9883	messageRoles: ["user"],
	9884	});
	9885	for (const run of refreshed.runs) {
	9886	if (
	9887	run.delegatedCompletion?.delivery !== null &&
	9888	run.delegatedCompletion !== undefined
	9889	) {
	9890	yield* offerDelegatedCompletionDelivery(threadId, run.id);
	9891	}
	9892	}
	9893	}),
	9894	)
	9895	.pipe(
	9896	Effect.catchCause((cause) =>
	9897	Effect.logWarning("Failed to recover delegated completion delivery", {
	9898	threadId,
	9899	cause,
	9900	}),
	9901	),
	9902	),
	9903	{ concurrency: 8, discard: true },
	9904	),
	9905	),
	9906	Effect.catchCause((cause) =>
	9907	Effect.logWarning("Failed to inspect delegated completion delivery during recovery", {
	9908	cause,
	9909	}),
	9910	),
	9911	);
	9912	});
	9913
	9914	const delegatedTaskResultPending = (childThreadId: ThreadId) =>
	9915	Effect.gen(function* () {
	9916	const child = yield* projectionStore.getThreadRecords(
	9917	childThreadId,
	9918	["runs", "messages", "subagents", "providerThreads", "providerTurns", "attempts"],
	9919	{ messageRoles: ["user"] },
	9920	);
	9921	const progress = delegatedTaskProgress(child);
	9922	// A caller's older read saw a result; newer work since then means it is not final.
	9923	if (progress.state !== "result_available") return true;
	9924	if (progress.resultRun === undefined) return false;
	9925	return yield* childAwaitsRestartContinuation(child.runs, progress.resultRun);
	9926	}).pipe(
	9927	Effect.mapError(
	9928	(cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
	9929	),
	9930	);
	9931
	9932	const recoverDelegatedTask = (threadId: ThreadId, sourceRunId: RunId) =>
	9933	Effect.gen(function* () {
	9934	const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
	9935	if (parentThreadId === undefined) return;
	9936	yield* threadDispatch.withLock(
	9937	parentThreadId,
	9938	finalizeAppOwnedSubagent(threadId, { settledContinuationOf: sourceRunId }),
	9939	);
	9940	}).pipe(
	9941	Effect.catchCause((cause) =>
	9942	Effect.logWarning("Failed to recover delegated task after restart", {
	9943	childThreadId: threadId,
	9944	cause,
	9945	}),
	9946	),
	9947	);
9840	9948
9841	9949	return OrchestratorV2.of({
9842	9950	resumeQueuedRuns,
	9951	recoverDelegatedTasks,
	9952	recoverDelegatedTask,
	9953	delegatedTaskResultPending,
9843	9954	dispatch: dispatchWithReceipt,
9844	9955	getTimelinePage: (threadId, options) =>
9845	9956	projectionStore
...	...	@@ -9933,6 +10044,7 @@ export const layer: Layer.Layer<
9933	10044	\| CommandPolicyV2
9934	10045	\| CommandReceiptStoreV2
9935	10046	\| ContextHandoffServiceV2
	10047	\| EffectOutbox.EffectOutboxV2
9936	10048	\| EventSinkV2
9937	10049	\| IdAllocatorV2
9938	10050	\| ProjectStore.ProjectStoreV2
...	...	@@ -9956,6 +10068,9 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
9956	10068	cause: "Orchestration V2 live runtime is not configured.",
9957	10069	}),
9958	10070	),
	10071	recoverDelegatedTasks: Effect.void,
	10072	recoverDelegatedTask: () => Effect.void,
	10073	delegatedTaskResultPending: () => Effect.succeed(false),
9959	10074	dispatch: (command) =>
9960	10075	Effect.fail(
9961	10076	new OrchestratorDispatchError({

apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+67

...	...	@@ -420,6 +420,73 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
420	420	}).pipe(Effect.provide(TestLayer)),
421	421	);
422	422
	423	it.effect("reads the run that owns a background roster, not a queued or resumed one", () =>
	424	Effect.gen(function* () {
	425	const projections = yield* ProjectionStore.ProjectionStoreV2;
	426	const now = yield* DateTime.now;
	427	const withRoster = Effect.fn(function* (threadId: ThreadId) {
	428	yield* projections.apply({
	429	id: EventId.make(`event:${threadId}:roster`),
	430	type: "provider-thread.updated",
	431	threadId,
	432	driver,
	433	providerInstanceId,
	434	occurredAt: now,
	435	payload: {
	436	id: ProviderThreadId.make(`provider-thread:${threadId}`),
	437	appThreadId: threadId,
	438	ownerNodeId: null,
	439	driver,
	440	providerInstanceId,
	441	providerSessionId: null,
	442	nativeThreadRef: null,
	443	nativeConversationHeadRef: null,
	444	status: "idle",
	445	firstRunOrdinal: null,
	446	lastRunOrdinal: null,
	447	handoffIds: [],
	448	forkedFrom: null,
	449	createdAt: now,
	450	updatedAt: now,
	451	pendingBackgroundTasks: [
	452	{ taskId: "background", description: "Still running", kind: "command" },
	453	],
	454	},
	455	});
	456	});
	457	const runIds = (threadId: ThreadId) =>
	458	projections
	459	.getRuntimeRecoveryProjection(threadId)
	460	.pipe(Effect.map((state) => state.runs.map((run) => run.id)));
	461	// Settled with background work left, then a queued follow-up.
	462	const settled = yield* createThread("roster-before-queue");
	463	const settledRun = yield* createRun(settled, "completed", {
	464	providerThreadId: ProviderThreadId.make(`provider-thread:${settled}`),
	465	});
	466	const queuedRun = yield* createRun(settled, "queued", { ordinal: 2, completedAt: null });
	467	yield* withRoster(settled);
	468	assert.deepEqual(yield* runIds(settled), [settledRun, queuedRun]);
	469	// A resumed queued run (ordinal 1) ended after a continuation (ordinal 2).
	470	const resumed = yield* createThread("roster-after-resume");
	471	const resumedRun = yield* createRun(resumed, "completed", {
	472	providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
	473	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
	474	});
	475	const continuationRun = yield* createRun(resumed, "completed", {
	476	providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
	477	ordinal: 2,
	478	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
	479	});
	480	yield* withRoster(resumed);
	481	assert.deepEqual(yield* runIds(resumed), [resumedRun, continuationRun]);
	482	// Without a roster, settled history is not read.
	483	const quiet = yield* createThread("no-roster-before-queue");
	484	yield* createRun(quiet, "completed");
	485	const quietQueued = yield* createRun(quiet, "queued", { ordinal: 2, completedAt: null });
	486	assert.deepEqual(yield* runIds(quiet), [quietQueued]);
	487	}).pipe(Effect.provide(TestLayer)),
	488	);
	489
423	490	it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>
424	491	Effect.gen(function* () {
425	492	const projections = yield* ProjectionStore.ProjectionStoreV2;

apps/server/src/orchestration-v2/ProjectionStore.test.ts+55

...	...	@@ -2276,8 +2276,33 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {
2276	2276	},
2277	2277	});
2278	2278	yield* assertSummary("Plan limit reached.", "usage_limit");
	2279	// A restart continuation ran ahead of the held queue and ended before the
	2280	// resumed failed run: the failure is still the latest executed run.
	2281	const aheadRunId = RunId.make("run:limit-shell:ran-ahead");
	2282	yield* store.apply({
	2283	id: EventId.make("event:limit-shell:ran-ahead"),
	2284	type: "run.created",
	2285	threadId,
	2286	runId: aheadRunId,
	2287	nodeId: NodeId.make("node:limit-shell:ran-ahead"),
	2288	driver,
	2289	providerInstanceId,
	2290	occurredAt: now,
	2291	payload: {
	2292	...original,
	2293	id: aheadRunId,
	2294	ordinal: original.ordinal + 3,
	2295	rootNodeId: NodeId.make("node:limit-shell:ran-ahead"),
	2296	userMessageId: MessageId.make("message:limit-shell:ran-ahead"),
	2297	status: "completed",
	2298	startedAt: DateTime.subtract(now, { minutes: 10 }),
	2299	completedAt: DateTime.subtract(now, { minutes: 5 }),
	2300	},
	2301	});
	2302	yield* assertSummary("Plan limit reached.", "usage_limit");
2279	2303	const sql = yield* SqlClient.SqlClient;
2280	2304	// The rest of this case treats the failed run as the latest run.
	2305	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`;
2281	2306	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`;
2282	2307	yield* assertSummary("Plan limit reached.", "usage_limit");
2283	2308	yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${cancelledRunId}`;
...	...	@@ -4372,6 +4397,36 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {
4372	4397	released && projectThreadAwarenessV2({ environmentId, project, thread: released });
4373	4398	assert.equal(state?.phase, "completed");
4374	4399	assert.equal(state?.updatedAt, DateTime.formatIso(releasedAt));
	4400
	4401	// A later provider must not hide this owner's roster from restart recovery.
	4402	yield* store.apply({
	4403	id: EventId.make("event:held-completion:monitor-restarted"),
	4404	type: "provider-thread.updated",
	4405	threadId,
	4406	occurredAt: releasedAt,
	4407	payload: { ...providerThread, updatedAt: releasedAt },
	4408	});
	4409	const newerRunId = RunId.make("run:held-completion:new-provider");
	4410	yield* store.apply({
	4411	id: EventId.make("event:held-completion:new-provider"),
	4412	type: "run.created",
	4413	threadId,
	4414	runId: newerRunId,
	4415	occurredAt: releasedAt,
	4416	payload: {
	4417	...(yield* store.getThreadProjection(threadId)).runs[0]!,
	4418	id: newerRunId,
	4419	ordinal: 2,
	4420	providerThreadId: ProviderThreadId.make("provider-thread:held-completion:new-provider"),
	4421	requestedAt: releasedAt,
	4422	startedAt: releasedAt,
	4423	completedAt: releasedAt,
	4424	},
	4425	});
	4426	assert.deepEqual(
	4427	(yield* store.getRuntimeRecoveryProjection(threadId)).runs.map((run) => run.id),
	4428	[runId, newerRunId],
	4429	);
4375	4430	}),
4376	4431	);
4377	4432	});

apps/server/src/orchestration-v2/ProjectionStore.ts+39-5

...	...	@@ -522,8 +522,10 @@ function needsRecovery(
522	522	}
523	523	case "runtime":
524	524	return (
525		projection.runs.some((run) =>
526		["queued", "preparing", "starting", "running", "waiting"].includes(run.status),
	525	projection.runs.some(
	526	(run) =>
	527	["preparing", "starting", "running", "waiting"].includes(run.status) \|\|
	528	(run.status === "queued" && run.queueHeld !== true),
527	529	) \|\|
528	530	projection.runtimeRequests.some((request) => request.status === "pending") \|\|
529	531	projection.providerSessions.some(
...	...	@@ -3307,7 +3309,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3307	3309	latest.status = 'cancelled'
3308	3310	AND json_extract(latest.payload_json, '$.startedAt') IS NULL
3309	3311	)
3310		ORDER BY latest.ordinal DESC, latest.run_id DESC LIMIT 1
	3312	-- latestExecutedRun: the run that ended last (runRanAfter).
	3313	ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC,
	3314	latest.ordinal DESC, latest.run_id DESC
	3315	LIMIT 1
3311	3316	) AND r.status = 'failed'
3312	3317	INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = (
3313	3318	SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error
...	...	@@ -3433,7 +3438,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3433	3438	ELSE 0 END
3434	3439	)
3435	3440	SELECT thread_id FROM orchestration_v2_projection_runs
3436		WHERE status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
	3441	WHERE status IN ('preparing', 'starting', 'running', 'waiting')
	3442	UNION
	3443	-- A held queue already went through recovery; rereading it on
	3444	-- every boot costs a projection read per held thread.
	3445	SELECT thread_id FROM orchestration_v2_projection_runs
	3446	WHERE status = 'queued'
	3447	AND CASE WHEN json_valid(payload_json)
	3448	THEN json_extract(payload_json, '$.queueHeld') IS NOT 1
	3449	ELSE 1 END
3437	3450	UNION
3438	3451	SELECT thread_id FROM orchestration_v2_projection_runtime_requests
3439	3452	WHERE status = 'pending'
...	...	@@ -3805,6 +3818,24 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3805	3818	WHERE latest.thread_id = ${threadId}
3806	3819	ORDER BY latest.ordinal DESC LIMIT 1
3807	3820	)
	3821	-- Retain the latest run for each provider thread with lost
	3822	-- background work, including owners used before a handoff.
	3823	OR run.run_id IN (
	3824	SELECT (
	3825	SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended
	3826	WHERE ended.thread_id = ${threadId}
	3827	AND ended.provider_thread_id = roster.provider_thread_id
	3828	AND ended.status NOT IN ('queued', 'rolled_back')
	3829	ORDER BY ended.completed_at IS NULL DESC, ended.completed_at DESC,
	3830	ended.ordinal DESC
	3831	LIMIT 1
	3832	)
	3833	FROM orchestration_v2_projection_provider_threads AS roster
	3834	WHERE roster.thread_id = ${threadId}
	3835	AND CASE WHEN json_valid(roster.payload_json)
	3836	THEN json_array_length(roster.payload_json, '$.pendingBackgroundTasks') > 0
	3837	ELSE 0 END
	3838	)
3808	3839	OR run.run_id IN (
3809	3840	SELECT item.run_id FROM orchestration_v2_projection_turn_items AS item
3810	3841	WHERE item.thread_id = ${threadId}
...	...	@@ -4899,7 +4930,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4899	4930	candidate.status = 'cancelled'
4900	4931	AND json_extract(candidate.payload_json, '$.startedAt') IS NULL
4901	4932	)
4902		ORDER BY candidate.ordinal DESC, candidate.run_id DESC LIMIT 1
	4933	-- latestExecutedRun: the run that ended last (runRanAfter).
	4934	ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC,
	4935	candidate.ordinal DESC, candidate.run_id DESC
	4936	LIMIT 1
4903	4937	) AND blocked.status = 'failed'
4904	4938	WHERE t.deleted_at IS NULL${threadId === undefined ? sql`` : sql` AND t.thread_id = ${threadId}`}${
4905	4939	location === "active"

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryPerformance.test.ts created+510

...	...	@@ -0,0 +1,510 @@
	1	import { assert, it } from "@effect/vitest";
	2	import {
	3	CheckpointScopeId,
	4	CommandId,
	5	EventId,
	6	MessageId,
	7	NodeId,
	8	type OrchestrationV2AppThread,
	9	type OrchestrationV2DomainEvent,
	10	type OrchestrationV2Run,
	11	ProjectId,
	12	ProviderDriverKind,
	13	ProviderInstanceId,
	14	ProviderSessionId,
	15	ProviderThreadId,
	16	ProviderTurnId,
	17	RunAttemptId,
	18	RunId,
	19	ThreadId,
	20	TurnItemId,
	21	} from "@t3tools/contracts";
	22	import * as Console from "effect/Console";
	23	import * as DateTime from "effect/DateTime";
	24	import * as Effect from "effect/Effect";
	25	import * as Layer from "effect/Layer";
	26	import * as SqlClient from "effect/unstable/sql/SqlClient";
	27
	28	import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts";
	29	import * as ServerSettings from "../serverSettings.ts";
	30	import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
	31	import * as EffectOutbox from "./EffectOutbox.ts";
	32	import * as EventSink from "./EventSink.ts";
	33	import * as EventStore from "./EventStore.ts";
	34	import * as IdAllocator from "./IdAllocator.ts";
	35	import * as ProjectionStore from "./ProjectionStore.ts";
	36	import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
	37
	38	// Startup recovery cost for restart continuation. The default case is small
	39	// enough for CI and asserts correctness; T3_BENCH_RECOVERY=1 adds the matrix.
	40	// `it.live` keeps a real clock: each reconcile gets a fresh command id, so the
	41	// second recover cannot hide behind command receipt dedup.
	42
	43	const stores = Layer.mergeAll(
	44	EventStore.layer,
	45	ProjectionStore.layer,
	46	EffectOutbox.layer,
	47	IdAllocator.layer,
	48	).pipe(Layer.provideMerge(SqlitePersistenceMemory));
	49	const TestLayer = ProviderRuntimeRecovery.layer.pipe(
	50	Layer.provideMerge(EventSink.layer.pipe(Layer.provideMerge(stores))),
	51	Layer.provideMerge(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })),
	52	);
	53
	54	const providerInstanceId = ProviderInstanceId.make("codex");
	55	const driver = ProviderDriverKind.make("codex");
	56	const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
	57	const projectId = ProjectId.make("project:recovery-bench");
	58	const ITEMS_PER_SETTLED_THREAD = 10;
	59	const OPEN_ITEMS_PER_ACTIVE_THREAD = 3;
	60	const WAITING_THREADS = 3;
	61	const SEED_BATCH = 500;
	62
	63	type UnstampedEvent = OrchestrationV2DomainEvent extends infer Event
	64	? Event extends OrchestrationV2DomainEvent
	65	? Omit<Event, "id" \| "occurredAt">
	66	: never
	67	: never;
	68
	69	interface Scenario {
	70	readonly settled: number;
	71	readonly active: number;
	72	}
	73
	74	interface Seeded {
	75	readonly unheldQueuedThreads: number;
	76	readonly waitingThreads: number;
	77	}
	78
	79	const seedScenario = Effect.fn(function* (scenario: Scenario) {
	80	const projections = yield* ProjectionStore.ProjectionStoreV2;
	81	const outbox = yield* EffectOutbox.EffectOutboxV2;
	82	const sql = yield* SqlClient.SqlClient;
	83	const now = yield* DateTime.now;
	84	let eventIndex = 0;
	85	const apply = (event: UnstampedEvent) =>
	86	projections.apply({
	87	...event,
	88	id: EventId.make(`event:bench:${eventIndex++}`),
	89	occurredAt: now,
	90	} as OrchestrationV2DomainEvent);
	91
	92	const thread = (id: ThreadId, overrides: Partial<OrchestrationV2AppThread> = {}) =>
	93	apply({
	94	type: "thread.created",
	95	threadId: id,
	96	payload: {
	97	createdBy: "user",
	98	creationSource: "web",
	99	id,
	100	projectId,
	101	title: id,
	102	providerInstanceId,
	103	modelSelection,
	104	runtimeMode: "full-access",
	105	interactionMode: "default",
	106	branch: null,
	107	worktreePath: null,
	108	activeProviderThreadId: null,
	109	lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: id },
	110	forkedFrom: null,
	111	createdAt: now,
	112	updatedAt: now,
	113	archivedAt: null,
	114	settledOverride: null,
	115	settledAt: null,
	116	lastVisitedAt: null,
	117	deletedAt: null,
	118	...overrides,
	119	},
	120	});
	121
	122	const run = (
	123	threadId: ThreadId,
	124	ordinal: number,
	125	status: OrchestrationV2Run["status"],
	126	overrides: Partial<OrchestrationV2Run> = {},
	127	) => {
	128	const runId = RunId.make(`run:${threadId}:${ordinal}`);
	129	return apply({
	130	type: "run.created",
	131	threadId,
	132	runId,
	133	payload: {
	134	id: runId,
	135	threadId,
	136	ordinal,
	137	providerInstanceId,
	138	modelSelection,
	139	providerThreadId: null,
	140	userMessageId: MessageId.make(`message:${runId}`),
	141	rootNodeId: null,
	142	activeAttemptId: null,
	143	status,
	144	requestedAt: now,
	145	startedAt: status === "queued" ? null : now,
	146	completedAt: status === "completed" ? now : null,
	147	checkpointId: null,
	148	contextHandoffId: null,
	149	...overrides,
	150	},
	151	}).pipe(Effect.as(runId));
	152	};
	153
	154	const commandItem = (input: {
	155	readonly threadId: ThreadId;
	156	readonly runId: RunId;
	157	readonly ordinal: number;
	158	readonly status: "running" \| "completed";
	159	readonly nodeId?: NodeId;
	160	readonly providerThreadId?: ProviderThreadId;
	161	readonly providerTurnId?: ProviderTurnId;
	162	}) =>
	163	apply({
	164	type: "turn-item.updated",
	165	threadId: input.threadId,
	166	runId: input.runId,
	167	payload: {
	168	id: TurnItemId.make(`item:${input.runId}:${input.ordinal}`),
	169	threadId: input.threadId,
	170	runId: input.runId,
	171	nodeId: input.nodeId ?? null,
	172	providerThreadId: input.providerThreadId ?? null,
	173	providerTurnId: input.providerTurnId ?? null,
	174	nativeItemRef: null,
	175	parentItemId: null,
	176	ordinal: input.ordinal,
	177	type: "command_execution",
	178	status: input.status,
	179	title: `Command ${input.ordinal}`,
	180	input: `echo ${input.ordinal}`,
	181	output: input.status === "completed" ? `${input.ordinal}\n` : "",
	182	...(input.status === "completed" ? { exitCode: 0 } : {}),
	183	startedAt: now,
	184	completedAt: input.status === "completed" ? now : null,
	185	updatedAt: now,
	186	},
	187	});
	188
	189	const settledThread = Effect.fn(function* (index: number) {
	190	const threadId = ThreadId.make(`thread:bench:settled:${index}`);
	191	yield* thread(threadId);
	192	const runId = yield* run(threadId, 1, "completed");
	193	for (let ordinal = 1; ordinal <= ITEMS_PER_SETTLED_THREAD; ordinal += 1) {
	194	yield* commandItem({ threadId, runId, ordinal, status: "completed" });
	195	}
	196	// ~5% of threads carry a queued follow-up; half are already held, and an
	197	// earlier recovery already handled those, so only unheld ones are candidates.
	198	if (index % 20 === 0) {
	199	const queueHeld = index % 40 === 0;
	200	yield* run(threadId, 2, "queued", { queueHeld });
	201	return !queueHeld;
	202	}
	203	return false;
	204	});
	205
	206	const activeThread = Effect.fn(function* (index: number) {
	207	const threadId = ThreadId.make(`thread:bench:active:${index}`);
	208	const providerSessionId = ProviderSessionId.make(`session:bench:${index}`);
	209	const providerThreadId = ProviderThreadId.make(`provider-thread:bench:${index}`);
	210	const runId = RunId.make(`run:${threadId}:1`);
	211	const attemptId = RunAttemptId.make(`attempt:bench:${index}`);
	212	const nodeId = NodeId.make(`node:bench:${index}`);
	213	const providerTurnId = ProviderTurnId.make(`provider-turn:bench:${index}`);
	214	yield* thread(threadId, { activeProviderThreadId: providerThreadId });
	215	yield* apply({
	216	type: "provider-session.attached",
	217	threadId,
	218	driver,
	219	providerInstanceId,
	220	payload: {
	221	id: providerSessionId,
	222	driver,
	223	providerInstanceId,
	224	status: "ready",
	225	cwd: "/workspace",
	226	model: modelSelection.model,
	227	capabilities: CodexProviderCapabilitiesV2,
	228	createdAt: now,
	229	updatedAt: now,
	230	lastError: null,
	231	},
	232	});
	233	yield* apply({
	234	type: "provider-thread.updated",
	235	threadId,
	236	driver,
	237	providerInstanceId,
	238	payload: {
	239	id: providerThreadId,
	240	appThreadId: threadId,
	241	ownerNodeId: null,
	242	driver,
	243	providerInstanceId,
	244	providerSessionId,
	245	nativeThreadRef: { driver, nativeId: `native:bench:${index}`, strength: "strong" },
	246	nativeConversationHeadRef: null,
	247	status: "active",
	248	firstRunOrdinal: 1,
	249	lastRunOrdinal: 1,
	250	handoffIds: [],
	251	forkedFrom: null,
	252	createdAt: now,
	253	updatedAt: now,
	254	pendingBackgroundTasks: [],
	255	},
	256	});
	257	yield* run(threadId, 1, "running", {
	258	providerThreadId,
	259	rootNodeId: nodeId,
	260	activeAttemptId: attemptId,
	261	});
	262	yield* apply({
	263	type: "run-attempt.created",
	264	threadId,
	265	runId,
	266	payload: {
	267	id: attemptId,
	268	runId,
	269	attemptOrdinal: 1,
	270	rootNodeId: nodeId,
	271	providerInstanceId,
	272	providerThreadId,
	273	providerTurnId,
	274	reason: "initial",
	275	status: "running",
	276	startedAt: now,
	277	completedAt: null,
	278	},
	279	});
	280	yield* apply({
	281	type: "node.updated",
	282	threadId,
	283	runId,
	284	nodeId,
	285	providerInstanceId,
	286	payload: {
	287	id: nodeId,
	288	threadId,
	289	runId,
	290	parentNodeId: null,
	291	rootNodeId: nodeId,
	292	kind: "root_turn",
	293	status: "running",
	294	countsForRun: true,
	295	providerThreadId,
	296	providerTurnId,
	297	nativeItemRef: null,
	298	runtimeRequestId: null,
	299	checkpointScopeId: null,
	300	startedAt: now,
	301	completedAt: null,
	302	},
	303	});
	304	yield* apply({
	305	type: "provider-turn.updated",
	306	threadId,
	307	runId,
	308	nodeId,
	309	providerInstanceId,
	310	payload: {
	311	id: providerTurnId,
	312	providerThreadId,
	313	nodeId,
	314	runAttemptId: attemptId,
	315	nativeTurnRef: { driver, nativeId: `native-turn:bench:${index}`, strength: "strong" },
	316	ordinal: 1,
	317	status: "running",
	318	startedAt: now,
	319	completedAt: null,
	320	},
	321	});
	322	for (let ordinal = 1; ordinal <= OPEN_ITEMS_PER_ACTIVE_THREAD; ordinal += 1) {
	323	yield* commandItem({
	324	threadId,
	325	runId,
	326	ordinal,
	327	status: "running",
	328	nodeId,
	329	providerThreadId,
	330	providerTurnId,
	331	});
	332	}
	333	});
	334
	335	const waitingThread = Effect.fn(function* (index: number) {
	336	const threadId = ThreadId.make(`thread:bench:waiting:${index}`);
	337	yield* thread(threadId);
	338	yield* run(threadId, 1, "completed");
	339	const runId = yield* run(threadId, 2, "waiting");
	340	yield* outbox.enqueue([
	341	{
	342	id: `effect:bench:checkpoint:${runId}`,
	343	commandId: CommandId.make(`command:effect:checkpoint.capture:${runId}`),
	344	threadId,
	345	request: {
	346	type: "checkpoint.capture",
	347	runId,
	348	scopeId: CheckpointScopeId.make(`scope:bench:${index}`),
	349	},
	350	},
	351	]);
	352	});
	353
	354	let unheldQueuedThreads = 0;
	355	for (let start = 0; start < scenario.settled; start += SEED_BATCH) {
	356	const end = Math.min(scenario.settled, start + SEED_BATCH);
	357	yield* sql.withTransaction(
	358	Effect.gen(function* () {
	359	for (let index = start; index < end; index += 1) {
	360	if (yield* settledThread(index)) unheldQueuedThreads += 1;
	361	}
	362	}),
	363	);
	364	}
	365	yield* sql.withTransaction(
	366	Effect.gen(function* () {
	367	for (let index = 0; index < scenario.active; index += 1) yield* activeThread(index);
	368	for (let index = 0; index < WAITING_THREADS; index += 1) yield* waitingThread(index);
	369	}),
	370	);
	371	return { unheldQueuedThreads, waitingThreads: WAITING_THREADS } satisfies Seeded;
	372	});
	373
	374	const timed = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
	375	Effect.gen(function* () {
	376	const start = performance.now();
	377	const value = yield* effect;
	378	return [value, performance.now() - start] as const;
	379	});
	380
	381	const continuationEffectCount = Effect.gen(function* () {
	382	const sql = yield* SqlClient.SqlClient;
	383	const rows = yield* sql<{ readonly count: number }>`
	384	SELECT COUNT(*) AS count FROM orchestration_v2_effect_outbox
	385	WHERE effect_id LIKE 'effect:restart-continuation:%' AND status = 'pending'
	386	`;
	387	return Number(rows[0]?.count ?? 0);
	388	});
	389
	390	interface Measurement {
	391	readonly settled: number;
	392	readonly active: number;
	393	readonly seedMs: number;
	394	readonly candidates: number;
	395	readonly selectMs: number;
	396	readonly prepareMs: number;
	397	readonly shutdownMs: number;
	398	readonly startupAfterShutdownMs: number;
	399	readonly crashRecoverMs: number;
	400	readonly secondRecoverMs: number;
	401	}
	402
	403	/** A graceful restart: prepare intent, reconcile on shutdown, recover on boot. */
	404	const measureGraceful = Effect.fn(function* (scenario: Scenario) {
	405	const [seeded, seedMs] = yield* timed(seedScenario(scenario));
	406	const projections = yield* ProjectionStore.ProjectionStoreV2;
	407	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
	408	const [candidates, selectMs] = yield* timed(projections.getRecoveryThreadIds("runtime"));
	409	assert.equal(
	410	candidates.length,
	411	scenario.active + seeded.unheldQueuedThreads + seeded.waitingThreads,
	412	"runtime recovery candidates",
	413	);
	414	const [, prepareMs] = yield* timed(recovery.prepareForShutdown);
	415	assert.equal(yield* continuationEffectCount, scenario.active, "prepared continuations");
	416	const [, shutdownMs] = yield* timed(recovery.reconcile("shutdown"));
	417	const [, startupAfterShutdownMs] = yield* timed(recovery.recover);
	418	assert.equal(yield* continuationEffectCount, scenario.active, "continuations after boot");
	419	return {
	420	seedMs,
	421	candidates: candidates.length,
	422	selectMs,
	423	prepareMs,
	424	shutdownMs,
	425	startupAfterShutdownMs,
	426	};
	427	});
	428
	429	/** A crash: no shutdown hook ran, so boot recovery records the continuations. */
	430	const measureCrash = Effect.fn(function* (scenario: Scenario) {
	431	yield* seedScenario(scenario);
	432	const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
	433	const [summary, crashRecoverMs] = yield* timed(recovery.recover);
	434	assert.equal(summary.terminalizedRuns, scenario.active, "terminalized active runs");
	435	assert.equal(summary.stoppedSessions, scenario.active, "stopped active sessions");
	436	assert.equal(yield* continuationEffectCount, scenario.active, "recorded continuations");
	437	const [, secondRecoverMs] = yield* timed(recovery.recover);
	438	assert.equal(yield* continuationEffectCount, scenario.active, "second recover is idempotent");
	439	return { crashRecoverMs, secondRecoverMs };
	440	});
	441
	442	const measure = Effect.fn(function* (scenario: Scenario) {
	443	const graceful = yield* measureGraceful(scenario).pipe(Effect.provide(Layer.fresh(TestLayer)));
	444	const crash = yield* measureCrash(scenario).pipe(Effect.provide(Layer.fresh(TestLayer)));
	445	return { ...scenario, ...graceful, ...crash } satisfies Measurement;
	446	});
	447
	448	const formatTable = (rows: ReadonlyArray<Measurement>) => {
	449	const ms = (value: number) => value.toFixed(1);
	450	const perActive = (value: number, active: number) => (value / Math.max(active, 1)).toFixed(2);
	451	const header = [
	452	"N",
	453	"A",
	454	"seed",
	455	"cand",
	456	"select",
	457	"prepare",
	458	"prep/A",
	459	"shutdown",
	460	"boot-after",
	461	"crash-recover",
	462	"recover/A",
	463	"recover-2",
	464	];
	465	const lines = rows.map((row) => [
	466	String(row.settled),
	467	String(row.active),
	468	ms(row.seedMs),
	469	String(row.candidates),
	470	ms(row.selectMs),
	471	ms(row.prepareMs),
	472	perActive(row.prepareMs, row.active),
	473	ms(row.shutdownMs),
	474	ms(row.startupAfterShutdownMs),
	475	ms(row.crashRecoverMs),
	476	perActive(row.crashRecoverMs, row.active),
	477	ms(row.secondRecoverMs),
	478	]);
	479	const widths = header.map((cell, column) =>
	480	Math.max(cell.length, ...lines.map((line) => line[column]!.length)),
	481	);
	482	return [header, ...lines]
	483	.map((line) => line.map((cell, column) => cell.padStart(widths[column]!)).join(" "))
	484	.join("\n");
	485	};
	486
	487	it.live(
	488	"restart recovery records one continuation per active thread and is idempotent",
	489	() =>
	490	Effect.gen(function* () {
	491	const row = yield* measure({ settled: 500, active: 20 });
	492	yield* Console.log(`provider runtime recovery (ms)\n${formatTable([row])}`);
	493	}),
	494	60_000,
	495	);
	496
	497	it.live.skipIf(process.env.T3_BENCH_RECOVERY !== "1")(
	498	"restart recovery scales with active threads, not settled history",
	499	() =>
	500	Effect.gen(function* () {
	501	const rows: Array<Measurement> = [];
	502	for (const settled of [5_000, 50_000]) {
	503	for (const active of [10, 100, 1_000]) {
	504	rows.push(yield* measure({ settled, active }));
	505	yield* Console.log(`provider runtime recovery (ms)\n${formatTable(rows)}`);
	506	}
	507	}
	508	}),
	509	3_600_000,
	510	);

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts+148

...	...	@@ -1272,3 +1272,151 @@ it.effect(
1272	1272	}).pipe(Effect.provide(layer));
1273	1273	},
1274	1274	);
	1275
	1276	it.effect("leaves delegated tasks to their own child threads after process loss", () => {
	1277	const threadId = ThreadId.make("thread_recovery_delegation");
	1278	const settledRunId = RunId.make("run_recovery_delegation_settled");
	1279	const runningRunId = RunId.make("run_recovery_delegation_running");
	1280	const providerThreadId = ProviderThreadId.make("provider_thread_recovery_delegation");
	1281	const claudeInstanceId = ProviderInstanceId.make("claude");
	1282	const driver = ProviderDriverKind.make("claude");
	1283	const settledTaskId = NodeId.make("node_recovery_delegation_settled");
	1284	const runningTaskId = NodeId.make("node_recovery_delegation_running");
	1285	const nativeSubagentId = NodeId.make("node_recovery_native_subagent");
	1286	const delegatedTask = (id: NodeId, runId: RunId) => ({
	1287	id,
	1288	runId,
	1289	origin: "app_owned",
	1290	childThreadId: ThreadId.make(`thread:delegated-task:${id}`),
	1291	driver,
	1292	providerInstanceId: claudeInstanceId,
	1293	status: "running",
	1294	});
	1295	const subagentItem = (subagentId: NodeId, runId: RunId, origin: string) => ({
	1296	id: TurnItemId.make(`turn_item:${subagentId}`),
	1297	runId,
	1298	nodeId: subagentId,
	1299	providerThreadId,
	1300	type: "subagent",
	1301	status: "running",
	1302	subagentId,
	1303	origin,
	1304	childThreadId:
	1305	origin === "app_owned" ? ThreadId.make(`thread:delegated-task:${subagentId}`) : null,
	1306	providerInstanceId: claudeInstanceId,
	1307	title: `subagent ${subagentId}`,
	1308	});
	1309	let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] \| null =
	1310	null;
	1311	const projection = {
	1312	thread: { id: threadId, providerInstanceId: claudeInstanceId },
	1313	runtimeRequests: [],
	1314	providerSessions: [],
	1315	providerThreads: [
	1316	{
	1317	id: providerThreadId,
	1318	driver,
	1319	providerInstanceId: claudeInstanceId,
	1320	ownerNodeId: null,
	1321	status: "idle",
	1322	pendingBackgroundTasks: [],
	1323	},
	1324	],
	1325	providerTurns: [],
	1326	runs: [
	1327	{
	1328	id: settledRunId,
	1329	ordinal: 1,
	1330	status: "completed",
	1331	providerThreadId,
	1332	providerInstanceId: claudeInstanceId,
	1333	},
	1334	{
	1335	id: runningRunId,
	1336	ordinal: 2,
	1337	status: "running",
	1338	providerThreadId,
	1339	providerInstanceId: claudeInstanceId,
	1340	},
	1341	],
	1342	attempts: [],
	1343	nodes: [
	1344	{ id: settledTaskId, runId: settledRunId, status: "running", kind: "subagent" },
	1345	{ id: runningTaskId, runId: runningRunId, status: "running", kind: "subagent" },
	1346	{ id: nativeSubagentId, runId: settledRunId, status: "running", kind: "subagent" },
	1347	],
	1348	subagents: [
	1349	delegatedTask(settledTaskId, settledRunId),
	1350	delegatedTask(runningTaskId, runningRunId),
	1351	{
	1352	...delegatedTask(nativeSubagentId, settledRunId),
	1353	origin: "provider_native",
	1354	childThreadId: null,
	1355	},
	1356	],
	1357	messages: [],
	1358	turnItems: [
	1359	subagentItem(settledTaskId, settledRunId, "app_owned"),
	1360	subagentItem(runningTaskId, runningRunId, "app_owned"),
	1361	subagentItem(nativeSubagentId, settledRunId, "provider_native"),
	1362	],
	1363	} as unknown as OrchestrationV2ThreadProjection;
	1364	const layer = ProviderRuntimeRecovery.layer.pipe(
	1365	Layer.provide(ServerSettings.layerTest()),
	1366	Layer.provide(
	1367	Layer.mergeAll(
	1368	Layer.mock(ProjectionStore.ProjectionStoreV2)({
	1369	getRecoveryThreadIds: () => Effect.succeed([threadId]),
	1370	getRuntimeRecoveryProjection: () => Effect.succeed(projection),
	1371	}),
	1372	Layer.mock(EventSink.EventSinkV2)({
	1373	commitCommand: (input) => {
	1374	committedInput = input;
	1375	return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
	1376	},
	1377	}),
	1378	IdAllocator.layer,
	1379	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
	1380	runRecoveryOnce: Effect.succeed(false),
	1381	}),
	1382	Layer.mock(EffectOutbox.EffectOutboxV2)({
	1383	listByCommandId: () => Effect.succeed([]),
	1384	reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
	1385	}),
	1386	),
	1387	),
	1388	);
	1389
	1390	return Effect.gen(function* () {
	1391	yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
	1392	const events = committedInput?.events ?? [];
	1393	const touched = events.flatMap((event) =>
	1394	event.type === "turn-item.updated"
	1395	? event.payload.type === "subagent"
	1396	? [event.payload.subagentId]
	1397	: []
	1398	: event.type === "subagent.updated" \|\| event.type === "node.updated"
	1399	? [event.payload.id]
	1400	: [],
	1401	);
	1402	// The cut run itself is cancelled; only the provider-native subagent dies with it.
	1403	assert.isTrue(
	1404	events.some(
	1405	(event) =>
	1406	event.type === "run.updated" &&
	1407	event.payload.id === runningRunId &&
	1408	event.payload.status === "cancelled",
	1409	),
	1410	);
	1411	assert.notInclude(touched, settledTaskId);
	1412	assert.notInclude(touched, runningTaskId);
	1413	assert.include(touched, nativeSubagentId);
	1414	// The note lists only provider-native work: delegated tasks report back on their own.
	1415	const noted = events.flatMap((event) =>
	1416	event.type === "run.background-work-cancelled"
	1417	? event.payload.restartCancelledBackgroundWork.map((work) => work.label)
	1418	: [],
	1419	);
	1420	assert.deepEqual(noted, [`subagent ${nativeSubagentId}`]);
	1421	}).pipe(Effect.provide(layer));
	1422	});

apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+71-28

...	...	@@ -1,12 +1,15 @@
	1	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1	2	import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
2	3	import {
3	4	CommandId,
4	5	type OrchestrationV2DomainEvent,
5	6	type ProviderThreadId,
6	7	type OrchestrationV2RestartCancelledBackgroundWork,
	8	type OrchestrationV2Subagent,
7	9	type OrchestrationV2ThreadProjection,
8	10	ThreadId,
9	11	} from "@t3tools/contracts";
	12	import * as Cause from "effect/Cause";
10	13	import * as Context from "effect/Context";
11	14	import * as DateTime from "effect/DateTime";
12	15	import * as Effect from "effect/Effect";
...	...	@@ -93,6 +96,23 @@ function isNonterminalNodeStatus(status: string): boolean {
93	96	return status === "pending" \|\| status === "running" \|\| status === "waiting";
94	97	}
95	98
	99	/**
	100	* A delegate_task child. Its own thread is reconciled and continued on its own
	101	* and reports back through the app, so it is not provider background work.
	102	*/
	103	function isAppOwnedDelegation(task: {
	104	readonly origin: OrchestrationV2Subagent["origin"];
	105	readonly childThreadId: ThreadId \| null;
	106	}): boolean {
	107	return task.origin === "app_owned" && task.childThreadId !== null;
	108	}
	109
	110	function isAppOwnedDelegationItem(
	111	item: OrchestrationV2ThreadProjection["turnItems"][number],
	112	): boolean {
	113	return item.type === "subagent" && isAppOwnedDelegation(item);
	114	}
	115
96	116	function providerThreadHasPendingBackgroundTasks(
97	117	providerThread: OrchestrationV2ThreadProjection["providerThreads"][number],
98	118	): boolean {
...	...	@@ -138,7 +158,11 @@ function providerThreadsWithOpenBackgroundWork(
138	158	): ReadonlySet<ProviderThreadId> {
139	159	const ids = new Set<ProviderThreadId>();
140	160	for (const item of projection.turnItems ?? []) {
141		if (!isBackgroundCapableTurnItemType(item.type) \|\| !isNonterminalTurnItemStatus(item.status))
	161	if (
	162	!isBackgroundCapableTurnItemType(item.type) \|\|
	163	!isNonterminalTurnItemStatus(item.status) \|\|
	164	isAppOwnedDelegationItem(item)
	165	)
142	166	continue;
143	167	const providerThreadId =
144	168	item.providerThreadId ??
...	...	@@ -166,7 +190,7 @@ function latestStartedRun(
166	190	run.providerThreadId === providerThreadId &&
167	191	run.status !== "queued" &&
168	192	run.status !== "rolled_back" &&
169		(latest === undefined \|\| run.ordinal > latest.ordinal)
	193	(latest === undefined \|\| runRanAfter(run, latest))
170	194	? run
171	195	: latest,
172	196	undefined,
...	...	@@ -222,6 +246,13 @@ export const make = Effect.gen(function* () {
222	246	const requests = projection.runtimeRequests.filter(
223	247	(request) => request.status === "pending" && request.responseCapability.type !== "message",
224	248	);
	249	// Delegated task rows, items and nodes stay open: the child settles them.
	250	const delegatedTaskNodeIds = new Set<string>([
	251	...(projection.subagents ?? []).filter(isAppOwnedDelegation).map((subagent) => subagent.id),
	252	...(projection.turnItems ?? []).flatMap((item) =>
	253	item.type === "subagent" && isAppOwnedDelegation(item) ? [item.subagentId] : [],
	254	),
	255	]);
225	256	const detail = `Cancelled because the server ${trigger === "startup" ? "restarted" : "shut down"} before the provider work completed.`;
226	257	const commandId = CommandId.make(
227	258	`command:runtime-reconcile:${trigger}:${projection.thread.id}:${DateTime.formatIso(now)}`,
...	...	@@ -333,6 +364,7 @@ export const make = Effect.gen(function* () {
333	364	(candidate) =>
334	365	candidate.runId === run.id &&
335	366	!messageRequestNodeIds.has(candidate.id) &&
	367	!delegatedTaskNodeIds.has(candidate.id) &&
336	368	(candidate.status === "pending" \|\|
337	369	candidate.status === "running" \|\|
338	370	candidate.status === "waiting"),
...	...	@@ -351,6 +383,7 @@ export const make = Effect.gen(function* () {
351	383	for (const subagent of projection.subagents.filter(
352	384	(candidate) =>
353	385	candidate.runId === run.id &&
	386	!isAppOwnedDelegation(candidate) &&
354	387	(candidate.status === "pending" \|\|
355	388	candidate.status === "running" \|\|
356	389	candidate.status === "waiting"),
...	...	@@ -404,6 +437,7 @@ export const make = Effect.gen(function* () {
404	437	(candidate) =>
405	438	candidate.runId === run.id &&
406	439	(candidate.nodeId === null \|\| !messageRequestNodeIds.has(candidate.nodeId)) &&
	440	!isAppOwnedDelegationItem(candidate) &&
407	441	(candidate.status === "pending" \|\|
408	442	candidate.status === "running" \|\|
409	443	candidate.status === "waiting"),
...	...	@@ -436,7 +470,7 @@ export const make = Effect.gen(function* () {
436	470	if (!isBackgroundCapableTurnItemType(item.type)) {
437	471	continue;
438	472	}
439		if (!isNonterminalTurnItemStatus(item.status)) {
	473	if (!isNonterminalTurnItemStatus(item.status) \|\| isAppOwnedDelegationItem(item)) {
440	474	continue;
441	475	}
442	476	const providerInstanceId = resolveStaleBackgroundItemProviderInstanceId(item, projection);
...	...	@@ -763,32 +797,41 @@ export const make = Effect.gen(function* () {
763	797	if (!enabled) return;
764	798	const threadIds = yield* projections.getRecoveryThreadIds("runtime");
765	799	for (const threadId of threadIds) {
766		const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
767		if (
768		!resolveProjectSettings(enabled, projection.thread.projectId).settings
769		.continueThreadsAfterServerUpdate
770		)
771		continue;
772		// Shutdown reconciliation cancels the background work below, so a
773		// settled thread's continuation must be captured while it is still open.
774		const run = restartContinuationRun(
775		projection,
776		providerThreadsWithOpenBackgroundWork(projection),
	800	yield* Effect.gen(function* () {
	801	const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
	802	if (
	803	!resolveProjectSettings(enabled, projection.thread.projectId).settings
	804	.continueThreadsAfterServerUpdate
	805	)
	806	return;
	807	// Shutdown reconciliation cancels the background work below, so a
	808	// settled thread's continuation must be captured while it is still open.
	809	const run = restartContinuationRun(
	810	projection,
	811	providerThreadsWithOpenBackgroundWork(projection),
	812	);
	813	if (!run) return;
	814	const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
	815	yield* eventSink.writeWithEffects({
	816	commandId,
	817	events: [],
	818	effects: [
	819	{
	820	id: `effect:restart-continuation:${run.id}`,
	821	commandId,
	822	threadId,
	823	request: { type: "provider-runtime.continue", sourceRunId: run.id },
	824	},
	825	],
	826	});
	827	}).pipe(
	828	// One failing thread must not cost the threads after it their continuation.
	829	Effect.catchCauseIf(
	830	(cause) => !Cause.hasInterruptsOnly(cause),
	831	(cause) =>
	832	Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
	833	),
777	834	);
778		if (!run) continue;
779		const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
780		yield* eventSink.writeWithEffects({
781		commandId,
782		events: [],
783		effects: [
784		{
785		id: `effect:restart-continuation:${run.id}`,
786		commandId,
787		threadId,
788		request: { type: "provider-runtime.continue", sourceRunId: run.id },
789		},
790		],
791		});
792	835	}
793	836	}).pipe(
794	837	Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),

apps/server/src/orchestration-v2/ProviderSessionManager.ts+22

...	...	@@ -362,6 +362,25 @@ export const layerWithOptions = (
362	362	},
363	363	);
364	364	const layerScope = yield* Effect.scope;
	365	// Ctrl+C, or a stop that signals the whole process group, reaches the
	366	// provider CLIs with the server. They report their own background work
	367	// stopped before shutdown captures restart continuations, so provider
	368	// events after the signal are dropped; restart recovery owns that state.
	369	const shutdownSignal = { received: false };
	370	const onShutdownSignal = () => {
	371	shutdownSignal.received = true;
	372	};
	373	yield* Effect.acquireRelease(
	374	Effect.sync(() => {
	375	process.on("SIGINT", onShutdownSignal);
	376	process.on("SIGTERM", onShutdownSignal);
	377	}),
	378	() =>
	379	Effect.sync(() => {
	380	process.off("SIGINT", onShutdownSignal);
	381	process.off("SIGTERM", onShutdownSignal);
	382	}),
	383	);
365	384	const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
366	385	// One retry per released entry, so a later release with the same id
367	386	// cannot drop cleanup for threads only the earlier session served.
...	...	@@ -1542,6 +1561,7 @@ export const layerWithOptions = (
1542	1561	let stoppedByProvider = false;
1543	1562	return entry.runtime.events.pipe(
1544	1563	Stream.runForEach((event) => {
	1564	if (shutdownSignal.received) return Effect.void;
1545	1565	if (
1546	1566	event.type === "provider_session.updated" &&
1547	1567	event.providerSession.status === "stopped"
...	...	@@ -1600,6 +1620,8 @@ export const layerWithOptions = (
1600	1620	Effect.exit,
1601	1621	Effect.flatMap((exit) =>
1602	1622	Effect.gen(function* () {
	1623	// A provider that exits on the shutdown signal is released by shutdown.
	1624	if (shutdownSignal.received) return;
1603	1625	const current = (yield* Ref.get(sessions)).get(
1604	1626	sessionKey(entry.runtime.providerSessionId),
1605	1627	);

apps/server/src/orchestration-v2/ProviderTurnStartService.ts+2-3

...	...	@@ -953,6 +953,7 @@ export const layer: Layer.Layer<
953	953	run,
954	954	projection.runs,
955	955	projection.providerTurns,
	956	projection.attempts,
956	957	);
957	958	const restartCancelledWork = pendingRestartCancelledBackgroundWork({
958	959	runs: projection.runs,
...	...	@@ -967,9 +968,7 @@ export const layer: Layer.Layer<
967	968	.map((candidate) => candidate.id),
968	969	),
969	970	run,
970		runAttemptIds: projection.attempts
971		.filter((candidate) => candidate.runId === run.id)
972		.map((candidate) => candidate.id),
	971	attempts: projection.attempts,
973	972	});
974	973	const restartNote =
975	974	restartCancelledWork.length === 0

apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts+106-2

...	...	@@ -6,12 +6,15 @@ import {
6	6	RunId,
7	7	type OrchestrationV2Run,
8	8	} from "@t3tools/contracts";
	9	import * as DateTime from "effect/DateTime";
9	10
10	11	import {
11	12	cancelledRosterTaskWork,
12	13	cancelledTurnItemWork,
	14	isRestartNoteContinuation,
13	15	pendingRestartCancelledBackgroundWork,
14	16	restartCancelledBackgroundWorkNote,
	17	restartContinuationNote,
15	18	mergeRestartCancelledBackgroundWork,
16	19	} from "./RestartBackgroundNote.ts";
17	20
...	...	@@ -51,7 +54,7 @@ it("keeps the note for the provider thread that lost the work across a provider
51	54	providerTurns: runs.filter((source) => source.id !== target.id).map(turnFor),
52	55	compactionMessageIds: new Set(),
53	56	run: target,
54		runAttemptIds: target.activeAttemptId === null ? [] : [target.activeAttemptId],
	57	attempts: [{ id: target.activeAttemptId!, runId: target.id }],
55	58	});
56	59
57	60	// Codex never lost the work, so it neither receives nor consumes the note.
...	...	@@ -62,6 +65,30 @@ it("keeps the note for the provider thread that lost the work across a provider
62	65	assert.deepEqual(pending(later, [root, onCodex, backOnClaude, later]), []);
63	66	});
64	67
	68	it("delivers a resumed queued run's note even after a higher-ordinal run", () => {
	69	// Run 3 ran ahead of held run 2; run 2 then resumed, lost background work in
	70	// a second restart, and its continuation was superseded by a new message.
	71	const ranFirst = run(3, claudeThread, {
	72	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
	73	});
	74	const resumed = run(2, claudeThread, {
	75	completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
	76	restartCancelledBackgroundWork: lost,
	77	});
	78	const next = run(4, claudeThread);
	79	const runs = [resumed, ranFirst, next];
	80	assert.deepEqual(
	81	pendingRestartCancelledBackgroundWork({
	82	runs,
	83	providerTurns: [resumed, ranFirst].map(turnFor),
	84	compactionMessageIds: new Set(),
	85	run: next,
	86	attempts: [{ id: next.activeAttemptId!, runId: next.id }],
	87	}),
	88	lost,
	89	);
	90	});
	91
65	92	it("bounds the note so it cannot crowd out the turn's context", () => {
66	93	const work = Array.from({ length: 25 }, (_, index) => ({
67	94	kind: "shell" as const,
...	...	@@ -123,7 +150,7 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()
123	150	],
124	151	compactionMessageIds: new Set(),
125	152	run: steered,
126		runAttemptIds,
	153	attempts: runAttemptIds.map((id) => ({ id, runId: steered.id })),
127	154	});
128	155
129	156	// The first attempt reached the provider with the note; its replacement must not repeat it.
...	...	@@ -131,3 +158,80 @@ it("does not repeat the note when a steer restarts the run on a new attempt", ()
131	158	// A first attempt that never reached the provider did not deliver it.
132	159	assert.deepEqual(pending([firstAttempt, RunAttemptId.make("attempt:2b")], false), lost);
133	160	});
	161
	162	it("counts a prompted mid-turn or chained continuation as delivering the note", () => {
	163	const cut = run(1, claudeThread, { status: "cancelled", restartCancelledBackgroundWork: lost });
	164	const cutTurn = { ...turnFor(cut), status: "cancelled" as const };
	165	const continuation = run(2, claudeThread, { restartContinuationOfRunId: cut.id });
	166	// A turn cut mid-way that lost work is prompted with the note, not resumed natively.
	167	assert.isTrue(isRestartNoteContinuation(continuation, [cut, continuation], [cutTurn], []));
	168	assert.isFalse(
	169	isRestartNoteContinuation(
	170	continuation,
	171	[{ ...cut, restartCancelledBackgroundWork: [] }, continuation],
	172	[cutTurn],
	173	[],
	174	),
	175	);
	176	const later = run(3, claudeThread);
	177	const pending = (delivered: boolean) =>
	178	pendingRestartCancelledBackgroundWork({
	179	runs: [cut, continuation, later],
	180	providerTurns: [cutTurn, ...(delivered ? [turnFor(continuation)] : [])],
	181	compactionMessageIds: new Set(),
	182	run: later,
	183	attempts: [{ id: later.activeAttemptId!, runId: later.id }],
	184	});
	185	assert.deepEqual(pending(true), []);
	186	// A continuation that never reached the provider delivered nothing.
	187	assert.deepEqual(pending(false), lost);
	188
	189	// Its own continuation carries the original note forward.
	190	const unstarted = { ...continuation, status: "cancelled" as const };
	191	const chained = run(3, claudeThread, { restartContinuationOfRunId: unstarted.id });
	192	assert.deepEqual(restartContinuationNote(unstarted, [cut, unstarted], [cutTurn], []), {
	193	work: lost,
	194	settled: false,
	195	});
	196	assert.isTrue(isRestartNoteContinuation(chained, [cut, unstarted, chained], [cutTurn], []));
	197	// Claude announces running before accepting the prompt. A second restart
	198	// can cancel that turn without delivering anything to the provider.
	199	for (const status of ["running", "cancelled"] as const) {
	200	const turns = [cutTurn, { ...turnFor(unstarted), status }];
	201	assert.deepEqual(restartContinuationNote(unstarted, [cut, unstarted], turns, []), {
	202	work: lost,
	203	settled: false,
	204	});
	205	assert.deepEqual(
	206	pendingRestartCancelledBackgroundWork({
	207	runs: [cut, unstarted, later],
	208	providerTurns: turns,
	209	compactionMessageIds: new Set(),
	210	run: later,
	211	attempts: [{ id: later.activeAttemptId!, runId: later.id }],
	212	}),
	213	lost,
	214	);
	215	}
	216
	217	// A steer replaced the attempt after its completed turn delivered the note.
	218	const steered = { ...unstarted, activeAttemptId: RunAttemptId.make("attempt:replacement") };
	219	const attempts = [{ id: unstarted.activeAttemptId!, runId: steered.id }];
	220	const turns = [
	221	cutTurn,
	222	turnFor(unstarted),
	223	{ ...turnFor(steered), status: "cancelled" as const },
	224	];
	225	assert.deepEqual(restartContinuationNote(steered, [cut, steered], turns, attempts).work, []);
	226	assert.isFalse(isRestartNoteContinuation(chained, [cut, steered, chained], turns, attempts));
	227	assert.deepEqual(
	228	pendingRestartCancelledBackgroundWork({
	229	runs: [cut, steered, later],
	230	providerTurns: turns,
	231	compactionMessageIds: new Set(),
	232	run: later,
	233	attempts,
	234	}),
	235	[],
	236	);
	237	});

apps/server/src/orchestration-v2/RestartBackgroundNote.ts+67-13

...	...	@@ -3,10 +3,13 @@ import type {
3	3	OrchestrationV2ProviderTurn,
4	4	OrchestrationV2RestartCancelledBackgroundWork,
5	5	OrchestrationV2Run,
	6	OrchestrationV2RunAttempt,
6	7	OrchestrationV2TurnItem,
7	8	} from "@t3tools/contracts";
	9	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
8	10
9	11	type Work = OrchestrationV2RestartCancelledBackgroundWork;
	12	type Attempt = Pick<OrchestrationV2RunAttempt, "id" \| "runId">;
10	13
11	14	const MAX_LABEL_LENGTH = 160;
12	15
...	...	@@ -120,25 +123,69 @@ export function isRestartNoteSource(
120	123	);
121	124	}
122	125
123		/** A restart continuation whose prompt is the note rather than a resume. */
	126	/**
	127	* The note a restart continuation of `source` carries, and whether the turn it
	128	* continues had settled (the note is then the whole prompt). A continuation cut
	129	* before its provider turn completed may not have delivered its own note:
	130	* adapters can announce a running turn before accepting the prompt. Carry the
	131	* note forward until a completed turn proves delivery.
	132	*/
	133	export function restartContinuationNote(
	134	source: OrchestrationV2Run,
	135	runs: ReadonlyArray<OrchestrationV2Run>,
	136	providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" \| "status">>,
	137	attempts: ReadonlyArray<Attempt>,
	138	): { readonly work: ReadonlyArray<Work>; readonly settled: boolean } {
	139	const completedAttempts = new Set(
	140	providerTurns.filter((turn) => turn.status === "completed").map((turn) => turn.runAttemptId),
	141	);
	142	const completedRuns = new Set(
	143	attempts.filter((attempt) => completedAttempts.has(attempt.id)).map((attempt) => attempt.runId),
	144	);
	145	let current = source;
	146	let work = current.restartCancelledBackgroundWork ?? [];
	147	const visited = new Set([current.id]);
	148	while (
	149	current.restartContinuationOfRunId !== undefined &&
	150	!completedRuns.has(current.id) &&
	151	!(current.activeAttemptId !== null && completedAttempts.has(current.activeAttemptId))
	152	) {
	153	const previous = runs.find((candidate) => candidate.id === current.restartContinuationOfRunId);
	154	if (previous === undefined \|\| visited.has(previous.id)) break;
	155	visited.add(previous.id);
	156	current = previous;
	157	work = mergeRestartCancelledBackgroundWork(current.restartCancelledBackgroundWork ?? [], work);
	158	}
	159	return { work, settled: isRestartNoteSource(current, providerTurns) };
	160	}
	161
	162	/**
	163	* A restart continuation prompted with the note rather than a native resume.
	164	* A turn cut mid-way that lost background work is prompted too, so the note
	165	* is delivered with it; only a continuation without a note resumes natively.
	166	*/
124	167	export function isRestartNoteContinuation(
125	168	run: Pick<OrchestrationV2Run, "restartContinuationOfRunId">,
126	169	runs: ReadonlyArray<OrchestrationV2Run>,
127	170	providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" \| "status">>,
	171	attempts: ReadonlyArray<Attempt>,
128	172	): boolean {
129	173	const source =
130	174	run.restartContinuationOfRunId === undefined
131	175	? undefined
132	176	: runs.find((candidate) => candidate.id === run.restartContinuationOfRunId);
133		return source !== undefined && isRestartNoteSource(source, providerTurns);
	177	return (
	178	source !== undefined &&
	179	restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0
	180	);
134	181	}
135	182
136	183	/**
137	184	* Work cancelled by a restart that the run's provider thread has not been told
138	185	* about yet. The note belongs to the provider thread that lost the work: turns
139	186	* on another provider (after a switch) neither owe it nor deliver it. A later
140		* run on the same provider thread delivers it once its attempt reaches the
141		* provider, so the pending set is derived rather than cleared. Compactions and
	187	* completed turn on the same provider thread proves delivery, so the pending
	188	* set is derived rather than cleared. Compactions and
142	189	* resumed turns carry no note, and a rolled-back run left native history, so
143	190	* none of them counts as delivery. A note continuation's own prompt is the note.
144	191	*/
...	...	@@ -150,13 +197,14 @@ export function pendingRestartCancelledBackgroundWork(input: {
150	197	OrchestrationV2Run,
151	198	\| "id"
152	199	\| "ordinal"
	200	\| "completedAt"
153	201	\| "userMessageId"
154	202	\| "providerThreadId"
155	203	\| "restartContinuationOfRunId"
156	204	\| "activeAttemptId"
157	205	>;
158		/** Every attempt id of `run`; a steer replaces the attempt but not the run. */
159		readonly runAttemptIds: ReadonlyArray<OrchestrationV2Run["activeAttemptId"] & string>;
	206	/** Include earlier attempts: a steer replaces the attempt but not the run. */
	207	readonly attempts: ReadonlyArray<Attempt>;
160	208	}): ReadonlyArray<Work> {
161	209	const isCompaction = (run: typeof input.run) => input.compactionMessageIds.has(run.userMessageId);
162	210	// The current run prepends the note unless it is a compaction or a
...	...	@@ -170,7 +218,7 @@ export function pendingRestartCancelledBackgroundWork(input: {
170	218	const providerThreadId = input.run.providerThreadId;
171	219	const deliveredAttemptIds = new Set(
172	220	input.providerTurns
173		.filter((turn) => turn.providerThreadId === providerThreadId)
	221	.filter((turn) => turn.providerThreadId === providerThreadId && turn.status === "completed")
174	222	.map((turn) => turn.runAttemptId),
175	223	);
176	224	const sameThread = input.runs.filter((run) => run.providerThreadId === providerThreadId);
...	...	@@ -179,23 +227,29 @@ export function pendingRestartCancelledBackgroundWork(input: {
179	227	candidate.id !== input.run.id &&
180	228	candidate.activeAttemptId !== null &&
181	229	candidate.status !== "rolled_back" &&
182		deliveredAttemptIds.has(candidate.activeAttemptId) &&
	230	(deliveredAttemptIds.has(candidate.activeAttemptId) \|\|
	231	input.attempts.some(
	232	(attempt) => attempt.runId === candidate.id && deliveredAttemptIds.has(attempt.id),
	233	)) &&
183	234	!isCompaction(candidate) &&
184	235	(candidate.restartContinuationOfRunId === undefined \|\|
185		isRestartNoteContinuation(candidate, input.runs, input.providerTurns)),
	236	isRestartNoteContinuation(candidate, input.runs, input.providerTurns, input.attempts)),
186	237	);
187	238	// A steer restarts this run on a new attempt: an earlier attempt that already
188	239	// reached the provider delivered the note, so the replacement must not repeat it.
189		const alreadyDelivered = input.runAttemptIds.some(
190		(attemptId) => attemptId !== input.run.activeAttemptId && deliveredAttemptIds.has(attemptId),
	240	const alreadyDelivered = input.attempts.some(
	241	(attempt) =>
	242	attempt.runId === input.run.id &&
	243	attempt.id !== input.run.activeAttemptId &&
	244	deliveredAttemptIds.has(attempt.id),
191	245	);
192	246	if (alreadyDelivered) return [];
193	247	return sameThread
194	248	.filter(
195	249	(source) =>
196		source.ordinal < input.run.ordinal &&
	250	runRanAfter(input.run, source) &&
197	251	(source.restartCancelledBackgroundWork?.length ?? 0) > 0 &&
198		!prompted.some((later) => later.ordinal > source.ordinal),
	252	!prompted.some((later) => runRanAfter(later, source)),
199	253	)
200	254	.reduce<ReadonlyArray<Work>>(
201	255	(work, source) =>

apps/server/src/orchestration-v2/RestartContinuation.test.ts+217

...	...	@@ -1,5 +1,6 @@
1	1	import { assert, it } from "@effect/vitest";
2	2	import {
	3	MessageId,
3	4	ProjectId,
4	5	ProviderDriverKind,
5	6	ProviderInstanceId,
...	...	@@ -11,6 +12,7 @@ import {
11	12	ThreadId,
12	13	type OrchestrationV2ThreadProjection,
13	14	} from "@t3tools/contracts";
	15	import * as DateTime from "effect/DateTime";
14	16	import * as Effect from "effect/Effect";
15	17	import * as Layer from "effect/Layer";
16	18	import * as ServerSettings from "../serverSettings.ts";
...	...	@@ -201,6 +203,7 @@ it.effect("prompts a settled thread's continuation with the note of its lost wor
201	203	Layer.merge(
202	204	Layer.mock(ThreadManagementService.ThreadManagementService)({
203	205	getThreadRecords: () => Effect.succeed(projection),
	206	recoverDelegatedTask: () => Effect.void,
204	207	dispatch: (command) => {
205	208	commands.push(command);
206	209	return Effect.succeed({} as never);
...	...	@@ -246,6 +249,7 @@ it.effect("does not continue a failed run that lost background work", () =>
246	249	Layer.merge(
247	250	Layer.mock(ThreadManagementService.ThreadManagementService)({
248	251	getThreadRecords: () => Effect.succeed(projection),
	252	recoverDelegatedTask: () => Effect.void,
249	253	dispatch: (command) => {
250	254	commands.push(command);
251	255	return Effect.succeed({} as never);
...	...	@@ -328,6 +332,7 @@ it.effect("does not duplicate delivery and yields to newer user work or opt-out"
328	332	>[0][] = [];
329	333	const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({
330	334	getThreadRecords: () => Effect.succeed(projection),
	335	recoverDelegatedTask: () => Effect.void,
331	336	dispatch: (command) => {
332	337	commands.push(command);
333	338	if (command.type === "message.dispatch")
...	...	@@ -499,6 +504,7 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
499	504	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
500	505	Layer.mock(ThreadManagementService.ThreadManagementService)({
501	506	getThreadRecords: () => Effect.succeed(projection),
	507	recoverDelegatedTask: () => Effect.void,
502	508	dispatch: () =>
503	509	Effect.sync(() => {
504	510	dispatched = true;
...	...	@@ -511,3 +517,214 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
511	517	assert.isFalse(dispatched);
512	518	}),
513	519	);
	520
	521	const continuationTexts = (projection: OrchestrationV2ThreadProjection) =>
	522	Effect.gen(function* () {
	523	const texts: Array<string> = [];
	524	yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
	525	Effect.provide(
	526	Layer.merge(
	527	Layer.mock(ThreadManagementService.ThreadManagementService)({
	528	getThreadRecords: () => Effect.succeed(projection),
	529	recoverDelegatedTask: () => Effect.void,
	530	dispatch: (command) => {
	531	if (command.type === "message.dispatch") texts.push(command.text);
	532	return Effect.succeed({} as never);
	533	},
	534	}),
	535	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
	536	),
	537	),
	538	);
	539	return texts;
	540	});
	541
	542	const cutMidTurn = (extra: Record<string, unknown> = {}) => {
	543	const base = makeProjection();
	544	return {
	545	...base,
	546	runs: [
	547	{
	548	...base.runs[0]!,
	549	status: "cancelled",
	550	userMessageId: MessageId.make("message:user"),
	551	...extra,
	552	},
	553	],
	554	providerTurns: [{ ...base.providerTurns[0]!, status: "cancelled" }],
	555	} as unknown as OrchestrationV2ThreadProjection;
	556	};
	557
	558	const queuedFollowUp = {
	559	id: RunId.make("run:queued"),
	560	ordinal: 2,
	561	providerInstanceId: instanceId,
	562	providerThreadId,
	563	status: "queued",
	564	queueHeld: true,
	565	};
	566
	567	it.effect("continues a cut run past queued follow-ups, which stay held", () =>
	568	Effect.gen(function* () {
	569	const live = makeProjection();
	570	assert.equal(
	571	restartContinuationRun({
	572	...live,
	573	runs: [...live.runs, queuedFollowUp],
	574	} as unknown as OrchestrationV2ThreadProjection)?.id,
	575	runId,
	576	);
	577	const cut = cutMidTurn();
	578	const texts = yield* continuationTexts({
	579	...cut,
	580	runs: [...cut.runs, queuedFollowUp],
	581	} as unknown as OrchestrationV2ThreadProjection);
	582	assert.deepEqual(texts, ["Continue where you left off."]);
	583	}),
	584	);
	585
	586	it.effect("continues a resumed queued run that ran after an earlier continuation", () =>
	587	Effect.gen(function* () {
	588	// The continuation (ordinal 3) ran ahead of the held queue, then the user
	589	// resumed the queued run (ordinal 1 here) and the server restarted again.
	590	const finishedContinuation = {
	591	id: RunId.make("run:earlier-continuation"),
	592	ordinal: 3,
	593	providerInstanceId: instanceId,
	594	providerThreadId,
	595	status: "completed",
	596	completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
	597	};
	598	const live = makeProjection();
	599	assert.equal(
	600	restartContinuationRun({
	601	...live,
	602	runs: [...live.runs, finishedContinuation],
	603	} as unknown as OrchestrationV2ThreadProjection)?.id,
	604	runId,
	605	);
	606	const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });
	607	const texts = yield* continuationTexts({
	608	...cut,
	609	runs: [...cut.runs, finishedContinuation],
	610	} as unknown as OrchestrationV2ThreadProjection);
	611	assert.deepEqual(texts, ["Continue where you left off."]);
	612	}),
	613	);
	614
	615	it.effect("does not continue a run the user asked to stop before the restart", () =>
	616	Effect.gen(function* () {
	617	const texts = yield* continuationTexts({
	618	...cutMidTurn(),
	619	turnItems: [{ id: "turn-item:interrupt", runId, type: "run_interrupt_request" }],
	620	} as unknown as OrchestrationV2ThreadProjection);
	621	assert.deepEqual(texts, []);
	622	}),
	623	);
	624
	625	it.effect("does not continue a cut /compact or /logout turn", () =>
	626	Effect.gen(function* () {
	627	for (const text of ["/compact", " /LOGOUT "]) {
	628	const texts = yield* continuationTexts({
	629	...cutMidTurn(),
	630	messages: [{ id: MessageId.make("message:user"), text, attachments: [] }],
	631	} as unknown as OrchestrationV2ThreadProjection);
	632	assert.deepEqual(texts, [], text);
	633	}
	634	const texts = yield* continuationTexts({
	635	...cutMidTurn(),
	636	messages: [{ id: MessageId.make("message:user"), text: "/compact later", attachments: [] }],
	637	} as unknown as OrchestrationV2ThreadProjection);
	638	assert.lengthOf(texts, 1);
	639	}),
	640	);
	641
	642	it.effect("tells a turn cut mid-way about the background work it lost", () =>
	643	Effect.gen(function* () {
	644	const texts = yield* continuationTexts(
	645	cutMidTurn({
	646	restartCancelledBackgroundWork: [{ kind: "subagent", label: "Background reviewer" }],
	647	}),
	648	);
	649	assert.lengthOf(texts, 1);
	650	assert.include(texts[0]!, "Background reviewer");
	651	assert.isTrue(texts[0]!.endsWith("Continue where you left off."));
	652	}),
	653	);
	654
	655	it.effect(
	656	"carries the note forward when its continuation was cut before reaching the provider",
	657	() =>
	658	Effect.gen(function* () {
	659	const base = makeProjection();
	660	const original = {
	661	...base.runs[0]!,
	662	id: RunId.make("run:original"),
	663	status: "completed",
	664	activeAttemptId: RunAttemptId.make("attempt:original"),
	665	restartCancelledBackgroundWork: [{ kind: "shell", label: "sleep 25 && echo DONE" }],
	666	};
	667	const projection = {
	668	...base,
	669	runs: [
	670	original,
	671	{
	672	...base.runs[0]!,
	673	ordinal: 2,
	674	status: "cancelled",
	675	restartContinuationOfRunId: original.id,
	676	},
	677	],
	678	// The original turn settled; the continuation's attempt never started one.
	679	providerTurns: [
	680	{
	681	...base.providerTurns[0]!,
	682	runAttemptId: original.activeAttemptId,
	683	status: "completed",
	684	},
	685	],
	686	} as unknown as OrchestrationV2ThreadProjection;
	687	const texts = yield* continuationTexts(projection);
	688	assert.lengthOf(texts, 1);
	689	assert.include(texts[0]!, "sleep 25 && echo DONE");
	690	assert.notInclude(texts[0]!, "Continue where");
	691	}),
	692	);
	693
	694	it.effect("prepares later threads' continuations when one thread fails", () =>
	695	Effect.gen(function* () {
	696	const brokenThreadId = ThreadId.make("thread:broken");
	697	const writes: Parameters<EventSink.EventSinkV2["Service"]["writeWithEffects"]>[0][] = [];
	698	const recovery = yield* ProviderRuntimeRecovery.make.pipe(
	699	Effect.provide(
	700	Layer.mergeAll(
	701	ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
	702	Layer.mock(ProjectionStore.ProjectionStoreV2)({
	703	getRecoveryThreadIds: () => Effect.succeed([brokenThreadId, threadId]),
	704	getRuntimeRecoveryProjection: (id) =>
	705	id === brokenThreadId
	706	? Effect.fail(
	707	new ProjectionStore.ProjectionStoreThreadNotFoundError({ threadId: id }),
	708	)
	709	: Effect.succeed(makeProjection()),
	710	}),
	711	Layer.mock(EventSink.EventSinkV2)({
	712	writeWithEffects: (input) =>
	713	Effect.sync(() => {
	714	writes.push(input);
	715	return [];
	716	}),
	717	}),
	718	IdAllocator.layer,
	719	Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
	720	Layer.mock(EffectOutbox.EffectOutboxV2)({}),
	721	),
	722	),
	723	);
	724	yield* recovery.prepareForShutdown;
	725	assert.deepEqual(
	726	writes.map((write) => write.effects[0]?.request),
	727	[{ type: "provider-runtime.continue", sourceRunId: runId }],
	728	);
	729	}),
	730	);

apps/server/src/orchestration-v2/RestartContinuation.ts+64-7

...	...	@@ -1,3 +1,4 @@
	1	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1	2	import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
2	3	import {
3	4	CommandId,
...	...	@@ -11,12 +12,16 @@ import * as Effect from "effect/Effect";
11	12	import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
12	13
13	14	import * as ServerSettings from "../serverSettings.ts";
	15	import { isNativeMaintenanceCommand } from "./Orchestrator.ts";
14	16	import * as ThreadManagementService from "./ThreadManagementService.ts";
15	17	import {
16	18	isRestartNoteSource,
17	19	restartCancelledBackgroundWorkNote,
	20	restartContinuationNote,
18	21	} from "./RestartBackgroundNote.ts";
19	22
	23	const CONTINUE_PROMPT = "Continue where you left off.";
	24
20	25	/**
21	26	* The run a restart continuation resumes, if any: an unfinished root run, or a
22	27	* settled one whose own provider thread lost background work in the restart
...	...	@@ -30,8 +35,12 @@ export function restartContinuationRun(
30	35	cancelledWorkProviderThreadIds: ReadonlySet<ProviderThreadId> = new Set(),
31	36	): OrchestrationV2Run \| undefined {
32	37	if (projection.thread.archivedAt !== null \|\| projection.thread.deletedAt !== null) return;
	38	// Queued runs never started; recovery holds them behind the cut run.
33	39	const run = projection.runs.reduce<OrchestrationV2Run \| undefined>(
34		(latest, candidate) => (!latest \|\| candidate.ordinal > latest.ordinal ? candidate : latest),
	40	(latest, candidate) =>
	41	candidate.status !== "queued" && (!latest \|\| runRanAfter(candidate, latest))
	42	? candidate
	43	: latest,
35	44	undefined,
36	45	);
37	46	if (!run) return;
...	...	@@ -98,7 +107,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
98	107	const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);
99	108	const projection = yield* threads.getThreadRecords(
100	109	input.threadId,
101		["messages", "runs", "providerTurns"],
	110	["messages", "runs", "providerTurns", "attempts"],
102	111	{ messageIds: [messageId] },
103	112	);
104	113	if (
...	...	@@ -114,17 +123,54 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
114	123	const noteSource =
115	124	source !== undefined && isRestartNoteSource(source, projection.providerTurns);
116	125	if (!source \|\| (source.status !== "cancelled" && !noteSource)) return;
117		// A user submission after reconciliation takes precedence over an automatic prompt.
118		if (projection.runs.some((run) => run.ordinal > source.ordinal)) return;
	126	// A user submission after reconciliation takes precedence over an automatic
	127	// prompt. Queued runs never started and stay held behind this one.
	128	if (
	129	projection.runs.some(
	130	(run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
	131	)
	132	)
	133	return;
119	134	if (projection.thread.providerInstanceId !== source.providerInstanceId) return;
	135	const sourceRecords = yield* threads.getThreadRecords(
	136	input.threadId,
	137	["messages", "turnItems"],
	138	{
	139	messageIds: [source.userMessageId],
	140	turnItemRunIds: [source.id],
	141	turnItemTypes: ["run_interrupt_request"],
	142	},
	143	);
	144	// The user asked this run to stop before the restart cut it.
	145	if (
	146	sourceRecords.turnItems.some(
	147	(item) => item.runId === source.id && item.type === "run_interrupt_request",
	148	)
	149	)
	150	return;
	151	const sourceMessage = sourceRecords.messages.find(
	152	(message) => message.id === source.userMessageId,
	153	);
	154	if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
	155	const note = restartContinuationNote(
	156	source,
	157	projection.runs,
	158	projection.providerTurns,
	159	projection.attempts,
	160	);
	161	const noteText =
	162	note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work);
120	163	yield* threads.dispatch({
121	164	type: "message.dispatch",
122	165	commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`),
123	166	threadId: input.threadId,
124	167	messageId,
125		text: noteSource
126		? restartCancelledBackgroundWorkNote(source.restartCancelledBackgroundWork ?? [])
127		: "Continue where you left off.",
	168	text:
	169	noteText === undefined
	170	? CONTINUE_PROMPT
	171	: note.settled
	172	? noteText
	173	: `${noteText}\n\n${CONTINUE_PROMPT}`,
128	174	attachments: [],
129	175	modelSelection: source.modelSelection,
130	176	dispatchMode: { type: "start_immediately" },
...	...	@@ -133,4 +179,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
133	179	restartContinuationOfRunId: input.sourceRunId,
134	180	});
135	181	},
	182	// A delegated child this declined to continue still owes its parent a
	183	// result. Once a continuation run exists this is a no-op; that run settles it.
	184	(effect, input) =>
	185	effect.pipe(
	186	Effect.andThen(
	187	Effect.gen(function* () {
	188	const threads = yield* ThreadManagementService.ThreadManagementService;
	189	yield* threads.recoverDelegatedTask(input.threadId, input.sourceRunId);
	190	}),
	191	),
	192	),
136	193	);

apps/server/src/orchestration-v2/SubagentProjection.test.ts+22

...	...	@@ -166,6 +166,28 @@ function taskFixture() {
166	166	return { projection: { ...projection, runs: [run] }, run };
167	167	}
168	168
	169	it("reports the run that ended last, not the highest ordinal", () => {
	170	const { projection, run } = taskFixture();
	171	// A restart continuation (ordinal 4) ran ahead of held queued runs 2 and 3.
	172	const ended = (ordinal: number, completedAt: string): OrchestrationV2Run => ({
	173	...run,
	174	id: RunId.make(`run:${ordinal}`),
	175	ordinal,
	176	completedAt: DateTime.makeUnsafe(completedAt),
	177	});
	178	const progress = delegatedTaskProgress({
	179	...projection,
	180	runs: [
	181	{ ...run, status: "cancelled" },
	182	ended(4, "2026-07-24T10:00:00.000Z"),
	183	ended(2, "2026-07-24T10:05:00.000Z"),
	184	ended(3, "2026-07-24T10:10:00.000Z"),
	185	],
	186	});
	187	assert.equal(progress.state, "result_available");
	188	assert.equal(progress.resultRun?.ordinal, 3);
	189	});
	190
169	191	it("waits for nested work and retains the report across monitor acknowledgements", () => {
170	192	const { projection, run } = taskFixture();
171	193	assert.equal(

apps/server/src/orchestration-v2/SubagentProjection.ts+2-1

...	...	@@ -16,6 +16,7 @@ import type {
16	16	ThreadId,
17	17	TurnItemId,
18	18	} from "@t3tools/contracts";
	19	import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
19	20	import * as DateTime from "effect/DateTime";
20	21	import { isOrchestrationV2WorkActive } from "@t3tools/contracts";
21	22
...	...	@@ -239,7 +240,7 @@ export function delegatedTaskProgress(projection: {
239	240	projection.providerThreads.some((thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0);
240	241	const resultRun = workRuns
241	242	.filter((run) => terminal(run.status) && (run.startedAt !== null \|\| run.ordinal === 1))
242		.toSorted((a, b) => b.ordinal - a.ordinal)[0];
	243	.toSorted((a, b) => (runRanAfter(a, b) ? -1 : runRanAfter(b, a) ? 1 : 0))[0];
243	244	return {
244	245	state:
245	246	active \|\| resultRun === undefined

apps/server/src/orchestration-v2/ThreadManagementService.ts+4

...	...	@@ -313,6 +313,8 @@ export interface ThreadManagementServiceShape {
313	313	input: ThreadManagementInterruptInput,
314	314	) => Effect.Effect<ThreadManagementInterruptResult, ThreadManagementFailure>;
315	315	readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"];
	316	readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"];
	317	readonly delegatedTaskResultPending: Orchestrator.OrchestratorV2["Service"]["delegatedTaskResultPending"];
316	318	readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"];
317	319	readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"];
318	320	readonly streamDomainEvents: Orchestrator.OrchestratorV2["Service"]["streamDomainEvents"];
...	...	@@ -736,6 +738,8 @@ const make = Effect.gen(function* () {
736	738	waitForThread,
737	739	interruptThread,
738	740	getThreadEventSequence: orchestrator.getThreadEventSequence,
	741	recoverDelegatedTask: orchestrator.recoverDelegatedTask,
	742	delegatedTaskResultPending: orchestrator.delegatedTaskResultPending,
739	743	streamStoredEvents: orchestrator.streamStoredEvents,
740	744	streamStoredEventsFrom: orchestrator.streamStoredEventsFrom,
741	745	streamDomainEvents: orchestrator.streamDomainEvents,

apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts+4

...	...	@@ -433,6 +433,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
433	433	dispatch: orchestrator.dispatch,
434	434	getThreadRecords: orchestrator.getThreadRecords,
435	435	getThreadProjection: orchestrator.getThreadProjection,
	436	recoverDelegatedTask: orchestrator.recoverDelegatedTask,
	437	delegatedTaskResultPending: orchestrator.delegatedTaskResultPending,
436	438	});
437	439	}),
438	440	).pipe(Layer.provide(orchestratorProvided));
...	...	@@ -497,6 +499,8 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
497	499	Orchestrator.OrchestratorV2,
498	500	Effect.gen(function* () {
499	501	const orchestrator = yield* Orchestrator.OrchestratorV2;
	502	// As in serverRuntimeStartup: after runtime recovery, before the worker.
	503	yield* orchestrator.recoverDelegatedTasks;
500	504	yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
501	505	return orchestrator;
502	506	}),

apps/server/src/relay/AgentAwarenessRelay.test.ts+2

...	...	@@ -206,6 +206,8 @@ const makeTestRelay = Effect.fnUntraced(function* (
206	206	waitForThread: unused,
207	207	interruptThread: unused,
208	208	getThreadEventSequence: unused,
	209	recoverDelegatedTask: unused,
	210	delegatedTaskResultPending: unused,
209	211	streamStoredEvents: Stream.empty,
210	212	streamStoredEventsFrom: () => Stream.empty,
211	213	streamDomainEvents: options.domainEvents ?? Stream.empty,

apps/server/src/scheduling/Scheduler.test.ts+24

...	...	@@ -6,6 +6,7 @@ import * as Queue from "effect/Queue";
6	6	import * as Ref from "effect/Ref";
7	7	import * as TestClock from "effect/testing/TestClock";
8	8
	9	import * as ServerActivation from "../serverActivation.ts";
9	10	import * as Scheduler from "./Scheduler.ts";
10	11
11	12	it.effect("keeps other sources running after a source defects", () =>
...	...	@@ -84,3 +85,26 @@ it.effect("unregisters closed sources and interrupts their in-flight work", () =
84	85	assert.equal(yield* Ref.get(runs), 1);
85	86	}).pipe(Effect.provide(Scheduler.layer)),
86	87	);
	88
	89	it.effect("holds due work and the clock until startup recovery activates the server", () =>
	90	Effect.gen(function* () {
	91	const activation = yield* Deferred.make<void>();
	92	const runs = yield* Ref.make(0);
	93	const ran = yield* Deferred.make<void>();
	94	yield* Effect.gen(function* () {
	95	const scheduler = yield* Scheduler.Scheduler;
	96	yield* scheduler.register(
	97	"due",
	98	Ref.update(runs, (n) => n + 1).pipe(Effect.andThen(Deferred.succeed(ran, undefined))),
	99	);
	100	yield* TestClock.adjust("10 seconds");
	101	assert.equal(yield* Ref.get(runs), 0);
	102	yield* Deferred.succeed(activation, undefined);
	103	yield* Deferred.await(ran);
	104	}).pipe(
	105	Effect.provide(Scheduler.layer),
	106	Effect.provideService(ServerActivation.ServerActivation, Deferred.await(activation)),
	107	);
	108	assert.isAtLeast(yield* Ref.get(runs), 1);
	109	}),
	110	);

apps/server/src/scheduling/Scheduler.ts+5-2

...	...	@@ -6,6 +6,8 @@ import * as Ref from "effect/Ref";
6	6	import * as Scope from "effect/Scope";
7	7	import * as Semaphore from "effect/Semaphore";
8	8
	9	import { forkParked } from "../serverActivation.ts";
	10
9	11	/** Sources load due work from their durable state; registration owns its execution lifetime. */
10	12	export class Scheduler extends Context.Service<
11	13	Scheduler,
...	...	@@ -47,14 +49,15 @@ const make = Effect.gen(function* () {
47	49	return next;
48	50	}),
49	51	);
50		yield* run;
	52	// Due work waits for startup recovery, which would cancel runs it started.
	53	yield* forkParked(run);
51	54	});
52	55	const tick = Ref.get(sources).pipe(
53	56	Effect.flatMap((current) => Effect.forEach(current.values(), (run) => run, { discard: true })),
54	57	);
55	58	// One clock for all due-work sources. A slow source cannot block another
56	59	// source or overlap itself, and no extra missed-tick backlog is queued.
57		yield* Effect.sleep("5 seconds").pipe(Effect.andThen(tick), Effect.forever, Effect.forkScoped);
	60	yield* forkParked(Effect.sleep("5 seconds").pipe(Effect.andThen(tick), Effect.forever));
58	61	return Scheduler.of({ register });
59	62	});
60	63

apps/server/src/serverRuntimeStartup.test.ts+10-1

...	...	@@ -31,11 +31,20 @@ it.effect("starts without scanning or rebuilding projection history", () =>
31	31	const result = yield* ServerRuntimeStartup.runOrderedV2StartupPhases({
32	32	importLegacyShells: record("import"),
33	33	recover: record("recover").pipe(Effect.as({ closedRequests: 2 })),
	34	recoverDelegatedTasks: record("delegated"),
34	35	startEffectWorker: record("worker"),
35	36	autoBootstrap: record("bootstrap").pipe(Effect.as({ projectId: "project-1" })),
36	37	});
37	38
38		assert.deepEqual(yield* Ref.get(calls), ["import", "recover", "worker", "bootstrap"]);
	39	// Delegated recovery reads the runs recovery terminalizes, and settles them
	40	// before the worker runs restart continuations that would otherwise race it.
	41	assert.deepEqual(yield* Ref.get(calls), [
	42	"import",
	43	"recover",
	44	"delegated",
	45	"worker",
	46	"bootstrap",
	47	]);
39	48	assert.deepEqual(result, {
40	49	recovery: { closedRequests: 2 },
41	50	bootstrap: { projectId: "project-1" },

apps/server/src/serverRuntimeStartup.ts+11

...	...	@@ -32,6 +32,7 @@ import * as Keybindings from "./keybindings.ts";
32	32	import * as ExternalLauncher from "./process/externalLauncher.ts";
33	33	import * as EffectWorker from "./orchestration-v2/EffectWorker.ts";
34	34	import * as LegacyV1ThreadImporter from "./orchestration-v2/legacy/LegacyV1ThreadImporter.ts";
	35	import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
35	36	import * as ProviderRuntimeRecovery from "./orchestration-v2/ProviderRuntimeRecoveryService.ts";
36	37	import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
37	38	import * as ThreadLaunch from "./orchestration-v2/ThreadLaunchService.ts";
...	...	@@ -386,21 +387,26 @@ export function runOrderedV2StartupPhases<
386	387	Bootstrap,
387	388	ImportError,
388	389	RecoveryError,
	390	DelegationError,
389	391	WorkerError,
390	392	BootstrapError,
391	393	ImportContext,
392	394	RecoveryContext,
	395	DelegationContext,
393	396	WorkerContext,
394	397	BootstrapContext,
395	398	>(input: {
396	399	readonly importLegacyShells: Effect.Effect<Import, ImportError, ImportContext>;
397	400	readonly recover: Effect.Effect<Recovery, RecoveryError, RecoveryContext>;
	401	/** Settles delegated tasks whose runs recovery just terminalized. */
	402	readonly recoverDelegatedTasks: Effect.Effect<void, DelegationError, DelegationContext>;
398	403	readonly startEffectWorker: Effect.Effect<void, WorkerError, WorkerContext>;
399	404	readonly autoBootstrap: Effect.Effect<Bootstrap, BootstrapError, BootstrapContext>;
400	405	}) {
401	406	return Effect.gen(function* () {
402	407	yield* input.importLegacyShells;
403	408	const recovery = yield* input.recover;
	409	yield* input.recoverDelegatedTasks;
404	410	yield* input.startEffectWorker;
405	411	const bootstrap = yield* input.autoBootstrap;
406	412	return { recovery, bootstrap } as const;
...	...	@@ -413,6 +419,7 @@ const make = (options?: StartupOptions) =>
413	419	const keybindings = yield* Keybindings.Keybindings;
414	420	const legacyV1ThreadImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;
415	421	const providerRuntimeRecovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
	422	const orchestrator = yield* Orchestrator.OrchestratorV2;
416	423	const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
417	424	const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
418	425	const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents;
...	...	@@ -509,6 +516,10 @@ const make = (options?: StartupOptions) =>
509	516	),
510	517	),
511	518	recover: runStartupPhase("orchestration-v2.recovery", providerRuntimeRecovery.recover),
	519	recoverDelegatedTasks: runStartupPhase(
	520	"orchestration-v2.delegated-tasks.recover",
	521	orchestrator.recoverDelegatedTasks,
	522	),
512	523	startEffectWorker: runStartupPhase(
513	524	"orchestration-v2.effect-worker.start",
514	525	startEffectWorkerWithRelay({

docs/internals/server-updates.md+16-5

...	...	@@ -61,7 +61,9 @@ stopped backends and replays the failure for the same token.
61	61	Restart continuation is an environment-owned preference, off by default. The
62	62	[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)
63	63	requires matching durable run, provider thread, session, and native resume identity.
64		Ordinary queued work, finished runs, and background-only work do not qualify.
	64	Queued runs never started, so recovery holds them and continues the run they wait
	65	behind. A finished run qualifies only when the restart cancelled its background work;
	66	its continuation tells the provider what will not report back.
65	67
66	68	Recovery retires effects tied to the lost process and records continuation intent
67	69	in the durable outbox. That intent survives another restart before provider startup.
...	...	@@ -71,7 +73,16 @@ closing providers, then reconciles after ingestion has stopped so a late complet
71	73	cannot be overwritten by a stale cancellation.
72	74
73	75	The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)
74		rechecks the preference, archive state, provider selection, and newer user work before
75		dispatching. Stable command and message IDs prevent duplicate submissions after an
76		outbox retry. Codex resumes without adding provider prompt text; other adapters receive
77		the continuation message through their normal turn path.
	76	rechecks the preference, archive state, provider selection, newer user work, a stop
	77	the user requested, and maintenance turns such as `/compact` before dispatching. Stable
	78	command and message IDs prevent duplicate submissions after an outbox retry. Codex
	79	resumes without adding provider prompt text unless the turn lost background work;
	80	other adapters receive the continuation message through their normal turn path.
	81
	82	Delegated tasks (`delegate_task` child threads) are reconciled as their own threads,
	83	never as the parent's background work. The orchestrator settles child results and
	84	completion deliveries in a startup pass after reconciliation, because the terminal-run
	85	listener ignores reconciliation's cancellations. A cancelled child whose restart
	86	continuation is still pending in the outbox is not a result yet; the continuation's run
	87	settles it, or the handler settles it when it declines to continue. Schedulers wait for
	88	activation so they cannot start runs that reconciliation would then cancel.

packages/shared/src/orchestrationV2ThreadError.ts+18-1

...	...	@@ -55,11 +55,28 @@ export function latestExecutedRun(
55	55	for (const run of runs) {
56	56	if (run.status === "queued") continue;
57	57	if (run.status === "cancelled" && run.startedAt === null) continue;
58		if (latest === null \|\| run.ordinal > latest.ordinal) latest = run;
	58	if (latest === null \|\| runRanAfter(run, latest)) latest = run;
59	59	}
60	60	return latest;
61	61	}
62	62
	63	/**
	64	* Whether started `run` ran after `other`. Ordinals follow submission, but a
	65	* run can start ahead of a held queue (a restart continuation, or a message
	66	* sent while the queue is held), so a queued run resumed later can have a
	67	* lower ordinal than one that already ended. An unfinished run is the latest.
	68	*/
	69	export function runRanAfter(
	70	run: Pick<OrchestrationV2Run, "ordinal" \| "completedAt">,
	71	other: Pick<OrchestrationV2Run, "ordinal" \| "completedAt">,
	72	): boolean {
	73	const end = (candidate: typeof run) =>
	74	!candidate.completedAt
	75	? Number.POSITIVE_INFINITY
	76	: DateTime.toEpochMillis(candidate.completedAt);
	77	return end(run) === end(other) ? run.ordinal > other.ordinal : end(run) > end(other);
	78	}
	79
63	80	/**
64	81	* The latest run that actually started, when it stopped because the
65	82	* subscription limit was reached. Queued messages after that run must stay