1import { assert, it, vi } from "@effect/vitest";
2import {
3 CheckpointScopeId,
4 ChatAttachmentId,
5 ChatFileAttachment,
6 CommandId,
7 EventId,
8 MessageId,
9 NodeId,
10 type OrchestrationV2AppThread,
11 type OrchestrationV2CheckpointScope,
12 type OrchestrationV2DomainEvent,
13 type OrchestrationV2ExecutionNode,
14 type OrchestrationV2ProviderThread,
15 type OrchestrationV2Run,
16 type OrchestrationV2RunAttempt,
17 type OrchestrationV2Subagent,
18 type OrchestrationV2TurnItem,
19 ProviderDriverKind,
20 ProviderInstanceId,
21 ProviderSessionId,
22 ProviderThreadId,
23 ProviderTurnId,
24 RunAttemptId,
25 RunId,
26 ServerSettingsError,
27 ThreadId,
28 TurnItemId,
29} from "@t3tools/contracts";
30import * as Cause from "effect/Cause";
31import * as Exit from "effect/Exit";
32import * as DateTime from "effect/DateTime";
33import * as Deferred from "effect/Deferred";
34import * as Effect from "effect/Effect";
35import * as Layer from "effect/Layer";
36import * as Option from "effect/Option";
37import * as Ref from "effect/Ref";
38import * as Stream from "effect/Stream";
39
40import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
41import * as ServerSettings from "../serverSettings.ts";
42import * as CheckpointService from "./CheckpointService.ts";
43import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
44import * as EventSink from "./EventSink.ts";
45import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
46import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
47import * as ProjectionStore from "./ProjectionStore.ts";
48import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
49import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
50import * as RunExecutionService from "./RunExecutionService.ts";
51import * as RunFinalizationService from "./RunFinalizationService.ts";
52
53const driver = ProviderDriverKind.make("codex");
54
55const layerRunExecutionTest = RunExecutionService.layer.pipe(
56 Layer.provide(
57 Layer.mergeAll(
58 McpAppModelContext.layerEmpty,
59 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
60 Layer.mock(EventSink.EventSinkV2)({}),
61 IdAllocator.layer,
62 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
63 ingestNormalized: () => Effect.succeed([]),
64 }),
65 ServerSettings.layerTest(),
66 ),
67 ),
68);
69
70it("keeps recoverable turn failures reusable and reserves error for broken threads", () => {
71 assert.equal(RunExecutionService.finalProviderThreadStatus("reusable"), "idle");
72 assert.equal(RunExecutionService.finalProviderThreadStatus("broken"), "error");
73});
74
75it.effect("routes shared-runtime events only to their owning root run", () =>
76 Effect.gen(function* () {
77 const now = yield* DateTime.now;
78 const first: RunExecutionService.ProviderEventRouteIdentity = {
79 threadId: ThreadId.make("thread:shared-runtime:first"),
80 runId: RunId.make("run:shared-runtime:first"),
81 attemptId: RunAttemptId.make("attempt:shared-runtime:first"),
82 providerThreadId: ProviderThreadId.make("provider-thread:shared-runtime:first"),
83 };
84 const second: RunExecutionService.ProviderEventRouteIdentity = {
85 threadId: ThreadId.make("thread:shared-runtime:second"),
86 runId: RunId.make("run:shared-runtime:second"),
87 attemptId: RunAttemptId.make("attempt:shared-runtime:second"),
88 providerThreadId: ProviderThreadId.make("provider-thread:shared-runtime:second"),
89 };
90 const firstTurnId = ProviderTurnId.make("provider-turn:shared-runtime:first");
91 const turnEvent: ProviderAdapter.ProviderAdapterV2Event = {
92 type: "provider_turn.updated",
93 driver,
94 threadId: first.threadId,
95 providerTurn: {
96 id: firstTurnId,
97 providerThreadId: first.providerThreadId,
98 nodeId: NodeId.make("node:shared-runtime:first"),
99 runAttemptId: first.attemptId,
100 nativeTurnRef: null,
101 ordinal: 1,
102 status: "running",
103 startedAt: now,
104 completedAt: null,
105 },
106 };
107 const messageEvent: ProviderAdapter.ProviderAdapterV2Event = {
108 type: "message.updated",
109 driver,
110 message: {
111 createdBy: "agent",
112 creationSource: "provider",
113 id: MessageId.make("message:shared-runtime:first"),
114 threadId: first.threadId,
115 runId: first.runId,
116 nodeId: NodeId.make("node:shared-runtime:first"),
117 role: "assistant",
118 text: "first only",
119 attachments: [],
120 streaming: false,
121 createdAt: now,
122 updatedAt: now,
123 },
124 };
125 const terminalEvent: ProviderAdapter.ProviderAdapterV2Event = {
126 type: "turn.terminal",
127 driver,
128 providerThreadId: first.providerThreadId,
129 providerTurnId: firstTurnId,
130 runOrdinal: 1,
131 status: "completed",
132 failure: null,
133 threadDisposition: "reusable",
134 };
135
136 const firstInitial = RunExecutionService.makeProviderEventRoutingState({
137 identity: first,
138 providerTurnId: null,
139 });
140 const secondInitial = RunExecutionService.makeProviderEventRoutingState({
141 identity: second,
142 providerTurnId: null,
143 });
144 const [firstTurnAccepted, firstAfterTurn] = RunExecutionService.routeProviderEvent(
145 turnEvent,
146 first,
147 firstInitial,
148 );
149 const [secondTurnAccepted, secondAfterTurn] = RunExecutionService.routeProviderEvent(
150 turnEvent,
151 second,
152 secondInitial,
153 );
154
155 assert.isTrue(firstTurnAccepted);
156 assert.isFalse(secondTurnAccepted);
157 assert.isTrue(RunExecutionService.routeProviderEvent(messageEvent, first, firstAfterTurn)[0]);
158 assert.isFalse(
159 RunExecutionService.routeProviderEvent(messageEvent, second, secondAfterTurn)[0],
160 );
161 assert.isTrue(RunExecutionService.routeProviderEvent(terminalEvent, first, firstAfterTurn)[0]);
162 assert.isFalse(
163 RunExecutionService.routeProviderEvent(terminalEvent, second, secondAfterTurn)[0],
164 );
165 }),
166);
167
168it("routes only enrolled workflow members after the root turn ends", () => {
169 const threadId = ThreadId.make("thread:late-child");
170 const rootProviderTurnId = ProviderTurnId.make("provider-turn:late-child");
171 const identity: RunExecutionService.ProviderEventRouteIdentity = {
172 threadId,
173 runId: RunId.make("run:late-child"),
174 attemptId: RunAttemptId.make("attempt:late-child"),
175 providerThreadId: ProviderThreadId.make("provider-thread:late-child"),
176 };
177 const childCreated = (
178 childThreadId: ThreadId,
179 parentThreadId = threadId,
180 ): ProviderAdapter.ProviderAdapterV2Event =>
181 ({
182 type: "app_thread.created",
183 driver,
184 appThread: {
185 id: childThreadId,
186 lineage: {
187 parentThreadId,
188 relationshipToParent: "subagent",
189 rootThreadId: threadId,
190 },
191 },
192 }) as ProviderAdapter.ProviderAdapterV2Event;
193 const earlyChild = ThreadId.make("thread:late-child:early");
194 const lateChild = ThreadId.make("thread:late-child:late");
195
196 const initial = RunExecutionService.makeProviderEventRoutingState({
197 identity,
198 providerTurnId: rootProviderTurnId,
199 });
200 const [earlyAccepted, live] = RunExecutionService.routeProviderEvent(
201 childCreated(earlyChild),
202 identity,
203 initial,
204 );
205 assert.isTrue(earlyAccepted);
206 const existingDescendant = ThreadId.make("thread:late-child:existing-descendant");
207 const [descendantAccepted, withDescendant] = RunExecutionService.routeProviderEvent(
208 childCreated(existingDescendant, earlyChild),
209 identity,
210 live,
211 );
212 assert.isTrue(descendantAccepted);
213 const [terminalAccepted, ended] = RunExecutionService.routeProviderEvent(
214 {
215 type: "turn.terminal",
216 driver,
217 providerThreadId: identity.providerThreadId,
218 providerTurnId: rootProviderTurnId,
219 runOrdinal: 1,
220 status: "completed",
221 failure: null,
222 threadDisposition: "reusable",
223 },
224 identity,
225 withDescendant,
226 );
227 assert.isTrue(terminalAccepted);
228 // A child the root launched before it ended stays with this run.
229 assert.isTrue(ended.ownedThreadIds.has(earlyChild));
230 const [lateAccepted, afterLate] = RunExecutionService.routeProviderEvent(
231 childCreated(lateChild),
232 identity,
233 ended,
234 );
235 assert.isFalse(lateAccepted);
236 assert.isFalse(afterLate.ownedThreadIds.has(lateChild));
237
238 const memberId = ThreadId.make("thread:late-child:workflow-member");
239 assert.isFalse(
240 RunExecutionService.routeProviderEvent(childCreated(memberId, earlyChild), identity, ended)[0],
241 );
242 const coordinator: OrchestrationV2Subagent = {
243 id: NodeId.make("node:late-child:workflow"),
244 threadId,
245 runId: identity.runId,
246 parentNodeId: NodeId.make("node:late-child"),
247 origin: "provider_native",
248 createdBy: "agent",
249 driver,
250 providerInstanceId: ProviderInstanceId.make("codex"),
251 providerThreadId: null,
252 childThreadId: earlyChild,
253 nativeTaskRef: null,
254 prompt: "Review the code",
255 title: "Review",
256 model: null,
257 status: "running",
258 result: null,
259 startedAt: null,
260 completedAt: null,
261 updatedAt: DateTime.makeUnsafe(0),
262 workflow: {
263 phases: [],
264 agents: [{ index: 0, label: "Reviewer", state: "running", childThreadId: memberId }],
265 },
266 };
267 const [foreignAccepted, afterForeign] = RunExecutionService.routeProviderEvent(
268 {
269 type: "subagent.updated",
270 driver,
271 subagent: {
272 ...coordinator,
273 threadId: earlyChild,
274 childThreadId: existingDescendant,
275 runId: RunId.make("run:late-child:new"),
276 },
277 },
278 identity,
279 ended,
280 );
281 assert.isTrue(foreignAccepted);
282 assert.isFalse(
283 RunExecutionService.routeProviderEvent(
284 childCreated(memberId, existingDescendant),
285 identity,
286 afterForeign,
287 )[0],
288 );
289 const [workflowAccepted, afterWorkflow] = RunExecutionService.routeProviderEvent(
290 { type: "subagent.updated", driver, subagent: coordinator },
291 identity,
292 ended,
293 );
294 assert.isTrue(workflowAccepted);
295 assert.isFalse(
296 RunExecutionService.routeProviderEvent(childCreated(memberId), identity, afterWorkflow)[0],
297 );
298 const [memberAccepted, afterMember] = RunExecutionService.routeProviderEvent(
299 childCreated(memberId, earlyChild),
300 identity,
301 afterWorkflow,
302 );
303 assert.isTrue(memberAccepted);
304 assert.isTrue(afterMember.ownedThreadIds.has(memberId));
305 const [unrelatedAccepted] = RunExecutionService.routeProviderEvent(
306 childCreated(ThreadId.make("thread:late-child:unrelated-member"), lateChild),
307 identity,
308 ended,
309 );
310 assert.isFalse(unrelatedAccepted);
311 const [unrelatedDescendantAccepted, afterUnrelatedDescendant] =
312 RunExecutionService.routeProviderEvent(
313 childCreated(ThreadId.make("thread:late-child:unrelated-descendant"), earlyChild),
314 identity,
315 afterWorkflow,
316 );
317 assert.isFalse(unrelatedDescendantAccepted);
318 assert.isFalse(
319 afterUnrelatedDescendant.ownedThreadIds.has(
320 ThreadId.make("thread:late-child:unrelated-descendant"),
321 ),
322 );
323});
324
325it("does not route a superseded attempt through a reused provider thread", () => {
326 const threadId = ThreadId.make("thread:shared-runtime:restart");
327 const providerThreadId = ProviderThreadId.make("provider-thread:shared-runtime:restart");
328 const oldAttempt: RunExecutionService.ProviderEventRouteIdentity = {
329 threadId,
330 runId: RunId.make("run:shared-runtime:restart"),
331 attemptId: RunAttemptId.make("attempt:shared-runtime:restart:old"),
332 providerThreadId,
333 };
334 const newAttempt: RunExecutionService.ProviderEventRouteIdentity = {
335 ...oldAttempt,
336 attemptId: RunAttemptId.make("attempt:shared-runtime:restart:new"),
337 };
338 const oldTurnEvent: ProviderAdapter.ProviderAdapterV2Event = {
339 type: "provider_turn.updated",
340 driver,
341 threadId,
342 providerTurn: {
343 id: ProviderTurnId.make("provider-turn:shared-runtime:restart:old"),
344 providerThreadId,
345 nodeId: NodeId.make("node:shared-runtime:restart:old"),
346 runAttemptId: oldAttempt.attemptId,
347 nativeTurnRef: null,
348 ordinal: 1,
349 status: "interrupted",
350 startedAt: null,
351 completedAt: null,
352 },
353 };
354
355 const newState = RunExecutionService.makeProviderEventRoutingState({
356 identity: newAttempt,
357 providerTurnId: null,
358 });
359 assert.isFalse(RunExecutionService.routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]);
360});
361
362it("routes only exact same-thread background items inherited from settled runs", () => {
363 const threadId = ThreadId.make("thread:inherited-background-routing");
364 const otherThreadId = ThreadId.make("thread:inherited-background-routing:other");
365 const priorRunId = RunId.make("run:inherited-background-routing:prior");
366 const currentRunId = RunId.make("run:inherited-background-routing:current");
367 const itemId = TurnItemId.make("turn-item:inherited-background-routing");
368 const identity: RunExecutionService.ProviderEventRouteIdentity = {
369 threadId,
370 runId: currentRunId,
371 attemptId: RunAttemptId.make("attempt:inherited-background-routing:current"),
372 providerThreadId: ProviderThreadId.make("provider-thread:inherited-background-routing:current"),
373 };
374 const initial = RunExecutionService.makeProviderEventRoutingState({
375 identity,
376 inheritedBackgroundTurnItems: [{ id: itemId, runId: priorRunId }],
377 providerTurnId: null,
378 });
379 const inheritedRunning = {
380 type: "turn_item.updated",
381 driver,
382 turnItem: {
383 id: itemId,
384 threadId,
385 runId: priorRunId,
386 providerTurnId: null,
387 ordinal: 1,
388 type: "subagent",
389 status: "running",
390 },
391 } as Extract<ProviderAdapter.ProviderAdapterV2Event, { type: "turn_item.updated" }>;
392 const unrelatedRunItem = {
393 ...inheritedRunning,
394 turnItem: {
395 ...inheritedRunning.turnItem,
396 runId: RunId.make("run:inherited-background-routing:unrelated"),
397 },
398 } as ProviderAdapter.ProviderAdapterV2Event;
399 const unlistedPriorItem = {
400 ...inheritedRunning,
401 turnItem: {
402 ...inheritedRunning.turnItem,
403 id: TurnItemId.make("turn-item:inherited-background-routing:unrelated"),
404 },
405 } as ProviderAdapter.ProviderAdapterV2Event;
406 const unrelatedThreadItem = {
407 ...inheritedRunning,
408 turnItem: { ...inheritedRunning.turnItem, threadId: otherThreadId },
409 } as ProviderAdapter.ProviderAdapterV2Event;
410 const ordinaryItem = {
411 ...inheritedRunning,
412 turnItem: { ...inheritedRunning.turnItem, type: "reasoning" as const },
413 } as ProviderAdapter.ProviderAdapterV2Event;
414 const inheritedTerminal = {
415 ...inheritedRunning,
416 turnItem: { ...inheritedRunning.turnItem, status: "completed" as const },
417 } as ProviderAdapter.ProviderAdapterV2Event;
418
419 const [runningAccepted, afterRunning] = RunExecutionService.routeProviderEvent(
420 inheritedRunning,
421 identity,
422 initial,
423 );
424 assert.isTrue(runningAccepted);
425 assert.isFalse(
426 RunExecutionService.routeProviderEvent(unrelatedRunItem, identity, afterRunning)[0],
427 );
428 assert.isFalse(
429 RunExecutionService.routeProviderEvent(unlistedPriorItem, identity, afterRunning)[0],
430 );
431 assert.isFalse(
432 RunExecutionService.routeProviderEvent(unrelatedThreadItem, identity, afterRunning)[0],
433 );
434 assert.isFalse(RunExecutionService.routeProviderEvent(ordinaryItem, identity, afterRunning)[0]);
435
436 const [terminalAccepted, afterTerminal] = RunExecutionService.routeProviderEvent(
437 inheritedTerminal,
438 identity,
439 afterRunning,
440 );
441 assert.isTrue(terminalAccepted);
442 assert.isFalse(
443 RunExecutionService.routeProviderEvent(inheritedRunning, identity, afterTerminal)[0],
444 "a nonterminal replay must not resurrect an inherited terminal",
445 );
446});
447
448it("selects only live background items from non-completed settled prior runs", () => {
449 const threadId = ThreadId.make("thread:inherited-background-selection");
450 const otherThreadId = ThreadId.make("thread:inherited-background-selection:other");
451 const currentProviderThreadId = ProviderThreadId.make(
452 "provider-thread:inherited-background-selection:current",
453 );
454 const foreignProviderThreadId = ProviderThreadId.make(
455 "provider-thread:inherited-background-selection:foreign-session",
456 );
457 const interruptedRunId = RunId.make("run:inherited-background-selection:interrupted");
458 const failedRunId = RunId.make("run:inherited-background-selection:failed");
459 const cancelledRunId = RunId.make("run:inherited-background-selection:cancelled");
460 const completedRunId = RunId.make("run:inherited-background-selection:completed");
461 const rolledBackRunId = RunId.make("run:inherited-background-selection:rolled-back");
462 const currentRunId = RunId.make("run:inherited-background-selection:current");
463 const inheritedItemId = TurnItemId.make("turn-item:inherited-background-selection:live");
464 const failedItemId = TurnItemId.make("turn-item:inherited-background-selection:failed");
465 const cancelledItemId = TurnItemId.make("turn-item:inherited-background-selection:cancelled");
466 const makeRun = (
467 id: RunId,
468 ordinal: number,
469 status: OrchestrationV2Run["status"],
470 runThreadId = threadId,
471 ) =>
472 ({
473 id,
474 threadId: runThreadId,
475 ordinal,
476 status,
477 }) as OrchestrationV2Run;
478 const makeItem = (
479 id: TurnItemId,
480 runId: RunId,
481 status: OrchestrationV2TurnItem["status"],
482 type: OrchestrationV2TurnItem["type"] = "subagent",
483 itemThreadId = threadId,
484 providerThreadId = currentProviderThreadId,
485 ) =>
486 ({
487 id,
488 threadId: itemThreadId,
489 runId,
490 providerThreadId,
491 type,
492 status,
493 }) as OrchestrationV2TurnItem;
494
495 const selected = RunExecutionService.selectInheritedBackgroundTurnItems({
496 threadId,
497 currentProviderThreadId,
498 currentRunOrdinal: 6,
499 runs: [
500 makeRun(interruptedRunId, 1, "interrupted"),
501 makeRun(failedRunId, 2, "failed"),
502 makeRun(cancelledRunId, 3, "cancelled"),
503 makeRun(completedRunId, 4, "completed"),
504 makeRun(rolledBackRunId, 5, "rolled_back"),
505 makeRun(currentRunId, 6, "running"),
506 makeRun(
507 RunId.make("run:inherited-background-selection:other"),
508 1,
509 "interrupted",
510 otherThreadId,
511 ),
512 ],
513 turnItems: [
514 makeItem(inheritedItemId, interruptedRunId, "running"),
515 makeItem(failedItemId, failedRunId, "running"),
516 makeItem(cancelledItemId, cancelledRunId, "running"),
517 makeItem(
518 TurnItemId.make("turn-item:inherited-background-selection:completed-run"),
519 completedRunId,
520 "running",
521 ),
522 makeItem(
523 TurnItemId.make("turn-item:inherited-background-selection:terminal"),
524 interruptedRunId,
525 "completed",
526 ),
527 makeItem(
528 TurnItemId.make("turn-item:inherited-background-selection:ordinary"),
529 interruptedRunId,
530 "running",
531 "reasoning",
532 ),
533 makeItem(
534 TurnItemId.make("turn-item:inherited-background-selection:rolled-back"),
535 rolledBackRunId,
536 "running",
537 ),
538 makeItem(
539 TurnItemId.make("turn-item:inherited-background-selection:other-thread"),
540 interruptedRunId,
541 "running",
542 "subagent",
543 otherThreadId,
544 ),
545 makeItem(
546 TurnItemId.make("turn-item:inherited-background-selection:foreign-provider"),
547 interruptedRunId,
548 "running",
549 "subagent",
550 threadId,
551 foreignProviderThreadId,
552 ),
553 ],
554 });
555
556 assert.deepEqual(selected, [
557 { id: inheritedItemId, runId: interruptedRunId },
558 { id: failedItemId, runId: failedRunId },
559 { id: cancelledItemId, runId: cancelledRunId },
560 ]);
561});
562
563it("does not carry interrupted or still-running child ownership into later attempts", () => {
564 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("interrupted"));
565 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("failed"));
566 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("cancelled"));
567 assert.isTrue(RunExecutionService.canRouteRelatedSubagent("completed"));
568 // The launching run still ingests a running subagent's child thread.
569 assert.isFalse(RunExecutionService.canRouteRelatedSubagent("running"));
570
571 const threadId = ThreadId.make("thread:related-child:next-attempt");
572 const childThreadId = ThreadId.make("thread:related-child:interrupted");
573 const identity: RunExecutionService.ProviderEventRouteIdentity = {
574 threadId,
575 runId: RunId.make("run:related-child:next-attempt"),
576 attemptId: RunAttemptId.make("attempt:related-child:next-attempt"),
577 providerThreadId: ProviderThreadId.make("provider-thread:related-child:next-attempt"),
578 };
579 const state = RunExecutionService.makeProviderEventRoutingState({
580 identity,
581 providerTurnId: null,
582 relatedThreadIds: RunExecutionService.canRouteRelatedSubagent("interrupted")
583 ? [childThreadId]
584 : [],
585 });
586 const childNodeId = NodeId.make("node:related-child:interrupted");
587 const lateChildNode = {
588 type: "node.updated",
589 driver,
590 node: {
591 id: childNodeId,
592 threadId: childThreadId,
593 runId: null,
594 parentNodeId: null,
595 rootNodeId: childNodeId,
596 kind: "root_turn",
597 status: "completed",
598 countsForRun: false,
599 providerThreadId: null,
600 providerTurnId: null,
601 nativeItemRef: null,
602 runtimeRequestId: null,
603 checkpointScopeId: null,
604 startedAt: null,
605 completedAt: null,
606 },
607 } satisfies ProviderAdapter.ProviderAdapterV2Event;
608
609 assert.isFalse(RunExecutionService.routeProviderEvent(lateChildNode, identity, state)[0]);
610});
611
612it.effect("rechecks run ownership immediately before calling the provider", () =>
613 Effect.gen(function* () {
614 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
615 const guardCalls = yield* Ref.make(0);
616 const providerStarts = yield* Ref.make(0);
617 const threadId = ThreadId.make("thread:run-execution-start-guard");
618 const runId = RunId.make("run:run-execution-start-guard");
619 const attemptId = RunAttemptId.make("attempt:run-execution-start-guard");
620 const providerThreadId = ProviderThreadId.make("provider-thread:run-execution-start-guard");
621 const providerInstanceId = ProviderInstanceId.make("codex");
622 const providerSessionId = ProviderSessionId.make("session:run-execution-start-guard");
623 const rootNodeId = NodeId.make("node:run-execution-start-guard");
624 const run = {
625 id: runId,
626 threadId,
627 ordinal: 1,
628 providerInstanceId,
629 } as OrchestrationV2Run;
630 const rootNode = { id: rootNodeId } as OrchestrationV2ExecutionNode;
631 const providerThread = {
632 id: providerThreadId,
633 driver,
634 } as OrchestrationV2ProviderThread;
635 const attempt = {
636 id: attemptId,
637 providerTurnId: null,
638 } as OrchestrationV2RunAttempt;
639 const session = {
640 events: Stream.never,
641 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
642 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
643
644 yield* runExecution.startRootRun({
645 commandId: CommandId.make("command:run-execution-start-guard"),
646 appThread: { id: threadId } as OrchestrationV2AppThread,
647 providerSessionId,
648 session,
649 run,
650 rootNode,
651 checkpointScope: {
652 id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard"),
653 } as OrchestrationV2CheckpointScope,
654 providerThread,
655 attempt,
656 attemptId,
657 providerTurnOrdinal: 1,
658 shouldStartProviderTurn: () =>
659 Ref.modify(guardCalls, (calls) => [calls === 0, calls + 1] as const),
660 message: {
661 messageId: MessageId.make("message:run-execution-start-guard"),
662 text: "Do not start after ownership changes.",
663 attachments: [],
664 createdBy: "user",
665 creationSource: "web",
666 },
667 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
668 runtimePolicy: {
669 runtimeMode: "full-access",
670 interactionMode: "default",
671 cwd: process.cwd(),
672 approvalPolicy: "never",
673 sandboxPolicy: {
674 type: "readOnly",
675 access: { type: "fullAccess" },
676 networkAccess: false,
677 },
678 },
679 });
680
681 assert.equal(yield* Ref.get(guardCalls), 2);
682 assert.equal(yield* Ref.get(providerStarts), 0);
683 }).pipe(Effect.provide(layerRunExecutionTest)),
684);
685
686it.effect("passes the thread's MCP app context to the provider under a safe key", () =>
687 Effect.gen(function* () {
688 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
689 const started = yield* Deferred.make<ProviderAdapter.ProviderAdapterV2TurnInput>();
690 const threadId = ThreadId.make("thread:run-execution-app-context");
691 const providerInstanceId = ProviderInstanceId.make("codex");
692 const attemptId = RunAttemptId.make("attempt:run-execution-app-context");
693 const session = {
694 events: Stream.never,
695 startTurn: (input: ProviderAdapter.ProviderAdapterV2TurnInput) =>
696 Deferred.succeed(started, input),
697 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
698
699 yield* runExecution.startRootRun({
700 commandId: CommandId.make("command:run-execution-app-context"),
701 appThread: { id: threadId } as OrchestrationV2AppThread,
702 providerSessionId: ProviderSessionId.make("session:run-execution-app-context"),
703 session,
704 run: {
705 id: RunId.make("run:run-execution-app-context"),
706 threadId,
707 ordinal: 1,
708 providerInstanceId,
709 } as OrchestrationV2Run,
710 rootNode: {
711 id: NodeId.make("node:run-execution-app-context"),
712 } as OrchestrationV2ExecutionNode,
713 checkpointScope: {
714 id: CheckpointScopeId.make("checkpoint-scope:run-execution-app-context"),
715 } as OrchestrationV2CheckpointScope,
716 providerThread: {
717 id: ProviderThreadId.make("provider-thread:run-execution-app-context"),
718 driver,
719 } as OrchestrationV2ProviderThread,
720 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
721 attemptId,
722 providerTurnOrdinal: 1,
723 message: {
724 messageId: MessageId.make("message:run-execution-app-context"),
725 text: "What is on my list?",
726 attachments: [],
727 createdBy: "user",
728 creationSource: "web",
729 },
730 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
731 runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: process.cwd() },
732 });
733
734 const input = yield* Deferred.await(started);
735 // Item ids carry colons, and server names are free text; neither reaches the key.
736 assert.deepEqual(input.appContext, [
737 { key: "mcp_app_turn-item_provider_codex_native-item_call-1", text: "2 overdue" },
738 ]);
739 }).pipe(
740 Effect.provide(
741 RunExecutionService.layer.pipe(
742 Layer.provide(
743 Layer.mergeAll(
744 Layer.mock(McpAppModelContext.McpAppModelContext)({
745 forThread: () =>
746 Effect.succeed([
747 {
748 itemId: "turn-item:provider:codex:native-item:call-1",
749 server: "my tools>",
750 tool: "list",
751 text: "2 overdue",
752 },
753 ]),
754 }),
755 Layer.mock(CheckpointService.CheckpointServiceV2)({
756 captureBaseline: () => Effect.void,
757 }),
758 Layer.mock(EventSink.EventSinkV2)({}),
759 IdAllocator.layer,
760 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
761 ingestNormalized: () => Effect.succeed([]),
762 }),
763 ServerSettings.layerTest(),
764 ),
765 ),
766 ),
767 ),
768 ),
769);
770
771it.effect("fails the run when its ownership check cannot be read before calling the provider", () =>
772 Effect.gen(function* () {
773 const guardCalls = yield* Ref.make(0);
774 const providerStarts = yield* Ref.make(0);
775 const writes = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
776 const threadId = ThreadId.make("thread:run-execution-start-guard-read");
777 const runId = RunId.make("run:run-execution-start-guard-read");
778 const attemptId = RunAttemptId.make("attempt:run-execution-start-guard-read");
779 const providerInstanceId = ProviderInstanceId.make("codex");
780 const layerTest = RunExecutionService.layer.pipe(
781 Layer.provide(
782 Layer.mergeAll(
783 McpAppModelContext.layerEmpty,
784 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
785 Layer.mock(EventSink.EventSinkV2)({
786 writeIfRunCurrent: (input) =>
787 Ref.update(writes, (current) => [...current, ...input.events]).pipe(
788 Effect.as({ committed: true, storedEvents: [] }),
789 ),
790 }),
791 IdAllocator.layer,
792 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
793 ingestNormalized: () => Effect.succeed([]),
794 }),
795 ServerSettings.layerTest(),
796 ),
797 ),
798 );
799
800 yield* Effect.gen(function* () {
801 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
802 yield* runExecution.startRootRun({
803 commandId: CommandId.make("command:run-execution-start-guard-read"),
804 appThread: { id: threadId } as OrchestrationV2AppThread,
805 providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"),
806 session: {
807 events: Stream.never,
808 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
809 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
810 run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run,
811 rootNode: {
812 id: NodeId.make("node:run-execution-start-guard-read"),
813 } as OrchestrationV2ExecutionNode,
814 checkpointScope: {
815 id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard-read"),
816 } as OrchestrationV2CheckpointScope,
817 providerThread: {
818 id: ProviderThreadId.make("provider-thread:run-execution-start-guard-read"),
819 driver,
820 } as OrchestrationV2ProviderThread,
821 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
822 attemptId,
823 providerTurnOrdinal: 1,
824 // The preparation check passes; the check right before the provider
825 // call cannot read the run.
826 shouldStartProviderTurn: () =>
827 Ref.getAndUpdate(guardCalls, (calls) => calls + 1).pipe(
828 Effect.flatMap((calls) =>
829 calls === 0
830 ? Effect.succeed(true)
831 : Effect.fail(
832 new ProjectionStore.ProjectionStoreReadError({
833 threadId,
834 cause: "database unavailable",
835 }),
836 ),
837 ),
838 ),
839 // The failure is settled by the guarded write, not by another read.
840 shouldFinalizeRun: () =>
841 Effect.fail(
842 new ProjectionStore.ProjectionStoreReadError({
843 threadId,
844 cause: "database unavailable",
845 }),
846 ),
847 message: {
848 messageId: MessageId.make("message:run-execution-start-guard-read"),
849 text: "Start while the store is down.",
850 attachments: [],
851 createdBy: "user",
852 creationSource: "web",
853 },
854 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
855 runtimePolicy: {
856 runtimeMode: "full-access",
857 interactionMode: "default",
858 cwd: process.cwd(),
859 approvalPolicy: "never",
860 sandboxPolicy: {
861 type: "readOnly",
862 access: { type: "fullAccess" },
863 networkAccess: false,
864 },
865 },
866 });
867 }).pipe(Effect.provide(layerTest));
868
869 assert.equal(yield* Ref.get(guardCalls), 2);
870 assert.equal(yield* Ref.get(providerStarts), 0);
871 const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated");
872 assert.equal(
873 runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined,
874 "failed",
875 );
876 }),
877);
878
879it.effect(
880 "dispatches only attachment-free compact commands through the native compaction path",
881 () =>
882 Effect.gen(function* () {
883 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
884 const calls: Array<string> = [];
885 const attachment = ChatFileAttachment.make({
886 type: "file",
887 id: ChatAttachmentId.make("compact-file-12345678-1234-1234-1234-123456789abc"),
888 name: "notes.txt",
889 mimeType: "text/plain",
890 sizeBytes: 1,
891 });
892 const cases = [
893 { text: " /compact ", attachments: [], expected: "compact" },
894 { text: "/compact", attachments: [attachment], expected: "prompt" },
895 { text: "Continue the work", attachments: [], expected: "prompt" },
896 ];
897 for (const [index, testCase] of cases.entries()) {
898 const threadId = ThreadId.make(`thread:compact-routing:${index}`);
899 const attemptId = RunAttemptId.make(`attempt:compact-routing:${index}`);
900 yield* runExecution.startRootRun({
901 commandId: CommandId.make(`command:compact-routing:${index}`),
902 appThread: { id: threadId } as OrchestrationV2AppThread,
903 providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`),
904 session: {
905 events: Stream.never,
906 startTurn: () =>
907 Effect.sync(() => {
908 calls.push("prompt");
909 }),
910 compactThread: () =>
911 Effect.sync(() => {
912 calls.push("compact");
913 }),
914 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
915 run: {
916 id: RunId.make(`run:compact-routing:${index}`),
917 threadId,
918 ordinal: 1,
919 providerInstanceId: ProviderInstanceId.make("codex"),
920 } as OrchestrationV2Run,
921 rootNode: {
922 id: NodeId.make(`node:compact-routing:${index}`),
923 } as OrchestrationV2ExecutionNode,
924 checkpointScope: {
925 id: CheckpointScopeId.make(`checkpoint:compact-routing:${index}`),
926 } as OrchestrationV2CheckpointScope,
927 providerThread: {
928 id: ProviderThreadId.make(`provider-thread:compact-routing:${index}`),
929 driver,
930 } as OrchestrationV2ProviderThread,
931 attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt,
932 attemptId,
933 providerTurnOrdinal: 1,
934 message: {
935 messageId: MessageId.make(`message:compact-routing:${index}`),
936 text: testCase.text,
937 attachments: testCase.attachments,
938 createdBy: "user",
939 creationSource: "web",
940 },
941 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
942 runtimePolicy: {
943 runtimeMode: "full-access",
944 interactionMode: "default",
945 cwd: process.cwd(),
946 approvalPolicy: "never",
947 sandboxPolicy: {
948 type: "readOnly",
949 access: { type: "fullAccess" },
950 networkAccess: false,
951 },
952 },
953 });
954 }
955 assert.deepEqual(
956 calls,
957 cases.map((testCase) => testCase.expected),
958 );
959 }).pipe(Effect.provide(layerRunExecutionTest)),
960);
961
962it.effect("refreshes MCP credential liveness before calling the provider", () =>
963 Effect.gen(function* () {
964 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
965 const order = yield* Ref.make<ReadonlyArray<string>>([]);
966 const threadId = ThreadId.make("thread:run-execution-mcp-liveness");
967 const touchActiveMcpThread = vi
968 .spyOn(McpSessionRegistry, "touchActiveMcpThread")
969 .mockImplementation((touchedThreadId) =>
970 Ref.update(order, (entries) => [...entries, `touch:${touchedThreadId}`]),
971 );
972
973 yield* runExecution
974 .startRootRun({
975 commandId: CommandId.make("command:run-execution-mcp-liveness"),
976 appThread: { id: threadId } as OrchestrationV2AppThread,
977 providerSessionId: ProviderSessionId.make("session:run-execution-mcp-liveness"),
978 session: {
979 events: Stream.never,
980 startTurn: () => Ref.update(order, (entries) => [...entries, "start-turn"]),
981 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
982 run: {
983 id: RunId.make("run:run-execution-mcp-liveness"),
984 threadId,
985 ordinal: 1,
986 providerInstanceId: ProviderInstanceId.make("codex"),
987 } as OrchestrationV2Run,
988 rootNode: {
989 id: NodeId.make("node:run-execution-mcp-liveness"),
990 } as OrchestrationV2ExecutionNode,
991 checkpointScope: {
992 id: CheckpointScopeId.make("checkpoint-scope:run-execution-mcp-liveness"),
993 } as OrchestrationV2CheckpointScope,
994 providerThread: {
995 id: ProviderThreadId.make("provider-thread:run-execution-mcp-liveness"),
996 driver,
997 } as OrchestrationV2ProviderThread,
998 attempt: {
999 id: RunAttemptId.make("attempt:run-execution-mcp-liveness"),
1000 providerTurnId: null,
1001 } as OrchestrationV2RunAttempt,
1002 attemptId: RunAttemptId.make("attempt:run-execution-mcp-liveness"),
1003 providerTurnOrdinal: 1,
1004 message: {
1005 messageId: MessageId.make("message:run-execution-mcp-liveness"),
1006 text: "Keep the MCP credential alive.",
1007 attachments: [],
1008 createdBy: "user",
1009 creationSource: "web",
1010 },
1011 modelSelection: {
1012 instanceId: ProviderInstanceId.make("codex"),
1013 model: "gpt-5.4",
1014 },
1015 runtimePolicy: {
1016 runtimeMode: "full-access",
1017 interactionMode: "default",
1018 cwd: process.cwd(),
1019 approvalPolicy: "never",
1020 sandboxPolicy: {
1021 type: "readOnly",
1022 access: { type: "fullAccess" },
1023 networkAccess: false,
1024 },
1025 },
1026 })
1027 .pipe(Effect.ensuring(Effect.sync(() => touchActiveMcpThread.mockRestore())));
1028
1029 assert.deepEqual(yield* Ref.get(order), [`touch:${threadId}`, "start-turn"]);
1030 }).pipe(Effect.provide(layerRunExecutionTest)),
1031);
1032
1033it.effect("starts the provider when checkpoint baseline capture fails", () =>
1034 Effect.gen(function* () {
1035 const threadId = ThreadId.make("thread:run-execution-baseline-failure");
1036 const runId = RunId.make("run:run-execution-baseline-failure");
1037 const attemptId = RunAttemptId.make("attempt:run-execution-baseline-failure");
1038 const providerThreadId = ProviderThreadId.make(
1039 "provider-thread:run-execution-baseline-failure",
1040 );
1041 const providerInstanceId = ProviderInstanceId.make("codex");
1042 const providerSessionId = ProviderSessionId.make("session:run-execution-baseline-failure");
1043 const rootNodeId = NodeId.make("node:run-execution-baseline-failure");
1044 const checkpointScope = {
1045 id: CheckpointScopeId.make("checkpoint-scope:run-execution-baseline-failure"),
1046 } as OrchestrationV2CheckpointScope;
1047 const providerStarts = yield* Ref.make(0);
1048 const writes = yield* Ref.make<
1049 ReadonlyArray<{
1050 events: ReadonlyArray<OrchestrationV2DomainEvent>;
1051 effects: ReadonlyArray<PendingOrchestrationEffectV2>;
1052 }>
1053 >([]);
1054 const layerTest = RunExecutionService.layer.pipe(
1055 Layer.provide(
1056 Layer.mergeAll(
1057 McpAppModelContext.layerEmpty,
1058 Layer.mock(CheckpointService.CheckpointServiceV2)({
1059 captureBaseline: () =>
1060 Effect.fail(
1061 new CheckpointService.CheckpointBaselineCaptureError({
1062 scopeId: checkpointScope.id,
1063 ordinalWithinScope: 0,
1064 cause: new Error("VCS process timed out"),
1065 }),
1066 ),
1067 }),
1068 Layer.mock(EventSink.EventSinkV2)({
1069 writeWithEffects: (input) =>
1070 Ref.update(writes, (current) => [
1071 ...current,
1072 { events: input.events, effects: input.effects },
1073 ]).pipe(Effect.as([])),
1074 }),
1075 IdAllocator.layer,
1076 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1077 ingestNormalized: () => Effect.succeed([]),
1078 }),
1079 ServerSettings.layerTest(),
1080 ),
1081 ),
1082 );
1083
1084 yield* Effect.gen(function* () {
1085 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1086 yield* runExecution.startRootRun({
1087 commandId: CommandId.make("command:run-execution-baseline-failure"),
1088 appThread: { id: threadId } as OrchestrationV2AppThread,
1089 providerSessionId,
1090 session: {
1091 events: Stream.never,
1092 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
1093 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1094 run: {
1095 id: runId,
1096 threadId,
1097 ordinal: 1,
1098 providerInstanceId,
1099 } as OrchestrationV2Run,
1100 rootNode: { id: rootNodeId } as OrchestrationV2ExecutionNode,
1101 checkpointScope,
1102 providerThread: {
1103 id: providerThreadId,
1104 driver,
1105 } as OrchestrationV2ProviderThread,
1106 attempt: {
1107 id: attemptId,
1108 providerTurnId: null,
1109 } as OrchestrationV2RunAttempt,
1110 attemptId,
1111 providerTurnOrdinal: 1,
1112 message: {
1113 messageId: MessageId.make("message:run-execution-baseline-failure"),
1114 text: "Start the provider without a baseline.",
1115 attachments: [],
1116 createdBy: "user",
1117 creationSource: "web",
1118 },
1119 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1120 runtimePolicy: {
1121 runtimeMode: "full-access",
1122 interactionMode: "default",
1123 cwd: process.cwd(),
1124 approvalPolicy: "never",
1125 sandboxPolicy: {
1126 type: "readOnly",
1127 access: { type: "fullAccess" },
1128 networkAccess: false,
1129 },
1130 },
1131 });
1132 }).pipe(Effect.provide(layerTest));
1133
1134 assert.equal(yield* Ref.get(providerStarts), 1);
1135
1136 const events = (yield* Ref.get(writes)).flatMap((write) => [...write.events]);
1137 const failedEvent = events.find(
1138 (event) =>
1139 (event.type === "run.updated" ||
1140 event.type === "run-attempt.updated" ||
1141 event.type === "node.updated") &&
1142 event.payload.status === "failed",
1143 );
1144 assert.isUndefined(failedEvent);
1145 const errorItem = events.find(
1146 (event) => event.type === "turn-item.updated" && event.payload.type === "error",
1147 );
1148 assert.isUndefined(errorItem);
1149 }),
1150);
1151
1152it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as const)(
1153 "handles %s before the provider turn starts",
1154 (scenario) =>
1155 Effect.gen(function* () {
1156 const threadId = ThreadId.make("thread:run-execution-settings-failure");
1157 const runId = RunId.make("run:run-execution-settings-failure");
1158 const attemptId = RunAttemptId.make("attempt:run-execution-settings-failure");
1159 const providerInstanceId = ProviderInstanceId.make("codex");
1160 const providerSessionId = ProviderSessionId.make("session:run-execution-settings-failure");
1161 const providerThreadId = ProviderThreadId.make(
1162 "provider-thread:run-execution-settings-failure",
1163 );
1164 const rootNodeId = NodeId.make("node:run-execution-settings-failure");
1165 const checkpointScope = {
1166 id: CheckpointScopeId.make("checkpoint-scope:run-execution-settings-failure"),
1167 } as OrchestrationV2CheckpointScope;
1168 const providerStarts = yield* Ref.make(0);
1169 const refreshes = yield* Ref.make(0);
1170 const guardedWrites = yield* Ref.make(0);
1171 const writes = yield* Ref.make<ReadonlyArray<ReadonlyArray<OrchestrationV2DomainEvent>>>([]);
1172 const layerTest = RunExecutionService.layer.pipe(
1173 Layer.provide(
1174 Layer.mergeAll(
1175 McpAppModelContext.layerEmpty,
1176 Layer.mock(CheckpointService.CheckpointServiceV2)({
1177 captureBaseline: () =>
1178 scenario === "start-guard" ? Effect.void : Effect.die("not reached"),
1179 }),
1180 Layer.mock(EventSink.EventSinkV2)({
1181 writeIfRunCurrent: (input) =>
1182 Effect.gen(function* () {
1183 assert.equal(input.threadId, threadId);
1184 assert.equal(input.runId, runId);
1185 assert.equal(input.activeAttemptId, attemptId);
1186 assert.equal(input.expectedStatus, "running");
1187 yield* Ref.update(guardedWrites, (count) => count + 1);
1188 if (scenario === "stale-attempt") {
1189 return { committed: false, storedEvents: [] };
1190 }
1191 yield* Ref.update(writes, (current) => [...current, input.events]);
1192 return { committed: true, storedEvents: [] };
1193 }),
1194 }),
1195 IdAllocator.layer,
1196 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1197 ingestNormalized: () => Effect.succeed([]),
1198 }),
1199 scenario === "start-guard"
1200 ? ServerSettings.layerTest()
1201 : Layer.mock(ServerSettings.ServerSettingsService)({
1202 getSettings:
1203 scenario === "interruption"
1204 ? Effect.interrupt
1205 : Effect.fail(
1206 new ServerSettingsError({
1207 settingsPath: "<test>",
1208 operation: "read-file",
1209 cause: new Error("settings read failed"),
1210 }),
1211 ),
1212 }),
1213 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
1214 refresh: () => Effect.void,
1215 refreshAfterTurn: () => Ref.update(refreshes, (count) => count + 1),
1216 }),
1217 ),
1218 ),
1219 );
1220
1221 const result = yield* Effect.gen(function* () {
1222 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1223 yield* runExecution.startRootRun({
1224 commandId: CommandId.make("command:run-execution-settings-failure"),
1225 appThread: { id: threadId } as OrchestrationV2AppThread,
1226 providerSessionId,
1227 session: {
1228 events: Stream.never,
1229 startTurn: () => Ref.update(providerStarts, (count) => count + 1),
1230 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1231 run: {
1232 id: runId,
1233 threadId,
1234 ordinal: 1,
1235 providerInstanceId,
1236 status: "running",
1237 } as OrchestrationV2Run,
1238 rootNode: { id: rootNodeId, status: "running" } as OrchestrationV2ExecutionNode,
1239 checkpointScope,
1240 providerThread: {
1241 id: providerThreadId,
1242 driver,
1243 } as OrchestrationV2ProviderThread,
1244 attempt: {
1245 id: attemptId,
1246 providerTurnId: null,
1247 status: "running",
1248 } as OrchestrationV2RunAttempt,
1249 attemptId,
1250 providerTurnOrdinal: 1,
1251 // A declined start is a normal exit, not a preparation failure.
1252 ...(scenario === "start-guard"
1253 ? { shouldStartProviderTurn: () => Effect.succeed(false) }
1254 : {}),
1255 message: {
1256 messageId: MessageId.make("message:run-execution-settings-failure"),
1257 text: "Start after settings fail.",
1258 attachments: [],
1259 createdBy: "user",
1260 creationSource: "web",
1261 },
1262 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1263 runtimePolicy: {
1264 runtimeMode: "full-access",
1265 interactionMode: "default",
1266 cwd: process.cwd(),
1267 approvalPolicy: "never",
1268 sandboxPolicy: {
1269 type: "readOnly",
1270 access: { type: "fullAccess" },
1271 networkAccess: false,
1272 },
1273 },
1274 });
1275 }).pipe(Effect.provide(layerTest), Effect.exit);
1276
1277 assert.equal(yield* Ref.get(providerStarts), 0);
1278 const events = (yield* Ref.get(writes)).flat();
1279 if (scenario === "interruption") {
1280 assert.isTrue(Exit.isFailure(result));
1281 if (Exit.isFailure(result)) assert.isTrue(Cause.hasInterruptsOnly(result.cause));
1282 assert.equal(yield* Ref.get(guardedWrites), 0);
1283 assert.equal(yield* Ref.get(refreshes), 0);
1284 assert.isEmpty(events);
1285 return;
1286 }
1287 assert.isTrue(Exit.isSuccess(result));
1288 if (scenario === "start-guard") {
1289 assert.equal(yield* Ref.get(guardedWrites), 0);
1290 assert.equal(yield* Ref.get(refreshes), 0);
1291 assert.isEmpty(events);
1292 return;
1293 }
1294 assert.equal(yield* Ref.get(guardedWrites), 1);
1295 if (scenario === "stale-attempt") {
1296 assert.equal(yield* Ref.get(refreshes), 0);
1297 assert.isEmpty(events);
1298 return;
1299 }
1300 assert.equal(yield* Ref.get(refreshes), 1);
1301 assert.deepEqual(
1302 events
1303 .filter(
1304 (event) =>
1305 event.type === "run.updated" ||
1306 event.type === "run-attempt.updated" ||
1307 event.type === "node.updated",
1308 )
1309 .map((event) => event.payload.status),
1310 ["failed", "failed", "failed"],
1311 );
1312 const errorItem = events.find(
1313 (event) => event.type === "turn-item.updated" && event.payload.type === "error",
1314 );
1315 assert.isDefined(errorItem);
1316 if (errorItem?.type === "turn-item.updated" && errorItem.payload.type === "error") {
1317 // The persisted item carries a bounded curated message; the exact
1318 // underlying text stays in the logged cause.
1319 assert.equal(errorItem.payload.failure.message, "Run preparation failed.");
1320 }
1321 }),
1322);
1323
1324it.effect("ingests new member threads of an owned child after the root turn terminalizes", () =>
1325 Effect.gen(function* () {
1326 const threadId = ThreadId.make("thread:run-execution-late-child");
1327 const childThreadId = ThreadId.make("thread:run-execution-late-child:child");
1328 const memberThreadId = ThreadId.make("thread:run-execution-late-child:member");
1329 const runId = RunId.make("run:run-execution-late-child");
1330 const attemptId = RunAttemptId.make("attempt:run-execution-late-child");
1331 const providerInstanceId = ProviderInstanceId.make("codex");
1332 const providerSessionId = ProviderSessionId.make("session:run-execution-late-child");
1333 const providerThreadId = ProviderThreadId.make("provider-thread:run-execution-late-child");
1334 const childProviderThreadId = ProviderThreadId.make(
1335 "provider-thread:run-execution-late-child:child",
1336 );
1337 const rootProviderTurnId = ProviderTurnId.make("provider-turn:run-execution-late-child");
1338 const childProviderTurnId = ProviderTurnId.make("provider-turn:run-execution-late-child:child");
1339 const rootNodeId = NodeId.make("node:run-execution-late-child");
1340 const childNodeId = NodeId.make("node:run-execution-late-child:child");
1341 const subagentNodeId = NodeId.make("node:run-execution-late-child:subagent");
1342 const childMessageIngested = yield* Deferred.make<void>();
1343 const order = yield* Ref.make<ReadonlyArray<string>>([]);
1344 const layerTest = RunExecutionService.layer.pipe(
1345 Layer.provide(
1346 Layer.mergeAll(
1347 McpAppModelContext.layerEmpty,
1348 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1349 Layer.mock(EventSink.EventSinkV2)({
1350 write: () => Effect.succeed([]),
1351 writeWithEffects: (input) =>
1352 Effect.gen(function* () {
1353 if (
1354 input.events.some(
1355 (event) => event.type === "run.updated" && event.runId === runId,
1356 )
1357 ) {
1358 yield* Ref.update(order, (current) => [...current, "root-finalized"]);
1359 }
1360 return [];
1361 }),
1362 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
1363 }),
1364 IdAllocator.layer,
1365 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1366 ingestNormalized: (input) =>
1367 Effect.gen(function* () {
1368 if (
1369 input.event.type === "app_thread.created" &&
1370 input.event.appThread.id === memberThreadId
1371 ) {
1372 yield* Ref.update(order, (current) => [...current, "member-thread"]);
1373 }
1374 if (
1375 input.event.type === "message.updated" &&
1376 input.event.message.threadId === memberThreadId
1377 ) {
1378 yield* Ref.update(order, (current) => [...current, "child-message"]);
1379 yield* Deferred.succeed(childMessageIngested, undefined).pipe(Effect.ignore);
1380 }
1381 return [];
1382 }),
1383 }),
1384 ServerSettings.layerTest(),
1385 ),
1386 ),
1387 );
1388 const events: ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> = [
1389 {
1390 type: "app_thread.created",
1391 driver,
1392 appThread: {
1393 id: childThreadId,
1394 lineage: {
1395 parentThreadId: threadId,
1396 relationshipToParent: "subagent",
1397 rootThreadId: threadId,
1398 },
1399 },
1400 } as ProviderAdapter.ProviderAdapterV2Event,
1401 {
1402 type: "provider_thread.updated",
1403 driver,
1404 providerThread: {
1405 id: childProviderThreadId,
1406 appThreadId: childThreadId,
1407 },
1408 } as ProviderAdapter.ProviderAdapterV2Event,
1409 {
1410 type: "subagent.updated",
1411 driver,
1412 subagent: {
1413 ...makeRunOwnedSubagentFixture({
1414 ids: backgroundScenarioIds("run-execution-late-child"),
1415 providerInstanceId,
1416 childThreadId,
1417 driver,
1418 status: "running",
1419 }),
1420 workflow: {
1421 phases: [],
1422 agents: [
1423 { index: 0, label: "Reviewer", state: "running", childThreadId: memberThreadId },
1424 ],
1425 },
1426 },
1427 },
1428 {
1429 type: "provider_turn.updated",
1430 driver,
1431 threadId: childThreadId,
1432 providerTurn: {
1433 id: childProviderTurnId,
1434 providerThreadId: childProviderThreadId,
1435 nodeId: childNodeId,
1436 runAttemptId: null,
1437 status: "running",
1438 },
1439 } as ProviderAdapter.ProviderAdapterV2Event,
1440 {
1441 type: "turn.terminal",
1442 driver,
1443 providerThreadId,
1444 providerTurnId: rootProviderTurnId,
1445 runOrdinal: 1,
1446 status: "completed",
1447 failure: null,
1448 threadDisposition: "reusable",
1449 },
1450 {
1451 type: "app_thread.created",
1452 driver,
1453 appThread: {
1454 id: memberThreadId,
1455 lineage: {
1456 parentThreadId: childThreadId,
1457 relationshipToParent: "subagent",
1458 rootThreadId: threadId,
1459 },
1460 },
1461 } as ProviderAdapter.ProviderAdapterV2Event,
1462 {
1463 type: "message.updated",
1464 driver,
1465 message: {
1466 id: MessageId.make("message:run-execution-late-child:child"),
1467 threadId: memberThreadId,
1468 runId: null,
1469 nodeId: childNodeId,
1470 role: "assistant",
1471 text: "Hello.",
1472 streaming: false,
1473 },
1474 } as ProviderAdapter.ProviderAdapterV2Event,
1475 {
1476 type: "provider_turn.updated",
1477 driver,
1478 threadId: childThreadId,
1479 providerTurn: {
1480 id: childProviderTurnId,
1481 providerThreadId: childProviderThreadId,
1482 nodeId: childNodeId,
1483 runAttemptId: null,
1484 status: "completed",
1485 },
1486 } as ProviderAdapter.ProviderAdapterV2Event,
1487 {
1488 type: "subagent.updated",
1489 driver,
1490 subagent: {
1491 id: subagentNodeId,
1492 threadId,
1493 runId,
1494 status: "completed",
1495 },
1496 } as ProviderAdapter.ProviderAdapterV2Event,
1497 ];
1498
1499 yield* Effect.gen(function* () {
1500 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1501 yield* runExecution.startRootRun({
1502 commandId: CommandId.make("command:run-execution-late-child"),
1503 appThread: { id: threadId } as OrchestrationV2AppThread,
1504 providerSessionId,
1505 session: {
1506 events: Stream.fromIterable(events),
1507 startTurn: () => Effect.void,
1508 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1509 run: {
1510 id: runId,
1511 threadId,
1512 ordinal: 1,
1513 providerInstanceId,
1514 } as OrchestrationV2Run,
1515 rootNode: { id: rootNodeId } as OrchestrationV2ExecutionNode,
1516 checkpointScope: {
1517 id: CheckpointScopeId.make("checkpoint-scope:run-execution-late-child"),
1518 } as OrchestrationV2CheckpointScope,
1519 providerThread: {
1520 id: providerThreadId,
1521 driver,
1522 } as OrchestrationV2ProviderThread,
1523 attempt: {
1524 id: attemptId,
1525 providerTurnId: rootProviderTurnId,
1526 } as OrchestrationV2RunAttempt,
1527 attemptId,
1528 providerTurnOrdinal: 1,
1529 message: {
1530 messageId: MessageId.make("message:run-execution-late-child:user"),
1531 text: "Spawn a child and finish.",
1532 attachments: [],
1533 createdBy: "user",
1534 creationSource: "web",
1535 },
1536 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1537 runtimePolicy: {
1538 runtimeMode: "full-access",
1539 interactionMode: "default",
1540 cwd: process.cwd(),
1541 approvalPolicy: "never",
1542 sandboxPolicy: {
1543 type: "readOnly",
1544 access: { type: "fullAccess" },
1545 networkAccess: false,
1546 },
1547 },
1548 });
1549 }).pipe(Effect.provide(layerTest));
1550
1551 const observed = yield* Deferred.await(childMessageIngested).pipe(
1552 Effect.timeoutOption("2 seconds"),
1553 );
1554 assert.isTrue(Option.isSome(observed), "child message was not ingested after root terminal");
1555 assert.deepEqual(yield* Ref.get(order), ["root-finalized", "member-thread", "child-message"]);
1556 }),
1557);
1558
1559it.effect("ends a nested native subagent and its thread when the root run is interrupted", () =>
1560 Effect.gen(function* () {
1561 const key = "run-execution-nested-native";
1562 const threadId = ThreadId.make(`thread:${key}`);
1563 const childThreadId = ThreadId.make(`thread:${key}:child`);
1564 const grandchildThreadId = ThreadId.make(`thread:${key}:grandchild`);
1565 const runId = RunId.make(`run:${key}`);
1566 const attemptId = RunAttemptId.make(`attempt:${key}`);
1567 const providerInstanceId = ProviderInstanceId.make("codex");
1568 const providerThreadId = ProviderThreadId.make(`provider-thread:${key}`);
1569 const rootProviderTurnId = ProviderTurnId.make(`provider-turn:${key}`);
1570 const nestedId = NodeId.make(`node:${key}:nested`);
1571 const grandchildRootId = NodeId.make(`node:${key}:grandchild-root`);
1572 const finalized = yield* Deferred.make<ReadonlyArray<OrchestrationV2DomainEvent>>();
1573 const layerTest = RunExecutionService.layer.pipe(
1574 Layer.provide(
1575 Layer.mergeAll(
1576 McpAppModelContext.layerEmpty,
1577 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1578 Layer.mock(EventSink.EventSinkV2)({
1579 write: () => Effect.succeed([]),
1580 writeWithEffects: (input) =>
1581 Effect.as(
1582 input.events.some((event) => event.type === "run.updated")
1583 ? Deferred.succeed(finalized, input.events)
1584 : Effect.void,
1585 [],
1586 ),
1587 }),
1588 IdAllocator.layer,
1589 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1590 ingestNormalized: () => Effect.succeed([]),
1591 }),
1592 ServerSettings.layerTest(),
1593 ),
1594 ),
1595 );
1596 const subagentThread = (id: ThreadId, parentThreadId: ThreadId) =>
1597 ({
1598 type: "app_thread.created",
1599 driver,
1600 appThread: {
1601 id,
1602 lineage: { parentThreadId, relationshipToParent: "subagent", rootThreadId: threadId },
1603 },
1604 }) as ProviderAdapter.ProviderAdapterV2Event;
1605 const events: ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event> = [
1606 subagentThread(childThreadId, threadId),
1607 subagentThread(grandchildThreadId, childThreadId),
1608 {
1609 type: "subagent.updated",
1610 driver,
1611 subagent: {
1612 id: nestedId,
1613 threadId: childThreadId,
1614 runId: null,
1615 origin: "provider_native",
1616 childThreadId: grandchildThreadId,
1617 status: "running",
1618 },
1619 } as ProviderAdapter.ProviderAdapterV2Event,
1620 {
1621 type: "node.updated",
1622 driver,
1623 node: {
1624 id: grandchildRootId,
1625 threadId: grandchildThreadId,
1626 runId: null,
1627 kind: "root_turn",
1628 status: "running",
1629 },
1630 } as ProviderAdapter.ProviderAdapterV2Event,
1631 {
1632 type: "turn.terminal",
1633 driver,
1634 providerThreadId,
1635 providerTurnId: rootProviderTurnId,
1636 runOrdinal: 1,
1637 status: "interrupted",
1638 failure: null,
1639 threadDisposition: "reusable",
1640 },
1641 ];
1642
1643 yield* Effect.gen(function* () {
1644 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1645 yield* runExecution.startRootRun({
1646 commandId: CommandId.make(`command:${key}`),
1647 appThread: { id: threadId } as OrchestrationV2AppThread,
1648 providerSessionId: ProviderSessionId.make(`session:${key}`),
1649 session: {
1650 events: Stream.fromIterable(events),
1651 startTurn: () => Effect.void,
1652 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
1653 run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run,
1654 rootNode: { id: NodeId.make(`node:${key}`) } as OrchestrationV2ExecutionNode,
1655 checkpointScope: {
1656 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
1657 } as OrchestrationV2CheckpointScope,
1658 providerThread: { id: providerThreadId, driver } as OrchestrationV2ProviderThread,
1659 attempt: { id: attemptId, providerTurnId: rootProviderTurnId } as OrchestrationV2RunAttempt,
1660 attemptId,
1661 providerTurnOrdinal: 1,
1662 message: {
1663 messageId: MessageId.make(`message:${key}:user`),
1664 text: "Spawn a nested subagent.",
1665 attachments: [],
1666 createdBy: "user",
1667 creationSource: "web",
1668 },
1669 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
1670 runtimePolicy: {
1671 runtimeMode: "full-access",
1672 interactionMode: "default",
1673 cwd: process.cwd(),
1674 approvalPolicy: "never",
1675 sandboxPolicy: { type: "readOnly", access: { type: "fullAccess" }, networkAccess: false },
1676 },
1677 });
1678 }).pipe(Effect.provide(layerTest));
1679
1680 const ended = (yield* Deferred.await(finalized)).flatMap((event) =>
1681 event.type === "subagent.updated" || event.type === "node.updated"
1682 ? [[event.type, event.payload.id, event.payload.status]]
1683 : [],
1684 );
1685 assert.deepInclude(ended, ["subagent.updated", nestedId, "interrupted"]);
1686 assert.deepInclude(ended, ["node.updated", grandchildRootId, "interrupted"]);
1687 }),
1688);
1689
1690it.effect("keeps ingesting a late background command item completion after root terminal", () =>
1691 Effect.gen(function* () {
1692 const observed = yield* runBackgroundItemScenario("bg-command", (ids) => [
1693 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
1694 rootTerminalEvent(ids, "completed"),
1695 backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
1696 ]);
1697 assert.deepEqual(observed, ["turn_item:running", "root-finalized", "turn_item:completed"]);
1698 }),
1699);
1700
1701it.effect("ingests the trailing subagent item completion after the subagent row completes", () =>
1702 Effect.gen(function* () {
1703 const observed = yield* runBackgroundItemScenario("subagent-trailing-item", (ids) => [
1704 subagentEvent(ids, "running"),
1705 backgroundTurnItemEvent(ids, "subagent", "running", 1),
1706 rootTerminalEvent(ids, "completed"),
1707 subagentEvent(ids, "completed"),
1708 backgroundTurnItemEvent(ids, "subagent", "completed", 2),
1709 ]);
1710 assert.deepEqual(observed, [
1711 "subagent:running",
1712 "turn_item:running",
1713 "root-finalized",
1714 "subagent:completed",
1715 "turn_item:completed",
1716 ]);
1717 }),
1718);
1719
1720it.effect("keeps ingesting a child thread's late background item completion", () =>
1721 Effect.gen(function* () {
1722 const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [
1723 childThreadCreatedEvent(ids),
1724 subagentEvent(ids, "running"),
1725 childBackgroundTurnItemEvent(ids, "running", 1),
1726 rootTerminalEvent(ids, "completed"),
1727 subagentEvent(ids, "completed"),
1728 childBackgroundTurnItemEvent(ids, "completed", 2),
1729 ]);
1730 assert.deepEqual(observed, [
1731 "subagent:running",
1732 "turn_item:running",
1733 "root-finalized",
1734 "subagent:completed",
1735 "turn_item:completed",
1736 ]);
1737 }),
1738);
1739
1740it.effect("keeps ingesting until the last of several background items terminalizes", () =>
1741 Effect.gen(function* () {
1742 const secondItemId = TurnItemId.make("turn-item:bg-multi:second");
1743 const observed = yield* runBackgroundItemScenario("bg-multi", (ids) => [
1744 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
1745 backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, secondItemId),
1746 rootTerminalEvent(ids, "completed"),
1747 backgroundTurnItemEvent(ids, "command_execution", "completed", 3),
1748 backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 4, secondItemId),
1749 ]);
1750 assert.deepEqual(observed, [
1751 "turn_item:running",
1752 "turn_item:running",
1753 "root-finalized",
1754 "turn_item:completed",
1755 "turn_item:completed",
1756 ]);
1757 }),
1758);
1759
1760it.effect("does not pin ingestion on background items when the root turn is interrupted", () =>
1761 Effect.gen(function* () {
1762 const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [
1763 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
1764 rootTerminalEvent(ids, "interrupted"),
1765 backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
1766 ]);
1767 assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
1768 }),
1769);
1770
1771it.effect("seeds inherited background items before their next update", () =>
1772 Effect.gen(function* () {
1773 const key = "inherited-background-seeded";
1774 const priorRunId = RunId.make(`run:${key}:prior`);
1775 const observed = yield* runBackgroundItemScenario(
1776 key,
1777 (ids) => [
1778 rootTerminalEvent(ids, "completed"),
1779 backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 1),
1780 ],
1781 {
1782 loadInheritedBackgroundTurnItems: () =>
1783 Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]),
1784 },
1785 );
1786
1787 assert.deepEqual(observed, ["root-finalized", "turn_item:completed"]);
1788 }),
1789);
1790
1791it.effect("releases the live run after an inherited background item terminalizes", () =>
1792 Effect.gen(function* () {
1793 const key = "inherited-background-terminal";
1794 const priorRunId = RunId.make(`run:${key}:prior`);
1795 const observed = yield* runBackgroundItemScenario(
1796 key,
1797 (ids) => [
1798 backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "running", 1),
1799 rootTerminalEvent(ids, "completed"),
1800 backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 2),
1801 ],
1802 {
1803 loadInheritedBackgroundTurnItems: () =>
1804 Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]),
1805 },
1806 );
1807
1808 assert.deepEqual(observed, ["turn_item:running", "root-finalized", "turn_item:completed"]);
1809 }),
1810);
1811
1812it.effect("does not hold the live stream open for a foreign provider's background item", () =>
1813 Effect.gen(function* () {
1814 const key = "inherited-background-foreign-provider";
1815 const ids = backgroundScenarioIds(key);
1816 const priorRunId = RunId.make(`run:${key}:prior`);
1817 const foreignProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:foreign-session`);
1818 const foreignItem = {
1819 id: ids.itemId,
1820 threadId: ids.threadId,
1821 runId: priorRunId,
1822 providerThreadId: foreignProviderThreadId,
1823 type: "subagent",
1824 status: "running",
1825 } as OrchestrationV2TurnItem;
1826 const inherited = RunExecutionService.selectInheritedBackgroundTurnItems({
1827 threadId: ids.threadId,
1828 currentProviderThreadId: ids.providerThreadId,
1829 currentRunOrdinal: 2,
1830 runs: [
1831 {
1832 id: priorRunId,
1833 threadId: ids.threadId,
1834 ordinal: 1,
1835 status: "interrupted",
1836 } as OrchestrationV2Run,
1837 ],
1838 turnItems: [foreignItem],
1839 });
1840
1841 const observed = yield* runBackgroundItemScenario(
1842 key,
1843 (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")],
1844 {
1845 keepEventStreamOpen: true,
1846 loadInheritedBackgroundTurnItems: () => Effect.succeed(inherited),
1847 },
1848 );
1849
1850 assert.deepEqual(inherited, []);
1851 assert.deepEqual(observed, ["root-finalized"]);
1852 }),
1853);
1854
1855it.effect("refreshes inherited background items after event subscription", () =>
1856 Effect.gen(function* () {
1857 const key = "inherited-background-subscription-refresh";
1858 const ids = backgroundScenarioIds(key);
1859 const priorRunId = RunId.make(`run:${key}:prior`);
1860 const itemStatus = yield* Ref.make<OrchestrationV2TurnItem["status"]>("running");
1861 const loadInheritedBackgroundTurnItems = () =>
1862 Ref.get(itemStatus).pipe(
1863 Effect.map((status) =>
1864 RunExecutionService.selectInheritedBackgroundTurnItems({
1865 threadId: ids.threadId,
1866 currentProviderThreadId: ids.providerThreadId,
1867 currentRunOrdinal: 2,
1868 runs: [
1869 {
1870 id: priorRunId,
1871 threadId: ids.threadId,
1872 ordinal: 1,
1873 status: "interrupted",
1874 } as OrchestrationV2Run,
1875 ],
1876 turnItems: [
1877 {
1878 id: ids.itemId,
1879 threadId: ids.threadId,
1880 runId: priorRunId,
1881 providerThreadId: ids.providerThreadId,
1882 type: "subagent",
1883 status,
1884 } as OrchestrationV2TurnItem,
1885 ],
1886 }),
1887 ),
1888 );
1889
1890 const observed = yield* runBackgroundItemScenario(
1891 key,
1892 (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")],
1893 {
1894 keepEventStreamOpen: true,
1895 loadInheritedBackgroundTurnItems,
1896 onSubscribe: Ref.set(itemStatus, "completed"),
1897 },
1898 );
1899
1900 assert.deepEqual(yield* loadInheritedBackgroundTurnItems(), []);
1901 assert.deepEqual(observed, ["root-finalized"]);
1902 }),
1903);
1904
1905it.effect(
1906 "keeps ingesting a late empty provider-thread roster while thread-scoped pending work is true",
1907 () =>
1908 Effect.gen(function* () {
1909 const key = "bg-roster-pending-work";
1910 const ids = backgroundScenarioIds(key);
1911 const providerInstanceId = ProviderInstanceId.make("codex");
1912 const now = yield* DateTime.now;
1913 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
1914 const pendingByProviderThreadId = yield* Ref.make(new Map([[ids.providerThreadId, true]]));
1915 const scopedProbeArgs = yield* Ref.make<ReadonlyArray<ProviderThreadId>>([]);
1916 const ingestCalls = yield* Ref.make<
1917 ReadonlyArray<{
1918 readonly activeAttemptId: RunAttemptId | null;
1919 readonly eventType: string;
1920 readonly hasWriteIfRunCurrent: boolean;
1921 readonly hasWriteIfProviderThreadOwner: boolean;
1922 readonly expectedLastRunOrdinal: number | null;
1923 readonly runId: RunId | null;
1924 readonly rosterLength: number | null;
1925 }>
1926 >([]);
1927 const ingestionDone = yield* Deferred.make<void>();
1928 const layerTest = RunExecutionService.layer.pipe(
1929 Layer.provide(
1930 Layer.mergeAll(
1931 McpAppModelContext.layerEmpty,
1932 Layer.mock(CheckpointService.CheckpointServiceV2)({
1933 captureBaseline: () => Effect.void,
1934 }),
1935 Layer.mock(EventSink.EventSinkV2)({
1936 write: () => Effect.succeed([]),
1937 writeWithEffects: (input) =>
1938 Effect.gen(function* () {
1939 if (
1940 input.events.some(
1941 (event) => event.type === "run.updated" && event.runId === ids.runId,
1942 )
1943 ) {
1944 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
1945 }
1946 return [];
1947 }),
1948 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
1949 writeIfProviderThreadOwner: () =>
1950 Effect.succeed({ committed: true, storedEvents: [] }),
1951 }),
1952 IdAllocator.layer,
1953 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1954 ingestNormalized: (input) =>
1955 Effect.gen(function* () {
1956 const event = input.event;
1957 const rosterLength =
1958 event.type === "provider_thread.updated"
1959 ? (event.providerThread.pendingBackgroundTasks?.length ?? 0)
1960 : null;
1961 yield* Ref.update(ingestCalls, (current) => [
1962 ...current,
1963 {
1964 activeAttemptId: input.writeIfProviderThreadOwner?.activeAttemptId ?? null,
1965 eventType: event.type,
1966 hasWriteIfRunCurrent: input.writeIfRunCurrent !== undefined,
1967 hasWriteIfProviderThreadOwner: input.writeIfProviderThreadOwner !== undefined,
1968 expectedLastRunOrdinal:
1969 input.writeIfProviderThreadOwner?.expectedLastRunOrdinal ?? null,
1970 runId: input.writeIfProviderThreadOwner?.runId ?? null,
1971 rosterLength,
1972 },
1973 ]);
1974 if (event.type === "provider_thread.updated" && rosterLength === 0) {
1975 yield* Ref.update(pendingByProviderThreadId, (current) => {
1976 const next = new Map(current);
1977 next.set(event.providerThread.id, false);
1978 return next;
1979 });
1980 yield* Ref.update(observed, (current) => [...current, "roster-cleared"]);
1981 }
1982 if (event.type === "turn.terminal") {
1983 yield* Ref.update(observed, (current) => [...current, "terminal"]);
1984 }
1985 return [];
1986 }),
1987 }),
1988 ServerSettings.layerTest(),
1989 ),
1990 ),
1991 );
1992
1993 const providerThreadBase = {
1994 id: ids.providerThreadId,
1995 driver,
1996 providerInstanceId,
1997 providerSessionId: ProviderSessionId.make(`session:${key}`),
1998 appThreadId: ids.threadId,
1999 ownerNodeId: null,
2000 nativeThreadRef: null,
2001 nativeConversationHeadRef: null,
2002 status: "idle" as const,
2003 firstRunOrdinal: 1,
2004 lastRunOrdinal: 1,
2005 handoffIds: [],
2006 forkedFrom: null,
2007 createdAt: now,
2008 updatedAt: now,
2009 };
2010
2011 yield* Effect.gen(function* () {
2012 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2013 yield* runExecution.startRootRun({
2014 commandId: CommandId.make(`command:${key}`),
2015 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2016 providerSessionId: ProviderSessionId.make(`session:${key}`),
2017 session: {
2018 events: Stream.empty,
2019 // Session-wide stays true forever; the root must consult the
2020 // thread-scoped probe instead of being pinned by siblings.
2021 hasPendingBackgroundWork: Effect.succeed(true),
2022 hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) =>
2023 Effect.gen(function* () {
2024 yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]);
2025 return (yield* Ref.get(pendingByProviderThreadId)).get(providerThread.id) === true;
2026 }),
2027 subscribeEvents: Effect.succeed({
2028 events: Stream.fromIterable([
2029 {
2030 type: "provider_thread.updated",
2031 driver,
2032 providerThread: {
2033 ...providerThreadBase,
2034 status: "active" as const,
2035 pendingBackgroundTasks: [
2036 { taskId: "bg-1", description: "sleep 20", kind: "command" },
2037 ],
2038 updatedAt: now,
2039 },
2040 } as ProviderAdapter.ProviderAdapterV2Event,
2041 rootTerminalEvent(ids, "completed"),
2042 {
2043 type: "provider_thread.updated",
2044 driver,
2045 providerThread: {
2046 ...providerThreadBase,
2047 status: "idle" as const,
2048 pendingBackgroundTasks: [],
2049 updatedAt: now,
2050 },
2051 } as ProviderAdapter.ProviderAdapterV2Event,
2052 ]),
2053 close: Deferred.succeed(ingestionDone, undefined),
2054 }),
2055 startTurn: () => Effect.void,
2056 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2057 run: {
2058 id: ids.runId,
2059 threadId: ids.threadId,
2060 ordinal: 1,
2061 providerInstanceId,
2062 } as OrchestrationV2Run,
2063 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2064 checkpointScope: {
2065 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
2066 } as OrchestrationV2CheckpointScope,
2067 providerThread: providerThreadBase as OrchestrationV2ProviderThread,
2068 attempt: {
2069 id: ids.attemptId,
2070 providerTurnId: ids.rootProviderTurnId,
2071 } as OrchestrationV2RunAttempt,
2072 attemptId: ids.attemptId,
2073 providerTurnOrdinal: 1,
2074 message: {
2075 messageId: MessageId.make(`message:${key}:user`),
2076 text: "Start background work and settle.",
2077 attachments: [],
2078 createdBy: "user",
2079 creationSource: "web",
2080 },
2081 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2082 runtimePolicy: {
2083 runtimeMode: "full-access",
2084 interactionMode: "default",
2085 cwd: process.cwd(),
2086 approvalPolicy: "never",
2087 sandboxPolicy: {
2088 type: "readOnly",
2089 access: { type: "fullAccess" },
2090 networkAccess: false,
2091 },
2092 },
2093 });
2094 }).pipe(Effect.provide(layerTest));
2095
2096 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2097 assert.isTrue(Option.isSome(closed), "event subscription did not release");
2098 assert.deepEqual(yield* Ref.get(observed), ["terminal", "root-finalized", "roster-cleared"]);
2099 assert.isTrue((yield* Ref.get(scopedProbeArgs)).includes(ids.providerThreadId));
2100
2101 const calls = yield* Ref.get(ingestCalls);
2102 const preTerminalRoster = calls.find(
2103 (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 1,
2104 );
2105 const lateClear = calls.find(
2106 (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 0,
2107 );
2108 assert.isDefined(preTerminalRoster);
2109 assert.isTrue(preTerminalRoster?.hasWriteIfRunCurrent);
2110 assert.isFalse(preTerminalRoster?.hasWriteIfProviderThreadOwner);
2111 assert.isDefined(lateClear);
2112 assert.isFalse(
2113 lateClear?.hasWriteIfRunCurrent,
2114 "late empty roster must not use stale writeIfRunCurrent running-gate",
2115 );
2116 assert.isTrue(
2117 lateClear?.hasWriteIfProviderThreadOwner,
2118 "late empty roster must gate on provider-thread ownership",
2119 );
2120 assert.equal(lateClear?.expectedLastRunOrdinal, 1);
2121 assert.equal(lateClear?.runId, ids.runId);
2122 assert.equal(lateClear?.activeAttemptId, ids.attemptId);
2123 }),
2124);
2125
2126it.effect("drops late root provider-thread writes from a superseded attempt", () =>
2127 Effect.gen(function* () {
2128 const key = "bg-roster-attempt-owner-lost";
2129 const ids = backgroundScenarioIds(key);
2130 const replacementAttemptId = RunAttemptId.make(`attempt:${key}:replacement`);
2131 const providerInstanceId = ProviderInstanceId.make("codex");
2132 const now = yield* DateTime.now;
2133 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
2134 // Probe stays true forever so only ownership-loss can release the stream.
2135 const ingestionDone = yield* Deferred.make<void>();
2136 const ingestCalls = yield* Ref.make<
2137 ReadonlyArray<{
2138 readonly activeAttemptId: RunAttemptId | null;
2139 readonly eventType: string;
2140 readonly hasWriteIfProviderThreadOwner: boolean;
2141 readonly expectedLastRunOrdinal: number | null;
2142 readonly rosterLength: number | null;
2143 readonly committed: boolean;
2144 }>
2145 >([]);
2146 const layerTest = RunExecutionService.layer.pipe(
2147 Layer.provide(
2148 Layer.mergeAll(
2149 McpAppModelContext.layerEmpty,
2150 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
2151 Layer.mock(EventSink.EventSinkV2)({
2152 write: () => Effect.succeed([]),
2153 writeWithEffects: (input) =>
2154 Effect.gen(function* () {
2155 if (
2156 input.events.some(
2157 (event) => event.type === "run.updated" && event.runId === ids.runId,
2158 )
2159 ) {
2160 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
2161 }
2162 return [];
2163 }),
2164 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
2165 writeIfProviderThreadOwner: () =>
2166 Effect.succeed({ committed: false, storedEvents: [] }),
2167 }),
2168 IdAllocator.layer,
2169 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2170 ingestNormalized: (input) =>
2171 Effect.gen(function* () {
2172 const event = input.event;
2173 const rosterLength =
2174 event.type === "provider_thread.updated"
2175 ? (event.providerThread.pendingBackgroundTasks?.length ?? 0)
2176 : null;
2177 const ownerGate = input.writeIfProviderThreadOwner;
2178 // Simulate the EventSink reject after a same-run replacement
2179 // changes activeAttemptId without advancing lastRunOrdinal.
2180 const rejectAsStaleOwner =
2181 ownerGate !== undefined && ownerGate.activeAttemptId !== replacementAttemptId;
2182 yield* Ref.update(ingestCalls, (current) => [
2183 ...current,
2184 {
2185 activeAttemptId: ownerGate?.activeAttemptId ?? null,
2186 eventType: event.type,
2187 hasWriteIfProviderThreadOwner: ownerGate !== undefined,
2188 expectedLastRunOrdinal: ownerGate?.expectedLastRunOrdinal ?? null,
2189 rosterLength,
2190 committed: !rejectAsStaleOwner,
2191 },
2192 ]);
2193 if (event.type === "turn.terminal") {
2194 yield* Ref.update(observed, (current) => [...current, "terminal"]);
2195 }
2196 if (event.type === "provider_thread.updated" && rejectAsStaleOwner) {
2197 yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]);
2198 return [];
2199 }
2200 if (event.type === "provider_thread.updated") {
2201 yield* Ref.update(observed, (current) => [...current, "roster-written"]);
2202 }
2203 return [];
2204 }),
2205 }),
2206 ServerSettings.layerTest(),
2207 ),
2208 ),
2209 );
2210
2211 const providerThreadBase = {
2212 id: ids.providerThreadId,
2213 driver,
2214 providerInstanceId,
2215 providerSessionId: ProviderSessionId.make(`session:${key}`),
2216 appThreadId: ids.threadId,
2217 ownerNodeId: null,
2218 nativeThreadRef: null,
2219 nativeConversationHeadRef: null,
2220 status: "idle" as const,
2221 firstRunOrdinal: 1,
2222 lastRunOrdinal: 1,
2223 handoffIds: [],
2224 forkedFrom: null,
2225 createdAt: now,
2226 updatedAt: now,
2227 };
2228
2229 yield* Effect.gen(function* () {
2230 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2231 yield* runExecution.startRootRun({
2232 commandId: CommandId.make(`command:${key}`),
2233 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2234 providerSessionId: ProviderSessionId.make(`session:${key}`),
2235 session: {
2236 events: Stream.empty,
2237 hasPendingBackgroundWork: Effect.succeed(true),
2238 hasPendingBackgroundWorkForThread: () => Effect.succeed(true),
2239 subscribeEvents: Effect.succeed({
2240 events: Stream.fromIterable([
2241 {
2242 type: "provider_thread.updated",
2243 driver,
2244 providerThread: {
2245 ...providerThreadBase,
2246 status: "active" as const,
2247 pendingBackgroundTasks: [
2248 { taskId: "bg-stale", description: "sleep 20", kind: "command" },
2249 ],
2250 updatedAt: now,
2251 },
2252 } as ProviderAdapter.ProviderAdapterV2Event,
2253 rootTerminalEvent(ids, "completed"),
2254 // Late snapshot after a replacement attempt claimed the same
2255 // run ordinal. Without attempt gating this would clobber it.
2256 {
2257 type: "provider_thread.updated",
2258 driver,
2259 providerThread: {
2260 ...providerThreadBase,
2261 status: "idle" as const,
2262 lastRunOrdinal: 1,
2263 pendingBackgroundTasks: [
2264 { taskId: "bg-stale", description: "sleep 20", kind: "command" },
2265 ],
2266 updatedAt: now,
2267 },
2268 } as ProviderAdapter.ProviderAdapterV2Event,
2269 ]),
2270 close: Deferred.succeed(ingestionDone, undefined),
2271 }),
2272 startTurn: () => Effect.void,
2273 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2274 run: {
2275 id: ids.runId,
2276 threadId: ids.threadId,
2277 ordinal: 1,
2278 providerInstanceId,
2279 } as OrchestrationV2Run,
2280 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2281 checkpointScope: {
2282 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
2283 } as OrchestrationV2CheckpointScope,
2284 providerThread: providerThreadBase as OrchestrationV2ProviderThread,
2285 attempt: {
2286 id: ids.attemptId,
2287 providerTurnId: ids.rootProviderTurnId,
2288 } as OrchestrationV2RunAttempt,
2289 attemptId: ids.attemptId,
2290 providerTurnOrdinal: 1,
2291 message: {
2292 messageId: MessageId.make(`message:${key}:user`),
2293 text: "Background work that loses ownership.",
2294 attachments: [],
2295 createdBy: "user",
2296 creationSource: "web",
2297 },
2298 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2299 runtimePolicy: {
2300 runtimeMode: "full-access",
2301 interactionMode: "default",
2302 cwd: process.cwd(),
2303 approvalPolicy: "never",
2304 sandboxPolicy: {
2305 type: "readOnly",
2306 access: { type: "fullAccess" },
2307 networkAccess: false,
2308 },
2309 },
2310 });
2311 }).pipe(Effect.provide(layerTest));
2312
2313 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2314 assert.isTrue(
2315 Option.isSome(closed),
2316 "subscription must release after ownership-loss reject even when probe stays true",
2317 );
2318 assert.deepEqual(yield* Ref.get(observed), [
2319 "roster-written",
2320 "terminal",
2321 "root-finalized",
2322 "stale-owner-rejected",
2323 ]);
2324
2325 const calls = yield* Ref.get(ingestCalls);
2326 const lateStale = calls.find(
2327 (call) =>
2328 call.eventType === "provider_thread.updated" &&
2329 call.hasWriteIfProviderThreadOwner &&
2330 call.rosterLength === 1,
2331 );
2332 assert.isDefined(lateStale);
2333 assert.equal(lateStale?.expectedLastRunOrdinal, 1);
2334 assert.equal(lateStale?.activeAttemptId, ids.attemptId);
2335 assert.isFalse(lateStale?.committed);
2336 }),
2337);
2338
2339it.effect(
2340 "keeps ingesting a late background turn-item completion after ownership-loss rejects roster writes",
2341 () =>
2342 Effect.gen(function* () {
2343 const key = "bg-item-after-owner-lost";
2344 const ids = backgroundScenarioIds(key);
2345 const providerInstanceId = ProviderInstanceId.make("codex");
2346 const now = yield* DateTime.now;
2347 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
2348 // Probe stays true forever; open background items must pin the stream
2349 // past ownership-loss so late turn_item completions still land.
2350 const ingestionDone = yield* Deferred.make<void>();
2351 const layerTest = RunExecutionService.layer.pipe(
2352 Layer.provide(
2353 Layer.mergeAll(
2354 McpAppModelContext.layerEmpty,
2355 Layer.mock(CheckpointService.CheckpointServiceV2)({
2356 captureBaseline: () => Effect.void,
2357 }),
2358 Layer.mock(EventSink.EventSinkV2)({
2359 write: () => Effect.succeed([]),
2360 writeWithEffects: (input) =>
2361 Effect.gen(function* () {
2362 if (
2363 input.events.some(
2364 (event) => event.type === "run.updated" && event.runId === ids.runId,
2365 )
2366 ) {
2367 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
2368 }
2369 return [];
2370 }),
2371 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
2372 writeIfProviderThreadOwner: () =>
2373 Effect.succeed({ committed: false, storedEvents: [] }),
2374 }),
2375 IdAllocator.layer,
2376 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2377 ingestNormalized: (input) =>
2378 Effect.gen(function* () {
2379 const event = input.event;
2380 if (event.type === "turn.terminal") {
2381 yield* Ref.update(observed, (current) => [...current, "terminal"]);
2382 }
2383 if (
2384 event.type === "provider_thread.updated" &&
2385 input.writeIfProviderThreadOwner !== undefined
2386 ) {
2387 yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]);
2388 return [];
2389 }
2390 if (event.type === "turn_item.updated") {
2391 yield* Ref.update(observed, (current) => [
2392 ...current,
2393 `turn_item:${event.turnItem.status}`,
2394 ]);
2395 }
2396 return [];
2397 }),
2398 }),
2399 ServerSettings.layerTest(),
2400 ),
2401 ),
2402 );
2403
2404 const providerThreadBase = {
2405 id: ids.providerThreadId,
2406 driver,
2407 providerInstanceId,
2408 providerSessionId: ProviderSessionId.make(`session:${key}`),
2409 appThreadId: ids.threadId,
2410 ownerNodeId: null,
2411 nativeThreadRef: null,
2412 nativeConversationHeadRef: null,
2413 status: "idle" as const,
2414 firstRunOrdinal: 1,
2415 lastRunOrdinal: 1,
2416 handoffIds: [],
2417 forkedFrom: null,
2418 createdAt: now,
2419 updatedAt: now,
2420 };
2421
2422 yield* Effect.gen(function* () {
2423 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2424 yield* runExecution.startRootRun({
2425 commandId: CommandId.make(`command:${key}`),
2426 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2427 providerSessionId: ProviderSessionId.make(`session:${key}`),
2428 session: {
2429 events: Stream.empty,
2430 hasPendingBackgroundWork: Effect.succeed(true),
2431 hasPendingBackgroundWorkForThread: () => Effect.succeed(true),
2432 subscribeEvents: Effect.succeed({
2433 events: Stream.fromIterable([
2434 backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2435 rootTerminalEvent(ids, "completed"),
2436 // Ownership reject after a newer run claimed lastRunOrdinal.
2437 {
2438 type: "provider_thread.updated",
2439 driver,
2440 providerThread: {
2441 ...providerThreadBase,
2442 status: "idle" as const,
2443 lastRunOrdinal: 1,
2444 pendingBackgroundTasks: [
2445 { taskId: "bg-stale", description: "sleep 20", kind: "command" },
2446 ],
2447 updatedAt: now,
2448 },
2449 } as ProviderAdapter.ProviderAdapterV2Event,
2450 // Late completion still writable (turn_item writes are not
2451 // ownership-gated); stream must stay open for it.
2452 backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2453 ]),
2454 close: Deferred.succeed(ingestionDone, undefined),
2455 }),
2456 startTurn: () => Effect.void,
2457 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2458 run: {
2459 id: ids.runId,
2460 threadId: ids.threadId,
2461 ordinal: 1,
2462 providerInstanceId,
2463 } as OrchestrationV2Run,
2464 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2465 checkpointScope: {
2466 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
2467 } as OrchestrationV2CheckpointScope,
2468 providerThread: providerThreadBase as OrchestrationV2ProviderThread,
2469 attempt: {
2470 id: ids.attemptId,
2471 providerTurnId: ids.rootProviderTurnId,
2472 } as OrchestrationV2RunAttempt,
2473 attemptId: ids.attemptId,
2474 providerTurnOrdinal: 1,
2475 message: {
2476 messageId: MessageId.make(`message:${key}:user`),
2477 text: "Background item after ownership loss.",
2478 attachments: [],
2479 createdBy: "user",
2480 creationSource: "web",
2481 },
2482 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2483 runtimePolicy: {
2484 runtimeMode: "full-access",
2485 interactionMode: "default",
2486 cwd: process.cwd(),
2487 approvalPolicy: "never",
2488 sandboxPolicy: {
2489 type: "readOnly",
2490 access: { type: "fullAccess" },
2491 networkAccess: false,
2492 },
2493 },
2494 });
2495 }).pipe(Effect.provide(layerTest));
2496
2497 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2498 assert.isTrue(
2499 Option.isSome(closed),
2500 "subscription must release after background item completes past ownership loss",
2501 );
2502 assert.deepEqual(yield* Ref.get(observed), [
2503 "turn_item:running",
2504 "terminal",
2505 "root-finalized",
2506 "stale-owner-rejected",
2507 "turn_item:completed",
2508 ]);
2509 }),
2510);
2511
2512it.effect(
2513 "does not pin ingestion on a sibling session-wide pending state when this thread has no roster",
2514 () =>
2515 Effect.gen(function* () {
2516 const key = "bg-roster-sibling-not-pin";
2517 const ids = backgroundScenarioIds(key);
2518 const providerInstanceId = ProviderInstanceId.make("codex");
2519 const now = yield* DateTime.now;
2520 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
2521 const ingestionDone = yield* Deferred.make<void>();
2522 const scopedProbeArgs = yield* Ref.make<ReadonlyArray<ProviderThreadId>>([]);
2523 const layerTest = RunExecutionService.layer.pipe(
2524 Layer.provide(
2525 Layer.mergeAll(
2526 McpAppModelContext.layerEmpty,
2527 Layer.mock(CheckpointService.CheckpointServiceV2)({
2528 captureBaseline: () => Effect.void,
2529 }),
2530 Layer.mock(EventSink.EventSinkV2)({
2531 write: () => Effect.succeed([]),
2532 writeWithEffects: (input) =>
2533 Effect.gen(function* () {
2534 if (
2535 input.events.some(
2536 (event) => event.type === "run.updated" && event.runId === ids.runId,
2537 )
2538 ) {
2539 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
2540 }
2541 return [];
2542 }),
2543 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
2544 }),
2545 IdAllocator.layer,
2546 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2547 ingestNormalized: (input) =>
2548 Effect.gen(function* () {
2549 if (input.event.type === "turn.terminal") {
2550 yield* Ref.update(observed, (current) => [...current, "terminal"]);
2551 }
2552 return [];
2553 }),
2554 }),
2555 ServerSettings.layerTest(),
2556 ),
2557 ),
2558 );
2559
2560 const providerThreadBase = {
2561 id: ids.providerThreadId,
2562 driver,
2563 providerInstanceId,
2564 providerSessionId: ProviderSessionId.make(`session:${key}`),
2565 appThreadId: ids.threadId,
2566 ownerNodeId: null,
2567 nativeThreadRef: {
2568 driver,
2569 nativeId: "native-self",
2570 strength: "strong" as const,
2571 },
2572 nativeConversationHeadRef: null,
2573 status: "idle" as const,
2574 firstRunOrdinal: 1,
2575 lastRunOrdinal: 1,
2576 handoffIds: [],
2577 forkedFrom: null,
2578 createdAt: now,
2579 updatedAt: now,
2580 };
2581 const siblingProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:sibling`);
2582
2583 yield* Effect.gen(function* () {
2584 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2585 yield* runExecution.startRootRun({
2586 commandId: CommandId.make(`command:${key}`),
2587 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2588 providerSessionId: ProviderSessionId.make(`session:${key}`),
2589 session: {
2590 events: Stream.empty,
2591 // Session-wide stays true (sibling has work). Stop must use only
2592 // the scoped probe for this root's provider thread.
2593 hasPendingBackgroundWork: Effect.succeed(true),
2594 hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) =>
2595 Effect.gen(function* () {
2596 yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]);
2597 // Own thread has no roster; sibling would report true if probed.
2598 return providerThread.id !== ids.providerThreadId;
2599 }),
2600 subscribeEvents: Effect.succeed({
2601 events: Stream.fromIterable([
2602 rootTerminalEvent(ids, "completed"),
2603 // Sibling thread update after terminal. Own-thread scoped
2604 // pending is false, so this root must release without waiting
2605 // for sibling-driven session-wide pending work.
2606 {
2607 type: "provider_thread.updated",
2608 driver,
2609 providerThread: {
2610 ...providerThreadBase,
2611 id: siblingProviderThreadId,
2612 appThreadId: ThreadId.make(`thread:${key}:sibling`),
2613 nativeThreadRef: {
2614 driver,
2615 nativeId: "native-sibling",
2616 strength: "strong" as const,
2617 },
2618 pendingBackgroundTasks: [
2619 { taskId: "sibling-bg", description: "other thread", kind: "command" },
2620 ],
2621 updatedAt: now,
2622 },
2623 } as ProviderAdapter.ProviderAdapterV2Event,
2624 ]),
2625 close: Deferred.succeed(ingestionDone, undefined),
2626 }),
2627 startTurn: () => Effect.void,
2628 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2629 run: {
2630 id: ids.runId,
2631 threadId: ids.threadId,
2632 ordinal: 1,
2633 providerInstanceId,
2634 } as OrchestrationV2Run,
2635 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2636 checkpointScope: {
2637 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
2638 } as OrchestrationV2CheckpointScope,
2639 providerThread: providerThreadBase as OrchestrationV2ProviderThread,
2640 attempt: {
2641 id: ids.attemptId,
2642 providerTurnId: ids.rootProviderTurnId,
2643 } as OrchestrationV2RunAttempt,
2644 attemptId: ids.attemptId,
2645 providerTurnOrdinal: 1,
2646 message: {
2647 messageId: MessageId.make(`message:${key}:user`),
2648 text: "Settle without local pending work.",
2649 attachments: [],
2650 createdBy: "user",
2651 creationSource: "web",
2652 },
2653 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2654 runtimePolicy: {
2655 runtimeMode: "full-access",
2656 interactionMode: "default",
2657 cwd: process.cwd(),
2658 approvalPolicy: "never",
2659 sandboxPolicy: {
2660 type: "readOnly",
2661 access: { type: "fullAccess" },
2662 networkAccess: false,
2663 },
2664 },
2665 });
2666 }).pipe(Effect.provide(layerTest));
2667
2668 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2669 assert.isTrue(Option.isSome(closed), "event subscription did not release");
2670 // Critical: release while session-wide hasPendingBackgroundWork stays true
2671 // and the scoped probe reports false for this root's provider thread.
2672 const observedEvents = [...(yield* Ref.get(observed))];
2673 assert.includeMembers(observedEvents, ["terminal", "root-finalized"]);
2674 const probedIds = yield* Ref.get(scopedProbeArgs);
2675 assert.isTrue(probedIds.includes(ids.providerThreadId));
2676 assert.isFalse(probedIds.includes(siblingProviderThreadId));
2677 }),
2678);
2679
2680it.effect(
2681 "cascade-terminalizes run-owned subagent rows on interrupt before root finalization",
2682 () =>
2683 Effect.gen(function* () {
2684 const ids = backgroundScenarioIds("subagent-interrupt-cascade");
2685 const childThreadId = ids.childThreadId;
2686 const unrelatedChildThreadId = ThreadId.make(
2687 "thread:subagent-interrupt-cascade:unrelated-child",
2688 );
2689 const providerInstanceId = ProviderInstanceId.make("codex");
2690 const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
2691 const ingested = yield* Ref.make<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>>([]);
2692 const ingestionDone = yield* Deferred.make<void>();
2693 const layerTest = RunExecutionService.layer.pipe(
2694 Layer.provide(
2695 Layer.mergeAll(
2696 McpAppModelContext.layerEmpty,
2697 Layer.mock(CheckpointService.CheckpointServiceV2)({
2698 captureBaseline: () => Effect.void,
2699 }),
2700 Layer.mock(EventSink.EventSinkV2)({
2701 write: () => Effect.succeed([]),
2702 writeWithEffects: (input) =>
2703 Effect.gen(function* () {
2704 yield* Ref.update(written, (current) => [...current, ...input.events]);
2705 return [];
2706 }),
2707 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
2708 }),
2709 IdAllocator.layer,
2710 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
2711 ingestNormalized: (input) =>
2712 Ref.update(ingested, (current) => [...current, input.event]).pipe(Effect.as([])),
2713 }),
2714 ServerSettings.layerTest(),
2715 ),
2716 ),
2717 );
2718
2719 const runningSubagent = makeRunOwnedSubagentFixture({
2720 ids,
2721 providerInstanceId,
2722 childThreadId,
2723 driver,
2724 status: "running",
2725 });
2726 const runningTurnItem = makeRunOwnedSubagentTurnItemFixture({
2727 ids,
2728 providerInstanceId,
2729 childThreadId,
2730 driver,
2731 status: "running",
2732 });
2733 const runningNode = makeRunOwnedSubagentNodeFixture({
2734 ids,
2735 status: "running",
2736 });
2737 const runningChildNode = makeRunOwnedSubagentChildNodeFixture({
2738 ids,
2739 status: "running",
2740 });
2741 const memberThreadId = ThreadId.make("thread:subagent-interrupt-cascade:member");
2742 const runningMember = {
2743 ...runningSubagent,
2744 id: NodeId.make("node:subagent-interrupt-cascade:member"),
2745 threadId: childThreadId,
2746 runId: null,
2747 parentNodeId: runningChildNode.id,
2748 childThreadId: memberThreadId,
2749 };
2750 const foreignSubagent = {
2751 ...runningMember,
2752 id: NodeId.make("node:subagent-interrupt-cascade:foreign-subagent"),
2753 runId: RunId.make("run:subagent-interrupt-cascade:foreign"),
2754 childThreadId: unrelatedChildThreadId,
2755 };
2756 const runningMemberRoot = {
2757 ...runningChildNode,
2758 id: NodeId.make("node:subagent-interrupt-cascade:member-root"),
2759 threadId: memberThreadId,
2760 rootNodeId: NodeId.make("node:subagent-interrupt-cascade:member-root"),
2761 };
2762 const foreignChildNode = {
2763 ...runningChildNode,
2764 id: NodeId.make("node:subagent-interrupt-cascade:foreign-child"),
2765 runId: foreignSubagent.runId,
2766 };
2767 const runningChildTurnItem = makeLinkedChildTurnItemFixture({
2768 ids,
2769 driver,
2770 type: "command_execution",
2771 });
2772 const foreignChildTurnItem = {
2773 ...runningChildTurnItem,
2774 id: TurnItemId.make("turn-item:subagent-interrupt-cascade:foreign-child"),
2775 runId: foreignSubagent.runId,
2776 };
2777 const suppressedChildAssistantTurnItem = makeLinkedChildTurnItemFixture({
2778 ids: {
2779 ...ids,
2780 childItemId: TurnItemId.make("turn-item:subagent-interrupt-cascade:suppressed-assistant"),
2781 },
2782 driver,
2783 type: "assistant_message",
2784 });
2785 const unrelatedChildNode = {
2786 ...runningChildNode,
2787 id: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"),
2788 threadId: unrelatedChildThreadId,
2789 runId: ids.runId,
2790 rootNodeId: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"),
2791 };
2792 const unrelatedChildTurnItem = {
2793 ...runningChildTurnItem,
2794 id: TurnItemId.make("turn-item:subagent-interrupt-cascade:unrelated-child"),
2795 threadId: unrelatedChildThreadId,
2796 nodeId: unrelatedChildNode.id,
2797 };
2798
2799 yield* Effect.gen(function* () {
2800 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2801 yield* runExecution.startRootRun({
2802 commandId: CommandId.make("command:subagent-interrupt-cascade"),
2803 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2804 providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"),
2805 session: {
2806 events: Stream.empty,
2807 subscribeEvents: Effect.succeed({
2808 events: Stream.fromIterable([
2809 childThreadCreatedEvent(ids),
2810 {
2811 type: "subagent.updated",
2812 driver,
2813 subagent: runningSubagent,
2814 },
2815 {
2816 type: "node.updated",
2817 driver,
2818 node: runningNode,
2819 },
2820 {
2821 type: "node.updated",
2822 driver,
2823 node: runningChildNode,
2824 },
2825 {
2826 type: "node.updated",
2827 driver,
2828 node: unrelatedChildNode,
2829 },
2830 {
2831 type: "turn_item.updated",
2832 driver,
2833 turnItem: runningTurnItem,
2834 },
2835 {
2836 type: "turn_item.updated",
2837 driver,
2838 turnItem: runningChildTurnItem,
2839 },
2840 {
2841 type: "turn_item.updated",
2842 driver,
2843 turnItem: suppressedChildAssistantTurnItem,
2844 },
2845 {
2846 type: "turn_item.updated",
2847 driver,
2848 turnItem: unrelatedChildTurnItem,
2849 },
2850 { type: "subagent.updated", driver, subagent: runningMember },
2851 { type: "subagent.updated", driver, subagent: foreignSubagent },
2852 { type: "node.updated", driver, node: runningMemberRoot },
2853 { type: "node.updated", driver, node: foreignChildNode },
2854 { type: "turn_item.updated", driver, turnItem: foreignChildTurnItem },
2855 rootTerminalEvent(ids, "interrupted"),
2856 // Late provider completion after interrupt must not be ingested.
2857 {
2858 type: "subagent.updated",
2859 driver,
2860 subagent: {
2861 ...runningSubagent,
2862 status: "completed" as const,
2863 result: "should-not-apply",
2864 completedAt: runningSubagent.updatedAt,
2865 },
2866 },
2867 {
2868 type: "turn_item.updated",
2869 driver,
2870 turnItem: {
2871 ...runningTurnItem,
2872 status: "completed" as const,
2873 result: "should-not-apply",
2874 completedAt: runningTurnItem.updatedAt,
2875 updatedAt: runningTurnItem.updatedAt,
2876 },
2877 },
2878 {
2879 type: "turn_item.updated",
2880 driver,
2881 turnItem: {
2882 ...runningChildTurnItem,
2883 status: "completed" as const,
2884 completedAt: runningChildTurnItem.updatedAt,
2885 updatedAt: runningChildTurnItem.updatedAt,
2886 },
2887 },
2888 {
2889 type: "node.updated",
2890 driver,
2891 node: {
2892 ...runningChildNode,
2893 status: "completed" as const,
2894 completedAt: runningChildNode.startedAt,
2895 },
2896 },
2897 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
2898 close: Deferred.succeed(ingestionDone, undefined),
2899 }),
2900 startTurn: () => Effect.void,
2901 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2902 run: {
2903 id: ids.runId,
2904 threadId: ids.threadId,
2905 ordinal: 1,
2906 providerInstanceId,
2907 } as OrchestrationV2Run,
2908 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
2909 checkpointScope: {
2910 id: CheckpointScopeId.make("checkpoint-scope:subagent-interrupt-cascade"),
2911 } as OrchestrationV2CheckpointScope,
2912 providerThread: {
2913 id: ids.providerThreadId,
2914 driver,
2915 } as OrchestrationV2ProviderThread,
2916 attempt: {
2917 id: ids.attemptId,
2918 providerTurnId: ids.rootProviderTurnId,
2919 } as OrchestrationV2RunAttempt,
2920 attemptId: ids.attemptId,
2921 relatedThreadIds: [unrelatedChildThreadId, memberThreadId],
2922 providerTurnOrdinal: 1,
2923 message: {
2924 messageId: MessageId.make("message:subagent-interrupt-cascade:user"),
2925 text: "Spawn a subagent then stop.",
2926 attachments: [],
2927 createdBy: "user",
2928 creationSource: "web",
2929 },
2930 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
2931 runtimePolicy: {
2932 runtimeMode: "full-access",
2933 interactionMode: "default",
2934 cwd: process.cwd(),
2935 approvalPolicy: "never",
2936 sandboxPolicy: {
2937 type: "readOnly",
2938 access: { type: "fullAccess" },
2939 networkAccess: false,
2940 },
2941 },
2942 });
2943 }).pipe(Effect.provide(layerTest));
2944
2945 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
2946 assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
2947
2948 const events = yield* Ref.get(written);
2949 const subagentEvents = events.flatMap((event) =>
2950 event.type === "subagent.updated" ? [event] : [],
2951 );
2952 const turnItemEvents = events.flatMap((event) => {
2953 if (event.type !== "turn-item.updated" || event.payload.type !== "subagent") {
2954 return [];
2955 }
2956 return [
2957 {
2958 ...event,
2959 payload: event.payload,
2960 },
2961 ];
2962 });
2963 const nodeEvents = events.flatMap((event) =>
2964 event.type === "node.updated" &&
2965 event.payload.status === "interrupted" &&
2966 (event.payload.id === runningNode.id || event.payload.id === runningChildNode.id)
2967 ? [event]
2968 : [],
2969 );
2970 const runUpdatedIndex = events.findIndex((event) => event.type === "run.updated");
2971 assert.isAtLeast(runUpdatedIndex, 0, "root run.updated must be written");
2972
2973 assert.lengthOf(subagentEvents, 2);
2974 const terminalSubagent = subagentEvents.find(
2975 (event) => event.payload.id === runningSubagent.id,
2976 );
2977 const terminalMember = subagentEvents.find((event) => event.payload.id === runningMember.id);
2978 assert.isDefined(terminalMember);
2979 assert.equal(terminalMember.payload.status, "interrupted");
2980 assert.equal(terminalMember.payload.runId, null);
2981 assert.equal(terminalMember.payload.childThreadId, memberThreadId);
2982 assert.isTrue(
2983 events.some(
2984 (event) =>
2985 event.type === "node.updated" &&
2986 event.payload.id === runningMemberRoot.id &&
2987 event.payload.status === "interrupted",
2988 ),
2989 );
2990 assert.isFalse(
2991 events.some(
2992 (event) => event.type === "subagent.updated" && event.payload.id === foreignSubagent.id,
2993 ),
2994 );
2995 assert.isFalse(
2996 events.some(
2997 (event) => event.type === "node.updated" && event.payload.id === foreignChildNode.id,
2998 ),
2999 );
3000 assert.isFalse(
3001 events.some(
3002 (event) =>
3003 event.type === "turn-item.updated" && event.payload.id === foreignChildTurnItem.id,
3004 ),
3005 );
3006 assert.isDefined(terminalSubagent);
3007 assert.equal(terminalSubagent.payload.status, "interrupted");
3008 assert.equal(terminalSubagent.payload.childThreadId, childThreadId);
3009 assert.equal(terminalSubagent.payload.result, null);
3010 assert.isNotNull(terminalSubagent.payload.completedAt);
3011
3012 assert.lengthOf(turnItemEvents, 1);
3013 const terminalTurnItem = turnItemEvents[0];
3014 assert.isDefined(terminalTurnItem);
3015 assert.equal(terminalTurnItem.payload.status, "interrupted");
3016 assert.equal(terminalTurnItem.payload.childThreadId, childThreadId);
3017 assert.equal(terminalTurnItem.payload.result, null);
3018
3019 const terminalChildTurnItems = events.flatMap((event) =>
3020 event.type === "turn-item.updated" &&
3021 event.payload.id === runningChildTurnItem.id &&
3022 event.payload.status === "interrupted"
3023 ? [event]
3024 : [],
3025 );
3026 assert.lengthOf(terminalChildTurnItems, 1);
3027 const terminalChildTurnItem = terminalChildTurnItems[0];
3028 assert.isDefined(terminalChildTurnItem);
3029 assert.equal(terminalChildTurnItem.threadId, childThreadId);
3030 assert.equal(terminalChildTurnItem.payload.threadId, childThreadId);
3031 assert.equal(terminalChildTurnItem.payload.runId, null);
3032 assert.equal(terminalChildTurnItem.payload.type, "command_execution");
3033 assert.isFalse(
3034 events.some(
3035 (event) =>
3036 event.type === "turn-item.updated" &&
3037 event.payload.id === unrelatedChildTurnItem.id &&
3038 event.payload.status === "interrupted",
3039 ),
3040 "related but unlinked child turn items must not cascade",
3041 );
3042 assert.isFalse(
3043 events.some(
3044 (event) =>
3045 event.type === "turn-item.updated" &&
3046 event.payload.id === suppressedChildAssistantTurnItem.id,
3047 ),
3048 "suppressed streaming child items must not be created by the cascade",
3049 );
3050
3051 assert.lengthOf(nodeEvents, 2);
3052 assert.isTrue(
3053 nodeEvents.some(
3054 (event) => event.payload.id === runningNode.id && event.payload.kind === "subagent",
3055 ),
3056 );
3057 assert.isFalse(
3058 events.some(
3059 (event) =>
3060 event.type === "node.updated" &&
3061 event.payload.id === unrelatedChildNode.id &&
3062 event.payload.status === "interrupted",
3063 ),
3064 "owned child threads without a live run-owned subagent link must not cascade",
3065 );
3066 assert.isTrue(
3067 nodeEvents.some(
3068 (event) =>
3069 event.payload.id === runningChildNode.id &&
3070 event.runId === (runningChildNode.runId ?? ids.runId) &&
3071 event.payload.threadId === childThreadId &&
3072 event.payload.kind === "root_turn",
3073 ),
3074 );
3075
3076 const cascadeIndexes = events.flatMap((event, index) => {
3077 if (event.type === "subagent.updated" && event.payload.status === "interrupted") {
3078 return [index];
3079 }
3080 if (
3081 event.type === "turn-item.updated" &&
3082 event.payload.status === "interrupted" &&
3083 (event.payload.type === "subagent" || event.payload.id === runningChildTurnItem.id)
3084 ) {
3085 return [index];
3086 }
3087 if (event.type === "node.updated" && event.payload.status === "interrupted") {
3088 return event.payload.id === runningNode.id || event.payload.id === runningChildNode.id
3089 ? [index]
3090 : [];
3091 }
3092 return [];
3093 });
3094 assert.isTrue(
3095 cascadeIndexes.every((index) => index < runUpdatedIndex),
3096 "subagent cascade must precede run.updated",
3097 );
3098 assert.isFalse(
3099 events.some((event) => {
3100 if (event.type === "subagent.updated") {
3101 return event.payload.status === "completed";
3102 }
3103 if (event.type === "turn-item.updated" && event.payload.type === "subagent") {
3104 return event.payload.status === "completed";
3105 }
3106 return false;
3107 }),
3108 "late provider completion must not reopen cascaded subagent rows",
3109 );
3110 assert.isFalse(
3111 (yield* Ref.get(ingested)).some(
3112 (event) =>
3113 (event.type === "subagent.updated" && event.subagent.status === "completed") ||
3114 (event.type === "turn_item.updated" && event.turnItem.status === "completed") ||
3115 (event.type === "node.updated" && event.node.status === "completed"),
3116 ),
3117 "late provider completion must not be ingested after interrupt",
3118 );
3119 }),
3120);
3121
3122it.effect(
3123 "cascades linked child-thread nodes after run-owned subagent and turn-item terminalize",
3124 () =>
3125 Effect.gen(function* () {
3126 const ids = backgroundScenarioIds("subagent-link-survives-terminal");
3127 const childThreadId = ids.childThreadId;
3128 const unrelatedChildThreadId = ThreadId.make(
3129 "thread:subagent-link-survives-terminal:unrelated-child",
3130 );
3131 const providerInstanceId = ProviderInstanceId.make("codex");
3132 const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
3133 const ingestionDone = yield* Deferred.make<void>();
3134 const layerTest = RunExecutionService.layer.pipe(
3135 Layer.provide(
3136 Layer.mergeAll(
3137 McpAppModelContext.layerEmpty,
3138 Layer.mock(CheckpointService.CheckpointServiceV2)({
3139 captureBaseline: () => Effect.void,
3140 }),
3141 Layer.mock(EventSink.EventSinkV2)({
3142 write: () => Effect.succeed([]),
3143 writeWithEffects: (input) =>
3144 Effect.gen(function* () {
3145 yield* Ref.update(written, (current) => [...current, ...input.events]);
3146 return [];
3147 }),
3148 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
3149 }),
3150 IdAllocator.layer,
3151 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3152 ingestNormalized: () => Effect.succeed([]),
3153 }),
3154 ServerSettings.layerTest(),
3155 ),
3156 ),
3157 );
3158
3159 const runningSubagent = makeRunOwnedSubagentFixture({
3160 ids,
3161 providerInstanceId,
3162 childThreadId,
3163 driver,
3164 status: "running",
3165 });
3166 const runningTurnItem = makeRunOwnedSubagentTurnItemFixture({
3167 ids,
3168 providerInstanceId,
3169 childThreadId,
3170 driver,
3171 status: "running",
3172 });
3173 const runningNode = makeRunOwnedSubagentNodeFixture({
3174 ids,
3175 status: "running",
3176 });
3177 const runningChildNode = makeRunOwnedSubagentChildNodeFixture({
3178 ids,
3179 status: "running",
3180 });
3181 const unrelatedChildNode = {
3182 ...runningChildNode,
3183 id: NodeId.make("node:subagent-link-survives-terminal:unrelated-child"),
3184 threadId: unrelatedChildThreadId,
3185 runId: ids.runId,
3186 rootNodeId: NodeId.make("node:subagent-link-survives-terminal:unrelated-child"),
3187 };
3188 const completedAt = runningSubagent.updatedAt;
3189
3190 yield* Effect.gen(function* () {
3191 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3192 yield* runExecution.startRootRun({
3193 commandId: CommandId.make("command:subagent-link-survives-terminal"),
3194 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
3195 providerSessionId: ProviderSessionId.make("session:subagent-link-survives-terminal"),
3196 session: {
3197 events: Stream.empty,
3198 subscribeEvents: Effect.succeed({
3199 events: Stream.fromIterable([
3200 childThreadCreatedEvent(ids),
3201 {
3202 type: "subagent.updated",
3203 driver,
3204 subagent: runningSubagent,
3205 },
3206 {
3207 type: "node.updated",
3208 driver,
3209 node: runningNode,
3210 },
3211 {
3212 type: "node.updated",
3213 driver,
3214 node: runningChildNode,
3215 },
3216 {
3217 type: "node.updated",
3218 driver,
3219 node: unrelatedChildNode,
3220 },
3221 {
3222 type: "turn_item.updated",
3223 driver,
3224 turnItem: runningTurnItem,
3225 },
3226 // Subagent + turn-item settle before root interrupt; linkage
3227 // must still prove the open child-thread node is cascadeable.
3228 {
3229 type: "subagent.updated",
3230 driver,
3231 subagent: {
3232 ...runningSubagent,
3233 status: "completed" as const,
3234 result: "subagent finished first",
3235 completedAt,
3236 },
3237 },
3238 {
3239 type: "turn_item.updated",
3240 driver,
3241 turnItem: {
3242 ...runningTurnItem,
3243 status: "completed" as const,
3244 result: "turn item finished first",
3245 completedAt,
3246 updatedAt: completedAt,
3247 },
3248 },
3249 {
3250 type: "node.updated",
3251 driver,
3252 node: {
3253 ...runningNode,
3254 status: "completed" as const,
3255 completedAt,
3256 },
3257 },
3258 rootTerminalEvent(ids, "interrupted"),
3259 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
3260 close: Deferred.succeed(ingestionDone, undefined),
3261 }),
3262 startTurn: () => Effect.void,
3263 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
3264 run: {
3265 id: ids.runId,
3266 threadId: ids.threadId,
3267 ordinal: 1,
3268 providerInstanceId,
3269 } as OrchestrationV2Run,
3270 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
3271 checkpointScope: {
3272 id: CheckpointScopeId.make("checkpoint-scope:subagent-link-survives-terminal"),
3273 } as OrchestrationV2CheckpointScope,
3274 providerThread: {
3275 id: ids.providerThreadId,
3276 driver,
3277 } as OrchestrationV2ProviderThread,
3278 attempt: {
3279 id: ids.attemptId,
3280 providerTurnId: ids.rootProviderTurnId,
3281 } as OrchestrationV2RunAttempt,
3282 attemptId: ids.attemptId,
3283 relatedThreadIds: [unrelatedChildThreadId],
3284 providerTurnOrdinal: 1,
3285 message: {
3286 messageId: MessageId.make("message:subagent-link-survives-terminal:user"),
3287 text: "Subagent settles before root interrupt.",
3288 attachments: [],
3289 createdBy: "user",
3290 creationSource: "web",
3291 },
3292 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
3293 runtimePolicy: {
3294 runtimeMode: "full-access",
3295 interactionMode: "default",
3296 cwd: process.cwd(),
3297 approvalPolicy: "never",
3298 sandboxPolicy: {
3299 type: "readOnly",
3300 access: { type: "fullAccess" },
3301 networkAccess: false,
3302 },
3303 },
3304 });
3305 }).pipe(Effect.provide(layerTest));
3306
3307 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
3308 assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
3309
3310 const events = yield* Ref.get(written);
3311 const runUpdatedIndex = events.findIndex((event) => event.type === "run.updated");
3312 assert.isAtLeast(runUpdatedIndex, 0, "root run.updated must be written");
3313
3314 const cascadedChildNodeEvents = events.flatMap((event, index) =>
3315 event.type === "node.updated" &&
3316 event.payload.id === runningChildNode.id &&
3317 event.payload.status === "interrupted"
3318 ? [{ event, index }]
3319 : [],
3320 );
3321 assert.lengthOf(
3322 cascadedChildNodeEvents,
3323 1,
3324 "open linked child-thread node must cascade after subagent/turn-item terminalize",
3325 );
3326 const cascadedChild = cascadedChildNodeEvents[0];
3327 assert.isDefined(cascadedChild);
3328 assert.isTrue(
3329 cascadedChild.index < runUpdatedIndex,
3330 "child-thread cascade must precede run.updated",
3331 );
3332 assert.equal(cascadedChild.event.payload.threadId, childThreadId);
3333 assert.equal(cascadedChild.event.payload.kind, "root_turn");
3334 assert.isFalse(
3335 events.some(
3336 (event) =>
3337 event.type === "node.updated" &&
3338 event.payload.id === unrelatedChildNode.id &&
3339 event.payload.status === "interrupted",
3340 ),
3341 "related but unlinked child threads must not cascade",
3342 );
3343 assert.isFalse(
3344 events.some(
3345 (event) => event.type === "subagent.updated" && event.payload.status === "interrupted",
3346 ),
3347 "already-terminal subagent rows must not be re-cascaded",
3348 );
3349 assert.isFalse(
3350 events.some(
3351 (event) =>
3352 event.type === "turn-item.updated" &&
3353 event.payload.type === "subagent" &&
3354 event.payload.status === "interrupted",
3355 ),
3356 "already-terminal subagent turn items must not be re-cascaded",
3357 );
3358 }),
3359);
3360
3361it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child projections", () =>
3362 Effect.gen(function* () {
3363 const now = yield* DateTime.now;
3364 let nextId = 0;
3365 const allocateEventId = () =>
3366 Effect.sync(() => EventId.make(`event:cascade-helper:${nextId++}`));
3367
3368 for (const driverKind of [
3369 ProviderDriverKind.make("claudeAgent"),
3370 ProviderDriverKind.make("codex"),
3371 ] as const) {
3372 const runId = RunId.make(`run:cascade-helper:${driverKind}`);
3373 const threadId = ThreadId.make(`thread:cascade-helper:${driverKind}`);
3374 const childThreadId = ThreadId.make(`thread:cascade-helper:${driverKind}:child`);
3375 const subagentId = NodeId.make(`node:cascade-helper:${driverKind}:subagent`);
3376 const childNodeId = NodeId.make(`node:cascade-helper:${driverKind}:child-root`);
3377 const providerInstanceId = ProviderInstanceId.make(String(driverKind));
3378 const terminalStatus = driverKind === "claudeAgent" ? "failed" : "cancelled";
3379 const subagent: OrchestrationV2Subagent = {
3380 id: subagentId,
3381 threadId,
3382 runId,
3383 parentNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`),
3384 origin: "provider_native",
3385 createdBy: "agent",
3386 driver: driverKind,
3387 providerInstanceId,
3388 providerThreadId: null,
3389 childThreadId,
3390 nativeTaskRef: {
3391 driver: driverKind,
3392 nativeId: `native-${driverKind}`,
3393 strength: "strong",
3394 },
3395 prompt: "hold",
3396 title: "hold",
3397 model: null,
3398 status: "running",
3399 progress: "partial progress",
3400 result: "partial result",
3401 startedAt: now,
3402 completedAt: null,
3403 updatedAt: now,
3404 };
3405 const turnItem = {
3406 id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}`),
3407 threadId,
3408 runId,
3409 nodeId: subagentId,
3410 providerThreadId: null,
3411 providerTurnId: null,
3412 nativeItemRef: subagent.nativeTaskRef,
3413 parentItemId: null,
3414 ordinal: 3,
3415 status: "running" as const,
3416 title: "hold",
3417 startedAt: now,
3418 completedAt: null,
3419 updatedAt: now,
3420 type: "subagent" as const,
3421 subagentId,
3422 origin: "provider_native" as const,
3423 driver: driverKind,
3424 providerInstanceId,
3425 childThreadId,
3426 prompt: "hold",
3427 progress: "partial progress",
3428 result: "partial result",
3429 } satisfies Extract<OrchestrationV2TurnItem, { type: "subagent" }>;
3430 const node: OrchestrationV2ExecutionNode = {
3431 id: subagentId,
3432 threadId,
3433 runId,
3434 parentNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`),
3435 rootNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`),
3436 kind: "subagent",
3437 status: "running",
3438 countsForRun: false,
3439 providerThreadId: null,
3440 providerTurnId: null,
3441 nativeItemRef: subagent.nativeTaskRef,
3442 runtimeRequestId: null,
3443 checkpointScopeId: null,
3444 startedAt: now,
3445 completedAt: null,
3446 };
3447 const openChildNode: OrchestrationV2ExecutionNode = {
3448 id: childNodeId,
3449 threadId: childThreadId,
3450 runId: null,
3451 parentNodeId: null,
3452 rootNodeId: childNodeId,
3453 kind: "root_turn",
3454 status: "running",
3455 countsForRun: false,
3456 providerThreadId: null,
3457 providerTurnId: null,
3458 nativeItemRef: null,
3459 runtimeRequestId: null,
3460 checkpointScopeId: null,
3461 startedAt: now,
3462 completedAt: null,
3463 };
3464 const childTurnItem: OrchestrationV2TurnItem =
3465 driverKind === "claudeAgent"
3466 ? {
3467 id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}:child-reasoning`),
3468 threadId: childThreadId,
3469 runId: null,
3470 nodeId: childNodeId,
3471 providerThreadId: null,
3472 providerTurnId: null,
3473 nativeItemRef: null,
3474 parentItemId: null,
3475 ordinal: 1,
3476 status: "running",
3477 title: "Working",
3478 startedAt: now,
3479 completedAt: null,
3480 updatedAt: now,
3481 type: "reasoning",
3482 text: "partial progress",
3483 streaming: true,
3484 }
3485 : {
3486 id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}:child-command`),
3487 threadId: childThreadId,
3488 runId: null,
3489 nodeId: childNodeId,
3490 providerThreadId: null,
3491 providerTurnId: null,
3492 nativeItemRef: null,
3493 parentItemId: null,
3494 ordinal: 1,
3495 status: "running",
3496 title: "sleep 300",
3497 startedAt: now,
3498 completedAt: null,
3499 updatedAt: now,
3500 type: "command_execution",
3501 input: "sleep 300",
3502 };
3503
3504 const events = yield* RunExecutionService.cascadeTerminalizeRunOwnedSubagents({
3505 run: {
3506 id: runId,
3507 threadId,
3508 ordinal: 1,
3509 providerInstanceId,
3510 } as OrchestrationV2Run,
3511 open: {
3512 subagents: new Map([[subagentId, subagent]]),
3513 turnItems: new Map([[subagentId, turnItem]]),
3514 childTurnItems: new Map(),
3515 nodes: new Map([[subagentId, node]]),
3516 linkedChildThreadIds: new Set([childThreadId]),
3517 },
3518 status: terminalStatus,
3519 completedAt: now,
3520 allocateEventId,
3521 });
3522
3523 assert.equal(events.length, 3, `${driverKind}: subagent + node + turn item`);
3524 const terminalSubagent = events.find((event) => event.type === "subagent.updated");
3525 assert.isDefined(terminalSubagent);
3526 if (terminalSubagent?.type !== "subagent.updated") {
3527 assert.fail("expected subagent.updated event");
3528 return;
3529 }
3530 assert.equal(terminalSubagent.payload.status, terminalStatus);
3531 assert.equal(terminalSubagent.payload.childThreadId, childThreadId);
3532 assert.equal(terminalSubagent.payload.progress, "partial progress");
3533 assert.equal(terminalSubagent.payload.result, "partial result");
3534 assert.equal(terminalSubagent.payload.driver, driverKind);
3535
3536 const terminalItem = events.find(
3537 (event) => event.type === "turn-item.updated" && event.payload.type === "subagent",
3538 );
3539 assert.isDefined(terminalItem);
3540 if (terminalItem?.type !== "turn-item.updated" || terminalItem.payload.type !== "subagent") {
3541 assert.fail("expected subagent turn-item.updated event");
3542 return;
3543 }
3544 assert.equal(terminalItem.payload.status, terminalStatus);
3545 assert.equal(terminalItem.payload.childThreadId, childThreadId);
3546 assert.equal(terminalItem.payload.progress, "partial progress");
3547 assert.equal(terminalItem.payload.result, "partial result");
3548
3549 // Shared cascade path: after subagent/turn-item rows are gone, only the
3550 // preserved linkage may prove an open child-thread node is cascadeable.
3551 const afterTerminalLinkEvents =
3552 yield* RunExecutionService.cascadeTerminalizeRunOwnedSubagents({
3553 run: {
3554 id: runId,
3555 threadId,
3556 ordinal: 1,
3557 providerInstanceId,
3558 } as OrchestrationV2Run,
3559 open: {
3560 subagents: new Map(),
3561 turnItems: new Map(),
3562 childTurnItems: new Map([[childTurnItem.id, childTurnItem]]),
3563 nodes: new Map([[childNodeId, openChildNode]]),
3564 linkedChildThreadIds: new Set([childThreadId]),
3565 },
3566 status: terminalStatus,
3567 completedAt: now,
3568 allocateEventId,
3569 });
3570 assert.equal(
3571 afterTerminalLinkEvents.length,
3572 2,
3573 `${driverKind}: linked child node and turn item cascade after link rows terminalize`,
3574 );
3575 const cascadedChild = afterTerminalLinkEvents.find((event) => event.type === "node.updated");
3576 assert.isDefined(cascadedChild);
3577 if (cascadedChild?.type !== "node.updated") {
3578 assert.fail("expected node.updated for linked child thread");
3579 return;
3580 }
3581 assert.equal(cascadedChild.payload.id, childNodeId);
3582 assert.equal(cascadedChild.payload.threadId, childThreadId);
3583 assert.equal(cascadedChild.payload.status, terminalStatus);
3584 assert.equal(cascadedChild.payload.kind, "root_turn");
3585 const cascadedChildTurnItem = afterTerminalLinkEvents.find(
3586 (event) => event.type === "turn-item.updated",
3587 );
3588 assert.isDefined(cascadedChildTurnItem);
3589 if (cascadedChildTurnItem?.type !== "turn-item.updated") {
3590 assert.fail("expected turn-item.updated for linked child thread");
3591 return;
3592 }
3593 assert.equal(cascadedChildTurnItem.threadId, childThreadId);
3594 assert.equal(cascadedChildTurnItem.payload.id, childTurnItem.id);
3595 assert.equal(cascadedChildTurnItem.payload.status, terminalStatus);
3596 assert.equal(cascadedChildTurnItem.payload.runId, null);
3597 assert.equal(cascadedChildTurnItem.payload.type, childTurnItem.type);
3598 if (cascadedChildTurnItem.payload.type === "reasoning") {
3599 assert.isFalse(cascadedChildTurnItem.payload.streaming);
3600 }
3601 }
3602 }),
3603);
3604
3605it.effect("omits interrupt results and subagent cascade for a superseded attempt", () =>
3606 Effect.gen(function* () {
3607 const { written, observed } = yield* captureRootRunTermination({
3608 key: "steer-supersede",
3609 shouldFinalizeRun: () => Effect.succeed(false),
3610 seedOpenSubagent: true,
3611 });
3612 assert.deepEqual(
3613 written.map((item) => item.type),
3614 [],
3615 );
3616 assert.deepEqual(observed, []);
3617 }),
3618);
3619
3620it.effect("emits run_interrupt_result when superseded attempt still has a hard-stop request", () =>
3621 Effect.gen(function* () {
3622 const { written, observed } = yield* captureRootRunTermination({
3623 key: "stop-then-steer-supersede",
3624 shouldFinalizeRun: () => Effect.succeed(false),
3625 hasUnpairedRunInterruptRequest: () => Effect.succeed(true),
3626 });
3627 assert.deepEqual(
3628 written.map((item) => item.type),
3629 ["run_interrupt_result"],
3630 );
3631 assert.deepEqual(observed, ["pull-requests-refreshed"]);
3632 const ids = backgroundScenarioIds("stop-then-steer-supersede");
3633 const expectedRequestId = yield* Effect.gen(function* () {
3634 const idAllocator = yield* IdAllocator.IdAllocatorV2;
3635 return idAllocator.derive.runSignalTurnItem({
3636 runId: ids.runId,
3637 signal: "interrupt-request",
3638 });
3639 }).pipe(Effect.provide(IdAllocator.layer));
3640 assert.equal(written[0]?.parentItemId, expectedRequestId);
3641 }),
3642);
3643
3644it.effect("omits run_interrupt_result when superseded attempt request is already paired", () =>
3645 Effect.gen(function* () {
3646 const { written, observed } = yield* captureRootRunTermination({
3647 key: "stop-then-steer-already-paired",
3648 shouldFinalizeRun: () => Effect.succeed(false),
3649 hasUnpairedRunInterruptRequest: () => Effect.succeed(false),
3650 });
3651 assert.deepEqual(
3652 written.map((item) => item.type),
3653 [],
3654 );
3655 assert.deepEqual(observed, []);
3656 }),
3657);
3658
3659it.effect("does not overwrite Stop when ownership changes after the finalization read", () =>
3660 Effect.gen(function* () {
3661 const { written, observed, submittedEffects, committedEffects } =
3662 yield* captureRootRunTermination({
3663 key: "stop-wins-finalization-gap",
3664 shouldFinalizeRun: () => Effect.succeed(true),
3665 rejectTerminalWrite: true,
3666 });
3667 assert.deepEqual(written, []);
3668 assert.deepEqual(observed, []);
3669 assert.deepEqual(
3670 submittedEffects.map((effect) => effect.request.type),
3671 ["checkpoint.capture"],
3672 );
3673 assert.deepEqual(committedEffects, []);
3674 }),
3675);
3676
3677it.effect("emits run_interrupt_result when hard-stop finalizes the active attempt", () =>
3678 Effect.gen(function* () {
3679 const { written, observed, committedEffects } = yield* captureRootRunTermination({
3680 key: "hard-stop",
3681 shouldFinalizeRun: () => Effect.succeed(true),
3682 });
3683 assert.deepEqual(
3684 written.map((item) => item.type),
3685 ["run_interrupt_result"],
3686 );
3687 assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]);
3688 assert.deepEqual(
3689 committedEffects.map((effect) => effect.request.type),
3690 ["checkpoint.capture"],
3691 );
3692 }),
3693);
3694
3695it.effect.each(["completed", "interrupted", "cancelled", "failed"] as const)(
3696 "refreshes pull requests after the current root run %s",
3697 (status) =>
3698 Effect.gen(function* () {
3699 const { observed } = yield* captureRootRunTermination({
3700 key: `pull-request-refresh:${status}`,
3701 shouldFinalizeRun: () => Effect.succeed(true),
3702 events: (ids) => Stream.make(rootTerminalEvent(ids, status)),
3703 });
3704 assert.deepEqual(observed, [
3705 `run:${status === "completed" ? "waiting" : status}`,
3706 "pull-requests-refreshed",
3707 ]);
3708 }),
3709);
3710
3711it.effect("records a finished run as failed when its ownership check cannot be read", () =>
3712 Effect.gen(function* () {
3713 const { observed } = yield* captureRootRunTermination({
3714 key: "finalize-guard-read-failure",
3715 shouldFinalizeRun: () =>
3716 Effect.fail(
3717 new ProjectionStore.ProjectionStoreReadError({
3718 threadId: ThreadId.make("thread:finalize-guard-read-failure"),
3719 cause: "database unavailable",
3720 }),
3721 ),
3722 events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
3723 });
3724 // The fallback settles through the guarded write instead of the same
3725 // failing read, so the run does not stay running.
3726 assert.include(observed, "run:failed");
3727 }),
3728);
3729
3730it.effect("does not refresh pull requests for auxiliary or stale provider terminals", () =>
3731 Effect.gen(function* () {
3732 const { observed } = yield* captureRootRunTermination({
3733 key: "pull-request-refresh:auxiliary",
3734 shouldFinalizeRun: () => Effect.succeed(true),
3735 events: (ids) =>
3736 Stream.fromIterable([
3737 rootTerminalEvent(
3738 { ...ids, rootProviderTurnId: ProviderTurnId.make("turn:child") },
3739 "completed",
3740 ),
3741 rootTerminalEvent(
3742 { ...ids, rootProviderTurnId: ProviderTurnId.make("turn:previous") },
3743 "interrupted",
3744 ),
3745 ]),
3746 });
3747 assert.deepEqual(observed, []);
3748 }),
3749);
3750
3751it.effect("refreshes pull requests after a provider stream exits with an error", () =>
3752 Effect.gen(function* () {
3753 const { written, observed } = yield* captureRootRunTermination({
3754 key: "pull-request-refresh:stream-error",
3755 shouldFinalizeRun: () => Effect.succeed(true),
3756 events: () =>
3757 Stream.fail(
3758 new ProviderAdapter.ProviderAdapterEventStreamError({
3759 driver,
3760 providerSessionId: ProviderSessionId.make("session:exited"),
3761 cause: "provider process exited",
3762 }),
3763 ),
3764 });
3765 assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3766 assert.deepEqual(
3767 written.map((item) => item.type),
3768 ["error"],
3769 );
3770 const error = written.find((item) => item.type === "error");
3771 assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly");
3772 }),
3773);
3774
3775it.effect("refreshes pull requests only once when startup failure closes its event stream", () =>
3776 Effect.gen(function* () {
3777 const ingestionStarted = yield* Deferred.make<void>();
3778 const { observed, written } = yield* captureRootRunTermination({
3779 key: "pull-request-refresh:startup-error",
3780 shouldFinalizeRun: () => Effect.succeed(true),
3781 events: () =>
3782 Stream.unwrap(Deferred.succeed(ingestionStarted, undefined).pipe(Effect.as(Stream.never))),
3783 startTurn: (input) =>
3784 Deferred.await(ingestionStarted).pipe(
3785 Effect.andThen(
3786 Effect.fail(
3787 new ProviderAdapter.ProviderAdapterTurnStartError({
3788 driver,
3789 threadId: input.threadId,
3790 providerThreadId: input.providerThread.id,
3791 runId: input.runId,
3792 cause: "provider rejected the turn",
3793 }),
3794 ),
3795 ),
3796 ),
3797 });
3798 assert.equal(observed.filter((item) => item === "pull-requests-refreshed").length, 1);
3799 assert.equal(observed[0], "run:failed");
3800 const error = written.find((item) => item.type === "error");
3801 assert.include(error?.failure.message ?? "", "provider could not start this turn");
3802 }),
3803);
3804
3805it.effect("keeps completed runs completed when pull request refresh fails", () =>
3806 Effect.gen(function* () {
3807 const { observed } = yield* captureRootRunTermination({
3808 key: "pull-request-refresh:refresh-failure",
3809 shouldFinalizeRun: () => Effect.succeed(true),
3810 events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
3811 refreshAfterTurn: Effect.die("refresh failed"),
3812 });
3813 assert.deepEqual(observed, ["run:waiting", "pull-requests-refreshed"]);
3814 }),
3815);
3816
3817function captureRootRunTermination(input: {
3818 readonly key: string;
3819 readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
3820 readonly rejectTerminalWrite?: boolean;
3821 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
3822 readonly seedOpenSubagent?: boolean;
3823 readonly events?: (
3824 ids: BackgroundScenarioIds,
3825 ) => Stream.Stream<
3826 ProviderAdapter.ProviderAdapterV2Event,
3827 ProviderAdapter.ProviderAdapterV2Error
3828 >;
3829 readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
3830 readonly refreshAfterTurn?: Effect.Effect<void>;
3831}) {
3832 return Effect.gen(function* () {
3833 const ids = backgroundScenarioIds(input.key);
3834 const providerInstanceId = ProviderInstanceId.make("codex");
3835 const runningSubagent = makeRunOwnedSubagentFixture({
3836 ids,
3837 providerInstanceId,
3838 childThreadId: ids.childThreadId,
3839 driver,
3840 status: "running",
3841 });
3842 const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
3843 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
3844 const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
3845 const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
3846 const ingestionDone = yield* Deferred.make<void>();
3847 const captureTurnItem = (payload: OrchestrationV2TurnItem) =>
3848 Ref.update(writtenItems, (current) => [...current, payload]);
3849 const captureFinalEvents = (events: ReadonlyArray<OrchestrationV2DomainEvent>) =>
3850 Effect.gen(function* () {
3851 for (const event of events) {
3852 if (event.type === "turn-item.updated") {
3853 yield* captureTurnItem(event.payload);
3854 }
3855 if (event.type === "run.updated") {
3856 yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
3857 }
3858 }
3859 });
3860 const layerTest = RunExecutionService.layer.pipe(
3861 Layer.provide(
3862 Layer.mergeAll(
3863 McpAppModelContext.layerEmpty,
3864 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3865 Layer.mock(EventSink.EventSinkV2)({
3866 write: (payload) =>
3867 Effect.gen(function* () {
3868 for (const event of payload.events) {
3869 if (event.type === "turn-item.updated") {
3870 yield* captureTurnItem(event.payload);
3871 }
3872 }
3873 return [];
3874 }),
3875 writeWithEffects: (payload) =>
3876 Effect.gen(function* () {
3877 yield* Ref.update(submittedEffects, (current) => [...current, ...payload.effects]);
3878 yield* Ref.update(committedEffects, (current) => [...current, ...payload.effects]);
3879 yield* captureFinalEvents(payload.events);
3880 return [];
3881 }),
3882 writeIfRunCurrent: (payload) =>
3883 Effect.gen(function* () {
3884 const effects = payload.effects ?? [];
3885 yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
3886 if (input.rejectTerminalWrite === true) {
3887 return { committed: false, storedEvents: [] };
3888 }
3889 yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
3890 yield* captureFinalEvents(payload.events);
3891 return { committed: true, storedEvents: [] };
3892 }),
3893 }),
3894 IdAllocator.layer,
3895 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3896 ingestNormalized: () => Effect.succeed([]),
3897 }),
3898 ServerSettings.layerTest(),
3899 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
3900 refresh: () => Effect.void,
3901 refreshAfterTurn: () =>
3902 Ref.update(observed, (current) => [...current, "pull-requests-refreshed"]).pipe(
3903 Effect.andThen(input.refreshAfterTurn ?? Effect.void),
3904 ),
3905 }),
3906 ),
3907 ),
3908 );
3909
3910 yield* Effect.gen(function* () {
3911 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3912 yield* runExecution.startRootRun({
3913 commandId: CommandId.make(`command:${input.key}`),
3914 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
3915 providerSessionId: ProviderSessionId.make(`session:${input.key}`),
3916 session: {
3917 events: Stream.empty,
3918 subscribeEvents: Effect.succeed({
3919 events:
3920 input.events?.(ids) ??
3921 Stream.fromIterable([
3922 ...(input.seedOpenSubagent
3923 ? [
3924 { type: "subagent.updated", driver, subagent: runningSubagent } as const,
3925 {
3926 type: "node.updated",
3927 driver,
3928 node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }),
3929 } as const,
3930 {
3931 type: "turn_item.updated",
3932 driver,
3933 turnItem: makeRunOwnedSubagentTurnItemFixture({
3934 ids,
3935 providerInstanceId,
3936 childThreadId: ids.childThreadId,
3937 driver,
3938 status: "running",
3939 }),
3940 } as const,
3941 ]
3942 : []),
3943 rootTerminalEvent(ids, "interrupted"),
3944 ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
3945 close: Deferred.succeed(ingestionDone, undefined),
3946 }),
3947 startTurn: input.startTurn ?? (() => Effect.void),
3948 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
3949 run: {
3950 id: ids.runId,
3951 threadId: ids.threadId,
3952 ordinal: 1,
3953 providerInstanceId,
3954 } as OrchestrationV2Run,
3955 rootNode: {
3956 id: ids.rootNodeId,
3957 providerTurnId: ids.rootProviderTurnId,
3958 } as OrchestrationV2ExecutionNode,
3959 checkpointScope: {
3960 id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`),
3961 } as OrchestrationV2CheckpointScope,
3962 providerThread: {
3963 id: ids.providerThreadId,
3964 driver,
3965 } as OrchestrationV2ProviderThread,
3966 attempt: {
3967 id: ids.attemptId,
3968 providerTurnId: ids.rootProviderTurnId,
3969 } as OrchestrationV2RunAttempt,
3970 attemptId: ids.attemptId,
3971 providerTurnOrdinal: 1,
3972 shouldFinalizeRun: input.shouldFinalizeRun,
3973 ...(input.hasUnpairedRunInterruptRequest === undefined
3974 ? {}
3975 : {
3976 hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest,
3977 }),
3978 message: {
3979 messageId: MessageId.make(`message:${input.key}`),
3980 text: "interrupt projection",
3981 attachments: [],
3982 createdBy: "user",
3983 creationSource: "web",
3984 },
3985 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
3986 runtimePolicy: {
3987 runtimeMode: "full-access",
3988 interactionMode: "default",
3989 cwd: process.cwd(),
3990 approvalPolicy: "never",
3991 sandboxPolicy: {
3992 type: "readOnly",
3993 access: { type: "fullAccess" },
3994 networkAccess: false,
3995 },
3996 },
3997 });
3998 }).pipe(Effect.provide(layerTest));
3999
4000 yield* Deferred.await(ingestionDone);
4001 return {
4002 written: yield* Ref.get(writtenItems),
4003 observed: yield* Ref.get(observed),
4004 submittedEffects: yield* Ref.get(submittedEffects),
4005 committedEffects: yield* Ref.get(committedEffects),
4006 };
4007 });
4008}
4009
4010interface BackgroundScenarioIds {
4011 readonly threadId: ThreadId;
4012 readonly childThreadId: ThreadId;
4013 readonly runId: RunId;
4014 readonly attemptId: RunAttemptId;
4015 readonly providerThreadId: ProviderThreadId;
4016 readonly rootProviderTurnId: ProviderTurnId;
4017 readonly rootNodeId: NodeId;
4018 readonly itemId: TurnItemId;
4019 readonly childItemId: TurnItemId;
4020 readonly subagentNodeId: NodeId;
4021}
4022
4023function backgroundScenarioIds(key: string): BackgroundScenarioIds {
4024 return {
4025 threadId: ThreadId.make(`thread:${key}`),
4026 childThreadId: ThreadId.make(`thread:${key}:child`),
4027 runId: RunId.make(`run:${key}`),
4028 attemptId: RunAttemptId.make(`attempt:${key}`),
4029 providerThreadId: ProviderThreadId.make(`provider-thread:${key}`),
4030 rootProviderTurnId: ProviderTurnId.make(`provider-turn:${key}`),
4031 rootNodeId: NodeId.make(`node:${key}`),
4032 itemId: TurnItemId.make(`turn-item:${key}`),
4033 childItemId: TurnItemId.make(`turn-item:${key}:child`),
4034 subagentNodeId: NodeId.make(`node:${key}:subagent`),
4035 };
4036}
4037
4038function childThreadCreatedEvent(
4039 ids: BackgroundScenarioIds,
4040): ProviderAdapter.ProviderAdapterV2Event {
4041 return {
4042 type: "app_thread.created",
4043 driver,
4044 appThread: {
4045 id: ids.childThreadId,
4046 lineage: {
4047 parentThreadId: ids.threadId,
4048 relationshipToParent: "subagent",
4049 rootThreadId: ids.threadId,
4050 },
4051 },
4052 } as ProviderAdapter.ProviderAdapterV2Event;
4053}
4054
4055function childBackgroundTurnItemEvent(
4056 ids: BackgroundScenarioIds,
4057 status: "running" | "completed" | "idle",
4058 ordinal: number,
4059): ProviderAdapter.ProviderAdapterV2Event {
4060 return {
4061 type: "turn_item.updated",
4062 driver,
4063 turnItem: {
4064 id: ids.childItemId,
4065 threadId: ids.childThreadId,
4066 runId: null,
4067 providerTurnId: null,
4068 ordinal,
4069 type: "command_execution",
4070 status,
4071 },
4072 } as ProviderAdapter.ProviderAdapterV2Event;
4073}
4074
4075function backgroundTurnItemEvent(
4076 ids: BackgroundScenarioIds,
4077 type: "command_execution" | "dynamic_tool" | "subagent",
4078 status: "running" | "completed" | "idle",
4079 ordinal: number,
4080 itemId?: TurnItemId,
4081): ProviderAdapter.ProviderAdapterV2Event {
4082 return {
4083 type: "turn_item.updated",
4084 driver,
4085 turnItem: {
4086 id: itemId ?? ids.itemId,
4087 threadId: ids.threadId,
4088 runId: ids.runId,
4089 providerTurnId: ids.rootProviderTurnId,
4090 ordinal,
4091 type,
4092 status,
4093 },
4094 } as ProviderAdapter.ProviderAdapterV2Event;
4095}
4096
4097function backgroundTurnItemEventForRun(
4098 ids: BackgroundScenarioIds,
4099 runId: RunId,
4100 type: "command_execution" | "dynamic_tool" | "subagent",
4101 status: "running" | "completed" | "idle",
4102 ordinal: number,
4103): ProviderAdapter.ProviderAdapterV2Event {
4104 const event = backgroundTurnItemEvent(ids, type, status, ordinal);
4105 if (event.type !== "turn_item.updated") {
4106 return event;
4107 }
4108 return { ...event, turnItem: { ...event.turnItem, runId } };
4109}
4110
4111function subagentEvent(
4112 ids: BackgroundScenarioIds,
4113 status: "running" | "completed" | "idle",
4114): ProviderAdapter.ProviderAdapterV2Event {
4115 return {
4116 type: "subagent.updated",
4117 driver,
4118 subagent: {
4119 id: ids.subagentNodeId,
4120 threadId: ids.threadId,
4121 runId: ids.runId,
4122 status,
4123 },
4124 } as ProviderAdapter.ProviderAdapterV2Event;
4125}
4126
4127function makeRunOwnedSubagentFixture(input: {
4128 readonly ids: BackgroundScenarioIds;
4129 readonly providerInstanceId: ProviderInstanceId;
4130 readonly childThreadId: ThreadId;
4131 readonly driver: typeof driver;
4132 readonly status: "running" | "interrupted";
4133}): OrchestrationV2Subagent {
4134 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4135 return {
4136 id: input.ids.subagentNodeId,
4137 threadId: input.ids.threadId,
4138 runId: input.ids.runId,
4139 parentNodeId: input.ids.rootNodeId,
4140 origin: "provider_native",
4141 createdBy: "agent",
4142 driver: input.driver,
4143 providerInstanceId: input.providerInstanceId,
4144 providerThreadId: null,
4145 childThreadId: input.childThreadId,
4146 nativeTaskRef: {
4147 driver: input.driver,
4148 nativeId: `task:${input.ids.subagentNodeId}`,
4149 strength: "strong",
4150 },
4151 prompt: "hold",
4152 title: "Live-test subagent hold",
4153 model: null,
4154 status: input.status,
4155 result: null,
4156 startedAt: now,
4157 completedAt: null,
4158 updatedAt: now,
4159 };
4160}
4161
4162function makeRunOwnedSubagentTurnItemFixture(input: {
4163 readonly ids: BackgroundScenarioIds;
4164 readonly providerInstanceId: ProviderInstanceId;
4165 readonly childThreadId: ThreadId;
4166 readonly driver: typeof driver;
4167 readonly status: "running" | "interrupted";
4168}): Extract<OrchestrationV2TurnItem, { type: "subagent" }> {
4169 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4170 return {
4171 id: input.ids.itemId,
4172 threadId: input.ids.threadId,
4173 runId: input.ids.runId,
4174 nodeId: input.ids.subagentNodeId,
4175 providerThreadId: input.ids.providerThreadId,
4176 providerTurnId: input.ids.rootProviderTurnId,
4177 nativeItemRef: {
4178 driver: input.driver,
4179 nativeId: `task:${input.ids.subagentNodeId}`,
4180 strength: "strong",
4181 },
4182 parentItemId: null,
4183 ordinal: 3,
4184 status: input.status,
4185 title: "Live-test subagent hold",
4186 startedAt: now,
4187 completedAt: null,
4188 updatedAt: now,
4189 type: "subagent",
4190 subagentId: input.ids.subagentNodeId,
4191 origin: "provider_native",
4192 driver: input.driver,
4193 providerInstanceId: input.providerInstanceId,
4194 childThreadId: input.childThreadId,
4195 prompt: "hold",
4196 result: null,
4197 };
4198}
4199
4200function makeLinkedChildTurnItemFixture(input: {
4201 readonly ids: BackgroundScenarioIds;
4202 readonly driver: typeof driver;
4203 readonly type: "assistant_message" | "command_execution" | "reasoning";
4204}): OrchestrationV2TurnItem {
4205 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4206 const base = {
4207 id: input.ids.childItemId,
4208 threadId: input.ids.childThreadId,
4209 runId: null,
4210 nodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4211 providerThreadId: null,
4212 providerTurnId: null,
4213 nativeItemRef: null,
4214 parentItemId: null,
4215 ordinal: 1,
4216 status: "running" as const,
4217 title: "Working",
4218 startedAt: now,
4219 completedAt: null,
4220 updatedAt: now,
4221 };
4222 if (input.type === "assistant_message") {
4223 return {
4224 ...base,
4225 type: "assistant_message",
4226 messageId: MessageId.make(`${input.ids.childItemId}:message`),
4227 text: "partial response",
4228 streaming: true,
4229 };
4230 }
4231 if (input.type === "reasoning") {
4232 return {
4233 ...base,
4234 type: "reasoning",
4235 text: "partial progress",
4236 streaming: true,
4237 };
4238 }
4239 return {
4240 ...base,
4241 type: "command_execution",
4242 input: "sleep 300",
4243 };
4244}
4245
4246function makeRunOwnedSubagentNodeFixture(input: {
4247 readonly ids: BackgroundScenarioIds;
4248 readonly status: "running" | "interrupted";
4249}): OrchestrationV2ExecutionNode {
4250 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4251 return {
4252 id: input.ids.subagentNodeId,
4253 threadId: input.ids.threadId,
4254 runId: input.ids.runId,
4255 parentNodeId: input.ids.rootNodeId,
4256 rootNodeId: input.ids.rootNodeId,
4257 kind: "subagent",
4258 status: input.status,
4259 countsForRun: false,
4260 providerThreadId: input.ids.providerThreadId,
4261 providerTurnId: input.ids.rootProviderTurnId,
4262 nativeItemRef: {
4263 driver,
4264 nativeId: `task:${input.ids.subagentNodeId}`,
4265 strength: "strong",
4266 },
4267 runtimeRequestId: null,
4268 checkpointScopeId: null,
4269 startedAt: now,
4270 completedAt: null,
4271 };
4272}
4273
4274function makeRunOwnedSubagentChildNodeFixture(input: {
4275 readonly ids: BackgroundScenarioIds;
4276 readonly status: "running" | "interrupted";
4277}): OrchestrationV2ExecutionNode {
4278 const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z");
4279 return {
4280 id: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4281 threadId: input.ids.childThreadId,
4282 runId: null,
4283 parentNodeId: null,
4284 rootNodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`),
4285 kind: "root_turn",
4286 status: input.status,
4287 countsForRun: false,
4288 providerThreadId: null,
4289 providerTurnId: null,
4290 nativeItemRef: null,
4291 runtimeRequestId: null,
4292 checkpointScopeId: null,
4293 startedAt: now,
4294 completedAt: null,
4295 };
4296}
4297
4298function rootTerminalEvent(
4299 ids: BackgroundScenarioIds,
4300 status: "completed" | "interrupted" | "cancelled" | "failed",
4301): ProviderAdapter.ProviderAdapterV2Event {
4302 const common = {
4303 type: "turn.terminal" as const,
4304 driver,
4305 providerThreadId: ids.providerThreadId,
4306 providerTurnId: ids.rootProviderTurnId,
4307 runOrdinal: 1,
4308 threadDisposition: "reusable" as const,
4309 };
4310 return status === "failed"
4311 ? {
4312 ...common,
4313 status,
4314 failureItemOrdinal: 101,
4315 failure: {
4316 class: "provider_error",
4317 message: "Provider failed",
4318 code: null,
4319 retryable: null,
4320 },
4321 }
4322 : { ...common, status, failure: null };
4323}
4324
4325function runBackgroundItemScenario(
4326 key: string,
4327 makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>,
4328 options?: {
4329 readonly keepEventStreamOpen?: boolean;
4330 readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect<
4331 ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }>
4332 >;
4333 readonly onSubscribe?: Effect.Effect<void>;
4334 },
4335) {
4336 return Effect.gen(function* () {
4337 const ids = backgroundScenarioIds(key);
4338 const providerInstanceId = ProviderInstanceId.make("codex");
4339 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
4340 const ingestionDone = yield* Deferred.make<void>();
4341 const layerTest = RunExecutionService.layer.pipe(
4342 Layer.provide(
4343 Layer.mergeAll(
4344 McpAppModelContext.layerEmpty,
4345 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
4346 Layer.mock(EventSink.EventSinkV2)({
4347 write: () => Effect.succeed([]),
4348 writeWithEffects: (input) =>
4349 Effect.gen(function* () {
4350 if (
4351 input.events.some(
4352 (event) => event.type === "run.updated" && event.runId === ids.runId,
4353 )
4354 ) {
4355 yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
4356 }
4357 return [];
4358 }),
4359 writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }),
4360 }),
4361 IdAllocator.layer,
4362 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4363 ingestNormalized: (input) =>
4364 Effect.gen(function* () {
4365 const event = input.event;
4366 if (event.type === "turn_item.updated") {
4367 yield* Ref.update(observed, (current) => [
4368 ...current,
4369 `turn_item:${event.turnItem.status}`,
4370 ]);
4371 }
4372 if (event.type === "subagent.updated") {
4373 yield* Ref.update(observed, (current) => [
4374 ...current,
4375 `subagent:${event.subagent.status}`,
4376 ]);
4377 }
4378 return [];
4379 }),
4380 }),
4381 ServerSettings.layerTest(),
4382 ),
4383 ),
4384 );
4385
4386 yield* Effect.gen(function* () {
4387 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4388 yield* runExecution.startRootRun({
4389 commandId: CommandId.make(`command:${key}`),
4390 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
4391 providerSessionId: ProviderSessionId.make(`session:${key}`),
4392 session: {
4393 events: Stream.empty,
4394 subscribeEvents: Effect.gen(function* () {
4395 yield* options?.onSubscribe ?? Effect.void;
4396 const events = Stream.fromIterable(makeEvents(ids));
4397 return {
4398 events:
4399 options?.keepEventStreamOpen === true
4400 ? events.pipe(Stream.concat(Stream.never))
4401 : events,
4402 close: Deferred.succeed(ingestionDone, undefined),
4403 };
4404 }),
4405 startTurn: () => Effect.void,
4406 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
4407 run: {
4408 id: ids.runId,
4409 threadId: ids.threadId,
4410 ordinal: 1,
4411 providerInstanceId,
4412 } as OrchestrationV2Run,
4413 rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode,
4414 checkpointScope: {
4415 id: CheckpointScopeId.make(`checkpoint-scope:${key}`),
4416 } as OrchestrationV2CheckpointScope,
4417 providerThread: {
4418 id: ids.providerThreadId,
4419 driver,
4420 } as OrchestrationV2ProviderThread,
4421 attempt: {
4422 id: ids.attemptId,
4423 providerTurnId: ids.rootProviderTurnId,
4424 } as OrchestrationV2RunAttempt,
4425 attemptId: ids.attemptId,
4426 ...(options?.loadInheritedBackgroundTurnItems === undefined
4427 ? {}
4428 : {
4429 loadInheritedBackgroundTurnItems: options.loadInheritedBackgroundTurnItems,
4430 }),
4431 providerTurnOrdinal: 1,
4432 message: {
4433 messageId: MessageId.make(`message:${key}:user`),
4434 text: "Start a background item and finish.",
4435 attachments: [],
4436 createdBy: "user",
4437 creationSource: "web",
4438 },
4439 modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
4440 runtimePolicy: {
4441 runtimeMode: "full-access",
4442 interactionMode: "default",
4443 cwd: process.cwd(),
4444 approvalPolicy: "never",
4445 sandboxPolicy: {
4446 type: "readOnly",
4447 access: { type: "fullAccess" },
4448 networkAccess: false,
4449 },
4450 },
4451 });
4452 }).pipe(Effect.provide(layerTest));
4453
4454 const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds"));
4455 assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish");
4456 return yield* Ref.get(observed);
4457 });
4458}
4459
4460it.effect("releases ingestion after idle subagent rows and items settle", () =>
4461 Effect.gen(function* () {
4462 const observed = yield* runBackgroundItemScenario(
4463 "subagent-idle",
4464 (ids) => [
4465 subagentEvent(ids, "running"),
4466 backgroundTurnItemEvent(ids, "subagent", "running", 1),
4467 subagentEvent(ids, "idle"),
4468 backgroundTurnItemEvent(ids, "subagent", "idle", 2),
4469 rootTerminalEvent(ids, "completed"),
4470 ],
4471 { keepEventStreamOpen: true },
4472 );
4473 assert.deepEqual(observed, [
4474 "subagent:running",
4475 "turn_item:running",
4476 "subagent:idle",
4477 "turn_item:idle",
4478 "root-finalized",
4479 ]);
4480 }),
4481);