1import { isManagedWorkspaceArchiveCandidate } from "../workspace/managedWorkspaceArchive.ts";
2import type {
3 OrchestrationV2SearchThreadInput,
4 OrchestrationV2SearchThreadResult,
5 OrchestrationV2ThreadHistoryPage,
6} from "@t3tools/contracts";
7import {
8 latestExecutedRun,
9 latestRootProviderFailure,
10 runRanAfter,
11 usageLimitBlockedRun,
12} from "@t3tools/shared/orchestrationV2ThreadError";
13import { threadPullRequestsOf } from "@t3tools/shared/threadPullRequests";
14import {
15 normalizeThreadPullRequestKey,
16 visibleThreadPullRequests,
17 threadPullRequestKeysEqual,
18 legacyThreadPullRequestKey,
19} from "@t3tools/shared/threadPullRequests";
20import {
21 ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
22 type ChatAttachment,
23 CommandId,
24 isProviderNativeSubagentThread,
25 MessageId,
26 type NodeId,
27 type ModelSelection,
28 OrchestrationV2Command,
29 type OrchestrationV2InternalCommand,
30 type OrchestrationV2ServerCommand,
31 type ThreadPullRequestLink,
32 type ThreadPullRequestWatch,
33 type OrchestrationV2AppThread,
34 type OrchestrationV2ContextHandoff,
35 type OrchestrationV2ContextSourcePoint,
36 type OrchestrationV2ContextTransfer,
37 type OrchestrationV2ContextTransferResolution,
38 type OrchestrationV2ConversationMessage,
39 type OrchestrationV2DelegatedCompletionCohort,
40 type OrchestrationV2DelegatedCompletionDelivery,
41 type OrchestrationV2DomainEvent,
42 type OrchestrationV2ExecutionNode,
43 type OrchestrationV2ProviderThread,
44 type OrchestrationV2ProviderTurn,
45 type OrchestrationV2Run,
46 type OrchestrationV2RunAttempt,
47 type OrchestrationV2ThreadShell,
48 type OrchestrationV2ThreadShellSnapshot,
49 type OrchestrationV2StoredEvent,
50 type OrchestrationV2Subagent,
51 type OrchestrationV2ThreadProjection,
52 type OrchestrationV2TurnItem,
53 latestProviderTurnForAttempt,
54 orchestrationV2RunWorkStartedAt,
55 ProviderInstanceId,
56 ProviderInteractionMode,
57 resolveProviderInstanceEnabled,
58 type ProviderSessionId,
59 RunId,
60 RuntimeMode,
61 ThreadLinkedPullRequest,
62 ThreadId,
63 type TurnItemId,
64} from "@t3tools/contracts";
65import { modelSelectionsEqual } from "@t3tools/shared/model";
66import {
67 derivePendingBackgroundWork,
68 pendingBackgroundTurnItems,
69} from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
70import * as Context from "effect/Context";
71import * as DateTime from "effect/DateTime";
72import * as Effect from "effect/Effect";
73import * as Exit from "effect/Exit";
74import * as FileSystem from "effect/FileSystem";
75import * as Path from "effect/Path";
76import * as Layer from "effect/Layer";
77import * as Option from "effect/Option";
78import * as Ref from "effect/Ref";
79import * as Schema from "effect/Schema";
80import * as Stream from "effect/Stream";
81
82import * as ProjectStore from "./ProjectStore.ts";
83import {
84 isCheckpointRestoreIsolated,
85 SHARED_WORKSPACE_RESTORE_MESSAGE,
86} from "./CheckpointRestoreSafety.ts";
87import { CheckpointServiceV2 } from "./CheckpointService.ts";
88import { CommandPolicyV2, resolveMessageDispatchIntent } from "./CommandPolicy.ts";
89import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
90import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
91import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
92import { isRestartContinuationSource, runOutcome } from "./RestartBackgroundNote.ts";
93import {
94 isBufferedProviderNotification,
95 isUndeliveredMailboxSteer,
96} from "./NotificationMailbox.ts";
97import { EventSinkV2 } from "./EventSink.ts";
98import * as EffectOutbox from "./EffectOutbox.ts";
99import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
100import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
101import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
102import { DispatchModeLimit, exceededDispatchModeLimit } from "./DispatchModeLimit.ts";
103import {
104 applyToProjection,
105 emptyProjection,
106 threadShellFromProjection,
107 ProjectionStoreV2,
108 type ProjectionRecordField,
109 type ProjectionTimelinePage,
110 type ProjectionTimelinePageOptions,
111 type ProjectionRecordFilter,
112 type ProjectionRecords,
113 type ProjectionCheckpointContext,
114 type ShellSnapshotOptions,
115} from "./ProjectionStore.ts";
116import { ProviderAdapterRegistryV2 } from "./ProviderAdapterRegistry.ts";
117import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
118import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
119import * as RunExecutionService from "./RunExecutionService.ts";
120import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
121import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts";
122import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
123import { RuntimePolicyV2 } from "./RuntimePolicy.ts";
124import {
125 endOrphanedNativeSubagent,
126 endRunlessRootTurns,
127 makeSubagentChildThread,
128 subagentResultForRun,
129 delegatedTaskProgress,
130 subagentThreadTitle,
131} from "@t3tools/provider-core/server/subagentProjection";
132import {
133 forkableSourceRunStatusError,
134 isForkableSourceRunStatus,
135 ThreadForkServiceV2,
136} from "./ThreadForkService.ts";
137import { planThreadDeletion } from "./ThreadDeletion.ts";
138import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
139import * as ServerSettings from "../serverSettings.ts";
140
141export class OrchestratorDispatchError extends Schema.TaggedError<OrchestratorDispatchError>()(
142 "OrchestratorDispatchError",
143 {
144 commandId: CommandId,
145 commandType: Schema.String,
146 reason: Schema.optional(Schema.Literal("archive-condition-changed")),
147 cause: Schema.optional(Schema.Defect()),
148 },
149) {
150 override get message(): string {
151 return `Failed to dispatch orchestration command ${this.commandType} (${this.commandId}).`;
152 }
153}
154
155export class OrchestratorCommandRejectedError extends Schema.TaggedError<OrchestratorCommandRejectedError>()(
156 "OrchestratorCommandRejectedError",
157 { commandId: CommandId, commandType: Schema.String, cause: Schema.optional(Schema.Defect()) },
158) {
159 override get message(): string {
160 return `Orchestration command ${this.commandType} (${this.commandId}) was rejected before commit.`;
161 }
162}
163
164export class OrchestratorProjectionError extends Schema.TaggedError<OrchestratorProjectionError>()(
165 "OrchestratorProjectionError",
166 {
167 threadId: ThreadId,
168 cause: Schema.optional(Schema.Defect()),
169 },
170) {
171 override get message(): string {
172 return `Failed to load orchestration projection for thread ${this.threadId}.`;
173 }
174}
175
176export class OrchestratorDomainEventStreamError extends Schema.TaggedError<OrchestratorDomainEventStreamError>()(
177 "OrchestratorDomainEventStreamError",
178 {
179 cause: Schema.optional(Schema.Defect()),
180 },
181) {
182 override get message(): string {
183 return "Failed while streaming orchestration domain events.";
184 }
185}
186
187export class OrchestratorProviderAdapterError extends Schema.TaggedError<OrchestratorProviderAdapterError>()(
188 "OrchestratorProviderAdapterError",
189 {
190 commandId: CommandId,
191 providerInstanceId: ProviderInstanceId,
192 cause: Schema.optional(Schema.Defect()),
193 },
194) {
195 override get message(): string {
196 return `Provider adapter failed while dispatching orchestration command ${this.commandId}.`;
197 }
198}
199
200export class OrchestratorSubagentThreadReadOnlyError extends Schema.TaggedError<OrchestratorSubagentThreadReadOnlyError>()(
201 "OrchestratorSubagentThreadReadOnlyError",
202 { commandId: CommandId, threadId: ThreadId },
203) {
204 override get message(): string {
205 return "This subagent is run by its provider and cannot take messages. Message the parent thread instead.";
206 }
207}
208
209/** The command's thread runs above the modes its sender may touch (see `DispatchModeLimit`). */
210export class OrchestratorThreadAboveModeLimitError extends Schema.TaggedError<OrchestratorThreadAboveModeLimitError>()(
211 "OrchestratorThreadAboveModeLimitError",
212 {
213 commandId: CommandId,
214 threadId: ThreadId,
215 mode: Schema.Literals(["runtime", "interaction"]),
216 runtimeMode: RuntimeMode,
217 interactionMode: ProviderInteractionMode,
218 },
219) {
220 override get message(): string {
221 return `Thread ${this.threadId} now runs in ${this.runtimeMode}/${this.interactionMode} mode, above what this caller may change.`;
222 }
223}
224
225export class OrchestratorCommandPreviouslyRejectedError extends Schema.TaggedError<OrchestratorCommandPreviouslyRejectedError>()(
226 "OrchestratorCommandPreviouslyRejectedError",
227 {
228 commandId: CommandId,
229 commandType: Schema.String,
230 detail: Schema.String,
231 },
232) {
233 override get message(): string {
234 return `Command ${this.commandId} was previously rejected: ${this.detail}`;
235 }
236}
237
238export class OrchestratorCommandIdConflictError extends Schema.TaggedError<OrchestratorCommandIdConflictError>()(
239 "OrchestratorCommandIdConflictError",
240 {
241 commandId: CommandId,
242 commandType: Schema.String,
243 receiptThreadId: ThreadId,
244 commandThreadId: ThreadId,
245 },
246) {
247 override get message(): string {
248 return `Command ${this.commandId} was already handled for thread ${this.receiptThreadId} and cannot be replayed for ${this.commandThreadId}.`;
249 }
250}
251
252/**
253 * A command receipt only proves that this exact command already ran for the
254 * thread it was recorded against. Replaying it for a command aimed at another
255 * thread would report success for work that never happened there, so the
256 * dispatcher rejects the reuse instead (mirrors v1's command-id conflict).
257 */
258export function canReplayCommandReceipt(
259 receiptThreadId: ThreadId,
260 commandThreadId: ThreadId,
261): boolean {
262 return receiptThreadId === commandThreadId;
263}
264
265export const OrchestratorV2Error = Schema.Union([
266 OrchestratorDispatchError,
267 OrchestratorCommandRejectedError,
268 OrchestratorProjectionError,
269 OrchestratorDomainEventStreamError,
270 OrchestratorProviderAdapterError,
271 OrchestratorCommandPreviouslyRejectedError,
272 OrchestratorCommandIdConflictError,
273 OrchestratorSubagentThreadReadOnlyError,
274 OrchestratorThreadAboveModeLimitError,
275]);
276export type OrchestratorV2Error = typeof OrchestratorV2Error.Type;
277
278export interface OrchestratorV2DispatchResult {
279 readonly sequence: number;
280 readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
281}
282
283export interface OrchestratorV2Shape {
284 readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
285 /** Startup pass that settles delegated-task results and deliveries runs left behind. */
286 readonly recoverDelegatedTasks: Effect.Effect<void>;
287 /** Settles a delegated child whose restart continuation of `sourceRunId` declined or failed. */
288 readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
289 /**
290 * Whether a delegated child's apparent result is not final yet: a restart
291 * continuation is pending, or the child is working again.
292 * A taskId limits the check to that follow-up's runs.
293 */
294 readonly delegatedTaskResultPending: (
295 childThreadId: ThreadId,
296 taskId?: NodeId,
297 ) => Effect.Effect<boolean, OrchestratorProjectionError>;
298 /**
299 * Whether a restart continuation of this cut run has not started yet, so the
300 * run's status is not its outcome. A failed read counts as not pending.
301 */
302 readonly restartContinuationPending: (run: OrchestrationV2Run) => Effect.Effect<boolean>;
303 /**
304 * Resolves once that continuation has run or was declined. A declined one
305 * records no thread event, so a waiter needs this to wake.
306 */
307 readonly awaitRestartContinuation: (run: OrchestrationV2Run) => Effect.Effect<void>;
308 readonly dispatch: (
309 command: OrchestrationV2ServerCommand,
310 ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
311 readonly searchThreadStream: (
312 input: OrchestrationV2SearchThreadInput,
313 ) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
314 readonly searchThread: (
315 input: OrchestrationV2SearchThreadInput,
316 ) => Effect.Effect<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
317 readonly getThreadHistoryPage: (
318 threadId: ThreadId,
319 cursor: string,
320 throughEntryId?: string,
321 conversationOnly?: boolean,
322 ) => Effect.Effect<OrchestrationV2ThreadHistoryPage, OrchestratorV2Error>;
323 readonly getTimelinePage: (
324 threadId: ThreadId,
325 options: ProjectionTimelinePageOptions,
326 ) => Effect.Effect<ProjectionTimelinePage, OrchestratorProjectionError>;
327 readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, OrchestratorV2Error>;
328 readonly getTurnItem: (input: {
329 readonly threadId: ThreadId;
330 readonly itemId: TurnItemId;
331 }) => Effect.Effect<OrchestrationV2TurnItem | null, OrchestratorV2Error>;
332 readonly getThreadRecords: <K extends ProjectionRecordField>(
333 threadId: ThreadId,
334 fields: ReadonlyArray<K>,
335 filter?: ProjectionRecordFilter,
336 ) => Effect.Effect<ProjectionRecords<K>, OrchestratorV2Error>;
337 readonly getThreadProjection: (
338 threadId: ThreadId,
339 ) => Effect.Effect<OrchestrationV2ThreadProjection, OrchestratorV2Error>;
340 readonly getCheckpointContext: (
341 threadId: ThreadId,
342 ) => Effect.Effect<ProjectionCheckpointContext, OrchestratorV2Error>;
343 readonly getThreadSnapshot: (threadId: ThreadId) => Effect.Effect<
344 {
345 readonly schemaVersion: number;
346 readonly snapshotSequence: number;
347 readonly projection: OrchestrationV2ThreadProjection;
348 },
349 OrchestratorV2Error
350 >;
351 readonly getThreadSnapshotWindow: (
352 threadId: ThreadId,
353 options: Parameters<ProjectionStoreV2["Service"]["getThreadSnapshotWindow"]>[1],
354 ) => Effect.Effect<
355 {
356 readonly schemaVersion: number;
357 readonly snapshotSequence: number;
358 readonly projection: OrchestrationV2ThreadProjection;
359 },
360 OrchestratorV2Error
361 >;
362 readonly getShellSnapshot: (
363 options?: ShellSnapshotOptions,
364 ) => Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>;
365 /** See `ProjectionStoreV2Shape.readShellSnapshot`. */
366 readonly readShellSnapshot: (
367 options?: ShellSnapshotOptions,
368 ) => Effect.Effect<
369 Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>,
370 OrchestratorV2Error
371 >;
372 readonly getThreadShell: (
373 threadId: ThreadId,
374 ) => Effect.Effect<OrchestrationV2ThreadShell | null, OrchestratorV2Error>;
375 readonly getThreadEventSequence: (
376 threadId: ThreadId,
377 ) => Effect.Effect<number, OrchestratorV2Error>;
378 readonly streamStoredEvents: Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
379 readonly streamStoredEventsFrom: (input?: {
380 readonly threadId?: ThreadId;
381 readonly afterSequence?: number;
382 /** Keep only this type, before the bounded buffer retains anything. */
383 readonly eventType?: OrchestrationV2DomainEvent["type"];
384 }) => Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
385 readonly streamDomainEvents: Stream.Stream<OrchestrationV2DomainEvent, OrchestratorV2Error>;
386}
387
388export class OrchestratorV2 extends Context.Service<OrchestratorV2, OrchestratorV2Shape>()(
389 "t3/orchestration-v2/Orchestrator/OrchestratorV2",
390) {}
391
392function nextRunOrdinal(projection: Pick<OrchestrationV2ThreadProjection, "runs">): number {
393 return projection.runs.length + 1;
394}
395
396/**
397 * A wake (background notification, delegated task result, restart
398 * continuation) carries on the work of the run that started last, so it keeps
399 * that work's start. Stamp it when the wake run starts, not when it queues:
400 * a queued prompt ahead of it has no start yet, and delegated results jump
401 * the queue. Other runs start new work.
402 */
403function wakeWorkStartedAt(
404 runs: ReadonlyArray<OrchestrationV2Run>,
405 trigger: {
406 readonly notification?: unknown;
407 readonly delegatedCompletion?: unknown;
408 readonly restartContinuationOfRunId?: RunId | undefined;
409 },
410): Pick<OrchestrationV2Run, "workStartedAt"> {
411 if (
412 trigger.notification === undefined &&
413 trigger.delegatedCompletion === undefined &&
414 trigger.restartContinuationOfRunId === undefined
415 ) {
416 return {};
417 }
418 const previous = runs
419 .flatMap((run) => (run.startedAt === null ? [] : [{ run, startedAt: run.startedAt }]))
420 .toSorted(
421 (left, right) =>
422 DateTime.Order(right.startedAt, left.startedAt) || right.run.ordinal - left.run.ordinal,
423 )[0]?.run;
424 return previous === undefined ? {} : { workStartedAt: orchestrationV2RunWorkStartedAt(previous) };
425}
426
427/** A native /compact or /logout turn: provider maintenance, not agent work. */
428export function isNativeMaintenanceCommand(message: {
429 readonly text: string;
430 readonly attachments: ReadonlyArray<ChatAttachment>;
431 readonly context?: import("@t3tools/contracts").OrchestrationMessageContext | undefined;
432}): boolean {
433 return (
434 message.attachments.length === 0 &&
435 ["/compact", "/logout"].includes(message.text.trim().toLowerCase())
436 );
437}
438
439/** A native `/goal` command. It changes the provider's goal, so it never steers a running turn. */
440function isGoalCommand(message: {
441 readonly text: string;
442 readonly attachments: ReadonlyArray<ChatAttachment>;
443}): boolean {
444 return message.attachments.length === 0 && /^\/goal(?:\s|$)/u.test(message.text.trim());
445}
446
447const threadPullRequestLinksEqual = Schema.toEquivalence(Schema.NullOr(ThreadLinkedPullRequest));
448
449function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
450 switch (command.type) {
451 case "thread.create":
452 case "thread.archive":
453 case "thread.unarchive":
454 case "thread.delete":
455 case "thread.settle":
456 case "thread.auto-settle":
457 case "thread.ci-failed":
458 case "thread.unsettle":
459 case "thread.snooze":
460 case "thread.unsnooze":
461 case "thread.auto-settle.set":
462 case "thread.pin":
463 case "thread.unpin":
464 case "thread.pin.reorder":
465 case "thread.active.reorder":
466 case "thread.visit":
467 case "thread.mark-unread":
468 case "thread.metadata.update":
469 case "thread.pull-request.link":
470 case "thread.pull-request.unlink":
471 case "thread.pull-request-link.sync":
472 case "thread.pull-request.watch":
473 case "thread.pull-request-watch.sync":
474 case "thread.pull-request.sync":
475 case "thread.title.regeneration.complete":
476 case "thread.runtime-mode.set":
477 case "thread.interaction-mode.set":
478 case "thread.model-selection.set":
479 case "provider-session.detach":
480 case "message.dispatch":
481 case "notification.delivery.accept":
482 case "prepared-run.release":
483 case "prepared-run.progress":
484 case "prepared-run.fail":
485 case "prepared-run.retry":
486 case "subagent.stop":
487 case "run.interrupt":
488 case "queued-message.promote-to-steer":
489 case "queue.resume":
490 case "queued-run.reorder":
491 case "queued-run.cancel":
492 case "queued-run.edit":
493 case "runtime-request.respond":
494 case "thread.user-input.dismiss":
495 case "checkpoint.rollback":
496 case "checkpoint.rollback.fail":
497 case "thread.background-work.settle":
498 case "thread.stop":
499 case "provider.switch":
500 return command.threadId;
501 case "delegated_task.request":
502 case "delegated_task.wake-policy":
503 case "delegated_task.completion-delivery.acknowledge":
504 case "delegated_task.completion-delivery.dispose":
505 case "thread.created.record":
506 return command.parentThreadId;
507 case "secret_request.record":
508 return command.threadId;
509 case "thread.fork":
510 case "thread.merge_back":
511 return command.targetThreadId;
512 }
513}
514
515function pendingThreadTitleGenerationEffect(
516 commandId: CommandId,
517 threadId: ThreadId,
518 kind:
519 | { readonly type: "initial"; readonly messageId: MessageId }
520 | { readonly type: "regenerate" },
521): PendingOrchestrationEffectV2 {
522 return {
523 id: `effect:${commandId}:thread-title.generate`,
524 commandId,
525 threadId,
526 request: { type: "thread-title.generate", kind },
527 };
528}
529
530const WORKSPACE_PREPARATION_INPUT = "Preparing workspace";
531
532/** A reopened preparation item drops the output and exit code of the attempt it replaces. */
533function withoutPreparationResult(
534 item: Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }>,
535) {
536 const { output: _output, exitCode: _exitCode, outputIndicatesFailure: _failure, ...rest } = item;
537 return rest;
538}
539
540function isBlockingRun(run: OrchestrationV2Run): boolean {
541 return (
542 run.status === "preparing" ||
543 run.status === "starting" ||
544 run.status === "running" ||
545 run.status === "waiting"
546 );
547}
548
549/**
550 * A parent thread is "live" for wake purposes while a run is still producing
551 * agent output. A run parked at "waiting" is post-terminal drain, so its agent
552 * turn is over and a wake is still needed.
553 */
554function hasLiveRun(projection: Pick<OrchestrationV2ThreadProjection, "runs">): boolean {
555 return projection.runs.some(
556 (run) => run.status === "preparing" || run.status === "starting" || run.status === "running",
557 );
558}
559
560/** The link with its watch replaced, or removed when `watch` is undefined. */
561function withPullRequestWatch(
562 link: ThreadPullRequestLink,
563 watch: ThreadPullRequestWatch | undefined,
564): ThreadPullRequestLink {
565 const { watch: _previous, ...rest } = link;
566 return watch === undefined ? rest : { ...rest, watch };
567}
568
569/** A legacy single-PR link as a link entry. Re-linking a pull request keeps its watch. */
570function legacyPullRequestLink(
571 thread: OrchestrationV2AppThread,
572 linked: ThreadLinkedPullRequest,
573 now: DateTime.Utc,
574): ThreadPullRequestLink {
575 const key = legacyThreadPullRequestKey(linked);
576 return withPullRequestWatch(
577 {
578 ...key,
579 url: linked.url,
580 source: "manual",
581 linkedAt: DateTime.formatIso(now),
582 snapshot: null,
583 stack: null,
584 },
585 threadPullRequestsOf(thread).find((link) => threadPullRequestKeysEqual(link, key))?.watch,
586 );
587}
588
589function delegatedCompletionWakeDetail(taskIds: ReadonlyArray<string>): string {
590 const taskList = taskIds.join(", ");
591 return taskIds.length === 1
592 ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.`
593 : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`;
594}
595
596function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean {
597 return (
598 status === "completed" ||
599 status === "failed" ||
600 status === "cancelled" ||
601 status === "interrupted"
602 );
603}
604
605function delegatedTaskTerminalStatus(
606 status: OrchestrationV2Run["status"],
607): OrchestrationV2Subagent["status"] | null {
608 switch (status) {
609 case "completed":
610 case "failed":
611 case "cancelled":
612 case "interrupted":
613 return status;
614 case "rolled_back":
615 return "cancelled";
616 case "preparing":
617 case "queued":
618 case "starting":
619 case "running":
620 case "waiting":
621 return null;
622 }
623}
624
625function nextQueuedRun(
626 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
627): OrchestrationV2Run | undefined {
628 return queuedRunsInDeliveryOrder(projection)[0];
629}
630
631function latestStableRun(
632 projection: Pick<OrchestrationV2ThreadProjection, "runs">,
633): OrchestrationV2Run | null {
634 return (
635 projection.runs
636 .filter((run) => run.status === "completed" && run.checkpointId !== null)
637 .toSorted((left, right) => right.ordinal - left.ordinal)[0] ?? null
638 );
639}
640
641function runForSourcePoint(
642 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "checkpoints">,
643 sourcePoint: Extract<
644 OrchestrationV2Command,
645 { readonly type: "thread.fork" | "thread.merge_back" }
646 >["sourcePoint"],
647): OrchestrationV2Run | null {
648 switch (sourcePoint.type) {
649 case "latest_stable":
650 return latestStableRun(projection);
651 case "run":
652 return projection.runs.find((run) => run.id === sourcePoint.runId) ?? null;
653 case "checkpoint": {
654 const checkpoint = projection.checkpoints.find(
655 (candidate) => candidate.id === sourcePoint.checkpointId,
656 );
657 return checkpoint?.runId === null || checkpoint === undefined
658 ? null
659 : (projection.runs.find((run) => run.id === checkpoint.runId) ?? null);
660 }
661 }
662}
663
664function providerThreadForRun(
665 projection: Pick<OrchestrationV2ThreadProjection, "providerThreads">,
666 run: OrchestrationV2Run,
667): OrchestrationV2ProviderThread | undefined {
668 return run.providerThreadId === null
669 ? undefined
670 : projection.providerThreads.find((candidate) => candidate.id === run.providerThreadId);
671}
672
673function providerTurnForRun(
674 projection: Pick<OrchestrationV2ThreadProjection, "providerTurns" | "attempts">,
675 run: OrchestrationV2Run,
676): OrchestrationV2ProviderTurn | undefined {
677 if (run.activeAttemptId === null) {
678 return undefined;
679 }
680
681 return (
682 latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId) ??
683 projection.providerTurns.find((turn) => {
684 const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
685 return attempt?.providerTurnId === turn.id;
686 })
687 );
688}
689
690function contextSourcePointForRun(
691 projection: Pick<
692 OrchestrationV2ThreadProjection,
693 "thread" | "providerThreads" | "providerTurns" | "attempts"
694 >,
695 run: OrchestrationV2Run,
696): OrchestrationV2ContextSourcePoint {
697 const providerThread = providerThreadForRun(projection, run);
698 const providerTurn = providerTurnForRun(projection, run);
699 return {
700 threadId: projection.thread.id,
701 runId: run.id,
702 ...(run.checkpointId === null ? {} : { checkpointId: run.checkpointId }),
703 ...(providerThread?.nativeThreadRef === null || providerThread?.nativeThreadRef === undefined
704 ? {}
705 : { providerThreadRef: providerThread.nativeThreadRef }),
706 ...(providerTurn?.nativeTurnRef === null || providerTurn?.nativeTurnRef === undefined
707 ? {}
708 : { providerTurnRef: providerTurn.nativeTurnRef }),
709 };
710}
711
712function pendingForkTransferForThread(
713 projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
714): OrchestrationV2ContextTransfer | undefined {
715 return projection.contextTransfers.find(
716 (transfer) =>
717 transfer.type === "fork" &&
718 transfer.targetThreadId === projection.thread.id &&
719 transfer.status === "pending",
720 );
721}
722
723function pendingMergeBackTransfersForThread(
724 projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
725): ReadonlyArray<OrchestrationV2ContextTransfer> {
726 return projection.contextTransfers.filter(
727 (transfer) =>
728 transfer.type === "merge_back" &&
729 transfer.targetThreadId === projection.thread.id &&
730 transfer.status === "pending",
731 );
732}
733
734function latestContextTransfer(
735 transfers: ReadonlyArray<OrchestrationV2ContextTransfer>,
736): OrchestrationV2ContextTransfer | undefined {
737 return transfers.reduce<OrchestrationV2ContextTransfer | undefined>((latest, transfer) => {
738 if (latest === undefined) {
739 return transfer;
740 }
741 return DateTime.toEpochMillis(transfer.updatedAt) >= DateTime.toEpochMillis(latest.updatedAt)
742 ? transfer
743 : latest;
744 }, undefined);
745}
746
747function visibleDeltaRunOrdinals(
748 projection: Pick<OrchestrationV2ThreadProjection, "runs">,
749 items: ReadonlyArray<OrchestrationV2TurnItem>,
750): OrchestrationV2ContextHandoff["coveredRunOrdinals"] {
751 const ordinals = items.flatMap((item) => {
752 if (item.runId === null) {
753 return [];
754 }
755 const run = projection.runs.find((candidate) => candidate.id === item.runId);
756 return run === undefined ? [] : [run.ordinal];
757 });
758 if (ordinals.length === 0) {
759 return { from: 1, to: 1 };
760 }
761 return {
762 from: Math.min(...ordinals),
763 to: Math.max(...ordinals),
764 };
765}
766
767export function shouldPrepareLegacyImportHandoff(input: {
768 readonly hasCompletedRun: boolean;
769 readonly historyOrigin: OrchestrationV2AppThread["historyOrigin"];
770 readonly legacyImportItemCount: number;
771}): boolean {
772 return (
773 input.historyOrigin === "v1_import" && !input.hasCompletedRun && input.legacyImportItemCount > 0
774 );
775}
776
777export function appendContextHandoffId(
778 handoffIds: OrchestrationV2ProviderThread["handoffIds"],
779 handoffId: OrchestrationV2ContextHandoff["id"] | null,
780): OrchestrationV2ProviderThread["handoffIds"] {
781 return handoffId === null ? handoffIds : Array.from(new Set([...handoffIds, handoffId]));
782}
783
784function rootProviderThreadsForProvider(
785 projection: Pick<OrchestrationV2ThreadProjection, "providerThreads" | "thread">,
786 providerInstanceId: ModelSelection["instanceId"],
787): ReadonlyArray<OrchestrationV2ProviderThread> {
788 return projection.providerThreads
789 .filter(
790 (providerThread) =>
791 providerThread.providerInstanceId === providerInstanceId &&
792 providerThread.appThreadId === projection.thread.id &&
793 providerThread.ownerNodeId === null,
794 )
795 .toSorted(
796 (left, right) =>
797 (right.lastRunOrdinal ?? 0) - (left.lastRunOrdinal ?? 0) ||
798 DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
799 );
800}
801
802// Failed and interrupted turns still contain conversation the next provider needs.
803// Queued, cancelled, and rolled-back runs must not be replayed as conversation.
804function isHandoffSourceRun(run: OrchestrationV2Run): boolean {
805 return run.status === "completed" || run.status === "failed" || run.status === "interrupted";
806}
807
808function lastDeliveredRunForProviderThread(
809 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "providerTurns">,
810 providerThreadId: OrchestrationV2ProviderThread["id"],
811): OrchestrationV2Run | undefined {
812 return projection.runs.findLast(
813 (run) =>
814 isHandoffSourceRun(run) &&
815 run.providerThreadId === providerThreadId &&
816 (run.status === "completed" ||
817 projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)),
818 );
819}
820
821const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
822 const checkpointService = yield* CheckpointServiceV2;
823 const commandPolicy = yield* CommandPolicyV2;
824 const contextHandoffService = yield* ContextHandoffServiceV2;
825 const eventSink = yield* EventSinkV2;
826 const commandReceipts = yield* CommandReceiptStoreV2;
827 const idAllocator = yield* IdAllocator.IdAllocatorV2;
828 const projects = yield* ProjectStore.ProjectStoreV2;
829 const projectionStore = yield* ProjectionStoreV2;
830 const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
831 const nextTurnItemOrdinal = (
832 projection: Pick<OrchestrationV2ThreadProjection, "thread"> &
833 Partial<Pick<OrchestrationV2ThreadProjection, "turnItems">>,
834 ) =>
835 projectionStore.getNextTurnItemOrdinal(projection.thread.id).pipe(
836 Effect.mapError(
837 (cause) => new OrchestratorProjectionError({ threadId: projection.thread.id, cause }),
838 ),
839 Effect.map((next) =>
840 (projection.turnItems ?? []).reduce(
841 (ordinal, item) => Math.max(ordinal, item.ordinal + 1),
842 next,
843 ),
844 ),
845 );
846
847 const fileSystem = yield* FileSystem.FileSystem;
848 const path = yield* Path.Path;
849 const providerAdapters = yield* ProviderAdapterRegistryV2;
850 const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
851 const providerSessions = yield* ProviderSessionManagerV2;
852 const providerSwitchService = yield* ProviderSwitchServiceV2;
853 const serverSettings = yield* ServerSettings.ServerSettingsService;
854 const runtimePolicy = yield* RuntimePolicyV2;
855 const threadForkService = yield* ThreadForkServiceV2;
856 const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
857
858 const mapDispatchError =
859 (command: OrchestrationV2ServerCommand) =>
860 <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, OrchestratorDispatchError, R> =>
861 effect.pipe(
862 Effect.mapError(
863 (cause) =>
864 new OrchestratorDispatchError({
865 commandId: command.commandId,
866 commandType: command.type,
867 cause,
868 }),
869 ),
870 );
871
872 const mapDelegatedCompletionError = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
873 effect.pipe(
874 Effect.mapError(
875 (cause) =>
876 new OrchestratorDispatchError({
877 commandId: CommandId.make("command:system:delegated-completion-delivery"),
878 commandType: "delegated_task.completion-delivery",
879 cause,
880 }),
881 ),
882 );
883
884 const providerSessionIdFor = (input: {
885 readonly adapter: ProviderAdapter.ProviderAdapterV2["Service"];
886 readonly providerInstanceId: ProviderInstanceId;
887 readonly threadId: ThreadId;
888 }) =>
889 input.adapter.getCapabilities().pipe(
890 Effect.flatMap((capabilities) =>
891 capabilities.sessions.supportsMultipleProviderThreadsPerSession
892 ? Effect.succeed(
893 idAllocator.derive.providerSession({
894 providerInstanceId: input.providerInstanceId,
895 }),
896 )
897 : idAllocator.allocate.providerSession({
898 providerInstanceId: input.providerInstanceId,
899 threadId: input.threadId,
900 }),
901 ),
902 );
903
904 const enforceCommandPolicy =
905 (command: OrchestrationV2Command) =>
906 <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, OrchestratorDispatchError, R> =>
907 effect.pipe(
908 Effect.mapError(
909 (cause) =>
910 new OrchestratorDispatchError({
911 commandId: command.commandId,
912 commandType: command.type,
913 cause,
914 }),
915 ),
916 );
917
918 const makeEvent = <Event extends OrchestrationV2DomainEvent>(
919 command: OrchestrationV2ServerCommand,
920 event: Omit<Event, "id">,
921 ) =>
922 Effect.gen(function* () {
923 const eventId = yield* mapDispatchError(command)(
924 idAllocator.allocate.event({
925 threadId: event.threadId,
926 commandId: command.commandId,
927 }),
928 );
929 return {
930 ...event,
931 id: eventId,
932 } as Event;
933 });
934
935 const emit =
936 (events: Ref.Ref<Array<OrchestrationV2DomainEvent>>, command: OrchestrationV2ServerCommand) =>
937 <Event extends OrchestrationV2DomainEvent>(event: Omit<Event, "id">) =>
938 Effect.gen(function* () {
939 const withId = yield* makeEvent(command, event);
940 yield* Ref.update(events, (existing) => [...existing, withId]);
941 return withId;
942 });
943
944 // Command decisions need control records, not historical assistant/tool output.
945 // Handoff preparation explicitly reads its history after choosing a strategy.
946 // This runs inside the write transaction for every command, so finished leaf
947 // nodes of finished runs stay unread: busy threads hold thousands of them.
948 const readCommandProjection = (threadId: ThreadId) =>
949 projectionStore
950 .getThreadRecords(
951 threadId,
952 [
953 "runs",
954 "attempts",
955 "nodes",
956 "subagents",
957 "providerSessions",
958 "providerThreads",
959 "providerTurns",
960 "runtimeRequests",
961 "messages",
962 "turnItems",
963 "checkpointScopes",
964 "contextTransfers",
965 ],
966 {
967 turnItemTypes: ["user_message", "error"],
968 messageRoles: ["user"],
969 omitSettledLeafNodes: true,
970 },
971 )
972 .pipe(
973 Effect.map((records): OrchestrationV2ThreadProjection => ({
974 ...records,
975 checkpoints: [],
976 plans: [],
977 contextHandoffs: [],
978 visibleTurnItems: [],
979 updatedAt: records.thread.updatedAt,
980 })),
981 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
982 );
983
984 const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
985 projectionStore
986 .getThreadRecords(
987 threadId,
988 ["turnItems"],
989 runIds === undefined ? undefined : { turnItemRunIds: runIds },
990 )
991 .pipe(
992 Effect.map((records) => records.turnItems),
993 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
994 );
995
996 const getProjectionWithPendingEvents = (
997 threadId: ThreadId,
998 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
999 ) =>
1000 Effect.gen(function* () {
1001 const pending = (yield* Ref.get(events)).filter((event) => event.threadId === threadId);
1002 const stored = yield* Effect.option(readCommandProjection(threadId));
1003 let projection: OrchestrationV2ThreadProjection;
1004 if (Option.isSome(stored)) {
1005 projection = stored.value;
1006 } else {
1007 const created = pending.find(
1008 (
1009 event,
1010 ): event is Extract<OrchestrationV2DomainEvent, { readonly type: "thread.created" }> =>
1011 event.type === "thread.created",
1012 );
1013 if (created === undefined) {
1014 return yield* new OrchestratorProjectionError({ threadId });
1015 }
1016 projection = emptyProjection(created);
1017 }
1018
1019 for (const event of pending) {
1020 if (event.type === "thread.created" && projection.thread.id === event.payload.id) {
1021 projection = { ...projection, thread: event.payload, updatedAt: event.occurredAt };
1022 continue;
1023 }
1024 projection = applyToProjection(projection, event);
1025 }
1026 return projection;
1027 });
1028
1029 const makeSystemEvent = <Event extends OrchestrationV2DomainEvent>(event: Omit<Event, "id">) =>
1030 Effect.gen(function* () {
1031 const eventId = yield* idAllocator.allocate.event({
1032 threadId: event.threadId,
1033 });
1034 return {
1035 ...event,
1036 id: eventId,
1037 } as Event;
1038 });
1039
1040 const writeSystemEvents = (
1041 events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
1042 effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
1043 ) =>
1044 Effect.gen(function* () {
1045 const withIds = yield* Effect.forEach(events, (event) =>
1046 makeSystemEvent(event as Omit<OrchestrationV2DomainEvent, "id">),
1047 );
1048 yield* eventSink.writeWithEffects({ events: withIds, effects });
1049 });
1050
1051 const completionDeliveryRun = (
1052 projection: Pick<OrchestrationV2ThreadProjection, "messages" | "runs">,
1053 delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
1054 ) => {
1055 if (delivery == null) return undefined;
1056 const message = projection.messages.find((candidate) => candidate.id === delivery.messageId);
1057 return projection.runs.find((candidate) => candidate.id === message?.runId);
1058 };
1059
1060 const completionDeliveryMessage = (
1061 projection: Pick<OrchestrationV2ThreadProjection, "messages">,
1062 delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
1063 ) =>
1064 delivery === null || delivery === undefined
1065 ? undefined
1066 : projection.messages.find((candidate) => candidate.id === delivery.messageId);
1067
1068 const offerDelegatedCompletionDelivery = (threadId: ThreadId, parentRunId: RunId) =>
1069 Effect.gen(function* () {
1070 const projection = yield* projectionStore.getThreadRecords(
1071 threadId,
1072 ["runs", "messages", "providerTurns", "providerThreads"],
1073 { messageRoles: ["user"] },
1074 );
1075 const parentRun = projection.runs.find((candidate) => candidate.id === parentRunId);
1076 const cohort = parentRun?.delegatedCompletion;
1077 const delivery = cohort?.delivery;
1078 if (
1079 parentRun === undefined ||
1080 cohort?.disposition !== "open" ||
1081 delivery === null ||
1082 delivery === undefined ||
1083 delivery.taskIds.length === 0 ||
1084 projection.thread.archivedAt !== null ||
1085 projection.thread.deletedAt !== null ||
1086 (completionDeliveryMessage(projection, delivery) !== undefined &&
1087 !isUndeliveredMailboxSteer(projection, delivery.messageId))
1088 ) {
1089 return;
1090 }
1091 const providerThread =
1092 parentRun.providerThreadId === null
1093 ? undefined
1094 : projection.providerThreads.find(
1095 (candidate) => candidate.id === parentRun.providerThreadId,
1096 );
1097 if (providerThread === undefined) {
1098 return;
1099 }
1100 yield* continuationRequests.offer({
1101 threadId,
1102 providerThreadId: providerThread.id,
1103 driver: providerThread.driver,
1104 detail: null,
1105 delivery: "message_text",
1106 delegatedCompletion: {
1107 parentRunId,
1108 generation: delivery.generation,
1109 messageId: delivery.messageId,
1110 },
1111 });
1112 });
1113
1114 const offerDelegatedCompletionDeliveries = (threadId: ThreadId) =>
1115 Effect.gen(function* () {
1116 const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
1117 for (const run of projection.runs) {
1118 if (
1119 run.delegatedCompletion?.delivery !== undefined &&
1120 run.delegatedCompletion.delivery !== null
1121 ) {
1122 yield* offerDelegatedCompletionDelivery(threadId, run.id);
1123 }
1124 }
1125 });
1126
1127 const emitQueuedRunCancellation = (input: {
1128 readonly command: OrchestrationV2ServerCommand;
1129 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
1130 readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">;
1131 readonly run: OrchestrationV2Run;
1132 readonly now: DateTime.Utc;
1133 }) =>
1134 Effect.gen(function* () {
1135 const rootNode =
1136 input.run.rootNodeId === null
1137 ? undefined
1138 : input.projection.nodes.find((candidate) => candidate.id === input.run.rootNodeId);
1139 const attempt =
1140 input.run.activeAttemptId === null
1141 ? undefined
1142 : input.projection.attempts.find(
1143 (candidate) => candidate.id === input.run.activeAttemptId,
1144 );
1145 const emitEvent = emit(input.events, input.command);
1146 yield* emitEvent({
1147 type: "run.updated",
1148 threadId: input.run.threadId,
1149 runId: input.run.id,
1150 ...(input.run.rootNodeId === null ? {} : { nodeId: input.run.rootNodeId }),
1151 providerInstanceId: input.run.providerInstanceId,
1152 occurredAt: input.now,
1153 payload: {
1154 ...input.run,
1155 status: "cancelled",
1156 queuePosition: null,
1157 completedAt: input.now,
1158 },
1159 });
1160 if (attempt !== undefined && rootNode !== undefined) {
1161 yield* emitEvent({
1162 type: "run-attempt.updated",
1163 threadId: input.run.threadId,
1164 runId: input.run.id,
1165 nodeId: rootNode.id,
1166 providerInstanceId: input.run.providerInstanceId,
1167 occurredAt: input.now,
1168 payload: {
1169 ...attempt,
1170 status: "cancelled",
1171 completedAt: input.now,
1172 },
1173 });
1174 }
1175 if (rootNode !== undefined) {
1176 yield* emitEvent({
1177 type: "node.updated",
1178 threadId: input.run.threadId,
1179 runId: input.run.id,
1180 nodeId: rootNode.id,
1181 providerInstanceId: input.run.providerInstanceId,
1182 occurredAt: input.now,
1183 payload: {
1184 ...rootNode,
1185 status: "cancelled",
1186 completedAt: input.now,
1187 },
1188 });
1189 }
1190 });
1191
1192 const failQueuedRunStart = (threadId: ThreadId, cause: unknown) =>
1193 Effect.gen(function* () {
1194 const projection = yield* projectionStore.getThreadRecords(
1195 threadId,
1196 ["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
1197 { turnItemTypes: [], messageRoles: ["user"] },
1198 );
1199 const queuedRun = nextQueuedRun(projection);
1200 if (queuedRun === undefined) return;
1201 const now = yield* DateTime.now;
1202 const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId);
1203 const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId);
1204 const providerThread = projection.providerThreads.find(
1205 (entry) => entry.id === queuedRun.providerThreadId,
1206 );
1207 const handoffUnsupported =
1208 typeof cause === "object" &&
1209 cause !== null &&
1210 "_tag" in cause &&
1211 cause._tag === "CommandPolicyCapabilityUnsupportedError";
1212 const failureCause =
1213 typeof cause === "object" &&
1214 cause !== null &&
1215 "_tag" in cause &&
1216 cause._tag === "OrchestratorDispatchError" &&
1217 "cause" in cause
1218 ? cause.cause
1219 : cause;
1220 const failure = makeProviderFailure({
1221 cause: failureCause,
1222 code: handoffUnsupported ? "context_handoff_unsupported" : "queued_start_failed",
1223 class: handoffUnsupported ? "validation_error" : "unknown",
1224 });
1225 yield* writeSystemEvents([
1226 ...(attempt !== undefined && rootNode !== undefined
1227 ? [
1228 {
1229 type: "run-attempt.updated" as const,
1230 threadId,
1231 runId: queuedRun.id,
1232 nodeId: rootNode.id,
1233 providerInstanceId: queuedRun.providerInstanceId,
1234 occurredAt: now,
1235 payload: { ...attempt, status: "failed" as const, completedAt: now },
1236 },
1237 ]
1238 : []),
1239 ...(rootNode === undefined
1240 ? []
1241 : [
1242 {
1243 type: "node.updated" as const,
1244 threadId,
1245 runId: queuedRun.id,
1246 nodeId: rootNode.id,
1247 providerInstanceId: queuedRun.providerInstanceId,
1248 occurredAt: now,
1249 payload: { ...rootNode, status: "failed" as const, completedAt: now },
1250 },
1251 ]),
1252 ...(rootNode === undefined || providerThread === undefined
1253 ? []
1254 : [
1255 {
1256 type: "turn-item.updated" as const,
1257 threadId,
1258 runId: queuedRun.id,
1259 nodeId: rootNode.id,
1260 providerInstanceId: queuedRun.providerInstanceId,
1261 occurredAt: now,
1262 payload: {
1263 id: idAllocator.derive.turnItemFromProviderItem({
1264 driver: providerThread.driver,
1265 nativeItemId: `queued-start-failure:${queuedRun.id}`,
1266 }),
1267 threadId,
1268 runId: queuedRun.id,
1269 nodeId: rootNode.id,
1270 providerThreadId: providerThread.id,
1271 providerTurnId: null,
1272 nativeItemRef: null,
1273 parentItemId: null,
1274 ordinal: yield* nextTurnItemOrdinal(projection),
1275 status: "failed" as const,
1276 title: "Queued provider could not start",
1277 startedAt: now,
1278 completedAt: now,
1279 updatedAt: now,
1280 type: "error" as const,
1281 failure,
1282 },
1283 },
1284 ]),
1285 {
1286 type: "run.updated",
1287 threadId,
1288 runId: queuedRun.id,
1289 ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
1290 providerInstanceId: queuedRun.providerInstanceId,
1291 occurredAt: now,
1292 payload: { ...queuedRun, status: "failed", queuePosition: null, completedAt: now },
1293 },
1294 ]);
1295 });
1296
1297 const startNextQueuedRun = (
1298 threadId: ThreadId,
1299 options?: { readonly failedRunId?: RunId; readonly endedRunId?: RunId },
1300 ) =>
1301 Effect.gen(function* () {
1302 // Every terminal run checks the queue. Only a deliverable queued run
1303 // needs the transcript for provider handoff and legacy import context.
1304 if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
1305 const projection = yield* readCommandProjection(threadId);
1306 if (
1307 projection.thread.archivedAt !== null ||
1308 projection.thread.deletedAt !== null ||
1309 projection.runs.some(isBlockingRun) ||
1310 projection.runs.some((run) => run.status === "queued" && run.queueHeld === true)
1311 ) {
1312 return;
1313 }
1314
1315 // The limit already stopped this thread. Starting the queue would send
1316 // every waiting message and drop it from the queue as each one fails.
1317 const sessionError =
1318 projection.providerSessions
1319 .filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
1320 .toSorted(
1321 (left, right) =>
1322 DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1323 )[0]?.lastError ?? null;
1324 if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
1325 return;
1326 }
1327 const queuedRun = nextQueuedRun(projection);
1328 if (queuedRun === undefined) {
1329 return;
1330 }
1331 // A provider that just failed will likely fail the next message too.
1332 // Hold the queue so the user decides when to resume it. Validation
1333 // failures (setup, unsupported handoff) belong to that message alone,
1334 // and a message queued for another provider is how users recover.
1335 // A message queued behind a worktree that was cancelled or never created
1336 // would run in the project checkout instead, so it waits for the user.
1337 const failedRun = latestExecutedRun(projection.runs);
1338 const failureClass =
1339 failedRun?.id === options?.failedRunId
1340 ? latestRootProviderFailure(failedRun, projection.turnItems)?.class
1341 : undefined;
1342 const worktreeMissing =
1343 failedRun?.id === options?.endedRunId &&
1344 failedRun?.workspacePreparation?.type === "worktree" &&
1345 projection.thread.worktreePath === null;
1346 // A project can turn a provider off while a message waits in the queue,
1347 // including between a run ending and this check, or while the queue is
1348 // held and someone clicks resume. Neither path may start that run. Only
1349 // an explicit turn-off holds the queue: an instance with no settings
1350 // entry was already admitted when the message was queued.
1351 const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
1352 const projectOverride =
1353 settings.projectSettingsOverrides[projection.thread.projectId]?.providerInstances?.[
1354 queuedRun.providerInstanceId
1355 ]?.enabled;
1356 const machineInstance = Object.hasOwn(
1357 settings.providerInstances,
1358 queuedRun.providerInstanceId,
1359 )
1360 ? settings.providerInstances[queuedRun.providerInstanceId]
1361 : undefined;
1362 const instanceDisabledForProject =
1363 projectOverride !== undefined
1364 ? !projectOverride
1365 : machineInstance !== undefined && !resolveProviderInstanceEnabled(machineInstance);
1366 if (
1367 worktreeMissing ||
1368 instanceDisabledForProject ||
1369 (failureClass !== undefined &&
1370 failureClass !== "validation_error" &&
1371 failedRun?.providerInstanceId === queuedRun.providerInstanceId)
1372 ) {
1373 const now = yield* DateTime.now;
1374 yield* writeSystemEvents(
1375 projection.runs
1376 .filter((run) => run.status === "queued")
1377 .map((run) => ({
1378 type: "run.updated" as const,
1379 threadId,
1380 runId: run.id,
1381 providerInstanceId: run.providerInstanceId,
1382 occurredAt: now,
1383 payload: { ...run, queueHeld: true },
1384 })),
1385 );
1386 return;
1387 }
1388 const rootNodeId = queuedRun.rootNodeId;
1389 const attemptId = queuedRun.activeAttemptId;
1390 const providerThreadId = queuedRun.providerThreadId;
1391 if (rootNodeId === null || attemptId === null || providerThreadId === null) {
1392 return yield* new OrchestratorDispatchError({
1393 commandId: CommandId.make(`command:system:start-queued:${queuedRun.id}`),
1394 commandType: "message.dispatch",
1395 cause: `Queued run ${queuedRun.id} is missing execution identity.`,
1396 });
1397 }
1398
1399 const rootNode = projection.nodes.find((candidate) => candidate.id === rootNodeId);
1400 const attempt = projection.attempts.find((candidate) => candidate.id === attemptId);
1401 const queuedMessage = projection.messages.find(
1402 (candidate) => candidate.id === queuedRun.userMessageId,
1403 );
1404 const legacyQueuedTurnItem = projection.turnItems.find(
1405 (
1406 candidate,
1407 ): candidate is Extract<OrchestrationV2TurnItem, { readonly type: "user_message" }> =>
1408 candidate.type === "user_message" &&
1409 candidate.runId === queuedRun.id &&
1410 candidate.messageId === queuedRun.userMessageId,
1411 );
1412 const queuedProviderThread = projection.providerThreads.find(
1413 (candidate) => candidate.id === providerThreadId,
1414 );
1415 const storedCheckpointScope = projection.checkpointScopes.find(
1416 (scope) => scope.id === rootNode?.checkpointScopeId,
1417 );
1418 if (
1419 rootNode === undefined ||
1420 attempt === undefined ||
1421 queuedMessage === undefined ||
1422 queuedProviderThread === undefined ||
1423 (rootNode.checkpointScopeId !== null && storedCheckpointScope === undefined)
1424 ) {
1425 return yield* new OrchestratorDispatchError({
1426 commandId: CommandId.make(`command:system:start-queued:${queuedRun.id}`),
1427 commandType: "message.dispatch",
1428 cause: `Queued run ${queuedRun.id} is missing projection state.`,
1429 });
1430 }
1431
1432 const commandId = CommandId.make(`command:system:start-queued:${queuedRun.id}`);
1433 const now = yield* DateTime.now;
1434 const selectionChanged = !modelSelectionsEqual(
1435 projection.thread.modelSelection,
1436 queuedRun.modelSelection,
1437 );
1438 const switchPlan = selectionChanged
1439 ? yield* providerSwitchService
1440 .plan({ projection, targetModelSelection: queuedRun.modelSelection })
1441 .pipe(
1442 Effect.mapError(
1443 (cause) =>
1444 new OrchestratorDispatchError({
1445 commandId,
1446 commandType: "message.dispatch",
1447 cause,
1448 }),
1449 ),
1450 )
1451 : null;
1452 const activeProviderThread = projection.providerThreads.find(
1453 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
1454 );
1455 const canResumeAcrossInstances =
1456 switchPlan?.instanceChanged === true &&
1457 switchPlan.transition.type === "restart_and_resume" &&
1458 activeProviderThread !== undefined &&
1459 activeProviderThread.nativeThreadRef !== null;
1460 const deliveryProviderThread =
1461 canResumeAcrossInstances && activeProviderThread !== undefined
1462 ? {
1463 ...queuedProviderThread,
1464 nativeThreadRef: activeProviderThread.nativeThreadRef,
1465 nativeConversationHeadRef: activeProviderThread.nativeConversationHeadRef,
1466 nativeMetadata: activeProviderThread.nativeMetadata,
1467 }
1468 : queuedProviderThread;
1469 const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1470 Effect.mapError(
1471 (cause) =>
1472 new OrchestratorDispatchError({
1473 commandId,
1474 commandType: "message.dispatch",
1475 cause,
1476 }),
1477 ),
1478 );
1479 const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
1480 Effect.mapError(
1481 (cause) =>
1482 new OrchestratorDispatchError({
1483 commandId,
1484 commandType: "message.dispatch",
1485 cause,
1486 }),
1487 ),
1488 );
1489 const latestCompletedRun = projection.runs.findLast((run) => run.status === "completed");
1490 const latestHandoffRun = projection.runs.findLast(isHandoffSourceRun);
1491 const targetLastCompletedRun = lastDeliveredRunForProviderThread(
1492 projection,
1493 queuedProviderThread.id,
1494 );
1495 const coveredRuns =
1496 canResumeAcrossInstances ||
1497 latestHandoffRun === undefined ||
1498 latestHandoffRun.providerInstanceId === queuedRun.providerInstanceId
1499 ? []
1500 : projection.runs.filter(
1501 (run) =>
1502 isHandoffSourceRun(run) &&
1503 run.ordinal > (targetLastCompletedRun?.ordinal ?? 0) &&
1504 run.ordinal <= latestHandoffRun.ordinal,
1505 );
1506 const needsFullContext = deliveryProviderThread.nativeThreadRef === null;
1507 const legacyImportItems =
1508 projection.thread.historyOrigin === "v1_import"
1509 ? yield* readHandoffItems(threadId, [null])
1510 : [];
1511 const handoffStrategy = needsFullContext
1512 ? ("full_thread_summary" as const)
1513 : ("delta_since_target_last_seen" as const);
1514 const transferId =
1515 coveredRuns.length === 0
1516 ? null
1517 : yield* idAllocator.allocate
1518 .contextTransfer({
1519 sourceThreadId: threadId,
1520 targetThreadId: threadId,
1521 type: "provider_handoff",
1522 })
1523 .pipe(
1524 Effect.mapError(
1525 (cause) =>
1526 new OrchestratorDispatchError({
1527 commandId,
1528 commandType: "message.dispatch",
1529 cause,
1530 }),
1531 ),
1532 );
1533 if (transferId !== null) {
1534 yield* commandPolicy.ensureContextHandoff({
1535 commandId,
1536 threadId,
1537 providerInstanceId: queuedRun.providerInstanceId,
1538 capabilities: targetCapabilities,
1539 strategy: needsFullContext ? "full_thread_summary" : "delta_context",
1540 });
1541 }
1542 const handoff =
1543 transferId === null || latestHandoffRun === undefined
1544 ? null
1545 : yield* contextHandoffService
1546 .prepareProviderHandoff({
1547 threadId,
1548 targetRunId: queuedRun.id,
1549 transferId,
1550 fromProviderThreadIds: Array.from(
1551 new Set(
1552 coveredRuns.flatMap((run) =>
1553 run.providerThreadId === null ? [] : [run.providerThreadId],
1554 ),
1555 ),
1556 ),
1557 toProviderThreadId: queuedProviderThread.id,
1558 fromProviderInstanceId: latestHandoffRun.providerInstanceId,
1559 toProviderInstanceId: queuedRun.providerInstanceId,
1560 coveredRunOrdinals: {
1561 from: coveredRuns[0]!.ordinal,
1562 to: coveredRuns.at(-1)!.ordinal,
1563 },
1564 runs: projection.runs,
1565 strategy: handoffStrategy,
1566 items: [
1567 ...(needsFullContext && latestCompletedRun !== undefined
1568 ? legacyImportItems
1569 : []),
1570 ...(yield* readHandoffItems(
1571 threadId,
1572 coveredRuns.map((run) => run.id),
1573 )),
1574 ],
1575 createdAt: now,
1576 })
1577 .pipe(
1578 Effect.mapError(
1579 (cause) =>
1580 new OrchestratorDispatchError({
1581 commandId,
1582 commandType: "message.dispatch",
1583 cause,
1584 }),
1585 ),
1586 );
1587 const legacyImportRecoveryHandoff =
1588 latestCompletedRun === undefined && needsFullContext && legacyImportItems.length > 0
1589 ? yield* contextHandoffService
1590 .prepareLegacyImport({
1591 threadId,
1592 targetRunId: queuedRun.id,
1593 toProviderThreadId: queuedProviderThread.id,
1594 toProviderInstanceId: queuedRun.providerInstanceId,
1595 items: legacyImportItems,
1596 createdAt: now,
1597 })
1598 .pipe(
1599 Effect.mapError(
1600 (cause) =>
1601 new OrchestratorDispatchError({
1602 commandId,
1603 commandType: "message.dispatch",
1604 cause,
1605 }),
1606 ),
1607 )
1608 : null;
1609 const activeHandoff = handoff ?? legacyImportRecoveryHandoff;
1610 const checkpointScope =
1611 storedCheckpointScope ??
1612 (yield* runtimePolicy
1613 .resolve({ thread: projection.thread, modelSelection: queuedRun.modelSelection })
1614 .pipe(
1615 Effect.flatMap((resolvedRuntimePolicy) =>
1616 checkpointService.prepareRootRunScope({
1617 threadId,
1618 runId: queuedRun.id,
1619 rootNodeId: rootNode.id,
1620 providerThreadId: queuedProviderThread.id,
1621 cwd: resolvedRuntimePolicy.cwd ?? projection.thread.worktreePath ?? process.cwd(),
1622 createdAt: now,
1623 }),
1624 ),
1625 Effect.mapError(
1626 (cause) =>
1627 new OrchestratorDispatchError({
1628 commandId,
1629 commandType: "message.dispatch",
1630 cause,
1631 }),
1632 ),
1633 ));
1634 const providerSessionId =
1635 (!canResumeAcrossInstances &&
1636 queuedProviderThread.providerSessionId !== null &&
1637 !switchPlan?.releaseProviderSessionIds.includes(queuedProviderThread.providerSessionId)
1638 ? queuedProviderThread.providerSessionId
1639 : null) ??
1640 (yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1641 Effect.flatMap((adapter) =>
1642 providerSessionIdFor({
1643 adapter,
1644 providerInstanceId: queuedRun.providerInstanceId,
1645 threadId,
1646 }),
1647 ),
1648 Effect.mapError(
1649 (cause) =>
1650 new OrchestratorDispatchError({
1651 commandId,
1652 commandType: "message.dispatch",
1653 cause,
1654 }),
1655 ),
1656 ));
1657 const providerThread: OrchestrationV2ProviderThread = {
1658 ...deliveryProviderThread,
1659 providerSessionId,
1660 status: "not_loaded",
1661 firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? queuedRun.ordinal,
1662 lastRunOrdinal: queuedRun.ordinal,
1663 handoffIds: appendContextHandoffId(
1664 appendContextHandoffId(queuedProviderThread.handoffIds, handoff?.id ?? null),
1665 legacyImportRecoveryHandoff?.id ?? null,
1666 ),
1667 updatedAt: now,
1668 };
1669 const startingRun: OrchestrationV2Run = {
1670 ...queuedRun,
1671 status: "starting",
1672 queuePosition: null,
1673 startedAt: null,
1674 contextHandoffId: activeHandoff?.id ?? null,
1675 ...wakeWorkStartedAt(projection.runs, {
1676 notification: queuedMessage.notification,
1677 delegatedCompletion: queuedMessage.delegatedCompletion,
1678 restartContinuationOfRunId: queuedRun.restartContinuationOfRunId,
1679 }),
1680 };
1681 const userTurnItem: OrchestrationV2TurnItem = {
1682 ...(legacyQueuedTurnItem ?? {
1683 id: idAllocator.derive.userTurnItem({ messageId: queuedMessage.id }),
1684 threadId,
1685 runId: queuedRun.id,
1686 nodeId: rootNodeId,
1687 providerThreadId: queuedProviderThread.id,
1688 providerTurnId: null,
1689 nativeItemRef: null,
1690 parentItemId: null,
1691 ordinal: queuedRun.ordinal * 100,
1692 status: "completed",
1693 title: null,
1694 type: "user_message",
1695 messageId: queuedMessage.id,
1696 text: queuedMessage.text,
1697 attachments: queuedMessage.attachments,
1698 ...(queuedMessage.context ? { context: queuedMessage.context } : {}),
1699 createdBy: queuedMessage.createdBy,
1700 creationSource: queuedMessage.creationSource,
1701 ...(queuedMessage.scheduledTaskId === undefined
1702 ? {}
1703 : { scheduledTaskId: queuedMessage.scheduledTaskId }),
1704 ...(queuedMessage.senderThreadId === undefined
1705 ? {}
1706 : { senderThreadId: queuedMessage.senderThreadId }),
1707 }),
1708 inputIntent: "queued_turn",
1709 startedAt: now,
1710 completedAt: now,
1711 updatedAt: now,
1712 };
1713 const handoffTurnItem: OrchestrationV2TurnItem | null =
1714 activeHandoff === null
1715 ? null
1716 : {
1717 id: idAllocator.derive.runSignalTurnItem({
1718 runId: queuedRun.id,
1719 signal: `context-handoff:${activeHandoff.id}`,
1720 }),
1721 threadId,
1722 runId: queuedRun.id,
1723 nodeId: rootNodeId,
1724 providerThreadId: queuedProviderThread.id,
1725 providerTurnId: null,
1726 nativeItemRef: null,
1727 parentItemId: null,
1728 ordinal: queuedRun.ordinal * 100 - 1,
1729 status: "completed",
1730 title: handoff === null ? "Imported context" : "Provider handoff",
1731 startedAt: now,
1732 completedAt: now,
1733 updatedAt: now,
1734 type: "handoff",
1735 contextHandoffId: activeHandoff.id,
1736 fromProviderThreadIds: activeHandoff.fromProviderThreadIds,
1737 toProviderThreadId: activeHandoff.toProviderThreadId,
1738 fromProviderInstanceIds: Array.from(
1739 new Set(coveredRuns.map((run) => run.providerInstanceId)),
1740 ),
1741 toProviderInstanceId: queuedRun.providerInstanceId,
1742 fromModelSelections: Array.from(
1743 new Map(
1744 coveredRuns.map((run) => [
1745 `${run.modelSelection.instanceId}\0${run.modelSelection.model}`,
1746 run.modelSelection,
1747 ]),
1748 ).values(),
1749 ),
1750 toModel: queuedRun.modelSelection.model,
1751 strategy: activeHandoff.strategy,
1752 summary: activeHandoff.summaryText,
1753 };
1754 const checkpointEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1755 storedCheckpointScope === undefined
1756 ? [
1757 {
1758 type: "checkpoint-scope.created",
1759 threadId,
1760 runId: queuedRun.id,
1761 nodeId: rootNode.id,
1762 providerInstanceId: queuedRun.providerInstanceId,
1763 occurredAt: now,
1764 payload: checkpointScope,
1765 },
1766 {
1767 type: "node.updated",
1768 threadId,
1769 runId: queuedRun.id,
1770 nodeId: rootNode.id,
1771 providerInstanceId: queuedRun.providerInstanceId,
1772 occurredAt: now,
1773 payload: { ...rootNode, checkpointScopeId: checkpointScope.id },
1774 },
1775 ]
1776 : [];
1777 const sessionsToDetach = projection.providerSessions.filter(
1778 (session) =>
1779 switchPlan?.releaseProviderSessionIds.includes(session.id) &&
1780 session.status !== "stopped" &&
1781 session.status !== "error",
1782 );
1783 yield* writeSystemEvents(
1784 [
1785 ...(selectionChanged
1786 ? [
1787 {
1788 type:
1789 queuedRun.providerInstanceId === projection.thread.providerInstanceId
1790 ? ("thread.model-selection-updated" as const)
1791 : ("thread.provider-switched" as const),
1792 threadId,
1793 providerInstanceId: queuedRun.providerInstanceId,
1794 occurredAt: now,
1795 payload: {
1796 ...projection.thread,
1797 providerInstanceId: queuedRun.providerInstanceId,
1798 modelSelection: queuedRun.modelSelection,
1799 updatedAt: now,
1800 },
1801 },
1802 ]
1803 : []),
1804 ...(handoff === null || transferId === null || latestHandoffRun === undefined
1805 ? []
1806 : [
1807 {
1808 type: "context-transfer.created" as const,
1809 threadId,
1810 runId: queuedRun.id,
1811 providerInstanceId: queuedRun.providerInstanceId,
1812 occurredAt: now,
1813 payload: {
1814 id: transferId,
1815 type: "provider_handoff" as const,
1816 sourceThreadId: threadId,
1817 targetThreadId: threadId,
1818 sourcePoint: contextSourcePointForRun(projection, latestHandoffRun),
1819 basePoint:
1820 needsFullContext || targetLastCompletedRun === undefined
1821 ? null
1822 : contextSourcePointForRun(projection, targetLastCompletedRun),
1823 sourceProviderInstanceId: latestHandoffRun.providerInstanceId,
1824 targetProviderInstanceId: queuedRun.providerInstanceId,
1825 targetRunId: queuedRun.id,
1826 status: "consumed" as const,
1827 resolution: {
1828 strategy: needsFullContext
1829 ? ("portable_context" as const)
1830 : ("delta_context" as const),
1831 contextHandoffId: handoff.id,
1832 },
1833 createdBy: queuedMessage.createdBy,
1834 error: null,
1835 createdAt: now,
1836 updatedAt: now,
1837 consumedAt: now,
1838 },
1839 },
1840 {
1841 type: "context-handoff.updated" as const,
1842 threadId,
1843 runId: queuedRun.id,
1844 providerInstanceId: queuedRun.providerInstanceId,
1845 occurredAt: now,
1846 payload: handoff,
1847 },
1848 ]),
1849 ...(legacyImportRecoveryHandoff === null
1850 ? []
1851 : [
1852 {
1853 type: "context-handoff.updated" as const,
1854 threadId,
1855 runId: queuedRun.id,
1856 providerInstanceId: queuedRun.providerInstanceId,
1857 occurredAt: now,
1858 payload: legacyImportRecoveryHandoff,
1859 },
1860 ]),
1861 ...(handoffTurnItem === null
1862 ? []
1863 : [
1864 {
1865 type: "turn-item.updated" as const,
1866 threadId,
1867 runId: queuedRun.id,
1868 nodeId: rootNodeId,
1869 providerInstanceId: queuedRun.providerInstanceId,
1870 occurredAt: now,
1871 payload: handoffTurnItem,
1872 },
1873 ]),
1874 ...sessionsToDetach.map((session) => ({
1875 type: "provider-session.detached" as const,
1876 threadId,
1877 driver: session.driver,
1878 providerInstanceId: session.providerInstanceId,
1879 occurredAt: now,
1880 payload: {
1881 providerSessionId: session.id,
1882 detachedAt: now,
1883 reason: "Provider or model selection changed.",
1884 },
1885 })),
1886 ...checkpointEvents,
1887 {
1888 type: "provider-thread.updated",
1889 threadId,
1890 providerInstanceId: queuedRun.providerInstanceId,
1891 occurredAt: now,
1892 payload: providerThread,
1893 },
1894 {
1895 type: "turn-item.updated",
1896 threadId,
1897 runId: queuedRun.id,
1898 nodeId: rootNodeId,
1899 providerInstanceId: queuedRun.providerInstanceId,
1900 occurredAt: now,
1901 payload: notificationTurnItem(userTurnItem, queuedMessage, projection.subagents),
1902 },
1903 {
1904 type: "run.updated",
1905 threadId,
1906 runId: queuedRun.id,
1907 nodeId: rootNodeId,
1908 providerInstanceId: queuedRun.providerInstanceId,
1909 occurredAt: now,
1910 payload: startingRun,
1911 },
1912 ],
1913 [
1914 ...sessionsToDetach.map((session) => ({
1915 id: `effect:${commandId}:provider-session.detach:${session.id}`,
1916 commandId,
1917 threadId,
1918 request: {
1919 type: "provider-session.detach" as const,
1920 providerSessionId: session.id,
1921 detail: "Provider or model selection changed.",
1922 },
1923 })),
1924 {
1925 id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,
1926 commandId,
1927 threadId,
1928 request: { type: "provider-turn.start", runId: queuedRun.id },
1929 },
1930 ],
1931 );
1932 }).pipe(Effect.catch((cause) => failQueuedRunStart(threadId, cause)));
1933
1934 const resumeQueuedRuns = Effect.gen(function* () {
1935 const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
1936 let resumed = 0;
1937 for (const threadId of threadIds) {
1938 const resumedThread = yield* Effect.gen(function* () {
1939 if (!(yield* projectionStore.canStartQueuedRun(threadId))) return false;
1940 yield* threadDispatch.withLock(threadId, startNextQueuedRun(threadId));
1941 return true;
1942 }).pipe(
1943 Effect.catch((cause) =>
1944 Effect.logWarning("Failed to resume queued V2 run after recovery", {
1945 threadId,
1946 cause,
1947 }).pipe(Effect.as(false)),
1948 ),
1949 );
1950 if (resumedThread) {
1951 resumed += 1;
1952 }
1953 }
1954 return resumed;
1955 }).pipe(
1956 Effect.mapError(
1957 (cause) =>
1958 new OrchestratorDispatchError({
1959 commandId: CommandId.make("command:system:resume-queued-runs"),
1960 commandType: "message.dispatch",
1961 cause,
1962 }),
1963 ),
1964 );
1965
1966 const dispatchDelegatedTaskCompletionDeliveryResolution = (
1967 command: Extract<
1968 OrchestrationV2Command,
1969 {
1970 readonly type:
1971 | "delegated_task.completion-delivery.acknowledge"
1972 | "delegated_task.completion-delivery.dispose";
1973 }
1974 >,
1975 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
1976 ) =>
1977 Effect.gen(function* () {
1978 const projection = yield* projectionStore
1979 .getThreadRecords(
1980 command.parentThreadId,
1981 ["subagents", "runs", "messages", "nodes", "attempts", "turnItems"],
1982 { turnItemTypes: [], messageRoles: ["user"] },
1983 )
1984 .pipe(
1985 Effect.mapError(
1986 (cause) =>
1987 new OrchestratorProjectionError({
1988 threadId: command.parentThreadId,
1989 cause,
1990 }),
1991 ),
1992 );
1993 const task = projection.subagents.find(
1994 (candidate) => candidate.id === command.taskId && candidate.origin === "app_owned",
1995 );
1996 if (task === undefined) {
1997 return yield* new OrchestratorDispatchError({
1998 commandId: command.commandId,
1999 commandType: command.type,
2000 cause: `Delegated task ${command.taskId} is not an app-owned task of thread ${command.parentThreadId}.`,
2001 });
2002 }
2003 const state =
2004 command.type === "delegated_task.completion-delivery.acknowledge"
2005 ? "acknowledged"
2006 : "disposed";
2007 const now = yield* DateTime.now;
2008 const emitEvent = emit(events, command);
2009 // task_status and t3_thread_read use distinct command IDs, so two
2010 // valid observations can race after their read preflight. Re-emit the
2011 // existing task row so the second dispatch is a successful idempotent
2012 // no-op rather than "already acknowledged/disposed" or empty-events.
2013 if (
2014 task.completionDelivery?.state === state ||
2015 (command.type === "delegated_task.completion-delivery.acknowledge" &&
2016 task.completionDelivery?.state === "disposed")
2017 ) {
2018 yield* emitEvent({
2019 type: "subagent.updated",
2020 threadId: command.parentThreadId,
2021 ...(task.runId === null ? {} : { runId: task.runId }),
2022 nodeId: task.id,
2023 driver: task.driver,
2024 providerInstanceId: task.providerInstanceId,
2025 occurredAt: now,
2026 payload: task,
2027 });
2028 return;
2029 }
2030 const updatedTask: OrchestrationV2Subagent = {
2031 ...task,
2032 completionDelivery: {
2033 state,
2034 observedByRunId:
2035 command.type === "delegated_task.completion-delivery.acknowledge"
2036 ? command.observedByRunId
2037 : null,
2038 },
2039 updatedAt: now,
2040 };
2041 yield* emitEvent({
2042 type: "subagent.updated",
2043 threadId: command.parentThreadId,
2044 ...(task.runId === null ? {} : { runId: task.runId }),
2045 nodeId: task.id,
2046 driver: task.driver,
2047 providerInstanceId: task.providerInstanceId,
2048 occurredAt: now,
2049 payload: updatedTask,
2050 });
2051
2052 const parentRun =
2053 task.runId === null
2054 ? undefined
2055 : projection.runs.find((candidate) => candidate.id === task.runId);
2056 const cohort = parentRun?.delegatedCompletion;
2057 const delivery = cohort?.delivery;
2058 if (
2059 parentRun === undefined ||
2060 cohort === undefined ||
2061 delivery === null ||
2062 delivery === undefined
2063 ) {
2064 return;
2065 }
2066 if (!delivery.taskIds.includes(task.id)) {
2067 return;
2068 }
2069
2070 const remainingTaskIds = delivery.taskIds.filter((taskId) => taskId !== task.id);
2071 const deliveryRun = completionDeliveryRun(projection, delivery);
2072 const clearDelivery =
2073 remainingTaskIds.length === 0 &&
2074 (deliveryRun === undefined || deliveryRun.status === "queued");
2075 const updatedCohort: OrchestrationV2DelegatedCompletionCohort = {
2076 ...cohort,
2077 delivery: clearDelivery
2078 ? null
2079 : {
2080 ...delivery,
2081 taskIds: remainingTaskIds,
2082 },
2083 };
2084 yield* emitEvent({
2085 type: "run.updated",
2086 threadId: command.parentThreadId,
2087 runId: parentRun.id,
2088 ...(parentRun.rootNodeId === null ? {} : { nodeId: parentRun.rootNodeId }),
2089 providerInstanceId: parentRun.providerInstanceId,
2090 occurredAt: now,
2091 payload: {
2092 ...parentRun,
2093 delegatedCompletion: updatedCohort,
2094 },
2095 });
2096
2097 if (deliveryRun?.status === "queued") {
2098 if (remainingTaskIds.length === 0) {
2099 yield* emitQueuedRunCancellation({
2100 command,
2101 events,
2102 projection,
2103 run: deliveryRun,
2104 now,
2105 });
2106 return;
2107 }
2108 const message = completionDeliveryMessage(projection, delivery);
2109 if (message !== undefined) {
2110 yield* emitEvent({
2111 type: "message.updated",
2112 threadId: command.parentThreadId,
2113 runId: deliveryRun.id,
2114 ...(deliveryRun.rootNodeId === null ? {} : { nodeId: deliveryRun.rootNodeId }),
2115 providerInstanceId: deliveryRun.providerInstanceId,
2116 occurredAt: now,
2117 payload: {
2118 ...message,
2119 text: delegatedCompletionWakeDetail(remainingTaskIds),
2120 delegatedCompletion: {
2121 parentRunId: parentRun.id,
2122 generation: delivery.generation,
2123 taskIds: remainingTaskIds,
2124 },
2125 updatedAt: now,
2126 },
2127 });
2128 }
2129 }
2130 });
2131
2132 const disposeDelegatedCompletionCohort = (input: {
2133 readonly command: OrchestrationV2ServerCommand;
2134 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
2135 readonly projection: Pick<
2136 OrchestrationV2ThreadProjection,
2137 "runs" | "subagents" | "messages" | "nodes" | "attempts"
2138 >;
2139 readonly parentRunId: RunId;
2140 readonly disposition: "stopped" | "disposed";
2141 readonly now: DateTime.Utc;
2142 readonly cancelQueuedDelivery?: boolean;
2143 }) =>
2144 Effect.gen(function* () {
2145 const parentRun = input.projection.runs.find(
2146 (candidate) => candidate.id === input.parentRunId,
2147 );
2148 if (parentRun === undefined) {
2149 return;
2150 }
2151 const cohort = parentRun.delegatedCompletion;
2152 const tasks = input.projection.subagents.filter(
2153 (candidate) => candidate.origin === "app_owned" && candidate.runId === input.parentRunId,
2154 );
2155 if (cohort === undefined && tasks.length === 0) {
2156 return;
2157 }
2158 const emitEvent = emit(input.events, input.command);
2159 const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition;
2160 const nextCohort = {
2161 disposition: nextDisposition,
2162 nextGeneration: cohort?.nextGeneration ?? 1,
2163 delivery: null,
2164 } as const;
2165 yield* emitEvent({
2166 type: "run.updated",
2167 threadId: parentRun.threadId,
2168 runId: parentRun.id,
2169 ...(parentRun.rootNodeId === null ? {} : { nodeId: parentRun.rootNodeId }),
2170 providerInstanceId: parentRun.providerInstanceId,
2171 occurredAt: input.now,
2172 payload: {
2173 ...parentRun,
2174 delegatedCompletion: nextCohort,
2175 },
2176 });
2177 for (const task of tasks) {
2178 if (
2179 task.completionDelivery?.state === "acknowledged" ||
2180 task.completionDelivery?.state === "delivered" ||
2181 task.completionDelivery?.state === "disposed"
2182 ) {
2183 continue;
2184 }
2185 yield* emitEvent({
2186 type: "subagent.updated",
2187 threadId: parentRun.threadId,
2188 ...(task.runId === null ? {} : { runId: task.runId }),
2189 nodeId: task.id,
2190 driver: task.driver,
2191 providerInstanceId: task.providerInstanceId,
2192 occurredAt: input.now,
2193 payload: {
2194 ...task,
2195 completionDelivery: {
2196 state: "disposed",
2197 observedByRunId: null,
2198 },
2199 updatedAt: input.now,
2200 },
2201 });
2202 }
2203 const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null);
2204 if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") {
2205 yield* emitQueuedRunCancellation({
2206 command: input.command,
2207 events: input.events,
2208 projection: input.projection,
2209 run: deliveryRun,
2210 now: input.now,
2211 });
2212 }
2213 });
2214
2215 const disposeAllDelegatedCompletionCohorts = (input: {
2216 readonly command: OrchestrationV2Command;
2217 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
2218 readonly projection: Pick<
2219 OrchestrationV2ThreadProjection,
2220 "runs" | "subagents" | "messages" | "nodes" | "attempts"
2221 >;
2222 readonly now: DateTime.Utc;
2223 readonly cancelQueuedDelivery?: boolean;
2224 }) =>
2225 Effect.forEach(
2226 Array.from(
2227 new Set([
2228 ...input.projection.runs
2229 .filter((run) => run.delegatedCompletion !== undefined)
2230 .map((run) => run.id),
2231 ...input.projection.subagents
2232 .filter((task) => task.origin === "app_owned" && task.runId !== null)
2233 .map((task) => task.runId!),
2234 ]),
2235 ),
2236 (parentRunId) =>
2237 disposeDelegatedCompletionCohort({
2238 ...input,
2239 parentRunId,
2240 disposition: "disposed",
2241 }),
2242 { concurrency: 1, discard: true },
2243 );
2244
2245 const dispatchThreadCreate = Effect.fn("orchestrationV2.dispatch.threadCreate")(function* (
2246 command: Extract<OrchestrationV2Command, { readonly type: "thread.create" }>,
2247 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
2248 ) {
2249 yield* Effect.annotateCurrentSpan({
2250 "orchestration_v2.command_id": command.commandId,
2251 "orchestration_v2.command_type": command.type,
2252 "orchestration_v2.thread_id": command.threadId,
2253 "orchestration_v2.driver": command.modelSelection.instanceId,
2254 });
2255
2256 const now = yield* DateTime.now;
2257 const emitEvent = emit(events, command);
2258 const thread: OrchestrationV2AppThread = {
2259 createdBy: command.createdBy,
2260 creationSource: command.creationSource,
2261 id: command.threadId,
2262 projectId: command.projectId,
2263 title: command.title,
2264 providerInstanceId: command.modelSelection.instanceId,
2265 modelSelection: command.modelSelection,
2266 runtimeMode: command.runtimeMode,
2267 interactionMode: command.interactionMode,
2268 branch: command.branch,
2269 worktreePath: command.worktreePath,
2270 activeProviderThreadId: null,
2271 lineage: {
2272 parentThreadId: null,
2273 relationshipToParent: null,
2274 rootThreadId: command.threadId,
2275 },
2276 forkedFrom: null,
2277 createdAt: now,
2278 updatedAt: now,
2279 archivedAt: null,
2280 settledOverride: null,
2281 settledAt: null,
2282 snoozedUntil: null,
2283 snoozedAt: null,
2284 lastVisitedAt: null,
2285 deletedAt: null,
2286 };
2287
2288 yield* emitEvent({
2289 type: "thread.created",
2290 threadId: command.threadId,
2291 providerInstanceId: command.modelSelection.instanceId,
2292 occurredAt: now,
2293 payload: thread,
2294 });
2295 if (command.importedNativeThread !== undefined) {
2296 yield* emitEvent({
2297 type: "provider-thread.updated",
2298 threadId: command.threadId,
2299 driver: command.importedNativeThread.ref.driver,
2300 providerInstanceId: command.modelSelection.instanceId,
2301 occurredAt: now,
2302 payload: {
2303 id: idAllocator.derive.providerThread({
2304 driver: command.importedNativeThread.ref.driver,
2305 providerInstanceId: command.modelSelection.instanceId,
2306 nativeThreadId: command.importedNativeThread.ref.nativeId,
2307 }),
2308 driver: command.importedNativeThread.ref.driver,
2309 providerInstanceId: command.modelSelection.instanceId,
2310 providerSessionId: null,
2311 appThreadId: command.threadId,
2312 ownerNodeId: null,
2313 nativeThreadRef: command.importedNativeThread.ref,
2314 nativeConversationHeadRef: null,
2315 status: "not_loaded",
2316 firstRunOrdinal: null,
2317 lastRunOrdinal: null,
2318 handoffIds: [],
2319 forkedFrom: null,
2320 contextUsage: null,
2321 nativeMetadata: command.importedNativeThread.metadata ?? null,
2322 createdAt: now,
2323 updatedAt: now,
2324 },
2325 });
2326 }
2327 });
2328
2329 const dispatchThreadVisit = Effect.fn("orchestrationV2.dispatch.threadVisit")(function* (
2330 command: Extract<OrchestrationV2Command, { readonly type: "thread.visit" }>,
2331 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
2332 ) {
2333 const thread = yield* projectionStore
2334 .getThread(command.threadId)
2335 .pipe(
2336 Effect.mapError(
2337 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
2338 ),
2339 );
2340 if (thread.deletedAt !== null) {
2341 return yield* new OrchestratorDispatchError({
2342 commandId: command.commandId,
2343 commandType: command.type,
2344 cause: `Thread ${command.threadId} is deleted.`,
2345 });
2346 }
2347 const visitedAt = DateTime.make(command.visitedAt);
2348 if (Option.isNone(visitedAt)) {
2349 return yield* new OrchestratorDispatchError({
2350 commandId: command.commandId,
2351 commandType: command.type,
2352 cause: `Thread ${command.threadId} visit time ${command.visitedAt} is not a valid timestamp.`,
2353 });
2354 }
2355 const movesForward =
2356 thread.lastVisitedAt === null ||
2357 DateTime.toEpochMillis(visitedAt.value) > DateTime.toEpochMillis(thread.lastVisitedAt);
2358 // Viewing a thread changes read state only. Loading its transcript (or
2359 // bumping updatedAt) makes a routine read receipt scale with its history.
2360 yield* emit(
2361 events,
2362 command,
2363 )({
2364 type: "thread.visited",
2365 threadId: command.threadId,
2366 providerInstanceId: thread.providerInstanceId,
2367 occurredAt: yield* DateTime.now,
2368 payload: movesForward ? { ...thread, lastVisitedAt: visitedAt.value } : thread,
2369 });
2370 });
2371
2372 // Checked under the thread lock: the watch or the thread can change while the host is read.
2373 // The watch is recorded first so the wake's own thread events carry it.
2374 const dispatchPullRequestWatchSync = Effect.fn("orchestrationV2.dispatch.pullRequestWatchSync")(
2375 function* (
2376 command: Extract<
2377 OrchestrationV2ServerCommand,
2378 { readonly type: "thread.pull-request-watch.sync" }
2379 >,
2380 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
2381 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
2382 ) {
2383 const thread = yield* projectionStore
2384 .getThread(command.threadId)
2385 .pipe(
2386 Effect.mapError(
2387 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
2388 ),
2389 );
2390 const key = normalizeThreadPullRequestKey(command);
2391 const link = threadPullRequestsOf(thread).find(
2392 (candidate) =>
2393 candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, key),
2394 );
2395 // Same rule as a direct message.dispatch: a provider-native subagent takes no messages.
2396 const inactive =
2397 thread.archivedAt !== null ||
2398 thread.settledOverride === "settled" ||
2399 thread.settledAt !== null ||
2400 isProviderNativeSubagentThread(thread);
2401 if (link?.watch?.startedAt !== command.startedAt || (command.wake && inactive)) {
2402 return yield* new OrchestratorDispatchError({
2403 commandId: command.commandId,
2404 commandType: command.type,
2405 cause: "The pull request watch ended or its thread settled while it was read.",
2406 });
2407 }
2408 yield* dispatchThreadMutation(command, events, effects);
2409 if (command.wake === undefined) return;
2410 yield* dispatchMessage(
2411 {
2412 type: "message.dispatch",
2413 commandId: command.commandId,
2414 threadId: command.threadId,
2415 messageId: command.wake.messageId,
2416 text: command.wake.text,
2417 notification: command.wake.notification,
2418 attachments: [],
2419 dispatchMode: { type: "queue_after_active" },
2420 createdBy: "agent",
2421 creationSource: "server",
2422 },
2423 events,
2424 effects,
2425 );
2426 },
2427 );
2428
2429 const dispatchThreadMutation = Effect.fn("orchestrationV2.dispatch.threadMutation")(function* (
2430 command: Extract<
2431 OrchestrationV2ServerCommand,
2432 {
2433 readonly type:
2434 | "thread.archive"
2435 | "thread.unarchive"
2436 | "thread.settle"
2437 | "thread.unsettle"
2438 | "thread.snooze"
2439 | "thread.unsnooze"
2440 | "thread.auto-settle.set"
2441 | "thread.pin"
2442 | "thread.unpin"
2443 | "thread.pin.reorder"
2444 | "thread.active.reorder"
2445 | "thread.mark-unread"
2446 | "thread.metadata.update"
2447 | "thread.pull-request.link"
2448 | "thread.pull-request.unlink"
2449 | "thread.pull-request-link.sync"
2450 | "thread.pull-request.watch"
2451 | "thread.pull-request-watch.sync"
2452 | "thread.pull-request.sync"
2453 | "thread.title.regeneration.complete"
2454 | "thread.runtime-mode.set"
2455 | "thread.interaction-mode.set"
2456 | "thread.model-selection.set"
2457 | "provider.switch";
2458 }
2459 >,
2460 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
2461 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
2462 ) {
2463 const thread = yield* projectionStore.getThread(command.threadId).pipe(
2464 Effect.mapError(
2465 (cause) =>
2466 new OrchestratorProjectionError({
2467 threadId: command.threadId,
2468 cause,
2469 }),
2470 ),
2471 );
2472 if (thread.deletedAt !== null) {
2473 return yield* new OrchestratorDispatchError({
2474 commandId: command.commandId,
2475 commandType: command.type,
2476 ...(command.type === "thread.archive" && command.snapshotSequence !== undefined
2477 ? { reason: "archive-condition-changed" as const }
2478 : {}),
2479 cause: `Thread ${command.threadId} is deleted.`,
2480 });
2481 }
2482 if (
2483 command.type === "thread.pull-request.watch" &&
2484 command.watching &&
2485 isProviderNativeSubagentThread(thread)
2486 ) {
2487 return yield* new OrchestratorSubagentThreadReadOnlyError({
2488 commandId: command.commandId,
2489 threadId: command.threadId,
2490 });
2491 }
2492 if (
2493 command.type === "thread.pull-request.watch" &&
2494 command.watching &&
2495 (thread.settledOverride === "settled" ||
2496 thread.settledAt !== null ||
2497 thread.archivedAt !== null)
2498 ) {
2499 return yield* new OrchestratorDispatchError({
2500 commandId: command.commandId,
2501 commandType: command.type,
2502 cause: `Thread ${command.threadId} is settled or archived and cannot watch pull requests.`,
2503 });
2504 }
2505 // A subagent or delegated task reports to its parent thread, which owns the pull request.
2506 // Its own watch would wake it on every change of a PR it was not asked to babysit.
2507 if (
2508 command.type === "thread.pull-request.watch" &&
2509 command.watching &&
2510 thread.lineage.relationshipToParent === "subagent"
2511 ) {
2512 return yield* new OrchestratorDispatchError({
2513 commandId: command.commandId,
2514 commandType: command.type,
2515 cause: `Thread ${command.threadId} is a subagent; its parent thread watches pull requests.`,
2516 });
2517 }
2518 // Only the agent's watch_pull_request links as "agent". A call that raced a Stop may land
2519 // after its run ended, so the latest run that executed decides. A user can still watch.
2520 if (
2521 command.type === "thread.pull-request.watch" &&
2522 command.watching &&
2523 command.link?.source === "agent"
2524 ) {
2525 const { runs } = yield* projectionStore
2526 .getThreadRecords(command.threadId, ["runs"])
2527 .pipe(mapDispatchError(command));
2528 const latest = latestExecutedRun(runs);
2529 if (latest !== null && (yield* stopReachedRun(command, command.threadId, latest.id))) {
2530 return yield* new OrchestratorDispatchError({
2531 commandId: command.commandId,
2532 commandType: command.type,
2533 cause: `Thread ${command.threadId} was stopped.`,
2534 });
2535 }
2536 }
2537 if (
2538 command.type === "thread.metadata.update" &&
2539 command.expectedWorktreePath !== undefined &&
2540 command.expectedWorktreePath !== thread.worktreePath
2541 ) {
2542 return yield* new OrchestratorDispatchError({
2543 commandId: command.commandId,
2544 commandType: command.type,
2545 cause: `Thread ${command.threadId} worktree changed before the metadata update could be applied.`,
2546 });
2547 }
2548 if (command.type === "thread.metadata.update" && command.expectedEmpty === true) {
2549 const records = yield* projectionStore
2550 .getThreadRecords(command.threadId, ["runs"])
2551 .pipe(mapDispatchError(command));
2552 const messageCount = yield* projectionStore
2553 .getMessageCount(command.threadId)
2554 .pipe(mapDispatchError(command));
2555 if (messageCount > 0 || records.runs.length > 0)
2556 return yield* new OrchestratorDispatchError({
2557 commandId: command.commandId,
2558 commandType: command.type,
2559 cause: `Thread ${command.threadId} is no longer empty.`,
2560 });
2561 }
2562 if (command.type === "thread.unsettle" && command.snapshotSequence !== undefined) {
2563 const sequence = yield* eventSink
2564 .latestSequence({ threadId: command.threadId })
2565 .pipe(mapDispatchError(command));
2566 if (thread.settledOverride !== "settled" || sequence > command.snapshotSequence) {
2567 return yield* new OrchestratorDispatchError({
2568 commandId: command.commandId,
2569 commandType: command.type,
2570 cause: `Thread ${command.threadId} changed before automatic settlement could be cancelled.`,
2571 });
2572 }
2573 }
2574 if (command.type === "thread.archive" && command.snapshotSequence !== undefined) {
2575 const shell = yield* projectionStore
2576 .getThreadShell(command.threadId)
2577 .pipe(mapDispatchError(command));
2578 const records = yield* projectionStore
2579 .getThreadRecords(command.threadId, ["runs", "runtimeRequests"])
2580 .pipe(mapDispatchError(command));
2581 const sequence = yield* eventSink
2582 .latestSequence({ threadId: command.threadId })
2583 .pipe(mapDispatchError(command));
2584 if (
2585 shell === null ||
2586 !isManagedWorkspaceArchiveCandidate(shell) ||
2587 sequence > command.snapshotSequence ||
2588 records.runs.some((run) =>
2589 ["queued", "preparing", "starting", "running", "waiting"].includes(run.status),
2590 ) ||
2591 records.runtimeRequests.some((request) => request.status === "pending")
2592 ) {
2593 return yield* new OrchestratorDispatchError({
2594 commandId: command.commandId,
2595 commandType: command.type,
2596 reason: "archive-condition-changed",
2597 cause: `Thread ${command.threadId} changed before automatic archival.`,
2598 });
2599 }
2600 }
2601 if (command.type === "thread.archive" && thread.archivedAt !== null) {
2602 return yield* new OrchestratorDispatchError({
2603 commandId: command.commandId,
2604 commandType: command.type,
2605 cause: `Thread ${command.threadId} is already archived.`,
2606 });
2607 }
2608 if (command.type === "thread.unarchive" && thread.archivedAt === null) {
2609 return yield* new OrchestratorDispatchError({
2610 commandId: command.commandId,
2611 commandType: command.type,
2612 cause: `Thread ${command.threadId} is not archived.`,
2613 });
2614 }
2615 if (
2616 (command.type === "thread.settle" ||
2617 command.type === "thread.unsettle" ||
2618 command.type === "thread.snooze" ||
2619 command.type === "thread.unsnooze" ||
2620 command.type === "thread.auto-settle.set" ||
2621 command.type === "thread.pin" ||
2622 command.type === "thread.unpin" ||
2623 command.type === "thread.pin.reorder" ||
2624 command.type === "thread.active.reorder" ||
2625 command.type === "thread.pull-request.sync") &&
2626 thread.archivedAt !== null
2627 ) {
2628 return yield* new OrchestratorDispatchError({
2629 commandId: command.commandId,
2630 commandType: command.type,
2631 cause: `Thread ${command.threadId} is archived.`,
2632 });
2633 }
2634 // Only pinned threads have a slot in the arranged order. Rejecting
2635 // (rather than silently pinning) keeps a raced reorder-after-unpin from
2636 // resurrecting a pin the user just cleared.
2637 if (command.type === "thread.pin.reorder" && thread.pinnedAt == null) {
2638 return yield* new OrchestratorDispatchError({
2639 commandId: command.commandId,
2640 commandType: command.type,
2641 cause: `Thread ${command.threadId} is not pinned and cannot be reordered.`,
2642 });
2643 }
2644 if (
2645 command.type === "thread.active.reorder" &&
2646 (thread.pinnedAt != null || thread.settledOverride === "settled")
2647 ) {
2648 return yield* new OrchestratorDispatchError({
2649 commandId: command.commandId,
2650 commandType: command.type,
2651 cause: `Thread ${command.threadId} is not active and cannot be reordered.`,
2652 });
2653 }
2654 if (command.type === "thread.pull-request.sync") {
2655 const project = yield* projects.get(command.projectId).pipe(mapDispatchError(command));
2656 const currentSequence = yield* eventSink.latestSequence({ threadId: command.threadId }).pipe(
2657 Effect.mapError(
2658 (cause) =>
2659 new OrchestratorProjectionError({
2660 threadId: command.threadId,
2661 cause,
2662 }),
2663 ),
2664 );
2665 if (
2666 Option.isNone(project) ||
2667 project.value.deletedAt !== null ||
2668 project.value.workspaceRoot !== command.expected.workspaceRoot ||
2669 currentSequence > command.snapshotSequence ||
2670 thread.projectId !== command.projectId ||
2671 thread.branch !== command.expected.branch ||
2672 thread.worktreePath !== command.expected.worktreePath ||
2673 !threadPullRequestLinksEqual(
2674 thread.linkedPullRequest ?? null,
2675 command.expected.linkedPullRequest,
2676 ) ||
2677 !threadPullRequestLinksEqual(
2678 thread.branchPullRequest ?? null,
2679 command.expected.branchPullRequest,
2680 )
2681 ) {
2682 return yield* new OrchestratorDispatchError({
2683 commandId: command.commandId,
2684 commandType: command.type,
2685 cause: `Thread ${command.threadId} changed before pull request discovery.`,
2686 });
2687 }
2688 }
2689 if (command.type === "thread.settle") {
2690 const projection = yield* loadProjectionForCommand(
2691 command,
2692 ["runs", "runtimeRequests", "messages"],
2693 { turnItemTypes: [], messageRoles: ["user"] },
2694 );
2695 // Queued notification and delegated-completion runs only wake the agent.
2696 // They are not user messages and are hidden from the queue UI, so they
2697 // must not block settling; they are cancelled below instead.
2698 const automaticMessageIds = new Set(
2699 projection.messages
2700 .filter(
2701 (message) =>
2702 message.notification !== undefined || message.delegatedCompletion !== undefined,
2703 )
2704 .map((message) => message.id),
2705 );
2706 const automaticQueuedRuns = projection.runs.filter(
2707 (run) => run.status === "queued" && automaticMessageIds.has(run.userMessageId),
2708 );
2709 const runningRunExists = projection.runs.some((run) =>
2710 ["preparing", "starting", "running", "waiting"].includes(run.status),
2711 );
2712 // Includes a queue held after a Stop, which otherwise shows no activity.
2713 const queuedMessageExists = projection.runs.some(
2714 (run) => run.status === "queued" && !automaticQueuedRuns.includes(run),
2715 );
2716 const pendingRequests = projection.runtimeRequests.filter(
2717 (request) => request.status === "pending",
2718 );
2719 const blockingRequestExists = pendingRequests.some(
2720 (request) => request.kind !== "user_input" || request.responseCapability.type !== "message",
2721 );
2722 // The reason is shown to the user, so it names what to clear first.
2723 const blocker = blockingRequestExists
2724 ? "is waiting on an approval or question. Answer it before settling."
2725 : runningRunExists
2726 ? "is still running. Stop it before settling."
2727 : queuedMessageExists
2728 ? "has a queued message. Send it or remove it from the queue before settling."
2729 : null;
2730 if (blocker !== null) {
2731 return yield* new OrchestratorDispatchError({
2732 commandId: command.commandId,
2733 commandType: command.type,
2734 cause: `Thread ${command.threadId} ${blocker}`,
2735 });
2736 }
2737
2738 // Message-capable async questions do not keep provider callbacks alive.
2739 // Resolve them in the settle transaction so a settled thread never has
2740 // an actionable question attached to it.
2741 for (const request of pendingRequests) {
2742 yield* dispatchRuntimeRequestRespond(
2743 {
2744 type: "runtime-request.respond",
2745 commandId: command.commandId,
2746 threadId: command.threadId,
2747 requestId: request.id,
2748 decision: "cancel",
2749 },
2750 events,
2751 effects,
2752 );
2753 }
2754 for (const run of automaticQueuedRuns) {
2755 yield* dispatchQueuedRunCancel(
2756 {
2757 type: "queued-run.cancel",
2758 commandId: command.commandId,
2759 threadId: command.threadId,
2760 runId: run.id,
2761 },
2762 events,
2763 );
2764 }
2765 }
2766
2767 const needsProviderState =
2768 command.type === "thread.runtime-mode.set" ||
2769 command.type === "thread.model-selection.set" ||
2770 command.type === "provider.switch" ||
2771 command.type === "thread.archive" ||
2772 command.type === "thread.settle" ||
2773 (command.type === "thread.metadata.update" &&
2774 command.worktreePath !== undefined &&
2775 command.worktreePath !== thread.worktreePath);
2776 const providerContext = needsProviderState
2777 ? yield* projectionStore
2778 .getThreadProviderContext(
2779 command.threadId,
2780 command.type === "thread.model-selection.set" || command.type === "provider.switch"
2781 ? command.modelSelection.instanceId
2782 : undefined,
2783 )
2784 .pipe(mapDispatchError(command))
2785 : null;
2786 const providerSwitchPlan =
2787 command.type === "thread.model-selection.set" || command.type === "provider.switch"
2788 ? yield* Effect.gen(function* () {
2789 yield* providerAdapters.get(command.modelSelection.instanceId).pipe(
2790 Effect.mapError(
2791 (cause) =>
2792 new OrchestratorProviderAdapterError({
2793 commandId: command.commandId,
2794 providerInstanceId: command.modelSelection.instanceId,
2795 cause,
2796 }),
2797 ),
2798 );
2799 return yield* providerSwitchService
2800 .plan({
2801 projection: providerContext!,
2802 targetModelSelection: command.modelSelection,
2803 })
2804 .pipe(mapDispatchError(command));
2805 })
2806 : null;
2807
2808 const now = yield* DateTime.now;
2809 if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
2810 const projection = yield* loadProjectionForCommand(
2811 command,
2812 ["runs", "runtimeRequests", "turnItems"],
2813 { turnItemTypes: ["error"] },
2814 );
2815 const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
2816 const failure = latestRootProviderFailure(run, projection.turnItems);
2817 const resetMs = Date.parse(command.limitRecovery.resetAt);
2818 if (command.limitRecovery.snooze === true && resetMs <= DateTime.toEpochMillis(now)) {
2819 return yield* new OrchestratorDispatchError({
2820 commandId: command.commandId,
2821 commandType: command.type,
2822 cause: "The reset time has passed. Retry the thread manually.",
2823 });
2824 }
2825 if (
2826 !Number.isFinite(resetMs) ||
2827 thread.archivedAt !== null ||
2828 thread.settledOverride === "settled" ||
2829 run?.id !== command.limitRecovery.runId ||
2830 failure?.class !== "usage_limit" ||
2831 failure.resetAt !== command.limitRecovery.resetAt ||
2832 resetMs <= DateTime.toEpochMillis(run.completedAt ?? run.requestedAt) ||
2833 projection.runtimeRequests.some((request) => request.status === "pending")
2834 ) {
2835 return yield* new OrchestratorDispatchError({
2836 commandId: command.commandId,
2837 commandType: command.type,
2838 cause: "The provider limit changed before recovery could be configured.",
2839 });
2840 }
2841 }
2842 let snoozedUntil: DateTime.Utc | null = null;
2843 if (command.type === "thread.snooze") {
2844 const projection = yield* loadProjectionForCommand(command, ["runs", "runtimeRequests"], {
2845 turnItemTypes: [],
2846 });
2847 const parsedSnoozedUntil = DateTime.make(command.snoozedUntil);
2848 if (
2849 Option.isNone(parsedSnoozedUntil) ||
2850 DateTime.toEpochMillis(parsedSnoozedUntil.value) <= DateTime.toEpochMillis(now)
2851 ) {
2852 return yield* new OrchestratorDispatchError({
2853 commandId: command.commandId,
2854 commandType: command.type,
2855 cause: `Thread ${command.threadId} snooze wake time ${command.snoozedUntil} is not in the future.`,
2856 });
2857 }
2858 if (projection.runtimeRequests.some((request) => request.status === "pending")) {
2859 return yield* new OrchestratorDispatchError({
2860 commandId: command.commandId,
2861 commandType: command.type,
2862 cause: `Thread ${command.threadId} has a pending approval or user-input request and cannot be snoozed.`,
2863 });
2864 }
2865 const queuedRuns = projection.runs.filter((run) => run.status === "queued");
2866 const queuedMessages =
2867 queuedRuns.length === 0
2868 ? []
2869 : (yield* loadProjectionForCommand(command, ["messages"], {
2870 messageIds: queuedRuns.map((run) => run.userMessageId),
2871 })).messages;
2872 // Automatic wakes are hidden from the user queue and must not block snooze.
2873 // Snooze only changes visibility, so keep those deliveries intact.
2874 const automaticMessageIds = new Set(
2875 queuedMessages
2876 .filter(
2877 (message) =>
2878 message.notification !== undefined || message.delegatedCompletion !== undefined,
2879 )
2880 .map((message) => message.id),
2881 );
2882 if (queuedRuns.some((run) => !automaticMessageIds.has(run.userMessageId))) {
2883 return yield* new OrchestratorDispatchError({
2884 commandId: command.commandId,
2885 commandType: command.type,
2886 cause: `Thread ${command.threadId} has a queued run and cannot be snoozed.`,
2887 });
2888 }
2889 snoozedUntil = parsedSnoozedUntil.value;
2890 }
2891 let markUnreadVisitedAt: DateTime.Utc | null = null;
2892 if (command.type === "thread.mark-unread") {
2893 const projection = yield* loadProjectionForCommand(command, ["runs"]);
2894 const latestRunCompletedAt = projection.runs.at(-1)?.completedAt ?? null;
2895 if (latestRunCompletedAt === null) {
2896 return yield* new OrchestratorDispatchError({
2897 commandId: command.commandId,
2898 commandType: command.type,
2899 cause: `Thread ${command.threadId} has no completed run to mark unread.`,
2900 });
2901 }
2902 markUnreadVisitedAt = DateTime.subtract(latestRunCompletedAt, { milliseconds: 1 });
2903 }
2904 const updatedThread: OrchestrationV2AppThread = (() => {
2905 switch (command.type) {
2906 case "thread.archive":
2907 // An archived thread takes no wakes, so its watches end like a settled thread's.
2908 return {
2909 ...thread,
2910 archivedAt: now,
2911 titleRegeneration: null,
2912 pullRequests: thread.pullRequests?.map((link) => withPullRequestWatch(link, undefined)),
2913 updatedAt: now,
2914 };
2915 case "thread.unarchive":
2916 return { ...thread, archivedAt: null, updatedAt: now };
2917 case "thread.settle": {
2918 // Settling is "I'm done with this": it clears a pin the same way it
2919 // parks the thread (mirrors the v1 decider's settle/pin exclusion).
2920 const wasPinned = thread.pinnedAt != null;
2921 const alreadySettled =
2922 thread.settledOverride === "settled" && thread.settledAt !== null && !wasPinned;
2923 return {
2924 ...thread,
2925 settledOverride: "settled",
2926 settledAt: alreadySettled ? thread.settledAt : (command.settledAt ?? now),
2927 pullRequests: thread.pullRequests?.map((link) => withPullRequestWatch(link, undefined)),
2928 unsettledAt: null,
2929 pinnedAt: null,
2930 pinOrderKey: null,
2931 activeOrderKey: null,
2932 updatedAt: alreadySettled ? thread.updatedAt : now,
2933 };
2934 }
2935 case "thread.unsettle": {
2936 const alreadyPinnedActive = thread.settledOverride === "active";
2937 return {
2938 ...thread,
2939 settledOverride: "active",
2940 settledAt: null,
2941 unsettledAt: alreadyPinnedActive ? (thread.unsettledAt ?? null) : now,
2942 updatedAt: alreadyPinnedActive ? thread.updatedAt : now,
2943 };
2944 }
2945 case "thread.snooze": {
2946 const sameWakeTime =
2947 thread.snoozedUntil != null &&
2948 snoozedUntil !== null &&
2949 DateTime.toEpochMillis(thread.snoozedUntil) === DateTime.toEpochMillis(snoozedUntil);
2950 const existingSnoozedAt = sameWakeTime ? (thread.snoozedAt ?? null) : null;
2951 return {
2952 ...thread,
2953 snoozedUntil,
2954 limitRecovery: thread.limitRecovery ? { ...thread.limitRecovery, snooze: false } : null,
2955 snoozedAt: existingSnoozedAt ?? now,
2956 updatedAt: existingSnoozedAt === null ? now : thread.updatedAt,
2957 };
2958 }
2959 case "thread.unsnooze": {
2960 const alreadyAwake = thread.snoozedUntil == null;
2961 return {
2962 ...thread,
2963 snoozedUntil: null,
2964 snoozedAt: null,
2965 lastSnoozeWakeAt: alreadyAwake ? thread.lastSnoozeWakeAt : now,
2966 updatedAt: alreadyAwake ? thread.updatedAt : now,
2967 };
2968 }
2969 case "thread.auto-settle.set": {
2970 const unchanged = command.enabled === (thread.autoSettleDisabledAt == null);
2971 return {
2972 ...thread,
2973 autoSettleDisabledAt: command.enabled ? null : (thread.autoSettleDisabledAt ?? now),
2974 updatedAt: unchanged ? thread.updatedAt : now,
2975 };
2976 }
2977 case "thread.pin": {
2978 // Pinning is a promotion: it clears the parked states rather than
2979 // silently outranking them — an explicit settle is un-settled and a
2980 // snooze's return ticket is spent (the thread is on top NOW).
2981 const alreadyPinned = thread.pinnedAt != null;
2982 const promotes = thread.settledOverride === "settled" || thread.snoozedUntil != null;
2983 return {
2984 ...thread,
2985 pinnedAt: alreadyPinned ? thread.pinnedAt : now,
2986 // A fresh pin takes the client's slot in the arranged order; on a
2987 // re-pin the existing key wins so raced duplicates cannot move a
2988 // thread the user already placed.
2989 ...(alreadyPinned || command.orderKey === undefined
2990 ? {}
2991 : { pinOrderKey: command.orderKey }),
2992 settledOverride:
2993 thread.settledOverride === "settled" ? "active" : thread.settledOverride,
2994 settledAt: thread.settledOverride === "settled" ? null : thread.settledAt,
2995 snoozedUntil: null,
2996 snoozedAt: null,
2997 lastSnoozeWakeAt: thread.snoozedUntil == null ? thread.lastSnoozeWakeAt : now,
2998 updatedAt: alreadyPinned && !promotes ? thread.updatedAt : now,
2999 };
3000 }
3001 case "thread.unpin": {
3002 const alreadyUnpinned = thread.pinnedAt == null;
3003 return {
3004 ...thread,
3005 pinnedAt: null,
3006 // Unpin clears the slot: re-pinning is "pin again", not "restore
3007 // an ancient position".
3008 pinOrderKey: null,
3009 updatedAt: alreadyUnpinned ? thread.updatedAt : now,
3010 };
3011 }
3012 case "thread.pin.reorder": {
3013 // Idempotent by re-emission (see thread.settle): a duplicate drop on
3014 // the same slot keeps the existing updatedAt so it projects as a
3015 // no-op.
3016 const keyUnchanged = thread.pinOrderKey === command.orderKey;
3017 return {
3018 ...thread,
3019 pinOrderKey: command.orderKey,
3020 updatedAt: keyUnchanged ? thread.updatedAt : now,
3021 };
3022 }
3023 case "thread.active.reorder": {
3024 return {
3025 ...thread,
3026 activeOrderKey: command.orderKey,
3027 // Arranging the active list is not thread activity.
3028 updatedAt: thread.updatedAt,
3029 };
3030 }
3031 case "thread.mark-unread":
3032 return { ...thread, lastVisitedAt: markUnreadVisitedAt };
3033 case "thread.metadata.update": {
3034 const previousRecovery =
3035 thread.limitRecovery?.runId === command.limitRecovery?.runId &&
3036 thread.limitRecovery?.resetAt === command.limitRecovery?.resetAt
3037 ? thread.limitRecovery
3038 : null;
3039 const limitRecovery =
3040 command.limitRecovery === undefined
3041 ? thread.limitRecovery
3042 : command.limitRecovery === null
3043 ? null
3044 : {
3045 ...command.limitRecovery,
3046 autoResume:
3047 command.limitRecovery.autoResume ?? previousRecovery?.autoResume ?? false,
3048 snooze: command.limitRecovery.snooze ?? previousRecovery?.snooze ?? false,
3049 requestId: command.commandId,
3050 };
3051 return {
3052 ...thread,
3053 ...(command.title === undefined ? {} : { title: command.title }),
3054 ...(command.limitRecovery === undefined ? {} : { limitRecovery }),
3055 ...(command.limitRecovery !== undefined &&
3056 limitRecovery?.snooze === true &&
3057 Date.parse(limitRecovery.resetAt) > DateTime.toEpochMillis(now)
3058 ? {
3059 snoozedUntil: DateTime.makeUnsafe(limitRecovery.resetAt),
3060 // Recovery changes acknowledge the same stopped run; keep its
3061 // metadata timestamp from appearing as a fresh failure wake.
3062 snoozedAt: now,
3063 }
3064 : command.limitRecovery !== undefined &&
3065 thread.limitRecovery?.snooze &&
3066 thread.snoozedUntil != null &&
3067 DateTime.toEpochMillis(thread.snoozedUntil) ===
3068 Date.parse(thread.limitRecovery.resetAt)
3069 ? { snoozedUntil: null, snoozedAt: null, lastSnoozeWakeAt: now }
3070 : {}),
3071 ...(command.branch === undefined ? {} : { branch: command.branch }),
3072 ...(command.worktreePath === undefined ? {} : { worktreePath: command.worktreePath }),
3073 ...(command.linkedPullRequest === undefined
3074 ? {}
3075 : {
3076 linkedPullRequest: command.linkedPullRequest,
3077 pullRequests: [
3078 ...threadPullRequestsOf(thread).filter(
3079 (link) =>
3080 !(
3081 thread.linkedPullRequest &&
3082 threadPullRequestKeysEqual(
3083 link,
3084 legacyThreadPullRequestKey(thread.linkedPullRequest),
3085 )
3086 ) &&
3087 !(
3088 command.linkedPullRequest &&
3089 threadPullRequestKeysEqual(
3090 link,
3091 legacyThreadPullRequestKey(command.linkedPullRequest),
3092 )
3093 ),
3094 ),
3095 ...(command.linkedPullRequest
3096 ? [legacyPullRequestLink(thread, command.linkedPullRequest, now)]
3097 : []),
3098 ],
3099 }),
3100 // regenerateTitle: true arms the in-flight marker; a landing title
3101 // or an explicit false (generation failed/abandoned) clears it.
3102 ...(command.regenerateTitle === true
3103 ? { titleRegeneration: { requestId: command.commandId, startedAt: now } }
3104 : command.regenerateTitle === false || command.title !== undefined
3105 ? { titleRegeneration: null }
3106 : {}),
3107 updatedAt: now,
3108 };
3109 }
3110 case "thread.pull-request.link":
3111 case "thread.pull-request.unlink":
3112 case "thread.pull-request-link.sync": {
3113 const key = normalizeThreadPullRequestKey(command);
3114 const links = threadPullRequestsOf(thread);
3115 const existing = links.find((link) => threadPullRequestKeysEqual(link, key));
3116 let pullRequests = links;
3117 if (command.type === "thread.pull-request.link") {
3118 const undismisses =
3119 existing?.source === "stack-dismissed" &&
3120 command.source !== "stack" &&
3121 command.source !== "stack-dismissed";
3122 if (existing && !undismisses) return thread;
3123 const link = existing
3124 ? { ...existing, source: command.source, url: command.url }
3125 : {
3126 ...key,
3127 url: command.url,
3128 source: command.source,
3129 linkedAt: DateTime.formatIso(now),
3130 snapshot: null,
3131 stack: null,
3132 };
3133 const branch = thread.branchPullRequest;
3134 const branchKey = branch ? legacyThreadPullRequestKey(branch) : null;
3135 pullRequests = [
3136 ...(command.source === "manual" &&
3137 branch &&
3138 branchKey &&
3139 visibleThreadPullRequests(links).length === 0 &&
3140 !threadPullRequestKeysEqual(branchKey, key) &&
3141 !links.some((entry) => threadPullRequestKeysEqual(entry, branchKey))
3142 ? [
3143 {
3144 ...branchKey,
3145 url: branch.url,
3146 source: "manual" as const,
3147 linkedAt: DateTime.formatIso(now),
3148 snapshot: null,
3149 stack: null,
3150 },
3151 ]
3152 : []),
3153 ...links.filter((entry) => !threadPullRequestKeysEqual(entry, key)),
3154 link,
3155 ];
3156 } else if (command.type === "thread.pull-request.unlink") {
3157 if (!existing) return thread;
3158 const belongsToStack =
3159 existing.source === "stack" ||
3160 existing.stack !== null ||
3161 links.some(
3162 (link) =>
3163 link.host.toLowerCase() === key.host &&
3164 link.repository.toLowerCase() === key.repository &&
3165 link.stack?.layers.some((layer) => layer.number === key.number),
3166 );
3167 pullRequests = belongsToStack
3168 ? links.map((link) =>
3169 link === existing
3170 ? {
3171 ...withPullRequestWatch(link, undefined),
3172 source: "stack-dismissed" as const,
3173 }
3174 : link,
3175 )
3176 : links.filter((link) => link !== existing);
3177 } else {
3178 if (!existing) return thread;
3179 pullRequests = links.map((link) =>
3180 link === existing
3181 ? { ...link, snapshot: command.snapshot, stack: command.stack }
3182 : link,
3183 );
3184 }
3185 return {
3186 ...thread,
3187 pullRequests,
3188 linkedPullRequest:
3189 thread.linkedPullRequest &&
3190 pullRequests.some(
3191 (link) =>
3192 link.source !== "stack-dismissed" &&
3193 threadPullRequestKeysEqual(
3194 link,
3195 legacyThreadPullRequestKey(thread.linkedPullRequest!),
3196 ),
3197 )
3198 ? thread.linkedPullRequest
3199 : null,
3200 updatedAt: command.type === "thread.pull-request-link.sync" ? thread.updatedAt : now,
3201 };
3202 }
3203 case "thread.pull-request.watch":
3204 case "thread.pull-request-watch.sync": {
3205 const key = normalizeThreadPullRequestKey(command);
3206 const startedAt = DateTime.formatIso(now);
3207 const linked = threadPullRequestsOf(thread);
3208 const visible = (link: ThreadPullRequestLink) =>
3209 link.source !== "stack-dismissed" && threadPullRequestKeysEqual(link, key);
3210 // A watch started on an unlinked (or dismissed) pull request links it in the same step.
3211 const links =
3212 command.type === "thread.pull-request.watch" &&
3213 command.watching &&
3214 command.link !== undefined &&
3215 !linked.some(visible)
3216 ? [
3217 ...linked.filter((link) => !threadPullRequestKeysEqual(link, key)),
3218 {
3219 ...key,
3220 url: command.link.url,
3221 source: command.link.source,
3222 linkedAt: startedAt,
3223 snapshot: null,
3224 stack: null,
3225 },
3226 ]
3227 : linked;
3228 const existing = links.find(visible);
3229 if (existing === undefined) return thread;
3230 const watch =
3231 command.type === "thread.pull-request-watch.sync"
3232 ? // Progress read before a stop or restart must not bring the old watch back.
3233 existing.watch?.startedAt === command.startedAt
3234 ? (command.watch ?? undefined)
3235 : existing.watch
3236 : !command.watching
3237 ? undefined
3238 : (existing.watch ?? {
3239 startedAt,
3240 headSha: null,
3241 failedChecks: [],
3242 passed: false,
3243 passedChecks: [],
3244 remarksThrough: startedAt,
3245 remarkIds: [],
3246 conflicting: false,
3247 wakes: 0,
3248 });
3249 if (watch === existing.watch && links === linked) return thread;
3250 return {
3251 ...thread,
3252 pullRequests: links.map((link) =>
3253 link === existing ? withPullRequestWatch(link, watch) : link,
3254 ),
3255 // A user or agent starting or stopping a watch is activity; recorded progress is not.
3256 updatedAt: command.type === "thread.pull-request.watch" ? now : thread.updatedAt,
3257 };
3258 }
3259 case "thread.pull-request.sync":
3260 return {
3261 ...thread,
3262 branchPullRequest: command.branchPullRequest,
3263 ...(command.linkedPullRequest === undefined
3264 ? {}
3265 : {
3266 linkedPullRequest: command.linkedPullRequest,
3267 pullRequests: [
3268 ...threadPullRequestsOf(thread).filter(
3269 (link) =>
3270 !(
3271 thread.linkedPullRequest &&
3272 threadPullRequestKeysEqual(
3273 link,
3274 legacyThreadPullRequestKey(thread.linkedPullRequest),
3275 )
3276 ) &&
3277 !(
3278 command.linkedPullRequest &&
3279 threadPullRequestKeysEqual(
3280 link,
3281 legacyThreadPullRequestKey(command.linkedPullRequest),
3282 )
3283 ),
3284 ),
3285 ...(command.linkedPullRequest
3286 ? [legacyPullRequestLink(thread, command.linkedPullRequest, now)]
3287 : []),
3288 ],
3289 }),
3290 updatedAt: thread.updatedAt,
3291 };
3292 case "thread.title.regeneration.complete":
3293 return thread.titleRegeneration?.requestId === command.requestId
3294 ? {
3295 ...thread,
3296 ...(command.title === undefined ? {} : { title: command.title }),
3297 titleRegeneration: null,
3298 updatedAt: now,
3299 }
3300 : thread;
3301 case "thread.runtime-mode.set":
3302 return { ...thread, runtimeMode: command.runtimeMode, updatedAt: now };
3303 case "thread.interaction-mode.set":
3304 return { ...thread, interactionMode: command.interactionMode, updatedAt: now };
3305 case "thread.model-selection.set":
3306 case "provider.switch":
3307 return {
3308 ...thread,
3309 providerInstanceId: command.modelSelection.instanceId,
3310 modelSelection: command.modelSelection,
3311 updatedAt: now,
3312 };
3313 }
3314 })();
3315 if (
3316 command.type === "thread.pull-request-link.sync" ||
3317 command.type === "thread.pull-request-watch.sync"
3318 ) {
3319 if (updatedThread === thread) return;
3320 const link = threadPullRequestsOf(updatedThread).find((candidate) =>
3321 threadPullRequestKeysEqual(candidate, normalizeThreadPullRequestKey(command)),
3322 );
3323 if (link === undefined) return;
3324 yield* emit(
3325 events,
3326 command,
3327 )({
3328 type: "thread.pull-request-link-updated",
3329 threadId: command.threadId,
3330 providerInstanceId: updatedThread.providerInstanceId,
3331 occurredAt: now,
3332 payload: link,
3333 });
3334 return;
3335 }
3336 const eventType = (() => {
3337 switch (command.type) {
3338 case "thread.archive":
3339 return "thread.archived" as const;
3340 case "thread.unarchive":
3341 return "thread.unarchived" as const;
3342 case "thread.settle":
3343 return "thread.settled" as const;
3344 case "thread.unsettle":
3345 return "thread.unsettled" as const;
3346 case "thread.snooze":
3347 return "thread.snoozed" as const;
3348 case "thread.unsnooze":
3349 return "thread.unsnoozed" as const;
3350 case "thread.auto-settle.set":
3351 return "thread.auto-settle-set" as const;
3352 case "thread.pin":
3353 return "thread.pinned" as const;
3354 case "thread.unpin":
3355 return "thread.unpinned" as const;
3356 case "thread.pin.reorder":
3357 return "thread.pin-reordered" as const;
3358 case "thread.active.reorder":
3359 return "thread.active-reordered" as const;
3360 case "thread.mark-unread":
3361 return "thread.marked-unread" as const;
3362 case "thread.metadata.update":
3363 case "thread.title.regeneration.complete":
3364 return "thread.metadata-updated" as const;
3365 case "thread.pull-request.link":
3366 case "thread.pull-request.unlink":
3367 case "thread.pull-request.watch":
3368 case "thread.pull-request.sync":
3369 return "thread.pull-request-synced" as const;
3370 case "thread.runtime-mode.set":
3371 return "thread.runtime-mode-updated" as const;
3372 case "thread.interaction-mode.set":
3373 return "thread.interaction-mode-updated" as const;
3374 case "thread.model-selection.set":
3375 return thread.providerInstanceId === command.modelSelection.instanceId
3376 ? ("thread.model-selection-updated" as const)
3377 : ("thread.provider-switched" as const);
3378 case "provider.switch":
3379 return "thread.provider-switched" as const;
3380 }
3381 })();
3382 yield* emit(
3383 events,
3384 command,
3385 )({
3386 type: eventType,
3387 threadId: command.threadId,
3388 providerInstanceId: updatedThread.providerInstanceId,
3389 occurredAt: now,
3390 payload: updatedThread,
3391 });
3392
3393 if (command.type === "thread.metadata.update" && command.regenerateTitle === true) {
3394 yield* Ref.update(effects, (existing) => [
3395 ...existing,
3396 pendingThreadTitleGenerationEffect(command.commandId, command.threadId, {
3397 type: "regenerate",
3398 }),
3399 ]);
3400 }
3401
3402 if (command.type === "thread.archive") {
3403 const projection = yield* loadProjectionForCommand(
3404 command,
3405 ["runs", "attempts", "nodes", "subagents", "messages"],
3406 { turnItemTypes: [], messageRoles: ["user"] },
3407 );
3408 const emitEvent = emit(events, command);
3409 const activeRunIds = new Set(
3410 projection.runs.filter((run) => run.status === "queued").map((run) => run.id),
3411 );
3412 for (const run of projection.runs.filter((candidate) => activeRunIds.has(candidate.id))) {
3413 yield* emitEvent({
3414 type: "run.updated",
3415 threadId: command.threadId,
3416 runId: run.id,
3417 providerInstanceId: run.providerInstanceId,
3418 occurredAt: now,
3419 payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now },
3420 });
3421 }
3422 for (const attempt of projection.attempts.filter(
3423 (candidate) =>
3424 activeRunIds.has(candidate.runId) &&
3425 (candidate.status === "pending" || candidate.status === "running"),
3426 )) {
3427 const run = projection.runs.find((candidate) => candidate.id === attempt.runId)!;
3428 yield* emitEvent({
3429 type: "run-attempt.updated",
3430 threadId: command.threadId,
3431 runId: attempt.runId,
3432 nodeId: attempt.rootNodeId,
3433 providerInstanceId: run.providerInstanceId,
3434 occurredAt: now,
3435 payload: { ...attempt, status: "cancelled", completedAt: now },
3436 });
3437 }
3438 for (const node of projection.nodes.filter(
3439 (candidate) =>
3440 candidate.runId !== null &&
3441 activeRunIds.has(candidate.runId) &&
3442 ["pending", "running", "waiting"].includes(candidate.status),
3443 )) {
3444 const run = projection.runs.find((candidate) => candidate.id === node.runId)!;
3445 yield* emitEvent({
3446 type: "node.updated",
3447 threadId: command.threadId,
3448 runId: run.id,
3449 nodeId: node.id,
3450 providerInstanceId: run.providerInstanceId,
3451 occurredAt: now,
3452 payload: { ...node, status: "cancelled", completedAt: now },
3453 });
3454 }
3455 yield* disposeAllDelegatedCompletionCohorts({
3456 command,
3457 events,
3458 projection: yield* getProjectionWithPendingEvents(command.threadId, events),
3459 now,
3460 cancelQueuedDelivery: false,
3461 });
3462 }
3463
3464 // Settle joins archive here: both mean "done with this
3465 // thread", so a live provider session must not keep running background
3466 // work (PR monitors, dev servers, subagent fleets) after any of them
3467 // lands. The settle guard above already rejects active or blocked runs,
3468 // so for settle this only ever stops an idle session; commands are
3469 // decided serially against the projection, so a turn start that
3470 // re-engages the thread cannot race this detach.
3471 const detachSessionIds = new Set(
3472 command.type === "thread.archive" || command.type === "thread.settle"
3473 ? (providerContext?.providerSessions ?? []).map((session) => session.id)
3474 : command.type === "thread.metadata.update" &&
3475 command.worktreePath !== undefined &&
3476 command.worktreePath !== thread.worktreePath
3477 ? (providerContext?.providerSessions ?? []).map((session) => session.id)
3478 : command.type === "thread.runtime-mode.set"
3479 ? (providerContext?.providerSessions ?? [])
3480 .filter(
3481 (session) => !session.capabilities.sessions.supportsRuntimeModeSwitchInSession,
3482 )
3483 .map((session) => session.id)
3484 : (providerSwitchPlan?.releaseProviderSessionIds ?? []),
3485 );
3486 if (detachSessionIds.size > 0) {
3487 const liveSessions = (providerContext?.providerSessions ?? []).filter(
3488 (session) =>
3489 detachSessionIds.has(session.id) &&
3490 session.status !== "stopped" &&
3491 session.status !== "error",
3492 );
3493 yield* Effect.forEach(
3494 liveSessions,
3495 (session) =>
3496 Effect.gen(function* () {
3497 yield* emit(
3498 events,
3499 command,
3500 )({
3501 type: "provider-session.detached",
3502 threadId: command.threadId,
3503 driver: session.driver,
3504 providerInstanceId: session.providerInstanceId,
3505 occurredAt: now,
3506 payload: {
3507 providerSessionId: session.id,
3508 detachedAt: now,
3509 reason:
3510 command.type === "thread.archive"
3511 ? "Thread archived."
3512 : command.type === "thread.settle"
3513 ? "Thread settled."
3514 : command.type === "thread.metadata.update"
3515 ? "Workspace changed."
3516 : command.type === "thread.runtime-mode.set"
3517 ? "Runtime mode changed."
3518 : "Provider or model selection changed.",
3519 },
3520 });
3521 const pendingEffect = {
3522 id: `effect:${command.commandId}:provider-session.detach:${session.id}`,
3523 commandId: command.commandId,
3524 threadId: command.threadId,
3525 request: {
3526 type: "provider-session.detach",
3527 providerSessionId: session.id,
3528 detail:
3529 command.type === "thread.archive"
3530 ? "Thread archived."
3531 : command.type === "thread.settle"
3532 ? "Thread settled."
3533 : command.type === "thread.metadata.update"
3534 ? "Workspace changed."
3535 : command.type === "thread.runtime-mode.set"
3536 ? "Runtime mode changed."
3537 : "Provider or model selection changed.",
3538 // Terminal detaches revoke the thread's MCP credentials; other
3539 // detach reasons keep them so a re-attaching provider process
3540 // stays authorized.
3541 ...(command.type === "thread.archive" ? { revokeMcpCredential: true } : {}),
3542 },
3543 } satisfies PendingOrchestrationEffectV2;
3544 yield* Ref.update(effects, (existing) => [...existing, pendingEffect]);
3545 }),
3546 { concurrency: 1, discard: true },
3547 );
3548 }
3549
3550 if (command.type === "thread.archive") {
3551 yield* Ref.update(effects, (existing) => [
3552 ...existing,
3553 {
3554 id: `effect:${command.commandId}:terminal.cleanup`,
3555 commandId: command.commandId,
3556 threadId: command.threadId,
3557 request: { type: "terminal.cleanup" },
3558 } satisfies PendingOrchestrationEffectV2,
3559 ]);
3560 }
3561 });
3562
3563 const dispatchProviderSessionDetach = Effect.fn("orchestrationV2.dispatch.providerSessionDetach")(
3564 function* (
3565 command: Extract<OrchestrationV2Command, { readonly type: "provider-session.detach" }>,
3566 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
3567 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
3568 ) {
3569 const projection = yield* projectionStore
3570 .getThreadRecords(command.threadId, ["providerSessions"])
3571 .pipe(
3572 Effect.mapError(
3573 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
3574 ),
3575 );
3576 const session = projection.providerSessions.find(
3577 (candidate) => candidate.id === command.providerSessionId,
3578 );
3579 if (session === undefined) {
3580 return yield* new OrchestratorDispatchError({
3581 commandId: command.commandId,
3582 commandType: command.type,
3583 cause: `Provider session ${command.providerSessionId} does not belong to thread ${command.threadId}.`,
3584 });
3585 }
3586 const now = yield* DateTime.now;
3587 yield* emit(
3588 events,
3589 command,
3590 )({
3591 type: "provider-session.detached",
3592 threadId: command.threadId,
3593 driver: session.driver,
3594 providerInstanceId: session.providerInstanceId,
3595 occurredAt: now,
3596 payload: {
3597 providerSessionId: session.id,
3598 detachedAt: now,
3599 ...(command.reason === undefined ? {} : { reason: command.reason }),
3600 },
3601 });
3602 const pendingEffect = {
3603 id: `effect:${command.commandId}:provider-session.detach:${command.providerSessionId}`,
3604 commandId: command.commandId,
3605 threadId: command.threadId,
3606 request: {
3607 type: "provider-session.detach",
3608 providerSessionId: command.providerSessionId,
3609 ...(command.reason === undefined ? {} : { detail: command.reason }),
3610 },
3611 } satisfies PendingOrchestrationEffectV2;
3612 yield* Ref.update(effects, (existing) => [...existing, pendingEffect]);
3613 },
3614 );
3615
3616 /** Fails when a limited sender's command would touch a thread running above its limit. */
3617 const refuseAboveDispatchModeLimit = Effect.fn("orchestrationV2.dispatch.refuseAboveModeLimit")(
3618 function* (
3619 command: OrchestrationV2ServerCommand,
3620 threadId: ThreadId,
3621 modes: {
3622 readonly runtimeMode: RuntimeMode;
3623 readonly interactionMode: ProviderInteractionMode;
3624 },
3625 ) {
3626 const limit = yield* DispatchModeLimit;
3627 const exceeded = limit === undefined ? undefined : exceededDispatchModeLimit(limit, modes);
3628 if (limit === undefined || exceeded === undefined) return;
3629 const refusal = {
3630 threadId,
3631 mode: exceeded,
3632 runtimeMode: modes.runtimeMode,
3633 interactionMode: modes.interactionMode,
3634 };
3635 if (limit.refused !== undefined) yield* Ref.set(limit.refused, refusal);
3636 return yield* new OrchestratorThreadAboveModeLimitError({
3637 commandId: command.commandId,
3638 ...refusal,
3639 });
3640 },
3641 );
3642
3643 const dispatchThreadFork = Effect.fn("orchestrationV2.dispatch.threadFork")(function* (
3644 command: Extract<OrchestrationV2Command, { readonly type: "thread.fork" }>,
3645 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
3646 ) {
3647 yield* Effect.annotateCurrentSpan({
3648 "orchestration_v2.command_id": command.commandId,
3649 "orchestration_v2.command_type": command.type,
3650 "orchestration_v2.source_thread_id": command.sourceThreadId,
3651 "orchestration_v2.target_thread_id": command.targetThreadId,
3652 "orchestration_v2.source_point_type": command.sourcePoint.type,
3653 });
3654
3655 const sourceProjection = yield* projectionStore
3656 .getThreadRecords(command.sourceThreadId, [
3657 "runs",
3658 "checkpoints",
3659 "providerThreads",
3660 "providerTurns",
3661 "attempts",
3662 "contextTransfers",
3663 ])
3664 .pipe(
3665 Effect.mapError(
3666 (cause) =>
3667 new OrchestratorProjectionError({
3668 threadId: command.sourceThreadId,
3669 cause,
3670 }),
3671 ),
3672 );
3673 // The source is not under this command's lock, so check the modes this
3674 // command copies, not an earlier read the source's user could outrun.
3675 yield* refuseAboveDispatchModeLimit(command, command.sourceThreadId, sourceProjection.thread);
3676
3677 const sourceRun = runForSourcePoint(sourceProjection, command.sourcePoint);
3678
3679 if (sourceRun === null) {
3680 return yield* new OrchestratorDispatchError({
3681 commandId: command.commandId,
3682 commandType: command.type,
3683 cause: `No stable source run was found for fork source ${command.sourcePoint.type}.`,
3684 });
3685 }
3686 if (!isForkableSourceRunStatus(sourceRun.status)) {
3687 return yield* new OrchestratorDispatchError({
3688 commandId: command.commandId,
3689 commandType: command.type,
3690 cause: forkableSourceRunStatusError(sourceRun),
3691 });
3692 }
3693 const sourceProviderThread = providerThreadForRun(sourceProjection, sourceRun);
3694 const now = command.createdAt ?? (yield* DateTime.now);
3695 const emitEvent = emit(events, command);
3696 const transferId = yield* mapDispatchError(command)(
3697 idAllocator.allocate.contextTransfer({
3698 sourceThreadId: sourceProjection.thread.id,
3699 targetThreadId: command.targetThreadId,
3700 type: "fork",
3701 }),
3702 );
3703 const { targetThread, transfer } = yield* threadForkService
3704 .plan({
3705 sourceProjection,
3706 sourceRun,
3707 sourceProviderThread,
3708 canonicalSourcePoint: contextSourcePointForRun(sourceProjection, sourceRun),
3709 transferId,
3710 targetThreadId: command.targetThreadId,
3711 ...(command.title === undefined ? {} : { title: command.title }),
3712 createdBy: command.createdBy,
3713 creationSource: command.creationSource,
3714 createdAt: now,
3715 })
3716 .pipe(mapDispatchError(command));
3717
3718 yield* emitEvent({
3719 type: "thread.created",
3720 threadId: command.targetThreadId,
3721 providerInstanceId: targetThread.providerInstanceId,
3722 occurredAt: now,
3723 payload: targetThread,
3724 });
3725 yield* emitEvent({
3726 type: "context-transfer.created",
3727 threadId: command.targetThreadId,
3728 providerInstanceId: sourceRun.providerInstanceId,
3729 occurredAt: now,
3730 payload: transfer,
3731 });
3732 });
3733
3734 const dispatchThreadMergeBack = Effect.fn("orchestrationV2.dispatch.threadMergeBack")(function* (
3735 command: Extract<OrchestrationV2Command, { readonly type: "thread.merge_back" }>,
3736 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
3737 ) {
3738 yield* Effect.annotateCurrentSpan({
3739 "orchestration_v2.command_id": command.commandId,
3740 "orchestration_v2.command_type": command.type,
3741 "orchestration_v2.source_thread_id": command.sourceThreadId,
3742 "orchestration_v2.target_thread_id": command.targetThreadId,
3743 "orchestration_v2.source_point_type": command.sourcePoint.type,
3744 });
3745
3746 const sourceProjection = yield* projectionStore
3747 .getThreadRecords(command.sourceThreadId, [
3748 "runs",
3749 "checkpoints",
3750 "providerThreads",
3751 "providerTurns",
3752 "attempts",
3753 "contextTransfers",
3754 ])
3755 .pipe(
3756 Effect.mapError(
3757 (cause) =>
3758 new OrchestratorProjectionError({
3759 threadId: command.sourceThreadId,
3760 cause,
3761 }),
3762 ),
3763 );
3764 // The source is not under this command's lock, so check the modes this
3765 // command copies, not an earlier read the source's user could outrun.
3766 yield* refuseAboveDispatchModeLimit(command, command.sourceThreadId, sourceProjection.thread);
3767 const targetProjection = yield* projectionStore
3768 .getThreadRecords(command.targetThreadId, ["contextTransfers"])
3769 .pipe(
3770 Effect.mapError(
3771 (cause) =>
3772 new OrchestratorProjectionError({
3773 threadId: command.targetThreadId,
3774 cause,
3775 }),
3776 ),
3777 );
3778
3779 if (
3780 sourceProjection.thread.lineage.relationshipToParent !== "fork" ||
3781 sourceProjection.thread.lineage.parentThreadId !== command.targetThreadId
3782 ) {
3783 return yield* new OrchestratorDispatchError({
3784 commandId: command.commandId,
3785 commandType: command.type,
3786 cause: `Thread ${command.sourceThreadId} is not a fork of ${command.targetThreadId}.`,
3787 });
3788 }
3789
3790 const sourceRun = runForSourcePoint(sourceProjection, command.sourcePoint);
3791 if (sourceRun === null) {
3792 return yield* new OrchestratorDispatchError({
3793 commandId: command.commandId,
3794 commandType: command.type,
3795 cause: `No stable source run was found for merge-back source ${command.sourcePoint.type}.`,
3796 });
3797 }
3798 if (sourceRun.status !== "completed" && sourceRun.status !== "waiting") {
3799 return yield* new OrchestratorDispatchError({
3800 commandId: command.commandId,
3801 commandType: command.type,
3802 cause: `Merge-back source run ${sourceRun.id} is ${sourceRun.status}; only provider-finished runs are supported.`,
3803 });
3804 }
3805
3806 const forkTransfer = sourceProjection.contextTransfers.findLast(
3807 (transfer) =>
3808 transfer.type === "fork" &&
3809 transfer.sourceThreadId === command.targetThreadId &&
3810 transfer.targetThreadId === command.sourceThreadId,
3811 );
3812 if (forkTransfer === undefined) {
3813 return yield* new OrchestratorDispatchError({
3814 commandId: command.commandId,
3815 commandType: command.type,
3816 cause: `No fork transfer exists between ${command.targetThreadId} and ${command.sourceThreadId}.`,
3817 });
3818 }
3819
3820 const sourceProviderThread = providerThreadForRun(sourceProjection, sourceRun);
3821 const now = command.createdAt ?? (yield* DateTime.now);
3822 const emitEvent = emit(events, command);
3823 const transferId = yield* mapDispatchError(command)(
3824 idAllocator.allocate.contextTransfer({
3825 sourceThreadId: command.sourceThreadId,
3826 targetThreadId: command.targetThreadId,
3827 type: "merge_back",
3828 }),
3829 );
3830 const pendingMergeBackTransfersForPair = targetProjection.contextTransfers.filter(
3831 (transfer) =>
3832 transfer.type === "merge_back" &&
3833 transfer.status === "pending" &&
3834 transfer.sourceThreadId === command.sourceThreadId &&
3835 transfer.targetThreadId === command.targetThreadId,
3836 );
3837 const transfer: OrchestrationV2ContextTransfer = {
3838 id: transferId,
3839 type: "merge_back",
3840 sourceThreadId: command.sourceThreadId,
3841 targetThreadId: command.targetThreadId,
3842 sourcePoint: contextSourcePointForRun(sourceProjection, sourceRun),
3843 basePoint: forkTransfer.sourcePoint,
3844 sourceProviderInstanceId: sourceRun.providerInstanceId,
3845 targetProviderInstanceId: targetProjection.thread.modelSelection.instanceId,
3846 targetRunId: null,
3847 status: "pending",
3848 resolution: null,
3849 createdBy: command.createdBy,
3850 error:
3851 sourceProviderThread === undefined ? "Source merge-back run has no provider thread." : null,
3852 createdAt: now,
3853 updatedAt: now,
3854 consumedAt: null,
3855 };
3856
3857 for (const pendingTransfer of pendingMergeBackTransfersForPair) {
3858 yield* emitEvent({
3859 type: "context-transfer.updated",
3860 threadId: command.targetThreadId,
3861 providerInstanceId: sourceRun.providerInstanceId,
3862 occurredAt: now,
3863 payload: {
3864 ...pendingTransfer,
3865 status: "superseded",
3866 error: `Superseded by merge-back transfer ${transferId}.`,
3867 updatedAt: now,
3868 },
3869 });
3870 }
3871 yield* emitEvent({
3872 type: "context-transfer.created",
3873 threadId: command.targetThreadId,
3874 providerInstanceId: sourceRun.providerInstanceId,
3875 occurredAt: now,
3876 payload: transfer,
3877 });
3878 });
3879
3880 const isRunCompacting = (
3881 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs" | "messages">,
3882 targetRunId: RunId,
3883 ) =>
3884 Effect.gen(function* () {
3885 const run = projection.runs.find((candidate) => candidate.id === targetRunId);
3886 if (run === undefined || !isBlockingRun(run)) return false;
3887
3888 const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
3889 if (
3890 message !== undefined &&
3891 message.attachments.length === 0 &&
3892 message.text.trim().toLowerCase() === "/compact"
3893 ) {
3894 return true;
3895 }
3896
3897 const records = yield* projectionStore
3898 .getThreadRecords(projection.thread.id, ["turnItems"], {
3899 turnItemTypes: ["compaction"],
3900 turnItemRunId: run.id,
3901 })
3902 .pipe(
3903 Effect.mapError(
3904 (cause) => new OrchestratorProjectionError({ threadId: projection.thread.id, cause }),
3905 ),
3906 );
3907 return records.turnItems.some(
3908 (item) =>
3909 item.nodeId === run.rootNodeId &&
3910 (item.status === "pending" || item.status === "running"),
3911 );
3912 });
3913
3914 const dispatchSteerIntoRun = (input: {
3915 readonly command: Extract<
3916 OrchestrationV2Command,
3917 { readonly type: "message.dispatch" | "queued-message.promote-to-steer" }
3918 >;
3919 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
3920 readonly effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>;
3921 readonly projection: Pick<
3922 OrchestrationV2ThreadProjection,
3923 | "runs"
3924 | "messages"
3925 | "providerSessions"
3926 | "providerThreads"
3927 | "providerTurns"
3928 | "subagents"
3929 | "attempts"
3930 | "nodes"
3931 | "thread"
3932 | "turnItems"
3933 >;
3934 readonly modelSelection: ModelSelection;
3935 readonly targetRunId: OrchestrationV2Run["id"];
3936 readonly messageId: OrchestrationV2ConversationMessage["id"];
3937 readonly text: string;
3938 readonly attachments: ReadonlyArray<ChatAttachment>;
3939 readonly context?: import("@t3tools/contracts").OrchestrationMessageContext | undefined;
3940 readonly createdBy: OrchestrationV2ConversationMessage["createdBy"];
3941 readonly creationSource: OrchestrationV2ConversationMessage["creationSource"];
3942 readonly scheduledTaskId?: OrchestrationV2ConversationMessage["scheduledTaskId"];
3943 readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"];
3944 readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"];
3945 readonly forceRestart: boolean;
3946 }) =>
3947 Effect.gen(function* () {
3948 const targetRun = input.projection.runs.find(
3949 (candidate) => candidate.id === input.targetRunId,
3950 );
3951 if (targetRun === undefined) {
3952 return yield* new OrchestratorDispatchError({
3953 commandId: input.command.commandId,
3954 commandType: input.command.type,
3955 cause: `Target run ${input.targetRunId} was not found.`,
3956 });
3957 }
3958 if (isNativeMaintenanceCommand(input)) {
3959 return yield* new OrchestratorDispatchError({
3960 commandId: input.command.commandId,
3961 commandType: input.command.type,
3962 cause:
3963 input.text.trim().toLowerCase() === "/compact"
3964 ? "Context compaction must run as a separate turn. Queue it or wait for the active turn to finish."
3965 : "Signing out must run as a separate turn. Queue it or wait for the active turn to finish.",
3966 });
3967 }
3968 if (isGoalCommand(input)) {
3969 return yield* new OrchestratorDispatchError({
3970 commandId: input.command.commandId,
3971 commandType: input.command.type,
3972 cause:
3973 "Goal commands must run as a separate turn. Queue it or wait for the active turn to finish.",
3974 });
3975 }
3976 const targetMessage = input.projection.messages.find(
3977 (message) => message.id === targetRun.userMessageId,
3978 );
3979 if (
3980 targetMessage !== undefined &&
3981 targetMessage.attachments.length === 0 &&
3982 targetMessage.text.trim().toLowerCase() === "/logout"
3983 ) {
3984 return yield* new OrchestratorDispatchError({
3985 commandId: input.command.commandId,
3986 commandType: input.command.type,
3987 cause: "Wait for sign-out to finish before steering the thread.",
3988 });
3989 }
3990 const rootNodeId = targetRun.rootNodeId;
3991 if (rootNodeId === null) {
3992 return yield* new OrchestratorDispatchError({
3993 commandId: input.command.commandId,
3994 commandType: input.command.type,
3995 cause: `Target run ${targetRun.id} has no root node.`,
3996 });
3997 }
3998 if (targetRun.status !== "running") {
3999 return yield* new OrchestratorDispatchError({
4000 commandId: input.command.commandId,
4001 commandType: input.command.type,
4002 cause: `Target run ${targetRun.id} is ${targetRun.status} and cannot be steered.`,
4003 });
4004 }
4005 const providerThread = input.projection.providerThreads.find(
4006 (candidate) => candidate.id === targetRun.providerThreadId,
4007 );
4008 if (providerThread === undefined || providerThread.providerSessionId === null) {
4009 return yield* new OrchestratorDispatchError({
4010 commandId: input.command.commandId,
4011 commandType: input.command.type,
4012 cause: `Provider thread ${targetRun.providerThreadId} has no active provider session for steering.`,
4013 });
4014 }
4015 const providerSessionId = providerThread.providerSessionId;
4016 const providerTurn = input.projection.providerTurns.find(
4017 (candidate) =>
4018 candidate.runAttemptId === targetRun.activeAttemptId && candidate.status === "running",
4019 );
4020 if (providerTurn === undefined) {
4021 return yield* new OrchestratorDispatchError({
4022 commandId: input.command.commandId,
4023 commandType: input.command.type,
4024 cause: `No running provider turn found for active run ${targetRun.id}.`,
4025 });
4026 }
4027 const sessionOption = yield* providerSessions.get(providerSessionId).pipe(
4028 Effect.mapError(
4029 (cause) =>
4030 new OrchestratorDispatchError({
4031 commandId: input.command.commandId,
4032 commandType: input.command.type,
4033 cause,
4034 }),
4035 ),
4036 );
4037 if (Option.isNone(sessionOption)) {
4038 return yield* new OrchestratorDispatchError({
4039 commandId: input.command.commandId,
4040 commandType: input.command.type,
4041 cause: `Provider session ${providerThread.providerSessionId} is not active.`,
4042 });
4043 }
4044
4045 const session = sessionOption.value;
4046 const now = yield* DateTime.now;
4047 const emitEvent = emit(input.events, input.command);
4048 const selectionChanged = !modelSelectionsEqual(
4049 targetRun.modelSelection,
4050 input.modelSelection,
4051 );
4052 const providerInstanceChanged =
4053 targetRun.providerInstanceId !== input.modelSelection.instanceId;
4054 const selectionTransition =
4055 selectionChanged && !providerInstanceChanged
4056 ? yield* providerAdapters.get(targetRun.providerInstanceId).pipe(
4057 Effect.flatMap((adapter) =>
4058 adapter.planSelectionTransition({
4059 current: targetRun.modelSelection,
4060 target: input.modelSelection,
4061 sessionCapabilities: session.providerSession.capabilities,
4062 }),
4063 ),
4064 Effect.mapError(
4065 (cause) =>
4066 new OrchestratorProviderAdapterError({
4067 commandId: input.command.commandId,
4068 providerInstanceId: targetRun.providerInstanceId,
4069 cause,
4070 }),
4071 ),
4072 )
4073 : null;
4074 if (selectionTransition?.type === "reject") {
4075 return yield* new OrchestratorDispatchError({
4076 commandId: input.command.commandId,
4077 commandType: input.command.type,
4078 cause: selectionTransition.reason,
4079 });
4080 }
4081 const appendSteeringMessage = (messageInput: {
4082 readonly runId: OrchestrationV2Run["id"];
4083 readonly nodeId: OrchestrationV2ExecutionNode["id"];
4084 readonly providerTurnId: typeof providerTurn.id | null;
4085 readonly providerThreadId: OrchestrationV2ProviderThread["id"];
4086 readonly providerInstanceId: ProviderInstanceId;
4087 }) =>
4088 Effect.gen(function* () {
4089 const message: OrchestrationV2ConversationMessage = {
4090 createdBy: input.createdBy,
4091 creationSource: input.creationSource,
4092 ...(input.delegatedCompletion === undefined
4093 ? {}
4094 : { delegatedCompletion: input.delegatedCompletion }),
4095 ...(input.scheduledTaskId === undefined
4096 ? {}
4097 : { scheduledTaskId: input.scheduledTaskId }),
4098 ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }),
4099 id: input.messageId,
4100 threadId: input.command.threadId,
4101 runId: messageInput.runId,
4102 nodeId: messageInput.nodeId,
4103 role: "user",
4104 text: input.text,
4105 attachments: input.attachments,
4106 ...(input.context ? { context: input.context } : {}),
4107 streaming: false,
4108 createdAt: now,
4109 updatedAt: now,
4110 };
4111 const turnItem: OrchestrationV2TurnItem = {
4112 createdBy: input.createdBy,
4113 creationSource: input.creationSource,
4114 ...(input.scheduledTaskId === undefined
4115 ? {}
4116 : { scheduledTaskId: input.scheduledTaskId }),
4117 ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }),
4118 id: idAllocator.derive.userTurnItem({ messageId: input.messageId }),
4119 threadId: input.command.threadId,
4120 runId: messageInput.runId,
4121 nodeId: messageInput.nodeId,
4122 providerThreadId: messageInput.providerThreadId,
4123 providerTurnId: messageInput.providerTurnId,
4124 nativeItemRef: null,
4125 parentItemId: null,
4126 ordinal: yield* nextTurnItemOrdinal(input.projection),
4127 status: "completed",
4128 title: null,
4129 startedAt: now,
4130 completedAt: now,
4131 updatedAt: now,
4132 type: "user_message",
4133 messageId: input.messageId,
4134 inputIntent:
4135 input.command.type === "queued-message.promote-to-steer"
4136 ? "promoted_queued_to_steer"
4137 : "steer",
4138 text: input.text,
4139 attachments: input.attachments,
4140 ...(input.context ? { context: input.context } : {}),
4141 };
4142 yield* emitEvent({
4143 type: "message.updated",
4144 threadId: input.command.threadId,
4145 runId: messageInput.runId,
4146 nodeId: messageInput.nodeId,
4147 providerInstanceId: messageInput.providerInstanceId,
4148 occurredAt: now,
4149 payload: message,
4150 });
4151 yield* emitEvent({
4152 type: "turn-item.updated",
4153 threadId: input.command.threadId,
4154 runId: messageInput.runId,
4155 nodeId: messageInput.nodeId,
4156 providerInstanceId: messageInput.providerInstanceId,
4157 occurredAt: now,
4158 payload: notificationTurnItem(turnItem, message, input.projection.subagents),
4159 });
4160 });
4161
4162 // A selection the provider applies on its next turn restarts the run when
4163 // the provider can restart it. Otherwise the steer joins the running turn
4164 // and the selection waits for the next one, rather than failing the steer.
4165 const turnCapabilities = session.providerSession.capabilities.turns;
4166 const selectionMustApplyNow =
4167 selectionChanged &&
4168 (providerInstanceChanged || selectionTransition?.type !== "apply_on_next_turn");
4169 const steeringPolicy = yield* enforceCommandPolicy(input.command)(
4170 commandPolicy.decideSteeringExecution({
4171 commandId: input.command.commandId,
4172 threadId: input.command.threadId,
4173 providerInstanceId: targetRun.providerInstanceId,
4174 capabilities: session.providerSession.capabilities,
4175 forceRestart:
4176 input.forceRestart ||
4177 selectionMustApplyNow ||
4178 (selectionChanged &&
4179 turnCapabilities.supportsInterrupt &&
4180 turnCapabilities.supportsSteeringByInterruptRestart),
4181 }),
4182 );
4183
4184 if (steeringPolicy === "active_steering") {
4185 // The steer's selection becomes the saved next-turn choice, even when it
4186 // matches the running run again. A delegated completion carries the
4187 // run's selection, not a user choice, so it never replaces the saved one.
4188 // The saved choice may name another instance, so it moves with the steer.
4189 const instanceChanged =
4190 input.projection.thread.providerInstanceId !== input.modelSelection.instanceId;
4191 if (
4192 input.delegatedCompletion === undefined &&
4193 (instanceChanged ||
4194 !modelSelectionsEqual(input.projection.thread.modelSelection, input.modelSelection))
4195 ) {
4196 yield* emitEvent({
4197 type: instanceChanged ? "thread.provider-switched" : "thread.model-selection-updated",
4198 threadId: input.command.threadId,
4199 providerInstanceId: input.modelSelection.instanceId,
4200 occurredAt: now,
4201 payload: {
4202 ...input.projection.thread,
4203 providerInstanceId: input.modelSelection.instanceId,
4204 modelSelection: input.modelSelection,
4205 updatedAt: now,
4206 },
4207 });
4208 }
4209 yield* appendSteeringMessage({
4210 runId: targetRun.id,
4211 nodeId: rootNodeId,
4212 providerTurnId: providerTurn.id,
4213 providerThreadId: providerThread.id,
4214 providerInstanceId: targetRun.providerInstanceId,
4215 });
4216 yield* Ref.update(input.effects, (existing) => [
4217 ...existing,
4218 {
4219 id: `effect:${input.command.commandId}:provider-turn.steer:${providerTurn.id}`,
4220 commandId: input.command.commandId,
4221 threadId: input.command.threadId,
4222 request: {
4223 type: "provider-turn.steer",
4224 providerSessionId,
4225 providerThreadId: providerThread.id,
4226 providerTurnId: providerTurn.id,
4227 messageId: input.messageId,
4228 },
4229 } satisfies PendingOrchestrationEffectV2,
4230 ]);
4231 return;
4232 }
4233
4234 const currentAttempt = input.projection.attempts.find(
4235 (candidate) => candidate.id === targetRun.activeAttemptId,
4236 );
4237 const currentRootNode = input.projection.nodes.find(
4238 (candidate) => candidate.id === rootNodeId,
4239 );
4240 const attemptOrdinal =
4241 Math.max(
4242 0,
4243 ...input.projection.attempts
4244 .filter((candidate) => candidate.runId === targetRun.id)
4245 .map((candidate) => candidate.attemptOrdinal),
4246 ) + 1;
4247 const nextAttemptId = idAllocator.derive.runAttempt({
4248 runId: targetRun.id,
4249 attemptOrdinal,
4250 });
4251 const nextRootNodeId = idAllocator.derive.rootNodeAttempt({
4252 runId: targetRun.id,
4253 attemptOrdinal,
4254 });
4255 let restartProviderThread = providerThread;
4256 let restartSessionTransition:
4257 | {
4258 readonly type: "replace";
4259 readonly replacementProviderSessionId: ProviderSessionId;
4260 }
4261 | { readonly type: "detach" }
4262 | null = null;
4263 let restartHandoff: OrchestrationV2ContextHandoff | null = null;
4264 let restartTransfer: OrchestrationV2ContextTransfer | null = null;
4265 const canResumeAcrossInstances =
4266 providerInstanceChanged &&
4267 providerThread.nativeThreadRef !== null &&
4268 (yield* providerSwitchService
4269 .plan({
4270 projection: {
4271 ...input.projection,
4272 thread: { ...input.projection.thread, modelSelection: targetRun.modelSelection },
4273 },
4274 targetModelSelection: input.modelSelection,
4275 })
4276 .pipe(mapDispatchError(input.command))).transition.type === "restart_and_resume";
4277 const requiresProviderThreadHandoff =
4278 (providerInstanceChanged && !canResumeAcrossInstances) ||
4279 selectionTransition?.type === "create_with_handoff";
4280 const requiresProviderSessionRestart =
4281 canResumeAcrossInstances || selectionTransition?.type === "restart_session";
4282 if (requiresProviderThreadHandoff) {
4283 const targetAdapter = yield* providerAdapters.get(input.modelSelection.instanceId).pipe(
4284 Effect.mapError(
4285 (cause) =>
4286 new OrchestratorProviderAdapterError({
4287 commandId: input.command.commandId,
4288 providerInstanceId: input.modelSelection.instanceId,
4289 cause,
4290 }),
4291 ),
4292 );
4293 const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
4294 Effect.mapError(
4295 (cause) =>
4296 new OrchestratorProviderAdapterError({
4297 commandId: input.command.commandId,
4298 providerInstanceId: input.modelSelection.instanceId,
4299 cause,
4300 }),
4301 ),
4302 );
4303 yield* enforceCommandPolicy(input.command)(
4304 commandPolicy.ensureContextHandoff({
4305 commandId: input.command.commandId,
4306 threadId: input.command.threadId,
4307 providerInstanceId: input.modelSelection.instanceId,
4308 capabilities: targetCapabilities,
4309 strategy: "full_thread_summary",
4310 }),
4311 );
4312 const existingTargetProviderThread = rootProviderThreadsForProvider(
4313 input.projection,
4314 input.modelSelection.instanceId,
4315 ).find((candidate) => candidate.id !== providerThread.id);
4316 const targetProviderSessionId =
4317 existingTargetProviderThread?.providerSessionId ??
4318 (yield* mapDispatchError(input.command)(
4319 providerSessionIdFor({
4320 adapter: targetAdapter,
4321 providerInstanceId: input.modelSelection.instanceId,
4322 threadId: input.command.threadId,
4323 }),
4324 ));
4325 const targetProviderThreadBase: OrchestrationV2ProviderThread =
4326 existingTargetProviderThread === undefined
4327 ? {
4328 id: idAllocator.derive.providerThread({
4329 driver: targetAdapter.driver,
4330 nativeThreadId: `pending:${targetRun.id}:attempt:${attemptOrdinal}`,
4331 }),
4332 driver: targetAdapter.driver,
4333 providerInstanceId: input.modelSelection.instanceId,
4334 providerSessionId: targetProviderSessionId,
4335 appThreadId: input.command.threadId,
4336 ownerNodeId: null,
4337 nativeThreadRef: null,
4338 nativeConversationHeadRef: null,
4339 status: "not_loaded",
4340 firstRunOrdinal: targetRun.ordinal,
4341 lastRunOrdinal: targetRun.ordinal,
4342 handoffIds: [],
4343 forkedFrom: null,
4344 createdAt: now,
4345 updatedAt: now,
4346 }
4347 : {
4348 ...existingTargetProviderThread,
4349 providerSessionId: targetProviderSessionId,
4350 lastRunOrdinal: targetRun.ordinal,
4351 updatedAt: now,
4352 };
4353 const transferId = yield* mapDispatchError(input.command)(
4354 idAllocator.allocate.contextTransfer({
4355 sourceThreadId: input.command.threadId,
4356 targetThreadId: input.command.threadId,
4357 type: "provider_handoff",
4358 }),
4359 );
4360 restartHandoff = yield* contextHandoffService
4361 .prepareProviderHandoff({
4362 threadId: input.command.threadId,
4363 targetRunId: targetRun.id,
4364 transferId,
4365 fromProviderThreadIds: [providerThread.id],
4366 toProviderThreadId: targetProviderThreadBase.id,
4367 fromProviderInstanceId: targetRun.providerInstanceId,
4368 toProviderInstanceId: input.modelSelection.instanceId,
4369 coveredRunOrdinals: { from: 1, to: targetRun.ordinal },
4370 strategy: "full_thread_summary",
4371 items: yield* readHandoffItems(input.command.threadId),
4372 runs: input.projection.runs,
4373 createdAt: now,
4374 })
4375 .pipe(mapDispatchError(input.command));
4376 restartProviderThread = {
4377 ...targetProviderThreadBase,
4378 handoffIds: Array.from(
4379 new Set([...targetProviderThreadBase.handoffIds, restartHandoff.id]),
4380 ),
4381 };
4382 restartTransfer = {
4383 id: transferId,
4384 type: "provider_handoff",
4385 sourceThreadId: input.command.threadId,
4386 targetThreadId: input.command.threadId,
4387 sourcePoint: contextSourcePointForRun(input.projection, targetRun),
4388 basePoint: null,
4389 sourceProviderInstanceId: targetRun.providerInstanceId,
4390 targetProviderInstanceId: input.modelSelection.instanceId,
4391 targetRunId: targetRun.id,
4392 status: "consumed",
4393 resolution: {
4394 strategy: "portable_context",
4395 contextHandoffId: restartHandoff.id,
4396 },
4397 createdBy: input.createdBy,
4398 error: null,
4399 createdAt: now,
4400 updatedAt: now,
4401 consumedAt: now,
4402 };
4403 restartSessionTransition = { type: "detach" };
4404 } else if (requiresProviderSessionRestart) {
4405 const nextProviderSessionId = yield* mapDispatchError(input.command)(
4406 idAllocator.allocate.providerSession({
4407 providerInstanceId: input.modelSelection.instanceId,
4408 threadId: input.command.threadId,
4409 }),
4410 );
4411 restartProviderThread = {
4412 ...providerThread,
4413 providerInstanceId: input.modelSelection.instanceId,
4414 providerSessionId: nextProviderSessionId,
4415 status: "not_loaded",
4416 updatedAt: now,
4417 };
4418 restartSessionTransition = {
4419 type: "replace",
4420 replacementProviderSessionId: nextProviderSessionId,
4421 };
4422 }
4423 const resolvedRuntimePolicy = yield* runtimePolicy
4424 .resolve({ thread: input.projection.thread, modelSelection: input.modelSelection })
4425 .pipe(
4426 Effect.mapError(
4427 (cause) =>
4428 new OrchestratorDispatchError({
4429 commandId: input.command.commandId,
4430 commandType: input.command.type,
4431 cause,
4432 }),
4433 ),
4434 );
4435 const checkpointScope = yield* checkpointService
4436 .prepareRootRunScope({
4437 threadId: input.command.threadId,
4438 runId: targetRun.id,
4439 rootNodeId: nextRootNodeId,
4440 providerThreadId: restartProviderThread.id,
4441 cwd:
4442 resolvedRuntimePolicy.cwd ??
4443 input.projection.thread.worktreePath ??
4444 session.providerSession.cwd,
4445 createdAt: now,
4446 })
4447 .pipe(
4448 Effect.mapError(
4449 (cause) =>
4450 new OrchestratorDispatchError({
4451 commandId: input.command.commandId,
4452 commandType: input.command.type,
4453 cause,
4454 }),
4455 ),
4456 );
4457 const ensuredCheckpointScope = yield* checkpointService.ensureScope(checkpointScope).pipe(
4458 Effect.mapError(
4459 (cause) =>
4460 new OrchestratorDispatchError({
4461 commandId: input.command.commandId,
4462 commandType: input.command.type,
4463 cause,
4464 }),
4465 ),
4466 );
4467 const restartedRun: OrchestrationV2Run = {
4468 ...targetRun,
4469 providerInstanceId: input.modelSelection.instanceId,
4470 modelSelection: input.modelSelection,
4471 providerThreadId: restartProviderThread.id,
4472 rootNodeId: nextRootNodeId,
4473 activeAttemptId: nextAttemptId,
4474 userMessageId: input.messageId,
4475 status: "starting",
4476 contextHandoffId: restartHandoff?.id ?? targetRun.contextHandoffId,
4477 };
4478 const nextAttempt: OrchestrationV2RunAttempt = {
4479 id: nextAttemptId,
4480 runId: targetRun.id,
4481 attemptOrdinal,
4482 rootNodeId: nextRootNodeId,
4483 providerInstanceId: input.modelSelection.instanceId,
4484 providerThreadId: restartProviderThread.id,
4485 providerTurnId: null,
4486 reason: "steering_restart",
4487 status: "pending",
4488 startedAt: null,
4489 completedAt: null,
4490 };
4491 const nextRootNode: OrchestrationV2ExecutionNode = {
4492 id: nextRootNodeId,
4493 threadId: input.command.threadId,
4494 runId: targetRun.id,
4495 parentNodeId: null,
4496 rootNodeId: nextRootNodeId,
4497 kind: "root_turn",
4498 status: "pending",
4499 countsForRun: true,
4500 providerThreadId: restartProviderThread.id,
4501 providerTurnId: null,
4502 nativeItemRef: null,
4503 runtimeRequestId: null,
4504 checkpointScopeId: ensuredCheckpointScope.id,
4505 startedAt: null,
4506 completedAt: null,
4507 };
4508 if (currentAttempt !== undefined) {
4509 yield* emitEvent({
4510 type: "run-attempt.updated",
4511 threadId: input.command.threadId,
4512 runId: targetRun.id,
4513 nodeId: rootNodeId,
4514 providerInstanceId: targetRun.providerInstanceId,
4515 occurredAt: now,
4516 payload: { ...currentAttempt, status: "superseded", completedAt: now },
4517 });
4518 }
4519 if (currentRootNode !== undefined) {
4520 yield* emitEvent({
4521 type: "node.updated",
4522 threadId: input.command.threadId,
4523 runId: targetRun.id,
4524 nodeId: rootNodeId,
4525 providerInstanceId: targetRun.providerInstanceId,
4526 occurredAt: now,
4527 payload: { ...currentRootNode, status: "interrupted", completedAt: now },
4528 });
4529 }
4530 if (selectionChanged) {
4531 yield* emitEvent({
4532 type: providerInstanceChanged
4533 ? "thread.provider-switched"
4534 : "thread.model-selection-updated",
4535 threadId: input.command.threadId,
4536 providerInstanceId: input.modelSelection.instanceId,
4537 occurredAt: now,
4538 payload: {
4539 ...input.projection.thread,
4540 providerInstanceId: input.modelSelection.instanceId,
4541 modelSelection: input.modelSelection,
4542 updatedAt: now,
4543 },
4544 });
4545 }
4546 if (requiresProviderThreadHandoff || requiresProviderSessionRestart) {
4547 yield* emitEvent({
4548 type: "provider-thread.updated",
4549 threadId: input.command.threadId,
4550 driver: restartProviderThread.driver,
4551 providerInstanceId: input.modelSelection.instanceId,
4552 occurredAt: now,
4553 payload: restartProviderThread,
4554 });
4555 }
4556 if (restartHandoff !== null) {
4557 yield* emitEvent({
4558 type: "context-handoff.updated",
4559 threadId: input.command.threadId,
4560 runId: targetRun.id,
4561 providerInstanceId: input.modelSelection.instanceId,
4562 occurredAt: now,
4563 payload: restartHandoff,
4564 });
4565 }
4566 if (restartTransfer !== null) {
4567 yield* emitEvent({
4568 type: "context-transfer.created",
4569 threadId: input.command.threadId,
4570 runId: targetRun.id,
4571 providerInstanceId: input.modelSelection.instanceId,
4572 occurredAt: now,
4573 payload: restartTransfer,
4574 });
4575 }
4576 yield* emitEvent({
4577 type: "run.updated",
4578 threadId: input.command.threadId,
4579 runId: targetRun.id,
4580 nodeId: nextRootNodeId,
4581 providerInstanceId: input.modelSelection.instanceId,
4582 occurredAt: now,
4583 payload: restartedRun,
4584 });
4585 yield* emitEvent({
4586 type: "run-attempt.created",
4587 threadId: input.command.threadId,
4588 runId: targetRun.id,
4589 nodeId: nextRootNodeId,
4590 providerInstanceId: input.modelSelection.instanceId,
4591 occurredAt: now,
4592 payload: nextAttempt,
4593 });
4594 yield* emitEvent({
4595 type: "node.updated",
4596 threadId: input.command.threadId,
4597 runId: targetRun.id,
4598 nodeId: nextRootNodeId,
4599 providerInstanceId: input.modelSelection.instanceId,
4600 occurredAt: now,
4601 payload: nextRootNode,
4602 });
4603 yield* emitEvent({
4604 type: "checkpoint-scope.created",
4605 threadId: input.command.threadId,
4606 runId: targetRun.id,
4607 nodeId: nextRootNodeId,
4608 providerInstanceId: input.modelSelection.instanceId,
4609 occurredAt: now,
4610 payload: ensuredCheckpointScope,
4611 });
4612 yield* appendSteeringMessage({
4613 runId: targetRun.id,
4614 nodeId: nextRootNodeId,
4615 providerTurnId: null,
4616 providerThreadId: restartProviderThread.id,
4617 providerInstanceId: input.modelSelection.instanceId,
4618 });
4619 const interruptedAttemptId = targetRun.activeAttemptId;
4620 if (interruptedAttemptId === null) {
4621 return yield* new OrchestratorDispatchError({
4622 commandId: input.command.commandId,
4623 commandType: input.command.type,
4624 cause: `Active run ${targetRun.id} has no attempt to interrupt.`,
4625 });
4626 }
4627 yield* Ref.update(input.effects, (existing) => [
4628 ...existing,
4629 {
4630 id: `effect:${input.command.commandId}:provider-turn.restart:${providerTurn.id}`,
4631 commandId: input.command.commandId,
4632 threadId: input.command.threadId,
4633 request: {
4634 type: "provider-turn.restart",
4635 providerSessionId,
4636 providerThreadId: providerThread.id,
4637 providerTurnId: providerTurn.id,
4638 interruptedAttemptId,
4639 runId: targetRun.id,
4640 ...(restartSessionTransition === null
4641 ? {}
4642 : { sessionTransition: restartSessionTransition }),
4643 },
4644 } satisfies PendingOrchestrationEffectV2,
4645 ]);
4646 });
4647
4648 const dispatchMessage = (
4649 command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
4650 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4651 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
4652 ) =>
4653 Effect.gen(function* () {
4654 let projection = yield* getProjectionWithPendingEvents(command.threadId, events);
4655 if (command.manualContinuationOfRunId !== undefined) {
4656 const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId);
4657 const limited = latestRootProviderFailure(source ?? null, projection.turnItems);
4658 if (
4659 command.dispatchMode.type !== "start_immediately" ||
4660 source === undefined ||
4661 (source.status !== "interrupted" &&
4662 !(source.status === "failed" && limited?.class === "usage_limit")) ||
4663 latestExecutedRun(projection.runs)?.id !== source.id ||
4664 projection.thread.archivedAt !== null ||
4665 projection.thread.deletedAt !== null ||
4666 projection.runtimeRequests.some((request) => request.status === "pending")
4667 ) {
4668 return yield* new OrchestratorDispatchError({
4669 commandId: command.commandId,
4670 commandType: command.type,
4671 cause: "This thread can no longer be resumed from that run.",
4672 });
4673 }
4674 }
4675 if (command.usageLimitContinuationOfRunId !== undefined) {
4676 const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
4677 const failure = latestRootProviderFailure(run, projection.turnItems);
4678 const recovery = projection.thread.limitRecovery;
4679 const now = yield* DateTime.now;
4680 if (
4681 run?.id !== command.usageLimitContinuationOfRunId ||
4682 failure?.class !== "usage_limit" ||
4683 threadShellFromProjection(projection).lastErrorClass !== "usage_limit" ||
4684 !recovery?.autoResume ||
4685 recovery.requestId !== command.usageLimitRecoveryRequestId ||
4686 recovery.runId !== run.id ||
4687 recovery.resetAt !== failure.resetAt ||
4688 Date.parse(recovery.resetAt) > DateTime.toEpochMillis(now) ||
4689 projection.thread.archivedAt !== null ||
4690 projection.thread.deletedAt !== null ||
4691 projection.thread.settledOverride === "settled" ||
4692 projection.thread.providerInstanceId !== run.providerInstanceId ||
4693 projection.runtimeRequests.some((request) => request.status === "pending") ||
4694 (projection.thread.snoozedUntil != null &&
4695 DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now))
4696 ) {
4697 yield* emit(
4698 events,
4699 command,
4700 )({
4701 type: "thread.metadata-updated",
4702 threadId: command.threadId,
4703 occurredAt: now,
4704 payload: {
4705 ...projection.thread,
4706 ...(recovery?.autoResume &&
4707 recovery.requestId === command.usageLimitRecoveryRequestId &&
4708 recovery.runId === command.usageLimitContinuationOfRunId &&
4709 projection.thread.snoozedUntil != null &&
4710 DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now)
4711 ? { limitRecovery: { ...recovery, requestId: command.commandId } }
4712 : {}),
4713 },
4714 });
4715 return;
4716 }
4717 }
4718
4719 if (command.restartContinuationOfRunId !== undefined) {
4720 const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
4721 if (
4722 !source ||
4723 !isRestartContinuationSource(source, projection.providerTurns) ||
4724 projection.thread.archivedAt !== null ||
4725 projection.thread.deletedAt !== null ||
4726 projection.thread.providerInstanceId !== source.providerInstanceId ||
4727 // Held queued runs never started; they wait behind the continuation.
4728 projection.runs.some(
4729 (run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
4730 ) ||
4731 (yield* stopReachedRun(command, command.threadId, source.id))
4732 ) {
4733 // Preserve the current row so stale automatic deliveries receive an
4734 // accepted receipt without changing work or repeatedly retrying.
4735 yield* emit(
4736 events,
4737 command,
4738 )({
4739 type: "thread.metadata-updated",
4740 threadId: command.threadId,
4741 occurredAt: yield* DateTime.now,
4742 payload: projection.thread,
4743 });
4744 return;
4745 }
4746 }
4747
4748 const continuedTaskId = projection.runs.find(
4749 (run) => run.id === command.restartContinuationOfRunId,
4750 )?.delegatedTaskId;
4751
4752 if (projection.thread.settledOverride !== null) {
4753 const now = yield* DateTime.now;
4754 const thread: OrchestrationV2AppThread = {
4755 ...projection.thread,
4756 settledOverride: null,
4757 settledAt: null,
4758 unsettledAt:
4759 projection.thread.settledOverride === "active"
4760 ? (projection.thread.unsettledAt ?? null)
4761 : now,
4762 updatedAt: now,
4763 };
4764 yield* emit(
4765 events,
4766 command,
4767 )({
4768 type: "thread.unsettled",
4769 threadId: command.threadId,
4770 providerInstanceId: thread.providerInstanceId,
4771 occurredAt: now,
4772 payload: thread,
4773 });
4774 projection = yield* getProjectionWithPendingEvents(command.threadId, events);
4775 }
4776 if (projection.thread.snoozedUntil != null) {
4777 const now = yield* DateTime.now;
4778 const thread: OrchestrationV2AppThread = {
4779 ...projection.thread,
4780 snoozedUntil: null,
4781 snoozedAt: null,
4782 updatedAt: now,
4783 };
4784 yield* emit(
4785 events,
4786 command,
4787 )({
4788 type: "thread.unsnoozed",
4789 threadId: command.threadId,
4790 providerInstanceId: thread.providerInstanceId,
4791 occurredAt: now,
4792 payload: thread,
4793 });
4794 projection = yield* getProjectionWithPendingEvents(command.threadId, events);
4795 }
4796 const userMessages = projection.messages.filter((message) => message.role === "user");
4797 const onlyMaintenanceHistory =
4798 userMessages.length > 0 && userMessages.every(isNativeMaintenanceCommand);
4799 if (
4800 !isNativeMaintenanceCommand(command) &&
4801 ((command.titleSeed !== undefined &&
4802 (yield* projectionStore
4803 .getMessageCount(command.threadId)
4804 .pipe(mapDispatchError(command))) === 0) ||
4805 (command.createdBy === "user" && onlyMaintenanceHistory))
4806 ) {
4807 const now = yield* DateTime.now;
4808 const thread: OrchestrationV2AppThread = {
4809 ...projection.thread,
4810 title: command.titleSeed ?? projection.thread.title,
4811 titleRegeneration: { requestId: command.commandId, startedAt: now },
4812 updatedAt: now,
4813 };
4814 yield* emit(
4815 events,
4816 command,
4817 )({
4818 type: "thread.metadata-updated",
4819 threadId: command.threadId,
4820 providerInstanceId: thread.providerInstanceId,
4821 occurredAt: now,
4822 payload: thread,
4823 });
4824 yield* Ref.update(effects, (existing) => [
4825 ...existing,
4826 pendingThreadTitleGenerationEffect(command.commandId, command.threadId, {
4827 type: "initial",
4828 messageId: command.messageId,
4829 }),
4830 ]);
4831 projection = yield* getProjectionWithPendingEvents(command.threadId, events);
4832 }
4833 const modelSelection = command.modelSelection ?? projection.thread.modelSelection;
4834 let dispatchMode = resolveMessageDispatchIntent(
4835 projection,
4836 command.dispatchMode,
4837 command.deliveryIntent,
4838 );
4839 if (dispatchMode.type === "steer_active") {
4840 const targetRunId = dispatchMode.targetRunId;
4841 const target = projection.runs.find((run) => run.id === targetRunId);
4842 const turn = latestProviderTurnForAttempt(
4843 projection.providerTurns.filter((candidate) => candidate.nodeId === target?.rootNodeId),
4844 target?.activeAttemptId,
4845 );
4846 // The client may still show a running turn while its completion is being
4847 // projected. Preserve the submission as a new turn when steering is too late.
4848 if (
4849 target !== undefined &&
4850 (target.status === "completed" ||
4851 ((target.status === "running" || target.status === "waiting") &&
4852 turn !== undefined &&
4853 turn.status === "completed"))
4854 ) {
4855 dispatchMode = { type: "start_immediately" };
4856 }
4857 }
4858 // A restart continuation resumes the interrupted turn, so it starts
4859 // immediately rather than queueing like other wakes.
4860 if (
4861 command.notification !== undefined &&
4862 (command.createdBy !== "agent" ||
4863 (command.creationSource !== "server" && command.creationSource !== "provider") ||
4864 (dispatchMode.type !== "queue_after_active" &&
4865 !(
4866 command.restartContinuationOfRunId !== undefined &&
4867 dispatchMode.type === "start_immediately"
4868 )))
4869 ) {
4870 return yield* new OrchestratorDispatchError({
4871 commandId: command.commandId,
4872 commandType: command.type,
4873 cause: "Notifications must be server- or provider-created queued messages.",
4874 });
4875 }
4876 let delegatedCompletion:
4877 | OrchestrationV2ConversationMessage["delegatedCompletion"]
4878 | undefined;
4879 if (command.delegatedCompletion !== undefined) {
4880 const requestedCompletion = command.delegatedCompletion;
4881 if (
4882 command.createdBy !== "agent" ||
4883 command.creationSource !== "server" ||
4884 dispatchMode.type !== "queue_after_active"
4885 ) {
4886 return yield* new OrchestratorDispatchError({
4887 commandId: command.commandId,
4888 commandType: command.type,
4889 cause: "Delegated completion delivery must be a server-created queued message.",
4890 });
4891 }
4892 const parentRun = projection.runs.find(
4893 (candidate) => candidate.id === requestedCompletion.parentRunId,
4894 );
4895 const delivery = parentRun?.delegatedCompletion?.delivery;
4896 if (
4897 parentRun?.delegatedCompletion?.disposition !== "open" ||
4898 delivery === null ||
4899 delivery === undefined ||
4900 delivery.generation !== requestedCompletion.generation ||
4901 delivery.messageId !== command.messageId ||
4902 (projection.messages.some((candidate) => candidate.id === command.messageId) &&
4903 !isUndeliveredMailboxSteer(projection, command.messageId))
4904 ) {
4905 return yield* new OrchestratorDispatchError({
4906 commandId: command.commandId,
4907 commandType: command.type,
4908 cause: "Delegated completion delivery is no longer dispatchable.",
4909 });
4910 }
4911 delegatedCompletion = {
4912 parentRunId: requestedCompletion.parentRunId,
4913 generation: delivery.generation,
4914 taskIds: delivery.taskIds,
4915 };
4916 }
4917 // Route durable mailbox deliveries under the thread lock, using the live
4918 // session's capabilities. Never interrupt/restart a turn for a notification.
4919 if (
4920 delegatedCompletion !== undefined &&
4921 delegatedCompletion.taskIds.every(
4922 (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always",
4923 )
4924 ) {
4925 const active = projection.runs.find((run) => run.status === "running");
4926 const providerThread = projection.providerThreads.find(
4927 (row) => row.id === active?.providerThreadId,
4928 );
4929 const activeTurn = projection.providerTurns.find(
4930 (turn) => turn.runAttemptId === active?.activeAttemptId && turn.status === "running",
4931 );
4932 const activeMessage = projection.messages.find(
4933 (message) => message.id === active?.userMessageId,
4934 );
4935 if (
4936 active !== undefined &&
4937 activeTurn !== undefined &&
4938 providerThread?.providerSessionId != null &&
4939 (activeMessage === undefined || !isNativeMaintenanceCommand(activeMessage))
4940 ) {
4941 const session = yield* providerSessions
4942 .get(providerThread.providerSessionId)
4943 .pipe(Effect.orElseSucceed(() => Option.none()));
4944 if (
4945 Option.isSome(session) &&
4946 session.value.providerSession.capabilities.turns.supportsActiveSteering &&
4947 session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== true
4948 ) {
4949 dispatchMode = { type: "steer_active", targetRunId: active.id };
4950 }
4951 }
4952 }
4953 const dispatchText =
4954 delegatedCompletion === undefined
4955 ? command.text
4956 : delegatedCompletionWakeDetail(delegatedCompletion.taskIds);
4957 const sourcePlanProjection =
4958 command.sourcePlanRef === undefined
4959 ? null
4960 : yield* getProjectionWithPendingEvents(command.sourcePlanRef.threadId, events);
4961 // Command projections leave plans out, so read the source plan directly.
4962 const sourcePlanArtifact =
4963 command.sourcePlanRef === undefined
4964 ? undefined
4965 : yield* projectionStore
4966 .getPlan(command.sourcePlanRef.threadId, command.sourcePlanRef.planId)
4967 .pipe(mapDispatchError(command));
4968 const sourcePlan = sourcePlanArtifact?.kind === "proposed_plan" ? sourcePlanArtifact : null;
4969 if (command.sourcePlanRef !== undefined && sourcePlan === null) {
4970 return yield* new OrchestratorDispatchError({
4971 commandId: command.commandId,
4972 commandType: command.type,
4973 cause: `Proposed plan ${command.sourcePlanRef.planId} does not exist on thread ${command.sourcePlanRef.threadId}.`,
4974 });
4975 }
4976 if (
4977 sourcePlanProjection !== null &&
4978 sourcePlanProjection.thread.projectId !== projection.thread.projectId
4979 ) {
4980 return yield* new OrchestratorDispatchError({
4981 commandId: command.commandId,
4982 commandType: command.type,
4983 cause: `Proposed plan ${command.sourcePlanRef?.planId} belongs to a different project.`,
4984 });
4985 }
4986 if (sourcePlan !== null && sourcePlan.status !== "active") {
4987 return yield* new OrchestratorDispatchError({
4988 commandId: command.commandId,
4989 commandType: command.type,
4990 cause: `Proposed plan ${sourcePlan.id} is not active.`,
4991 });
4992 }
4993 const completeSourcePlan = (occurredAt: DateTime.Utc) =>
4994 sourcePlan === null
4995 ? Effect.void
4996 : emit(
4997 events,
4998 command,
4999 )({
5000 type: "plan.updated",
5001 threadId: sourcePlan.threadId,
5002 ...(sourcePlan.runId === null ? {} : { runId: sourcePlan.runId }),
5003 nodeId: sourcePlan.nodeId,
5004 occurredAt,
5005 payload: { ...sourcePlan, status: "completed" },
5006 });
5007
5008 if (
5009 (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") &&
5010 (yield* isRunCompacting(projection, dispatchMode.targetRunId))
5011 ) {
5012 dispatchMode = { type: "queue_after_active" };
5013 }
5014
5015 if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") {
5016 yield* dispatchSteerIntoRun({
5017 command,
5018 events,
5019 effects,
5020 projection,
5021 modelSelection:
5022 delegatedCompletion === undefined
5023 ? modelSelection
5024 : (projection.runs.find((run) => run.id === dispatchMode.targetRunId)
5025 ?.modelSelection ?? modelSelection),
5026 delegatedCompletion,
5027 targetRunId: dispatchMode.targetRunId,
5028 messageId: command.messageId,
5029 text: dispatchText,
5030 ...(command.context ? { context: command.context } : {}),
5031 attachments: command.attachments,
5032 createdBy: command.createdBy,
5033 creationSource: command.creationSource,
5034 ...(command.scheduledTaskId === undefined
5035 ? {}
5036 : { scheduledTaskId: command.scheduledTaskId }),
5037 ...(command.senderThreadId === undefined
5038 ? {}
5039 : { senderThreadId: command.senderThreadId }),
5040 forceRestart: dispatchMode.type === "restart_active",
5041 });
5042 return;
5043 }
5044
5045 const activeProviderThread = projection.providerThreads.find(
5046 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
5047 );
5048 const activeRun = projection.runs.find(isBlockingRun);
5049 // A message queued during worktree setup can arrive after that setup failed
5050 // without a worktree. It waits, held, like the messages queued before the
5051 // failure, instead of starting in the project checkout.
5052 const latestRun = latestExecutedRun(projection.runs);
5053 const worktreeMissingRun =
5054 activeRun === undefined &&
5055 dispatchMode.type === "queue_after_active" &&
5056 (latestRun?.status === "failed" || latestRun?.status === "interrupted") &&
5057 latestRun.workspacePreparation?.type === "worktree" &&
5058 projection.thread.worktreePath === null
5059 ? latestRun
5060 : undefined;
5061 const queueBehindRun = activeRun ?? worktreeMissingRun;
5062 const pendingMergeBackTransfers = pendingMergeBackTransfersForThread(projection);
5063 const shouldQueue =
5064 queueBehindRun !== undefined &&
5065 (dispatchMode.type === "defer_start" ||
5066 dispatchMode.type === "start_immediately" ||
5067 dispatchMode.type === "queue_after_active");
5068 if (shouldQueue) {
5069 if (pendingMergeBackTransfers.length > 0) {
5070 return yield* new OrchestratorDispatchError({
5071 commandId: command.commandId,
5072 commandType: command.type,
5073 cause: `Thread ${command.threadId} has a pending merge-back transfer; queued merge-back consumption is not implemented yet.`,
5074 });
5075 }
5076 const queueProviderThread =
5077 activeProviderThread ??
5078 projection.providerThreads.find(
5079 (candidate) => candidate.id === queueBehindRun.providerThreadId,
5080 );
5081 if (queueProviderThread === undefined) {
5082 return yield* new OrchestratorDispatchError({
5083 commandId: command.commandId,
5084 commandType: command.type,
5085 cause: `Active run ${queueBehindRun.id} has no provider thread for queued dispatch.`,
5086 });
5087 }
5088 const now = yield* DateTime.now;
5089 const ordinal = nextRunOrdinal(projection);
5090 const runId = idAllocator.derive.run({ threadId: command.threadId, ordinal });
5091 const targetProviderThread =
5092 modelSelection.instanceId === queueProviderThread.providerInstanceId
5093 ? queueProviderThread
5094 : rootProviderThreadsForProvider(projection, modelSelection.instanceId)[0];
5095 const queuedAdapter = yield* providerAdapters
5096 .get(modelSelection.instanceId)
5097 .pipe(mapDispatchError(command));
5098 const selectedProviderSession =
5099 targetProviderThread?.providerSessionId === null ||
5100 targetProviderThread?.providerSessionId === undefined
5101 ? undefined
5102 : projection.providerSessions.find(
5103 (candidate) => candidate.id === targetProviderThread.providerSessionId,
5104 );
5105 const queuedCapabilities =
5106 selectedProviderSession?.capabilities ??
5107 (yield* queuedAdapter.getCapabilities().pipe(mapDispatchError(command)));
5108 yield* enforceCommandPolicy(command)(
5109 commandPolicy.ensureQueuedMessages({
5110 commandId: command.commandId,
5111 threadId: command.threadId,
5112 providerInstanceId: modelSelection.instanceId,
5113 capabilities: queuedCapabilities,
5114 }),
5115 );
5116 const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? {
5117 id: idAllocator.derive.providerThread({
5118 driver: queuedAdapter.driver,
5119 nativeThreadId: `pending:${runId}`,
5120 }),
5121 driver: queuedAdapter.driver,
5122 providerInstanceId: modelSelection.instanceId,
5123 providerSessionId: null,
5124 appThreadId: command.threadId,
5125 ownerNodeId: null,
5126 nativeThreadRef: null,
5127 nativeConversationHeadRef: null,
5128 status: "not_loaded",
5129 firstRunOrdinal: null,
5130 lastRunOrdinal: null,
5131 handoffIds: [],
5132 forkedFrom: null,
5133 createdAt: now,
5134 updatedAt: now,
5135 };
5136 const attemptId = idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 });
5137 const rootNodeId = idAllocator.derive.rootNode({ runId });
5138 const checkpointScope =
5139 queueBehindRun.status === "preparing" || worktreeMissingRun !== undefined
5140 ? null
5141 : yield* runtimePolicy.resolve({ thread: projection.thread, modelSelection }).pipe(
5142 Effect.flatMap((resolvedRuntimePolicy) =>
5143 checkpointService.prepareRootRunScope({
5144 threadId: command.threadId,
5145 runId,
5146 rootNodeId,
5147 providerThreadId: queuedProviderThread.id,
5148 cwd:
5149 resolvedRuntimePolicy.cwd ??
5150 selectedProviderSession?.cwd ??
5151 projection.thread.worktreePath ??
5152 process.cwd(),
5153 createdAt: now,
5154 }),
5155 ),
5156 Effect.mapError(
5157 (cause) =>
5158 new OrchestratorDispatchError({
5159 commandId: command.commandId,
5160 commandType: command.type,
5161 cause,
5162 }),
5163 ),
5164 );
5165 const run: OrchestrationV2Run = {
5166 id: runId,
5167 threadId: command.threadId,
5168 ordinal,
5169 providerInstanceId: modelSelection.instanceId,
5170 modelSelection,
5171 providerThreadId: queuedProviderThread.id,
5172 userMessageId: command.messageId,
5173 rootNodeId,
5174 activeAttemptId: attemptId,
5175 status: "queued",
5176 ...(worktreeMissingRun !== undefined ||
5177 projection.runs.some(
5178 (candidate) => candidate.status === "queued" && candidate.queueHeld === true,
5179 )
5180 ? { queueHeld: true }
5181 : {}),
5182 queuePosition:
5183 Math.max(
5184 0,
5185 ...projection.runs
5186 .filter((candidate) => candidate.status === "queued")
5187 .map((candidate) => candidate.queuePosition ?? candidate.ordinal),
5188 ) + 1,
5189 requestedAt: now,
5190 startedAt: null,
5191 completedAt: null,
5192 checkpointId: null,
5193 contextHandoffId: null,
5194 ...(command.sourcePlanRef === undefined ? {} : { sourcePlanRef: command.sourcePlanRef }),
5195 ...(command.restartContinuationOfRunId === undefined
5196 ? {}
5197 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
5198 ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
5199 };
5200 const attempt: OrchestrationV2RunAttempt = {
5201 id: attemptId,
5202 runId,
5203 attemptOrdinal: 1,
5204 rootNodeId,
5205 providerInstanceId: modelSelection.instanceId,
5206 providerThreadId: queuedProviderThread.id,
5207 providerTurnId: null,
5208 reason: "initial",
5209 status: "pending",
5210 startedAt: null,
5211 completedAt: null,
5212 };
5213 const rootNode: OrchestrationV2ExecutionNode = {
5214 id: rootNodeId,
5215 threadId: command.threadId,
5216 runId,
5217 parentNodeId: null,
5218 rootNodeId,
5219 kind: "root_turn",
5220 status: "pending",
5221 countsForRun: true,
5222 providerThreadId: queuedProviderThread.id,
5223 providerTurnId: null,
5224 nativeItemRef: null,
5225 runtimeRequestId: null,
5226 checkpointScopeId: checkpointScope?.id ?? null,
5227 startedAt: null,
5228 completedAt: null,
5229 };
5230 const message: OrchestrationV2ConversationMessage = {
5231 createdBy: command.createdBy,
5232 creationSource: command.creationSource,
5233 ...(command.scheduledTaskId === undefined
5234 ? {}
5235 : { scheduledTaskId: command.scheduledTaskId }),
5236 ...(command.senderThreadId === undefined
5237 ? {}
5238 : { senderThreadId: command.senderThreadId }),
5239 id: command.messageId,
5240 threadId: command.threadId,
5241 runId,
5242 nodeId: rootNodeId,
5243 role: "user",
5244 text: dispatchText,
5245 ...(command.context ? { context: command.context } : {}),
5246 attachments: command.attachments,
5247 streaming: false,
5248 createdAt: now,
5249 updatedAt: now,
5250 ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }),
5251 ...(command.notification === undefined ? {} : { notification: command.notification }),
5252 };
5253 const emitEvent = emit(events, command);
5254 if (targetProviderThread === undefined) {
5255 yield* emitEvent({
5256 type: "provider-thread.updated",
5257 threadId: command.threadId,
5258 providerInstanceId: modelSelection.instanceId,
5259 occurredAt: now,
5260 payload: queuedProviderThread,
5261 });
5262 }
5263 yield* emitEvent({
5264 type: "run.created",
5265 threadId: command.threadId,
5266 runId,
5267 nodeId: rootNodeId,
5268 providerInstanceId: modelSelection.instanceId,
5269 occurredAt: now,
5270 payload: run,
5271 });
5272 yield* completeSourcePlan(now);
5273 yield* emitEvent({
5274 type: "run-attempt.created",
5275 threadId: command.threadId,
5276 runId,
5277 nodeId: rootNodeId,
5278 providerInstanceId: modelSelection.instanceId,
5279 occurredAt: now,
5280 payload: attempt,
5281 });
5282 yield* emitEvent({
5283 type: "node.updated",
5284 threadId: command.threadId,
5285 runId,
5286 nodeId: rootNodeId,
5287 providerInstanceId: modelSelection.instanceId,
5288 occurredAt: now,
5289 payload: rootNode,
5290 });
5291 if (checkpointScope !== null) {
5292 yield* emitEvent({
5293 type: "checkpoint-scope.created",
5294 threadId: command.threadId,
5295 runId,
5296 nodeId: rootNodeId,
5297 providerInstanceId: modelSelection.instanceId,
5298 occurredAt: now,
5299 payload: yield* checkpointService.ensureScope(checkpointScope).pipe(
5300 Effect.mapError(
5301 (cause) =>
5302 new OrchestratorDispatchError({
5303 commandId: command.commandId,
5304 commandType: command.type,
5305 cause,
5306 }),
5307 ),
5308 ),
5309 });
5310 }
5311 yield* emitEvent({
5312 type: "message.updated",
5313 threadId: command.threadId,
5314 runId,
5315 nodeId: rootNodeId,
5316 providerInstanceId: modelSelection.instanceId,
5317 occurredAt: now,
5318 payload: message,
5319 });
5320 const existingItem = projection.turnItems.find(
5321 (item) => item.type === "user_message" && item.messageId === command.messageId,
5322 );
5323 if (existingItem?.type === "user_message") {
5324 yield* emitEvent({
5325 type: "turn-item.updated",
5326 threadId: command.threadId,
5327 runId,
5328 nodeId: rootNodeId,
5329 providerInstanceId: modelSelection.instanceId,
5330 occurredAt: now,
5331 payload: {
5332 ...existingItem,
5333 runId,
5334 nodeId: rootNodeId,
5335 providerThreadId: queuedProviderThread.id,
5336 providerTurnId: null,
5337 inputIntent: "queued_turn",
5338 updatedAt: now,
5339 },
5340 });
5341 }
5342 return;
5343 }
5344 const pendingForkTransfer = pendingForkTransferForThread(projection);
5345 const pendingMergeBackSourceThreadIds = new Set(
5346 pendingMergeBackTransfers.map((transfer) => transfer.sourceThreadId),
5347 );
5348 if (pendingMergeBackSourceThreadIds.size > 1) {
5349 return yield* new OrchestratorDispatchError({
5350 commandId: command.commandId,
5351 commandType: command.type,
5352 cause: `Thread ${command.threadId} has pending merge-back transfers from multiple forks.`,
5353 });
5354 }
5355 const pendingMergeBackTransfer = latestContextTransfer(pendingMergeBackTransfers);
5356 const supersededMergeBackTransfers = pendingMergeBackTransfers.filter(
5357 (transfer) => transfer.id !== pendingMergeBackTransfer?.id,
5358 );
5359 const now = yield* DateTime.now;
5360 if (
5361 !modelSelectionsEqual(projection.thread.modelSelection, modelSelection) ||
5362 projection.thread.providerInstanceId !== modelSelection.instanceId
5363 ) {
5364 yield* emit(
5365 events,
5366 command,
5367 )({
5368 type:
5369 projection.thread.providerInstanceId === modelSelection.instanceId
5370 ? "thread.model-selection-updated"
5371 : "thread.provider-switched",
5372 threadId: command.threadId,
5373 providerInstanceId: modelSelection.instanceId,
5374 occurredAt: now,
5375 payload: {
5376 ...projection.thread,
5377 providerInstanceId: modelSelection.instanceId,
5378 modelSelection,
5379 updatedAt: now,
5380 },
5381 });
5382 }
5383 const ordinal = nextRunOrdinal(projection);
5384 const runId = idAllocator.derive.run({ threadId: command.threadId, ordinal });
5385 const latestCompletedRun = projection.runs.findLast((run) => run.status === "completed");
5386 const latestHandoffRun = projection.runs.findLast(isHandoffSourceRun);
5387 const legacyImportItems =
5388 projection.thread.historyOrigin === "v1_import"
5389 ? yield* readHandoffItems(command.threadId, [null])
5390 : [];
5391 const isProviderSwitch =
5392 activeProviderThread !== undefined &&
5393 activeProviderThread.providerInstanceId !== modelSelection.instanceId;
5394 // Account overlays share native history. Selection commands may already
5395 // have updated the app thread, so classify against the native thread's owner.
5396 const canResumeAcrossInstances =
5397 isProviderSwitch &&
5398 activeProviderThread.nativeThreadRef !== null &&
5399 (yield* providerSwitchService
5400 .plan({
5401 projection: {
5402 ...projection,
5403 thread: {
5404 ...projection.thread,
5405 modelSelection: {
5406 ...projection.thread.modelSelection,
5407 instanceId: activeProviderThread.providerInstanceId,
5408 },
5409 },
5410 },
5411 targetModelSelection: modelSelection,
5412 })
5413 .pipe(mapDispatchError(command))).transition.type === "restart_and_resume";
5414
5415 if (
5416 pendingForkTransfer === undefined &&
5417 pendingMergeBackTransfer === undefined &&
5418 !isProviderSwitch
5419 ) {
5420 const adapter = yield* providerAdapters.get(modelSelection.instanceId).pipe(
5421 Effect.mapError(
5422 (cause) =>
5423 new OrchestratorProviderAdapterError({
5424 commandId: command.commandId,
5425 providerInstanceId: modelSelection.instanceId,
5426 cause,
5427 }),
5428 ),
5429 );
5430 const providerSessionId =
5431 activeProviderThread?.providerSessionId ??
5432 (yield* mapDispatchError(command)(
5433 providerSessionIdFor({
5434 adapter,
5435 providerInstanceId: modelSelection.instanceId,
5436 threadId: command.threadId,
5437 }),
5438 ));
5439 const providerThreadId =
5440 activeProviderThread?.id ??
5441 idAllocator.derive.providerThread({
5442 driver: adapter.driver,
5443 nativeThreadId: `pending:${runId}`,
5444 });
5445 const legacyImportHandoff = shouldPrepareLegacyImportHandoff({
5446 historyOrigin: projection.thread.historyOrigin,
5447 hasCompletedRun: latestCompletedRun !== undefined,
5448 legacyImportItemCount: legacyImportItems.length,
5449 })
5450 ? yield* contextHandoffService
5451 .prepareLegacyImport({
5452 threadId: command.threadId,
5453 targetRunId: runId,
5454 toProviderThreadId: providerThreadId,
5455 toProviderInstanceId: modelSelection.instanceId,
5456 items: legacyImportItems,
5457 createdAt: now,
5458 })
5459 .pipe(mapDispatchError(command))
5460 : null;
5461 const providerThread: OrchestrationV2ProviderThread =
5462 activeProviderThread === undefined
5463 ? {
5464 id: providerThreadId,
5465 driver: adapter.driver,
5466 providerInstanceId: modelSelection.instanceId,
5467 providerSessionId,
5468 appThreadId: command.threadId,
5469 ownerNodeId: null,
5470 nativeThreadRef: null,
5471 nativeConversationHeadRef: null,
5472 status: "not_loaded",
5473 firstRunOrdinal: ordinal,
5474 lastRunOrdinal: ordinal,
5475 handoffIds: legacyImportHandoff === null ? [] : [legacyImportHandoff.id],
5476 forkedFrom: null,
5477 createdAt: now,
5478 updatedAt: now,
5479 }
5480 : {
5481 ...activeProviderThread,
5482 providerSessionId,
5483 lastRunOrdinal: ordinal,
5484 handoffIds: appendContextHandoffId(
5485 activeProviderThread.handoffIds,
5486 legacyImportHandoff?.id ?? null,
5487 ),
5488 updatedAt: now,
5489 };
5490 const attemptId = idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 });
5491 const rootNodeId = idAllocator.derive.rootNode({ runId });
5492 const checkpointScope =
5493 dispatchMode.type === "defer_start"
5494 ? null
5495 : yield* runtimePolicy
5496 .resolve({
5497 thread: projection.thread,
5498 modelSelection,
5499 })
5500 .pipe(
5501 mapDispatchError(command),
5502 Effect.flatMap((resolvedRuntimePolicy) =>
5503 checkpointService.prepareRootRunScope({
5504 threadId: command.threadId,
5505 runId,
5506 rootNodeId,
5507 providerThreadId,
5508 cwd:
5509 resolvedRuntimePolicy.cwd ??
5510 projection.thread.worktreePath ??
5511 process.cwd(),
5512 createdAt: now,
5513 }),
5514 ),
5515 mapDispatchError(command),
5516 );
5517 const run: OrchestrationV2Run = {
5518 id: runId,
5519 threadId: command.threadId,
5520 ordinal,
5521 providerInstanceId: modelSelection.instanceId,
5522 modelSelection,
5523 providerThreadId,
5524 userMessageId: command.messageId,
5525 rootNodeId,
5526 activeAttemptId: attemptId,
5527 status: dispatchMode.type === "defer_start" ? "preparing" : "starting",
5528 queuePosition: null,
5529 requestedAt: now,
5530 startedAt: null,
5531 completedAt: null,
5532 checkpointId: null,
5533 contextHandoffId: legacyImportHandoff?.id ?? null,
5534 ...(command.sourcePlanRef === undefined ? {} : { sourcePlanRef: command.sourcePlanRef }),
5535 ...(command.restartContinuationOfRunId === undefined
5536 ? {}
5537 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
5538 ...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
5539 ? { workspacePreparation: dispatchMode.workspaceStrategy }
5540 : {}),
5541 ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
5542 ...wakeWorkStartedAt(projection.runs, command),
5543 };
5544 const attempt: OrchestrationV2RunAttempt = {
5545 id: attemptId,
5546 runId,
5547 attemptOrdinal: 1,
5548 rootNodeId,
5549 providerInstanceId: modelSelection.instanceId,
5550 providerThreadId,
5551 providerTurnId: null,
5552 reason: "initial",
5553 status: "pending",
5554 startedAt: null,
5555 completedAt: null,
5556 };
5557 const rootNode: OrchestrationV2ExecutionNode = {
5558 id: rootNodeId,
5559 threadId: command.threadId,
5560 runId,
5561 parentNodeId: null,
5562 rootNodeId,
5563 kind: "root_turn",
5564 status: "pending",
5565 countsForRun: true,
5566 providerThreadId,
5567 providerTurnId: null,
5568 nativeItemRef: null,
5569 runtimeRequestId: null,
5570 checkpointScopeId: checkpointScope?.id ?? null,
5571 startedAt: null,
5572 completedAt: null,
5573 };
5574 const message: OrchestrationV2ConversationMessage = {
5575 createdBy: command.createdBy,
5576 creationSource: command.creationSource,
5577 ...(command.scheduledTaskId === undefined
5578 ? {}
5579 : { scheduledTaskId: command.scheduledTaskId }),
5580 ...(command.senderThreadId === undefined
5581 ? {}
5582 : { senderThreadId: command.senderThreadId }),
5583 id: command.messageId,
5584 threadId: command.threadId,
5585 runId,
5586 nodeId: rootNodeId,
5587 role: "user",
5588 text: dispatchText,
5589 ...(command.context ? { context: command.context } : {}),
5590 attachments: command.attachments,
5591 streaming: false,
5592 createdAt: now,
5593 updatedAt: now,
5594 ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }),
5595 ...(command.notification === undefined ? {} : { notification: command.notification }),
5596 };
5597 const turnItem: OrchestrationV2TurnItem = {
5598 createdBy: command.createdBy,
5599 creationSource: command.creationSource,
5600 ...(command.scheduledTaskId === undefined
5601 ? {}
5602 : { scheduledTaskId: command.scheduledTaskId }),
5603 ...(command.senderThreadId === undefined
5604 ? {}
5605 : { senderThreadId: command.senderThreadId }),
5606 id: idAllocator.derive.userTurnItem({ messageId: command.messageId }),
5607 threadId: command.threadId,
5608 runId,
5609 nodeId: rootNodeId,
5610 providerThreadId,
5611 providerTurnId: null,
5612 nativeItemRef: null,
5613 parentItemId: null,
5614 ordinal: yield* nextTurnItemOrdinal(projection),
5615 status: "completed",
5616 title: null,
5617 startedAt: now,
5618 completedAt: now,
5619 updatedAt: now,
5620 type: "user_message",
5621 messageId: command.messageId,
5622 inputIntent: "turn_start",
5623 text: dispatchText,
5624 ...(command.context ? { context: command.context } : {}),
5625 attachments: command.attachments,
5626 };
5627 const preparationTurnItem: OrchestrationV2TurnItem | null =
5628 dispatchMode.type === "defer_start"
5629 ? {
5630 id: idAllocator.derive.turnItemFromProviderItem({
5631 driver: adapter.driver,
5632 nativeItemId: `workspace-preparation:${runId}`,
5633 }),
5634 threadId: command.threadId,
5635 runId,
5636 nodeId: rootNodeId,
5637 providerThreadId,
5638 providerTurnId: null,
5639 nativeItemRef: null,
5640 parentItemId: null,
5641 ordinal: turnItem.ordinal + 1,
5642 status: "running",
5643 title: WORKSPACE_PREPARATION_INPUT,
5644 startedAt: now,
5645 completedAt: null,
5646 updatedAt: now,
5647 type: "command_execution",
5648 input: WORKSPACE_PREPARATION_INPUT,
5649 }
5650 : null;
5651 const emitEvent = emit(events, command);
5652 yield* emitEvent({
5653 type: "provider-thread.updated",
5654 threadId: command.threadId,
5655 driver: adapter.driver,
5656 providerInstanceId: modelSelection.instanceId,
5657 occurredAt: now,
5658 payload: providerThread,
5659 });
5660 if (legacyImportHandoff !== null) {
5661 yield* emitEvent({
5662 type: "context-handoff.updated",
5663 threadId: command.threadId,
5664 runId,
5665 providerInstanceId: modelSelection.instanceId,
5666 occurredAt: now,
5667 payload: legacyImportHandoff,
5668 });
5669 }
5670 yield* emitEvent({
5671 type: "run.created",
5672 threadId: command.threadId,
5673 runId,
5674 nodeId: rootNodeId,
5675 providerInstanceId: modelSelection.instanceId,
5676 occurredAt: now,
5677 payload: run,
5678 });
5679 yield* completeSourcePlan(now);
5680 yield* emitEvent({
5681 type: "run-attempt.created",
5682 threadId: command.threadId,
5683 runId,
5684 nodeId: rootNodeId,
5685 providerInstanceId: modelSelection.instanceId,
5686 occurredAt: now,
5687 payload: attempt,
5688 });
5689 yield* emitEvent({
5690 type: "node.updated",
5691 threadId: command.threadId,
5692 runId,
5693 nodeId: rootNodeId,
5694 providerInstanceId: modelSelection.instanceId,
5695 occurredAt: now,
5696 payload: rootNode,
5697 });
5698 if (checkpointScope !== null) {
5699 yield* emitEvent({
5700 type: "checkpoint-scope.created",
5701 threadId: command.threadId,
5702 runId,
5703 nodeId: rootNodeId,
5704 providerInstanceId: modelSelection.instanceId,
5705 occurredAt: now,
5706 payload: checkpointScope,
5707 });
5708 }
5709 yield* emitEvent({
5710 type: "message.updated",
5711 threadId: command.threadId,
5712 runId,
5713 nodeId: rootNodeId,
5714 providerInstanceId: modelSelection.instanceId,
5715 occurredAt: now,
5716 payload: message,
5717 });
5718 yield* emitEvent({
5719 type: "turn-item.updated",
5720 threadId: command.threadId,
5721 runId,
5722 nodeId: rootNodeId,
5723 providerInstanceId: modelSelection.instanceId,
5724 occurredAt: now,
5725 payload: notificationTurnItem(turnItem, message, projection.subagents),
5726 });
5727 if (preparationTurnItem !== null) {
5728 yield* emitEvent({
5729 type: "turn-item.updated",
5730 threadId: command.threadId,
5731 runId,
5732 nodeId: rootNodeId,
5733 providerInstanceId: modelSelection.instanceId,
5734 occurredAt: now,
5735 payload: preparationTurnItem,
5736 });
5737 }
5738 const pendingEffect = {
5739 id: `effect:${command.commandId}:provider-turn.start:${runId}`,
5740 commandId: command.commandId,
5741 threadId: command.threadId,
5742 request: { type: "provider-turn.start", runId },
5743 } satisfies PendingOrchestrationEffectV2;
5744 if (dispatchMode.type !== "defer_start") {
5745 yield* Ref.update(effects, (existing) => [...existing, pendingEffect]);
5746 }
5747 return;
5748 }
5749 const sourceProjection =
5750 pendingForkTransfer === undefined
5751 ? null
5752 : yield* projectionStore
5753 .getThreadRecords(pendingForkTransfer.sourceThreadId, [
5754 "runs",
5755 "attempts",
5756 "providerThreads",
5757 "providerTurns",
5758 ])
5759 .pipe(
5760 Effect.mapError(
5761 (cause) =>
5762 new OrchestratorProjectionError({
5763 threadId: pendingForkTransfer.sourceThreadId,
5764 cause,
5765 }),
5766 ),
5767 );
5768 const sourceRun =
5769 pendingForkTransfer?.sourcePoint.runId === undefined || sourceProjection === null
5770 ? null
5771 : (sourceProjection.runs.find(
5772 (candidate) => candidate.id === pendingForkTransfer.sourcePoint.runId,
5773 ) ?? null);
5774 const sourceProviderThread =
5775 sourceProjection === null || sourceRun === null
5776 ? undefined
5777 : providerThreadForRun(sourceProjection, sourceRun);
5778 const sourceProviderTurnId =
5779 sourceProjection === null || sourceRun === null || sourceRun.activeAttemptId === null
5780 ? undefined
5781 : (latestProviderTurnForAttempt(sourceProjection.providerTurns, sourceRun.activeAttemptId)
5782 ?.id ??
5783 sourceProjection.attempts.find(
5784 (candidate) => candidate.id === sourceRun.activeAttemptId,
5785 )?.providerTurnId ??
5786 undefined);
5787 if (pendingForkTransfer !== undefined) {
5788 if (sourceRun === null || sourceProviderThread === undefined) {
5789 return yield* new OrchestratorDispatchError({
5790 commandId: command.commandId,
5791 commandType: command.type,
5792 cause: `Pending fork transfer ${pendingForkTransfer.id} has no resolvable source provider thread.`,
5793 });
5794 }
5795 if (pendingForkTransfer.sourceProviderInstanceId === null) {
5796 return yield* new OrchestratorDispatchError({
5797 commandId: command.commandId,
5798 commandType: command.type,
5799 cause: `Pending fork transfer ${pendingForkTransfer.id} has no source provider.`,
5800 });
5801 }
5802 }
5803
5804 const adapter = yield* providerAdapters.get(modelSelection.instanceId).pipe(
5805 Effect.mapError(
5806 (cause) =>
5807 new OrchestratorProviderAdapterError({
5808 commandId: command.commandId,
5809 providerInstanceId: modelSelection.instanceId,
5810 cause,
5811 }),
5812 ),
5813 );
5814 const targetProviderThread =
5815 isProviderSwitch && !canResumeAcrossInstances
5816 ? rootProviderThreadsForProvider(projection, modelSelection.instanceId)[0]
5817 : activeProviderThread;
5818 const providerSessionId =
5819 (canResumeAcrossInstances ? undefined : targetProviderThread?.providerSessionId) ??
5820 (yield* mapDispatchError(command)(
5821 providerSessionIdFor({
5822 adapter,
5823 providerInstanceId: modelSelection.instanceId,
5824 threadId: command.threadId,
5825 }),
5826 ));
5827 const existingProviderSession = projection.providerSessions.find(
5828 (candidate) => candidate.id === providerSessionId,
5829 );
5830 const resolvedRuntimePolicy = yield* runtimePolicy
5831 .resolve({ thread: projection.thread, modelSelection })
5832 .pipe(
5833 Effect.mapError(
5834 (cause) =>
5835 new OrchestratorDispatchError({
5836 commandId: command.commandId,
5837 commandType: command.type,
5838 cause,
5839 }),
5840 ),
5841 );
5842
5843 const capabilities = yield* adapter.getCapabilities().pipe(
5844 Effect.mapError(
5845 (cause) =>
5846 new OrchestratorProviderAdapterError({
5847 commandId: command.commandId,
5848 providerInstanceId: modelSelection.instanceId,
5849 cause,
5850 }),
5851 ),
5852 );
5853 const forkExecution =
5854 pendingForkTransfer === undefined || sourceRun === null
5855 ? null
5856 : yield* enforceCommandPolicy(command)(
5857 commandPolicy.decideForkExecution({
5858 commandId: command.commandId,
5859 threadId: command.threadId,
5860 providerInstanceId: modelSelection.instanceId,
5861 capabilities,
5862 sameProvider:
5863 pendingForkTransfer.sourceProviderInstanceId === modelSelection.instanceId,
5864 hasStrongNativeSource: sourceProviderThread?.nativeThreadRef?.strength === "strong",
5865 sourceRunStatus: sourceRun.status,
5866 fromSpecificTurn: sourceRun !== null,
5867 }),
5868 );
5869 const canResolveForkNatively = forkExecution === "native_fork";
5870 const requiresPortableFork = forkExecution === "portable_context";
5871
5872 if (canResolveForkNatively) {
5873 yield* enforceCommandPolicy(command)(
5874 commandPolicy.ensureNativeFork({
5875 commandId: command.commandId,
5876 threadId: command.threadId,
5877 providerInstanceId: modelSelection.instanceId,
5878 capabilities,
5879 fromSpecificTurn: sourceRun !== null,
5880 }),
5881 );
5882 }
5883
5884 const ensuredProviderThread: OrchestrationV2ProviderThread =
5885 targetProviderThread === undefined
5886 ? {
5887 id: idAllocator.derive.providerThread({
5888 driver: adapter.driver,
5889 nativeThreadId: `pending:${runId}`,
5890 }),
5891 driver: adapter.driver,
5892 providerInstanceId: modelSelection.instanceId,
5893 providerSessionId,
5894 appThreadId: command.threadId,
5895 ownerNodeId: null,
5896 nativeThreadRef: null,
5897 nativeConversationHeadRef: null,
5898 status: "not_loaded",
5899 firstRunOrdinal: ordinal,
5900 lastRunOrdinal: ordinal,
5901 handoffIds: [],
5902 forkedFrom:
5903 canResolveForkNatively && sourceProviderThread !== undefined
5904 ? {
5905 providerThreadId: sourceProviderThread.id,
5906 ...(sourceProviderTurnId === undefined
5907 ? {}
5908 : { providerTurnId: sourceProviderTurnId }),
5909 }
5910 : null,
5911 createdAt: now,
5912 updatedAt: now,
5913 }
5914 : {
5915 ...targetProviderThread,
5916 providerInstanceId: modelSelection.instanceId,
5917 providerSessionId,
5918 updatedAt: now,
5919 };
5920 const portableForkItems =
5921 !requiresPortableFork || sourceProjection === null || sourceRun === null
5922 ? []
5923 : yield* readHandoffItems(sourceProjection.thread.id, [
5924 ...sourceProjection.runs
5925 .filter((run) => run.ordinal <= sourceRun.ordinal)
5926 .map((run) => run.id),
5927 ...(sourceProjection.thread.historyOrigin === "v1_import" ? [null] : []),
5928 ]);
5929 const portableForkHandoff =
5930 !requiresPortableFork ||
5931 pendingForkTransfer === undefined ||
5932 sourceProjection === null ||
5933 sourceRun === null
5934 ? null
5935 : yield* contextHandoffService
5936 .prepareProviderHandoff({
5937 threadId: command.threadId,
5938 targetRunId: runId,
5939 transferId: pendingForkTransfer.id,
5940 fromProviderThreadIds:
5941 sourceProviderThread === undefined ? [] : [sourceProviderThread.id],
5942 toProviderThreadId: ensuredProviderThread.id,
5943 fromProviderInstanceId: sourceRun.providerInstanceId,
5944 toProviderInstanceId: modelSelection.instanceId,
5945 coveredRunOrdinals: visibleDeltaRunOrdinals(sourceProjection, portableForkItems),
5946 strategy: "full_thread_summary",
5947 items: portableForkItems,
5948 runs: sourceProjection.runs,
5949 createdAt: now,
5950 })
5951 .pipe(
5952 Effect.mapError(
5953 (cause) =>
5954 new OrchestratorDispatchError({
5955 commandId: command.commandId,
5956 commandType: command.type,
5957 cause,
5958 }),
5959 ),
5960 );
5961 const requiresFullProviderSwitchContext =
5962 isProviderSwitch && pendingMergeBackTransfer !== undefined;
5963 const targetLastCompletedRun =
5964 targetProviderThread === undefined
5965 ? undefined
5966 : lastDeliveredRunForProviderThread(projection, targetProviderThread.id);
5967 const providerSwitchCoveredRuns =
5968 !isProviderSwitch || canResumeAcrossInstances || latestHandoffRun === undefined
5969 ? []
5970 : projection.runs.filter(
5971 (run) =>
5972 isHandoffSourceRun(run) &&
5973 run.ordinal >
5974 (requiresFullProviderSwitchContext
5975 ? 0
5976 : (targetLastCompletedRun?.ordinal ?? 0)) &&
5977 run.ordinal <= latestHandoffRun.ordinal,
5978 );
5979 const providerSwitchItems =
5980 providerSwitchCoveredRuns.length === 0
5981 ? []
5982 : [
5983 ...(latestCompletedRun !== undefined &&
5984 (targetProviderThread === undefined || requiresFullProviderSwitchContext)
5985 ? legacyImportItems
5986 : []),
5987 ...(yield* readHandoffItems(
5988 command.threadId,
5989 providerSwitchCoveredRuns.map((run) => run.id),
5990 )),
5991 ];
5992 const providerSwitchTransferId =
5993 providerSwitchCoveredRuns.length === 0 || latestHandoffRun === undefined
5994 ? null
5995 : yield* mapDispatchError(command)(
5996 idAllocator.allocate.contextTransfer({
5997 sourceThreadId: command.threadId,
5998 targetThreadId: command.threadId,
5999 type: "provider_handoff",
6000 }),
6001 );
6002 if (providerSwitchTransferId !== null) {
6003 yield* enforceCommandPolicy(command)(
6004 commandPolicy.ensureContextHandoff({
6005 commandId: command.commandId,
6006 threadId: command.threadId,
6007 providerInstanceId: modelSelection.instanceId,
6008 capabilities,
6009 strategy:
6010 targetProviderThread === undefined || requiresFullProviderSwitchContext
6011 ? "full_thread_summary"
6012 : "delta_context",
6013 }),
6014 );
6015 }
6016 const providerSwitchHandoff =
6017 providerSwitchTransferId === null || latestHandoffRun === undefined
6018 ? null
6019 : yield* contextHandoffService
6020 .prepareProviderHandoff({
6021 threadId: command.threadId,
6022 targetRunId: runId,
6023 transferId: providerSwitchTransferId,
6024 fromProviderThreadIds: Array.from(
6025 new Set(
6026 providerSwitchCoveredRuns.flatMap((run) =>
6027 run.providerThreadId === null ? [] : [run.providerThreadId],
6028 ),
6029 ),
6030 ),
6031 toProviderThreadId: ensuredProviderThread.id,
6032 fromProviderInstanceId: latestHandoffRun.providerInstanceId,
6033 toProviderInstanceId: modelSelection.instanceId,
6034 coveredRunOrdinals: {
6035 from: providerSwitchCoveredRuns[0]!.ordinal,
6036 to: providerSwitchCoveredRuns.at(-1)!.ordinal,
6037 },
6038 strategy:
6039 targetProviderThread === undefined || requiresFullProviderSwitchContext
6040 ? "full_thread_summary"
6041 : "delta_since_target_last_seen",
6042 items: providerSwitchItems,
6043 runs: projection.runs,
6044 createdAt: now,
6045 })
6046 .pipe(
6047 Effect.mapError(
6048 (cause) =>
6049 new OrchestratorDispatchError({
6050 commandId: command.commandId,
6051 commandType: command.type,
6052 cause,
6053 }),
6054 ),
6055 );
6056 const legacyImportRecoveryHandoff =
6057 isProviderSwitch &&
6058 !canResumeAcrossInstances &&
6059 latestCompletedRun === undefined &&
6060 legacyImportItems.length > 0
6061 ? yield* contextHandoffService
6062 .prepareLegacyImport({
6063 threadId: command.threadId,
6064 targetRunId: runId,
6065 toProviderThreadId: ensuredProviderThread.id,
6066 toProviderInstanceId: modelSelection.instanceId,
6067 items: legacyImportItems,
6068 createdAt: now,
6069 })
6070 .pipe(mapDispatchError(command))
6071 : null;
6072 const providerThread: OrchestrationV2ProviderThread = {
6073 ...ensuredProviderThread,
6074 status: "active",
6075 firstRunOrdinal: ensuredProviderThread.firstRunOrdinal ?? ordinal,
6076 lastRunOrdinal: ordinal,
6077 handoffIds: [
6078 ...ensuredProviderThread.handoffIds,
6079 ...[portableForkHandoff, providerSwitchHandoff, legacyImportRecoveryHandoff].flatMap(
6080 (handoff) => (handoff === null ? [] : [handoff.id]),
6081 ),
6082 ],
6083 updatedAt: now,
6084 };
6085
6086 const attemptId = idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 });
6087 const rootNodeId = idAllocator.derive.rootNode({ runId });
6088 const emitEvent = emit(events, command);
6089 const mergeBackSourceProjection =
6090 pendingMergeBackTransfer === undefined
6091 ? null
6092 : yield* projectionStore
6093 .getThreadRecords(pendingMergeBackTransfer.sourceThreadId, [
6094 "runs",
6095 "attempts",
6096 "providerThreads",
6097 "providerTurns",
6098 ])
6099 .pipe(
6100 Effect.mapError(
6101 (cause) =>
6102 new OrchestratorProjectionError({
6103 threadId: pendingMergeBackTransfer.sourceThreadId,
6104 cause,
6105 }),
6106 ),
6107 );
6108 const mergeBackSourceRun =
6109 pendingMergeBackTransfer?.sourcePoint.runId === undefined ||
6110 mergeBackSourceProjection === null
6111 ? null
6112 : (mergeBackSourceProjection.runs.find(
6113 (candidate) => candidate.id === pendingMergeBackTransfer.sourcePoint.runId,
6114 ) ?? null);
6115 if (pendingMergeBackTransfer !== undefined && mergeBackSourceRun === null) {
6116 return yield* new OrchestratorDispatchError({
6117 commandId: command.commandId,
6118 commandType: command.type,
6119 cause: `Pending merge-back transfer ${pendingMergeBackTransfer.id} has no resolvable source run.`,
6120 });
6121 }
6122 const mergeBackSourceProviderThread =
6123 mergeBackSourceProjection === null || mergeBackSourceRun === null
6124 ? undefined
6125 : providerThreadForRun(mergeBackSourceProjection, mergeBackSourceRun);
6126 if (pendingMergeBackTransfer !== undefined && mergeBackSourceProviderThread === undefined) {
6127 return yield* new OrchestratorDispatchError({
6128 commandId: command.commandId,
6129 commandType: command.type,
6130 cause: `Pending merge-back transfer ${pendingMergeBackTransfer.id} has no resolvable source provider thread.`,
6131 });
6132 }
6133 if (pendingMergeBackTransfer !== undefined) {
6134 yield* enforceCommandPolicy(command)(
6135 commandPolicy.ensureContextHandoff({
6136 commandId: command.commandId,
6137 threadId: command.threadId,
6138 providerInstanceId: modelSelection.instanceId,
6139 capabilities,
6140 strategy: "fork_delta_context",
6141 }),
6142 );
6143 }
6144 const mergeBackDeltaItems =
6145 mergeBackSourceProjection === null || mergeBackSourceRun === null
6146 ? []
6147 : yield* readHandoffItems(
6148 mergeBackSourceProjection.thread.id,
6149 mergeBackSourceProjection.runs
6150 .filter((run) => run.ordinal <= mergeBackSourceRun.ordinal)
6151 .map((run) => run.id),
6152 );
6153 const mergeBackHandoff =
6154 pendingMergeBackTransfer === undefined ||
6155 mergeBackSourceProjection === null ||
6156 mergeBackSourceRun === null ||
6157 mergeBackSourceProviderThread === undefined
6158 ? null
6159 : yield* contextHandoffService
6160 .prepareForkDelta({
6161 sourceThreadId: pendingMergeBackTransfer.sourceThreadId,
6162 targetThreadId: command.threadId,
6163 targetRunId: runId,
6164 transferId: pendingMergeBackTransfer.id,
6165 fromProviderThreadIds: [mergeBackSourceProviderThread.id],
6166 toProviderThreadId: providerThread.id,
6167 fromProviderInstanceId: mergeBackSourceRun.providerInstanceId,
6168 toProviderInstanceId: modelSelection.instanceId,
6169 coveredRunOrdinals: visibleDeltaRunOrdinals(
6170 mergeBackSourceProjection,
6171 mergeBackDeltaItems,
6172 ),
6173 deltaItems: mergeBackDeltaItems,
6174 createdAt: now,
6175 })
6176 .pipe(
6177 Effect.mapError(
6178 (cause) =>
6179 new OrchestratorDispatchError({
6180 commandId: command.commandId,
6181 commandType: command.type,
6182 cause,
6183 }),
6184 ),
6185 );
6186 const checkpointScope = yield* checkpointService
6187 .prepareRootRunScope({
6188 threadId: command.threadId,
6189 runId,
6190 rootNodeId,
6191 providerThreadId: providerThread.id,
6192 cwd:
6193 resolvedRuntimePolicy.cwd ??
6194 existingProviderSession?.cwd ??
6195 projection.thread.worktreePath ??
6196 process.cwd(),
6197 createdAt: now,
6198 })
6199 .pipe(
6200 Effect.mapError(
6201 (cause) =>
6202 new OrchestratorDispatchError({
6203 commandId: command.commandId,
6204 commandType: command.type,
6205 cause,
6206 }),
6207 ),
6208 );
6209 const run: OrchestrationV2Run = {
6210 id: runId,
6211 threadId: command.threadId,
6212 ordinal,
6213 providerInstanceId: modelSelection.instanceId,
6214 modelSelection,
6215 providerThreadId: providerThread.id,
6216 userMessageId: command.messageId,
6217 rootNodeId,
6218 activeAttemptId: attemptId,
6219 status: "starting",
6220 queuePosition: null,
6221 requestedAt: now,
6222 startedAt: null,
6223 completedAt: null,
6224 checkpointId: null,
6225 contextHandoffId:
6226 portableForkHandoff?.id ??
6227 providerSwitchHandoff?.id ??
6228 mergeBackHandoff?.id ??
6229 legacyImportRecoveryHandoff?.id ??
6230 null,
6231 ...(command.sourcePlanRef === undefined ? {} : { sourcePlanRef: command.sourcePlanRef }),
6232 ...(command.restartContinuationOfRunId === undefined
6233 ? {}
6234 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6235 ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
6236 ...wakeWorkStartedAt(projection.runs, command),
6237 };
6238 const attempt: OrchestrationV2RunAttempt = {
6239 id: attemptId,
6240 runId,
6241 attemptOrdinal: 1,
6242 rootNodeId,
6243 providerInstanceId: modelSelection.instanceId,
6244 providerThreadId: providerThread.id,
6245 providerTurnId: null,
6246 reason: "initial",
6247 status: "pending",
6248 startedAt: null,
6249 completedAt: null,
6250 };
6251 const rootNode: OrchestrationV2ExecutionNode = {
6252 id: rootNodeId,
6253 threadId: command.threadId,
6254 runId,
6255 parentNodeId: null,
6256 rootNodeId,
6257 kind: "root_turn",
6258 status: "pending",
6259 countsForRun: true,
6260 providerThreadId: providerThread.id,
6261 providerTurnId: null,
6262 nativeItemRef: null,
6263 runtimeRequestId: null,
6264 checkpointScopeId: checkpointScope.id,
6265 startedAt: null,
6266 completedAt: null,
6267 };
6268 const message: OrchestrationV2ConversationMessage = {
6269 createdBy: command.createdBy,
6270 creationSource: command.creationSource,
6271 ...(command.scheduledTaskId === undefined
6272 ? {}
6273 : { scheduledTaskId: command.scheduledTaskId }),
6274 ...(command.senderThreadId === undefined ? {} : { senderThreadId: command.senderThreadId }),
6275 id: command.messageId,
6276 threadId: command.threadId,
6277 runId,
6278 nodeId: rootNodeId,
6279 role: "user",
6280 text: dispatchText,
6281 ...(command.context ? { context: command.context } : {}),
6282 attachments: command.attachments,
6283 streaming: false,
6284 createdAt: now,
6285 updatedAt: now,
6286 ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }),
6287 ...(command.notification === undefined ? {} : { notification: command.notification }),
6288 };
6289 const turnItem: OrchestrationV2TurnItem = {
6290 createdBy: command.createdBy,
6291 creationSource: command.creationSource,
6292 ...(command.scheduledTaskId === undefined
6293 ? {}
6294 : { scheduledTaskId: command.scheduledTaskId }),
6295 ...(command.senderThreadId === undefined ? {} : { senderThreadId: command.senderThreadId }),
6296 id: idAllocator.derive.userTurnItem({ messageId: command.messageId }),
6297 threadId: command.threadId,
6298 runId,
6299 nodeId: rootNodeId,
6300 providerThreadId: providerThread.id,
6301 providerTurnId: null,
6302 nativeItemRef: null,
6303 parentItemId: null,
6304 ordinal: ordinal * 100,
6305 status: "completed",
6306 title: null,
6307 startedAt: now,
6308 completedAt: now,
6309 updatedAt: now,
6310 type: "user_message",
6311 messageId: command.messageId,
6312 inputIntent: "turn_start",
6313 text: dispatchText,
6314 ...(command.context ? { context: command.context } : {}),
6315 attachments: command.attachments,
6316 };
6317 const activeHandoff = portableForkHandoff ?? mergeBackHandoff ?? providerSwitchHandoff;
6318 const handoffSourceRuns =
6319 portableForkHandoff !== null
6320 ? sourceRun === null
6321 ? []
6322 : [sourceRun]
6323 : providerSwitchHandoff === null
6324 ? mergeBackSourceRun === null
6325 ? []
6326 : [mergeBackSourceRun]
6327 : providerSwitchCoveredRuns;
6328 const handoffFromModelSelections = Array.from(
6329 new Map(
6330 handoffSourceRuns.map((run) => [
6331 `${run.modelSelection.instanceId}\0${run.modelSelection.model}`,
6332 run.modelSelection,
6333 ]),
6334 ).values(),
6335 );
6336 const handoffTurnItem: OrchestrationV2TurnItem | null =
6337 activeHandoff === null
6338 ? null
6339 : {
6340 id: idAllocator.derive.runSignalTurnItem({
6341 runId,
6342 signal: `context-handoff:${activeHandoff.id}`,
6343 }),
6344 threadId: command.threadId,
6345 runId,
6346 nodeId: rootNodeId,
6347 providerThreadId: providerThread.id,
6348 providerTurnId: null,
6349 nativeItemRef: null,
6350 parentItemId: null,
6351 ordinal: ordinal * 100 - 1,
6352 status: "completed",
6353 title:
6354 portableForkHandoff !== null
6355 ? "Fork context"
6356 : providerSwitchHandoff !== null
6357 ? "Provider handoff"
6358 : "Merge-back context",
6359 startedAt: now,
6360 completedAt: now,
6361 updatedAt: now,
6362 type: "handoff",
6363 contextHandoffId: activeHandoff.id,
6364 fromProviderThreadIds: activeHandoff.fromProviderThreadIds,
6365 toProviderThreadId: activeHandoff.toProviderThreadId,
6366 fromProviderInstanceIds: Array.from(
6367 new Set(handoffSourceRuns.map((run) => run.providerInstanceId)),
6368 ),
6369 toProviderInstanceId: modelSelection.instanceId,
6370 fromModelSelections: handoffFromModelSelections,
6371 toModel: modelSelection.model,
6372 strategy: activeHandoff.strategy,
6373 summary: activeHandoff.summaryText,
6374 };
6375 const nativeForkResolution: OrchestrationV2ContextTransferResolution | null =
6376 !canResolveForkNatively || providerThread.nativeThreadRef === null
6377 ? null
6378 : {
6379 strategy: "native_fork",
6380 providerThreadRef: providerThread.nativeThreadRef,
6381 };
6382 const portableForkResolution: OrchestrationV2ContextTransferResolution | null =
6383 pendingForkTransfer === undefined || portableForkHandoff === null
6384 ? null
6385 : {
6386 strategy: "portable_context",
6387 contextHandoffId: portableForkHandoff.id,
6388 };
6389 const mergeBackResolution: OrchestrationV2ContextTransferResolution | null =
6390 pendingMergeBackTransfer === undefined || mergeBackHandoff === null
6391 ? null
6392 : {
6393 strategy: "fork_delta_context",
6394 contextHandoffId: mergeBackHandoff.id,
6395 };
6396
6397 if (pendingForkTransfer !== undefined && canResolveForkNatively) {
6398 yield* emitEvent({
6399 type: "context-transfer.updated",
6400 threadId: command.threadId,
6401 runId,
6402 providerInstanceId: modelSelection.instanceId,
6403 occurredAt: now,
6404 payload: {
6405 ...pendingForkTransfer,
6406 targetProviderInstanceId: modelSelection.instanceId,
6407 targetRunId: runId,
6408 status: "pending",
6409 resolution: null,
6410 error: null,
6411 updatedAt: now,
6412 },
6413 });
6414 }
6415 if (pendingForkTransfer !== undefined && portableForkResolution !== null) {
6416 yield* emitEvent({
6417 type: "context-transfer.updated",
6418 threadId: command.threadId,
6419 runId,
6420 providerInstanceId: modelSelection.instanceId,
6421 occurredAt: now,
6422 payload: {
6423 ...pendingForkTransfer,
6424 targetProviderInstanceId: modelSelection.instanceId,
6425 targetRunId: runId,
6426 status: "resolved_portable",
6427 resolution: portableForkResolution,
6428 error: null,
6429 updatedAt: now,
6430 },
6431 });
6432 }
6433 yield* emitEvent({
6434 type: "provider-thread.updated",
6435 threadId: command.threadId,
6436 providerInstanceId: modelSelection.instanceId,
6437 occurredAt: now,
6438 payload: providerThread,
6439 });
6440 if (portableForkHandoff !== null) {
6441 yield* emitEvent({
6442 type: "context-handoff.updated",
6443 threadId: command.threadId,
6444 runId,
6445 providerInstanceId: modelSelection.instanceId,
6446 occurredAt: now,
6447 payload: portableForkHandoff,
6448 });
6449 }
6450 if (legacyImportRecoveryHandoff !== null) {
6451 yield* emitEvent({
6452 type: "context-handoff.updated",
6453 threadId: command.threadId,
6454 runId,
6455 providerInstanceId: modelSelection.instanceId,
6456 occurredAt: now,
6457 payload: legacyImportRecoveryHandoff,
6458 });
6459 }
6460 if (
6461 providerSwitchTransferId !== null &&
6462 providerSwitchHandoff !== null &&
6463 latestHandoffRun !== undefined
6464 ) {
6465 const transfer: OrchestrationV2ContextTransfer = {
6466 id: providerSwitchTransferId,
6467 type: "provider_handoff",
6468 sourceThreadId: command.threadId,
6469 targetThreadId: command.threadId,
6470 sourcePoint: contextSourcePointForRun(projection, latestHandoffRun),
6471 basePoint:
6472 requiresFullProviderSwitchContext || targetLastCompletedRun === undefined
6473 ? null
6474 : contextSourcePointForRun(projection, targetLastCompletedRun),
6475 sourceProviderInstanceId: latestHandoffRun.providerInstanceId,
6476 targetProviderInstanceId: modelSelection.instanceId,
6477 targetRunId: runId,
6478 status: "consumed",
6479 resolution: {
6480 strategy:
6481 providerSwitchHandoff.strategy === "full_thread_summary"
6482 ? "portable_context"
6483 : "delta_context",
6484 contextHandoffId: providerSwitchHandoff.id,
6485 },
6486 createdBy: command.createdBy,
6487 error: null,
6488 createdAt: now,
6489 updatedAt: now,
6490 consumedAt: now,
6491 };
6492 yield* emitEvent({
6493 type: "context-transfer.created",
6494 threadId: command.threadId,
6495 runId,
6496 providerInstanceId: modelSelection.instanceId,
6497 occurredAt: now,
6498 payload: transfer,
6499 });
6500 yield* emitEvent({
6501 type: "context-handoff.updated",
6502 threadId: command.threadId,
6503 runId,
6504 providerInstanceId: modelSelection.instanceId,
6505 occurredAt: now,
6506 payload: providerSwitchHandoff,
6507 });
6508 }
6509 if (mergeBackHandoff !== null) {
6510 yield* emitEvent({
6511 type: "context-handoff.updated",
6512 threadId: command.threadId,
6513 runId,
6514 providerInstanceId: modelSelection.instanceId,
6515 occurredAt: now,
6516 payload: mergeBackHandoff,
6517 });
6518 }
6519 for (const supersededTransfer of supersededMergeBackTransfers) {
6520 yield* emitEvent({
6521 type: "context-transfer.updated",
6522 threadId: command.threadId,
6523 runId,
6524 providerInstanceId: modelSelection.instanceId,
6525 occurredAt: now,
6526 payload: {
6527 ...supersededTransfer,
6528 status: "superseded",
6529 error:
6530 pendingMergeBackTransfer === undefined
6531 ? "Superseded while consuming merge-back transfer."
6532 : `Superseded by merge-back transfer ${pendingMergeBackTransfer.id}.`,
6533 updatedAt: now,
6534 },
6535 });
6536 }
6537 if (pendingMergeBackTransfer !== undefined && mergeBackResolution !== null) {
6538 yield* emitEvent({
6539 type: "context-transfer.updated",
6540 threadId: command.threadId,
6541 runId,
6542 providerInstanceId: modelSelection.instanceId,
6543 occurredAt: now,
6544 payload: {
6545 ...pendingMergeBackTransfer,
6546 targetProviderInstanceId: modelSelection.instanceId,
6547 targetRunId: runId,
6548 status: "consumed",
6549 resolution: mergeBackResolution,
6550 error: null,
6551 updatedAt: now,
6552 consumedAt: now,
6553 },
6554 });
6555 }
6556 yield* emitEvent({
6557 type: "run.created",
6558 threadId: command.threadId,
6559 runId,
6560 nodeId: rootNodeId,
6561 providerInstanceId: modelSelection.instanceId,
6562 occurredAt: now,
6563 payload: run,
6564 });
6565 yield* completeSourcePlan(now);
6566 yield* emitEvent({
6567 type: "run-attempt.created",
6568 threadId: command.threadId,
6569 runId,
6570 nodeId: rootNodeId,
6571 providerInstanceId: modelSelection.instanceId,
6572 occurredAt: now,
6573 payload: attempt,
6574 });
6575 yield* emitEvent({
6576 type: "node.updated",
6577 threadId: command.threadId,
6578 runId,
6579 nodeId: rootNodeId,
6580 providerInstanceId: modelSelection.instanceId,
6581 occurredAt: now,
6582 payload: rootNode,
6583 });
6584 yield* emitEvent({
6585 type: "checkpoint-scope.created",
6586 threadId: command.threadId,
6587 runId,
6588 nodeId: rootNodeId,
6589 providerInstanceId: modelSelection.instanceId,
6590 occurredAt: now,
6591 payload: yield* checkpointService.ensureScope(checkpointScope).pipe(
6592 Effect.mapError(
6593 (cause) =>
6594 new OrchestratorDispatchError({
6595 commandId: command.commandId,
6596 commandType: command.type,
6597 cause,
6598 }),
6599 ),
6600 ),
6601 });
6602 if (handoffTurnItem !== null) {
6603 yield* emitEvent({
6604 type: "turn-item.updated",
6605 threadId: command.threadId,
6606 runId,
6607 nodeId: rootNodeId,
6608 providerInstanceId: modelSelection.instanceId,
6609 occurredAt: now,
6610 payload: handoffTurnItem,
6611 });
6612 }
6613 yield* emitEvent({
6614 type: "message.updated",
6615 threadId: command.threadId,
6616 runId,
6617 nodeId: rootNodeId,
6618 providerInstanceId: modelSelection.instanceId,
6619 occurredAt: now,
6620 payload: message,
6621 });
6622 yield* emitEvent({
6623 type: "turn-item.updated",
6624 threadId: command.threadId,
6625 runId,
6626 nodeId: rootNodeId,
6627 providerInstanceId: modelSelection.instanceId,
6628 occurredAt: now,
6629 payload: notificationTurnItem(turnItem, message, projection.subagents),
6630 });
6631 const forkResolution = nativeForkResolution ?? portableForkResolution;
6632 if (pendingForkTransfer !== undefined && forkResolution !== null) {
6633 yield* emitEvent({
6634 type: "context-transfer.updated",
6635 threadId: command.threadId,
6636 runId,
6637 providerInstanceId: modelSelection.instanceId,
6638 occurredAt: now,
6639 payload: {
6640 ...pendingForkTransfer,
6641 targetProviderInstanceId: modelSelection.instanceId,
6642 targetRunId: runId,
6643 status: "consumed",
6644 resolution: forkResolution,
6645 error: null,
6646 updatedAt: now,
6647 consumedAt: now,
6648 },
6649 });
6650 }
6651
6652 if (canResumeAcrossInstances && activeProviderThread.providerSessionId !== null) {
6653 const previousProviderSessionId = activeProviderThread.providerSessionId;
6654 yield* emitEvent({
6655 type: "provider-session.detached",
6656 threadId: command.threadId,
6657 driver: activeProviderThread.driver,
6658 providerInstanceId: activeProviderThread.providerInstanceId,
6659 occurredAt: now,
6660 payload: {
6661 providerSessionId: previousProviderSessionId,
6662 detachedAt: now,
6663 reason: "Provider account changed; continuing the native thread.",
6664 },
6665 });
6666 yield* Ref.update(effects, (existing) => [
6667 ...existing,
6668 {
6669 id: `effect:${command.commandId}:provider-session.detach:${previousProviderSessionId}`,
6670 commandId: command.commandId,
6671 threadId: command.threadId,
6672 request: {
6673 type: "provider-session.detach",
6674 providerSessionId: previousProviderSessionId,
6675 detail: "Provider account changed; continuing the native thread.",
6676 },
6677 } satisfies PendingOrchestrationEffectV2,
6678 ]);
6679 }
6680 const pendingEffect = {
6681 id: `effect:${command.commandId}:provider-turn.start:${runId}`,
6682 commandId: command.commandId,
6683 threadId: command.threadId,
6684 request: { type: "provider-turn.start", runId },
6685 } satisfies PendingOrchestrationEffectV2;
6686 yield* Ref.update(effects, (existing) => [...existing, pendingEffect]);
6687 });
6688
6689 const dispatchDelegatedTaskRequest = Effect.fn("orchestrationV2.dispatch.delegatedTaskRequest")(
6690 function* (
6691 command: Extract<OrchestrationV2Command, { readonly type: "delegated_task.request" }>,
6692 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
6693 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
6694 ) {
6695 const parentProjection = yield* projectionStore
6696 .getThreadRecords(command.parentThreadId, [
6697 "runs",
6698 "nodes",
6699 "subagents",
6700 "providerThreads",
6701 "providerTurns",
6702 "attempts",
6703 ])
6704 .pipe(
6705 Effect.mapError(
6706 (cause) =>
6707 new OrchestratorProjectionError({
6708 threadId: command.parentThreadId,
6709 cause,
6710 }),
6711 ),
6712 );
6713 const parentRun = parentProjection.runs.find(
6714 (candidate) => candidate.id === command.parentRunId,
6715 );
6716 if (parentRun === undefined || !isBlockingRun(parentRun)) {
6717 return yield* new OrchestratorDispatchError({
6718 commandId: command.commandId,
6719 commandType: command.type,
6720 cause: `Parent run ${command.parentRunId} is not active.`,
6721 });
6722 }
6723 if (yield* stopReachedRun(command, command.parentThreadId, parentRun.id)) {
6724 return yield* new OrchestratorDispatchError({
6725 commandId: command.commandId,
6726 commandType: command.type,
6727 cause: `Parent run ${command.parentRunId} is stopping.`,
6728 });
6729 }
6730 const parentNode = parentProjection.nodes.find(
6731 (candidate) => candidate.id === command.parentNodeId,
6732 );
6733 if (
6734 parentNode === undefined ||
6735 parentNode.runId !== parentRun.id ||
6736 parentRun.rootNodeId === null
6737 ) {
6738 return yield* new OrchestratorDispatchError({
6739 commandId: command.commandId,
6740 commandType: command.type,
6741 cause: `Parent node ${command.parentNodeId} is not part of active run ${parentRun.id}.`,
6742 });
6743 }
6744
6745 const targetAdapter = yield* providerAdapters.get(command.modelSelection.instanceId).pipe(
6746 Effect.mapError(
6747 (cause) =>
6748 new OrchestratorProviderAdapterError({
6749 commandId: command.commandId,
6750 providerInstanceId: command.modelSelection.instanceId,
6751 cause,
6752 }),
6753 ),
6754 );
6755
6756 const now = command.createdAt ?? (yield* DateTime.now);
6757 const taskNodeId = idAllocator.derive.delegatedTaskNode({
6758 commandId: command.commandId,
6759 });
6760 const childThreadId = idAllocator.derive.delegatedTaskThread({
6761 commandId: command.commandId,
6762 });
6763 const childMessageId = idAllocator.derive.delegatedTaskMessage({
6764 commandId: command.commandId,
6765 });
6766 const taskTurnItemId = idAllocator.derive.delegatedTaskTurnItem({
6767 commandId: command.commandId,
6768 });
6769 const taskTitle = subagentThreadTitle({
6770 parentTitle: parentProjection.thread.title,
6771 prompt: command.task,
6772 ...(command.title === undefined ? {} : { title: command.title }),
6773 ordinal: parentProjection.subagents.length + 1,
6774 });
6775 const childThread: OrchestrationV2AppThread = {
6776 ...makeSubagentChildThread({
6777 parentThread: parentProjection.thread,
6778 childThreadId,
6779 parentNodeId: taskNodeId,
6780 activeProviderThreadId: null,
6781 providerInstanceId: command.modelSelection.instanceId,
6782 modelSelection: command.modelSelection,
6783 title: taskTitle,
6784 now,
6785 createdBy: command.createdBy,
6786 creationSource: command.creationSource,
6787 }),
6788 runtimeMode: command.runtimeMode,
6789 interactionMode: command.interactionMode,
6790 ...(command.workspaceStrategy === undefined ? {} : { branch: null, worktreePath: null }),
6791 };
6792 const task: OrchestrationV2Subagent = {
6793 id: taskNodeId,
6794 threadId: command.parentThreadId,
6795 runId: parentRun.id,
6796 parentNodeId: command.parentNodeId,
6797 origin: "app_owned",
6798 createdBy: command.createdBy,
6799 driver: targetAdapter.driver,
6800 providerInstanceId: command.modelSelection.instanceId,
6801 providerThreadId: null,
6802 childThreadId,
6803 nativeTaskRef: null,
6804 prompt: command.task,
6805 title: command.title ?? null,
6806 model: command.modelSelection.model,
6807 ...(command.completionWake === undefined ? {} : { completionWake: command.completionWake }),
6808 status: "running",
6809 result: null,
6810 startedAt: now,
6811 completedAt: null,
6812 updatedAt: now,
6813 };
6814 const taskNode: OrchestrationV2ExecutionNode = {
6815 id: taskNodeId,
6816 threadId: command.parentThreadId,
6817 runId: parentRun.id,
6818 parentNodeId: command.parentNodeId,
6819 rootNodeId: parentRun.rootNodeId,
6820 kind: "subagent",
6821 status: "running",
6822 countsForRun: false,
6823 providerThreadId: null,
6824 providerTurnId: null,
6825 nativeItemRef: null,
6826 runtimeRequestId: null,
6827 checkpointScopeId: null,
6828 startedAt: now,
6829 completedAt: null,
6830 };
6831 const parentProviderTurn = providerTurnForRun(parentProjection, parentRun);
6832 const taskTurnItem: OrchestrationV2TurnItem = {
6833 id: taskTurnItemId,
6834 threadId: command.parentThreadId,
6835 runId: parentRun.id,
6836 nodeId: taskNodeId,
6837 providerThreadId: parentRun.providerThreadId,
6838 providerTurnId: parentProviderTurn?.id ?? null,
6839 nativeItemRef: null,
6840 parentItemId: null,
6841 ordinal: yield* nextTurnItemOrdinal(parentProjection),
6842 status: "running",
6843 title: command.title ?? taskTitle,
6844 startedAt: now,
6845 completedAt: null,
6846 updatedAt: now,
6847 type: "subagent",
6848 subagentId: taskNodeId,
6849 origin: "app_owned",
6850 driver: targetAdapter.driver,
6851 providerInstanceId: command.modelSelection.instanceId,
6852 childThreadId,
6853 prompt: command.task,
6854 result: null,
6855 };
6856 const emitEvent = emit(events, command);
6857
6858 yield* emitEvent({
6859 type: "thread.created",
6860 threadId: childThreadId,
6861 driver: targetAdapter.driver,
6862 providerInstanceId: command.modelSelection.instanceId,
6863 occurredAt: now,
6864 payload: childThread,
6865 });
6866 yield* emitEvent({
6867 type: "node.updated",
6868 threadId: command.parentThreadId,
6869 runId: parentRun.id,
6870 nodeId: taskNodeId,
6871 driver: targetAdapter.driver,
6872 providerInstanceId: command.modelSelection.instanceId,
6873 occurredAt: now,
6874 payload: taskNode,
6875 });
6876 yield* emitEvent({
6877 type: "subagent.updated",
6878 threadId: command.parentThreadId,
6879 runId: parentRun.id,
6880 nodeId: taskNodeId,
6881 driver: targetAdapter.driver,
6882 providerInstanceId: command.modelSelection.instanceId,
6883 occurredAt: now,
6884 payload: task,
6885 });
6886 yield* emitEvent({
6887 type: "turn-item.updated",
6888 threadId: command.parentThreadId,
6889 runId: parentRun.id,
6890 nodeId: taskNodeId,
6891 driver: targetAdapter.driver,
6892 providerInstanceId: command.modelSelection.instanceId,
6893 occurredAt: now,
6894 payload: taskTurnItem,
6895 });
6896
6897 const childMessageCommand = {
6898 type: "message.dispatch",
6899 createdBy: command.createdBy,
6900 creationSource: command.creationSource,
6901 commandId: command.commandId,
6902 threadId: childThreadId,
6903 senderThreadId: command.parentThreadId,
6904 messageId: childMessageId,
6905 text: command.task,
6906 attachments: [],
6907 modelSelection: command.modelSelection,
6908 dispatchMode:
6909 command.workspaceStrategy === undefined
6910 ? { type: "start_immediately" }
6911 : { type: "defer_start", workspaceStrategy: command.workspaceStrategy },
6912 } satisfies Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>;
6913 yield* dispatchMessage(childMessageCommand, events, effects);
6914
6915 const childProjection = yield* getProjectionWithPendingEvents(childThreadId, events);
6916 const childRun = childProjection.runs[0];
6917 if (childRun === undefined) {
6918 return yield* new OrchestratorDispatchError({
6919 commandId: command.commandId,
6920 commandType: command.type,
6921 cause: `Delegated child thread ${childThreadId} did not create a run.`,
6922 });
6923 }
6924 const spawnTransferId = yield* mapDispatchError(command)(
6925 idAllocator.allocate.contextTransfer({
6926 sourceThreadId: command.parentThreadId,
6927 targetThreadId: childThreadId,
6928 type: "subagent_spawn",
6929 }),
6930 );
6931 const spawnTransfer: OrchestrationV2ContextTransfer = {
6932 id: spawnTransferId,
6933 type: "subagent_spawn",
6934 sourceThreadId: command.parentThreadId,
6935 targetThreadId: childThreadId,
6936 sourcePoint: {
6937 ...contextSourcePointForRun(parentProjection, parentRun),
6938 turnItemId: taskTurnItemId,
6939 },
6940 basePoint: null,
6941 sourceProviderInstanceId: parentRun.providerInstanceId,
6942 targetProviderInstanceId: command.modelSelection.instanceId,
6943 targetRunId: childRun.id,
6944 status: "consumed",
6945 resolution: null,
6946 createdBy: command.createdBy,
6947 error: null,
6948 createdAt: now,
6949 updatedAt: now,
6950 consumedAt: now,
6951 };
6952 yield* emitEvent({
6953 type: "context-transfer.created",
6954 threadId: childThreadId,
6955 runId: childRun.id,
6956 providerInstanceId: command.modelSelection.instanceId,
6957 occurredAt: now,
6958 payload: spawnTransfer,
6959 });
6960 },
6961 );
6962
6963 // A follow-up is new requested work in the same child conversation. Give it
6964 // its own task so results, acknowledgments, Stop, and recovery use the same
6965 // lifecycle as a fresh delegation without replacing the original result.
6966 const recordSubagentFollowup = Effect.fnUntraced(function* (
6967 command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
6968 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
6969 ) {
6970 if (
6971 command.scheduledTaskId !== undefined ||
6972 command.notification !== undefined ||
6973 command.restartContinuationOfRunId !== undefined ||
6974 (command.createdBy !== "user" && command.creationSource !== "mcp")
6975 )
6976 return;
6977 const pending = yield* Ref.get(events);
6978 const message = pending
6979 .flatMap((event) =>
6980 event.type === "message.updated" && event.payload.id === command.messageId
6981 ? [event.payload]
6982 : [],
6983 )
6984 .at(-1);
6985 if (message?.runId == null) return;
6986 const child = yield* projectionStore.getThread(command.threadId);
6987 const parentThreadId = child.lineage.parentThreadId;
6988 if (
6989 child.lineage.relationshipToParent !== "subagent" ||
6990 parentThreadId === null ||
6991 (command.createdBy !== "user" && command.senderThreadId !== parentThreadId)
6992 )
6993 return;
6994 const run =
6995 pending
6996 .flatMap((event) =>
6997 (event.type === "run.created" || event.type === "run.updated") &&
6998 event.payload.id === message.runId
6999 ? [event.payload]
7000 : [],
7001 )
7002 .at(-1) ??
7003 (yield* projectionStore.getThreadRecords(command.threadId, ["runs"])).runs.find(
7004 (row) => row.id === message.runId,
7005 );
7006 if (run === undefined || run.delegatedTaskId !== undefined) return;
7007 const parent = yield* projectionStore
7008 .getThreadRecords(parentThreadId, ["runs", "subagents", "providerTurns", "attempts"])
7009 .pipe(mapDispatchError(command));
7010 const original = parent.subagents.find(
7011 (task) =>
7012 task.origin === "app_owned" &&
7013 task.childThreadId === child.id &&
7014 task.id === (child.forkedFrom?.type === "node" ? child.forkedFrom.nodeId : undefined),
7015 );
7016 // Before first publication, follow-ups remain part of the original task.
7017 if (original === undefined || original.result === null) return;
7018 const owner =
7019 parent.runs.filter(isBlockingRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ??
7020 latestExecutedRun(parent.runs);
7021 if (owner === null || owner.rootNodeId === null) return;
7022 const now = message.createdAt;
7023 const adapter = yield* providerAdapters.get(run.providerInstanceId);
7024 const taskId = idAllocator.derive.delegatedTaskNode({ commandId: command.commandId });
7025 const task: OrchestrationV2Subagent = {
7026 ...original,
7027 createdBy: command.createdBy,
7028 driver: adapter.driver,
7029 id: taskId,
7030 runId: owner.id,
7031 parentNodeId: owner.rootNodeId,
7032 providerInstanceId: run.providerInstanceId,
7033 providerThreadId: run.providerThreadId,
7034 model: run.modelSelection.model,
7035 modelSelection: run.modelSelection,
7036 prompt: command.text,
7037 title: child.title,
7038 completionWake: "always",
7039 completionDelivery: { state: "pending", observedByRunId: null },
7040 status: run.status === "queued" ? "pending" : "running",
7041 progress: undefined,
7042 result: null,
7043 startedAt: now,
7044 completedAt: null,
7045 updatedAt: now,
7046 };
7047 const node: OrchestrationV2ExecutionNode = {
7048 id: taskId,
7049 threadId: parentThreadId,
7050 runId: owner.id,
7051 parentNodeId: owner.rootNodeId,
7052 rootNodeId: owner.rootNodeId,
7053 kind: "subagent",
7054 status: task.status,
7055 countsForRun: false,
7056 providerThreadId: run.providerThreadId,
7057 providerTurnId: null,
7058 nativeItemRef: null,
7059 runtimeRequestId: null,
7060 checkpointScopeId: null,
7061 startedAt: now,
7062 completedAt: null,
7063 };
7064 const turnItem: OrchestrationV2TurnItem = {
7065 id: idAllocator.derive.delegatedTaskTurnItem({ commandId: command.commandId }),
7066 threadId: parentThreadId,
7067 runId: owner.id,
7068 nodeId: taskId,
7069 providerThreadId: owner.providerThreadId,
7070 providerTurnId: providerTurnForRun(parent, owner)?.id ?? null,
7071 nativeItemRef: null,
7072 parentItemId: null,
7073 ordinal: yield* nextTurnItemOrdinal(parent),
7074 status: task.status,
7075 title: child.title,
7076 startedAt: now,
7077 completedAt: null,
7078 updatedAt: now,
7079 type: "subagent",
7080 subagentId: taskId,
7081 origin: "app_owned",
7082 driver: task.driver,
7083 providerInstanceId: task.providerInstanceId,
7084 childThreadId: child.id,
7085 prompt: command.text,
7086 result: null,
7087 };
7088 const emitEvent = emit(events, command);
7089 yield* emitEvent({
7090 type: "run.updated",
7091 threadId: child.id,
7092 runId: run.id,
7093 occurredAt: now,
7094 payload: { ...run, delegatedTaskId: taskId },
7095 });
7096 yield* emitEvent({
7097 type: "node.updated",
7098 threadId: parentThreadId,
7099 runId: owner.id,
7100 nodeId: taskId,
7101 occurredAt: now,
7102 payload: node,
7103 });
7104 yield* emitEvent({
7105 type: "subagent.updated",
7106 threadId: parentThreadId,
7107 runId: owner.id,
7108 nodeId: taskId,
7109 occurredAt: now,
7110 payload: task,
7111 });
7112 yield* emitEvent({
7113 type: "turn-item.updated",
7114 threadId: parentThreadId,
7115 runId: owner.id,
7116 nodeId: taskId,
7117 occurredAt: now,
7118 payload: turnItem,
7119 });
7120 });
7121
7122 // Rewrites a delegated task's completionWake after creation. The wait path
7123 // uses this when its blocking window ends without a terminal (timeout), so
7124 // a child that later terminalizes mid-parent-turn still wakes the parent.
7125 // Runs under the parent thread's dispatch lock, which is also what finalize
7126 // takes for its parent-side writes, so the two never interleave on this row.
7127 const dispatchDelegatedTaskWakePolicy = Effect.fn(
7128 "orchestrationV2.dispatch.delegatedTaskWakePolicy",
7129 )(function* (
7130 command: Extract<OrchestrationV2Command, { readonly type: "delegated_task.wake-policy" }>,
7131 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7132 ) {
7133 const parentProjection = yield* projectionStore
7134 .getThreadRecords(command.parentThreadId, ["subagents", "runs", "messages", "providerTurns"])
7135 .pipe(
7136 Effect.mapError(
7137 (cause) =>
7138 new OrchestratorProjectionError({
7139 threadId: command.parentThreadId,
7140 cause,
7141 }),
7142 ),
7143 );
7144 const task = parentProjection.subagents.find(
7145 (candidate) => candidate.id === command.taskId && candidate.origin === "app_owned",
7146 );
7147 // No-op commands reject with a descriptive cause, matching the thread
7148 // mutation handlers ("already archived", "not archived").
7149 if (task === undefined) {
7150 return yield* new OrchestratorDispatchError({
7151 commandId: command.commandId,
7152 commandType: command.type,
7153 cause: `Delegated task ${command.taskId} is not an app-owned task of thread ${command.parentThreadId}.`,
7154 });
7155 }
7156 if (task.completionWake === command.completionWake) {
7157 return yield* new OrchestratorDispatchError({
7158 commandId: command.commandId,
7159 commandType: command.type,
7160 cause: `Delegated task ${command.taskId} already wakes the parent with completionWake ${command.completionWake}.`,
7161 });
7162 }
7163 const now = yield* DateTime.now;
7164 const emitEvent = emit(events, command);
7165 const updatedTask: OrchestrationV2Subagent = {
7166 ...task,
7167 completionWake: command.completionWake,
7168 updatedAt: now,
7169 };
7170 // A non-terminal task needs no offer here: finalize reads the upgraded
7171 // policy when the child terminalizes. Both writers hold this parent lock,
7172 // so a terminal task means finalize already committed the terminal row and
7173 // already made its offer decision under the pre-upgrade policy: under
7174 // settled_only it offered iff the spawning run was not live. Plan a
7175 // delivery only when that run is live now, which is precisely the case
7176 // where finalize skipped. The mailbox steers a capable active session or
7177 // queues behind that run. When that run is not live, finalize already
7178 // offered and a second offer would wake the parent twice. (If the parent settled in between,
7179 // this skips a wake that finalize also skipped; a missed wake is cheaper
7180 // than a duplicate one, and the result is already in the projection.)
7181 const parentRun =
7182 task.runId === null
7183 ? undefined
7184 : parentProjection.runs.find((candidate) => candidate.id === task.runId);
7185 const completionPlan =
7186 command.completionWake === "always" &&
7187 isTerminalDelegatedTaskStatus(task.status) &&
7188 parentRun !== undefined &&
7189 hasLiveRun({ runs: [parentRun] })
7190 ? yield* planDelegatedCompletionDelivery({
7191 parentProjection,
7192 parentRun,
7193 task: updatedTask,
7194 updatedTask,
7195 now,
7196 })
7197 : undefined;
7198 yield* emitEvent({
7199 type: "subagent.updated",
7200 threadId: command.parentThreadId,
7201 ...(task.runId === null ? {} : { runId: task.runId }),
7202 nodeId: task.id,
7203 driver: task.driver,
7204 providerInstanceId: task.providerInstanceId,
7205 occurredAt: now,
7206 payload: completionPlan?.task ?? updatedTask,
7207 });
7208 if (completionPlan === undefined) {
7209 return;
7210 }
7211 if (completionPlan.parentRun !== undefined) {
7212 yield* emitEvent({
7213 type: "run.updated",
7214 threadId: command.parentThreadId,
7215 runId: completionPlan.parentRun.id,
7216 ...(completionPlan.parentRun.rootNodeId === null
7217 ? {}
7218 : { nodeId: completionPlan.parentRun.rootNodeId }),
7219 providerInstanceId: completionPlan.parentRun.providerInstanceId,
7220 occurredAt: now,
7221 payload: completionPlan.parentRun,
7222 });
7223 }
7224 if (completionPlan.message !== undefined) {
7225 yield* emitEvent({
7226 type: "message.updated",
7227 threadId: command.parentThreadId,
7228 ...(completionPlan.message.runId === null ? {} : { runId: completionPlan.message.runId }),
7229 ...(completionPlan.message.nodeId === null
7230 ? {}
7231 : { nodeId: completionPlan.message.nodeId }),
7232 providerInstanceId:
7233 completionPlan.parentRun?.providerInstanceId ??
7234 parentProjection.thread.providerInstanceId,
7235 occurredAt: now,
7236 payload: completionPlan.message,
7237 });
7238 }
7239 });
7240
7241 const dispatchCreatedThreadRecord = Effect.fn("orchestrationV2.dispatch.createdThreadRecord")(
7242 function* (
7243 command: Extract<OrchestrationV2Command, { readonly type: "thread.created.record" }>,
7244 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7245 ) {
7246 const parentProjection = yield* projectionStore
7247 .getThreadRecords(
7248 command.parentThreadId,
7249 ["runs", "nodes", "turnItems", "attempts", "providerTurns"],
7250 { turnItemTypes: [], messageRoles: ["user"] },
7251 )
7252 .pipe(
7253 Effect.mapError(
7254 (cause) =>
7255 new OrchestratorProjectionError({
7256 threadId: command.parentThreadId,
7257 cause,
7258 }),
7259 ),
7260 );
7261 const targetProjection = yield* projectionStore
7262 .getThreadRecords(
7263 command.targetThreadId,
7264 ["runs", "nodes", "turnItems", "attempts", "providerTurns"],
7265 { turnItemTypes: [], messageRoles: ["user"] },
7266 )
7267 .pipe(
7268 Effect.mapError(
7269 (cause) =>
7270 new OrchestratorProjectionError({
7271 threadId: command.targetThreadId,
7272 cause,
7273 }),
7274 ),
7275 );
7276 const parentRun = parentProjection.runs.find(
7277 (candidate) => candidate.id === command.parentRunId,
7278 );
7279 const parentNode = parentProjection.nodes.find(
7280 (candidate) => candidate.id === command.parentNodeId,
7281 );
7282 if (
7283 parentRun === undefined ||
7284 parentNode === undefined ||
7285 parentNode.runId !== command.parentRunId ||
7286 parentRun.rootNodeId !== command.parentNodeId
7287 ) {
7288 return yield* new OrchestratorDispatchError({
7289 commandId: command.commandId,
7290 commandType: command.type,
7291 cause: `Parent node ${command.parentNodeId} is not the root of run ${command.parentRunId}.`,
7292 });
7293 }
7294 if (parentProjection.thread.projectId !== targetProjection.thread.projectId) {
7295 return yield* new OrchestratorDispatchError({
7296 commandId: command.commandId,
7297 commandType: command.type,
7298 cause: `Target thread ${command.targetThreadId} belongs to another project.`,
7299 });
7300 }
7301 if (
7302 command.targetRunId !== null &&
7303 !targetProjection.runs.some((candidate) => candidate.id === command.targetRunId)
7304 ) {
7305 return yield* new OrchestratorDispatchError({
7306 commandId: command.commandId,
7307 commandType: command.type,
7308 cause: `Target run ${command.targetRunId} does not belong to thread ${command.targetThreadId}.`,
7309 });
7310 }
7311
7312 const now = yield* DateTime.now;
7313 const parentProviderTurn = providerTurnForRun(parentProjection, parentRun);
7314 const turnItem: OrchestrationV2TurnItem = {
7315 id: idAllocator.derive.createdThreadTurnItem({ commandId: command.commandId }),
7316 threadId: command.parentThreadId,
7317 runId: command.parentRunId,
7318 nodeId: command.parentNodeId,
7319 providerThreadId: parentRun.providerThreadId,
7320 providerTurnId: parentProviderTurn?.id ?? null,
7321 nativeItemRef: null,
7322 parentItemId: null,
7323 ordinal: yield* nextTurnItemOrdinal(parentProjection),
7324 status: "completed",
7325 title: targetProjection.thread.title,
7326 startedAt: now,
7327 completedAt: now,
7328 updatedAt: now,
7329 type: "thread_created",
7330 targetThreadId: command.targetThreadId,
7331 targetRunId: command.targetRunId,
7332 targetProviderInstanceId: targetProjection.thread.modelSelection.instanceId,
7333 targetModel: targetProjection.thread.modelSelection.model,
7334 };
7335
7336 yield* emit(
7337 events,
7338 command,
7339 )({
7340 type: "turn-item.updated",
7341 threadId: command.parentThreadId,
7342 runId: command.parentRunId,
7343 nodeId: command.parentNodeId,
7344 providerInstanceId: parentRun.providerInstanceId,
7345 occurredAt: now,
7346 payload: turnItem,
7347 });
7348 },
7349 );
7350
7351 /**
7352 * Records or updates the card for a secret an agent asked the user for. The
7353 * item carries the request and its status only; the value goes straight to
7354 * the server's secret store and never through orchestration.
7355 */
7356 const dispatchSecretRequestRecord = Effect.fn("orchestrationV2.dispatch.secretRequestRecord")(
7357 function* (
7358 command: Extract<OrchestrationV2InternalCommand, { readonly type: "secret_request.record" }>,
7359 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7360 ) {
7361 const projection = yield* projectionStore
7362 .getThreadRecords(
7363 command.threadId,
7364 ["runs", "nodes", "turnItems", "attempts", "providerTurns"],
7365 { turnItemTypes: ["secret_request"], messageRoles: [] },
7366 )
7367 .pipe(
7368 Effect.mapError(
7369 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
7370 ),
7371 );
7372 const run = projection.runs.find((candidate) => candidate.id === command.runId);
7373 if (run === undefined || run.rootNodeId !== command.nodeId) {
7374 return yield* new OrchestratorDispatchError({
7375 commandId: command.commandId,
7376 commandType: command.type,
7377 cause: `Node ${command.nodeId} is not the root of run ${command.runId}.`,
7378 });
7379 }
7380 const existing = projection.turnItems.find((item) => item.id === command.turnItemId);
7381 if (existing !== undefined && existing.type !== "secret_request") {
7382 return yield* new OrchestratorDispatchError({
7383 commandId: command.commandId,
7384 commandType: command.type,
7385 cause: `Turn item ${command.turnItemId} is not a secret request.`,
7386 });
7387 }
7388 if (existing !== undefined && existing.runId !== command.runId) {
7389 return yield* new OrchestratorDispatchError({
7390 commandId: command.commandId,
7391 commandType: command.type,
7392 cause: `Secret request ${command.turnItemId} belongs to another run.`,
7393 });
7394 }
7395 const now = yield* DateTime.now;
7396 // A request is answered once, and a retry finds its card as it was
7397 // asked: either one records the card unchanged.
7398 const unchanged =
7399 existing !== undefined &&
7400 (existing.secretStatus !== "pending" || command.secretStatus === "pending");
7401 const pending = command.secretStatus === "pending";
7402 const turnItem: OrchestrationV2TurnItem = unchanged
7403 ? existing
7404 : {
7405 id: command.turnItemId,
7406 threadId: command.threadId,
7407 runId: command.runId,
7408 nodeId: command.nodeId,
7409 providerThreadId: run.providerThreadId,
7410 providerTurnId: providerTurnForRun(projection, run)?.id ?? null,
7411 nativeItemRef: null,
7412 parentItemId: null,
7413 ordinal: existing?.ordinal ?? (yield* nextTurnItemOrdinal(projection)),
7414 status: pending
7415 ? "waiting"
7416 : command.secretStatus === "saved"
7417 ? "completed"
7418 : "cancelled",
7419 title: command.label,
7420 startedAt: existing?.startedAt ?? now,
7421 completedAt: pending ? null : now,
7422 updatedAt: now,
7423 type: "secret_request",
7424 label: command.label,
7425 reason: command.reason,
7426 ...(command.placeholder === undefined ? {} : { placeholder: command.placeholder }),
7427 secretStatus: command.secretStatus,
7428 };
7429 yield* emit(
7430 events,
7431 command,
7432 )({
7433 type: "turn-item.updated",
7434 threadId: command.threadId,
7435 runId: command.runId,
7436 nodeId: command.nodeId,
7437 providerInstanceId: run.providerInstanceId,
7438 occurredAt: now,
7439 payload: turnItem,
7440 });
7441 },
7442 );
7443
7444 const dispatchRuntimeRequestRespond = (
7445 command: Extract<OrchestrationV2Command, { readonly type: "runtime-request.respond" }>,
7446 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7447 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
7448 ) =>
7449 Effect.gen(function* () {
7450 const context = yield* projectionStore
7451 .getRuntimeResponseContext(command.threadId, command.requestId)
7452 .pipe(
7453 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
7454 );
7455 const runtimeRequest = context.request;
7456 if (runtimeRequest === undefined) {
7457 return yield* new OrchestratorDispatchError({
7458 commandId: command.commandId,
7459 commandType: command.type,
7460 cause: `Runtime request ${command.requestId} was not found.`,
7461 });
7462 }
7463 if (runtimeRequest.status !== "pending") {
7464 return yield* new OrchestratorDispatchError({
7465 commandId: command.commandId,
7466 commandType: command.type,
7467 cause: `Runtime request ${command.requestId} is ${runtimeRequest.status}.`,
7468 });
7469 }
7470 if (runtimeRequest.responseCapability.type === "not_resumable") {
7471 return yield* new OrchestratorDispatchError({
7472 commandId: command.commandId,
7473 commandType: command.type,
7474 cause: runtimeRequest.responseCapability.reason,
7475 });
7476 }
7477 const providerSessionId =
7478 runtimeRequest.responseCapability.type === "live"
7479 ? runtimeRequest.responseCapability.providerSessionId
7480 : null;
7481 const providerSession = context.session;
7482 if (providerSessionId !== null && providerSession === undefined) {
7483 return yield* new OrchestratorDispatchError({
7484 commandId: command.commandId,
7485 commandType: command.type,
7486 cause: `Provider session ${providerSessionId} was not found.`,
7487 });
7488 }
7489
7490 const now = yield* DateTime.now;
7491 const resolvedRequest = {
7492 ...runtimeRequest,
7493 status: "resolved" as const,
7494 resolvedAt: now,
7495 ...(command.decision === undefined ? {} : { decision: command.decision }),
7496 ...(command.answers === undefined ? {} : { answers: command.answers }),
7497 };
7498 const emitEvent = emit(events, command);
7499 const requestNode = context.node;
7500 const resolvedNodeStatus =
7501 command.decision === "decline" || command.decision === "cancel"
7502 ? ("cancelled" as const)
7503 : ("completed" as const);
7504 yield* emitEvent({
7505 type: "runtime-request.updated",
7506 threadId: command.threadId,
7507 ...(requestNode?.runId == null ? {} : { runId: requestNode.runId }),
7508 nodeId: runtimeRequest.nodeId,
7509 ...(providerSession === undefined
7510 ? {}
7511 : {
7512 driver: providerSession.driver,
7513 providerInstanceId: providerSession.providerInstanceId,
7514 }),
7515 occurredAt: now,
7516 payload: resolvedRequest,
7517 });
7518 if (requestNode !== undefined) {
7519 yield* emitEvent({
7520 type: "node.updated",
7521 threadId: command.threadId,
7522 ...(requestNode.runId === null ? {} : { runId: requestNode.runId }),
7523 nodeId: requestNode.id,
7524 ...(providerSession === undefined
7525 ? {}
7526 : {
7527 driver: providerSession.driver,
7528 providerInstanceId: providerSession.providerInstanceId,
7529 }),
7530 occurredAt: now,
7531 payload: {
7532 ...requestNode,
7533 status: resolvedNodeStatus,
7534 completedAt: now,
7535 },
7536 });
7537 }
7538
7539 const approvalTurnItem = context.item;
7540 if (approvalTurnItem !== undefined) {
7541 yield* emitEvent({
7542 type: "turn-item.updated",
7543 threadId: command.threadId,
7544 ...(approvalTurnItem.runId === null ? {} : { runId: approvalTurnItem.runId }),
7545 ...(approvalTurnItem.nodeId === null ? {} : { nodeId: approvalTurnItem.nodeId }),
7546 ...(providerSession === undefined
7547 ? {}
7548 : {
7549 driver: providerSession.driver,
7550 providerInstanceId: providerSession.providerInstanceId,
7551 }),
7552 occurredAt: now,
7553 payload: {
7554 ...approvalTurnItem,
7555 ...(approvalTurnItem.type === "user_input_request" && command.answers !== undefined
7556 ? {
7557 questionAnswer: {
7558 requestId: command.requestId,
7559 answers: command.answers ?? {},
7560 attachmentsByQuestionId: command.attachmentsByQuestionId ?? {},
7561 questionTextById: Object.fromEntries(
7562 approvalTurnItem.questions.map((question) => [
7563 question.id,
7564 question.question,
7565 ]),
7566 ),
7567 },
7568 }
7569 : {}),
7570 status: resolvedNodeStatus,
7571 completedAt: now,
7572 updatedAt: now,
7573 },
7574 });
7575 }
7576 if (runtimeRequest.responseCapability.type === "message") {
7577 if (resolvedNodeStatus === "cancelled") return;
7578 if (approvalTurnItem?.type !== "user_input_request") {
7579 return yield* new OrchestratorDispatchError({
7580 commandId: command.commandId,
7581 commandType: command.type,
7582 cause: "The question for this request was not found.",
7583 });
7584 }
7585 const replies: string[] = [];
7586 for (const question of approvalTurnItem.questions) {
7587 const answer = command.answers?.[question.id];
7588 if (typeof answer !== "string" || answer.trim().length === 0) {
7589 if (question.required === false) continue;
7590 return yield* new OrchestratorDispatchError({
7591 commandId: command.commandId,
7592 commandType: command.type,
7593 cause: "Answer each question before sending.",
7594 });
7595 }
7596 replies.push(`${question.question}\n${answer.trim()}`);
7597 }
7598 if (replies.length === 0) {
7599 return yield* new OrchestratorDispatchError({
7600 commandId: command.commandId,
7601 commandType: command.type,
7602 cause: "Enter an answer before sending.",
7603 });
7604 }
7605 let dispatchMode: Extract<
7606 OrchestrationV2Command,
7607 { type: "message.dispatch" }
7608 >["dispatchMode"] = {
7609 type: "queue_after_active",
7610 };
7611 const {
7612 run: activeRun,
7613 providerThread: activeProviderThread,
7614 providerTurn: activeProviderTurn,
7615 } = yield* projectionStore
7616 .getRunningTurnContext(command.threadId)
7617 .pipe(mapDispatchError(command));
7618 if (activeRun && activeProviderTurn && activeProviderThread?.providerSessionId) {
7619 const activeSession = yield* providerSessions
7620 .get(activeProviderThread.providerSessionId)
7621 .pipe(
7622 Effect.mapError(
7623 (cause) =>
7624 new OrchestratorDispatchError({
7625 commandId: command.commandId,
7626 commandType: command.type,
7627 cause,
7628 }),
7629 ),
7630 );
7631 if (
7632 Option.isSome(activeSession) &&
7633 activeSession.value.providerSession.capabilities.turns.supportsActiveSteering
7634 ) {
7635 dispatchMode = { type: "steer_active", targetRunId: activeRun.id };
7636 }
7637 }
7638 // The resolution and normal message dispatch share one event transaction.
7639 return yield* dispatchMessage(
7640 {
7641 type: "message.dispatch",
7642 commandId: command.commandId,
7643 threadId: command.threadId,
7644 messageId: MessageId.make(`async-answer:${command.requestId}`),
7645 text: replies.join("\n\n"),
7646 attachments: [],
7647 createdBy: "user",
7648 creationSource: "server",
7649 dispatchMode,
7650 },
7651 events,
7652 effects,
7653 );
7654 }
7655 if (providerSessionId === null) return;
7656 yield* Ref.update(effects, (existing) => [
7657 ...existing,
7658 {
7659 id: `effect:${command.commandId}:runtime-request.respond:${command.requestId}`,
7660 commandId: command.commandId,
7661 threadId: command.threadId,
7662 request: {
7663 type: "runtime-request.respond",
7664 providerSessionId,
7665 requestId: command.requestId,
7666 ...(command.decision === undefined ? {} : { decision: command.decision }),
7667 ...(command.answers === undefined ? {} : { answers: command.answers }),
7668 },
7669 } satisfies PendingOrchestrationEffectV2,
7670 ]);
7671 });
7672
7673 const dispatchThreadUserInputDismiss = (
7674 command: Extract<OrchestrationV2Command, { readonly type: "thread.user-input.dismiss" }>,
7675 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7676 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
7677 ) =>
7678 Effect.gen(function* () {
7679 const request = yield* projectionStore
7680 .getRuntimeRequest(command.threadId, command.requestId)
7681 .pipe(
7682 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
7683 );
7684 if (request === undefined || request.status !== "pending") {
7685 return yield* new OrchestratorDispatchError({
7686 commandId: command.commandId,
7687 commandType: command.type,
7688 cause: "This question has already been answered.",
7689 });
7690 }
7691 if (request.kind !== "user_input" || request.responseCapability.type !== "message") {
7692 return yield* new OrchestratorDispatchError({
7693 commandId: command.commandId,
7694 commandType: command.type,
7695 cause: "This question needs an answer. Answer it or stop the turn.",
7696 });
7697 }
7698 yield* dispatchRuntimeRequestRespond(
7699 {
7700 type: "runtime-request.respond",
7701 commandId: command.commandId,
7702 threadId: command.threadId,
7703 requestId: command.requestId,
7704 decision: "cancel",
7705 },
7706 events,
7707 effects,
7708 );
7709 });
7710
7711 const dispatchQueuedMessagePromoteToSteer = (
7712 command: Extract<OrchestrationV2Command, { readonly type: "queued-message.promote-to-steer" }>,
7713 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7714 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
7715 ) =>
7716 Effect.gen(function* () {
7717 const projection = yield* projectionStore
7718 .getThreadRecords(
7719 command.threadId,
7720 [
7721 "runs",
7722 "nodes",
7723 "attempts",
7724 "messages",
7725 "providerThreads",
7726 "providerTurns",
7727 "providerSessions",
7728 "turnItems",
7729 "runtimeRequests",
7730 "subagents",
7731 ],
7732 { turnItemTypes: [], messageRoles: ["user"] },
7733 )
7734 .pipe(
7735 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
7736 );
7737 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
7738 return yield* new OrchestratorDispatchError({
7739 commandId: command.commandId,
7740 commandType: command.type,
7741 cause: `Thread ${command.threadId} is not active.`,
7742 });
7743 }
7744 const queuedRun = projection.runs.find((candidate) => candidate.id === command.queuedRunId);
7745 if (queuedRun === undefined || queuedRun.status !== "queued") {
7746 return yield* new OrchestratorDispatchError({
7747 commandId: command.commandId,
7748 commandType: command.type,
7749 cause: `Queued run ${command.queuedRunId} is not queued.`,
7750 });
7751 }
7752 const queuedRootNode =
7753 queuedRun.rootNodeId === null
7754 ? undefined
7755 : projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId);
7756 const queuedAttempt =
7757 queuedRun.activeAttemptId === null
7758 ? undefined
7759 : projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId);
7760 const queuedMessage = projection.messages.find(
7761 (candidate) => candidate.id === queuedRun.userMessageId,
7762 );
7763 if (
7764 queuedRootNode === undefined ||
7765 queuedAttempt === undefined ||
7766 queuedMessage === undefined
7767 ) {
7768 return yield* new OrchestratorDispatchError({
7769 commandId: command.commandId,
7770 commandType: command.type,
7771 cause: `Queued run ${queuedRun.id} is missing message or execution state.`,
7772 });
7773 }
7774 if (
7775 queuedMessage.delegatedCompletion !== undefined ||
7776 queuedMessage.notification !== undefined
7777 ) {
7778 return yield* new OrchestratorDispatchError({
7779 commandId: command.commandId,
7780 commandType: command.type,
7781 cause: "Automatic completion deliveries cannot be promoted to Steer.",
7782 });
7783 }
7784
7785 if (yield* isRunCompacting(projection, command.targetRunId)) return;
7786
7787 const now = yield* DateTime.now;
7788 const emitEvent = emit(events, command);
7789 yield* emitEvent({
7790 type: "run.updated",
7791 threadId: command.threadId,
7792 runId: queuedRun.id,
7793 nodeId: queuedRootNode.id,
7794 providerInstanceId: queuedRun.providerInstanceId,
7795 occurredAt: now,
7796 payload: {
7797 ...queuedRun,
7798 status: "cancelled",
7799 queuePosition: null,
7800 completedAt: now,
7801 },
7802 });
7803 yield* emitEvent({
7804 type: "run-attempt.updated",
7805 threadId: command.threadId,
7806 runId: queuedRun.id,
7807 nodeId: queuedRootNode.id,
7808 providerInstanceId: queuedRun.providerInstanceId,
7809 occurredAt: now,
7810 payload: {
7811 ...queuedAttempt,
7812 status: "cancelled",
7813 completedAt: now,
7814 },
7815 });
7816 yield* emitEvent({
7817 type: "node.updated",
7818 threadId: command.threadId,
7819 runId: queuedRun.id,
7820 nodeId: queuedRootNode.id,
7821 providerInstanceId: queuedRun.providerInstanceId,
7822 occurredAt: now,
7823 payload: {
7824 ...queuedRootNode,
7825 status: "cancelled",
7826 completedAt: now,
7827 },
7828 });
7829
7830 yield* dispatchSteerIntoRun({
7831 command,
7832 events,
7833 effects,
7834 projection,
7835 modelSelection: projection.thread.modelSelection,
7836 targetRunId: command.targetRunId,
7837 messageId: queuedMessage.id,
7838 text: queuedMessage.text,
7839 attachments: queuedMessage.attachments,
7840 ...(queuedMessage.context ? { context: queuedMessage.context } : {}),
7841 createdBy: queuedMessage.createdBy,
7842 creationSource: queuedMessage.creationSource,
7843 ...(queuedMessage.scheduledTaskId === undefined
7844 ? {}
7845 : { scheduledTaskId: queuedMessage.scheduledTaskId }),
7846 ...(queuedMessage.senderThreadId === undefined
7847 ? {}
7848 : { senderThreadId: queuedMessage.senderThreadId }),
7849 forceRestart: false,
7850 });
7851 });
7852
7853 const dispatchQueuedRunReorder = (
7854 command: Extract<OrchestrationV2Command, { readonly type: "queued-run.reorder" }>,
7855 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7856 ) =>
7857 Effect.gen(function* () {
7858 const projection = yield* projectionStore
7859 .getThreadRecords(command.threadId, ["runs", "messages"], { messageRoles: ["user"] })
7860 .pipe(
7861 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
7862 );
7863 const queuedRuns = queuedRunsInDeliveryOrder(projection);
7864 const moving = queuedRuns.find((run) => run.id === command.runId);
7865 if (moving === undefined) {
7866 return yield* new OrchestratorDispatchError({
7867 commandId: command.commandId,
7868 commandType: command.type,
7869 cause: `Run ${command.runId} is not queued.`,
7870 });
7871 }
7872 const movingMessage = projection.messages.find(
7873 (candidate) => candidate.id === moving.userMessageId,
7874 );
7875 if (
7876 movingMessage?.delegatedCompletion !== undefined ||
7877 movingMessage?.notification !== undefined
7878 ) {
7879 return yield* new OrchestratorDispatchError({
7880 commandId: command.commandId,
7881 commandType: command.type,
7882 cause: "Automatic completion deliveries cannot be reordered.",
7883 });
7884 }
7885 const automaticRuns = queuedRuns.filter((run) => isAutomaticCompletionRun(projection, run));
7886 if (
7887 command.beforeRunId !== null &&
7888 automaticRuns.some((run) => run.id === command.beforeRunId)
7889 ) {
7890 return yield* new OrchestratorDispatchError({
7891 commandId: command.commandId,
7892 commandType: command.type,
7893 cause: "Queued messages cannot be reordered ahead of automatic completion delivery.",
7894 });
7895 }
7896 const reorderableRuns = queuedRuns.filter(
7897 (run) => !isAutomaticCompletionRun(projection, run),
7898 );
7899 const withoutMoving = reorderableRuns.filter((run) => run.id !== command.runId);
7900 const beforeIndex =
7901 command.beforeRunId === null
7902 ? withoutMoving.length
7903 : withoutMoving.findIndex((run) => run.id === command.beforeRunId);
7904 if (beforeIndex === -1) {
7905 return yield* new OrchestratorDispatchError({
7906 commandId: command.commandId,
7907 commandType: command.type,
7908 cause: `Queue target ${command.beforeRunId} is not queued.`,
7909 });
7910 }
7911 const reordered = [
7912 ...automaticRuns,
7913 ...withoutMoving.slice(0, beforeIndex),
7914 moving,
7915 ...withoutMoving.slice(beforeIndex),
7916 ];
7917 const now = yield* DateTime.now;
7918 const emitEvent = emit(events, command);
7919 yield* Effect.forEach(
7920 reordered,
7921 (run, index) =>
7922 Effect.gen(function* () {
7923 const queuePosition = index + 1;
7924 if (run.queuePosition === queuePosition) {
7925 return;
7926 }
7927 yield* emitEvent({
7928 type: "run.updated",
7929 threadId: command.threadId,
7930 runId: run.id,
7931 ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
7932 providerInstanceId: run.providerInstanceId,
7933 occurredAt: now,
7934 payload: {
7935 ...run,
7936 queuePosition,
7937 },
7938 });
7939 }),
7940 { concurrency: 1 },
7941 );
7942 });
7943
7944 const dispatchQueuedRunCancel = (
7945 command: Extract<OrchestrationV2Command, { readonly type: "queued-run.cancel" }>,
7946 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7947 ) =>
7948 Effect.gen(function* () {
7949 const projection = yield* projectionStore
7950 .getThreadRecords(
7951 command.threadId,
7952 ["runs", "messages", "nodes", "attempts", "subagents", "turnItems"],
7953 { turnItemTypes: [], messageRoles: ["user"] },
7954 )
7955 .pipe(
7956 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
7957 );
7958 const queuedRun = projection.runs.find((candidate) => candidate.id === command.runId);
7959 if (queuedRun === undefined || queuedRun.status !== "queued") {
7960 return yield* new OrchestratorDispatchError({
7961 commandId: command.commandId,
7962 commandType: command.type,
7963 cause: `Run ${command.runId} is not queued.`,
7964 });
7965 }
7966 const queuedMessage = projection.messages.find(
7967 (candidate) => candidate.id === queuedRun.userMessageId,
7968 );
7969 if (queuedMessage?.delegatedCompletion !== undefined) {
7970 const now = yield* DateTime.now;
7971 yield* disposeDelegatedCompletionCohort({
7972 command,
7973 events,
7974 projection,
7975 parentRunId: queuedMessage.delegatedCompletion.parentRunId,
7976 disposition: "disposed",
7977 now,
7978 });
7979 return;
7980 }
7981 const queuedRootNode =
7982 queuedRun.rootNodeId === null
7983 ? undefined
7984 : projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId);
7985 const queuedAttempt =
7986 queuedRun.activeAttemptId === null
7987 ? undefined
7988 : projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId);
7989
7990 const now = yield* DateTime.now;
7991 const emitEvent = emit(events, command);
7992 yield* emitEvent({
7993 type: "run.updated",
7994 threadId: command.threadId,
7995 runId: queuedRun.id,
7996 ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
7997 providerInstanceId: queuedRun.providerInstanceId,
7998 occurredAt: now,
7999 payload: {
8000 ...queuedRun,
8001 status: "cancelled",
8002 queuePosition: null,
8003 completedAt: now,
8004 },
8005 });
8006 if (queuedAttempt !== undefined && queuedRootNode !== undefined) {
8007 yield* emitEvent({
8008 type: "run-attempt.updated",
8009 threadId: command.threadId,
8010 runId: queuedRun.id,
8011 nodeId: queuedRootNode.id,
8012 providerInstanceId: queuedRun.providerInstanceId,
8013 occurredAt: now,
8014 payload: {
8015 ...queuedAttempt,
8016 status: "cancelled",
8017 completedAt: now,
8018 },
8019 });
8020 }
8021 if (queuedRootNode !== undefined) {
8022 yield* emitEvent({
8023 type: "node.updated",
8024 threadId: command.threadId,
8025 runId: queuedRun.id,
8026 nodeId: queuedRootNode.id,
8027 providerInstanceId: queuedRun.providerInstanceId,
8028 occurredAt: now,
8029 payload: {
8030 ...queuedRootNode,
8031 status: "cancelled",
8032 completedAt: now,
8033 },
8034 });
8035 }
8036 });
8037
8038 const dispatchQueuedRunEdit = (
8039 command: Extract<OrchestrationV2Command, { readonly type: "queued-run.edit" }>,
8040 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8041 ) =>
8042 Effect.gen(function* () {
8043 if (command.text.trim().length === 0) {
8044 return yield* new OrchestratorDispatchError({
8045 commandId: command.commandId,
8046 commandType: command.type,
8047 cause: `Queued run ${command.runId} cannot be edited to an empty message.`,
8048 });
8049 }
8050 const projection = yield* projectionStore
8051 .getThreadRecords(command.threadId, ["runs", "messages", "turnItems"], {
8052 turnItemTypes: ["user_message"],
8053 messageRoles: ["user"],
8054 })
8055 .pipe(
8056 Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })),
8057 );
8058 const queuedRun = projection.runs.find((candidate) => candidate.id === command.runId);
8059 if (queuedRun === undefined || queuedRun.status !== "queued") {
8060 return yield* new OrchestratorDispatchError({
8061 commandId: command.commandId,
8062 commandType: command.type,
8063 cause: `Run ${command.runId} is not queued.`,
8064 });
8065 }
8066 const queuedMessage = projection.messages.find(
8067 (candidate) => candidate.id === queuedRun.userMessageId,
8068 );
8069 if (queuedMessage === undefined) {
8070 return yield* new OrchestratorDispatchError({
8071 commandId: command.commandId,
8072 commandType: command.type,
8073 cause: `Queued run ${queuedRun.id} has no user message.`,
8074 });
8075 }
8076 if (
8077 queuedMessage.delegatedCompletion !== undefined ||
8078 queuedMessage.notification !== undefined
8079 ) {
8080 return yield* new OrchestratorDispatchError({
8081 commandId: command.commandId,
8082 commandType: command.type,
8083 cause: "Automatic completion deliveries cannot be edited.",
8084 });
8085 }
8086 const queuedTurnItem = projection.turnItems.find(
8087 (candidate) =>
8088 candidate.type === "user_message" && candidate.messageId === queuedMessage.id,
8089 );
8090
8091 const now = yield* DateTime.now;
8092 const emitEvent = emit(events, command);
8093 const editedAttachments =
8094 command.attachments === undefined ? {} : { attachments: command.attachments };
8095 yield* emitEvent({
8096 type: "message.updated",
8097 threadId: command.threadId,
8098 runId: queuedRun.id,
8099 ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
8100 providerInstanceId: queuedRun.providerInstanceId,
8101 occurredAt: now,
8102 payload: {
8103 ...queuedMessage,
8104 text: command.text,
8105 ...editedAttachments,
8106 ...(command.context ? { context: command.context } : {}),
8107 updatedAt: now,
8108 },
8109 });
8110 if (queuedTurnItem !== undefined && queuedTurnItem.type === "user_message") {
8111 yield* emitEvent({
8112 type: "turn-item.updated",
8113 threadId: command.threadId,
8114 runId: queuedRun.id,
8115 ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
8116 providerInstanceId: queuedRun.providerInstanceId,
8117 occurredAt: now,
8118 payload: {
8119 ...queuedTurnItem,
8120 text: command.text,
8121 ...editedAttachments,
8122 ...(command.context ? { context: command.context } : {}),
8123 updatedAt: now,
8124 },
8125 });
8126 }
8127 });
8128
8129 const loadProjectionForCommand = <K extends ProjectionRecordField>(
8130 command: OrchestrationV2Command,
8131 fields: ReadonlyArray<K>,
8132 filter?: ProjectionRecordFilter,
8133 ) =>
8134 projectionStore
8135 .getThreadRecords(commandThreadId(command), fields, filter)
8136 .pipe(
8137 Effect.mapError(
8138 () => new OrchestratorProjectionError({ threadId: commandThreadId(command) }),
8139 ),
8140 );
8141
8142 const preparedRunState = (
8143 command: Extract<
8144 OrchestrationV2Command,
8145 {
8146 readonly type:
8147 | "prepared-run.release"
8148 | "prepared-run.progress"
8149 | "prepared-run.fail"
8150 | "prepared-run.retry";
8151 }
8152 >,
8153 projection: Pick<
8154 OrchestrationV2ThreadProjection,
8155 "runs" | "attempts" | "nodes" | "providerThreads" | "turnItems"
8156 >,
8157 ) => {
8158 const run = projection.runs.find((candidate) => candidate.id === command.runId);
8159 const attempt = projection.attempts.find((candidate) => candidate.id === run?.activeAttemptId);
8160 const rootNode = projection.nodes.find((candidate) => candidate.id === run?.rootNodeId);
8161 const providerThread = projection.providerThreads.find(
8162 (candidate) => candidate.id === run?.providerThreadId,
8163 );
8164 const preparationItem = projection.turnItems.find(
8165 (
8166 candidate,
8167 ): candidate is Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }> =>
8168 candidate.runId === command.runId &&
8169 candidate.type === "command_execution" &&
8170 candidate.input === WORKSPACE_PREPARATION_INPUT,
8171 );
8172 if (
8173 run?.status !== (command.type === "prepared-run.retry" ? "failed" : "preparing") ||
8174 attempt === undefined ||
8175 rootNode === undefined ||
8176 providerThread === undefined ||
8177 preparationItem === undefined
8178 ) {
8179 return null;
8180 }
8181 return { run, attempt, rootNode, providerThread, preparationItem } as const;
8182 };
8183
8184 const dispatchPreparedRunProgress = (
8185 command: Extract<OrchestrationV2Command, { readonly type: "prepared-run.progress" }>,
8186 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8187 ) =>
8188 Effect.gen(function* () {
8189 const projection = yield* loadProjectionForCommand(
8190 command,
8191 ["runs", "attempts", "nodes", "providerThreads", "turnItems"],
8192 { turnItemTypes: ["command_execution"], turnItemRunId: command.runId },
8193 );
8194 const state = preparedRunState(command, projection);
8195 if (state === null) {
8196 return yield* new OrchestratorDispatchError({
8197 commandId: command.commandId,
8198 commandType: command.type,
8199 cause: `Run ${command.runId} is not awaiting workspace preparation.`,
8200 });
8201 }
8202 const now = yield* DateTime.now;
8203 yield* emit(
8204 events,
8205 command,
8206 )({
8207 type: "turn-item.updated",
8208 threadId: command.threadId,
8209 runId: state.run.id,
8210 nodeId: state.rootNode.id,
8211 providerInstanceId: state.run.providerInstanceId,
8212 occurredAt: now,
8213 payload: {
8214 ...state.preparationItem,
8215 title: command.phase === "worktree" ? "Preparing worktree" : "Starting setup script",
8216 updatedAt: now,
8217 },
8218 });
8219 });
8220
8221 const dispatchPreparedRunRelease = (
8222 command: Extract<OrchestrationV2Command, { readonly type: "prepared-run.release" }>,
8223 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8224 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
8225 ) =>
8226 Effect.gen(function* () {
8227 const projection = yield* loadProjectionForCommand(
8228 command,
8229 ["runs", "attempts", "nodes", "providerThreads", "turnItems"],
8230 { turnItemTypes: ["command_execution"], turnItemRunId: command.runId },
8231 );
8232 const state = preparedRunState(command, projection);
8233 if (state === null) {
8234 return yield* new OrchestratorDispatchError({
8235 commandId: command.commandId,
8236 commandType: command.type,
8237 cause: `Run ${command.runId} is not awaiting workspace preparation.`,
8238 });
8239 }
8240 const now = yield* DateTime.now;
8241 const resolvedRuntimePolicy = yield* runtimePolicy
8242 .resolve({ thread: projection.thread, modelSelection: state.run.modelSelection })
8243 .pipe(mapDispatchError(command));
8244 const checkpointScope = yield* checkpointService
8245 .prepareRootRunScope({
8246 threadId: command.threadId,
8247 runId: state.run.id,
8248 rootNodeId: state.rootNode.id,
8249 providerThreadId: state.providerThread.id,
8250 cwd: resolvedRuntimePolicy.cwd ?? projection.thread.worktreePath ?? process.cwd(),
8251 createdAt: now,
8252 })
8253 .pipe(mapDispatchError(command));
8254 const emitEvent = emit(events, command);
8255 yield* emitEvent({
8256 type: "checkpoint-scope.created",
8257 threadId: command.threadId,
8258 runId: state.run.id,
8259 nodeId: state.rootNode.id,
8260 providerInstanceId: state.run.providerInstanceId,
8261 occurredAt: now,
8262 payload: checkpointScope,
8263 });
8264 yield* emitEvent({
8265 type: "node.updated",
8266 threadId: command.threadId,
8267 runId: state.run.id,
8268 nodeId: state.rootNode.id,
8269 providerInstanceId: state.run.providerInstanceId,
8270 occurredAt: now,
8271 payload: { ...state.rootNode, checkpointScopeId: checkpointScope.id },
8272 });
8273 yield* emitEvent({
8274 type: "turn-item.updated",
8275 threadId: command.threadId,
8276 runId: state.run.id,
8277 nodeId: state.rootNode.id,
8278 providerInstanceId: state.run.providerInstanceId,
8279 occurredAt: now,
8280 payload: {
8281 ...state.preparationItem,
8282 status: "completed",
8283 title: "Workspace ready",
8284 output: "Workspace preparation completed.",
8285 exitCode: 0,
8286 completedAt: now,
8287 updatedAt: now,
8288 },
8289 });
8290 yield* emitEvent({
8291 type: "run.updated",
8292 threadId: command.threadId,
8293 runId: state.run.id,
8294 nodeId: state.rootNode.id,
8295 providerInstanceId: state.run.providerInstanceId,
8296 occurredAt: now,
8297 payload: { ...state.run, status: "starting" },
8298 });
8299 yield* Ref.update(effects, (existing) => [
8300 ...existing,
8301 {
8302 id: `effect:${command.commandId}:provider-turn.start:${state.run.id}`,
8303 commandId: command.commandId,
8304 threadId: command.threadId,
8305 request: { type: "provider-turn.start", runId: state.run.id },
8306 } satisfies PendingOrchestrationEffectV2,
8307 ]);
8308 });
8309
8310 const dispatchPreparedRunFail = (
8311 command: Extract<OrchestrationV2Command, { readonly type: "prepared-run.fail" }>,
8312 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8313 ) =>
8314 Effect.gen(function* () {
8315 const projection = yield* loadProjectionForCommand(
8316 command,
8317 ["runs", "attempts", "nodes", "providerThreads", "turnItems"],
8318 { turnItemTypes: ["command_execution"], turnItemRunId: command.runId },
8319 );
8320 const state = preparedRunState(command, projection);
8321 if (state === null) {
8322 return yield* new OrchestratorDispatchError({
8323 commandId: command.commandId,
8324 commandType: command.type,
8325 cause: `Run ${command.runId} is not awaiting workspace preparation.`,
8326 });
8327 }
8328 const now = yield* DateTime.now;
8329 const emitEvent = emit(events, command);
8330 yield* emitEvent({
8331 type: "run-attempt.updated",
8332 threadId: command.threadId,
8333 runId: state.run.id,
8334 nodeId: state.rootNode.id,
8335 providerInstanceId: state.run.providerInstanceId,
8336 occurredAt: now,
8337 payload: { ...state.attempt, status: "failed", completedAt: now },
8338 });
8339 yield* emitEvent({
8340 type: "node.updated",
8341 threadId: command.threadId,
8342 runId: state.run.id,
8343 nodeId: state.rootNode.id,
8344 providerInstanceId: state.run.providerInstanceId,
8345 occurredAt: now,
8346 payload: { ...state.rootNode, status: "failed", completedAt: now },
8347 });
8348 yield* emitEvent({
8349 type: "turn-item.updated",
8350 threadId: command.threadId,
8351 runId: state.run.id,
8352 nodeId: state.rootNode.id,
8353 providerInstanceId: state.run.providerInstanceId,
8354 occurredAt: now,
8355 payload: {
8356 ...state.preparationItem,
8357 status: "failed",
8358 title: "Workspace preparation failed",
8359 output: command.failure.message,
8360 exitCode: 1,
8361 completedAt: now,
8362 updatedAt: now,
8363 },
8364 });
8365 yield* emitEvent({
8366 type: "turn-item.updated",
8367 threadId: command.threadId,
8368 runId: state.run.id,
8369 nodeId: state.rootNode.id,
8370 providerInstanceId: state.run.providerInstanceId,
8371 occurredAt: now,
8372 payload: {
8373 id: idAllocator.derive.turnItemFromProviderItem({
8374 driver: state.providerThread.driver,
8375 nativeItemId: `workspace-preparation-failure:${state.run.id}`,
8376 }),
8377 threadId: command.threadId,
8378 runId: state.run.id,
8379 nodeId: state.rootNode.id,
8380 providerThreadId: state.providerThread.id,
8381 providerTurnId: null,
8382 nativeItemRef: null,
8383 parentItemId: null,
8384 ordinal: yield* nextTurnItemOrdinal(projection),
8385 status: "failed",
8386 title: "Workspace preparation failed",
8387 startedAt: now,
8388 completedAt: now,
8389 updatedAt: now,
8390 type: "error",
8391 failure: {
8392 ...command.failure,
8393 code: ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
8394 },
8395 },
8396 });
8397 yield* emitEvent({
8398 type: "run.updated",
8399 threadId: command.threadId,
8400 runId: state.run.id,
8401 nodeId: state.rootNode.id,
8402 providerInstanceId: state.run.providerInstanceId,
8403 occurredAt: now,
8404 payload: { ...state.run, status: "failed", completedAt: now },
8405 });
8406 });
8407
8408 /**
8409 * Returns a run whose workspace preparation failed to preparing. The failure
8410 * item turns cancelled so clients stop offering the retry; ThreadLaunchService
8411 * runs the recorded preparation again once this commits.
8412 */
8413 const dispatchPreparedRunRetry = (
8414 command: Extract<OrchestrationV2Command, { readonly type: "prepared-run.retry" }>,
8415 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8416 ) =>
8417 Effect.gen(function* () {
8418 const projection = yield* loadProjectionForCommand(
8419 command,
8420 ["runs", "attempts", "nodes", "providerThreads", "turnItems"],
8421 { turnItemTypes: ["command_execution", "error"], turnItemRunId: command.runId },
8422 );
8423 const state = preparedRunState(command, projection);
8424 const failureItem = projection.turnItems.find(
8425 (candidate): candidate is Extract<OrchestrationV2TurnItem, { readonly type: "error" }> =>
8426 candidate.type === "error" &&
8427 candidate.status === "failed" &&
8428 candidate.failure.code === ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
8429 );
8430 if (
8431 state === null ||
8432 state.run.workspacePreparation === undefined ||
8433 failureItem === undefined ||
8434 projection.thread.archivedAt !== null ||
8435 projection.thread.deletedAt !== null
8436 ) {
8437 return yield* new OrchestratorDispatchError({
8438 commandId: command.commandId,
8439 commandType: command.type,
8440 cause: `Run ${command.runId} has no failed workspace preparation to retry.`,
8441 });
8442 }
8443 if (projection.runs.some((run) => run.id !== state.run.id && isBlockingRun(run))) {
8444 return yield* new OrchestratorDispatchError({
8445 commandId: command.commandId,
8446 commandType: command.type,
8447 cause: "Another run is active on this thread.",
8448 });
8449 }
8450 const now = yield* DateTime.now;
8451 const emitEvent = emit(events, command);
8452 const scope = {
8453 threadId: command.threadId,
8454 runId: state.run.id,
8455 nodeId: state.rootNode.id,
8456 providerInstanceId: state.run.providerInstanceId,
8457 occurredAt: now,
8458 };
8459 yield* emitEvent({
8460 ...scope,
8461 type: "turn-item.updated",
8462 payload: { ...failureItem, status: "cancelled", updatedAt: now },
8463 });
8464 yield* emitEvent({
8465 ...scope,
8466 type: "turn-item.updated",
8467 payload: {
8468 ...withoutPreparationResult(state.preparationItem),
8469 status: "running",
8470 title: WORKSPACE_PREPARATION_INPUT,
8471 completedAt: null,
8472 updatedAt: now,
8473 },
8474 });
8475 yield* emitEvent({
8476 ...scope,
8477 type: "run-attempt.updated",
8478 payload: { ...state.attempt, status: "pending", completedAt: null },
8479 });
8480 yield* emitEvent({
8481 ...scope,
8482 type: "node.updated",
8483 payload: { ...state.rootNode, status: "pending", completedAt: null },
8484 });
8485 yield* emitEvent({
8486 ...scope,
8487 type: "run.updated",
8488 payload: { ...state.run, status: "preparing", completedAt: null },
8489 });
8490 // Messages held when this preparation failed follow the run again. A hold
8491 // from a later run (a stop, say) is not this failure's to release.
8492 const failureOwnsQueue = latestExecutedRun(projection.runs)?.id === state.run.id;
8493 for (const run of projection.runs) {
8494 if (!failureOwnsQueue || run.status !== "queued" || run.queueHeld !== true) continue;
8495 yield* emitEvent({
8496 type: "run.updated",
8497 threadId: command.threadId,
8498 runId: run.id,
8499 providerInstanceId: run.providerInstanceId,
8500 occurredAt: now,
8501 payload: { ...run, queueHeld: false },
8502 });
8503 }
8504 });
8505
8506 /**
8507 * Ends the background work a settled thread still shows that no provider
8508 * process will report on: work on the provider thread whose interrupt just
8509 * returned (`stoppedProviderThreadId`), and work on provider threads with no
8510 * live session at all. Only the stopped run's work and older runs' is ended
8511 * (`throughRunOrdinal`); a later run's work is its own. A dead process's
8512 * roster goes too, as on restart.
8513 */
8514 const settleBackgroundWork = (input: {
8515 readonly command: Extract<
8516 OrchestrationV2ServerCommand,
8517 { readonly type: "run.interrupt" | "thread.background-work.settle" }
8518 >;
8519 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
8520 readonly projection: Pick<
8521 OrchestrationV2ThreadProjection,
8522 "runs" | "nodes" | "subagents" | "turnItems" | "providerThreads"
8523 >;
8524 readonly stoppedProviderThreadId: OrchestrationV2ProviderThread["id"] | null;
8525 readonly throughRunOrdinal: number;
8526 readonly now: DateTime.Utc;
8527 }) =>
8528 Effect.gen(function* () {
8529 const emitEvent = emit(input.events, input.command);
8530 const runOrdinals = new Map(input.projection.runs.map((run) => [run.id, run.ordinal]));
8531 const allocateEventId = () =>
8532 idAllocator.allocate.event({
8533 threadId: input.command.threadId,
8534 commandId: input.command.commandId,
8535 });
8536 const childThreadIds = new Set<ThreadId>();
8537 const liveness = new Map<string, boolean>();
8538 const hasLiveSession = (providerThreadId: OrchestrationV2ProviderThread["id"]) =>
8539 Effect.gen(function* () {
8540 const known = liveness.get(providerThreadId);
8541 if (known !== undefined) return known;
8542 const sessionId = input.projection.providerThreads.find(
8543 (candidate) => candidate.id === providerThreadId,
8544 )?.providerSessionId;
8545 const live =
8546 sessionId !== null &&
8547 sessionId !== undefined &&
8548 Option.isSome(
8549 yield* providerSessions
8550 .get(sessionId)
8551 .pipe(Effect.orElseSucceed(() => Option.none())),
8552 );
8553 liveness.set(providerThreadId, live);
8554 return live;
8555 });
8556 for (const item of pendingBackgroundTurnItems({
8557 turnItems: input.projection.turnItems,
8558 runs: input.projection.runs,
8559 })) {
8560 // An item without a run counts as the stopped run's.
8561 const itemRunOrdinal = item.runId === null ? undefined : runOrdinals.get(item.runId);
8562 if (itemRunOrdinal !== undefined && itemRunOrdinal > input.throughRunOrdinal) continue;
8563 const providerThreadId = item.providerThreadId ?? null;
8564 if (
8565 providerThreadId !== null &&
8566 providerThreadId !== input.stoppedProviderThreadId &&
8567 (yield* hasLiveSession(providerThreadId))
8568 ) {
8569 continue;
8570 }
8571 const run = input.projection.runs.find((candidate) => candidate.id === item.runId);
8572 yield* emitEvent({
8573 type: "turn-item.updated",
8574 threadId: input.command.threadId,
8575 ...(item.runId === null ? {} : { runId: item.runId }),
8576 ...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
8577 ...(run === undefined ? {} : { providerInstanceId: run.providerInstanceId }),
8578 occurredAt: input.now,
8579 payload: {
8580 ...item,
8581 status: "interrupted",
8582 completedAt: input.now,
8583 updatedAt: input.now,
8584 },
8585 });
8586 if (item.type !== "subagent") continue;
8587 // A native subagent died with the process too: end its record, its own
8588 // thread's runless root turn, and the subagents it started there, which
8589 // a provider can record on that thread rather than this one.
8590 const dying: Array<{
8591 readonly projection: Pick<
8592 OrchestrationV2ThreadProjection,
8593 "nodes" | "subagents" | "turnItems"
8594 >;
8595 readonly subagent: Pick<
8596 Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
8597 "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
8598 >;
8599 }> = [{ projection: input.projection, subagent: item }];
8600 for (const { projection, subagent } of dying) {
8601 const ended = yield* endOrphanedNativeSubagent({
8602 projection,
8603 subagentId: subagent.subagentId,
8604 status: "interrupted",
8605 now: input.now,
8606 emitted: yield* Ref.get(input.events),
8607 allocateEventId,
8608 }).pipe(mapDispatchError(input.command));
8609 yield* Ref.update(input.events, (events) => [...events, ...ended]);
8610 const childThreadId = subagent.childThreadId;
8611 if (
8612 subagent.origin !== "provider_native" ||
8613 childThreadId === null ||
8614 childThreadIds.has(childThreadId)
8615 ) {
8616 continue;
8617 }
8618 childThreadIds.add(childThreadId);
8619 const child = yield* projectionStore
8620 .getThreadRecords(childThreadId, ["nodes", "subagents", "turnItems"], {
8621 // Runless items are the subagent's; any run there is the user's own.
8622 turnItemRunIds: [null],
8623 turnItemStatuses: ["pending", "running", "waiting"],
8624 })
8625 .pipe(
8626 Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
8627 mapDispatchError(input.command),
8628 );
8629 if (child === null) continue;
8630 const childEnded = yield* endRunlessRootTurns({
8631 threadId: childThreadId,
8632 providerInstanceId: subagent.providerInstanceId,
8633 projection: child,
8634 status: "interrupted",
8635 now: input.now,
8636 emitted: yield* Ref.get(input.events),
8637 allocateEventId,
8638 }).pipe(mapDispatchError(input.command));
8639 yield* Ref.update(input.events, (events) => [...events, ...childEnded]);
8640 for (const nested of child.turnItems) {
8641 if (nested.type === "subagent") dying.push({ projection: child, subagent: nested });
8642 }
8643 // Workflow members have records and child roots without subagent turn items.
8644 for (const nested of child.subagents) {
8645 if (nested.origin === "provider_native" && nested.runId === null) {
8646 dying.push({
8647 projection: child,
8648 subagent: {
8649 subagentId: nested.id,
8650 origin: nested.origin,
8651 childThreadId: nested.childThreadId,
8652 providerInstanceId: nested.providerInstanceId,
8653 },
8654 });
8655 }
8656 }
8657 }
8658 }
8659 for (const providerThread of input.projection.providerThreads) {
8660 // A live process owns its roster and reports clearing it.
8661 if (
8662 (providerThread.pendingBackgroundTasks?.length ?? 0) === 0 ||
8663 (yield* hasLiveSession(providerThread.id))
8664 ) {
8665 continue;
8666 }
8667 yield* emitEvent({
8668 type: "provider-thread.updated",
8669 threadId: input.command.threadId,
8670 driver: providerThread.driver,
8671 providerInstanceId: providerThread.providerInstanceId,
8672 occurredAt: input.now,
8673 payload: {
8674 ...providerThread,
8675 status:
8676 providerThread.status === "active" &&
8677 providerThread.lastRunOrdinal !== null &&
8678 providerThread.lastRunOrdinal <= input.throughRunOrdinal
8679 ? "idle"
8680 : providerThread.status,
8681 pendingBackgroundTasks: [],
8682 updatedAt: input.now,
8683 },
8684 });
8685 }
8686 });
8687
8688 // A successful Stop may have no terminal event to ingest: the session or
8689 // native turn already died, or its final write failed when storage filled.
8690 const settleInterruptedRun = (input: {
8691 readonly command: Extract<
8692 OrchestrationV2ServerCommand,
8693 { readonly type: "run.interrupt" | "thread.background-work.settle" }
8694 >;
8695 readonly projection: Pick<
8696 OrchestrationV2ThreadProjection,
8697 | "runs"
8698 | "attempts"
8699 | "nodes"
8700 | "subagents"
8701 | "runtimeRequests"
8702 | "messages"
8703 | "turnItems"
8704 | "providerThreads"
8705 >;
8706 readonly providerTurn: OrchestrationV2ProviderTurn;
8707 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
8708 readonly effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>;
8709 readonly now: DateTime.Utc;
8710 }) =>
8711 Effect.gen(function* () {
8712 const attempt = input.projection.attempts.find(
8713 (candidate) => candidate.id === input.providerTurn.runAttemptId,
8714 );
8715 const run = input.projection.runs.find((candidate) => candidate.id === attempt?.runId);
8716 const rootNode = input.projection.nodes.find((candidate) => candidate.id === run?.rootNodeId);
8717 if (
8718 attempt === undefined ||
8719 run === undefined ||
8720 rootNode === undefined ||
8721 run.activeAttemptId !== attempt.id ||
8722 run.status !== "running" ||
8723 attempt.status !== "running"
8724 )
8725 return;
8726 const emitEvent = emit(input.events, input.command);
8727 const base = {
8728 threadId: input.command.threadId,
8729 runId: run.id,
8730 nodeId: rootNode.id,
8731 providerInstanceId: run.providerInstanceId,
8732 occurredAt: input.now,
8733 };
8734 if (input.providerTurn.status === "running") {
8735 yield* emitEvent({
8736 ...base,
8737 type: "provider-turn.updated",
8738 payload: {
8739 ...input.providerTurn,
8740 status: "interrupted",
8741 completedAt: input.now,
8742 },
8743 });
8744 }
8745 yield* emitEvent({
8746 ...base,
8747 type: "run-attempt.updated",
8748 payload: {
8749 ...attempt,
8750 status: "interrupted",
8751 completedAt: input.now,
8752 },
8753 });
8754 const providerThread = input.projection.providerThreads.find(
8755 (candidate) => candidate.id === input.providerTurn.providerThreadId,
8756 );
8757 if (providerThread?.status === "active" && providerThread.lastRunOrdinal === run.ordinal) {
8758 yield* emitEvent({
8759 ...base,
8760 type: "provider-thread.updated",
8761 payload: { ...providerThread, status: "idle", updatedAt: input.now },
8762 });
8763 }
8764 for (const message of input.projection.messages) {
8765 if (message.runId !== run.id || !message.streaming) continue;
8766 yield* emitEvent({
8767 ...base,
8768 ...(message.nodeId === null ? {} : { nodeId: message.nodeId }),
8769 type: "message.updated",
8770 payload: { ...message, streaming: false, updatedAt: input.now },
8771 });
8772 }
8773 const delegatedNodeIds = new Set(
8774 input.projection.subagents
8775 .filter((subagent) => subagent.origin === "app_owned")
8776 .map((subagent) => subagent.id),
8777 );
8778 const cascaded = yield* RunExecutionService.cascadeTerminalizeRunOwnedSubagents({
8779 run,
8780 status: "interrupted",
8781 completedAt: input.now,
8782 allocateEventId: () =>
8783 idAllocator.allocate.event({
8784 threadId: input.command.threadId,
8785 commandId: input.command.commandId,
8786 }),
8787 open: {
8788 subagents: new Map(
8789 input.projection.subagents
8790 .filter((task) => task.runId === run.id && task.origin !== "app_owned")
8791 .map((task) => [task.id, task]),
8792 ),
8793 nodes: new Map(
8794 input.projection.nodes
8795 .filter((node) => node.runId === run.id && !delegatedNodeIds.has(node.id))
8796 .map((node) => [node.id, node]),
8797 ),
8798 turnItems: new Map(),
8799 childTurnItems: new Map(
8800 input.projection.turnItems
8801 .filter(
8802 (item) =>
8803 item.runId === run.id &&
8804 (item.type === "assistant_message" || item.type === "reasoning"),
8805 )
8806 .map((item) => [item.id, item]),
8807 ),
8808 linkedChildThreadIds: new Set([run.threadId]),
8809 },
8810 }).pipe(mapDispatchError(input.command));
8811 yield* Ref.update(input.events, (events) => [...events, ...cascaded]);
8812 for (const request of input.projection.runtimeRequests) {
8813 if (
8814 request.status !== "pending" ||
8815 !input.projection.nodes.some(
8816 (node) => node.id === request.nodeId && node.runId === run.id,
8817 )
8818 )
8819 continue;
8820 yield* emitEvent({
8821 ...base,
8822 nodeId: request.nodeId,
8823 type: "runtime-request.updated",
8824 payload: {
8825 ...request,
8826 status: "cancelled",
8827 resolvedAt: input.now,
8828 responseCapability: { type: "not_resumable", reason: "The run was interrupted." },
8829 },
8830 });
8831 }
8832 if (providerThread !== undefined) {
8833 yield* emitEvent({
8834 ...base,
8835 type: "turn-item.updated",
8836 payload: RunExecutionService.makeInterruptResultTurnItem({
8837 idAllocator,
8838 run,
8839 rootNode,
8840 providerThread,
8841 completedAt: input.now,
8842 }),
8843 });
8844 }
8845 const { delegatedCompletion: _delegatedCompletion, ...runWithoutDelegatedCompletion } = run;
8846 yield* emitEvent({
8847 ...base,
8848 type: "run.updated",
8849 payload: {
8850 ...runWithoutDelegatedCompletion,
8851 status: "interrupted",
8852 completedAt: input.now,
8853 },
8854 });
8855 const scopeId = rootNode.checkpointScopeId;
8856 if (scopeId !== null) {
8857 yield* Ref.update(input.effects, (effects) => [
8858 ...effects,
8859 {
8860 id: `effect:checkpoint.capture:${run.id}`,
8861 commandId: CommandId.make(`command:effect:checkpoint.capture:${run.id}`),
8862 threadId: input.command.threadId,
8863 request: {
8864 type: "checkpoint.capture",
8865 runId: run.id,
8866 scopeId,
8867 },
8868 } satisfies PendingOrchestrationEffectV2,
8869 ]);
8870 }
8871 });
8872
8873 /**
8874 * Whether Stop reached a run, as its interrupt request records. Its agent can call tools
8875 * until the provider stops it, and a slow call can land after that, but nothing it starts
8876 * may outlive the Stop. A restart continuation of that run must not start either.
8877 */
8878 const stopReachedRun = (
8879 command: OrchestrationV2ServerCommand,
8880 threadId: ThreadId,
8881 runId: RunId,
8882 ) =>
8883 projectionStore
8884 .getThreadRecords(threadId, ["turnItems"], {
8885 turnItemTypes: ["run_interrupt_request"],
8886 turnItemRunIds: [runId],
8887 })
8888 .pipe(
8889 Effect.map(({ turnItems }) => turnItems.length > 0),
8890 mapDispatchError(command),
8891 );
8892
8893 /** Records Stop for a run without replacing a marker already written by its interrupt. */
8894 const markStoppedRun = Effect.fnUntraced(function* (input: {
8895 readonly command: Extract<
8896 OrchestrationV2ServerCommand,
8897 { readonly type: "run.interrupt" | "thread.stop" }
8898 >;
8899 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
8900 readonly thread: OrchestrationV2AppThread;
8901 readonly run: OrchestrationV2Run;
8902 readonly now: DateTime.Utc;
8903 }) {
8904 const { command, events, thread, run, now } = input;
8905 if (run.rootNodeId === null || run.providerThreadId === null) return;
8906 const pendingItems = (yield* Ref.get(events)).flatMap((event) =>
8907 event.type === "turn-item.updated" ? [event.payload] : [],
8908 );
8909 if (
8910 pendingItems.some((item) => item.runId === run.id && item.type === "run_interrupt_request") ||
8911 (yield* stopReachedRun(command, thread.id, run.id))
8912 ) {
8913 return;
8914 }
8915 yield* emit(
8916 events,
8917 command,
8918 )({
8919 type: "turn-item.updated",
8920 threadId: thread.id,
8921 runId: run.id,
8922 nodeId: run.rootNodeId,
8923 providerInstanceId: run.providerInstanceId,
8924 occurredAt: now,
8925 payload: {
8926 id: idAllocator.derive.runSignalTurnItem({ runId: run.id, signal: "interrupt-request" }),
8927 threadId: thread.id,
8928 runId: run.id,
8929 nodeId: run.rootNodeId,
8930 providerThreadId: run.providerThreadId,
8931 providerTurnId: null,
8932 nativeItemRef: null,
8933 parentItemId: null,
8934 ordinal: yield* nextTurnItemOrdinal({ thread, turnItems: pendingItems }),
8935 status: "completed",
8936 title: "Interrupt requested",
8937 startedAt: now,
8938 completedAt: now,
8939 updatedAt: now,
8940 type: "run_interrupt_request",
8941 message: command.reason ?? "Interrupt requested",
8942 },
8943 });
8944 });
8945
8946 /**
8947 * What Stop holds besides the run it interrupts: durable queued messages wait for the
8948 * user, buffered provider wakes are cancelled, and the thread's pull request watches end.
8949 * Every wake its delegated tasks still owe is dropped, since Stop stops those tasks too.
8950 * `cohortRunIds` are dropped even when they owe nothing, like a plain interrupt's.
8951 */
8952 const holdStoppedThread = (input: {
8953 readonly command: Extract<
8954 OrchestrationV2ServerCommand,
8955 { readonly type: "run.interrupt" | "thread.stop" }
8956 >;
8957 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
8958 readonly projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs" | "subagents">;
8959 readonly cohortRunIds: ReadonlyArray<RunId>;
8960 readonly now: DateTime.Utc;
8961 }) =>
8962 Effect.gen(function* () {
8963 const emitEvent = emit(input.events, input.command);
8964 const { thread, runs, subagents } = input.projection;
8965 const queuedRuns = runs.filter((run) => run.status === "queued");
8966 const queuedMessages =
8967 queuedRuns.length === 0
8968 ? []
8969 : (yield* projectionStore
8970 .getThreadRecords(thread.id, ["messages"], {
8971 messageIds: queuedRuns.map((run) => run.userMessageId),
8972 })
8973 .pipe(
8974 Effect.mapError(
8975 (cause) => new OrchestratorProjectionError({ threadId: thread.id, cause }),
8976 ),
8977 )).messages;
8978 const bufferedMessageIds = new Set(
8979 queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
8980 );
8981 for (const run of runs) {
8982 // A resumed queue can have a lower ordinal than a pending continuation source.
8983 // Stop must reach those sources even when interrupting the resumed run succeeds.
8984 if (
8985 run.status === "cancelled" &&
8986 (yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
8987 ) {
8988 yield* markStoppedRun({ ...input, thread, run });
8989 }
8990 if (run.status !== "queued") continue;
8991 // Stop discards the provider's wake buffer, so its queued placeholders cannot resume.
8992 if (bufferedMessageIds.has(run.userMessageId)) {
8993 yield* dispatchQueuedRunCancel(
8994 {
8995 type: "queued-run.cancel",
8996 commandId: input.command.commandId,
8997 threadId: thread.id,
8998 runId: run.id,
8999 },
9000 input.events,
9001 );
9002 continue;
9003 }
9004 if (run.queueHeld === true) continue;
9005 yield* emitEvent({
9006 type: "run.updated",
9007 threadId: thread.id,
9008 runId: run.id,
9009 providerInstanceId: run.providerInstanceId,
9010 occurredAt: input.now,
9011 payload: { ...run, queueHeld: true },
9012 });
9013 }
9014 const pullRequests = thread.pullRequests ?? [];
9015 if (pullRequests.some((link) => link.watch !== undefined)) {
9016 yield* emitEvent({
9017 type: "thread.pull-request-synced",
9018 threadId: thread.id,
9019 providerInstanceId: thread.providerInstanceId,
9020 occurredAt: input.now,
9021 payload: {
9022 ...thread,
9023 pullRequests: pullRequests.map((link) => withPullRequestWatch(link, undefined)),
9024 },
9025 });
9026 }
9027 const cohortRunIds = new Set(input.cohortRunIds);
9028 for (const run of runs) {
9029 if (run.delegatedCompletion?.delivery != null) cohortRunIds.add(run.id);
9030 }
9031 for (const task of subagents) {
9032 const delivery = task.completionDelivery?.state;
9033 if (
9034 task.origin === "app_owned" &&
9035 task.runId !== null &&
9036 delivery !== "acknowledged" &&
9037 delivery !== "delivered" &&
9038 delivery !== "disposed"
9039 ) {
9040 cohortRunIds.add(task.runId);
9041 }
9042 }
9043 // Each disposal reads the events the ones before it wrote.
9044 for (const parentRunId of cohortRunIds) {
9045 yield* disposeDelegatedCompletionCohort({
9046 command: input.command,
9047 events: input.events,
9048 projection: yield* getProjectionWithPendingEvents(thread.id, input.events),
9049 parentRunId,
9050 disposition: "stopped",
9051 now: input.now,
9052 });
9053 }
9054 });
9055
9056 const dispatchBackgroundWorkSettle = (
9057 command: Extract<
9058 OrchestrationV2InternalCommand,
9059 { readonly type: "thread.background-work.settle" }
9060 >,
9061 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9062 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
9063 ) =>
9064 Effect.gen(function* () {
9065 const [projection, stopped] = yield* Effect.all([
9066 projectionStore.getThreadRecords(
9067 command.threadId,
9068 [
9069 "runs",
9070 "attempts",
9071 "nodes",
9072 "subagents",
9073 "runtimeRequests",
9074 "turnItems",
9075 "providerThreads",
9076 ],
9077 {
9078 turnItemTypes: [
9079 "command_execution",
9080 "dynamic_tool",
9081 "subagent",
9082 "assistant_message",
9083 "reasoning",
9084 ],
9085 turnItemStatuses: ["pending", "running", "waiting"],
9086 },
9087 ),
9088 projectionStore.getProviderControlContext(command.threadId, {
9089 providerThreadId: command.providerThreadId,
9090 providerTurnId: command.providerTurnId,
9091 }),
9092 ]).pipe(
9093 Effect.mapError(
9094 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
9095 ),
9096 );
9097 if (command.subagent !== undefined) {
9098 const outcome = command.subagent.outcome;
9099 const subagent = projection.subagents.find((task) => task.id === command.subagent?.id);
9100 if (
9101 subagent?.origin !== "provider_native" ||
9102 !["pending", "running", "waiting"].includes(subagent.status) ||
9103 (subagent.startedAt !== null &&
9104 DateTime.toEpochMillis(subagent.startedAt) >
9105 DateTime.toEpochMillis(command.subagent.stoppedAt))
9106 )
9107 return;
9108 const now = yield* DateTime.now;
9109 const allocateEventId = () =>
9110 idAllocator.allocate.event({
9111 threadId: command.threadId,
9112 commandId: command.commandId,
9113 });
9114 const ended = yield* endOrphanedNativeSubagent({
9115 projection,
9116 subagentId: subagent.id,
9117 result: outcome.result,
9118 status: outcome.status,
9119 completedAt: outcome.completedAt,
9120 now,
9121 emitted: yield* Ref.get(events),
9122 allocateEventId,
9123 }).pipe(mapDispatchError(command));
9124 yield* Ref.update(events, (current) => [...current, ...ended]);
9125 if (subagent.childThreadId !== null) {
9126 const child = yield* projectionStore
9127 .getThreadRecords(subagent.childThreadId, ["nodes", "turnItems"], {
9128 turnItemRunIds: [null],
9129 turnItemStatuses: ["pending", "running", "waiting"],
9130 })
9131 .pipe(
9132 Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
9133 mapDispatchError(command),
9134 );
9135 if (child !== null) {
9136 const childEnded = yield* endRunlessRootTurns({
9137 threadId: subagent.childThreadId,
9138 providerInstanceId: subagent.providerInstanceId,
9139 projection: child,
9140 status: outcome.status,
9141 completedAt: outcome.completedAt,
9142 now,
9143 emitted: yield* Ref.get(events),
9144 allocateEventId,
9145 }).pipe(mapDispatchError(command));
9146 yield* Ref.update(events, (current) => [...current, ...childEnded]);
9147 }
9148 }
9149 return;
9150 }
9151 const stoppedRunId = projection.attempts.find(
9152 (attempt) => attempt.id === stopped.providerTurn?.runAttemptId,
9153 )?.runId;
9154 const stoppedRun = projection.runs.find((run) => run.id === stoppedRunId);
9155 if (
9156 stoppedRun === undefined ||
9157 (["preparing", "starting", "running"].includes(stoppedRun.status) &&
9158 stoppedRun.activeAttemptId !== stopped.providerTurn?.runAttemptId)
9159 )
9160 return;
9161 const now = yield* DateTime.now;
9162 if (stopped.providerTurn !== undefined && stoppedRun.status === "running") {
9163 const output = yield* projectionStore
9164 .getThreadRecords(command.threadId, ["messages"], {
9165 messageRunIds: [stoppedRun.id],
9166 messageRoles: ["assistant"],
9167 })
9168 .pipe(mapDispatchError(command));
9169 yield* settleInterruptedRun({
9170 command,
9171 projection: { ...projection, messages: output.messages },
9172 providerTurn: stopped.providerTurn,
9173 events,
9174 effects,
9175 now,
9176 });
9177 }
9178 // A new turn may have started since Stop; its work is not this Stop's.
9179 if (
9180 projection.runs.some(
9181 (run) =>
9182 run.id !== stoppedRun.id &&
9183 (run.status === "preparing" || run.status === "starting" || run.status === "running"),
9184 )
9185 ) {
9186 return;
9187 }
9188 yield* settleBackgroundWork({
9189 command,
9190 events,
9191 projection,
9192 stoppedProviderThreadId: command.providerThreadId,
9193 throughRunOrdinal: stoppedRun.ordinal,
9194 now,
9195 });
9196 });
9197
9198 const dispatchSubagentStop = (
9199 command: Extract<OrchestrationV2Command, { readonly type: "subagent.stop" }>,
9200 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9201 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
9202 ) =>
9203 Effect.gen(function* () {
9204 const projection = yield* loadProjectionForCommand(command, [
9205 "runs",
9206 "subagents",
9207 "providerThreads",
9208 "providerTurns",
9209 ]);
9210 const run = projection.runs.find((candidate) => candidate.id === command.runId);
9211 const subagent = projection.subagents.find(
9212 (candidate) => candidate.id === command.subagentId,
9213 );
9214 if (
9215 subagent === undefined ||
9216 subagent.runId !== command.runId ||
9217 subagent.origin !== "provider_native" ||
9218 subagent.driver !== "claudeAgent" ||
9219 subagent.nativeTaskRef?.strength !== "strong" ||
9220 subagent.nativeTaskRef.nativeId === null
9221 ) {
9222 return yield* new OrchestratorDispatchError({
9223 commandId: command.commandId,
9224 commandType: command.type,
9225 cause: "The requested native subagent cannot be stopped on this run.",
9226 });
9227 }
9228 if (!["pending", "running", "waiting"].includes(subagent.status)) return;
9229 const providerThread = projection.providerThreads.find(
9230 (candidate) => candidate.id === run?.providerThreadId,
9231 );
9232 const providerTurn = projection.providerTurns.findLast(
9233 (candidate) => candidate.providerThreadId === providerThread?.id,
9234 );
9235 if (
9236 providerThread?.providerSessionId == null ||
9237 providerTurn === undefined ||
9238 providerThread.driver !== subagent.driver ||
9239 providerThread.providerInstanceId !== subagent.providerInstanceId
9240 ) {
9241 return yield* new OrchestratorDispatchError({
9242 commandId: command.commandId,
9243 commandType: command.type,
9244 cause: "The requested native subagent has no owning provider session.",
9245 });
9246 }
9247 const providerSessionId = providerThread.providerSessionId;
9248 const nativeTaskId = subagent.nativeTaskRef.nativeId;
9249 const now = yield* DateTime.now;
9250 // The effect can repair stale projections only after the provider confirms an outcome.
9251 yield* emit(
9252 events,
9253 command,
9254 )({
9255 type: "subagent.updated",
9256 threadId: command.threadId,
9257 runId: command.runId,
9258 nodeId: subagent.id,
9259 providerInstanceId: subagent.providerInstanceId,
9260 occurredAt: now,
9261 payload: { ...subagent, updatedAt: now },
9262 });
9263 yield* Ref.update(effects, (existing) => [
9264 ...existing,
9265 {
9266 id: `effect:${command.commandId}:subagent.stop:${subagent.id}`,
9267 commandId: command.commandId,
9268 threadId: command.threadId,
9269 request: {
9270 type: "provider-turn.interrupt",
9271 providerSessionId,
9272 providerThreadId: providerThread.id,
9273 providerTurnId: providerTurn.id,
9274 subagent: { id: subagent.id, nativeTaskId },
9275 },
9276 } satisfies PendingOrchestrationEffectV2,
9277 ]);
9278 return;
9279 });
9280
9281 const dispatchRunInterrupt = (
9282 command: Extract<OrchestrationV2Command, { readonly type: "run.interrupt" }>,
9283 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9284 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
9285 ) =>
9286 Effect.gen(function* () {
9287 const projection = yield* loadProjectionForCommand(
9288 command,
9289 [
9290 "runs",
9291 "nodes",
9292 "providerThreads",
9293 "providerTurns",
9294 "messages",
9295 "attempts",
9296 "turnItems",
9297 "subagents",
9298 "runtimeRequests",
9299 ],
9300 // Thread-wide, like the Waiting strip: a settled thread's background
9301 // work can belong to an earlier run than the one Stop targets.
9302 {
9303 turnItemTypes: [
9304 "command_execution",
9305 "dynamic_tool",
9306 "subagent",
9307 "assistant_message",
9308 "reasoning",
9309 ],
9310 turnItemStatuses: ["pending", "running", "waiting"],
9311 },
9312 );
9313 const run = projection.runs.find((candidate) => candidate.id === command.runId);
9314 const rootNode =
9315 run?.rootNodeId === null
9316 ? undefined
9317 : projection.nodes.find((candidate) => candidate.id === run?.rootNodeId);
9318 const providerThread =
9319 run?.providerThreadId === null
9320 ? undefined
9321 : projection.providerThreads.find((candidate) => candidate.id === run?.providerThreadId);
9322 const hasBackgroundWork =
9323 run?.id === projection.runs.findLast((candidate) => candidate.status !== "queued")?.id &&
9324 derivePendingBackgroundWork({
9325 latestRun: run,
9326 providerThreads: projection.providerThreads,
9327 turnItems: projection.turnItems,
9328 activeProviderThreadId: projection.thread.activeProviderThreadId,
9329 runs: projection.runs,
9330 }).length > 0;
9331 // A failed start has no provider turn. Background work still belongs
9332 // to the provider thread, so Stop reaches its latest accepted turn.
9333 const providerTurn =
9334 projection.providerTurns.findLast(
9335 (candidate) =>
9336 candidate.runAttemptId === run?.activeAttemptId &&
9337 (run?.status === "running" || candidate.status === "running" || hasBackgroundWork),
9338 ) ??
9339 (hasBackgroundWork
9340 ? projection.providerTurns.findLast(
9341 (candidate) => candidate.providerThreadId === run?.providerThreadId,
9342 )
9343 : undefined);
9344 if (run === undefined || rootNode === undefined || providerThread === undefined) {
9345 return yield* new OrchestratorDispatchError({
9346 commandId: command.commandId,
9347 commandType: command.type,
9348 cause: `Run ${command.runId} is not interruptible.`,
9349 });
9350 }
9351 const now = yield* DateTime.now;
9352 const completionMessage = projection.messages.find(
9353 (candidate) => candidate.id === run.userMessageId,
9354 );
9355 const completionCohortRunId = completionMessage?.delegatedCompletion?.parentRunId ?? run.id;
9356 // Stop holds the rest of the thread too. A plain interrupt only drops the wakes owed by
9357 // this run's delegated tasks, or on a wake run, by the cohort that woke it.
9358 const stopRemainingWork = () =>
9359 command.holdQueue === true
9360 ? holdStoppedThread({
9361 command,
9362 events,
9363 projection,
9364 cohortRunIds: [completionCohortRunId],
9365 now,
9366 })
9367 : Effect.gen(function* () {
9368 yield* disposeDelegatedCompletionCohort({
9369 command,
9370 events,
9371 projection: yield* getProjectionWithPendingEvents(command.threadId, events),
9372 parentRunId: completionCohortRunId,
9373 disposition: "stopped",
9374 now,
9375 });
9376 });
9377
9378 const emitEvent = emit(events, command);
9379 const interruptRequestItem: OrchestrationV2TurnItem = {
9380 id: idAllocator.derive.runSignalTurnItem({
9381 runId: run.id,
9382 signal: "interrupt-request",
9383 }),
9384 threadId: command.threadId,
9385 runId: run.id,
9386 nodeId: rootNode.id,
9387 providerThreadId: providerThread.id,
9388 providerTurnId: providerTurn?.id ?? null,
9389 nativeItemRef: null,
9390 parentItemId: null,
9391 ordinal: yield* nextTurnItemOrdinal(projection),
9392 status: "completed",
9393 title: "Interrupt requested",
9394 startedAt: now,
9395 completedAt: now,
9396 updatedAt: now,
9397 type: "run_interrupt_request",
9398 message: command.reason ?? "Interrupt requested",
9399 };
9400
9401 // Background work can outlive a provider switch, such as a Codex dev
9402 // server left running when the thread moved to Claude. Stop also reaches
9403 // each other live provider thread that owns pending work; that
9404 // interrupt's settle follow-up ends what its provider leaves behind.
9405 // Work on a dead session is settled with this run's below.
9406 const otherProviderInterrupts: Array<PendingOrchestrationEffectV2> = [];
9407 if (hasBackgroundWork || command.holdQueue === true) {
9408 const runOrdinals = new Map(
9409 projection.runs.map((candidate) => [candidate.id, candidate.ordinal]),
9410 );
9411 const runOrdinalOf = (item: (typeof projection.turnItems)[number]) =>
9412 item.runId === null ? -1 : (runOrdinals.get(item.runId) ?? -1);
9413 // Each provider thread is interrupted at its latest pending work: that
9414 // turn's run bounds the settle follow-up, which must cover all of the
9415 // thread's work. The target comes from the item's provider turn, since
9416 // a native subagent item names its own provider thread but its
9417 // parent's turn; interrupting the parent's turn reaches the subagent.
9418 const latestTurnByProviderThread = new Map<
9419 OrchestrationV2ProviderThread["id"],
9420 { readonly turn: (typeof projection.providerTurns)[number]; readonly runOrdinal: number }
9421 >();
9422 for (const item of pendingBackgroundTurnItems({
9423 turnItems: projection.turnItems,
9424 runs: projection.runs,
9425 })) {
9426 const turn = projection.providerTurns.find(
9427 (candidate) => candidate.id === item.providerTurnId,
9428 );
9429 if (turn === undefined || turn.providerThreadId === providerThread.id) continue;
9430 const runOrdinal = runOrdinalOf(item);
9431 const latest = latestTurnByProviderThread.get(turn.providerThreadId);
9432 if (latest === undefined || runOrdinal > latest.runOrdinal) {
9433 latestTurnByProviderThread.set(turn.providerThreadId, { turn, runOrdinal });
9434 }
9435 }
9436 for (const { turn } of latestTurnByProviderThread.values()) {
9437 const owner = projection.providerThreads.find(
9438 (candidate) => candidate.id === turn.providerThreadId,
9439 );
9440 if (owner === undefined || owner.providerSessionId === null) continue;
9441 const ownerSession = yield* providerSessions
9442 .get(owner.providerSessionId)
9443 .pipe(Effect.orElseSucceed(() => Option.none()));
9444 if (Option.isNone(ownerSession)) continue;
9445 otherProviderInterrupts.push({
9446 id: `effect:${command.commandId}:provider-turn.interrupt:${turn.id}`,
9447 commandId: command.commandId,
9448 threadId: command.threadId,
9449 request: {
9450 type: "provider-turn.interrupt",
9451 providerSessionId: owner.providerSessionId,
9452 providerThreadId: owner.id,
9453 providerTurnId: turn.id,
9454 },
9455 });
9456 }
9457 }
9458
9459 if (
9460 providerTurn === undefined &&
9461 (run.status === "preparing" || run.status === "starting" || run.status === "running")
9462 ) {
9463 const attempt = projection.attempts.find(
9464 (candidate) => candidate.id === run.activeAttemptId,
9465 );
9466 if (attempt === undefined) {
9467 return yield* new OrchestratorDispatchError({
9468 commandId: command.commandId,
9469 commandType: command.type,
9470 cause: `Run ${command.runId} has no active attempt to interrupt.`,
9471 });
9472 }
9473 const interruptResultItem: OrchestrationV2TurnItem = {
9474 id: idAllocator.derive.runSignalTurnItem({
9475 runId: run.id,
9476 signal: "interrupt-result",
9477 }),
9478 threadId: command.threadId,
9479 runId: run.id,
9480 nodeId: rootNode.id,
9481 providerThreadId: providerThread.id,
9482 providerTurnId: null,
9483 nativeItemRef: null,
9484 parentItemId: interruptRequestItem.id,
9485 ordinal: interruptRequestItem.ordinal + 1,
9486 status: "interrupted",
9487 title: "Interrupted",
9488 startedAt: now,
9489 completedAt: now,
9490 updatedAt: now,
9491 type: "run_interrupt_result",
9492 message: "Run interrupted before provider start",
9493 };
9494 yield* emitEvent({
9495 type: "turn-item.updated",
9496 threadId: command.threadId,
9497 runId: run.id,
9498 nodeId: rootNode.id,
9499 providerInstanceId: run.providerInstanceId,
9500 occurredAt: now,
9501 payload: interruptRequestItem,
9502 });
9503 yield* emitEvent({
9504 type: "turn-item.updated",
9505 threadId: command.threadId,
9506 runId: run.id,
9507 nodeId: rootNode.id,
9508 providerInstanceId: run.providerInstanceId,
9509 occurredAt: now,
9510 payload: interruptResultItem,
9511 });
9512 const preparationItem = projection.turnItems.find(
9513 (
9514 candidate,
9515 ): candidate is Extract<
9516 OrchestrationV2TurnItem,
9517 { readonly type: "command_execution" }
9518 > =>
9519 candidate.runId === run.id &&
9520 candidate.type === "command_execution" &&
9521 candidate.input === WORKSPACE_PREPARATION_INPUT &&
9522 candidate.status === "running",
9523 );
9524 if (preparationItem !== undefined) {
9525 yield* emitEvent({
9526 type: "turn-item.updated",
9527 threadId: command.threadId,
9528 runId: run.id,
9529 nodeId: rootNode.id,
9530 providerInstanceId: run.providerInstanceId,
9531 occurredAt: now,
9532 payload: {
9533 ...preparationItem,
9534 status: "interrupted",
9535 title: "Workspace preparation interrupted",
9536 output: command.reason ?? "Interrupted before provider start",
9537 completedAt: now,
9538 updatedAt: now,
9539 },
9540 });
9541 }
9542 yield* emitEvent({
9543 type: "run-attempt.updated",
9544 threadId: command.threadId,
9545 runId: run.id,
9546 nodeId: rootNode.id,
9547 providerInstanceId: run.providerInstanceId,
9548 occurredAt: now,
9549 payload: { ...attempt, status: "interrupted", completedAt: now },
9550 });
9551 yield* emitEvent({
9552 type: "node.updated",
9553 threadId: command.threadId,
9554 runId: run.id,
9555 nodeId: rootNode.id,
9556 providerInstanceId: run.providerInstanceId,
9557 occurredAt: now,
9558 payload: { ...rootNode, status: "interrupted", completedAt: now },
9559 });
9560 yield* emitEvent({
9561 type: "run.updated",
9562 threadId: command.threadId,
9563 runId: run.id,
9564 nodeId: rootNode.id,
9565 providerInstanceId: run.providerInstanceId,
9566 occurredAt: now,
9567 payload: { ...run, status: "interrupted", completedAt: now },
9568 });
9569 yield* stopRemainingWork();
9570 yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]);
9571 return {
9572 effectTypes: ["provider-turn.start", "provider-turn.restart"],
9573 reason: `Run ${run.id} was interrupted before its provider turn started.`,
9574 } satisfies {
9575 readonly effectTypes: ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
9576 readonly reason: string;
9577 };
9578 }
9579
9580 if (providerTurn === undefined) {
9581 return yield* new OrchestratorDispatchError({
9582 commandId: command.commandId,
9583 commandType: command.type,
9584 cause: `Run ${command.runId} is not interruptible.`,
9585 });
9586 }
9587 // A dead session cannot report a terminal event for either a running
9588 // turn or a settled turn's background work. Stop ends both locally.
9589 const sessionOption =
9590 providerThread.providerSessionId === null
9591 ? Option.none()
9592 : yield* providerSessions.get(providerThread.providerSessionId).pipe(
9593 Effect.mapError(
9594 (cause) =>
9595 new OrchestratorProviderAdapterError({
9596 commandId: command.commandId,
9597 providerInstanceId: run.providerInstanceId,
9598 cause,
9599 }),
9600 ),
9601 );
9602 if (Option.isNone(sessionOption)) {
9603 yield* emitEvent({
9604 type: "turn-item.updated",
9605 threadId: command.threadId,
9606 runId: run.id,
9607 nodeId: rootNode.id,
9608 providerInstanceId: run.providerInstanceId,
9609 occurredAt: now,
9610 payload: interruptRequestItem,
9611 });
9612 yield* stopRemainingWork();
9613 yield* settleInterruptedRun({ command, projection, providerTurn, events, effects, now });
9614 yield* settleBackgroundWork({
9615 command,
9616 events,
9617 projection,
9618 stoppedProviderThreadId: providerThread.id,
9619 throughRunOrdinal: run.ordinal,
9620 now,
9621 });
9622 yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]);
9623 return undefined;
9624 }
9625 if (providerThread.providerSessionId === null) {
9626 return yield* new OrchestratorDispatchError({
9627 commandId: command.commandId,
9628 commandType: command.type,
9629 cause: `Provider thread ${providerThread.id} has no active provider session.`,
9630 });
9631 }
9632 const providerSessionId = providerThread.providerSessionId;
9633 yield* enforceCommandPolicy(command)(
9634 commandPolicy.ensureInterrupt({
9635 commandId: command.commandId,
9636 threadId: command.threadId,
9637 providerInstanceId: run.providerInstanceId,
9638 capabilities: sessionOption.value.providerSession.capabilities,
9639 }),
9640 );
9641
9642 // Open interrupt edge cases are tracked in https://github.com/pingdotgg/t3code/issues/15013.
9643 yield* emitEvent({
9644 type: "turn-item.updated",
9645 threadId: command.threadId,
9646 runId: run.id,
9647 nodeId: rootNode.id,
9648 providerInstanceId: run.providerInstanceId,
9649 occurredAt: now,
9650 payload: interruptRequestItem,
9651 });
9652 yield* stopRemainingWork();
9653 yield* Ref.update(effects, (existing) => [
9654 ...existing,
9655 {
9656 id: `effect:${command.commandId}:provider-turn.interrupt:${providerTurn.id}`,
9657 commandId: command.commandId,
9658 threadId: command.threadId,
9659 request: {
9660 type: "provider-turn.interrupt",
9661 providerSessionId,
9662 providerThreadId: providerThread.id,
9663 providerTurnId: providerTurn.id,
9664 },
9665 } satisfies PendingOrchestrationEffectV2,
9666 ...otherProviderInterrupts,
9667 ]);
9668 return undefined;
9669 });
9670
9671 /**
9672 * Stop for a thread whatever it is doing, the way the Stop button stops the run it
9673 * shows: the running turn (or one whose background work is still pending) is interrupted,
9674 * and the rest of the thread is held. A turn that cannot be interrupted still leaves the
9675 * thread held. Delegated tasks under the thread are stopped by the sender.
9676 */
9677 const dispatchThreadStop = (
9678 command: Extract<OrchestrationV2ServerCommand, { readonly type: "thread.stop" }>,
9679 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9680 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
9681 ) =>
9682 Effect.gen(function* () {
9683 const projection = yield* projectionStore
9684 .getThreadRecords(command.threadId, ["runs", "providerThreads", "turnItems", "subagents"], {
9685 turnItemTypes: ["command_execution", "dynamic_tool", "subagent"],
9686 turnItemStatuses: ["pending", "running", "waiting"],
9687 })
9688 .pipe(
9689 Effect.mapError(
9690 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
9691 ),
9692 );
9693 if (projection.thread.deletedAt !== null) return undefined;
9694 // Queued messages never started and cannot own background work.
9695 const latestRun = projection.runs.findLast((run) => run.status !== "queued");
9696 const target =
9697 projection.runs.findLast(
9698 (run) =>
9699 run.status === "preparing" || run.status === "starting" || run.status === "running",
9700 ) ??
9701 (derivePendingBackgroundWork({
9702 latestRun,
9703 providerThreads: projection.providerThreads,
9704 turnItems: projection.turnItems,
9705 activeProviderThreadId: projection.thread.activeProviderThreadId,
9706 runs: projection.runs,
9707 }).length > 0
9708 ? latestRun
9709 : undefined);
9710 if (target !== undefined) {
9711 const interruptEvents = yield* Ref.make<Array<OrchestrationV2DomainEvent>>([]);
9712 const interruptEffects = yield* Ref.make<Array<PendingOrchestrationEffectV2>>([]);
9713 const interrupted = yield* Effect.exit(
9714 dispatchRunInterrupt(
9715 {
9716 type: "run.interrupt",
9717 commandId: command.commandId,
9718 threadId: command.threadId,
9719 runId: target.id,
9720 holdQueue: true,
9721 ...(command.reason === undefined ? {} : { reason: command.reason }),
9722 },
9723 interruptEvents,
9724 interruptEffects,
9725 ),
9726 );
9727 // A failed interrupt may have written part of its events, so they are kept apart.
9728 if (Exit.isSuccess(interrupted)) {
9729 const written = yield* Ref.get(interruptEvents);
9730 const enqueued = yield* Ref.get(interruptEffects);
9731 yield* Ref.update(events, (existing) => [...existing, ...written]);
9732 yield* Ref.update(effects, (existing) => [...existing, ...enqueued]);
9733 return interrupted.value;
9734 }
9735 yield* Effect.logWarning("thread.stop could not interrupt the running turn", {
9736 threadId: command.threadId,
9737 runId: target.id,
9738 cause: interrupted.cause,
9739 });
9740 }
9741 const now = yield* DateTime.now;
9742 // Record that Stop reached the turn it could not interrupt and the thread's last executed
9743 // run, so a late tool call from their agent starts nothing.
9744 const executed = latestExecutedRun(projection.runs);
9745 const reached = new Map<RunId, OrchestrationV2Run>();
9746 if (target !== undefined) reached.set(target.id, target);
9747 if (executed !== null && !hasLiveRun({ runs: [executed] }))
9748 reached.set(executed.id, executed);
9749 for (const run of reached.values()) {
9750 yield* markStoppedRun({ command, events, thread: projection.thread, run, now });
9751 }
9752 yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now });
9753 return undefined;
9754 });
9755
9756 const dispatchCheckpointRollback = (
9757 command: Extract<OrchestrationV2Command, { readonly type: "checkpoint.rollback" }>,
9758 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9759 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
9760 ) =>
9761 Effect.gen(function* () {
9762 const projection = yield* loadProjectionForCommand(
9763 command,
9764 ["providerThreads", "checkpoints", "checkpointScopes", "runs", "providerTurns", "attempts"],
9765 { turnItemTypes: [], messageRoles: ["user"] },
9766 );
9767 const providerThread = projection.providerThreads.find(
9768 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
9769 );
9770 if (providerThread === undefined) {
9771 return yield* new OrchestratorDispatchError({
9772 commandId: command.commandId,
9773 commandType: command.type,
9774 cause: "No active provider thread exists for rollback.",
9775 });
9776 }
9777 if (providerThread.providerSessionId === null) {
9778 return yield* new OrchestratorDispatchError({
9779 commandId: command.commandId,
9780 commandType: command.type,
9781 cause: `Provider thread ${providerThread.id} has no provider session.`,
9782 });
9783 }
9784
9785 const modelSelection = projection.thread.modelSelection;
9786 const capabilities = yield* providerAdapters.get(modelSelection.instanceId).pipe(
9787 Effect.flatMap((adapter) => adapter.getCapabilities()),
9788 Effect.mapError(
9789 (cause) =>
9790 new OrchestratorProviderAdapterError({
9791 commandId: command.commandId,
9792 providerInstanceId: modelSelection.instanceId,
9793 cause,
9794 }),
9795 ),
9796 );
9797 yield* enforceCommandPolicy(command)(
9798 commandPolicy.ensureRollback({
9799 commandId: command.commandId,
9800 threadId: command.threadId,
9801 providerInstanceId: modelSelection.instanceId,
9802 capabilities,
9803 }),
9804 );
9805
9806 const targetCheckpoint = projection.checkpoints.find(
9807 (candidate) => candidate.id === command.checkpointId,
9808 );
9809 if (targetCheckpoint === undefined) {
9810 return yield* new OrchestratorDispatchError({
9811 commandId: command.commandId,
9812 commandType: command.type,
9813 cause: `Checkpoint ${command.checkpointId} was not found.`,
9814 });
9815 }
9816 if (targetCheckpoint.status !== "ready") {
9817 return yield* new OrchestratorDispatchError({
9818 commandId: command.commandId,
9819 commandType: command.type,
9820 cause: `Checkpoint ${command.checkpointId} is ${targetCheckpoint.status} and cannot be restored.`,
9821 });
9822 }
9823 const targetScope = projection.checkpointScopes.find(
9824 (candidate) => candidate.id === targetCheckpoint.scopeId,
9825 );
9826 if (targetScope === undefined) {
9827 return yield* new OrchestratorDispatchError({
9828 commandId: command.commandId,
9829 commandType: command.type,
9830 cause: `Checkpoint scope ${targetCheckpoint.scopeId} was not found.`,
9831 });
9832 }
9833 if (targetScope.id !== command.scopeId) {
9834 return yield* new OrchestratorDispatchError({
9835 commandId: command.commandId,
9836 commandType: command.type,
9837 cause: `Checkpoint ${command.checkpointId} belongs to scope ${targetScope.id}, not ${command.scopeId}.`,
9838 });
9839 }
9840 if (command.restoreFiles !== false) {
9841 const isolated = yield* isCheckpointRestoreIsolated(projection.thread, targetScope, {
9842 projects,
9843 path,
9844 fileSystem,
9845 projections: projectionStore,
9846 }).pipe(
9847 Effect.mapError(
9848 (cause) =>
9849 new OrchestratorDispatchError({
9850 commandId: command.commandId,
9851 commandType: command.type,
9852 cause,
9853 }),
9854 ),
9855 );
9856 if (!isolated)
9857 return yield* new OrchestratorDispatchError({
9858 commandId: command.commandId,
9859 commandType: command.type,
9860 cause: SHARED_WORKSPACE_RESTORE_MESSAGE,
9861 });
9862 }
9863
9864 const targetOrdinal = targetCheckpoint.appRunOrdinal ?? 0;
9865 if (targetOrdinal > 0) {
9866 const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal);
9867 const targetProviderTurn =
9868 targetRun === undefined ? undefined : providerTurnForRun(projection, targetRun);
9869 if (targetRun === undefined || targetProviderTurn === undefined) {
9870 return yield* new OrchestratorDispatchError({
9871 commandId: command.commandId,
9872 commandType: command.type,
9873 cause: `Cannot roll back to checkpoint ${targetCheckpoint.id}: its provider turn is unavailable.`,
9874 });
9875 }
9876 if (targetProviderTurn.providerThreadId !== providerThread.id) {
9877 return yield* new OrchestratorDispatchError({
9878 commandId: command.commandId,
9879 commandType: command.type,
9880 cause: `Cannot roll back provider thread ${providerThread.id} to checkpoint ${targetCheckpoint.id}: target provider turn ${targetProviderTurn.id} belongs to provider thread ${targetProviderTurn.providerThreadId}.`,
9881 });
9882 }
9883 }
9884
9885 const now = yield* DateTime.now;
9886 // This rollback becomes the only one whose failure the thread records.
9887 yield* emit(
9888 events,
9889 command,
9890 )({
9891 type: "thread.metadata-updated",
9892 threadId: command.threadId,
9893 providerInstanceId: projection.thread.providerInstanceId,
9894 occurredAt: now,
9895 payload: {
9896 ...projection.thread,
9897 rollbackRequestId: command.commandId,
9898 rollbackFailure: null,
9899 updatedAt: now,
9900 },
9901 });
9902 yield* emit(
9903 events,
9904 command,
9905 )({
9906 type: "checkpoint.rollback-requested",
9907 threadId: command.threadId,
9908 providerInstanceId: modelSelection.instanceId,
9909 occurredAt: now,
9910 payload: {
9911 scopeId: targetScope.id,
9912 checkpointId: targetCheckpoint.id,
9913 requestedAt: now,
9914 },
9915 });
9916 yield* Ref.update(effects, (existing) => [
9917 ...existing,
9918 {
9919 id: `effect:${command.commandId}:provider-thread.rollback:${providerThread.id}:${targetCheckpoint.id}`,
9920 commandId: command.commandId,
9921 threadId: command.threadId,
9922 request: {
9923 type: "provider-thread.rollback",
9924 ...(command.restoreFiles === undefined ? {} : { restoreFiles: command.restoreFiles }),
9925 providerThreadId: providerThread.id,
9926 checkpointId: targetCheckpoint.id,
9927 scopeId: targetScope.id,
9928 },
9929 } satisfies PendingOrchestrationEffectV2,
9930 ]);
9931 });
9932
9933 /**
9934 * Records a provider rollback that failed after every retry, so clients
9935 * waiting on it stop and show the reason. A newer rollback clears it, and a
9936 * late failure from a rollback it superseded is rejected.
9937 */
9938 const dispatchCheckpointRollbackFail = (
9939 command: Extract<OrchestrationV2InternalCommand, { readonly type: "checkpoint.rollback.fail" }>,
9940 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9941 ) =>
9942 Effect.gen(function* () {
9943 const thread = yield* projectionStore
9944 .getThread(command.threadId)
9945 .pipe(mapDispatchError(command));
9946 if (thread.deletedAt !== null) return;
9947 if (thread.rollbackRequestId !== command.requestId) {
9948 return yield* new OrchestratorDispatchError({
9949 commandId: command.commandId,
9950 commandType: command.type,
9951 cause: `Rollback ${command.requestId} is no longer the thread's current rollback.`,
9952 });
9953 }
9954 const now = yield* DateTime.now;
9955 yield* emit(
9956 events,
9957 command,
9958 )({
9959 type: "thread.metadata-updated",
9960 threadId: command.threadId,
9961 providerInstanceId: thread.providerInstanceId,
9962 occurredAt: now,
9963 payload: {
9964 ...thread,
9965 rollbackFailure: { requestId: command.requestId, message: command.message },
9966 updatedAt: now,
9967 },
9968 });
9969 });
9970
9971 /**
9972 * Parent thread of an app-owned delegated child, or undefined when the
9973 * thread is not one. Thread lineage and fork origin are immutable, so this
9974 * is safe to read without holding either thread's dispatch lock.
9975 */
9976 const appOwnedSubagentParentThreadId = (childThreadId: ThreadId) =>
9977 Effect.gen(function* () {
9978 const childThread = yield* projectionStore.getThread(childThreadId);
9979 const lineage = childThread.lineage;
9980 return lineage.relationshipToParent === "subagent" &&
9981 lineage.parentThreadId !== null &&
9982 childThread.forkedFrom?.type === "node"
9983 ? lineage.parentThreadId
9984 : undefined;
9985 });
9986
9987 /**
9988 * A run with a pending restart continuation has no outcome yet: restart
9989 * reconciliation cancelled it mid-turn, or cancelled the background work a
9990 * settled turn was waiting on. The continuation's own run settles it, or
9991 * RestartContinuation recovers the thread when it declines to continue.
9992 */
9993 const awaitsRestartContinuation = (run: OrchestrationV2Run) =>
9994 effectOutbox
9995 .get(`effect:restart-continuation:${run.id}`)
9996 .pipe(
9997 Effect.map(
9998 Option.exists((effect) => effect.status === "pending" || effect.status === "running"),
9999 ),
10000 );
10001
10002 /**
10003 * A delegated child's result is held while its result run, or a run after it
10004 * that a second restart cut before it started, still has a continuation pending.
10005 */
10006 const childAwaitsRestartContinuation = (
10007 runs: ReadonlyArray<OrchestrationV2Run>,
10008 resultRun: OrchestrationV2Run,
10009 settledContinuationOf?: RunId,
10010 ) =>
10011 Effect.forEach(
10012 runs.filter(
10013 (run) =>
10014 run.id !== settledContinuationOf &&
10015 (run.id === resultRun.id || runRanAfter(run, resultRun)),
10016 ),
10017 awaitsRestartContinuation,
10018 ).pipe(Effect.map((pending) => pending.includes(true)));
10019
10020 const planDelegatedCompletionDelivery = Effect.fn(
10021 "orchestrationV2.planDelegatedCompletionDelivery",
10022 )(function* (input: {
10023 readonly parentProjection: Pick<
10024 OrchestrationV2ThreadProjection,
10025 "thread" | "messages" | "runs" | "subagents" | "providerTurns"
10026 >;
10027 readonly parentRun: OrchestrationV2Run | undefined;
10028 readonly task: OrchestrationV2Subagent;
10029 readonly updatedTask: OrchestrationV2Subagent;
10030 readonly now: DateTime.Utc;
10031 }) {
10032 const taskDelivery = input.updatedTask.completionDelivery;
10033 // delivered ownership has already settled through a completed wake run.
10034 // A later wake-policy upgrade must not re-claim the task or offer again.
10035 if (
10036 taskDelivery?.state === "acknowledged" ||
10037 taskDelivery?.state === "delivered" ||
10038 taskDelivery?.state === "disposed"
10039 ) {
10040 return {
10041 task: input.updatedTask,
10042 parentRun: undefined,
10043 message: undefined,
10044 offer: false,
10045 };
10046 }
10047 const cohort = input.parentRun?.delegatedCompletion;
10048 if (
10049 input.parentRun === undefined ||
10050 input.parentProjection.thread.archivedAt !== null ||
10051 input.parentProjection.thread.deletedAt !== null ||
10052 (cohort !== undefined && cohort.disposition !== "open")
10053 ) {
10054 return {
10055 task: {
10056 ...input.updatedTask,
10057 completionDelivery: {
10058 state: "disposed" as const,
10059 observedByRunId: null,
10060 },
10061 },
10062 parentRun: undefined,
10063 message: undefined,
10064 offer: false,
10065 };
10066 }
10067 // settled_only defers to a blocking delegate_task wait, which lives only as
10068 // long as the turn that spawned the task. Once that turn is over (a restart
10069 // continues it as a new run), nothing else delivers the result.
10070 if (
10071 (input.task.completionWake ?? "settled_only") === "settled_only" &&
10072 hasLiveRun({ runs: [input.parentRun] })
10073 ) {
10074 return {
10075 task: input.updatedTask,
10076 parentRun: undefined,
10077 message: undefined,
10078 offer: false,
10079 };
10080 }
10081
10082 const delivery = cohort?.delivery ?? null;
10083 const deliveryRun = completionDeliveryRun(input.parentProjection, delivery);
10084 if (delivery !== null) {
10085 if (deliveryRun?.status === "queued" && deliveryRun.userMessageId === delivery?.messageId) {
10086 const taskIds = Array.from(new Set([...delivery.taskIds, input.task.id]));
10087 const message = completionDeliveryMessage(input.parentProjection, delivery);
10088 const nextCohort = {
10089 ...cohort!,
10090 delivery: {
10091 ...delivery,
10092 taskIds,
10093 },
10094 };
10095 return {
10096 task: {
10097 ...input.updatedTask,
10098 completionDelivery: {
10099 state: "claimed" as const,
10100 observedByRunId: null,
10101 },
10102 },
10103 parentRun: {
10104 ...input.parentRun,
10105 delegatedCompletion: nextCohort,
10106 },
10107 message:
10108 message === undefined
10109 ? undefined
10110 : {
10111 ...message,
10112 text: delegatedCompletionWakeDetail(taskIds),
10113 delegatedCompletion: {
10114 parentRunId: input.parentRun.id,
10115 generation: delivery.generation,
10116 taskIds,
10117 },
10118 updatedAt: input.now,
10119 },
10120 offer: false,
10121 };
10122 }
10123 if (deliveryRun !== undefined && isBlockingRun(deliveryRun)) {
10124 return {
10125 task: {
10126 ...input.updatedTask,
10127 completionDelivery: {
10128 state: "pending" as const,
10129 observedByRunId: null,
10130 },
10131 },
10132 parentRun: undefined,
10133 message: undefined,
10134 offer: false,
10135 };
10136 }
10137 if (deliveryRun !== undefined) {
10138 // The terminal-run listener owns reconciliation of a completed wake.
10139 // A sibling that wins the parent lock first remains pending for the
10140 // successor that listener reserves, rather than creating a competing
10141 // delivery.
10142 return {
10143 task: {
10144 ...input.updatedTask,
10145 completionDelivery: {
10146 state: "pending" as const,
10147 observedByRunId: null,
10148 },
10149 },
10150 parentRun: undefined,
10151 message: undefined,
10152 offer: false,
10153 };
10154 }
10155 const taskIds = Array.from(new Set([...delivery.taskIds, input.task.id]));
10156 const nextCohort = {
10157 ...cohort!,
10158 delivery: {
10159 ...delivery,
10160 taskIds,
10161 },
10162 };
10163 return {
10164 task: {
10165 ...input.updatedTask,
10166 completionDelivery: {
10167 state: "claimed" as const,
10168 observedByRunId: null,
10169 },
10170 },
10171 parentRun: {
10172 ...input.parentRun,
10173 delegatedCompletion: nextCohort,
10174 },
10175 message: undefined,
10176 offer: true,
10177 };
10178 }
10179
10180 const generation = cohort?.nextGeneration ?? 1;
10181 const messageId = yield* mapDelegatedCompletionError(
10182 idAllocator.allocate.message({
10183 threadId: input.parentRun.threadId,
10184 ordinal:
10185 (yield* mapDelegatedCompletionError(
10186 projectionStore.getMessageCount(input.parentRun.threadId),
10187 )) + 1,
10188 }),
10189 );
10190 const nextCohort = {
10191 disposition: "open" as const,
10192 nextGeneration: generation + 1,
10193 delivery: {
10194 generation,
10195 messageId,
10196 taskIds: [input.task.id],
10197 },
10198 };
10199 return {
10200 task: {
10201 ...input.updatedTask,
10202 completionDelivery: {
10203 state: "claimed" as const,
10204 observedByRunId: null,
10205 },
10206 },
10207 parentRun: {
10208 ...input.parentRun,
10209 delegatedCompletion: nextCohort,
10210 },
10211 message: undefined,
10212 offer: true,
10213 };
10214 });
10215
10216 /**
10217 * Transfers a terminal child's result into its parent and offers the parent
10218 * wake. Every mutation here targets the PARENT thread, so callers must hold
10219 * the parent thread's dispatch lock rather than the child's: the
10220 * delegated_task.wake-policy handler rewrites the same subagent row under
10221 * that lock with a full-row payload, and unserialized writers clobber each
10222 * other (stale policy on the terminal row, or a terminal row regressed to
10223 * running). `settledContinuationOf` names the restart continuation that just
10224 * declined or failed, so its own still-running effect does not hold the child.
10225 */
10226 const finalizeAppOwnedSubagentTask = (
10227 childControls: ProjectionRecords<
10228 "runs" | "messages" | "subagents" | "providerThreads" | "providerTurns" | "attempts"
10229 >,
10230 options: {
10231 readonly settledContinuationOf?: RunId;
10232 readonly taskId: NodeId;
10233 readonly resultRun?: OrchestrationV2Run;
10234 },
10235 ) =>
10236 Effect.gen(function* () {
10237 const childThreadId = childControls.thread.id;
10238 const forkedFrom = childControls.thread.forkedFrom;
10239 if (
10240 childControls.thread.lineage.relationshipToParent !== "subagent" ||
10241 childControls.thread.lineage.parentThreadId === null ||
10242 forkedFrom?.type !== "node"
10243 ) {
10244 return;
10245 }
10246 const taskRuns = childControls.runs.filter((run) => run.delegatedTaskId === options.taskId);
10247 const childRun =
10248 taskRuns.length === 0
10249 ? options.taskId === forkedFrom.nodeId
10250 ? options.resultRun
10251 : undefined
10252 : taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0];
10253 if (childRun === undefined) return;
10254 const outcome = runOutcome(childRun);
10255 const terminalStatus = delegatedTaskTerminalStatus(outcome);
10256 if (terminalStatus === null) {
10257 return;
10258 }
10259 if (
10260 yield* childAwaitsRestartContinuation(
10261 taskRuns.length === 0 ? childControls.runs : taskRuns,
10262 childRun,
10263 options?.settledContinuationOf,
10264 )
10265 ) {
10266 return;
10267 }
10268
10269 const childResult = yield* projectionStore.getThreadRecords(
10270 childThreadId,
10271 ["messages", "turnItems"],
10272 {
10273 messageRoles: ["assistant"],
10274 messageRunIds: [childRun.id],
10275 turnItemRunId: childRun.id,
10276 turnItemTypes: ["assistant_message", "error"],
10277 },
10278 );
10279 const childProjection = {
10280 ...childControls,
10281 messages: childResult.messages,
10282 turnItems: childResult.turnItems,
10283 };
10284 const parentThreadId = childControls.thread.lineage.parentThreadId;
10285 const parentProjection = yield* projectionStore.getThreadRecords(
10286 parentThreadId,
10287 [
10288 "runs",
10289 "messages",
10290 "subagents",
10291 "providerThreads",
10292 "providerTurns",
10293 "nodes",
10294 "turnItems",
10295 "contextTransfers",
10296 ],
10297 { turnItemTypes: ["subagent"], messageRoles: ["user"] },
10298 );
10299 const task = parentProjection.subagents.find(
10300 (candidate) =>
10301 candidate.id === options.taskId &&
10302 candidate.origin === "app_owned" &&
10303 candidate.childThreadId === childThreadId,
10304 );
10305 if (task === undefined) {
10306 return;
10307 }
10308 const existingResultTransfer = parentProjection.contextTransfers.find(
10309 (transfer) =>
10310 transfer.type === "subagent_result" &&
10311 transfer.sourceThreadId === childThreadId &&
10312 transfer.targetThreadId === parentThreadId &&
10313 (transfer.sourcePoint.runId === childRun.id ||
10314 (options.taskId === forkedFrom.nodeId && transfer.sourcePoint.runId === undefined)),
10315 );
10316 if (existingResultTransfer !== undefined) {
10317 return;
10318 }
10319
10320 const now = yield* DateTime.now;
10321 const result = subagentResultForRun(childProjection, { ...childRun, status: outcome });
10322 const parentRun =
10323 task.runId === null
10324 ? undefined
10325 : parentProjection.runs.find((candidate) => candidate.id === task.runId);
10326 const parentNode = parentProjection.nodes.find((candidate) => candidate.id === task.id);
10327 const parentTurnItem = parentProjection.turnItems.find(
10328 (candidate) => candidate.type === "subagent" && candidate.subagentId === task.id,
10329 );
10330 const updatedTask: OrchestrationV2Subagent = {
10331 ...task,
10332 providerThreadId: childRun.providerThreadId,
10333 status: terminalStatus,
10334 result: result.text,
10335 completedAt: now,
10336 updatedAt: now,
10337 };
10338 const completionPlan = yield* planDelegatedCompletionDelivery({
10339 parentProjection,
10340 parentRun,
10341 task,
10342 updatedTask,
10343 now,
10344 });
10345 const resultTransferId = yield* idAllocator.allocate.contextTransfer({
10346 sourceThreadId: childThreadId,
10347 targetThreadId: parentThreadId,
10348 type: "subagent_result",
10349 });
10350 const childProviderThread =
10351 childRun.providerThreadId === null
10352 ? undefined
10353 : childProjection.providerThreads.find(
10354 (candidate) => candidate.id === childRun.providerThreadId,
10355 );
10356 const parentProviderThread =
10357 parentRun?.providerThreadId === null || parentRun?.providerThreadId === undefined
10358 ? undefined
10359 : parentProjection.providerThreads.find(
10360 (candidate) => candidate.id === parentRun.providerThreadId,
10361 );
10362 const resultHandoff: OrchestrationV2ContextHandoff | null =
10363 parentRun === undefined ||
10364 childProviderThread === undefined ||
10365 parentProviderThread === undefined
10366 ? null
10367 : {
10368 id: yield* idAllocator.allocate.contextHandoff({
10369 threadId: parentThreadId,
10370 fromProviderInstanceId: childRun.providerInstanceId,
10371 toProviderInstanceId: parentRun.providerInstanceId,
10372 }),
10373 transferId: resultTransferId,
10374 threadId: parentThreadId,
10375 targetRunId: parentRun.id,
10376 fromProviderThreadIds: [childProviderThread.id],
10377 toProviderThreadId: parentProviderThread.id,
10378 coveredRunOrdinals: {
10379 from: childRun.ordinal,
10380 to: childRun.ordinal,
10381 },
10382 strategy: "manual_context",
10383 status: "ready",
10384 summaryMessageId: result.messageId,
10385 summaryText: result.text,
10386 createdByProviderInstanceId: childRun.providerInstanceId,
10387 createdAt: now,
10388 updatedAt: now,
10389 };
10390 const resultTransfer: OrchestrationV2ContextTransfer = {
10391 id: resultTransferId,
10392 type: "subagent_result",
10393 sourceThreadId: childThreadId,
10394 targetThreadId: parentThreadId,
10395 sourcePoint: {
10396 ...contextSourcePointForRun(childProjection, childRun),
10397 ...(result.turnItemId === null ? {} : { turnItemId: result.turnItemId }),
10398 },
10399 basePoint: null,
10400 sourceProviderInstanceId: childRun.providerInstanceId,
10401 targetProviderInstanceId:
10402 parentRun?.providerInstanceId ?? parentProjection.thread.providerInstanceId,
10403 targetRunId: parentRun?.id ?? null,
10404 status: "consumed",
10405 resolution:
10406 resultHandoff === null
10407 ? null
10408 : {
10409 strategy: "portable_context",
10410 contextHandoffId: resultHandoff.id,
10411 },
10412 createdBy: "system",
10413 error: null,
10414 createdAt: now,
10415 updatedAt: now,
10416 consumedAt: now,
10417 };
10418
10419 yield* writeSystemEvents([
10420 {
10421 type: "subagent.updated",
10422 threadId: parentThreadId,
10423 ...(task.runId === null ? {} : { runId: task.runId }),
10424 nodeId: task.id,
10425 driver: task.driver,
10426 occurredAt: now,
10427 payload: completionPlan.task,
10428 },
10429 ...(completionPlan.parentRun === undefined
10430 ? []
10431 : [
10432 {
10433 type: "run.updated" as const,
10434 threadId: parentThreadId,
10435 runId: completionPlan.parentRun.id,
10436 ...(completionPlan.parentRun.rootNodeId === null
10437 ? {}
10438 : { nodeId: completionPlan.parentRun.rootNodeId }),
10439 providerInstanceId: completionPlan.parentRun.providerInstanceId,
10440 occurredAt: now,
10441 payload: completionPlan.parentRun,
10442 },
10443 ]),
10444 ...(completionPlan.message === undefined
10445 ? []
10446 : [
10447 {
10448 type: "message.updated" as const,
10449 threadId: parentThreadId,
10450 ...(completionPlan.message.runId === null
10451 ? {}
10452 : { runId: completionPlan.message.runId }),
10453 ...(completionPlan.message.nodeId === null
10454 ? {}
10455 : { nodeId: completionPlan.message.nodeId }),
10456 providerInstanceId:
10457 completionPlan.parentRun?.providerInstanceId ??
10458 parentProjection.thread.providerInstanceId,
10459 occurredAt: now,
10460 payload: completionPlan.message,
10461 },
10462 ]),
10463 ...(parentNode === undefined
10464 ? []
10465 : [
10466 {
10467 type: "node.updated" as const,
10468 threadId: parentThreadId,
10469 ...(parentNode.runId === null ? {} : { runId: parentNode.runId }),
10470 nodeId: parentNode.id,
10471 driver: task.driver,
10472 occurredAt: now,
10473 payload: {
10474 ...parentNode,
10475 status: terminalStatus,
10476 providerThreadId: childRun.providerThreadId,
10477 completedAt: now,
10478 },
10479 },
10480 ]),
10481 ...(parentTurnItem === undefined
10482 ? []
10483 : [
10484 {
10485 type: "turn-item.updated" as const,
10486 threadId: parentThreadId,
10487 ...(parentTurnItem.runId === null ? {} : { runId: parentTurnItem.runId }),
10488 ...(parentTurnItem.nodeId === null ? {} : { nodeId: parentTurnItem.nodeId }),
10489 driver: task.driver,
10490 occurredAt: now,
10491 payload: {
10492 ...parentTurnItem,
10493 status: terminalStatus,
10494 result: result.text,
10495 completedAt: now,
10496 updatedAt: now,
10497 },
10498 },
10499 ]),
10500 ...(resultHandoff === null
10501 ? []
10502 : [
10503 {
10504 type: "context-handoff.updated" as const,
10505 threadId: parentThreadId,
10506 ...(parentRun === undefined ? {} : { runId: parentRun.id }),
10507 providerInstanceId: childRun.providerInstanceId,
10508 occurredAt: now,
10509 payload: resultHandoff,
10510 },
10511 ]),
10512 {
10513 type: "context-transfer.created",
10514 threadId: parentThreadId,
10515 ...(parentRun === undefined ? {} : { runId: parentRun.id }),
10516 providerInstanceId: childRun.providerInstanceId,
10517 occurredAt: now,
10518 payload: resultTransfer,
10519 },
10520 ]);
10521
10522 if (completionPlan.offer && completionPlan.parentRun !== undefined) {
10523 yield* offerDelegatedCompletionDelivery(parentThreadId, completionPlan.parentRun.id);
10524 }
10525 });
10526
10527 const finalizeAppOwnedSubagent = (
10528 childThreadId: ThreadId,
10529 options?: { readonly settledContinuationOf?: RunId },
10530 ) =>
10531 Effect.gen(function* () {
10532 const childControls = yield* projectionStore.getThreadRecords(
10533 childThreadId,
10534 ["runs", "messages", "subagents", "providerThreads", "providerTurns", "attempts"],
10535 { messageRoles: ["user"] },
10536 );
10537 const parentThreadId = childControls.thread.lineage.parentThreadId;
10538 if (
10539 childControls.thread.lineage.relationshipToParent !== "subagent" ||
10540 parentThreadId === null ||
10541 childControls.thread.forkedFrom?.type !== "node"
10542 )
10543 return;
10544 const progress = delegatedTaskProgress(childControls);
10545 const parent = yield* projectionStore.getThreadRecords(parentThreadId, [
10546 "subagents",
10547 "contextTransfers",
10548 ]);
10549 for (const task of parent.subagents) {
10550 if (
10551 task.origin !== "app_owned" ||
10552 task.childThreadId !== childThreadId ||
10553 (task.id === childControls.thread.forkedFrom.nodeId &&
10554 progress.state !== "result_available") ||
10555 (task.result !== null &&
10556 parent.contextTransfers.some(
10557 (transfer) =>
10558 transfer.type === "subagent_result" && transfer.sourceThreadId === childThreadId,
10559 ))
10560 )
10561 continue;
10562 yield* finalizeAppOwnedSubagentTask(childControls, {
10563 ...options,
10564 taskId: task.id,
10565 ...(progress.resultRun === undefined ? {} : { resultRun: progress.resultRun }),
10566 });
10567 }
10568
10569 const { resultRun } = progress;
10570 if (progress.state !== "result_available" || resultRun === undefined) return;
10571 const status = delegatedTaskTerminalStatus(runOutcome(resultRun));
10572 if (status !== "cancelled" && status !== "interrupted") return;
10573 if (
10574 yield* childAwaitsRestartContinuation(
10575 childControls.runs,
10576 resultRun,
10577 options?.settledContinuationOf,
10578 )
10579 )
10580 return;
10581 // Cancelled or interrupted tasks must not leave async questions that resume the child.
10582 // Callers hold the parent lock; dismiss takes the child lock, as follow-up intake does.
10583 const { runtimeRequests } = yield* projectionStore.getThreadRecords(childThreadId, [
10584 "runtimeRequests",
10585 ]);
10586 const pendingQuestions = runtimeRequests.filter(
10587 (request) =>
10588 request.status === "pending" &&
10589 request.kind === "user_input" &&
10590 request.responseCapability.type === "message",
10591 );
10592 if (pendingQuestions.length === 0) return;
10593 const { nodes } = yield* projectionStore.getThreadRecords(childThreadId, ["nodes"]);
10594 const resultRunIds = new Set<RunId>();
10595 let run: OrchestrationV2Run | undefined = resultRun;
10596 while (run !== undefined && !resultRunIds.has(run.id)) {
10597 resultRunIds.add(run.id);
10598 const previousRunId: RunId | undefined = run.restartContinuationOfRunId;
10599 run =
10600 previousRunId === undefined
10601 ? undefined
10602 : childControls.runs.find((candidate) => candidate.id === previousRunId);
10603 }
10604 for (const request of pendingQuestions) {
10605 if (
10606 nodes.some(
10607 (node) =>
10608 node.id === request.nodeId && node.runId !== null && resultRunIds.has(node.runId),
10609 )
10610 ) {
10611 yield* dispatchWithReceipt({
10612 type: "thread.user-input.dismiss",
10613 commandId: CommandId.make(`command:system:delegated-task-question:${request.id}`),
10614 threadId: childThreadId,
10615 requestId: request.id,
10616 }).pipe(
10617 Effect.catchCause((cause) =>
10618 Effect.logWarning("Failed to dismiss task question", {
10619 requestId: request.id,
10620 cause,
10621 }),
10622 ),
10623 );
10624 }
10625 }
10626 });
10627
10628 const finalizeDelegatedCompletionDelivery = (threadId: ThreadId, runId: RunId) =>
10629 Effect.gen(function* () {
10630 // Ordinary turns do not own a delegated delivery. Read just their input
10631 // before loading the cohort state needed to settle an actual delivery.
10632 const message = yield* projectionStore.getRunMessage(threadId, runId);
10633 if (message?.delegatedCompletion === undefined) return;
10634 const projection = yield* projectionStore.getThreadRecords(
10635 threadId,
10636 ["runs", "messages", "subagents"],
10637 { messageRoles: ["user"] },
10638 );
10639 const deliveryRun = projection.runs.find((candidate) => candidate.id === runId);
10640 const deliveryMessage =
10641 deliveryRun === undefined
10642 ? undefined
10643 : projection.messages.find((candidate) => candidate.id === deliveryRun.userMessageId);
10644 const messageOwnership = deliveryMessage?.delegatedCompletion;
10645 if (deliveryRun === undefined || messageOwnership === undefined) {
10646 return;
10647 }
10648 const parentRun = projection.runs.find(
10649 (candidate) => candidate.id === messageOwnership.parentRunId,
10650 );
10651 const cohort = parentRun?.delegatedCompletion;
10652 const delivery = cohort?.delivery;
10653 if (
10654 parentRun === undefined ||
10655 cohort === undefined ||
10656 delivery === null ||
10657 delivery === undefined ||
10658 delivery.messageId !== deliveryRun.userMessageId ||
10659 delivery.generation !== messageOwnership.generation
10660 ) {
10661 return;
10662 }
10663
10664 const now = yield* DateTime.now;
10665 const nextTaskStates = new Map<
10666 OrchestrationV2Subagent["id"],
10667 OrchestrationV2Subagent["completionDelivery"]
10668 >();
10669 for (const task of projection.subagents) {
10670 if (
10671 task.origin !== "app_owned" ||
10672 task.runId !== parentRun.id ||
10673 !delivery.taskIds.includes(task.id) ||
10674 task.completionDelivery?.state !== "claimed"
10675 ) {
10676 continue;
10677 }
10678 nextTaskStates.set(task.id, {
10679 state: deliveryRun.status === "cancelled" ? "pending" : "delivered",
10680 observedByRunId: null,
10681 });
10682 }
10683 const pendingTaskIds = projection.subagents
10684 .filter(
10685 (task) =>
10686 task.origin === "app_owned" &&
10687 task.runId === parentRun.id &&
10688 isTerminalDelegatedTaskStatus(task.status) &&
10689 (task.completionDelivery?.state === "pending" ||
10690 nextTaskStates.get(task.id)?.state === "pending"),
10691 )
10692 .map((task) => task.id);
10693 // Results that arrived while this delivery was outstanding go out
10694 // together in one successor. Each child becomes pending once, so a
10695 // cohort's successors are bounded by its children.
10696 const canReserveFollowUp =
10697 cohort.disposition === "open" &&
10698 projection.thread.archivedAt === null &&
10699 projection.thread.deletedAt === null &&
10700 pendingTaskIds.length > 0;
10701 const nextDelivery = canReserveFollowUp
10702 ? {
10703 generation: cohort.nextGeneration,
10704 messageId: yield* mapDelegatedCompletionError(
10705 idAllocator.allocate.message({
10706 threadId,
10707 ordinal:
10708 (yield* mapDelegatedCompletionError(
10709 projectionStore.getMessageCount(projection.thread.id),
10710 )) + 1,
10711 }),
10712 ),
10713 taskIds: pendingTaskIds,
10714 }
10715 : null;
10716 if (nextDelivery !== null) {
10717 for (const taskId of pendingTaskIds) {
10718 nextTaskStates.set(taskId, {
10719 state: "claimed",
10720 observedByRunId: null,
10721 });
10722 }
10723 }
10724 const updatedParentRun: OrchestrationV2Run = {
10725 ...parentRun,
10726 delegatedCompletion: {
10727 ...cohort,
10728 nextGeneration: nextDelivery === null ? cohort.nextGeneration : cohort.nextGeneration + 1,
10729 delivery: nextDelivery,
10730 },
10731 };
10732 const taskEvents = projection.subagents.flatMap((task) => {
10733 const completionDelivery = nextTaskStates.get(task.id);
10734 if (completionDelivery === undefined) {
10735 return [];
10736 }
10737 return [
10738 {
10739 type: "subagent.updated" as const,
10740 threadId,
10741 ...(task.runId === null ? {} : { runId: task.runId }),
10742 nodeId: task.id,
10743 driver: task.driver,
10744 providerInstanceId: task.providerInstanceId,
10745 occurredAt: now,
10746 payload: {
10747 ...task,
10748 completionDelivery,
10749 updatedAt: now,
10750 },
10751 },
10752 ];
10753 });
10754 yield* writeSystemEvents([
10755 ...taskEvents,
10756 {
10757 type: "run.updated",
10758 threadId,
10759 runId: updatedParentRun.id,
10760 ...(updatedParentRun.rootNodeId === null ? {} : { nodeId: updatedParentRun.rootNodeId }),
10761 providerInstanceId: updatedParentRun.providerInstanceId,
10762 occurredAt: now,
10763 payload: updatedParentRun,
10764 },
10765 ]);
10766 if (nextDelivery !== null) {
10767 yield* offerDelegatedCompletionDelivery(threadId, parentRun.id);
10768 }
10769 });
10770
10771 const dispatchNotificationAccepted = Effect.fn("orchestrationV2.notificationAccepted")(function* (
10772 command: Extract<OrchestrationV2Command, { type: "notification.delivery.accept" }>,
10773 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
10774 ) {
10775 const projection = yield* getProjectionWithPendingEvents(command.threadId, events);
10776 const message = projection.messages.find((row) => row.id === command.messageId);
10777 const ownership = message?.delegatedCompletion;
10778 const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
10779 const cohort = parentRun?.delegatedCompletion;
10780 const delivery = cohort?.delivery;
10781 const now = yield* DateTime.now;
10782 const emitEvent = emit(events, command);
10783 if (
10784 parentRun === undefined ||
10785 cohort === undefined ||
10786 cohort.disposition !== "open" ||
10787 delivery == null ||
10788 delivery.messageId !== command.messageId ||
10789 delivery.generation !== ownership?.generation
10790 ) {
10791 yield* emitEvent({
10792 type: "thread.metadata-updated",
10793 threadId: command.threadId,
10794 occurredAt: now,
10795 payload: projection.thread,
10796 });
10797 return;
10798 }
10799 // settled_only tasks defer only to a blocking wait in their spawning run.
10800 const parentIsLive = hasLiveRun({ runs: [parentRun] });
10801 const pendingTaskIds =
10802 projection.thread.archivedAt === null && projection.thread.deletedAt === null
10803 ? projection.subagents
10804 .filter(
10805 (task) =>
10806 task.origin === "app_owned" &&
10807 task.runId === parentRun.id &&
10808 task.completionDelivery?.state === "pending" &&
10809 isTerminalDelegatedTaskStatus(task.status) &&
10810 (!parentIsLive || task.completionWake === "always"),
10811 )
10812 .map((task) => task.id)
10813 : [];
10814 const nextDelivery =
10815 pendingTaskIds.length === 0
10816 ? null
10817 : {
10818 generation: cohort.nextGeneration,
10819 messageId: yield* mapDelegatedCompletionError(
10820 idAllocator.allocate.message({
10821 threadId: command.threadId,
10822 ordinal:
10823 (yield* mapDelegatedCompletionError(
10824 projectionStore.getMessageCount(projection.thread.id),
10825 )) + 1,
10826 }),
10827 ),
10828 taskIds: pendingTaskIds,
10829 };
10830 const acceptedIds = new Set(delivery.taskIds);
10831 const pendingIds = new Set(pendingTaskIds);
10832 for (const task of projection.subagents) {
10833 const state = pendingIds.has(task.id)
10834 ? "claimed"
10835 : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
10836 ? "delivered"
10837 : undefined;
10838 if (state === undefined) continue;
10839 yield* emitEvent({
10840 type: "subagent.updated",
10841 threadId: command.threadId,
10842 runId: parentRun.id,
10843 nodeId: task.id,
10844 driver: task.driver,
10845 providerInstanceId: task.providerInstanceId,
10846 occurredAt: now,
10847 payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
10848 });
10849 }
10850 // Provider acceptance drains this batch but does not acknowledge its results.
10851 // task_status owns acknowledgment.
10852 yield* emitEvent({
10853 type: "run.updated",
10854 threadId: command.threadId,
10855 runId: parentRun.id,
10856 providerInstanceId: parentRun.providerInstanceId,
10857 occurredAt: now,
10858 payload: {
10859 ...parentRun,
10860 delegatedCompletion: {
10861 ...cohort,
10862 nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
10863 delivery: nextDelivery,
10864 },
10865 },
10866 });
10867 });
10868
10869 const dispatchUnsupported = (command: OrchestrationV2ServerCommand) =>
10870 Effect.fail(
10871 new OrchestratorDispatchError({
10872 commandId: command.commandId,
10873 commandType: command.type,
10874 }),
10875 );
10876
10877 const dispatchOnce = Effect.fn("orchestrationV2.dispatch.once")(function* (
10878 command: OrchestrationV2ServerCommand,
10879 ): Effect.fn.Return<
10880 {
10881 readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
10882 readonly effects: ReadonlyArray<PendingOrchestrationEffectV2>;
10883 readonly cancelUnsettledEffects?: {
10884 readonly effectTypes: ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
10885 readonly reason: string;
10886 };
10887 },
10888 OrchestratorV2Error
10889 > {
10890 yield* Effect.annotateCurrentSpan({
10891 "orchestration_v2.command_id": command.commandId,
10892 "orchestration_v2.command_type": command.type,
10893 "orchestration_v2.thread_id": commandThreadId(command),
10894 });
10895
10896 const events = yield* Ref.make<Array<OrchestrationV2DomainEvent>>([]);
10897 const effects = yield* Ref.make<Array<PendingOrchestrationEffectV2>>([]);
10898 let cancelUnsettledEffects:
10899 | {
10900 readonly effectTypes: ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
10901 readonly reason: string;
10902 }
10903 | undefined;
10904 switch (command.type) {
10905 case "thread.create":
10906 yield* dispatchThreadCreate(command, events);
10907 break;
10908 case "thread.visit":
10909 yield* dispatchThreadVisit(command, events);
10910 break;
10911 case "thread.ci-failed": {
10912 const thread = yield* projectionStore
10913 .getThreadShell(command.threadId)
10914 .pipe(mapDispatchError(command));
10915 const sequence = yield* eventSink
10916 .latestSequence({ threadId: command.threadId })
10917 .pipe(mapDispatchError(command));
10918 const records = yield* projectionStore
10919 .getThreadRecords(command.threadId, ["runs", "runtimeRequests"])
10920 .pipe(mapDispatchError(command));
10921 if (
10922 thread === null ||
10923 !isManagedWorkspaceArchiveCandidate(thread) ||
10924 sequence > command.snapshotSequence ||
10925 records.runs.some((run) =>
10926 ["queued", "preparing", "starting", "running", "waiting"].includes(run.status),
10927 ) ||
10928 records.runtimeRequests.some((request) => request.status === "pending")
10929 ) {
10930 return yield* new OrchestratorDispatchError({
10931 commandId: command.commandId,
10932 commandType: command.type,
10933 cause: "The thread changed before the CI failure was recorded.",
10934 });
10935 }
10936 if (
10937 yield* projectionStore
10938 .getTurnItem({ threadId: command.threadId, itemId: command.itemId })
10939 .pipe(mapDispatchError(command))
10940 )
10941 break;
10942 const now = yield* DateTime.now;
10943 yield* dispatchThreadMutation(
10944 {
10945 type: "thread.unsettle",
10946 commandId: command.commandId,
10947 threadId: command.threadId,
10948 reason: "user",
10949 },
10950 events,
10951 effects,
10952 );
10953 const ordinal = yield* projectionStore
10954 .getNextTurnItemOrdinal(command.threadId)
10955 .pipe(mapDispatchError(command));
10956 yield* emit(
10957 events,
10958 command,
10959 )({
10960 type: "turn-item.updated",
10961 threadId: command.threadId,
10962 occurredAt: now,
10963 payload: {
10964 type: "notification",
10965 source: { kind: "system" },
10966 outcome: "failed",
10967 summary: `CI failed for PR #${command.pullRequestNumber}`,
10968 detail: `[Open PR #${command.pullRequestNumber} in Mergebot](<${command.url}>)`,
10969 id: command.itemId,
10970 threadId: command.threadId,
10971 runId: null,
10972 nodeId: null,
10973 providerThreadId: null,
10974 providerTurnId: null,
10975 nativeItemRef: null,
10976 parentItemId: null,
10977 ordinal,
10978 status: "completed",
10979 title: null,
10980 startedAt: now,
10981 completedAt: now,
10982 updatedAt: now,
10983 },
10984 });
10985 break;
10986 }
10987 case "thread.auto-settle": {
10988 // Automatic settlement (#8600): the sweep evaluated a shell snapshot,
10989 // so re-check against the live thread before settling. Any change
10990 // after the snapshot — or any explicit override, including the
10991 // un-settle button's "active" — wins over the sweep.
10992 const thread = yield* projectionStore
10993 .getThread(command.threadId)
10994 .pipe(
10995 Effect.mapError(
10996 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
10997 ),
10998 );
10999 if (
11000 thread.settledOverride !== null ||
11001 DateTime.toEpochMillis(thread.updatedAt) > DateTime.toEpochMillis(command.snapshotAt)
11002 ) {
11003 return yield* new OrchestratorDispatchError({
11004 commandId: command.commandId,
11005 commandType: command.type,
11006 cause: `Thread ${command.threadId} changed before automatic settlement.`,
11007 });
11008 }
11009 yield* dispatchThreadMutation(
11010 {
11011 type: "thread.settle",
11012 commandId: command.commandId,
11013 threadId: command.threadId,
11014 ...(command.settledAt === undefined ? {} : { settledAt: command.settledAt }),
11015 },
11016 events,
11017 effects,
11018 );
11019 break;
11020 }
11021 case "thread.delete": {
11022 const projection = yield* projectionStore
11023 .getThreadRecords(command.threadId, [
11024 "runs",
11025 "attempts",
11026 "nodes",
11027 "runtimeRequests",
11028 "subagents",
11029 "providerSessions",
11030 ])
11031 .pipe(
11032 Effect.mapError(
11033 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
11034 ),
11035 );
11036 return yield* mapDispatchError(command)(
11037 planThreadDeletion({
11038 command,
11039 projection,
11040 attachmentIds: yield* projectionStore
11041 .getThreadAttachmentIds(command.threadId)
11042 .pipe(mapDispatchError(command)),
11043 now: yield* DateTime.now,
11044 idAllocator,
11045 }),
11046 );
11047 }
11048 case "thread.archive":
11049 case "thread.unarchive":
11050 case "thread.settle":
11051 case "thread.unsettle":
11052 case "thread.snooze":
11053 case "thread.unsnooze":
11054 case "thread.auto-settle.set":
11055 case "thread.pin":
11056 case "thread.unpin":
11057 case "thread.pin.reorder":
11058 case "thread.active.reorder":
11059 case "thread.mark-unread":
11060 case "thread.metadata.update":
11061 case "thread.pull-request.link":
11062 case "thread.pull-request.unlink":
11063 case "thread.pull-request-link.sync":
11064 case "thread.pull-request.watch":
11065 case "thread.pull-request.sync":
11066 case "thread.title.regeneration.complete":
11067 case "thread.runtime-mode.set":
11068 case "thread.interaction-mode.set":
11069 case "thread.model-selection.set":
11070 case "provider.switch":
11071 yield* dispatchThreadMutation(command, events, effects);
11072 break;
11073 case "thread.pull-request-watch.sync":
11074 yield* dispatchPullRequestWatchSync(command, events, effects);
11075 break;
11076 case "provider-session.detach":
11077 yield* dispatchProviderSessionDetach(command, events, effects);
11078 break;
11079 case "message.dispatch": {
11080 // The provider owns a native subagent's conversation, so a sent
11081 // message has nowhere to go. Answers to a subagent's questions never
11082 // target it either: adapters ask them on the top-level parent thread.
11083 const thread = yield* projectionStore
11084 .getThread(command.threadId)
11085 .pipe(
11086 Effect.mapError(
11087 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
11088 ),
11089 );
11090 if (isProviderNativeSubagentThread(thread)) {
11091 return yield* new OrchestratorSubagentThreadReadOnlyError({
11092 commandId: command.commandId,
11093 threadId: command.threadId,
11094 });
11095 }
11096 yield* dispatchMessage(command, events, effects);
11097 yield* recordSubagentFollowup(command, events).pipe(mapDispatchError(command));
11098 break;
11099 }
11100 case "notification.delivery.accept":
11101 yield* dispatchNotificationAccepted(command, events);
11102 break;
11103 case "prepared-run.release":
11104 yield* dispatchPreparedRunRelease(command, events, effects);
11105 break;
11106 case "prepared-run.progress":
11107 yield* dispatchPreparedRunProgress(command, events);
11108 break;
11109 case "prepared-run.fail":
11110 yield* dispatchPreparedRunFail(command, events);
11111 break;
11112 case "prepared-run.retry":
11113 yield* dispatchPreparedRunRetry(command, events);
11114 break;
11115 case "runtime-request.respond":
11116 yield* dispatchRuntimeRequestRespond(command, events, effects);
11117 break;
11118 case "thread.user-input.dismiss":
11119 yield* dispatchThreadUserInputDismiss(command, events, effects);
11120 break;
11121 case "subagent.stop":
11122 yield* dispatchSubagentStop(command, events, effects);
11123 break;
11124 case "run.interrupt":
11125 cancelUnsettledEffects = yield* dispatchRunInterrupt(command, events, effects);
11126 // Stop also stops every delegated task under the thread once it commits.
11127 if (command.holdQueue === true) {
11128 yield* Ref.update(effects, (existing) => [
11129 ...existing,
11130 {
11131 id: `effect:${command.commandId}:delegated-tasks.stop`,
11132 commandId: command.commandId,
11133 threadId: command.threadId,
11134 request: {
11135 type: "delegated-tasks.stop",
11136 ...(command.reason === undefined ? {} : { reason: command.reason }),
11137 },
11138 } satisfies PendingOrchestrationEffectV2,
11139 ]);
11140 }
11141 break;
11142 case "thread.stop":
11143 cancelUnsettledEffects = yield* dispatchThreadStop(command, events, effects);
11144 break;
11145 case "queued-message.promote-to-steer":
11146 yield* dispatchQueuedMessagePromoteToSteer(command, events, effects);
11147 break;
11148 case "queue.resume": {
11149 const projection = yield* loadProjectionForCommand(
11150 command,
11151 ["runs", "turnItems", "providerSessions"],
11152 { turnItemTypes: ["error"] },
11153 );
11154 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
11155 return yield* new OrchestratorDispatchError({
11156 commandId: command.commandId,
11157 commandType: command.type,
11158 cause: `Thread ${command.threadId} is not active.`,
11159 });
11160 }
11161 const sessionError =
11162 projection.providerSessions
11163 .filter(
11164 (session) => session.providerInstanceId === projection.thread.providerInstanceId,
11165 )
11166 .toSorted(
11167 (left, right) =>
11168 DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
11169 )[0]?.lastError ?? null;
11170 if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
11171 return yield* new OrchestratorDispatchError({
11172 commandId: command.commandId,
11173 commandType: command.type,
11174 cause: "Continue the limited thread before resuming its queue.",
11175 });
11176 }
11177 const now = yield* DateTime.now;
11178 const queued = projection.runs.filter((run) => run.status === "queued");
11179 for (const run of queued) {
11180 yield* emit(
11181 events,
11182 command,
11183 )({
11184 type: "run.updated",
11185 threadId: command.threadId,
11186 runId: run.id,
11187 providerInstanceId: run.providerInstanceId,
11188 occurredAt: now,
11189 payload: { ...run, queueHeld: false },
11190 });
11191 }
11192 if (queued.length === 0) {
11193 yield* emit(
11194 events,
11195 command,
11196 )({
11197 type: "thread.metadata-updated",
11198 threadId: command.threadId,
11199 occurredAt: now,
11200 payload: projection.thread,
11201 });
11202 }
11203 break;
11204 }
11205 case "queued-run.reorder":
11206 yield* dispatchQueuedRunReorder(command, events);
11207 break;
11208 case "queued-run.cancel":
11209 yield* dispatchQueuedRunCancel(command, events);
11210 break;
11211 case "queued-run.edit":
11212 yield* dispatchQueuedRunEdit(command, events);
11213 break;
11214 case "checkpoint.rollback":
11215 yield* dispatchCheckpointRollback(command, events, effects);
11216 break;
11217 case "checkpoint.rollback.fail":
11218 yield* dispatchCheckpointRollbackFail(command, events);
11219 break;
11220 case "thread.background-work.settle":
11221 yield* dispatchBackgroundWorkSettle(command, events, effects);
11222 break;
11223 case "thread.fork":
11224 yield* dispatchThreadFork(command, events);
11225 break;
11226 case "thread.merge_back":
11227 yield* dispatchThreadMergeBack(command, events);
11228 break;
11229 case "delegated_task.request":
11230 yield* dispatchDelegatedTaskRequest(command, events, effects);
11231 break;
11232 case "delegated_task.wake-policy":
11233 yield* dispatchDelegatedTaskWakePolicy(command, events);
11234 break;
11235 case "delegated_task.completion-delivery.acknowledge":
11236 case "delegated_task.completion-delivery.dispose":
11237 yield* dispatchDelegatedTaskCompletionDeliveryResolution(command, events);
11238 break;
11239 case "thread.created.record":
11240 yield* dispatchCreatedThreadRecord(command, events);
11241 break;
11242 case "secret_request.record":
11243 yield* dispatchSecretRequestRecord(command, events);
11244 break;
11245 default:
11246 return yield* dispatchUnsupported(command);
11247 }
11248 return {
11249 events: yield* Ref.get(events),
11250 effects: yield* Ref.get(effects),
11251 ...(cancelUnsettledEffects === undefined ? {} : { cancelUnsettledEffects }),
11252 };
11253 });
11254
11255 const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* (
11256 command: OrchestrationV2ServerCommand,
11257 ): Effect.fn.Return<OrchestratorV2DispatchResult, OrchestratorV2Error> {
11258 yield* Effect.annotateCurrentSpan({
11259 "orchestration_v2.command_id": command.commandId,
11260 "orchestration_v2.command_type": command.type,
11261 "orchestration_v2.thread_id": commandThreadId(command),
11262 });
11263
11264 const existingReceipt = yield* commandReceipts.getByCommandId(command.commandId).pipe(
11265 Effect.mapError(
11266 (cause) =>
11267 new OrchestratorDispatchError({
11268 commandId: command.commandId,
11269 commandType: command.type,
11270 cause,
11271 }),
11272 ),
11273 );
11274
11275 if (Option.isSome(existingReceipt)) {
11276 const receipt = existingReceipt.value;
11277 if (receipt.status === "rejected") {
11278 return yield* new OrchestratorCommandPreviouslyRejectedError({
11279 commandId: command.commandId,
11280 commandType: command.type,
11281 detail: receipt.error ?? "Previously rejected.",
11282 });
11283 }
11284 // A receipt only proves this exact command was handled for its own
11285 // thread. Replaying it for a command aimed at another thread would
11286 // report success for work that never happened.
11287 const dispatchThreadId = commandThreadId(command);
11288 if (!canReplayCommandReceipt(receipt.threadId, dispatchThreadId)) {
11289 return yield* new OrchestratorCommandIdConflictError({
11290 commandId: command.commandId,
11291 commandType: command.type,
11292 receiptThreadId: receipt.threadId,
11293 commandThreadId: dispatchThreadId,
11294 });
11295 }
11296 const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
11297 Stream.runCollect,
11298 Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
11299 Effect.mapError(
11300 (cause) =>
11301 new OrchestratorDispatchError({
11302 commandId: command.commandId,
11303 commandType: command.type,
11304 cause,
11305 }),
11306 ),
11307 );
11308 if (command.type === "queue.resume") {
11309 yield* mapDispatchError(command)(startNextQueuedRun(command.threadId));
11310 }
11311 return {
11312 sequence: receipt.resultSequence,
11313 storedEvents,
11314 } satisfies OrchestratorV2DispatchResult;
11315 }
11316
11317 // A limited sender checked these modes before dispatching; the thread's
11318 // user may have raised them since, and only here can they not change.
11319 const limit = yield* DispatchModeLimit;
11320 if (limit !== undefined) {
11321 // A fork or merge-back source is not under this lock: its dispatch
11322 // checks the copy it reads instead.
11323 const threadId = commandThreadId(command);
11324 const shell = yield* projectionStore
11325 .getThreadShell(threadId)
11326 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
11327 if (shell !== null) yield* refuseAboveDispatchModeLimit(command, threadId, shell);
11328 }
11329
11330 const plan = yield* dispatchOnce(command).pipe(
11331 Effect.flatMap((planned) =>
11332 // These commands can succeed without changing the current state.
11333 planned.events.length > 0 ||
11334 command.type === "thread.background-work.settle" ||
11335 command.type === "thread.stop" ||
11336 command.type === "thread.ci-failed" ||
11337 command.type === "queued-message.promote-to-steer" ||
11338 command.type === "subagent.stop"
11339 ? Effect.succeed(planned)
11340 : Effect.fail(
11341 new OrchestratorDispatchError({
11342 commandId: command.commandId,
11343 commandType: command.type,
11344 cause: "Command produced no domain events.",
11345 }),
11346 ),
11347 ),
11348 Effect.catch((cause) =>
11349 Effect.gen(function* () {
11350 // Refused like the check above: nothing recorded, so the same command
11351 // can go through once the thread's user lowers it again.
11352 if (cause._tag === "OrchestratorThreadAboveModeLimitError") return yield* cause;
11353 const rejectedAt = yield* DateTime.now;
11354 const receipt = yield* eventSink
11355 .commitRejectedCommand({
11356 commandId: command.commandId,
11357 threadId: commandThreadId(command),
11358 commandType: command.type,
11359 rejectedAt,
11360 error: cause instanceof Error ? cause.message : String(cause),
11361 })
11362 .pipe(
11363 Effect.mapError(
11364 (receiptCause) =>
11365 new OrchestratorDispatchError({
11366 commandId: command.commandId,
11367 commandType: command.type,
11368 cause: receiptCause,
11369 }),
11370 ),
11371 );
11372 if (
11373 command.type === "queued-run.edit" &&
11374 receipt.status === "rejected" &&
11375 cause._tag === "OrchestratorDispatchError"
11376 ) {
11377 return yield* new OrchestratorCommandRejectedError({
11378 commandId: cause.commandId,
11379 commandType: cause.commandType,
11380 cause: cause.cause,
11381 });
11382 }
11383 return yield* cause;
11384 }),
11385 ),
11386 );
11387
11388 if (plan.events.length === 0) {
11389 // A no-op consumes its command ID so a replay cannot change newer work.
11390 const resultSequence = yield* Effect.gen(function* () {
11391 const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) });
11392 yield* commandReceipts.insertIfAbsent({
11393 commandId: command.commandId,
11394 threadId: commandThreadId(command),
11395 commandType: command.type,
11396 acceptedAt: yield* DateTime.now,
11397 resultSequence: sequence,
11398 status: "accepted",
11399 error: null,
11400 });
11401 return sequence;
11402 }).pipe(
11403 Effect.mapError(
11404 (cause) =>
11405 new OrchestratorDispatchError({
11406 commandId: command.commandId,
11407 commandType: command.type,
11408 cause,
11409 }),
11410 ),
11411 );
11412 return { sequence: resultSequence, storedEvents: [] } satisfies OrchestratorV2DispatchResult;
11413 }
11414 const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);
11415 const committed = yield* eventSink
11416 .commitCommand({
11417 commandId: command.commandId,
11418 threadId: commandThreadId(command),
11419 commandType: command.type,
11420 acceptedAt,
11421 events: plan.events,
11422 effects: plan.effects,
11423 ...(plan.cancelUnsettledEffects === undefined
11424 ? {}
11425 : { cancelUnsettledEffects: plan.cancelUnsettledEffects }),
11426 })
11427 .pipe(
11428 Effect.mapError(
11429 (cause) =>
11430 new OrchestratorDispatchError({
11431 commandId: command.commandId,
11432 commandType: command.type,
11433 cause,
11434 }),
11435 ),
11436 );
11437
11438 if (committed.receipt.status === "rejected") {
11439 return yield* new OrchestratorCommandPreviouslyRejectedError({
11440 commandId: command.commandId,
11441 commandType: command.type,
11442 detail: committed.receipt.error ?? "Previously rejected.",
11443 });
11444 }
11445 if (command.type === "queue.resume") {
11446 yield* mapDispatchError(command)(startNextQueuedRun(command.threadId));
11447 }
11448 if (command.type === "notification.delivery.accept") {
11449 yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId));
11450 }
11451 if (command.type === "delegated_task.wake-policy") {
11452 yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.parentThreadId));
11453 }
11454
11455 return {
11456 sequence: committed.receipt.resultSequence,
11457 storedEvents: committed.storedEvents,
11458 } satisfies OrchestratorV2DispatchResult;
11459 });
11460
11461 const dispatchWithReceipt = (command: OrchestrationV2ServerCommand) =>
11462 Effect.gen(function* () {
11463 const dispatch = threadDispatch.withLock(
11464 commandThreadId(command),
11465 dispatchWithReceiptEffect(command),
11466 );
11467 if (
11468 command.type !== "message.dispatch" ||
11469 (command.createdBy !== "user" && command.creationSource !== "mcp")
11470 )
11471 return yield* dispatch;
11472 const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe(
11473 Effect.mapError(
11474 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
11475 ),
11476 );
11477 // Follow-up intake and Stop serialize on the parent before taking the child lock.
11478 return yield* parentThreadId === undefined
11479 ? dispatch
11480 : threadDispatch.withLock(parentThreadId, dispatch);
11481 });
11482
11483 const handleTerminalRun = (stored: OrchestrationV2StoredEvent) =>
11484 Effect.gen(function* () {
11485 const threadId = stored.event.threadId;
11486 // finalize writes the parent thread and startNextQueuedRun writes this
11487 // thread, so each takes its own thread's lock, sequentially and never
11488 // nested: dispatchDelegatedTaskRequest already writes child events
11489 // while holding the parent lock, so nesting the parent lock inside the
11490 // child lock here would invert that order, and the keyed executor's
11491 // semaphores are neither reentrant nor deadlock-aware.
11492 if (stored.event.type === "run.updated") {
11493 yield* threadDispatch.withLock(
11494 threadId,
11495 finalizeDelegatedCompletionDelivery(threadId, stored.event.payload.id),
11496 );
11497 }
11498 yield* threadDispatch
11499 .withLock(
11500 threadId,
11501 startNextQueuedRun(
11502 threadId,
11503 stored.event.type === "run.updated"
11504 ? {
11505 endedRunId: stored.event.payload.id,
11506 ...(stored.event.payload.status === "failed"
11507 ? { failedRunId: stored.event.payload.id }
11508 : {}),
11509 }
11510 : undefined,
11511 ),
11512 )
11513 .pipe(
11514 Effect.catchCause((cause) =>
11515 Effect.logWarning("Failed to start the next queued V2 run", { threadId, cause }),
11516 ),
11517 );
11518 // After the queue decision: a provider failure holds the child's queued
11519 // wakes, and only then is the failed run the task's result.
11520 const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
11521 if (parentThreadId !== undefined) {
11522 yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
11523 }
11524 }).pipe(
11525 Effect.catchCause((cause) =>
11526 Effect.logWarning("Failed to react to terminal V2 run", {
11527 threadId: stored.event.threadId,
11528 sequence: stored.sequence,
11529 cause,
11530 }),
11531 ),
11532 );
11533
11534 // Historical terminal events are already represented by the projections
11535 // below. Replaying the full event table on every server start delays live
11536 // queue promotion in proportion to the lifetime size of the database.
11537 const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
11538 // Queue promotion can wait on a provider or a thread lock. Subscribe to run
11539 // updates before buffering so that wait never retains unrelated tool bodies.
11540 yield* eventSink
11541 .stream({ afterSequence: terminalEventsAfterSequence, eventType: "run.updated" })
11542 .pipe(
11543 Stream.filter(
11544 (stored) =>
11545 stored.event.type === "run.updated" &&
11546 !String(stored.commandId).startsWith("command:runtime-reconcile:") &&
11547 (stored.event.payload.status === "completed" ||
11548 stored.event.payload.status === "interrupted" ||
11549 stored.event.payload.status === "failed" ||
11550 stored.event.payload.status === "cancelled" ||
11551 stored.event.payload.status === "rolled_back"),
11552 ),
11553 Stream.runForEach(handleTerminalRun),
11554 Effect.forkDetach,
11555 );
11556
11557 // Settles child results and completion deliveries whose runs ended without
11558 // the listener above: before this boot, or in runtime reconciliation, which
11559 // it skips. Startup runs this after reconciliation and before the effect
11560 // worker. Queue recovery instead holds unstarted runs until an explicit
11561 // queue.resume command arrives.
11562 const recoverDelegatedTasks = Effect.gen(function* () {
11563 yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(
11564 // Candidate counts let a startup trace show whether recovery scales
11565 // with pending work or with history.
11566 Effect.tap((threadIds) =>
11567 Effect.annotateCurrentSpan("recovery.subagentResultThreads", threadIds.length),
11568 ),
11569 Effect.flatMap((threadIds) =>
11570 Effect.forEach(
11571 threadIds,
11572 (threadId) =>
11573 Effect.gen(function* () {
11574 const thread = yield* projectionStore.getThreadShell(threadId);
11575 const parentThreadId = thread?.lineage.parentThreadId;
11576 if (parentThreadId === undefined || parentThreadId === null) return;
11577 yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
11578 }).pipe(
11579 Effect.catchCause((cause) =>
11580 Effect.logWarning("Failed to recover terminal app-owned subagent", {
11581 childThreadId: threadId,
11582 cause,
11583 }),
11584 ),
11585 ),
11586 { concurrency: 8, discard: true },
11587 ),
11588 ),
11589 Effect.catchCause((cause) =>
11590 Effect.logWarning("Failed to inspect app-owned subagents during recovery", {
11591 cause,
11592 }),
11593 ),
11594 );
11595 yield* projectionStore.getRecoveryThreadIds("delegated-completions").pipe(
11596 Effect.tap((threadIds) =>
11597 Effect.annotateCurrentSpan("recovery.delegatedCompletionThreads", threadIds.length),
11598 ),
11599 Effect.flatMap((threadIds) =>
11600 Effect.forEach(
11601 threadIds,
11602 (threadId) =>
11603 threadDispatch
11604 .withLock(
11605 threadId,
11606 Effect.gen(function* () {
11607 const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
11608 // Only the run that consumed a cohort's current delivery
11609 // message can settle it. A long-lived parent keeps every
11610 // past delivery run, and finalizing each of them would
11611 // reload the thread once per historical delivery.
11612 const openDeliveryMessageIds = new Set(
11613 projection.runs.flatMap((run) =>
11614 run.delegatedCompletion?.delivery == null
11615 ? []
11616 : [run.delegatedCompletion.delivery.messageId],
11617 ),
11618 );
11619 const terminalDeliveryRunIds = projection.runs
11620 .filter(
11621 (run) =>
11622 delegatedTaskTerminalStatus(run.status) !== null &&
11623 openDeliveryMessageIds.has(run.userMessageId),
11624 )
11625 .map((run) => run.id);
11626 for (const runId of terminalDeliveryRunIds) {
11627 yield* finalizeDelegatedCompletionDelivery(threadId, runId);
11628 }
11629 const refreshed =
11630 terminalDeliveryRunIds.length === 0
11631 ? projection
11632 : yield* projectionStore.getThreadRecords(threadId, ["runs"]);
11633 for (const run of refreshed.runs) {
11634 if (
11635 run.delegatedCompletion?.delivery !== null &&
11636 run.delegatedCompletion !== undefined
11637 ) {
11638 yield* offerDelegatedCompletionDelivery(threadId, run.id);
11639 }
11640 }
11641 }),
11642 )
11643 .pipe(
11644 Effect.catchCause((cause) =>
11645 Effect.logWarning("Failed to recover delegated completion delivery", {
11646 threadId,
11647 cause,
11648 }),
11649 ),
11650 ),
11651 { concurrency: 8, discard: true },
11652 ),
11653 ),
11654 Effect.catchCause((cause) =>
11655 Effect.logWarning("Failed to inspect delegated completion delivery during recovery", {
11656 cause,
11657 }),
11658 ),
11659 );
11660 });
11661
11662 const delegatedTaskResultPending = (childThreadId: ThreadId, taskId?: NodeId) =>
11663 Effect.gen(function* () {
11664 const child = yield* projectionStore.getThreadRecords(
11665 childThreadId,
11666 ["runs", "messages", "subagents", "providerThreads", "providerTurns", "attempts"],
11667 { messageRoles: ["user"] },
11668 );
11669 if (taskId !== undefined) {
11670 const runs = child.runs.filter((run) => run.delegatedTaskId === taskId);
11671 const run = runs.toSorted((a, b) => b.ordinal - a.ordinal)[0];
11672 if (run === undefined || delegatedTaskTerminalStatus(run.status) === null) return true;
11673 return yield* childAwaitsRestartContinuation(runs, run);
11674 }
11675 const progress = delegatedTaskProgress(child);
11676 // A caller's older read saw a result; newer work since then means it is not final.
11677 if (progress.state !== "result_available") return true;
11678 if (progress.resultRun === undefined) return false;
11679 return yield* childAwaitsRestartContinuation(child.runs, progress.resultRun);
11680 }).pipe(
11681 Effect.mapError(
11682 (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
11683 ),
11684 );
11685
11686 const recoverDelegatedTask = (threadId: ThreadId, sourceRunId: RunId) =>
11687 Effect.gen(function* () {
11688 const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
11689 if (parentThreadId === undefined) return;
11690 yield* threadDispatch.withLock(
11691 parentThreadId,
11692 finalizeAppOwnedSubagent(threadId, { settledContinuationOf: sourceRunId }),
11693 );
11694 }).pipe(
11695 Effect.catchCause((cause) =>
11696 Effect.logWarning("Failed to recover delegated task after restart", {
11697 childThreadId: threadId,
11698 cause,
11699 }),
11700 ),
11701 );
11702
11703 const shellProjectionError = (cause: unknown) =>
11704 new OrchestratorProjectionError({ threadId: ThreadId.make("thread:shell"), cause });
11705
11706 return OrchestratorV2.of({
11707 resumeQueuedRuns,
11708 recoverDelegatedTasks,
11709 recoverDelegatedTask,
11710 delegatedTaskResultPending,
11711 restartContinuationPending: (run) =>
11712 awaitsRestartContinuation(run).pipe(Effect.orElseSucceed(() => false)),
11713 awaitRestartContinuation: (run) =>
11714 effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
11715 dispatch: dispatchWithReceipt,
11716 searchThreadStream: (input) =>
11717 projectionStore
11718 .searchThreadStream(input)
11719 .pipe(
11720 Stream.mapError(
11721 (cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11722 ),
11723 ),
11724 searchThread: (input) =>
11725 projectionStore
11726 .searchThread(input)
11727 .pipe(
11728 Effect.mapError(
11729 (cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11730 ),
11731 ),
11732 getThreadHistoryPage: (threadId, cursor, throughEntryId, conversationOnly) =>
11733 projectionStore
11734 .getThreadHistoryPage(threadId, cursor, throughEntryId, conversationOnly)
11735 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11736 getTimelinePage: (threadId, options) =>
11737 projectionStore
11738 .getTimelinePage(threadId, options)
11739 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11740 getMessageCount: (threadId) =>
11741 projectionStore
11742 .getMessageCount(threadId)
11743 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11744 getTurnItem: (input) =>
11745 projectionStore
11746 .getTurnItem(input)
11747 .pipe(
11748 Effect.mapError(
11749 (cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11750 ),
11751 ),
11752 getThreadRecords: (threadId, fields, filter) =>
11753 projectionStore
11754 .getThreadRecords(threadId, fields, filter)
11755 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11756 getThreadProjection: (threadId) =>
11757 projectionStore
11758 .getThreadProjection(threadId)
11759 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11760 getCheckpointContext: (threadId) =>
11761 projectionStore
11762 .getCheckpointContext(threadId)
11763 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11764 getThreadSnapshot: (threadId) =>
11765 projectionStore
11766 .getThreadSnapshot(threadId)
11767 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11768 getThreadSnapshotWindow: (threadId, options) =>
11769 projectionStore
11770 .getThreadSnapshotWindow(threadId, options)
11771 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11772 getShellSnapshot: (options) =>
11773 projectionStore.getShellSnapshot(options).pipe(Effect.mapError(shellProjectionError)),
11774 readShellSnapshot: (options) =>
11775 projectionStore
11776 .readShellSnapshot(options)
11777 .pipe(
11778 Effect.mapError(shellProjectionError),
11779 Effect.map(Effect.mapError(shellProjectionError)),
11780 ),
11781 getThreadShell: (threadId) =>
11782 projectionStore
11783 .getThreadShell(threadId)
11784 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11785 getThreadEventSequence: (threadId) =>
11786 eventSink
11787 .latestSequence({ threadId })
11788 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11789 streamStoredEvents: eventSink.stream().pipe(
11790 Stream.mapError(
11791 (cause) =>
11792 new OrchestratorDomainEventStreamError({
11793 cause,
11794 }),
11795 ),
11796 ),
11797 streamStoredEventsFrom: (input) =>
11798 eventSink.stream({ ...input, bounded: true }).pipe(
11799 Stream.mapError(
11800 (cause) =>
11801 new OrchestratorDomainEventStreamError({
11802 cause,
11803 }),
11804 ),
11805 ),
11806 // Live tail only. eventSink.stream() with no cursor replays the whole
11807 // store from genesis first; domain-event subscribers (the awareness relay)
11808 // react to new activity, and startup replay made them grind through the
11809 // entire event history doing per-event work after every boot.
11810 streamDomainEvents: Stream.unwrap(
11811 eventSink
11812 .latestSequence()
11813 .pipe(Effect.map((latest) => eventSink.stream({ afterSequence: latest }))),
11814 ).pipe(
11815 Stream.map((stored) => stored.event),
11816 Stream.mapError(
11817 (cause) =>
11818 new OrchestratorDomainEventStreamError({
11819 cause,
11820 }),
11821 ),
11822 ),
11823 });
11824});
11825
11826export const layer: Layer.Layer<
11827 OrchestratorV2,
11828 never,
11829 | CheckpointServiceV2
11830 | FileSystem.FileSystem
11831 | Path.Path
11832 | CommandPolicyV2
11833 | CommandReceiptStoreV2
11834 | ContextHandoffServiceV2
11835 | EffectOutbox.EffectOutboxV2
11836 | EventSinkV2
11837 | IdAllocator.IdAllocatorV2
11838 | ProjectStore.ProjectStoreV2
11839 | ProviderAdapterRegistryV2
11840 | ProviderSessionManagerV2
11841 | ProviderSwitchServiceV2
11842 | ProjectionStoreV2
11843 | RuntimePolicyV2
11844 | ServerSettings.ServerSettingsService
11845 | ThreadForkServiceV2
11846> = Layer.effect(OrchestratorV2, makeOrchestrator()).pipe(
11847 Layer.provide(ThreadCommandExecutor.layer),
11848);