1import { assert } from "@effect/vitest";
2import {
3 type ChatAttachment,
4 CommandId,
5 isOrchestrationV2WorkActive,
6 MessageId,
7 ProjectId,
8 ThreadId,
9 type ModelSelection,
10 type OrchestrationV2Command,
11 type OrchestrationV2ExecutionNode,
12 type OrchestrationV2RunStatus,
13 type OrchestrationV2ThreadProjection,
14 type OrchestrationV2TurnItem,
15 type OrchestrationV2UserMessageInputIntent,
16 ProviderInstanceId,
17 type ProviderInteractionMode,
18 type ProviderDriverKind,
19 type ProviderReplayTranscript,
20 type ProviderUserInputAnswers,
21 type RuntimeMode,
22} from "@t3tools/contracts";
23import type * as Duration from "effect/Duration";
24import * as Effect from "effect/Effect";
25
26import type {
27 OrchestratorV2ScenarioResult,
28 OrchestratorV2ScenarioStep,
29} from "../OrchestratorScenario.ts";
30import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
31import type { RuntimePolicyV2Override } from "../../RuntimePolicy.ts";
32
33export const SIMPLE_PROMPT = "Respond with the following text: fixture simple ok";
34export const MULTI_TURN_FIRST_PROMPT = "Respond with exactly: first fixture turn complete";
35export const MULTI_TURN_SECOND_PROMPT = "Respond with exactly: second fixture turn complete";
36export const PROVIDER_THREAD_RESUME_FIRST_PROMPT =
37 "Respond with exactly: provider thread resume fixture first turn complete";
38export const PROVIDER_THREAD_RESUME_SECOND_PROMPT =
39 "Using the conversation history available in this resumed thread, first repeat the exact final answer you gave in the previous turn. Then on a new line write exactly: provider thread resume fixture second turn complete";
40export const TOOL_CALL_READ_ONLY_WORKSPACE_ROOT = "/tmp/claude-replay-tool_call_read_only";
41export const TOOL_CALL_READ_ONLY_PROMPT = `Read ${TOOL_CALL_READ_ONLY_WORKSPACE_ROOT}/package.json and ${TOOL_CALL_READ_ONLY_WORKSPACE_ROOT}/tsconfig.json, then answer exactly: read only tool fixture complete`;
42export const CLAUDE_LOCAL_BASH_TASK_PROMPT =
43 "Run a local Bash typecheck command, then answer exactly: claude local bash task fixture complete";
44export const CLAUDE_RESULT_IS_ERROR_PROMPT = "Say hello before the credentials expire.";
45export const CLAUDE_RESULT_IS_ERROR_FOLLOW_UP = "Try again now that auth is back.";
46export const TOOL_CALL_WRITE_PROMPT =
47 "Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP.";
48export const MESSAGE_STEERING_INITIAL_PROMPT =
49 "Respond with exactly: steering fixture initial response";
50export const SUBAGENT_PROMPT =
51 "Spawn 2 subagents, one to read package.json and one to read tsconfig.json";
52export const SUBAGENT_V2_NESTED_PROMPT =
53 "Spawn one subagent and tell it to spawn its own subagent, which must in turn spawn one more subagent whose only task is to reply with exactly: Hello. Each agent waits for its child and replies with exactly what the child said. Wait for your subagent, then reply with exactly what it said.";
54export const SUBAGENT_V2_PROMPT =
55 "Spawn one subagent whose only task is to reply with exactly: Hello. Wait for it to finish, then reply with exactly what it said.";
56export const SUBAGENT_V2_APPROVAL_PROMPT =
57 "Do not run any commands yourself. Spawn one subagent whose only task is to run this exact shell command: printf 'subagent approval fixture' > subagent-approval.txt and then reply with exactly: Written. Wait for it to finish, then reply with exactly what it said.";
58export const SUBAGENT_V2_NESTED_APPROVAL_PROMPT =
59 "Do not run any commands yourself. Spawn one subagent and tell it not to run any commands itself but to spawn its own subagent, whose only task is to run this exact shell command: printf 'nested approval fixture' > nested-approval.txt and then reply with exactly: Written. Each agent waits for its child and replies with exactly what the child said. Wait for your subagent, then reply with exactly what it said.";
60export const OPENCODE_SUBAGENT_PROMPT =
61 "Use the task tool exactly once. Delegate to the general subagent with this prompt: Respond exactly CHILD_OK. After the task completes, respond exactly PARENT_OK.";
62export const SUBAGENT_CONTINUE_PROMPT =
63 "Spawn one subagent and have it reply exactly: initial subagent response";
64export const SUBAGENT_CONTINUE_PARENT_PROMPT =
65 "Have the same subagent you spawned earlier reply exactly: continued subagent response";
66export const SUBAGENT_CONTINUE_CHILD_PROMPT = "Reply exactly: continued subagent response";
67/** Prompts the OpenCode 2 spike recorded against 2.0.18 (`opencode2_*` fixtures). */
68export const OPENCODE2_SIMPLE_PROMPT =
69 "Think carefully step by step about whether 391 is prime, showing your reasoning, then answer in one short sentence.";
70export const OPENCODE2_TOOL_CALL_PROMPT =
71 "Use the read tool to read hello.txt, then run the shell command `echo TOOL_OK` with the bash tool, then reply DONE.";
72export const OPENCODE2_INTERRUPT_PROMPT =
73 "Run the shell command `sleep 60 && echo LATE` with the bash tool, then reply DONE.";
74export const OPENCODE2_PERMISSION_PROMPT =
75 "Run the shell command `echo FIRST` with the bash tool. After it completes, run `echo SECOND` with the bash tool. Then reply with what happened.";
76export const OPENCODE2_STEER_PROMPT =
77 "Run the shell command `sleep 12 && echo A` with the bash tool, then reply DONE_A.";
78export const OPENCODE2_STEER_TEXT = "Also mention the word STEERED in your final reply.";
79export const OPENCODE2_QUEUED_PROMPT = "Reply exactly QUEUED_B.";
80export const OPENCODE2_CANCELLED_PROMPT = "Reply exactly QUEUED_C.";
81export const OPENCODE2_REVERT_FIRST_PROMPT =
82 "Create a file named reverted.txt containing the word ALPHA using the write tool, then reply DONE.";
83export const OPENCODE2_REVERT_SECOND_PROMPT =
84 "Overwrite reverted.txt so it contains the word BETA using the write tool, then reply DONE.";
85export const OPENCODE2_QUESTION_PROMPT =
86 "Before doing anything, use the question tool to ask me which color I prefer, offering the options red and blue. After I answer, reply with only the chosen color.";
87export const OPENCODE2_SUBAGENT_PROMPT =
88 "Use the subagent tool to delegate to the explore subagent with the prompt: 'List the files in the current directory and report their names.' Wait for it, then summarize its answer in one line.";
89export const OPENCODE2_NESTED_BACKGROUND_PROMPT =
90 "Use the subagent tool (foreground, do not set background) to delegate to the general subagent with this exact prompt: 'Use the subagent tool with background set to true to delegate to the general subagent with the prompt: Run the shell command `sleep 25` with the shell tool, then reply exactly GRANDCHILD_OK. As soon as it is launched, reply exactly MIDDLE_OK and end your turn without waiting for it.' Wait for that subagent to return, then reply exactly ROOT_OK.";
91export const OPENCODE2_BACKGROUND_PROMPT =
92 "Use the subagent tool with background enabled to delegate to the general subagent with the prompt: 'Run the shell command `sleep 20` with the bash tool and then reply exactly CHILD_OK.' As soon as it is launched, reply exactly PARENT_OK and end your turn without waiting for it.";
93export const OPENCODE2_COMPACTION_FIRST_PROMPT = "Remember the codeword PAPAYA. Reply OK.";
94export const OPENCODE2_COMPACTION_RECALL_PROMPT = "What was the codeword? One word.";
95export const OPENCODE2_RESTART_PROMPT =
96 "Run the shell command `sleep 25 && echo RESUMED` with the bash tool, then reply with its output.";
97export const OPENCODE2_RESTART_RECALL_PROMPT =
98 "What did I last ask you to run? Answer in one short sentence.";
99export const OPENCODE2_COMMAND_PROMPT = "/hello WORLD";
100export const OPENCODE2_SKILL_PROMPT = "Use $greet to say hi in three words.";
101export const TURN_INTERRUPT_PROMPT =
102 "Do not answer immediately. First run the local shell command `sleep 30`, then respond with exactly: interrupt fixture should not finish naturally.";
103export const TURN_INTERRUPT_MID_TOOL_PROMPT =
104 "Run this exact local command: `node -e \"console.log('interrupt fixture tool started'); setTimeout(() => {}, 30000)\"`. Do not answer until it completes, then respond exactly: interrupt fixture should not finish naturally.";
105export const TURN_INTERRUPT_RECOVERY_PROMPT =
106 "Respond with exactly: interrupt recovery fixture complete";
107export const MESSAGE_STEERING_STEER_PROMPT =
108 "Actually, respond with exactly: steering fixture observed";
109export const THREAD_ROLLBACK_FIRST_PROMPT =
110 "Respond with exactly: rollback fixture first turn complete";
111export const THREAD_ROLLBACK_SECOND_PROMPT =
112 "Respond with exactly: rollback fixture second turn complete";
113export const THREAD_ROLLBACK_AFTER_PROMPT = "Repeat the conversation verbatim.";
114export const THREAD_FORK_NATIVE_SOURCE_PROMPT =
115 "Respond with the following text: source fork seed ok";
116export const THREAD_FORK_NATIVE_TARGET_PROMPT = "Respond with the following text: fork native ok";
117export const THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER = "source-marker-7Q9V";
118export const THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER = "fork-marker-2K4M";
119export const THREAD_FORK_NATIVE_CONTINUE_RECALL = `${THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER}|${THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER}`;
120export const THREAD_FORK_NATIVE_CONTINUE_SOURCE_PROMPT = `Remember the opaque marker ${THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER} for later in this conversation. Respond with exactly: source marker stored`;
121export const THREAD_FORK_NATIVE_CONTINUE_FIRST_PROMPT = `Remember the second opaque marker ${THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER} for later in this conversation. Respond with exactly: fork marker stored`;
122export const THREAD_FORK_NATIVE_CONTINUE_SECOND_PROMPT =
123 "Return the two opaque markers previously provided in chronological order, separated by a single | character. Respond with only the markers and separator.";
124const THREAD_FORK_NATIVE_SIBLINGS_SOURCE_MARKER = "sibling-source-8R3D";
125const THREAD_FORK_NATIVE_SIBLINGS_FIRST_MARKER = "sibling-first-5L2P";
126const THREAD_FORK_NATIVE_SIBLINGS_SECOND_MARKER = "sibling-second-9N6C";
127export const THREAD_FORK_NATIVE_SIBLINGS_SOURCE_PROMPT = `Remember the opaque marker ${THREAD_FORK_NATIVE_SIBLINGS_SOURCE_MARKER} for later in this conversation. Respond with exactly: sibling source stored`;
128export const THREAD_FORK_NATIVE_SIBLINGS_FIRST_PROMPT = `Remember the fork-local marker ${THREAD_FORK_NATIVE_SIBLINGS_FIRST_MARKER}. Return the source marker followed by this marker, separated by |. Respond with only the markers and separator.`;
129export const THREAD_FORK_NATIVE_SIBLINGS_SECOND_PROMPT = `Remember the fork-local marker ${THREAD_FORK_NATIVE_SIBLINGS_SECOND_MARKER}. Return the source marker followed by this marker, separated by |. Respond with only the markers and separator.`;
130export const THREAD_MERGE_BACK_SOURCE_MARKER = "merge-source-4H8Q";
131export const THREAD_MERGE_BACK_FORK_MARKER = "merge-fork-7T2W";
132export const THREAD_MERGE_BACK_SOURCE_PROMPT = `Remember the opaque marker ${THREAD_MERGE_BACK_SOURCE_MARKER} for later in this conversation. Respond with exactly: merge source stored`;
133export const THREAD_MERGE_BACK_FORK_PROMPT = `Remember the fork-local marker ${THREAD_MERGE_BACK_FORK_MARKER}. Respond with exactly: merge fork stored`;
134export const THREAD_MERGE_BACK_HANDOFF_PROMPT = [
135 "Context handoff (merge_back / fork_delta_summary):",
136 "Merge-back context from forked conversation.",
137 "",
138 "Fork delta:",
139 `- User introduced opaque marker ${THREAD_MERGE_BACK_FORK_MARKER}.`,
140 "- Assistant confirmed: merge fork stored",
141 "",
142 "User message:",
143 "Retain the transferred fork marker for later. Respond with exactly: merge delta stored",
144].join("\n");
145export const THREAD_MERGE_BACK_RECALL = `${THREAD_MERGE_BACK_SOURCE_MARKER}|${THREAD_MERGE_BACK_FORK_MARKER}`;
146export const THREAD_MERGE_BACK_RECALL_PROMPT =
147 "Return the source marker followed by the transferred fork marker, separated by a single | character. Respond with only the markers and separator.";
148export const THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER = "merge-sibling-source-3C7K";
149export const THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER = "merge-sibling-first-6V2J";
150export const THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER = "merge-sibling-second-9X5B";
151export const THREAD_MERGE_BACK_SIBLINGS_SOURCE_PROMPT = `Remember the opaque marker ${THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER} for later in this conversation. Respond with exactly: merge sibling source stored`;
152export const THREAD_MERGE_BACK_SIBLINGS_FIRST_FORK_PROMPT = `Remember the fork-local marker ${THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER}. Respond with exactly: first merge sibling stored`;
153export const THREAD_MERGE_BACK_SIBLINGS_SECOND_FORK_PROMPT = `Remember the fork-local marker ${THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER}. Respond with exactly: second merge sibling stored`;
154export const THREAD_MERGE_BACK_SIBLINGS_FIRST_HANDOFF_PROMPT = [
155 "Context handoff (merge_back / fork_delta_summary):",
156 "Merge-back context from first forked conversation.",
157 "",
158 "Fork delta:",
159 `- User introduced opaque marker ${THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER}.`,
160 "- Assistant confirmed: first merge sibling stored",
161 "",
162 "User message:",
163 "Retain the first transferred marker for later. Respond with exactly: first merge delta stored",
164].join("\n");
165export const THREAD_MERGE_BACK_SIBLINGS_SECOND_HANDOFF_PROMPT = [
166 "Context handoff (merge_back / fork_delta_summary):",
167 "Merge-back context from second forked conversation.",
168 "",
169 "Fork delta:",
170 `- User introduced opaque marker ${THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER}.`,
171 "- Assistant confirmed: second merge sibling stored",
172 "",
173 "User message:",
174 "Retain the second transferred marker for later. Respond with exactly: second merge delta stored",
175].join("\n");
176export const THREAD_MERGE_BACK_SIBLINGS_RECALL = [
177 THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER,
178 THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER,
179 THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER,
180].join("|");
181export const THREAD_MERGE_BACK_SIBLINGS_RECALL_PROMPT =
182 "Return the source marker followed by both transferred fork markers in merge order, separated by single | characters. Respond with only the markers and separators.";
183export const THREAD_FORK_NATIVE_PRIOR_TURN_ALPHA_PROMPT =
184 "For this fork-boundary fixture, respond with exactly: fork boundary alpha";
185export const THREAD_FORK_NATIVE_PRIOR_TURN_BETA_PROMPT =
186 "For this fork-boundary fixture, respond with exactly: fork boundary beta";
187export const THREAD_FORK_NATIVE_PRIOR_TURN_REPEAT_PROMPT =
188 "Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.";
189export const TODO_LIST_PROMPT =
190 "Use the update_plan tool to track exactly three steps: inspect package.json, inspect tsconfig.json, report completion. Then read package.json and tsconfig.json, and answer exactly: todo list fixture complete";
191export const PLAN_QUESTIONS_PROMPT =
192 "Use request_user_input to ask one multiple-choice clarifying question about whether this fixture should prefer strict schemas or UI flexibility. After receiving the answer, respond exactly: plan questions fixture complete";
193export const PROPOSED_PLAN_PROMPT =
194 "Create a short implementation plan for adding deterministic replay fixtures. Do not ask questions. Present the final plan in a proposed plan block.";
195export const WEB_SEARCH_PROMPT =
196 "Search the web for FIFA World Cup ticket pricing, then answer exactly: web search fixture complete";
197export const SKILL_INVOCATION_PROMPT = "$review README.md";
198/** What Cursor receives once the adapter rewrites a discovered `$skill` mention. */
199export const SKILL_INVOCATION_CURSOR_MESSAGE = "/review README.md";
200
201export type OrchestratorFixtureInputStep =
202 | {
203 readonly type: "message";
204 readonly text: string;
205 readonly attachments?: ReadonlyArray<ChatAttachment>;
206 }
207 | {
208 readonly type: "queue_message";
209 readonly text: string;
210 readonly attachments?: ReadonlyArray<ChatAttachment>;
211 readonly createdBy?: Extract<
212 OrchestrationV2Command,
213 { readonly type: "message.dispatch" }
214 >["createdBy"];
215 readonly creationSource?: Extract<
216 OrchestrationV2Command,
217 { readonly type: "message.dispatch" }
218 >["creationSource"];
219 }
220 | {
221 readonly type: "cancel_queued_run";
222 readonly targetRunIndex: number;
223 }
224 | {
225 readonly type: "await_run_status";
226 readonly targetRunIndex: number;
227 readonly status: OrchestrationV2RunStatus;
228 /** Then also wait until that run has projected an item of this type. */
229 readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"];
230 }
231 | {
232 /**
233 * A run held open for background work finishes through the adapter's
234 * debounce, which replay passes on the adapter's receipt.
235 */
236 readonly type: "finish_held_run";
237 readonly targetRunIndex: number;
238 readonly status: OrchestrationV2RunStatus;
239 }
240 | {
241 readonly type: "capture_shell_snapshot";
242 readonly key: string;
243 }
244 | {
245 readonly type: "release_replay_gate";
246 readonly label: string;
247 }
248 | {
249 readonly type: "steer";
250 readonly text: string;
251 readonly attachments?: ReadonlyArray<ChatAttachment>;
252 readonly targetRunIndex: number;
253 }
254 | {
255 readonly type: "restart";
256 readonly text: string;
257 readonly attachments?: ReadonlyArray<ChatAttachment>;
258 readonly targetRunIndex: number;
259 }
260 | {
261 readonly type: "interrupt";
262 readonly targetRunIndex: number;
263 readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"];
264 }
265 | {
266 /** The Waiting strip's Stop: interrupts a settled run's leftover background work. */
267 readonly type: "stop_background_work";
268 readonly targetRunIndex: number;
269 }
270 | {
271 readonly type: "release_replay_gate_after_waiting";
272 readonly label: string;
273 readonly targetRunIndex: number;
274 }
275 | {
276 readonly type: "approve_next_runtime_request";
277 readonly decision?: Extract<
278 OrchestrationV2Command,
279 { readonly type: "runtime-request.respond" }
280 >["decision"];
281 /** Captures the shell snapshot under this key while the request is pending. */
282 readonly shellSnapshotKeyWhilePending?: string;
283 }
284 | {
285 readonly type: "answer_next_user_input_request";
286 readonly answers: ProviderUserInputAnswers;
287 }
288 | {
289 readonly type: "rollback";
290 readonly checkpointScopeSuffix: string;
291 readonly checkpointSuffix: string;
292 }
293 | {
294 /**
295 * Advance the deterministic test clock, e.g. past the provider session
296 * manager's idle timeout so the next message must reopen the session.
297 */
298 readonly type: "advance_clock";
299 readonly duration: Duration.Input;
300 };
301
302export interface OrchestratorFixtureInput {
303 readonly interactionMode?: ProviderInteractionMode;
304 /** The thread's permission mode; fixtures default to full access. */
305 readonly runtimeMode?: RuntimeMode;
306 /**
307 * Files committed into the replay workspace before the scenario runs, keyed
308 * by workspace-relative path. A recorder must seed the same files so adapter
309 * logic that reads the workspace (e.g. skill discovery) sees what the
310 * provider saw.
311 */
312 readonly workspaceFiles?: Readonly<Record<string, string>>;
313 readonly steps: ReadonlyArray<OrchestratorFixtureInputStep>;
314}
315
316export interface ProviderOrchestratorReplayVariant {
317 readonly driver: ProviderDriverKind;
318 readonly transcriptFile: URL;
319 readonly recordedScenario?: string;
320 readonly transcriptEntriesThroughLabel?: string;
321 readonly modelSelection: ModelSelection;
322 readonly runtimePolicyOverride?: RuntimePolicyV2Override;
323 /** Replays a provider wake turn as a continuation run, as the live runtime does. */
324 readonly runContinuationWorker?: boolean;
325 /**
326 * Workspace-relative paths that must not exist once the scenario finishes,
327 * e.g. the target of a tool call the run was configured to deny.
328 */
329 readonly expectedAbsentWorkspacePaths?: ReadonlyArray<string>;
330 readonly assertOutput: (
331 result: OrchestratorV2ScenarioResult,
332 transcript: ProviderReplayTranscript,
333 ) => void;
334}
335
336export interface OrchestratorReplayFixture {
337 readonly name: string;
338 readonly buildInput: () => OrchestratorFixtureInput;
339 readonly providers: ReadonlyArray<ProviderOrchestratorReplayVariant>;
340}
341
342export interface MaterializedOrchestratorFixtureInput {
343 readonly commands: ReadonlyArray<OrchestrationV2Command>;
344 readonly steps: ReadonlyArray<OrchestratorV2ScenarioStep>;
345 readonly projectionThreadIds: ReadonlyArray<ThreadId>;
346}
347
348export interface FixtureIds {
349 readonly threadId: ThreadId;
350 readonly projectId: ProjectId;
351}
352
353export const CODEX_MODEL_SELECTION = {
354 instanceId: ProviderInstanceId.make("codex"),
355 model: "gpt-6-luna",
356} satisfies ModelSelection;
357
358export const CLAUDE_MODEL_SELECTION = {
359 instanceId: ProviderInstanceId.make("claudeAgent"),
360 model: "claude-sonnet-4-6",
361} satisfies ModelSelection;
362
363export const CURSOR_MODEL_SELECTION = {
364 instanceId: ProviderInstanceId.make("cursor"),
365 model: "composer-2.5",
366} satisfies ModelSelection;
367
368/** Muse fixtures are recorded against this model; the account's listed models may differ. */
369export const MUSE_MODEL_SELECTION = {
370 instanceId: ProviderInstanceId.make("muse"),
371 model: "muse-spark-1.3-contributor",
372} satisfies ModelSelection;
373
374export const GROK_MODEL_SELECTION = {
375 instanceId: ProviderInstanceId.make("grok"),
376 model: "grok-build",
377} satisfies ModelSelection;
378
379export const OPENCODE_MODEL_SELECTION = {
380 instanceId: ProviderInstanceId.make("opencode"),
381 model: "openai/gpt-5.4-mini",
382 options: [{ id: "agent", value: "build" }],
383} satisfies ModelSelection;
384
385/** The free OpenCode Zen model and variant the OpenCode 2 spike recorded its reasoning run with. */
386export const OPENCODE2_MODEL_SELECTION = {
387 instanceId: ProviderInstanceId.make("opencode"),
388 model: "opencode/space-bunny-free",
389 options: [{ id: "variant", value: "high" }],
390} satisfies ModelSelection;
391
392/** Pi fixtures are recorded against this pinned OpenRouter model; the slug is `provider/model`. */
393export const PI_MODEL_SELECTION = {
394 instanceId: ProviderInstanceId.make("pi"),
395 model: "openrouter/deepseek/deepseek-v4-flash",
396} satisfies ModelSelection;
397
398export const ACP_REGISTRY_MODEL_SELECTION = {
399 instanceId: ProviderInstanceId.make("acpRegistry"),
400 model: "default",
401} satisfies ModelSelection;
402
403export const READ_ONLY_ON_REQUEST_POLICY = {
404 approvalPolicy: "on-request",
405 sandboxPolicy: {
406 type: "readOnly",
407 access: { type: "fullAccess" },
408 networkAccess: false,
409 },
410} as const satisfies RuntimePolicyV2Override;
411
412export const READ_ONLY_NEVER_POLICY = {
413 approvalPolicy: "never",
414 sandboxPolicy: {
415 type: "readOnly",
416 access: { type: "fullAccess" },
417 networkAccess: false,
418 },
419} as const satisfies RuntimePolicyV2Override;
420
421export const WORKSPACE_NEVER_POLICY = {
422 approvalPolicy: "never",
423 sandboxPolicy: {
424 type: "workspaceWrite",
425 writableRoots: [],
426 readOnlyAccess: { type: "fullAccess" },
427 networkAccess: false,
428 },
429} as const satisfies RuntimePolicyV2Override;
430
431export const RESTRICTED_GRANULAR_POLICY = {
432 approvalPolicy: {
433 granular: {
434 mcp_elicitations: true,
435 request_permissions: true,
436 rules: true,
437 sandbox_approval: true,
438 skill_approval: true,
439 },
440 },
441 sandboxPolicy: {
442 type: "readOnly",
443 access: {
444 type: "restricted",
445 includePlatformDefaults: false,
446 readableRoots: [],
447 },
448 networkAccess: false,
449 },
450} as const satisfies RuntimePolicyV2Override;
451
452function createThreadCommand(input: {
453 readonly commandId: CommandId;
454 readonly ids: FixtureIds;
455 readonly scenario: string;
456 readonly modelSelection: ModelSelection;
457 readonly interactionMode?: ProviderInteractionMode;
458 readonly runtimeMode?: RuntimeMode;
459}): OrchestrationV2Command {
460 return {
461 type: "thread.create",
462 createdBy: "user",
463 creationSource: "web",
464 commandId: input.commandId,
465 threadId: input.ids.threadId,
466 projectId: input.ids.projectId,
467 title: `Replay fixture: ${input.scenario}`,
468 modelSelection: input.modelSelection,
469 runtimeMode: input.runtimeMode ?? "full-access",
470 interactionMode: input.interactionMode ?? "default",
471 branch: null,
472 worktreePath: null,
473 };
474}
475
476function dispatchMessageCommand(input: {
477 readonly commandId: CommandId;
478 readonly ids: FixtureIds;
479 readonly modelSelection: ModelSelection;
480 readonly messageId: MessageId;
481 readonly text: string;
482 readonly attachments?: ReadonlyArray<ChatAttachment>;
483 readonly createdBy?: Extract<
484 OrchestrationV2Command,
485 { readonly type: "message.dispatch" }
486 >["createdBy"];
487 readonly creationSource?: Extract<
488 OrchestrationV2Command,
489 { readonly type: "message.dispatch" }
490 >["creationSource"];
491 readonly dispatchMode?: Extract<
492 OrchestrationV2Command,
493 { readonly type: "message.dispatch" }
494 >["dispatchMode"];
495}): OrchestrationV2Command {
496 return {
497 type: "message.dispatch",
498 createdBy: input.createdBy ?? "user",
499 creationSource: input.creationSource ?? "web",
500 commandId: input.commandId,
501 threadId: input.ids.threadId,
502 messageId: input.messageId,
503 text: input.text,
504 attachments: [...(input.attachments ?? [])],
505 modelSelection: input.modelSelection,
506 dispatchMode: input.dispatchMode ?? { type: "start_immediately" },
507 };
508}
509
510export function materializeFixtureInput(input: {
511 readonly scenario: string;
512 readonly fixtureInput: OrchestratorFixtureInput;
513 readonly driver: ProviderDriverKind;
514 readonly modelSelection: ModelSelection;
515}): Effect.Effect<
516 MaterializedOrchestratorFixtureInput,
517 IdAllocator.IdAllocatorV2Error,
518 IdAllocator.IdAllocatorV2
519> {
520 return Effect.gen(function* () {
521 const idAllocator = yield* IdAllocator.IdAllocatorV2;
522 const projectId = yield* idAllocator.allocate.project({ fixtureName: input.scenario });
523 const threadId = yield* idAllocator.allocate.thread({
524 fixtureName: input.scenario,
525 projectId,
526 });
527 const ids = { threadId, projectId } satisfies FixtureIds;
528 const commands: Array<OrchestrationV2Command> = [];
529 const steps: Array<OrchestratorV2ScenarioStep> = [];
530 let messageIndex = 0;
531 let runIndex = 0;
532 const activeRunDispatchKeys = new Set<string>();
533
534 const runIdFor = (runOrdinal: number) =>
535 idAllocator.derive.run({ threadId: ids.threadId, ordinal: runOrdinal });
536
537 const pushDispatch = (
538 command: OrchestrationV2Command,
539 options: {
540 readonly await?: boolean;
541 readonly key?: string;
542 readonly advanceClockAfter?: boolean;
543 } = {},
544 ) => {
545 commands.push(command);
546 steps.push({
547 type: "dispatch",
548 command,
549 await: options.await ?? true,
550 ...(options.key === undefined ? {} : { key: options.key }),
551 });
552 if (options.advanceClockAfter ?? true) {
553 steps.push({ type: "advance_clock", duration: "1 millis" });
554 }
555 };
556
557 pushDispatch(
558 createThreadCommand({
559 commandId: yield* idAllocator.allocate.command({
560 fixtureName: input.scenario,
561 commandName: "thread-create",
562 }),
563 ids,
564 scenario: input.scenario,
565 modelSelection: input.modelSelection,
566 ...(input.fixtureInput.interactionMode === undefined
567 ? {}
568 : { interactionMode: input.fixtureInput.interactionMode }),
569 ...(input.fixtureInput.runtimeMode === undefined
570 ? {}
571 : { runtimeMode: input.fixtureInput.runtimeMode }),
572 }),
573 );
574
575 // A run that asks several times stays busy between answers, so the next
576 // answer waits for its request instead of for the thread to go idle.
577 const answersNext = (stepIndex: number) => {
578 const next = input.fixtureInput.steps[stepIndex + 1]?.type;
579 return next === "approve_next_runtime_request" || next === "answer_next_user_input_request";
580 };
581 for (const [stepIndex, step] of input.fixtureInput.steps.entries()) {
582 switch (step.type) {
583 case "message":
584 messageIndex += 1;
585 runIndex += 1;
586 {
587 const nextStep = input.fixtureInput.steps[stepIndex + 1];
588 const shouldRunInBackground =
589 (nextStep !== undefined &&
590 ((nextStep.type === "interrupt" && nextStep.targetRunIndex === runIndex) ||
591 nextStep.type === "queue_message" ||
592 (nextStep.type === "restart" && nextStep.targetRunIndex === runIndex) ||
593 (nextStep.type === "release_replay_gate_after_waiting" &&
594 nextStep.targetRunIndex === runIndex) ||
595 // A provider continuation run starts while this thread is
596 // busy, so waiting for idle first would never return.
597 (nextStep.type === "await_run_status" && nextStep.targetRunIndex > runIndex) ||
598 // Held open until the test clock moves, so it cannot go idle first.
599 nextStep.type === "finish_held_run")) ||
600 nextStep?.type === "approve_next_runtime_request" ||
601 nextStep?.type === "answer_next_user_input_request";
602 const key = `run:${runIndex}`;
603 pushDispatch(
604 dispatchMessageCommand({
605 commandId: yield* idAllocator.allocate.command({
606 fixtureName: input.scenario,
607 commandName: `message-${messageIndex}`,
608 }),
609 ids,
610 modelSelection: input.modelSelection,
611 messageId: yield* idAllocator.allocate.message({
612 threadId: ids.threadId,
613 ordinal: messageIndex,
614 }),
615 text: step.text,
616 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
617 }),
618 shouldRunInBackground ? { await: false, key } : undefined,
619 );
620 if (shouldRunInBackground) {
621 activeRunDispatchKeys.add(key);
622 } else if (
623 !(
624 nextStep !== undefined &&
625 (nextStep.type === "steer" || nextStep.type === "restart") &&
626 nextStep.targetRunIndex === runIndex
627 )
628 ) {
629 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
630 }
631 }
632 break;
633 case "queue_message": {
634 messageIndex += 1;
635 runIndex += 1;
636 const nextStep = input.fixtureInput.steps[stepIndex + 1];
637 pushDispatch(
638 dispatchMessageCommand({
639 commandId: yield* idAllocator.allocate.command({
640 fixtureName: input.scenario,
641 commandName: `queue-message-${messageIndex}`,
642 }),
643 ids,
644 modelSelection: input.modelSelection,
645 messageId: yield* idAllocator.allocate.message({
646 threadId: ids.threadId,
647 ordinal: messageIndex,
648 }),
649 text: step.text,
650 ...(step.createdBy === undefined ? {} : { createdBy: step.createdBy }),
651 ...(step.creationSource === undefined ? {} : { creationSource: step.creationSource }),
652 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
653 dispatchMode: { type: "queue_after_active" },
654 }),
655 );
656 const shouldSkipQueueBarrier =
657 nextStep?.type === "queue_message" ||
658 (nextStep?.type === "cancel_queued_run" && nextStep.targetRunIndex === runIndex);
659 if (!shouldSkipQueueBarrier) {
660 const queueBarrierKey =
661 Array.from(activeRunDispatchKeys).at(-1) ?? `run:${runIndex - 1}`;
662 activeRunDispatchKeys.delete(queueBarrierKey);
663 steps.push({ type: "await", key: queueBarrierKey });
664 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
665 }
666 break;
667 }
668 case "cancel_queued_run":
669 pushDispatch({
670 type: "queued-run.cancel",
671 commandId: yield* idAllocator.allocate.command({
672 fixtureName: input.scenario,
673 commandName: `cancel-queued-run-${step.targetRunIndex}`,
674 }),
675 threadId: ids.threadId,
676 runId: runIdFor(step.targetRunIndex),
677 });
678 break;
679 case "await_run_status":
680 steps.push({
681 type: "await_run_status",
682 threadId: ids.threadId,
683 runId: runIdFor(step.targetRunIndex),
684 status: step.status,
685 });
686 if (step.waitForTurnItemType !== undefined) {
687 steps.push({
688 type: "await_run_turn_item",
689 threadId: ids.threadId,
690 runId: runIdFor(step.targetRunIndex),
691 itemType: step.waitForTurnItemType,
692 });
693 }
694 break;
695 case "finish_held_run":
696 steps.push({
697 type: "finish_held_run",
698 threadId: ids.threadId,
699 runId: runIdFor(step.targetRunIndex),
700 status: step.status,
701 });
702 break;
703 case "capture_shell_snapshot":
704 steps.push({ type: "capture_shell_snapshot", key: step.key });
705 break;
706 case "release_replay_gate":
707 steps.push({ type: "release_replay_gate", label: step.label });
708 break;
709 case "answer_next_user_input_request":
710 pushDispatch(
711 {
712 type: "runtime-request.respond",
713 commandId: yield* idAllocator.allocate.command({
714 fixtureName: input.scenario,
715 commandName: `answer-user-input-request-${messageIndex}`,
716 }),
717 threadId: ids.threadId,
718 requestId: yield* idAllocator.allocate.runtimeRequest({
719 driver: input.driver,
720 nativeRequestId: `fixture-placeholder-${messageIndex}`,
721 }),
722 answers: step.answers,
723 },
724 { advanceClockAfter: false },
725 );
726 steps[steps.length - 1] = {
727 type: "respond_to_next_runtime_request",
728 threadId: ids.threadId,
729 commandId: commands.at(-1)!.commandId,
730 answers: step.answers,
731 };
732 steps.push({ type: "advance_clock", duration: "1 millis" });
733 if (!answersNext(stepIndex)) {
734 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
735 }
736 break;
737 case "approve_next_runtime_request":
738 pushDispatch(
739 {
740 type: "runtime-request.respond",
741 commandId: yield* idAllocator.allocate.command({
742 fixtureName: input.scenario,
743 commandName: `approve-runtime-request-${messageIndex}`,
744 }),
745 threadId: ids.threadId,
746 requestId: yield* idAllocator.allocate.runtimeRequest({
747 driver: input.driver,
748 nativeRequestId: `fixture-placeholder-${messageIndex}`,
749 }),
750 decision: step.decision ?? "accept",
751 },
752 { advanceClockAfter: false },
753 );
754 steps[steps.length - 1] = {
755 type: "respond_to_next_runtime_request",
756 threadId: ids.threadId,
757 commandId: commands.at(-1)!.commandId,
758 decision: step.decision ?? "accept",
759 ...(step.shellSnapshotKeyWhilePending === undefined
760 ? {}
761 : { shellSnapshotKeyWhilePending: step.shellSnapshotKeyWhilePending }),
762 };
763 steps.push({ type: "advance_clock", duration: "1 millis" });
764 if (!answersNext(stepIndex)) {
765 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
766 }
767 break;
768 case "steer":
769 messageIndex += 1;
770 steps.push({
771 type: "await_run_steerable",
772 threadId: ids.threadId,
773 runId: runIdFor(step.targetRunIndex),
774 });
775 pushDispatch(
776 dispatchMessageCommand({
777 commandId: yield* idAllocator.allocate.command({
778 fixtureName: input.scenario,
779 commandName: `steer-${messageIndex}`,
780 }),
781 ids,
782 modelSelection: input.modelSelection,
783 messageId: yield* idAllocator.allocate.message({
784 threadId: ids.threadId,
785 ordinal: messageIndex,
786 }),
787 text: step.text,
788 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
789 dispatchMode: {
790 type: "steer_active",
791 targetRunId: runIdFor(step.targetRunIndex),
792 },
793 }),
794 );
795 {
796 const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type;
797 if (
798 nextStepType !== "approve_next_runtime_request" &&
799 nextStepType !== "answer_next_user_input_request"
800 ) {
801 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
802 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
803 }
804 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
805 }
806 }
807 break;
808 case "restart":
809 messageIndex += 1;
810 steps.push({
811 type: "await_run_steerable",
812 threadId: ids.threadId,
813 runId: runIdFor(step.targetRunIndex),
814 });
815 pushDispatch(
816 dispatchMessageCommand({
817 commandId: yield* idAllocator.allocate.command({
818 fixtureName: input.scenario,
819 commandName: `restart-${messageIndex}`,
820 }),
821 ids,
822 modelSelection: input.modelSelection,
823 messageId: yield* idAllocator.allocate.message({
824 threadId: ids.threadId,
825 ordinal: messageIndex,
826 }),
827 text: step.text,
828 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
829 dispatchMode: {
830 type: "restart_active",
831 targetRunId: runIdFor(step.targetRunIndex),
832 },
833 }),
834 );
835 {
836 const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type;
837 if (
838 nextStepType !== "approve_next_runtime_request" &&
839 nextStepType !== "answer_next_user_input_request"
840 ) {
841 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
842 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
843 }
844 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
845 }
846 }
847 break;
848 case "interrupt":
849 steps.push({
850 type: "await_run_steerable",
851 threadId: ids.threadId,
852 runId: runIdFor(step.targetRunIndex),
853 });
854 if (step.waitForTurnItemType !== undefined) {
855 steps.push({
856 type: "await_run_turn_item",
857 threadId: ids.threadId,
858 runId: runIdFor(step.targetRunIndex),
859 itemType: step.waitForTurnItemType,
860 });
861 }
862 pushDispatch(
863 {
864 type: "run.interrupt",
865 commandId: yield* idAllocator.allocate.command({
866 fixtureName: input.scenario,
867 commandName: `interrupt-${step.targetRunIndex}`,
868 }),
869 threadId: ids.threadId,
870 runId: runIdFor(step.targetRunIndex),
871 },
872 { advanceClockAfter: false },
873 );
874 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
875 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
876 }
877 steps.push({ type: "advance_clock", duration: "1 millis" });
878 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
879 break;
880 case "stop_background_work":
881 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
882 steps.push({ type: "capture_shell_snapshot", key: "before-stop" });
883 pushDispatch(
884 {
885 type: "run.interrupt",
886 commandId: yield* idAllocator.allocate.command({
887 fixtureName: input.scenario,
888 commandName: `stop-background-work-${step.targetRunIndex}`,
889 }),
890 threadId: ids.threadId,
891 runId: runIdFor(step.targetRunIndex),
892 holdQueue: true,
893 },
894 { advanceClockAfter: false },
895 );
896 steps.push({ type: "await_no_background_work", threadId: ids.threadId });
897 break;
898 case "release_replay_gate_after_waiting":
899 steps.push({
900 type: "release_replay_gate_after_waiting",
901 label: step.label,
902 threadId: ids.threadId,
903 runId: runIdFor(step.targetRunIndex),
904 });
905 break;
906 case "advance_clock":
907 steps.push({ type: "advance_clock", duration: step.duration });
908 break;
909 case "rollback":
910 {
911 const scopeId = yield* idAllocator.allocate.checkpointScope({
912 threadId: ids.threadId,
913 name: step.checkpointScopeSuffix,
914 });
915 pushDispatch({
916 type: "checkpoint.rollback",
917 restoreFiles: false,
918 commandId: yield* idAllocator.allocate.command({
919 fixtureName: input.scenario,
920 commandName: `rollback-${step.checkpointSuffix}`,
921 }),
922 threadId: ids.threadId,
923 scopeId,
924 checkpointId: yield* idAllocator.allocate.checkpoint({
925 checkpointScopeId: scopeId,
926 name: step.checkpointSuffix,
927 }),
928 });
929 }
930 break;
931 }
932 }
933
934 if (activeRunDispatchKeys.size > 0) {
935 steps.push({ type: "await_all" });
936 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
937 }
938
939 return {
940 commands,
941 steps,
942 projectionThreadIds: [ids.threadId],
943 };
944 });
945}
946
947export function projectionFor(
948 result: OrchestratorV2ScenarioResult,
949 scenario: string,
950): OrchestrationV2ThreadProjection {
951 const projections = [...result.projections.values()].filter(
952 (projection) => projection.thread.lineage.parentThreadId === null,
953 );
954
955 assert.equal(projections.length, 1, `expected one root projection for ${scenario}`);
956 const projection = projections[0];
957 assert.isDefined(projection, `missing projection for ${scenario}`);
958 return projection;
959}
960
961export function assertBaseProjection(input: {
962 readonly result: OrchestratorV2ScenarioResult;
963 readonly transcript: ProviderReplayTranscript;
964 readonly runCount: number;
965 readonly providerTurnCountAtLeast?: number;
966 readonly runStatuses?: ReadonlyArray<OrchestrationV2RunStatus>;
967}) {
968 const projection = projectionFor(input.result, input.transcript.scenario);
969
970 assert.equal(
971 projection.thread.providerInstanceId,
972 ProviderInstanceId.make(input.transcript.provider),
973 );
974 assert.lengthOf(projection.runs, input.runCount);
975 assert.isAtLeast(projection.providerThreads.length, 1);
976 assert.isAtLeast(
977 projection.providerTurns.length,
978 input.providerTurnCountAtLeast ?? input.runCount,
979 `expected provider turns; runs=${projection.runs.map((run) => `${run.id}:${run.status}`).join(",")}; sessions=${projection.providerSessions.map((session) => `${session.id}:${session.status}`).join(",")}; items=${projection.turnItems.map((item) => (item.type === "error" ? `${item.type}:${item.failure.message}` : item.type)).join(",")}`,
980 );
981 assert.isAtLeast(input.result.domainEvents.length, 1);
982 assert.deepEqual(
983 input.result.storedEvents.map((stored) => stored.sequence),
984 input.result.storedEvents.map((_, index) => index + 1),
985 );
986 assert.deepEqual(
987 input.result.storedEvents.map((stored) => stored.event.id),
988 input.result.domainEvents.map((event) => event.id),
989 );
990
991 if (input.runStatuses) {
992 assert.deepEqual(
993 projection.runs.map((run) => run.status),
994 input.runStatuses,
995 );
996 }
997}
998
999export function assertRunOrdinals(
1000 projection: OrchestrationV2ThreadProjection,
1001 expectedOrdinals: ReadonlyArray<number>,
1002) {
1003 assert.deepEqual(
1004 projection.runs.map((run) => run.ordinal),
1005 expectedOrdinals,
1006 );
1007}
1008
1009function assertRunsHaveRootNodes(projection: OrchestrationV2ThreadProjection) {
1010 for (const run of projection.runs) {
1011 assert.isNotNull(run.rootNodeId, `run ${run.id} must have a root node`);
1012 assert.isTrue(
1013 projection.nodes.some((node) => node.id === run.rootNodeId && node.kind === "root_turn"),
1014 `run ${run.id} root node must exist`,
1015 );
1016 }
1017}
1018
1019function assertRootNodesCountForRuns(projection: OrchestrationV2ThreadProjection) {
1020 const rootNodes = projection.nodes.filter((node) => node.kind === "root_turn");
1021 assert.isAtLeast(rootNodes.length, projection.runs.length);
1022 for (const node of rootNodes) {
1023 assert.equal(node.countsForRun, true, `root node ${node.id} must count for its app run`);
1024 }
1025}
1026
1027function assertProviderTurnsReferenceNodes(projection: OrchestrationV2ThreadProjection) {
1028 for (const providerTurn of projection.providerTurns) {
1029 assert.isTrue(
1030 projection.nodes.some((node) => node.id === providerTurn.nodeId),
1031 `provider turn ${providerTurn.id} must reference an execution node`,
1032 );
1033 assert.isTrue(
1034 projection.providerThreads.some((thread) => thread.id === providerTurn.providerThreadId),
1035 `provider turn ${providerTurn.id} must reference a provider thread`,
1036 );
1037 }
1038}
1039
1040function assertTurnItemsAreOrdered(projection: OrchestrationV2ThreadProjection) {
1041 const ordinals = projection.turnItems.map((item) => item.ordinal);
1042 assert.deepEqual(
1043 ordinals,
1044 [...ordinals].toSorted((left, right) => left - right),
1045 );
1046}
1047
1048function assertTurnItemsReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1049 for (const item of projection.turnItems) {
1050 if (item.runId !== null) {
1051 assert.isTrue(
1052 projection.runs.some((run) => run.id === item.runId),
1053 `turn item ${item.id} must reference an existing run`,
1054 );
1055 }
1056 if (item.nodeId !== null) {
1057 assert.isTrue(
1058 projection.nodes.some((node) => node.id === item.nodeId),
1059 `turn item ${item.id} must reference an existing node`,
1060 );
1061 }
1062 if (item.providerTurnId !== null) {
1063 assert.isTrue(
1064 projection.providerTurns.some((turn) => turn.id === item.providerTurnId),
1065 `turn item ${item.id} must reference an existing provider turn`,
1066 );
1067 }
1068 }
1069}
1070
1071export function assertVisibleTurnItemsMirrorLocalTurnItems(
1072 projection: OrchestrationV2ThreadProjection,
1073) {
1074 assert.lengthOf(
1075 projection.visibleTurnItems,
1076 projection.turnItems.length,
1077 "non-fork visible turn items must mirror local canonical turn items",
1078 );
1079
1080 for (const [index, item] of projection.turnItems.entries()) {
1081 const visibleItem = projection.visibleTurnItems[index];
1082 assert.isDefined(visibleItem, `missing visible turn item at position ${index}`);
1083 assert.equal(visibleItem.position, index);
1084 assert.equal(visibleItem.visibility, "local");
1085 assert.equal(visibleItem.sourceThreadId, item.threadId);
1086 assert.equal(visibleItem.sourceItemId, item.id);
1087 assert.deepEqual(visibleItem.item, item);
1088 }
1089}
1090
1091function assertMessagesReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1092 for (const message of projection.messages) {
1093 if (message.runId !== null) {
1094 assert.isTrue(
1095 projection.runs.some((run) => run.id === message.runId),
1096 `message ${message.id} must reference an existing run`,
1097 );
1098 }
1099 if (message.nodeId !== null) {
1100 assert.isTrue(
1101 projection.nodes.some((node) => node.id === message.nodeId),
1102 `message ${message.id} must reference an existing node`,
1103 );
1104 }
1105 }
1106}
1107
1108function assertRuntimeRequestsReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1109 for (const request of projection.runtimeRequests) {
1110 const requestNode = projection.nodes.find((node) => node.id === request.nodeId);
1111 assert.isTrue(
1112 requestNode !== undefined,
1113 `runtime request ${request.id} must reference an existing node`,
1114 );
1115 if (
1116 requestNode !== undefined &&
1117 (request.kind === "command" || request.kind === "file-read" || request.kind === "file-change")
1118 ) {
1119 assert.equal(
1120 requestNode.kind,
1121 "approval_request",
1122 `runtime request ${request.id} must reference an approval request node`,
1123 );
1124 }
1125 if (request.providerTurnId !== null) {
1126 assert.isTrue(
1127 projection.providerTurns.some((turn) => turn.id === request.providerTurnId),
1128 `runtime request ${request.id} must reference an existing provider turn`,
1129 );
1130 }
1131 }
1132}
1133
1134export function assertSemanticProjectionIntegrity(projection: OrchestrationV2ThreadProjection) {
1135 assertRunsHaveRootNodes(projection);
1136 assertRootNodesCountForRuns(projection);
1137 assertProviderTurnsReferenceNodes(projection);
1138 assertTurnItemsAreOrdered(projection);
1139 assertTurnItemsReferenceProjection(projection);
1140 assertMessagesReferenceProjection(projection);
1141 assertRuntimeRequestsReferenceProjection(projection);
1142}
1143
1144export function assertRunProviderTurnCardinality(input: {
1145 readonly projection: OrchestrationV2ThreadProjection;
1146 readonly rootRunCount: number;
1147 readonly providerTurnCountAtLeast?: number;
1148}) {
1149 assert.lengthOf(input.projection.runs, input.rootRunCount);
1150 assert.isAtLeast(
1151 input.projection.providerTurns.length,
1152 input.providerTurnCountAtLeast ?? input.rootRunCount,
1153 );
1154}
1155
1156export function assertNoExtraAppRunsForProviderChildren(input: {
1157 readonly projection: OrchestrationV2ThreadProjection;
1158 readonly expectedAppRuns: number;
1159}) {
1160 assert.lengthOf(
1161 input.projection.runs,
1162 input.expectedAppRuns,
1163 "provider child activity must not create additional app runs",
1164 );
1165}
1166
1167/**
1168 * Provider-native subagent threads have no runs; clients show them working
1169 * from the child's runless root turn. Pin that contract for every recorded
1170 * native subagent: the child hangs off the subagent node, every root turn is
1171 * runless, the root turn is live before the child's first item, and its
1172 * activity mirrors the subagent's (including a resume re-opening it).
1173 */
1174export function assertProviderNativeSubagentRootTurns(result: OrchestratorV2ScenarioResult) {
1175 const activity = (statuses: ReadonlyArray<OrchestrationV2ExecutionNode["status"]>) =>
1176 statuses
1177 .map((status) => (isOrchestrationV2WorkActive(status) ? "active" : status))
1178 .filter((status, index, all) => status !== all[index - 1]);
1179 for (const projection of result.projections.values()) {
1180 for (const subagent of projection.subagents) {
1181 if (subagent.origin !== "provider_native" || subagent.childThreadId === null) continue;
1182 const childThreadId = subagent.childThreadId;
1183 const child = result.projections.get(childThreadId);
1184 assert.isDefined(child, `missing child thread for subagent ${subagent.id}`);
1185 assert.equal(child.thread.creationSource, "provider");
1186 assert.deepEqual(child.thread.forkedFrom, { type: "node", nodeId: subagent.id });
1187 assert.lengthOf(child.runs, 0);
1188 const roots = child.nodes.filter((node) => node.kind === "root_turn");
1189 assert.isNotEmpty(roots, `child ${childThreadId} must have a root turn`);
1190 for (const root of roots) assert.isNull(root.runId);
1191
1192 // One run ingests a child thread at a time, so no update is stored by two.
1193 const runByChildUpdate = new Map<string, string | undefined>();
1194 for (const event of result.domainEvents) {
1195 if (event.threadId !== childThreadId) continue;
1196 const update = `${event.type}:${JSON.stringify(event.payload)}`;
1197 const storedBy = runByChildUpdate.get(update);
1198 if (runByChildUpdate.has(update) && storedBy !== event.runId) {
1199 assert.fail(
1200 `child ${childThreadId} stored ${event.type} in ${storedBy} and ${event.runId}`,
1201 );
1202 }
1203 runByChildUpdate.set(update, event.runId);
1204 }
1205
1206 const rootEvents = result.domainEvents.flatMap((event, index) =>
1207 event.type === "node.updated" &&
1208 event.payload.threadId === childThreadId &&
1209 event.payload.kind === "root_turn"
1210 ? [{ index, status: event.payload.status }]
1211 : [],
1212 );
1213 const firstItemIndex = result.domainEvents.findIndex(
1214 (event) =>
1215 event.type === "turn-item.updated" &&
1216 event.payload.threadId === childThreadId &&
1217 event.payload.type !== "user_message",
1218 );
1219 assert.equal(rootEvents[0]?.status, "running");
1220 if (firstItemIndex !== -1) {
1221 assert.isBelow(
1222 rootEvents[0]?.index ?? Infinity,
1223 firstItemIndex,
1224 `child ${childThreadId} must be working before its first item`,
1225 );
1226 }
1227 const subagentStatuses = result.domainEvents.flatMap((event) =>
1228 event.type === "subagent.updated" && event.payload.id === subagent.id
1229 ? [event.payload.status]
1230 : [],
1231 );
1232 const rootActivity = activity(rootEvents.map((event) => event.status));
1233 const subagentActivity = activity(subagentStatuses);
1234 // A subagent can be woken after its call ended, to answer the report of
1235 // a background subagent of its own; each of those turns ends too.
1236 const wokenAfterEnd =
1237 child.subagents.length > 0 && rootActivity.length > subagentActivity.length;
1238 assert.deepEqual(
1239 wokenAfterEnd ? rootActivity.slice(0, subagentActivity.length) : rootActivity,
1240 subagentActivity,
1241 `child ${childThreadId} root turn must follow subagent ${subagent.id}`,
1242 );
1243 if (wokenAfterEnd) {
1244 assert.notEqual(rootActivity.at(-1), "active", `child ${childThreadId} must end its turns`);
1245 // Each extra turn answers a report: one of the child's own subagents
1246 // ended before that turn started.
1247 const childSubagentIds = new Set(child.subagents.map((nested) => nested.id));
1248 const nestedEndIndexes = result.domainEvents.flatMap((event, index) =>
1249 event.type === "subagent.updated" &&
1250 childSubagentIds.has(event.payload.id) &&
1251 !isOrchestrationV2WorkActive(event.payload.status)
1252 ? [index]
1253 : [],
1254 );
1255 const subagentEndIndex = result.domainEvents.findLastIndex(
1256 (event) =>
1257 event.type === "subagent.updated" &&
1258 event.payload.id === subagent.id &&
1259 !isOrchestrationV2WorkActive(event.payload.status),
1260 );
1261 const wakeStarts = rootEvents.filter(
1262 (event, position) =>
1263 event.index > subagentEndIndex &&
1264 isOrchestrationV2WorkActive(event.status) &&
1265 !isOrchestrationV2WorkActive(rootEvents[position - 1]?.status ?? "completed"),
1266 );
1267 assert.isNotEmpty(wakeStarts, `child ${childThreadId} woke without a new turn`);
1268 assert.isAtMost(
1269 wakeStarts.length,
1270 nestedEndIndexes.length,
1271 `child ${childThreadId} woke more often than its subagents ended`,
1272 );
1273 for (const wake of wakeStarts) {
1274 assert.isTrue(
1275 nestedEndIndexes.some((endIndex) => endIndex < wake.index),
1276 `child ${childThreadId} woke before any of its subagents ended`,
1277 );
1278 }
1279 }
1280 }
1281 }
1282}
1283
1284export function assertExecutionNodeKinds(
1285 projection: OrchestrationV2ThreadProjection,
1286 expectedKinds: ReadonlyArray<OrchestrationV2ExecutionNode["kind"]>,
1287) {
1288 const kinds = projection.nodes.map((node) => node.kind);
1289 for (const expectedKind of expectedKinds) {
1290 assert.include(kinds, expectedKind);
1291 }
1292}
1293
1294export function assertTurnItemTypes(
1295 projection: OrchestrationV2ThreadProjection,
1296 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1297) {
1298 const actualTypes = projection.turnItems.map((item) => item.type);
1299 for (const expectedType of expectedTypes) {
1300 assert.include(actualTypes, expectedType);
1301 }
1302}
1303
1304export function assertTurnItemTypeSequence(
1305 projection: OrchestrationV2ThreadProjection,
1306 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1307) {
1308 assert.deepEqual(
1309 projection.turnItems.map((item) => item.type),
1310 expectedTypes,
1311 );
1312}
1313
1314export function assertVisibleTurnItemTypeSequence(
1315 projection: OrchestrationV2ThreadProjection,
1316 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1317) {
1318 assert.deepEqual(
1319 projection.visibleTurnItems.map((row) => row.item.type),
1320 expectedTypes,
1321 );
1322}
1323
1324export function assertAssistantTextIncludes(
1325 projection: OrchestrationV2ThreadProjection,
1326 expectedText: string,
1327) {
1328 assert.isTrue(
1329 projection.turnItems.some(
1330 (item) => item.type === "assistant_message" && item.text.includes(expectedText),
1331 ),
1332 `expected assistant output to include ${JSON.stringify(expectedText)}`,
1333 );
1334}
1335
1336export function assertRuntimeRequestCounts(
1337 projection: OrchestrationV2ThreadProjection,
1338 expected: { readonly total: number; readonly resolved?: number },
1339) {
1340 assert.lengthOf(projection.runtimeRequests, expected.total);
1341 if (expected.resolved !== undefined) {
1342 assert.equal(
1343 projection.runtimeRequests.filter((request) => request.status === "resolved").length,
1344 expected.resolved,
1345 );
1346 }
1347}
1348
1349function countReplayLabelsWithPrefix(transcript: ProviderReplayTranscript, prefix: string): number {
1350 return transcript.entries.filter(
1351 (entry) => entry.type !== "runtime_exit" && (entry.label?.startsWith(prefix) ?? false),
1352 ).length;
1353}
1354
1355export function assertReplayLabelPrefixCount(
1356 transcript: ProviderReplayTranscript,
1357 prefix: string,
1358 expected: number,
1359) {
1360 assert.equal(countReplayLabelsWithPrefix(transcript, prefix), expected);
1361}
1362
1363export function assertRuntimeRequestKinds(
1364 projection: OrchestrationV2ThreadProjection,
1365 expectedKinds: ReadonlyArray<string>,
1366) {
1367 assert.deepEqual(
1368 projection.runtimeRequests.map((request) => request.kind),
1369 expectedKinds,
1370 );
1371}
1372
1373export function assertAllRuntimeRequestsResolved(projection: OrchestrationV2ThreadProjection) {
1374 assert.deepEqual(
1375 projection.runtimeRequests.map((request) => request.status),
1376 projection.runtimeRequests.map(() => "resolved"),
1377 );
1378}
1379
1380export function assertConversationMessageRoles(
1381 projection: OrchestrationV2ThreadProjection,
1382 expectedRoles: ReadonlyArray<string>,
1383) {
1384 assert.deepEqual(
1385 projection.messages.map((message) => message.role),
1386 expectedRoles,
1387 );
1388}
1389
1390/**
1391 * ACP agents run their own file and shell work: T3 advertises neither
1392 * capability (the transcript pins its initialize) and the agent never asks.
1393 */
1394export function assertNoAcpClientFileOrTerminalRequests(transcript: ProviderReplayTranscript) {
1395 const frames = transcript.entries.flatMap((entry) =>
1396 entry.type === "runtime_exit"
1397 ? []
1398 : [entry.frame as { method?: unknown; params?: { clientCapabilities?: unknown } }],
1399 );
1400 assert.deepInclude(
1401 frames.find((frame) => frame.method === "initialize")?.params?.clientCapabilities ?? {},
1402 { fs: { readTextFile: false, writeTextFile: false }, terminal: false },
1403 "T3 must not advertise client fs or terminals",
1404 );
1405 assert.deepEqual(
1406 frames.flatMap((frame) =>
1407 typeof frame.method === "string" && /^(fs|terminal)\//u.test(frame.method)
1408 ? [frame.method]
1409 : [],
1410 ),
1411 [],
1412 "the agent must not route file or terminal work through T3",
1413 );
1414}
1415
1416export function assertUserMessagesInclude(
1417 projection: OrchestrationV2ThreadProjection,
1418 expectedTexts: ReadonlyArray<string>,
1419) {
1420 for (const expectedText of expectedTexts) {
1421 assert.isTrue(
1422 projection.turnItems.some(
1423 (item) => item.type === "user_message" && item.text.includes(expectedText),
1424 ),
1425 `expected user input to include ${JSON.stringify(expectedText)}`,
1426 );
1427 }
1428}
1429
1430function assertUserMessagesExclude(
1431 projection: OrchestrationV2ThreadProjection,
1432 rejectedTexts: ReadonlyArray<string>,
1433) {
1434 for (const rejectedText of rejectedTexts) {
1435 assert.isFalse(
1436 projection.turnItems.some(
1437 (item) => item.type === "user_message" && item.text.includes(rejectedText),
1438 ),
1439 `expected user input to exclude ${JSON.stringify(rejectedText)}`,
1440 );
1441 }
1442}
1443
1444export function assertVisibleUserMessagesInclude(
1445 projection: OrchestrationV2ThreadProjection,
1446 expectedTexts: ReadonlyArray<string>,
1447) {
1448 for (const expectedText of expectedTexts) {
1449 assert.isTrue(
1450 projection.visibleTurnItems.some(
1451 (row) => row.item.type === "user_message" && row.item.text.includes(expectedText),
1452 ),
1453 `expected visible user input to include ${JSON.stringify(expectedText)}`,
1454 );
1455 }
1456}
1457
1458export function assertVisibleUserMessagesExclude(
1459 projection: OrchestrationV2ThreadProjection,
1460 rejectedTexts: ReadonlyArray<string>,
1461) {
1462 for (const rejectedText of rejectedTexts) {
1463 assert.isFalse(
1464 projection.visibleTurnItems.some(
1465 (row) => row.item.type === "user_message" && row.item.text.includes(rejectedText),
1466 ),
1467 `expected visible user input to exclude ${JSON.stringify(rejectedText)}`,
1468 );
1469 }
1470}
1471
1472export function assertUserMessageInputIntents(
1473 projection: OrchestrationV2ThreadProjection,
1474 expectedIntents: ReadonlyArray<OrchestrationV2UserMessageInputIntent>,
1475) {
1476 assert.deepEqual(
1477 projection.turnItems
1478 .filter((item) => item.type === "user_message")
1479 .map((item) => item.inputIntent),
1480 expectedIntents,
1481 );
1482}
1483
1484/** The background-work notifications a thread's timeline shows, in order. */
1485export function backgroundNotifications(projection: OrchestrationV2ThreadProjection) {
1486 return projection.turnItems.flatMap((item) =>
1487 item.type === "notification"
1488 ? [{ summary: item.summary, outcome: item.outcome, source: item.source }]
1489 : [],
1490 );
1491}