1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, describe, it } from "@effect/vitest";
3import {
4 CommandId,
5 isProviderNativeSubagentThread,
6 MessageId,
7 ProviderDriverKind,
8 type ProviderReplayTranscript,
9} from "@t3tools/contracts";
10import * as Effect from "effect/Effect";
11
12import { classifyClaudeNativeTool } from "../Adapters/ClaudeAdapterV2.ts";
13import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts";
14import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
15import * as Orchestrator from "../Orchestrator.ts";
16import { userFacingDispatchErrorMessage } from "../UserFacingErrors.ts";
17import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
18import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
19import { subagentInput } from "./fixtures/subagent/input.ts";
20import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
21import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
22import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
23import { CLAUDE_MODEL_SELECTION, materializeFixtureInput } from "./fixtures/shared.ts";
24import {
25 THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER,
26 THREAD_FORK_NATIVE_CONTINUE_RECALL,
27 THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER,
28 THREAD_MERGE_BACK_FORK_MARKER,
29 THREAD_MERGE_BACK_RECALL,
30 THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER,
31 THREAD_MERGE_BACK_SIBLINGS_RECALL,
32 THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER,
33 THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER,
34 THREAD_MERGE_BACK_SOURCE_MARKER,
35} from "./fixtures/shared.ts";
36import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
37
38const readTranscript = Effect.fn("readClaudeReplayFixture")(function* (file: URL) {
39 return yield* readProviderReplayTranscript(file);
40}, Effect.provide(NodeServices.layer));
41
42function readClaudeTranscriptFixture(path: string) {
43 return readTranscript(new URL(`./fixtures/${path}/claude_transcript.ndjson`, import.meta.url));
44}
45
46function isRecord(value: unknown): value is Record<string, unknown> {
47 return typeof value === "object" && value !== null;
48}
49
50function metadataString(transcript: ProviderReplayTranscript, key: string): string {
51 const value = transcript.metadata?.[key];
52 if (typeof value !== "string") {
53 throw new Error(`${transcript.scenario} metadata.${key} must be a string.`);
54 }
55 return value;
56}
57
58function metadataStringArray(
59 transcript: ProviderReplayTranscript,
60 key: string,
61): ReadonlyArray<string> {
62 const value = transcript.metadata?.[key];
63 if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) {
64 throw new Error(`${transcript.scenario} metadata.${key} must be a string array.`);
65 }
66 return value;
67}
68
69type FramedReplayEntry = Extract<
70 ProviderReplayTranscript["entries"][number],
71 { readonly frame: unknown }
72>;
73
74function frameRecord(entry: FramedReplayEntry): Record<string, unknown> {
75 if (!isRecord(entry.frame)) {
76 throw new Error("Replay entry frame must be an object.");
77 }
78 return entry.frame;
79}
80
81function findEntryFrame(
82 transcript: ProviderReplayTranscript,
83 label: string,
84): Record<string, unknown> {
85 const entry = transcript.entries.find(
86 (candidate): candidate is FramedReplayEntry =>
87 "label" in candidate && candidate.label === label && "frame" in candidate,
88 );
89 assert.isDefined(entry, `${transcript.scenario} must include replay entry ${label}`);
90 return frameRecord(entry);
91}
92
93function successResultTexts(transcript: ProviderReplayTranscript): ReadonlyArray<string> {
94 return transcript.entries.flatMap((entry) => {
95 if (entry.type !== "emit_inbound" || !isRecord(entry.frame)) {
96 return [];
97 }
98 if (entry.frame.type !== "result" || entry.frame.subtype !== "success") {
99 return [];
100 }
101 return typeof entry.frame.result === "string" ? [entry.frame.result] : [];
102 });
103}
104
105function claudeToolUseNamesFromTranscript(
106 transcript: ProviderReplayTranscript,
107): ReadonlyArray<string> {
108 return transcript.entries.flatMap((entry) => {
109 if (
110 entry.type !== "emit_inbound" ||
111 !isRecord(entry.frame) ||
112 entry.frame.type !== "assistant"
113 ) {
114 return [];
115 }
116
117 const message = entry.frame.message;
118 const content = isRecord(message) ? message.content : undefined;
119 if (!Array.isArray(content)) {
120 return [];
121 }
122
123 return content.flatMap((part) =>
124 isRecord(part) &&
125 typeof part.id === "string" &&
126 typeof part.name === "string" &&
127 "input" in part
128 ? [part.name]
129 : [],
130 );
131 });
132}
133
134describe("Claude Agent SDK replay fixtures", () => {
135 it.effect("refuses messages to a native subagent thread without touching it", () =>
136 Effect.gen(function* () {
137 const raw = yield* readClaudeTranscriptFixture("subagent");
138 const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript(
139 materializeReplayTranscriptRuntimeInstructions(raw, {
140 driver: ProviderDriverKind.make("claudeAgent"),
141 model: CLAUDE_MODEL_SELECTION.model,
142 }),
143 );
144 const materialized = yield* materializeFixtureInput({
145 scenario: "subagent",
146 fixtureInput: subagentInput(),
147 driver: ProviderDriverKind.make("claudeAgent"),
148 modelSelection: CLAUDE_MODEL_SELECTION,
149 }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
150 const scenario = {
151 name: "subagent/claudeAgent:read-only-child",
152 transcript,
153 commands: materialized.commands,
154 steps: materialized.steps,
155 projectionThreadIds: materialized.projectionThreadIds,
156 };
157 yield* Effect.gen(function* () {
158 const result = yield* runOrchestratorV2Scenario(scenario);
159 const orchestrator = yield* Orchestrator.OrchestratorV2;
160 const child = [...result.projections.values()].find((projection) =>
161 isProviderNativeSubagentThread(projection.thread),
162 );
163 assert.isDefined(child);
164 const before = yield* orchestrator.getThreadEventSequence(child.thread.id);
165
166 const refused = yield* orchestrator
167 .dispatch({
168 type: "message.dispatch",
169 createdBy: "user",
170 creationSource: "web",
171 commandId: CommandId.make("command:subagent:message-native-child"),
172 threadId: child.thread.id,
173 messageId: MessageId.make("message:subagent:message-native-child"),
174 text: "Also check the tests.",
175 attachments: [],
176 dispatchMode: { type: "start_immediately" },
177 })
178 .pipe(Effect.flip);
179 assert.equal(refused._tag, "OrchestratorSubagentThreadReadOnlyError");
180 // The wire error carries this text to the web toast and mobile outbox.
181 assert.equal(
182 userFacingDispatchErrorMessage(refused),
183 "This subagent is run by its provider and cannot take messages. Message the parent thread instead.",
184 );
185 assert.equal(yield* orchestrator.getThreadEventSequence(child.thread.id), before);
186 const after = yield* orchestrator.getThreadProjection(child.thread.id);
187 assert.lengthOf(after.runs, 0);
188 assert.deepEqual(after.messages, child.messages);
189 }).pipe(
190 Effect.provide(
191 ProviderReplayHarness.layerProviderReplay(scenario, ClaudeOrchestratorReplayHarness),
192 ),
193 provideDeterministicTestRuntime,
194 Effect.scoped,
195 );
196 }),
197 );
198
199 it.effect("classifies every Claude fixture tool use through the native tool table", () =>
200 Effect.gen(function* () {
201 const unknownToolNames = new Set<string>();
202 const seenToolNames = new Set<string>();
203
204 for (const fixture of ORCHESTRATOR_REPLAY_FIXTURES) {
205 for (const provider of fixture.providers) {
206 if (provider.driver !== "claudeAgent") {
207 continue;
208 }
209
210 const transcript = yield* readTranscript(provider.transcriptFile);
211 for (const toolName of claudeToolUseNamesFromTranscript(transcript)) {
212 seenToolNames.add(toolName);
213 const classification = classifyClaudeNativeTool(toolName);
214 // MCP tools are open-ended and deliberately become dynamic tools.
215 if (!classification.known && !toolName.startsWith("mcp__")) {
216 unknownToolNames.add(`${fixture.name}:${toolName}`);
217 }
218 }
219 }
220 }
221
222 assert.isAtLeast(seenToolNames.size, 1, "expected Claude fixtures to contain tool uses");
223 assert.deepEqual([...unknownToolNames], []);
224 }),
225 );
226
227 it.effect("keeps unregistered native conversation-state transcripts reviewable", () =>
228 Effect.gen(function* () {
229 const rollback = yield* readClaudeTranscriptFixture("thread_rollback");
230 assert.equal(rollback.metadata?.queryMode, "resume_at_cursor");
231 const rollbackCursor = metadataString(rollback, "resumeSessionAt");
232 const rollbackResumeFrame = findEntryFrame(rollback, "query.open:resume_at_cursor");
233 const rollbackResumeOptions = rollbackResumeFrame.options;
234 if (!isRecord(rollbackResumeOptions)) {
235 throw new Error("Rollback resume query.open options must be an object.");
236 }
237 assert.equal(rollbackResumeOptions.resumeSessionAt, rollbackCursor);
238 const rollbackFinalText = successResultTexts(rollback).at(-1) ?? "";
239 assert.include(rollbackFinalText, "rollback fixture first turn complete");
240 assert.notInclude(rollbackFinalText, "rollback fixture second turn complete");
241
242 const latestFork = yield* readClaudeTranscriptFixture("thread_fork_native");
243 assert.equal(latestFork.metadata?.queryMode, "fork_session");
244 const latestForkedSessionId = metadataString(latestFork, "forkedNativeSessionId");
245 const latestForkedFrame = findEntryFrame(latestFork, "session.forked");
246 assert.equal(latestForkedFrame.sessionId, latestForkedSessionId);
247
248 const priorFork = yield* readClaudeTranscriptFixture("thread_fork_native_prior_turn");
249 assert.equal(priorFork.metadata?.queryMode, "fork_session_prior_turn");
250 const priorForkCursor = metadataString(priorFork, "forkUpToMessageId");
251 const priorForkFrame = findEntryFrame(priorFork, "session.fork");
252 const priorForkOptions = priorForkFrame.options;
253 if (!isRecord(priorForkOptions)) {
254 throw new Error("Prior-turn fork options must be an object.");
255 }
256 assert.equal(priorForkOptions.upToMessageId, priorForkCursor);
257 const priorForkFinalText = successResultTexts(priorFork).at(-1) ?? "";
258 assert.include(priorForkFinalText, "fork boundary alpha");
259 assert.notInclude(priorForkFinalText, "fork boundary beta");
260
261 const continuedFork = yield* readClaudeTranscriptFixture("thread_fork_native_continue");
262 assert.equal(continuedFork.metadata?.queryMode, "fork_session_continue");
263 const continuedForkSessionId = metadataString(continuedFork, "forkedNativeSessionId");
264 const continuedForkOpenFrame = findEntryFrame(continuedFork, "query.open:fork");
265 const continuedForkOptions = continuedForkOpenFrame.options;
266 if (!isRecord(continuedForkOptions)) {
267 throw new Error("Continued fork query.open options must be an object.");
268 }
269 assert.equal(continuedForkOptions.resume, continuedForkSessionId);
270 const continuedForkResults = successResultTexts(continuedFork);
271 assert.deepEqual(continuedForkResults.slice(0, 2), [
272 "source marker stored",
273 "fork marker stored",
274 ]);
275 assert.equal(
276 continuedForkResults.at(-1)?.replace(/\s*\|\s*/u, "|"),
277 THREAD_FORK_NATIVE_CONTINUE_RECALL,
278 );
279 const recallPromptFrame = findEntryFrame(continuedFork, "prompt.offer:3");
280 const recallMessage = recallPromptFrame.message;
281 const recallMessageBody = isRecord(recallMessage) ? recallMessage.message : undefined;
282 const recallPrompt =
283 isRecord(recallMessageBody) && typeof recallMessageBody.content === "string"
284 ? recallMessageBody.content
285 : "";
286 assert.notInclude(recallPrompt, THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER);
287 assert.notInclude(recallPrompt, THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER);
288
289 const siblingForks = yield* readClaudeTranscriptFixture("thread_fork_native_siblings");
290 assert.equal(siblingForks.metadata?.queryMode, "fork_session_siblings");
291 const siblingSessionIds = metadataStringArray(siblingForks, "forkedNativeSessionIds");
292 assert.lengthOf(siblingSessionIds, 2);
293 assert.notEqual(siblingSessionIds[0], siblingSessionIds[1]);
294 const siblingResults = successResultTexts(siblingForks).map((text) =>
295 text.replace(/\s*\|\s*/u, "|"),
296 );
297 assert.deepEqual(siblingResults, [
298 "sibling source stored",
299 "sibling-source-8R3D|sibling-first-5L2P",
300 "sibling-source-8R3D|sibling-second-9N6C",
301 ]);
302 assert.notInclude(siblingResults[1] ?? "", "sibling-second-9N6C");
303 assert.notInclude(siblingResults[2] ?? "", "sibling-first-5L2P");
304
305 const mergeBack = yield* readClaudeTranscriptFixture("thread_merge_back_continue");
306 assert.equal(mergeBack.metadata?.queryMode, "fork_session_merge_back");
307 const mergeBackSourceSessionId = metadataString(mergeBack, "nativeSessionId");
308 const mergeBackContinuationFrame = findEntryFrame(
309 mergeBack,
310 "query.open:source-continuation",
311 );
312 const mergeBackContinuationOptions = mergeBackContinuationFrame.options;
313 if (!isRecord(mergeBackContinuationOptions)) {
314 throw new Error("Merge-back source continuation options must be an object.");
315 }
316 assert.equal(mergeBackContinuationOptions.resume, mergeBackSourceSessionId);
317 const mergeBackRecallFrame = findEntryFrame(mergeBack, "prompt.offer:4");
318 const mergeBackRecallMessage = mergeBackRecallFrame.message;
319 const mergeBackRecallBody = isRecord(mergeBackRecallMessage)
320 ? mergeBackRecallMessage.message
321 : undefined;
322 const mergeBackRecallPrompt =
323 isRecord(mergeBackRecallBody) && typeof mergeBackRecallBody.content === "string"
324 ? mergeBackRecallBody.content
325 : "";
326 assert.notInclude(mergeBackRecallPrompt, THREAD_MERGE_BACK_SOURCE_MARKER);
327 assert.notInclude(mergeBackRecallPrompt, THREAD_MERGE_BACK_FORK_MARKER);
328 assert.equal(
329 successResultTexts(mergeBack)
330 .at(-1)
331 ?.replace(/\s*\|\s*/gu, "|"),
332 THREAD_MERGE_BACK_RECALL,
333 );
334
335 const siblingMergeBack = yield* readClaudeTranscriptFixture("thread_merge_back_siblings");
336 assert.equal(siblingMergeBack.metadata?.queryMode, "fork_session_merge_back_siblings");
337 const siblingMergeSessionIds = metadataStringArray(
338 siblingMergeBack,
339 "forkedNativeSessionIds",
340 );
341 assert.lengthOf(siblingMergeSessionIds, 2);
342 assert.notEqual(siblingMergeSessionIds[0], siblingMergeSessionIds[1]);
343 const siblingMergeRecallFrame = findEntryFrame(siblingMergeBack, "prompt.offer:6");
344 const siblingMergeRecallMessage = siblingMergeRecallFrame.message;
345 const siblingMergeRecallBody = isRecord(siblingMergeRecallMessage)
346 ? siblingMergeRecallMessage.message
347 : undefined;
348 const siblingMergeRecallPrompt =
349 isRecord(siblingMergeRecallBody) && typeof siblingMergeRecallBody.content === "string"
350 ? siblingMergeRecallBody.content
351 : "";
352 assert.notInclude(siblingMergeRecallPrompt, THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER);
353 assert.notInclude(siblingMergeRecallPrompt, THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER);
354 assert.notInclude(siblingMergeRecallPrompt, THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER);
355 assert.equal(
356 successResultTexts(siblingMergeBack)
357 .at(-1)
358 ?.replace(/\s*\|\s*/gu, "|"),
359 THREAD_MERGE_BACK_SIBLINGS_RECALL,
360 );
361
362 const forkLocalRollback = yield* readClaudeTranscriptFixture(
363 "thread_fork_native_fork_local_rollback",
364 );
365 assert.equal(forkLocalRollback.metadata?.queryMode, "fork_session_resume_at_fork_cursor");
366 const forkLocalRollbackCursor = metadataString(forkLocalRollback, "resumeSessionAt");
367 const forkLocalRollbackFrame = findEntryFrame(
368 forkLocalRollback,
369 "query.open:fork-resume-at-cursor",
370 );
371 const forkLocalRollbackOptions = forkLocalRollbackFrame.options;
372 if (!isRecord(forkLocalRollbackOptions)) {
373 throw new Error("Fork-local rollback resume query.open options must be an object.");
374 }
375 assert.equal(forkLocalRollbackOptions.resumeSessionAt, forkLocalRollbackCursor);
376 const forkLocalRollbackFinalText = successResultTexts(forkLocalRollback).at(-1) ?? "";
377 assert.include(forkLocalRollbackFinalText, "fork local source alpha");
378 assert.include(forkLocalRollbackFinalText, "fork local first");
379 assert.notInclude(forkLocalRollbackFinalText, "fork local second");
380 }),
381 );
382});