1import * as ScriptProgress from "@t3tools/provider-core/server/ScriptProgress";
2import * as OtelEnvironment from "@t3tools/shared/otelEnvironment";
3import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability";
4import * as NodeServices from "@effect/platform-node/NodeServices";
5import type { ProviderDriverKind, ProviderReplayTranscript } from "@t3tools/contracts";
6import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
10import * as PlatformError from "effect/PlatformError";
11import type * as SqlClient from "effect/sql/SqlClient";
12import type { MigrationError } from "effect/sql/Migrator";
13import type { SqlError } from "effect/sql/SqlError";
14
15import * as CheckpointStore from "../../checkpointing/CheckpointStore.ts";
16import * as ServerConfig from "../../config.ts";
17import * as SqlitePersistence from "../../persistence/Sqlite.ts";
18import * as ServerSettings from "../../serverSettings.ts";
19import * as ThreadManagementService from "../ThreadManagementService.ts";
20import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
21import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts";
22import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts";
23import * as VcsProcess from "../../vcs/VcsProcess.ts";
24import * as CheckpointCaptureService from "../CheckpointCaptureService.ts";
25import * as CheckpointService from "../CheckpointService.ts";
26import * as CheckpointRollbackService from "../CheckpointRollbackService.ts";
27import * as CommandPolicy from "../CommandPolicy.ts";
28import * as CommandReceiptStore from "../CommandReceiptStore.ts";
29import * as ContextHandoffService from "../ContextHandoffService.ts";
30import * as EffectOutbox from "../EffectOutbox.ts";
31import * as EffectWorker from "../EffectWorker.ts";
32import * as EventSink from "../EventSink.ts";
33import * as EventStore from "../EventStore.ts";
34import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
35import * as Orchestrator from "../Orchestrator.ts";
36import * as ProjectionStore from "../ProjectionStore.ts";
37import * as ProjectStore from "../ProjectStore.ts";
38import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts";
39import * as ProviderAuthService from "../../provider/ProviderAuthService.ts";
40import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
41import * as ProviderContinuationService from "../ProviderContinuationService.ts";
42import * as ProviderEventIngestor from "../ProviderEventIngestor.ts";
43import * as ProviderRuntimeRecoveryService from "../ProviderRuntimeRecoveryService.ts";
44import * as ProviderSessionManager from "../ProviderSessionManager.ts";
45import * as ProviderSwitchService from "../ProviderSwitchService.ts";
46import * as ProviderTurnControlService from "../ProviderTurnControlService.ts";
47import * as ProviderTurnStartService from "../ProviderTurnStartService.ts";
48import * as ProviderTurnStartServiceTestkit from "../ProviderTurnStartService.testkit.ts";
49import * as McpAppModelContext from "../../mcpApps/McpAppModelContext.ts";
50import * as RunExecutionService from "../RunExecutionService.ts";
51import * as RunFinalizationService from "../RunFinalizationService.ts";
52import * as ThreadTitleRegenerationService from "../ThreadTitleRegenerationService.ts";
53import * as RuntimePolicy from "../RuntimePolicy.ts";
54import * as TurnItemPositionStore from "../TurnItemPositionStore.ts";
55import * as RuntimeRequestService from "../RuntimeRequestService.ts";
56import * as ThreadCommandExecutor from "../ThreadCommandExecutor.ts";
57import * as ThreadForkService from "../ThreadForkService.ts";
58import {
59 runOrchestratorV2Scenario,
60 type OrchestratorV2ScenarioStepError,
61 type OrchestratorV2Scenario,
62 type OrchestratorV2ScenarioResult,
63} from "./OrchestratorScenario.ts";
64import {
65 makeProviderReplayGate,
66 type ProviderReplayGate,
67} from "@t3tools/provider-testing/replayGate";
68
69export function makeReplayServerConfig(
70 scenario: string,
71): Effect.Effect<
72 ServerConfig.ServerConfig["Service"],
73 PlatformError.PlatformError,
74 FileSystem.FileSystem | Path.Path
75> {
76 const safeScenario = scenario.replace(/[^a-z0-9_-]+/gi, "-");
77 return Effect.gen(function* () {
78 const fs = yield* FileSystem.FileSystem;
79 const path = yield* Path.Path;
80 const baseDir = yield* fs.makeTempDirectory({
81 prefix: `t3-orchestration-v2-replay-${safeScenario}-`,
82 });
83 const stateDir = path.join(baseDir, "userdata");
84 const logsDir = path.join(stateDir, "logs");
85 const providerLogsDir = path.join(logsDir, "provider");
86 const terminalLogsDir = path.join(logsDir, "terminals");
87 const attachmentsDir = path.join(stateDir, "attachments");
88 const environmentThemesDir = path.join(stateDir, "themes");
89 const worktreesDir = path.join(baseDir, "worktrees");
90 const providerStatusCacheDir = path.join(baseDir, "caches");
91
92 for (const directory of [
93 stateDir,
94 logsDir,
95 providerLogsDir,
96 terminalLogsDir,
97 attachmentsDir,
98 environmentThemesDir,
99 worktreesDir,
100 providerStatusCacheDir,
101 ]) {
102 yield* fs.makeDirectory(directory, { recursive: true });
103 }
104
105 return {
106 logLevel: "Error",
107 traceMinLevel: "Info",
108 traceTimingEnabled: true,
109 traceBatchWindowMs: 200,
110 traceMaxBytes: 10 * 1024 * 1024,
111 traceMaxFiles: 10,
112 otelEnvironment: OtelEnvironment.none,
113 otlpTracesUrl: undefined,
114 otlpMetricsUrl: undefined,
115 otlpLogsUrl: undefined,
116 otlpTracesExport: DEFAULT_SIGNAL_EXPORT,
117 otlpMetricsExport: DEFAULT_SIGNAL_EXPORT,
118 otlpLogsExport: DEFAULT_SIGNAL_EXPORT,
119 mode: "web",
120 port: 0,
121 host: undefined,
122 cwd: process.cwd(),
123 baseDir,
124 staticDir: undefined,
125 devUrl: undefined,
126 devAllowedOrigins: [],
127 noBrowser: false,
128 startupPresentation: "browser",
129 tailscaleServeEnabled: false,
130 tailscaleServePort: 443,
131 desktopBootstrapToken: undefined,
132 autoBootstrapProjectFromCwd: false,
133 logWebSocketEvents: false,
134 stateDir,
135 dbPath: path.join(stateDir, "state.sqlite"),
136 keybindingsConfigPath: path.join(stateDir, "keybindings.json"),
137 settingsPath: path.join(stateDir, "settings.json"),
138 providerStatusCacheDir,
139 worktreesDir,
140 attachmentsDir,
141 browserArtifactsDir: path.join(stateDir, "browser-artifacts"),
142 environmentThemesDir,
143 logsDir,
144 serverLogPath: path.join(logsDir, "server.log"),
145 serverTracePath: path.join(logsDir, "server.trace.ndjson"),
146 providerLogsDir,
147 providerEventLogPath: path.join(providerLogsDir, "events.log"),
148 terminalLogsDir,
149 environmentIdPath: path.join(stateDir, "environment-id"),
150 serverRuntimeStatePath: path.join(stateDir, "server-runtime.json"),
151 secretsDir: path.join(stateDir, "secrets"),
152 };
153 });
154}
155
156export interface OrchestratorV2ProviderReplayScenario<
157 Transcript extends ProviderReplayTranscript = ProviderReplayTranscript,
158> extends OrchestratorV2Scenario {
159 readonly transcript: Transcript;
160 readonly runtimePolicyOverride?: RuntimePolicy.RuntimePolicyV2Override;
161}
162
163export interface OrchestratorV2ProviderReplayHarness<
164 Transcript extends ProviderReplayTranscript = ProviderReplayTranscript,
165 Error = never,
166> {
167 readonly driver: ProviderDriverKind;
168 readonly decodeTranscript: (
169 transcript: ProviderReplayTranscript,
170 ) => Effect.Effect<Transcript, Error>;
171 readonly makeProviderAdapterRegistryLayer: (
172 transcript: Transcript,
173 options?: { readonly replayGate?: ProviderReplayGate },
174 ) => Layer.Layer<
175 ProviderAdapterRegistry.ProviderAdapterRegistryV2,
176 Error,
177 McpProviderSessions.McpProviderSessions | ScriptProgress.ScriptProgress
178 >;
179}
180
181export function runOrchestratorV2ProviderReplayScenario<
182 Transcript extends ProviderReplayTranscript,
183 Error,
184>(
185 scenario: OrchestratorV2ProviderReplayScenario<Transcript>,
186 harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>,
187 options: {
188 readonly databaseLayer?: Layer.Layer<
189 SqlClient.SqlClient,
190 MigrationError | PlatformError.PlatformError | SqlError
191 >;
192 readonly runEffectWorker?: boolean;
193 // Start continuation runs for provider wake turns, as the live runtime does.
194 // Off by default: most fixtures record no wake turn.
195 readonly runContinuationWorker?: boolean;
196 // Reconcile a previous runtime's state before the effect worker starts,
197 // as server startup does after a crash or restart.
198 readonly recoverOnStartup?: boolean;
199 readonly continueThreadsAfterServerUpdate?: boolean;
200 } = {},
201): Effect.Effect<
202 OrchestratorV2ScenarioResult,
203 | Orchestrator.OrchestratorV2Error
204 | OrchestratorV2ScenarioStepError
205 | Error
206 | MigrationError
207 | PlatformError.PlatformError
208 | SqlError,
209 never
210> {
211 const replayGate = makeProviderReplayGate(
212 scenario.steps?.flatMap((step) =>
213 step.type === "release_replay_gate" || step.type === "release_replay_gate_after_waiting"
214 ? [step.label]
215 : [],
216 ) ?? [],
217 );
218 const layer = layerProviderReplay(scenario, harness, {
219 ...options,
220 replayGate,
221 });
222
223 return runOrchestratorV2Scenario(scenario, { replayGate }).pipe(Effect.provide(layer));
224}
225
226export function layerProviderReplay<Transcript extends ProviderReplayTranscript, Error>(
227 scenario: OrchestratorV2ProviderReplayScenario<Transcript>,
228 harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>,
229 options: {
230 readonly databaseLayer?: Layer.Layer<
231 SqlClient.SqlClient,
232 MigrationError | PlatformError.PlatformError | SqlError
233 >;
234 readonly runEffectWorker?: boolean;
235 // Start continuation runs for provider wake turns, as the live runtime does.
236 // Off by default: most fixtures record no wake turn.
237 readonly runContinuationWorker?: boolean;
238 // Reconcile a previous runtime's state before the effect worker starts,
239 // as server startup does after a crash or restart.
240 readonly recoverOnStartup?: boolean;
241 readonly continueThreadsAfterServerUpdate?: boolean;
242 readonly replayGate?: ProviderReplayGate;
243 } = {},
244): Layer.Layer<
245 | Orchestrator.OrchestratorV2
246 | EffectWorker.OrchestrationEffectWorkerV2
247 | EventSink.EventSinkV2
248 | ProviderSessionManager.ProviderSessionManagerV2,
249 Error | MigrationError | PlatformError.PlatformError | SqlError
250> {
251 const layerRegistry = harness.makeProviderAdapterRegistryLayer(
252 scenario.transcript,
253 options.replayGate === undefined ? {} : { replayGate: options.replayGate },
254 );
255 return layerWithRegistry(scenario, layerRegistry, options);
256}
257
258export function layerWithRegistry<Error>(
259 scenario: Pick<OrchestratorV2ProviderReplayScenario, "name" | "runtimePolicyOverride">,
260 registryLayer: Layer.Layer<
261 ProviderAdapterRegistry.ProviderAdapterRegistryV2,
262 Error,
263 McpProviderSessions.McpProviderSessions | ScriptProgress.ScriptProgress
264 >,
265 options: {
266 readonly databaseLayer?: Layer.Layer<
267 SqlClient.SqlClient,
268 MigrationError | PlatformError.PlatformError | SqlError
269 >;
270 readonly runEffectWorker?: boolean;
271 // Start continuation runs for provider wake turns, as the live runtime does.
272 // Off by default: most fixtures record no wake turn.
273 readonly runContinuationWorker?: boolean;
274 // Reconcile a previous runtime's state before the effect worker starts,
275 // as server startup does after a crash or restart.
276 readonly recoverOnStartup?: boolean;
277 readonly continueThreadsAfterServerUpdate?: boolean;
278 } = {},
279): Layer.Layer<
280 | Orchestrator.OrchestratorV2
281 | EffectWorker.OrchestrationEffectWorkerV2
282 | EventSink.EventSinkV2
283 | ProviderSessionManager.ProviderSessionManagerV2,
284 Error | MigrationError | PlatformError.PlatformError | SqlError
285> {
286 const layerServerConfig = Layer.effect(
287 ServerConfig.ServerConfig,
288 makeReplayServerConfig(scenario.name).pipe(Effect.orDie),
289 ).pipe(Layer.provide(NodeServices.layer));
290 const layerRuntime =
291 scenario.runtimePolicyOverride === undefined
292 ? RuntimePolicy.layer
293 : RuntimePolicy.layerWithOverride(scenario.runtimePolicyOverride).pipe(
294 Layer.provide(RuntimePolicy.layer),
295 );
296 const layerDatabase = options.databaseLayer ?? SqlitePersistence.layerMemory;
297 // One queue shared by the adapters, the orchestrator, and the worker, like
298 // runtimeLayer.ts; layer memoization keeps it a single instance.
299 const layerContinuationRequests =
300 options.runContinuationWorker === true ? ProviderContinuationRequests.layer : Layer.empty;
301 const layerProvidedRegistry = registryLayer.pipe(
302 Layer.provide(
303 Layer.mergeAll(
304 layerContinuationRequests,
305 McpProviderSessions.layer,
306 ScriptProgress.layerUnavailable,
307 ),
308 ),
309 );
310 const layerServerSettings = ServerSettings.layerTest({
311 responseStreamingMode: "turn",
312 ...(options.continueThreadsAfterServerUpdate === undefined
313 ? {}
314 : { continueThreadsAfterServerUpdate: options.continueThreadsAfterServerUpdate }),
315 }).pipe(Layer.orDie);
316 const layerStores = Layer.mergeAll(
317 EventStore.layer,
318 ProjectionStore.layer,
319 ProjectStore.layer,
320 CommandReceiptStore.layer,
321 EffectOutbox.layer,
322 TurnItemPositionStore.layer,
323 ).pipe(Layer.provide(layerDatabase));
324 const layerEventSinkProvided = EventSink.layerFromStores.pipe(
325 Layer.provide(Layer.mergeAll(layerStores, layerDatabase)),
326 );
327 const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
328 Layer.provide(layerDatabase),
329 );
330 const layerProviderEventIngestorProvided = ProviderEventIngestor.layer.pipe(
331 Layer.provide(
332 Layer.mergeAll(
333 layerStores,
334 layerEventSinkProvided,
335 IdAllocator.layer,
336 ThreadCommandExecutor.layer,
337 ),
338 ),
339 );
340 const layerVcsDriverRegistry = VcsDriverRegistry.layer.pipe(
341 Layer.provide(VcsProcess.layer),
342 Layer.provide(layerServerConfig),
343 Layer.provide(NodeServices.layer),
344 );
345 const layerCheckpointStore = CheckpointStore.layer.pipe(
346 Layer.provide(layerVcsDriverRegistry),
347 Layer.provide(NodeServices.layer),
348 );
349 const layerCheckpointServiceProvided = CheckpointService.layer.pipe(
350 Layer.provide(Layer.mergeAll(layerCheckpointStore, IdAllocator.layer)),
351 );
352 const layerContextHandoffServiceProvided = ContextHandoffService.layer.pipe(
353 Layer.provide(IdAllocator.layer),
354 );
355 const layerPersistence = Layer.mergeAll(
356 layerStores,
357 layerEventSinkProvided,
358 layerCommandReceiptStoreProvided,
359 IdAllocator.layer,
360 layerProviderEventIngestorProvided,
361 );
362 const layerProviderSessionManagerProvided = ProviderSessionManager.layerWithOptions({
363 configureMcp: false,
364 }).pipe(
365 Layer.provide(
366 Layer.mergeAll(
367 layerProvidedRegistry,
368 layerEventSinkProvided,
369 IdAllocator.layer,
370 McpProviderSessions.layer,
371 ScriptProgress.layerUnavailable,
372 McpSessionRegistryTestkit.layer,
373 layerProviderEventIngestorProvided,
374 layerStores,
375 ),
376 ),
377 );
378 const layerProviderSwitchServiceProvided = ProviderSwitchService.layer.pipe(
379 Layer.provide(layerProvidedRegistry),
380 );
381 const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe(
382 Layer.provide(
383 Layer.mergeAll(
384 McpAppModelContext.layerEmpty,
385 layerCheckpointServiceProvided,
386 layerEventSinkProvided,
387 IdAllocator.layer,
388 layerProviderEventIngestorProvided,
389 layerServerSettings,
390 ),
391 ),
392 );
393 const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pipe(
394 Layer.provide(
395 Layer.mergeAll(
396 layerContextHandoffServiceProvided,
397 layerEventSinkProvided,
398 IdAllocator.layer,
399 layerStores,
400 layerProviderSessionManagerProvided,
401 Layer.mock(ProviderAuthService.ProviderAuthService)({
402 tryHandlePromptCommand: () => Effect.succeed(false),
403 }),
404 layerRunExecutionServiceProvided,
405 layerRuntime,
406 ),
407 ),
408 );
409 const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe(
410 Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
411 );
412 const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe(
413 Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
414 );
415 const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe(
416 Layer.provide(
417 Layer.mergeAll(
418 layerCheckpointServiceProvided,
419 layerEventSinkProvided,
420 IdAllocator.layer,
421 layerStores,
422 layerProviderSessionManagerProvided,
423 ThreadCommandExecutor.layer,
424 layerRuntime,
425 ),
426 ),
427 );
428 const layerCheckpointCaptureServiceProvided = CheckpointCaptureService.layer.pipe(
429 Layer.provide(
430 Layer.mergeAll(
431 layerCheckpointServiceProvided,
432 layerEventSinkProvided,
433 IdAllocator.layer,
434 layerStores,
435 ),
436 ),
437 );
438 const layerRunFinalizationServiceProvided = RunFinalizationService.layer.pipe(
439 Layer.provide(Layer.merge(layerCheckpointCaptureServiceProvided, layerStores)),
440 );
441 const layerThreadTitleRegenerationTest = Layer.succeed(
442 ThreadTitleRegenerationService.ThreadTitleRegenerationService,
443 ThreadTitleRegenerationService.ThreadTitleRegenerationService.of({
444 execute: () => Effect.void,
445 }),
446 );
447 const layerOrchestratorProvided = Orchestrator.layer.pipe(
448 Layer.provide(
449 Layer.mergeAll(
450 layerCheckpointServiceProvided,
451 CommandPolicy.layer,
452 layerContextHandoffServiceProvided,
453 layerPersistence,
454 layerProvidedRegistry,
455 layerContinuationRequests,
456 layerRuntime,
457 layerProviderSessionManagerProvided,
458 layerProviderSwitchServiceProvided,
459 layerRunExecutionServiceProvided,
460 layerServerSettings,
461 ThreadForkService.layer,
462 ),
463 ),
464 );
465 const layerThreadManagementProvided = Layer.unwrap(
466 Effect.gen(function* () {
467 const orchestrator = yield* Orchestrator.OrchestratorV2;
468 return Layer.mock(ThreadManagementService.ThreadManagementService)({
469 dispatch: orchestrator.dispatch,
470 getThreadRecords: orchestrator.getThreadRecords,
471 getThreadProjection: orchestrator.getThreadProjection,
472 recoverDelegatedTask: orchestrator.recoverDelegatedTask,
473 delegatedTaskResultPending: orchestrator.delegatedTaskResultPending,
474 });
475 }),
476 ).pipe(Layer.provide(layerOrchestratorProvided));
477 const layerContinuationWorkerProvided =
478 options.runContinuationWorker === true
479 ? ProviderContinuationService.layer.pipe(
480 Layer.provide(
481 Layer.mergeAll(
482 layerContinuationRequests,
483 layerThreadManagementProvided,
484 IdAllocator.layer,
485 ),
486 ),
487 )
488 : Layer.empty;
489 const layerEffectExecutorProvided = EffectWorker.layerExecutor.pipe(
490 Layer.provide(
491 Layer.mergeAll(
492 layerRunFinalizationServiceProvided,
493 layerCheckpointRollbackServiceProvided,
494 layerProviderSessionManagerProvided,
495 layerProviderTurnControlServiceProvided,
496 layerProviderTurnStartServiceProvided,
497 layerRuntimeRequestServiceProvided,
498 layerThreadTitleRegenerationTest,
499 layerServerSettings,
500 layerThreadManagementProvided,
501 ),
502 ),
503 );
504 const layerEffectWorkerProvided = EffectWorker.layer.pipe(
505 Layer.provide(Layer.merge(layerStores, layerEffectExecutorProvided)),
506 );
507 const layerReplayRuntime = Layer.mergeAll(
508 layerOrchestratorProvided,
509 layerProviderSessionManagerProvided,
510 layerEffectWorkerProvided,
511 layerEventSinkProvided,
512 layerContinuationWorkerProvided,
513 ).pipe(Layer.provide(ProviderTurnStartServiceTestkit.layer), Layer.provide(NodeServices.layer));
514
515 // Build the daemon from the exact worker instance exposed alongside the
516 // orchestrator. Keeping this acquisition in the replay layer makes the
517 // outbox lifecycle explicit and prevents test-only command-side draining.
518 if (options.runEffectWorker === false) {
519 return layerReplayRuntime;
520 }
521 // Built before the runtime it shares stores with, so recovery commits before
522 // the effect worker claims anything, as in serverRuntimeStartup.
523 const layerStartupRecovery: Layer.Layer<
524 never,
525 MigrationError | PlatformError.PlatformError | SqlError
526 > =
527 options.recoverOnStartup === true
528 ? Layer.effectDiscard(
529 ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService.use(
530 (recovery) => recovery.recover,
531 ).pipe(Effect.orDie),
532 ).pipe(
533 Layer.provide(ProviderRuntimeRecoveryService.layer),
534 Layer.provide(
535 Layer.mergeAll(
536 layerStores,
537 layerEventSinkProvided,
538 IdAllocator.layer,
539 layerServerSettings,
540 ),
541 ),
542 )
543 : Layer.empty;
544 return Layer.effect(
545 Orchestrator.OrchestratorV2,
546 Effect.gen(function* () {
547 const orchestrator = yield* Orchestrator.OrchestratorV2;
548 // As in serverRuntimeStartup: after runtime recovery, before the worker.
549 yield* orchestrator.recoverDelegatedTasks;
550 yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
551 return orchestrator;
552 }),
553 ).pipe(Layer.provideMerge(layerReplayRuntime), Layer.provide(layerStartupRecovery));
554}