1import { assert } from "@effect/vitest";
2import type { ProviderReplayTranscript } from "@t3tools/contracts";
3
4import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
5import {
6 assertAllRuntimeRequestsResolved,
7 assertAssistantTextIncludes,
8 assertBaseProjection,
9 assertExecutionNodeKinds,
10 assertRunOrdinals,
11 assertRuntimeRequestCounts,
12 assertRuntimeRequestKinds,
13 assertSemanticProjectionIntegrity,
14 assertTurnItemTypes,
15 assertUserMessagesInclude,
16 assertVisibleTurnItemsMirrorLocalTurnItems,
17 PLAN_QUESTIONS_PROMPT,
18 projectionFor,
19} from "../shared.ts";
20
21export function assertPlanQuestionsOutputBase(
22 result: OrchestratorV2ScenarioResult,
23 transcript: ProviderReplayTranscript,
24) {
25 assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] });
26
27 const projection = projectionFor(result, transcript.scenario);
28 assertSemanticProjectionIntegrity(projection);
29 assertVisibleTurnItemsMirrorLocalTurnItems(projection);
30 assertRunOrdinals(projection, [1]);
31 assertExecutionNodeKinds(projection, ["root_turn", "user_input_request", "assistant_message"]);
32 assertRuntimeRequestCounts(projection, { total: 1, resolved: 1 });
33 assertRuntimeRequestKinds(projection, ["user_input"]);
34 assertAllRuntimeRequestsResolved(projection);
35 assertTurnItemTypes(projection, ["user_message", "user_input_request", "assistant_message"]);
36 assertUserMessagesInclude(projection, [PLAN_QUESTIONS_PROMPT]);
37 assertAssistantTextIncludes(projection, "plan questions fixture complete");
38}
39
40function assertPlanQuestionId(
41 result: OrchestratorV2ScenarioResult,
42 transcript: ProviderReplayTranscript,
43 questionId: string,
44) {
45 const projection = projectionFor(result, transcript.scenario);
46 const requestItem = projection.turnItems.find((item) => item.type === "user_input_request");
47 assert.isDefined(requestItem);
48 assert.equal(requestItem?.questions[0]?.id, questionId);
49}
50
51/** Grok and the ACP registry replay the same recorded question id. */
52export function assertPlanQuestionsOutput(
53 result: OrchestratorV2ScenarioResult,
54 transcript: ProviderReplayTranscript,
55) {
56 assertPlanQuestionsOutputBase(result, transcript);
57 assertPlanQuestionId(result, transcript, "schema_vs_ui_flexibility");
58}
59
60/** Codex names the question itself; this is the id the recorded model chose. */
61export function assertCodexPlanQuestionsOutput(
62 result: OrchestratorV2ScenarioResult,
63 transcript: ProviderReplayTranscript,
64) {
65 assertPlanQuestionsOutputBase(result, transcript);
66 assertPlanQuestionId(result, transcript, "schema_preference");
67}