| author |  Theo Browne <me@t3.gg> 2026-09-04 07:54:23-07:00 |
| committer |  GitHub <noreply@github.com> 2026-09-04 07:54:23-07:00 |
| log | 1587f248dd81ed45e214d476451ebf16dbfadb1a |
| tree | 0ed79d327287efa511f35963773e8515be16764b |
| parent | dab5f6e6e02e78675655e69503aa89654e5b8050 |
| signature | Signed by PGP key B5690EEEBB952194 |
feat(server): measure provider turn token usage (#9132)
14 files changed, 3865 insertions(+), 20 deletions(-)
docs
- README.md
internals
user
packages
contracts
apps/server/src/provider/Layers/ClaudeAdapter.test.ts+250-6
| ... | @@ -24,6 +24,7 @@ import { | ... | @@ -24,6 +24,7 @@ import { |
| 24 | import { createModelSelection } from "@t3tools/shared/model"; | 24 | import { createModelSelection } from "@t3tools/shared/model"; |
| 25 | import { assert, describe, it } from "@effect/vitest"; | 25 | import { assert, describe, it } from "@effect/vitest"; |
| 26 | import * as Context from "effect/Context"; | 26 | import * as Context from "effect/Context"; |
| 27 | import * as Deferred from "effect/Deferred"; | ||
| 27 | import * as Effect from "effect/Effect"; | 28 | import * as Effect from "effect/Effect"; |
| 28 | import * as Fiber from "effect/Fiber"; | 29 | import * as Fiber from "effect/Fiber"; |
| 29 | import * as Layer from "effect/Layer"; | 30 | import * as Layer from "effect/Layer"; |
| ... | @@ -1899,7 +1900,8 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1899,7 +1900,8 @@ describe("ClaudeAdapterLive", () => { |
| 1899 | return Effect.gen(function* () { | 1900 | return Effect.gen(function* () { |
| 1900 | const adapter = yield* ClaudeAdapter; | 1901 | const adapter = yield* ClaudeAdapter; |
| 1901 | 1902 | ||
| 1902 | const runtimeEventsFiber = yield* Stream.take(adapter.streamEvents, 8).pipe( | 1903 | const runtimeEventsFiber = yield* adapter.streamEvents.pipe( |
| 1904 | Stream.takeUntil((event) => event.type === "turn.completed"), | ||
| 1903 | Stream.runCollect, | 1905 | Stream.runCollect, |
| 1904 | Effect.forkChild, | 1906 | Effect.forkChild, |
| 1905 | ); | 1907 | ); |
| ... | @@ -1937,6 +1939,17 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1937,6 +1939,17 @@ describe("ClaudeAdapterLive", () => { |
| 1937 | }, | 1939 | }, |
| 1938 | } as unknown as SDKMessage); | 1940 | } as unknown as SDKMessage); |
| 1939 | 1941 | ||
| 1942 | harness.query.emit({ | ||
| 1943 | type: "system", | ||
| 1944 | subtype: "task_started", | ||
| 1945 | task_id: "task-agent-1", | ||
| 1946 | description: "Review the database layer", | ||
| 1947 | task_type: "local_agent", | ||
| 1948 | tool_use_id: "tool-task-1", | ||
| 1949 | uuid: "task-agent-1-uuid", | ||
| 1950 | session_id: "sdk-session-task", | ||
| 1951 | } as unknown as SDKMessage); | ||
| 1952 | |||
| 1940 | harness.query.emit({ | 1953 | harness.query.emit({ |
| 1941 | type: "assistant", | 1954 | type: "assistant", |
| 1942 | session_id: "sdk-session-task", | 1955 | session_id: "sdk-session-task", |
| ... | @@ -1953,6 +1966,12 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1953,6 +1966,12 @@ describe("ClaudeAdapterLive", () => { |
| 1953 | subtype: "success", | 1966 | subtype: "success", |
| 1954 | is_error: false, | 1967 | is_error: false, |
| 1955 | errors: [], | 1968 | errors: [], |
| 1969 | usage: { | ||
| 1970 | input_tokens: 100, | ||
| 1971 | cache_read_input_tokens: 40, | ||
| 1972 | cache_creation_input_tokens: 10, | ||
| 1973 | output_tokens: 20, | ||
| 1974 | }, | ||
| 1956 | session_id: "sdk-session-task", | 1975 | session_id: "sdk-session-task", |
| 1957 | uuid: "result-task-1", | 1976 | uuid: "result-task-1", |
| 1958 | } as unknown as SDKMessage); | 1977 | } as unknown as SDKMessage); |
| ... | @@ -1964,6 +1983,11 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1964,6 +1983,11 @@ describe("ClaudeAdapterLive", () => { |
| 1964 | assert.equal(toolStarted.payload.itemType, "collab_agent_tool_call"); | 1983 | assert.equal(toolStarted.payload.itemType, "collab_agent_tool_call"); |
| 1965 | assert.equal(toolStarted.payload.title, "Subagent task"); | 1984 | assert.equal(toolStarted.payload.title, "Subagent task"); |
| 1966 | } | 1985 | } |
| 1986 | const completed = runtimeEvents.find((event) => event.type === "turn.completed"); | ||
| 1987 | assert.equal(completed?.type, "turn.completed"); | ||
| 1988 | if (completed?.type === "turn.completed") { | ||
| 1989 | assert.equal(completed.payload.tokenUsage?.hasSubagents, true); | ||
| 1990 | } | ||
| 1967 | }).pipe( | 1991 | }).pipe( |
| 1968 | Effect.provideService(Random.Random, makeDeterministicRandomService()), | 1992 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 1969 | Effect.provide(harness.layer), | 1993 | Effect.provide(harness.layer), |
| ... | @@ -1975,7 +1999,8 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1975,7 +1999,8 @@ describe("ClaudeAdapterLive", () => { |
| 1975 | return Effect.gen(function* () { | 1999 | return Effect.gen(function* () { |
| 1976 | const adapter = yield* ClaudeAdapter; | 2000 | const adapter = yield* ClaudeAdapter; |
| 1977 | 2001 | ||
| 1978 | const runtimeEventsFiber = yield* Stream.take(adapter.streamEvents, 6).pipe( | 2002 | const runtimeEventsFiber = yield* adapter.streamEvents.pipe( |
| 2003 | Stream.takeUntil((event) => event.type === "turn.completed"), | ||
| 1979 | Stream.runCollect, | 2004 | Stream.runCollect, |
| 1980 | Effect.forkChild, | 2005 | Effect.forkChild, |
| 1981 | ); | 2006 | ); |
| ... | @@ -1998,6 +2023,12 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -1998,6 +2023,12 @@ describe("ClaudeAdapterLive", () => { |
| 1998 | is_error: false, | 2023 | is_error: false, |
| 1999 | errors: ["Error: Request was aborted."], | 2024 | errors: ["Error: Request was aborted."], |
| 2000 | stop_reason: "tool_use", | 2025 | stop_reason: "tool_use", |
| 2026 | usage: { | ||
| 2027 | input_tokens: 12, | ||
| 2028 | cache_read_input_tokens: 3, | ||
| 2029 | cache_creation_input_tokens: 1, | ||
| 2030 | output_tokens: 4, | ||
| 2031 | }, | ||
| 2001 | session_id: "sdk-session-abort", | 2032 | session_id: "sdk-session-abort", |
| 2002 | uuid: "result-abort", | 2033 | uuid: "result-abort", |
| 2003 | } as unknown as SDKMessage); | 2034 | } as unknown as SDKMessage); |
| ... | @@ -2011,6 +2042,7 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -2011,6 +2042,7 @@ describe("ClaudeAdapterLive", () => { |
| 2011 | "session.state.changed", | 2042 | "session.state.changed", |
| 2012 | "turn.started", | 2043 | "turn.started", |
| 2013 | "thread.started", | 2044 | "thread.started", |
| 2045 | "thread.token-usage.updated", | ||
| 2014 | "turn.completed", | 2046 | "turn.completed", |
| 2015 | ], | 2047 | ], |
| 2016 | ); | 2048 | ); |
| ... | @@ -2022,6 +2054,15 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -2022,6 +2054,15 @@ describe("ClaudeAdapterLive", () => { |
| 2022 | assert.equal(turnCompleted.payload.state, "interrupted"); | 2054 | assert.equal(turnCompleted.payload.state, "interrupted"); |
| 2023 | assert.equal(turnCompleted.payload.errorMessage, "Error: Request was aborted."); | 2055 | assert.equal(turnCompleted.payload.errorMessage, "Error: Request was aborted."); |
| 2024 | assert.equal(turnCompleted.payload.stopReason, "tool_use"); | 2056 | assert.equal(turnCompleted.payload.stopReason, "tool_use"); |
| 2057 | assert.deepEqual(turnCompleted.payload.tokenUsage, { | ||
| 2058 | usageStatus: "partial", | ||
| 2059 | usageScope: "main_agent", | ||
| 2060 | inputTokens: 16, | ||
| 2061 | cachedInputTokens: 3, | ||
| 2062 | cacheCreationTokens: 1, | ||
| 2063 | outputTokens: 4, | ||
| 2064 | hasSubagents: false, | ||
| 2065 | }); | ||
| 2025 | } | 2066 | } |
| 2026 | }).pipe( | 2067 | }).pipe( |
| 2027 | Effect.provideService(Random.Random, makeDeterministicRandomService()), | 2068 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| ... | @@ -2509,7 +2550,10 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -2509,7 +2550,10 @@ describe("ClaudeAdapterLive", () => { |
| 2509 | session_id: "sdk-session-result-usage", | 2550 | session_id: "sdk-session-result-usage", |
| 2510 | usage: { | 2551 | usage: { |
| 2511 | input_tokens: 400, | 2552 | input_tokens: 400, |
| 2553 | cache_read_input_tokens: 90, | ||
| 2554 | cache_creation_input_tokens: 10, | ||
| 2512 | output_tokens: 50, | 2555 | output_tokens: 50, |
| 2556 | output_tokens_details: { thinking_tokens: 30 }, | ||
| 2513 | }, | 2557 | }, |
| 2514 | modelUsage: { | 2558 | modelUsage: { |
| 2515 | [SYNTHETIC_CLAUDE_CAPABLE_MODEL]: { | 2559 | [SYNTHETIC_CLAUDE_CAPABLE_MODEL]: { |
| ... | @@ -2527,16 +2571,211 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -2527,16 +2571,211 @@ describe("ClaudeAdapterLive", () => { |
| 2527 | assert.deepEqual(usageEvent.payload.usage, { | 2571 | assert.deepEqual(usageEvent.payload.usage, { |
| 2528 | usedTokens: 200, | 2572 | usedTokens: 200, |
| 2529 | lastUsedTokens: 200, | 2573 | lastUsedTokens: 200, |
| 2530 | totalProcessedTokens: 450, | 2574 | totalProcessedTokens: 550, |
| 2531 | inputTokens: 180, | 2575 | inputTokens: 180, |
| 2532 | outputTokens: 20, | 2576 | outputTokens: 20, |
| 2533 | maxTokens: 200000, | 2577 | maxTokens: 200000, |
| 2534 | }); | 2578 | }); |
| 2535 | } | 2579 | } |
| 2536 | assert.equal( | 2580 | const completed = runtimeEvents.find((event) => event.type === "turn.completed"); |
| 2537 | runtimeEvents.find((event) => event.type === "turn.completed")?.type, | 2581 | assert.equal(completed?.type, "turn.completed"); |
| 2538 | "turn.completed", | 2582 | if (completed?.type === "turn.completed") { |
| 2583 | assert.deepEqual(completed.payload.tokenUsage, { | ||
| 2584 | usageStatus: "complete", | ||
| 2585 | usageScope: "main_agent", | ||
| 2586 | inputTokens: 500, | ||
| 2587 | cachedInputTokens: 90, | ||
| 2588 | cacheCreationTokens: 10, | ||
| 2589 | reasoningTokens: 30, | ||
| 2590 | outputTokens: 50, | ||
| 2591 | hasSubagents: false, | ||
| 2592 | }); | ||
| 2593 | } | ||
| 2594 | }).pipe( | ||
| 2595 | Effect.provideService(Random.Random, makeDeterministicRandomService()), | ||
| 2596 | Effect.provide(harness.layer), | ||
| 2597 | ); | ||
| 2598 | }); | ||
| 2599 | |||
| 2600 | it.effect("treats omitted Claude cache counters as zero contributions", () => { | ||
| 2601 | const harness = makeHarness(); | ||
| 2602 | return Effect.gen(function* () { | ||
| 2603 | const adapter = yield* ClaudeAdapter; | ||
| 2604 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 2605 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 2606 | Stream.runHead, | ||
| 2607 | Effect.forkChild, | ||
| 2539 | ); | 2608 | ); |
| 2609 | yield* adapter.startSession({ | ||
| 2610 | threadId: THREAD_ID, | ||
| 2611 | provider: ProviderDriverKind.make("claudeAgent"), | ||
| 2612 | runtimeMode: "full-access", | ||
| 2613 | }); | ||
| 2614 | yield* adapter.sendTurn({ | ||
| 2615 | threadId: THREAD_ID, | ||
| 2616 | input: "uncached turn", | ||
| 2617 | attachments: [], | ||
| 2618 | }); | ||
| 2619 | harness.query.emit({ | ||
| 2620 | type: "result", | ||
| 2621 | subtype: "success", | ||
| 2622 | is_error: false, | ||
| 2623 | duration_ms: 100, | ||
| 2624 | duration_api_ms: 90, | ||
| 2625 | num_turns: 1, | ||
| 2626 | result: "done", | ||
| 2627 | stop_reason: "end_turn", | ||
| 2628 | session_id: "sdk-session-uncached-usage", | ||
| 2629 | usage: { | ||
| 2630 | input_tokens: 42, | ||
| 2631 | output_tokens: 9, | ||
| 2632 | }, | ||
| 2633 | } as unknown as SDKMessage); | ||
| 2634 | |||
| 2635 | const completed = yield* Fiber.join(completedFiber); | ||
| 2636 | assert.equal(completed._tag, "Some"); | ||
| 2637 | if (completed._tag === "Some" && completed.value.type === "turn.completed") { | ||
| 2638 | assert.deepEqual(completed.value.payload.tokenUsage, { | ||
| 2639 | usageStatus: "complete", | ||
| 2640 | usageScope: "main_agent", | ||
| 2641 | inputTokens: 42, | ||
| 2642 | outputTokens: 9, | ||
| 2643 | hasSubagents: false, | ||
| 2644 | }); | ||
| 2645 | } | ||
| 2646 | }).pipe( | ||
| 2647 | Effect.provideService(Random.Random, makeDeterministicRandomService()), | ||
| 2648 | Effect.provide(harness.layer), | ||
| 2649 | ); | ||
| 2650 | }); | ||
| 2651 | |||
| 2652 | it.effect("uses per-turn result usage across consecutive Claude turns", () => { | ||
| 2653 | const harness = makeHarness(); | ||
| 2654 | return Effect.gen(function* () { | ||
| 2655 | const adapter = yield* ClaudeAdapter; | ||
| 2656 | const firstTurnSettled = yield* Deferred.make<void>(); | ||
| 2657 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 2658 | Stream.tap((event) => | ||
| 2659 | event.type === "session.state.changed" && event.payload.reason === "api_retry:1/2" | ||
| 2660 | ? Deferred.succeed(firstTurnSettled, undefined).pipe(Effect.asVoid) | ||
| 2661 | : Effect.void, | ||
| 2662 | ), | ||
| 2663 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 2664 | Stream.take(2), | ||
| 2665 | Stream.runCollect, | ||
| 2666 | Effect.forkChild, | ||
| 2667 | ); | ||
| 2668 | |||
| 2669 | yield* adapter.startSession({ | ||
| 2670 | threadId: THREAD_ID, | ||
| 2671 | provider: ProviderDriverKind.make("claudeAgent"), | ||
| 2672 | runtimeMode: "full-access", | ||
| 2673 | }); | ||
| 2674 | yield* adapter.sendTurn({ | ||
| 2675 | threadId: THREAD_ID, | ||
| 2676 | input: "first turn", | ||
| 2677 | attachments: [], | ||
| 2678 | }); | ||
| 2679 | harness.query.emit({ | ||
| 2680 | type: "result", | ||
| 2681 | subtype: "success", | ||
| 2682 | is_error: false, | ||
| 2683 | duration_ms: 100, | ||
| 2684 | duration_api_ms: 90, | ||
| 2685 | num_turns: 1, | ||
| 2686 | result: "first", | ||
| 2687 | stop_reason: "end_turn", | ||
| 2688 | session_id: "sdk-session-consecutive-usage", | ||
| 2689 | usage: { | ||
| 2690 | input_tokens: 100, | ||
| 2691 | cache_read_input_tokens: 20, | ||
| 2692 | cache_creation_input_tokens: 5, | ||
| 2693 | output_tokens: 10, | ||
| 2694 | }, | ||
| 2695 | modelUsage: { | ||
| 2696 | "claude-opus-4-6": { | ||
| 2697 | inputTokens: 10_000, | ||
| 2698 | outputTokens: 1_000, | ||
| 2699 | cacheReadInputTokens: 2_000, | ||
| 2700 | cacheCreationInputTokens: 500, | ||
| 2701 | webSearchRequests: 0, | ||
| 2702 | costUSD: 1, | ||
| 2703 | contextWindow: 200_000, | ||
| 2704 | maxOutputTokens: 64_000, | ||
| 2705 | }, | ||
| 2706 | }, | ||
| 2707 | } as unknown as SDKMessage); | ||
| 2708 | harness.query.emit({ | ||
| 2709 | type: "system", | ||
| 2710 | subtype: "api_retry", | ||
| 2711 | attempt: 1, | ||
| 2712 | max_retries: 2, | ||
| 2713 | retry_delay_ms: 1, | ||
| 2714 | error_status: 502, | ||
| 2715 | error: { type: "api_error" }, | ||
| 2716 | session_id: "sdk-session-consecutive-usage", | ||
| 2717 | uuid: "consecutive-usage-barrier", | ||
| 2718 | } as unknown as SDKMessage); | ||
| 2719 | yield* Deferred.await(firstTurnSettled); | ||
| 2720 | |||
| 2721 | yield* adapter.sendTurn({ | ||
| 2722 | threadId: THREAD_ID, | ||
| 2723 | input: "second turn", | ||
| 2724 | attachments: [], | ||
| 2725 | }); | ||
| 2726 | harness.query.emit({ | ||
| 2727 | type: "result", | ||
| 2728 | subtype: "success", | ||
| 2729 | is_error: false, | ||
| 2730 | duration_ms: 80, | ||
| 2731 | duration_api_ms: 70, | ||
| 2732 | num_turns: 2, | ||
| 2733 | result: "second", | ||
| 2734 | stop_reason: "end_turn", | ||
| 2735 | session_id: "sdk-session-consecutive-usage", | ||
| 2736 | usage: { | ||
| 2737 | input_tokens: 30, | ||
| 2738 | cache_read_input_tokens: 2, | ||
| 2739 | cache_creation_input_tokens: 3, | ||
| 2740 | output_tokens: 7, | ||
| 2741 | }, | ||
| 2742 | modelUsage: { | ||
| 2743 | "claude-opus-4-6": { | ||
| 2744 | inputTokens: 20_000, | ||
| 2745 | outputTokens: 2_000, | ||
| 2746 | cacheReadInputTokens: 4_000, | ||
| 2747 | cacheCreationInputTokens: 1_000, | ||
| 2748 | webSearchRequests: 0, | ||
| 2749 | costUSD: 2, | ||
| 2750 | contextWindow: 200_000, | ||
| 2751 | maxOutputTokens: 64_000, | ||
| 2752 | }, | ||
| 2753 | }, | ||
| 2754 | } as unknown as SDKMessage); | ||
| 2755 | |||
| 2756 | const completed = Array.from(yield* Fiber.join(completedFiber)); | ||
| 2757 | assert.equal(completed[0]?.type, "turn.completed"); | ||
| 2758 | assert.equal(completed[1]?.type, "turn.completed"); | ||
| 2759 | if (completed[0]?.type === "turn.completed" && completed[1]?.type === "turn.completed") { | ||
| 2760 | assert.deepEqual(completed[0].payload.tokenUsage, { | ||
| 2761 | usageStatus: "complete", | ||
| 2762 | usageScope: "main_agent", | ||
| 2763 | inputTokens: 125, | ||
| 2764 | cachedInputTokens: 20, | ||
| 2765 | cacheCreationTokens: 5, | ||
| 2766 | outputTokens: 10, | ||
| 2767 | hasSubagents: false, | ||
| 2768 | }); | ||
| 2769 | assert.deepEqual(completed[1].payload.tokenUsage, { | ||
| 2770 | usageStatus: "complete", | ||
| 2771 | usageScope: "main_agent", | ||
| 2772 | inputTokens: 35, | ||
| 2773 | cachedInputTokens: 2, | ||
| 2774 | cacheCreationTokens: 3, | ||
| 2775 | outputTokens: 7, | ||
| 2776 | hasSubagents: false, | ||
| 2777 | }); | ||
| 2778 | } | ||
| 2540 | }).pipe( | 2779 | }).pipe( |
| 2541 | Effect.provideService(Random.Random, makeDeterministicRandomService()), | 2780 | Effect.provideService(Random.Random, makeDeterministicRandomService()), |
| 2542 | Effect.provide(harness.layer), | 2781 | Effect.provide(harness.layer), |
| ... | @@ -2941,6 +3180,11 @@ describe("ClaudeAdapterLive", () => { | ... | @@ -2941,6 +3180,11 @@ describe("ClaudeAdapterLive", () => { |
| 2941 | assert.equal(String(turnCompleted.turnId), String(turn.turnId)); | 3180 | assert.equal(String(turnCompleted.turnId), String(turn.turnId)); |
| 2942 | assert.equal(turnCompleted.payload.state, "interrupted"); | 3181 | assert.equal(turnCompleted.payload.state, "interrupted"); |
| 2943 | assert.equal(turnCompleted.payload.errorMessage, "Claude runtime interrupted."); | 3182 | assert.equal(turnCompleted.payload.errorMessage, "Claude runtime interrupted."); |
| 3183 | assert.deepEqual(turnCompleted.payload.tokenUsage, { | ||
| 3184 | usageStatus: "unavailable", | ||
| 3185 | usageScope: "main_agent", | ||
| 3186 | hasSubagents: false, | ||
| 3187 | }); | ||
| 2944 | } | 3188 | } |
| 2945 | 3189 | ||
| 2946 | const sessionExited = runtimeEvents[5]; | 3190 | const sessionExited = runtimeEvents[5]; |
apps/server/src/provider/Layers/ClaudeAdapter.ts+94
| ... | @@ -24,6 +24,7 @@ import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; | ... | @@ -24,6 +24,7 @@ import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; |
| 24 | import { type ClaudeScopedLimitNames, claudeRateLimitEventToUpdate } from "./claudeUsageLimits.ts"; | 24 | import { type ClaudeScopedLimitNames, claudeRateLimitEventToUpdate } from "./claudeUsageLimits.ts"; |
| 25 | import { | 25 | import { |
| 26 | ApprovalRequestId, | 26 | ApprovalRequestId, |
| 27 | classifyTaskAgentKind, | ||
| 27 | type CanonicalItemType, | 28 | type CanonicalItemType, |
| 28 | type CanonicalRequestType, | 29 | type CanonicalRequestType, |
| 29 | type ClaudeSettings, | 30 | type ClaudeSettings, |
| ... | @@ -38,6 +39,7 @@ import { | ... | @@ -38,6 +39,7 @@ import { |
| 38 | type ProviderSendTurnInput, | 39 | type ProviderSendTurnInput, |
| 39 | type ProviderSession, | 40 | type ProviderSession, |
| 40 | type ThreadTokenUsageSnapshot, | 41 | type ThreadTokenUsageSnapshot, |
| 42 | type TurnTokenUsage, | ||
| 41 | type ProviderUserInputAnswers, | 43 | type ProviderUserInputAnswers, |
| 42 | type RuntimeContentStreamKind, | 44 | type RuntimeContentStreamKind, |
| 43 | RuntimeItemId, | 45 | RuntimeItemId, |
| ... | @@ -154,6 +156,7 @@ interface ClaudeTurnState { | ... | @@ -154,6 +156,7 @@ interface ClaudeTurnState { |
| 154 | readonly capturedProposedPlanKeys: Set<string>; | 156 | readonly capturedProposedPlanKeys: Set<string>; |
| 155 | latestAssistantUsage: unknown | undefined; | 157 | latestAssistantUsage: unknown | undefined; |
| 156 | compactedSinceLatestAssistantUsage: boolean; | 158 | compactedSinceLatestAssistantUsage: boolean; |
| 159 | hasSubagents: boolean; | ||
| 157 | nextSyntheticAssistantBlockIndex: number; | 160 | nextSyntheticAssistantBlockIndex: number; |
| 158 | } | 161 | } |
| 159 | 162 | ||
| ... | @@ -643,6 +646,85 @@ function normalizeClaudeActiveTokenUsage( | ... | @@ -643,6 +646,85 @@ function normalizeClaudeActiveTokenUsage( |
| 643 | }); | 646 | }); |
| 644 | } | 647 | } |
| 645 | 648 | ||
| 649 | function normalizeClaudeTurnTokenUsage( | ||
| 650 | result: SDKResultMessage | undefined, | ||
| 651 | hasSubagents: boolean, | ||
| 652 | terminalStatus: ProviderRuntimeTurnStatus, | ||
| 653 | ): TurnTokenUsage { | ||
| 654 | const usage = result?.usage as Record<string, unknown> | undefined; | ||
| 655 | if (!usage) { | ||
| 656 | return { | ||
| 657 | usageStatus: "unavailable", | ||
| 658 | usageScope: "main_agent", | ||
| 659 | hasSubagents, | ||
| 660 | }; | ||
| 661 | } | ||
| 662 | |||
| 663 | const uncachedInputTokens = finiteNonNegativeInteger(usage.input_tokens); | ||
| 664 | const cachedInputTokens = finiteNonNegativeInteger(usage.cache_read_input_tokens); | ||
| 665 | const cacheCreationTokens = finiteNonNegativeInteger(usage.cache_creation_input_tokens); | ||
| 666 | const rawOutputTokens = finiteNonNegativeInteger(usage.output_tokens); | ||
| 667 | const outputDetails = usage.output_tokens_details as Record<string, unknown> | undefined; | ||
| 668 | const thinkingTokens = finiteNonNegativeInteger(outputDetails?.thinking_tokens); | ||
| 669 | const cachedInputContribution = usage.cache_read_input_tokens == null ? 0 : cachedInputTokens; | ||
| 670 | const cacheCreationContribution = | ||
| 671 | usage.cache_creation_input_tokens == null ? 0 : cacheCreationTokens; | ||
| 672 | const inputTokens = | ||
| 673 | uncachedInputTokens !== undefined && | ||
| 674 | cachedInputContribution !== undefined && | ||
| 675 | cacheCreationContribution !== undefined | ||
| 676 | ? uncachedInputTokens + cachedInputContribution + cacheCreationContribution | ||
| 677 | : undefined; | ||
| 678 | const hasKnownUsage = | ||
| 679 | uncachedInputTokens !== undefined || | ||
| 680 | cachedInputTokens !== undefined || | ||
| 681 | cacheCreationTokens !== undefined || | ||
| 682 | rawOutputTokens !== undefined; | ||
| 683 | const hasPositiveUsage = | ||
| 684 | (uncachedInputTokens ?? 0) + | ||
| 685 | (cachedInputTokens ?? 0) + | ||
| 686 | (cacheCreationTokens ?? 0) + | ||
| 687 | (rawOutputTokens ?? 0) > | ||
| 688 | 0; | ||
| 689 | |||
| 690 | if (!hasKnownUsage || (result?.subtype !== "success" && !hasPositiveUsage)) { | ||
| 691 | return { | ||
| 692 | usageStatus: "unavailable", | ||
| 693 | usageScope: "main_agent", | ||
| 694 | hasSubagents, | ||
| 695 | }; | ||
| 696 | } | ||
| 697 | |||
| 698 | const commonUsage = { | ||
| 699 | usageScope: "main_agent", | ||
| 700 | ...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}), | ||
| 701 | ...(cacheCreationTokens !== undefined ? { cacheCreationTokens } : {}), | ||
| 702 | ...(thinkingTokens !== undefined && rawOutputTokens !== undefined | ||
| 703 | ? { reasoningTokens: Math.min(rawOutputTokens, thinkingTokens) } | ||
| 704 | : {}), | ||
| 705 | hasSubagents, | ||
| 706 | } as const; | ||
| 707 | if ( | ||
| 708 | terminalStatus === "completed" && | ||
| 709 | result?.subtype === "success" && | ||
| 710 | inputTokens !== undefined && | ||
| 711 | rawOutputTokens !== undefined | ||
| 712 | ) { | ||
| 713 | return { | ||
| 714 | ...commonUsage, | ||
| 715 | usageStatus: "complete", | ||
| 716 | inputTokens, | ||
| 717 | outputTokens: rawOutputTokens, | ||
| 718 | }; | ||
| 719 | } | ||
| 720 | return { | ||
| 721 | ...commonUsage, | ||
| 722 | usageStatus: "partial", | ||
| 723 | ...(inputTokens !== undefined ? { inputTokens } : {}), | ||
| 724 | ...(rawOutputTokens !== undefined ? { outputTokens: rawOutputTokens } : {}), | ||
| 725 | }; | ||
| 726 | } | ||
| 727 | |||
| 646 | function compactBoundaryTokenUsageSnapshot( | 728 | function compactBoundaryTokenUsageSnapshot( |
| 647 | message: Record<string, unknown>, | 729 | message: Record<string, unknown>, |
| 648 | contextWindow?: number, | 730 | contextWindow?: number, |
| ... | @@ -2503,6 +2585,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( | ... | @@ -2503,6 +2585,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( |
| 2503 | ? { totalCostUsd: result.total_cost_usd } | 2585 | ? { totalCostUsd: result.total_cost_usd } |
| 2504 | : {}), | 2586 | : {}), |
| 2505 | ...(errorMessage ? { errorMessage } : {}), | 2587 | ...(errorMessage ? { errorMessage } : {}), |
| 2588 | tokenUsage: normalizeClaudeTurnTokenUsage(result, turnState.hasSubagents, status), | ||
| 2506 | }, | 2589 | }, |
| 2507 | providerRefs: nativeProviderRefs(context), | 2590 | providerRefs: nativeProviderRefs(context), |
| 2508 | }); | 2591 | }); |
| ... | @@ -3047,6 +3130,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( | ... | @@ -3047,6 +3130,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( |
| 3047 | capturedProposedPlanKeys: new Set(), | 3130 | capturedProposedPlanKeys: new Set(), |
| 3048 | latestAssistantUsage: undefined, | 3131 | latestAssistantUsage: undefined, |
| 3049 | compactedSinceLatestAssistantUsage: false, | 3132 | compactedSinceLatestAssistantUsage: false, |
| 3133 | hasSubagents: false, | ||
| 3050 | nextSyntheticAssistantBlockIndex: -1, | 3134 | nextSyntheticAssistantBlockIndex: -1, |
| 3051 | }; | 3135 | }; |
| 3052 | context.session = { | 3136 | context.session = { |
| ... | @@ -3353,6 +3437,15 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( | ... | @@ -3353,6 +3437,15 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( |
| 3353 | ) | 3437 | ) |
| 3354 | : undefined; | 3438 | : undefined; |
| 3355 | const owningAgentId = launchingTool?.agentId; | 3439 | const owningAgentId = launchingTool?.agentId; |
| 3440 | if ( | ||
| 3441 | context.turnState && | ||
| 3442 | classifyTaskAgentKind({ | ||
| 3443 | taskType: message.task_type, | ||
| 3444 | ...(owningAgentId ? { agentId: owningAgentId } : {}), | ||
| 3445 | }) === "agent" | ||
| 3446 | ) { | ||
| 3447 | context.turnState.hasSubagents = true; | ||
| 3448 | } | ||
| 3356 | // Model/effort: the Agent tool's input carries explicit overrides; | 3449 | // Model/effort: the Agent tool's input carries explicit overrides; |
| 3357 | // absent ones inherit the session's selection (SDK behavior). | 3450 | // absent ones inherit the session's selection (SDK behavior). |
| 3358 | // Subagent assistant snapshots refine model with the authoritative API | 3451 | // Subagent assistant snapshots refine model with the authoritative API |
| ... | @@ -4746,6 +4839,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( | ... | @@ -4746,6 +4839,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( |
| 4746 | capturedProposedPlanKeys: new Set(), | 4839 | capturedProposedPlanKeys: new Set(), |
| 4747 | latestAssistantUsage: undefined, | 4840 | latestAssistantUsage: undefined, |
| 4748 | compactedSinceLatestAssistantUsage: false, | 4841 | compactedSinceLatestAssistantUsage: false, |
| 4842 | hasSubagents: false, | ||
| 4749 | nextSyntheticAssistantBlockIndex: -1, | 4843 | nextSyntheticAssistantBlockIndex: -1, |
| 4750 | }; | 4844 | }; |
| 4751 | 4845 |
apps/server/src/provider/Layers/CodexAdapter.test.ts+446
| ... | @@ -597,7 +597,453 @@ function startLifecycleRuntime() { | ... | @@ -597,7 +597,453 @@ function startLifecycleRuntime() { |
| 597 | }); | 597 | }); |
| 598 | } | 598 | } |
| 599 | 599 | ||
| 600 | function codexTokenUsageEvent(input: { | ||
| 601 | readonly id: string; | ||
| 602 | readonly turnId: string; | ||
| 603 | readonly inputTokens: number; | ||
| 604 | readonly cachedInputTokens: number; | ||
| 605 | readonly cacheCreationTokens: number; | ||
| 606 | readonly outputTokens: number; | ||
| 607 | readonly reasoningTokens: number; | ||
| 608 | readonly last?: { | ||
| 609 | readonly inputTokens: number; | ||
| 610 | readonly cachedInputTokens: number; | ||
| 611 | readonly cacheCreationTokens: number; | ||
| 612 | readonly outputTokens: number; | ||
| 613 | readonly reasoningTokens: number; | ||
| 614 | }; | ||
| 615 | }): ProviderEvent { | ||
| 616 | const totalTokens = input.inputTokens + input.outputTokens; | ||
| 617 | const last = input.last ?? input; | ||
| 618 | return { | ||
| 619 | id: asEventId(input.id), | ||
| 620 | kind: "notification", | ||
| 621 | provider: ProviderDriverKind.make("codex"), | ||
| 622 | threadId: asThreadId("thread-1"), | ||
| 623 | turnId: asTurnId(input.turnId), | ||
| 624 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 625 | method: "thread/tokenUsage/updated", | ||
| 626 | payload: { | ||
| 627 | threadId: "thread-1", | ||
| 628 | turnId: input.turnId, | ||
| 629 | tokenUsage: { | ||
| 630 | total: { | ||
| 631 | inputTokens: input.inputTokens, | ||
| 632 | cachedInputTokens: input.cachedInputTokens, | ||
| 633 | cacheWriteInputTokens: input.cacheCreationTokens, | ||
| 634 | outputTokens: input.outputTokens, | ||
| 635 | reasoningOutputTokens: input.reasoningTokens, | ||
| 636 | totalTokens, | ||
| 637 | }, | ||
| 638 | last: { | ||
| 639 | inputTokens: last.inputTokens, | ||
| 640 | cachedInputTokens: last.cachedInputTokens, | ||
| 641 | cacheWriteInputTokens: last.cacheCreationTokens, | ||
| 642 | outputTokens: last.outputTokens, | ||
| 643 | reasoningOutputTokens: last.reasoningTokens, | ||
| 644 | totalTokens: last.inputTokens + last.outputTokens, | ||
| 645 | }, | ||
| 646 | }, | ||
| 647 | }, | ||
| 648 | }; | ||
| 649 | } | ||
| 650 | |||
| 651 | function codexTurnEvent(method: "turn/started" | "turn/completed", turnId: string): ProviderEvent { | ||
| 652 | return { | ||
| 653 | id: asEventId(`evt-${method}-${turnId}`), | ||
| 654 | kind: "notification", | ||
| 655 | provider: ProviderDriverKind.make("codex"), | ||
| 656 | threadId: asThreadId("thread-1"), | ||
| 657 | turnId: asTurnId(turnId), | ||
| 658 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 659 | method, | ||
| 660 | payload: | ||
| 661 | method === "turn/started" | ||
| 662 | ? {} | ||
| 663 | : { | ||
| 664 | threadId: "thread-1", | ||
| 665 | turn: { id: turnId, items: [], status: "completed" }, | ||
| 666 | }, | ||
| 667 | }; | ||
| 668 | } | ||
| 669 | |||
| 600 | lifecycleLayer("CodexAdapterLive lifecycle", (it) => { | 670 | lifecycleLayer("CodexAdapterLive lifecycle", (it) => { |
| 671 | it.effect("calculates one Codex turn total from cumulative counters", () => | ||
| 672 | Effect.gen(function* () { | ||
| 673 | const { adapter, runtime } = yield* startLifecycleRuntime(); | ||
| 674 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 675 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 676 | Stream.runHead, | ||
| 677 | Effect.forkChild, | ||
| 678 | ); | ||
| 679 | |||
| 680 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-usage")); | ||
| 681 | yield* runtime.emit( | ||
| 682 | codexTokenUsageEvent({ | ||
| 683 | id: "evt-usage-1", | ||
| 684 | turnId: "turn-usage", | ||
| 685 | inputTokens: 100, | ||
| 686 | cachedInputTokens: 40, | ||
| 687 | cacheCreationTokens: 10, | ||
| 688 | outputTokens: 20, | ||
| 689 | reasoningTokens: 8, | ||
| 690 | }), | ||
| 691 | ); | ||
| 692 | // Codex can repeat both notifications without new work. | ||
| 693 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-usage")); | ||
| 694 | yield* runtime.emit( | ||
| 695 | codexTokenUsageEvent({ | ||
| 696 | id: "evt-usage-duplicate", | ||
| 697 | turnId: "turn-usage", | ||
| 698 | inputTokens: 100, | ||
| 699 | cachedInputTokens: 40, | ||
| 700 | cacheCreationTokens: 10, | ||
| 701 | outputTokens: 20, | ||
| 702 | reasoningTokens: 8, | ||
| 703 | }), | ||
| 704 | ); | ||
| 705 | yield* runtime.emit({ | ||
| 706 | id: asEventId("evt-collab-activity"), | ||
| 707 | kind: "notification", | ||
| 708 | provider: ProviderDriverKind.make("codex"), | ||
| 709 | threadId: asThreadId("thread-1"), | ||
| 710 | turnId: asTurnId("turn-usage"), | ||
| 711 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 712 | method: "collabAgent/activity", | ||
| 713 | payload: { | ||
| 714 | agentThreadId: "child-1", | ||
| 715 | agentPath: "/root/child-1", | ||
| 716 | activityKind: "started", | ||
| 717 | }, | ||
| 718 | }); | ||
| 719 | yield* runtime.emit( | ||
| 720 | codexTokenUsageEvent({ | ||
| 721 | id: "evt-usage-2", | ||
| 722 | turnId: "turn-usage", | ||
| 723 | inputTokens: 150, | ||
| 724 | cachedInputTokens: 60, | ||
| 725 | cacheCreationTokens: 15, | ||
| 726 | outputTokens: 30, | ||
| 727 | reasoningTokens: 12, | ||
| 728 | }), | ||
| 729 | ); | ||
| 730 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-usage")); | ||
| 731 | |||
| 732 | const completed = yield* Fiber.join(completedFiber); | ||
| 733 | NodeAssert.equal(completed._tag, "Some"); | ||
| 734 | if (completed._tag === "Some" && completed.value.type === "turn.completed") { | ||
| 735 | NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, { | ||
| 736 | usageStatus: "complete", | ||
| 737 | usageScope: "main_agent", | ||
| 738 | inputTokens: 150, | ||
| 739 | cachedInputTokens: 60, | ||
| 740 | cacheCreationTokens: 15, | ||
| 741 | outputTokens: 30, | ||
| 742 | reasoningTokens: 12, | ||
| 743 | hasSubagents: true, | ||
| 744 | }); | ||
| 745 | } | ||
| 746 | }), | ||
| 747 | ); | ||
| 748 | |||
| 749 | it.effect("does not charge a late prior-turn update to the next Codex turn", () => | ||
| 750 | Effect.gen(function* () { | ||
| 751 | const { adapter, runtime } = yield* startLifecycleRuntime(); | ||
| 752 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 753 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 754 | Stream.take(2), | ||
| 755 | Stream.runCollect, | ||
| 756 | Effect.forkChild, | ||
| 757 | ); | ||
| 758 | |||
| 759 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-first")); | ||
| 760 | yield* runtime.emit( | ||
| 761 | codexTokenUsageEvent({ | ||
| 762 | id: "evt-late-1", | ||
| 763 | turnId: "turn-first", | ||
| 764 | inputTokens: 100, | ||
| 765 | cachedInputTokens: 40, | ||
| 766 | cacheCreationTokens: 10, | ||
| 767 | outputTokens: 20, | ||
| 768 | reasoningTokens: 8, | ||
| 769 | }), | ||
| 770 | ); | ||
| 771 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-first")); | ||
| 772 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-second")); | ||
| 773 | // A late update for the finished turn lands after the next turn starts. | ||
| 774 | yield* runtime.emit( | ||
| 775 | codexTokenUsageEvent({ | ||
| 776 | id: "evt-late-2", | ||
| 777 | turnId: "turn-first", | ||
| 778 | inputTokens: 150, | ||
| 779 | cachedInputTokens: 60, | ||
| 780 | cacheCreationTokens: 15, | ||
| 781 | outputTokens: 30, | ||
| 782 | reasoningTokens: 12, | ||
| 783 | }), | ||
| 784 | ); | ||
| 785 | yield* runtime.emit( | ||
| 786 | codexTokenUsageEvent({ | ||
| 787 | id: "evt-late-3", | ||
| 788 | turnId: "turn-second", | ||
| 789 | inputTokens: 170, | ||
| 790 | cachedInputTokens: 65, | ||
| 791 | cacheCreationTokens: 16, | ||
| 792 | outputTokens: 35, | ||
| 793 | reasoningTokens: 14, | ||
| 794 | }), | ||
| 795 | ); | ||
| 796 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-second")); | ||
| 797 | |||
| 798 | const completed = Array.from(yield* Fiber.join(completedFiber)); | ||
| 799 | const second = completed[1]; | ||
| 800 | NodeAssert.equal(second?.type, "turn.completed"); | ||
| 801 | if (second?.type === "turn.completed") { | ||
| 802 | NodeAssert.deepStrictEqual(second.payload.tokenUsage, { | ||
| 803 | usageStatus: "complete", | ||
| 804 | usageScope: "main_agent", | ||
| 805 | inputTokens: 20, | ||
| 806 | cachedInputTokens: 5, | ||
| 807 | cacheCreationTokens: 1, | ||
| 808 | outputTokens: 5, | ||
| 809 | reasoningTokens: 2, | ||
| 810 | hasSubagents: false, | ||
| 811 | }); | ||
| 812 | } | ||
| 813 | }), | ||
| 814 | ); | ||
| 815 | |||
| 816 | it.effect("clamps Codex cache and reasoning subsets to their totals", () => | ||
| 817 | Effect.gen(function* () { | ||
| 818 | const { adapter, runtime } = yield* startLifecycleRuntime(); | ||
| 819 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 820 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 821 | Stream.runHead, | ||
| 822 | Effect.forkChild, | ||
| 823 | ); | ||
| 824 | |||
| 825 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-clamp")); | ||
| 826 | yield* runtime.emit( | ||
| 827 | codexTokenUsageEvent({ | ||
| 828 | id: "evt-clamp-1", | ||
| 829 | turnId: "turn-clamp", | ||
| 830 | inputTokens: 100, | ||
| 831 | cachedInputTokens: 140, | ||
| 832 | cacheCreationTokens: 120, | ||
| 833 | outputTokens: 20, | ||
| 834 | reasoningTokens: 30, | ||
| 835 | }), | ||
| 836 | ); | ||
| 837 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-clamp")); | ||
| 838 | |||
| 839 | const completed = yield* Fiber.join(completedFiber); | ||
| 840 | NodeAssert.equal(completed._tag, "Some"); | ||
| 841 | if (completed._tag === "Some" && completed.value.type === "turn.completed") { | ||
| 842 | NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, { | ||
| 843 | usageStatus: "complete", | ||
| 844 | usageScope: "main_agent", | ||
| 845 | inputTokens: 100, | ||
| 846 | cachedInputTokens: 100, | ||
| 847 | cacheCreationTokens: 100, | ||
| 848 | outputTokens: 20, | ||
| 849 | reasoningTokens: 20, | ||
| 850 | hasSubagents: false, | ||
| 851 | }); | ||
| 852 | } | ||
| 853 | }), | ||
| 854 | ); | ||
| 855 | |||
| 856 | it.effect("counts the last response when Codex resets its running total mid-turn", () => | ||
| 857 | Effect.gen(function* () { | ||
| 858 | const { adapter, runtime } = yield* startLifecycleRuntime(); | ||
| 859 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 860 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 861 | Stream.runHead, | ||
| 862 | Effect.forkChild, | ||
| 863 | ); | ||
| 864 | |||
| 865 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-reset")); | ||
| 866 | yield* runtime.emit( | ||
| 867 | codexTokenUsageEvent({ | ||
| 868 | id: "evt-reset-1", | ||
| 869 | turnId: "turn-reset", | ||
| 870 | inputTokens: 5_000, | ||
| 871 | cachedInputTokens: 4_000, | ||
| 872 | cacheCreationTokens: 100, | ||
| 873 | outputTokens: 500, | ||
| 874 | reasoningTokens: 200, | ||
| 875 | last: { | ||
| 876 | inputTokens: 100, | ||
| 877 | cachedInputTokens: 80, | ||
| 878 | cacheCreationTokens: 10, | ||
| 879 | outputTokens: 20, | ||
| 880 | reasoningTokens: 8, | ||
| 881 | }, | ||
| 882 | }), | ||
| 883 | ); | ||
| 884 | // Codex restarted its cumulative total. The new total is smaller than | ||
| 885 | // the previous one, so only `last` is counted for this update. | ||
| 886 | yield* runtime.emit( | ||
| 887 | codexTokenUsageEvent({ | ||
| 888 | id: "evt-reset-2", | ||
| 889 | turnId: "turn-reset", | ||
| 890 | inputTokens: 150, | ||
| 891 | cachedInputTokens: 90, | ||
| 892 | cacheCreationTokens: 5, | ||
| 893 | outputTokens: 30, | ||
| 894 | reasoningTokens: 12, | ||
| 895 | }), | ||
| 896 | ); | ||
| 897 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-reset")); | ||
| 898 | |||
| 899 | const completed = yield* Fiber.join(completedFiber); | ||
| 900 | NodeAssert.equal(completed._tag, "Some"); | ||
| 901 | if (completed._tag === "Some" && completed.value.type === "turn.completed") { | ||
| 902 | NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, { | ||
| 903 | usageStatus: "complete", | ||
| 904 | usageScope: "main_agent", | ||
| 905 | inputTokens: 250, | ||
| 906 | cachedInputTokens: 170, | ||
| 907 | cacheCreationTokens: 15, | ||
| 908 | outputTokens: 50, | ||
| 909 | reasoningTokens: 20, | ||
| 910 | hasSubagents: false, | ||
| 911 | }); | ||
| 912 | } | ||
| 913 | }), | ||
| 914 | ); | ||
| 915 | |||
| 916 | it.effect("uses the last response usage when no prior Codex total exists", () => | ||
| 917 | Effect.gen(function* () { | ||
| 918 | const adapter = yield* CodexAdapter; | ||
| 919 | yield* adapter.startSession({ | ||
| 920 | provider: ProviderDriverKind.make("codex"), | ||
| 921 | threadId: asThreadId("thread-1"), | ||
| 922 | resumeCursor: { threadId: "provider-thread-1" }, | ||
| 923 | runtimeMode: "full-access", | ||
| 924 | }); | ||
| 925 | const runtime = lifecycleRuntimeFactory.lastRuntime; | ||
| 926 | NodeAssert.ok(runtime); | ||
| 927 | const firstCompletionsFiber = yield* adapter.streamEvents.pipe( | ||
| 928 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 929 | Stream.take(2), | ||
| 930 | Stream.runCollect, | ||
| 931 | Effect.forkChild, | ||
| 932 | ); | ||
| 933 | |||
| 934 | // Resumed thread: the cumulative total already holds old history, so the | ||
| 935 | // first update must count only `last`. | ||
| 936 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-resumed")); | ||
| 937 | yield* runtime.emit( | ||
| 938 | codexTokenUsageEvent({ | ||
| 939 | id: "evt-resume-baseline", | ||
| 940 | turnId: "turn-resumed", | ||
| 941 | inputTokens: 1_000, | ||
| 942 | cachedInputTokens: 400, | ||
| 943 | cacheCreationTokens: 100, | ||
| 944 | outputTokens: 200, | ||
| 945 | reasoningTokens: 80, | ||
| 946 | last: { | ||
| 947 | inputTokens: 300, | ||
| 948 | cachedInputTokens: 120, | ||
| 949 | cacheCreationTokens: 30, | ||
| 950 | outputTokens: 60, | ||
| 951 | reasoningTokens: 24, | ||
| 952 | }, | ||
| 953 | }), | ||
| 954 | ); | ||
| 955 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-resumed")); | ||
| 956 | |||
| 957 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-after-resume")); | ||
| 958 | yield* runtime.emit( | ||
| 959 | codexTokenUsageEvent({ | ||
| 960 | id: "evt-after-resume", | ||
| 961 | turnId: "turn-after-resume", | ||
| 962 | inputTokens: 1_100, | ||
| 963 | cachedInputTokens: 440, | ||
| 964 | cacheCreationTokens: 110, | ||
| 965 | outputTokens: 220, | ||
| 966 | reasoningTokens: 88, | ||
| 967 | }), | ||
| 968 | ); | ||
| 969 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-after-resume")); | ||
| 970 | |||
| 971 | const firstCompletions = Array.from(yield* Fiber.join(firstCompletionsFiber)); | ||
| 972 | |||
| 973 | yield* adapter.rollbackThread(asThreadId("thread-1"), 1); | ||
| 974 | const rollbackCompletionFiber = yield* adapter.streamEvents.pipe( | ||
| 975 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 976 | Stream.runHead, | ||
| 977 | Effect.forkChild, | ||
| 978 | ); | ||
| 979 | // Rollback drops the baseline and Codex shrinks its total, so the first | ||
| 980 | // update after it counts only `last` again. | ||
| 981 | yield* runtime.emit(codexTurnEvent("turn/started", "turn-after-rollback")); | ||
| 982 | yield* runtime.emit( | ||
| 983 | codexTokenUsageEvent({ | ||
| 984 | id: "evt-after-rollback", | ||
| 985 | turnId: "turn-after-rollback", | ||
| 986 | inputTokens: 1_050, | ||
| 987 | cachedInputTokens: 420, | ||
| 988 | cacheCreationTokens: 105, | ||
| 989 | outputTokens: 210, | ||
| 990 | reasoningTokens: 84, | ||
| 991 | last: { | ||
| 992 | inputTokens: 50, | ||
| 993 | cachedInputTokens: 20, | ||
| 994 | cacheCreationTokens: 5, | ||
| 995 | outputTokens: 10, | ||
| 996 | reasoningTokens: 4, | ||
| 997 | }, | ||
| 998 | }), | ||
| 999 | ); | ||
| 1000 | yield* runtime.emit(codexTurnEvent("turn/completed", "turn-after-rollback")); | ||
| 1001 | |||
| 1002 | const rollbackCompletion = yield* Fiber.join(rollbackCompletionFiber); | ||
| 1003 | const completions = [ | ||
| 1004 | ...firstCompletions, | ||
| 1005 | ...(rollbackCompletion._tag === "Some" ? [rollbackCompletion.value] : []), | ||
| 1006 | ]; | ||
| 1007 | NodeAssert.deepStrictEqual( | ||
| 1008 | completions.map((event) => | ||
| 1009 | event.type === "turn.completed" ? event.payload.tokenUsage : undefined, | ||
| 1010 | ), | ||
| 1011 | [ | ||
| 1012 | { | ||
| 1013 | usageStatus: "complete", | ||
| 1014 | usageScope: "main_agent", | ||
| 1015 | inputTokens: 300, | ||
| 1016 | cachedInputTokens: 120, | ||
| 1017 | cacheCreationTokens: 30, | ||
| 1018 | outputTokens: 60, | ||
| 1019 | reasoningTokens: 24, | ||
| 1020 | hasSubagents: false, | ||
| 1021 | }, | ||
| 1022 | { | ||
| 1023 | usageStatus: "complete", | ||
| 1024 | usageScope: "main_agent", | ||
| 1025 | inputTokens: 100, | ||
| 1026 | cachedInputTokens: 40, | ||
| 1027 | cacheCreationTokens: 10, | ||
| 1028 | outputTokens: 20, | ||
| 1029 | reasoningTokens: 8, | ||
| 1030 | hasSubagents: false, | ||
| 1031 | }, | ||
| 1032 | { | ||
| 1033 | usageStatus: "complete", | ||
| 1034 | usageScope: "main_agent", | ||
| 1035 | inputTokens: 50, | ||
| 1036 | cachedInputTokens: 20, | ||
| 1037 | cacheCreationTokens: 5, | ||
| 1038 | outputTokens: 10, | ||
| 1039 | reasoningTokens: 4, | ||
| 1040 | hasSubagents: false, | ||
| 1041 | }, | ||
| 1042 | ], | ||
| 1043 | ); | ||
| 1044 | }), | ||
| 1045 | ); | ||
| 1046 | |||
| 601 | it.effect("carries child model metadata through every task event", () => | 1047 | it.effect("carries child model metadata through every task event", () => |
| 602 | Effect.gen(function* () { | 1048 | Effect.gen(function* () { |
| 603 | const { adapter, runtime } = yield* startLifecycleRuntime(); | 1049 | const { adapter, runtime } = yield* startLifecycleRuntime(); |
apps/server/src/provider/Layers/CodexAdapter.ts+253-2
| ... | @@ -26,6 +26,7 @@ import { | ... | @@ -26,6 +26,7 @@ import { |
| 26 | RuntimeRequestId, | 26 | RuntimeRequestId, |
| 27 | RuntimeTaskId, | 27 | RuntimeTaskId, |
| 28 | type RuntimeTaskUsage, | 28 | type RuntimeTaskUsage, |
| 29 | type TurnTokenUsage, | ||
| 29 | ProviderApprovalDecision, | 30 | ProviderApprovalDecision, |
| 30 | ThreadId, | 31 | ThreadId, |
| 31 | ProviderSendTurnInput, | 32 | ProviderSendTurnInput, |
| ... | @@ -99,9 +100,34 @@ interface CodexAdapterSessionContext { | ... | @@ -99,9 +100,34 @@ interface CodexAdapterSessionContext { |
| 99 | readonly scope: Scope.Closeable; | 100 | readonly scope: Scope.Closeable; |
| 100 | readonly runtime: CodexSessionRuntimeShape; | 101 | readonly runtime: CodexSessionRuntimeShape; |
| 101 | readonly eventFiber: Fiber.Fiber<void, never>; | 102 | readonly eventFiber: Fiber.Fiber<void, never>; |
| 103 | readonly turnTokenUsage: CodexTurnTokenUsageState; | ||
| 102 | stopped: boolean; | 104 | stopped: boolean; |
| 103 | } | 105 | } |
| 104 | 106 | ||
| 107 | type CodexCumulativeTokenUsage = { | ||
| 108 | readonly inputTokens: number; | ||
| 109 | readonly cachedInputTokens: number; | ||
| 110 | readonly cacheCreationTokens?: number; | ||
| 111 | readonly outputTokens: number; | ||
| 112 | readonly reasoningTokens: number; | ||
| 113 | }; | ||
| 114 | |||
| 115 | interface CodexTurnTokenUsageAccumulator { | ||
| 116 | inputTokens: number; | ||
| 117 | cachedInputTokens: number; | ||
| 118 | cacheCreationTokens: number | undefined; | ||
| 119 | outputTokens: number; | ||
| 120 | reasoningTokens: number; | ||
| 121 | observed: boolean; | ||
| 122 | hasSubagents: boolean; | ||
| 123 | } | ||
| 124 | |||
| 125 | interface CodexTurnTokenUsageState { | ||
| 126 | baseline: CodexCumulativeTokenUsage | undefined; | ||
| 127 | activeTurnId: string | undefined; | ||
| 128 | readonly byTurnId: Map<string, CodexTurnTokenUsageAccumulator>; | ||
| 129 | } | ||
| 130 | |||
| 105 | function mapCodexRuntimeError( | 131 | function mapCodexRuntimeError( |
| 106 | threadId: ThreadId, | 132 | threadId: ThreadId, |
| 107 | method: string, | 133 | method: string, |
| ... | @@ -429,6 +455,162 @@ function normalizeCodexTokenUsage( | ... | @@ -429,6 +455,162 @@ function normalizeCodexTokenUsage( |
| 429 | }; | 455 | }; |
| 430 | } | 456 | } |
| 431 | 457 | ||
| 458 | function codexTokenUsageBreakdown( | ||
| 459 | usage: EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification__TokenUsageBreakdown, | ||
| 460 | ): CodexCumulativeTokenUsage { | ||
| 461 | return { | ||
| 462 | inputTokens: usage.inputTokens, | ||
| 463 | cachedInputTokens: usage.cachedInputTokens, | ||
| 464 | ...(usage.cacheWriteInputTokens !== undefined | ||
| 465 | ? { cacheCreationTokens: usage.cacheWriteInputTokens } | ||
| 466 | : {}), | ||
| 467 | outputTokens: usage.outputTokens, | ||
| 468 | reasoningTokens: usage.reasoningOutputTokens, | ||
| 469 | }; | ||
| 470 | } | ||
| 471 | |||
| 472 | function makeCodexTurnTokenUsageState(): CodexTurnTokenUsageState { | ||
| 473 | return { | ||
| 474 | baseline: undefined, | ||
| 475 | activeTurnId: undefined, | ||
| 476 | byTurnId: new Map(), | ||
| 477 | }; | ||
| 478 | } | ||
| 479 | |||
| 480 | function getCodexTurnAccumulator( | ||
| 481 | state: CodexTurnTokenUsageState, | ||
| 482 | turnId: string, | ||
| 483 | ): CodexTurnTokenUsageAccumulator { | ||
| 484 | const existing = state.byTurnId.get(turnId); | ||
| 485 | if (existing) return existing; | ||
| 486 | const created: CodexTurnTokenUsageAccumulator = { | ||
| 487 | inputTokens: 0, | ||
| 488 | cachedInputTokens: 0, | ||
| 489 | cacheCreationTokens: 0, | ||
| 490 | outputTokens: 0, | ||
| 491 | reasoningTokens: 0, | ||
| 492 | observed: false, | ||
| 493 | hasSubagents: false, | ||
| 494 | }; | ||
| 495 | state.byTurnId.set(turnId, created); | ||
| 496 | return created; | ||
| 497 | } | ||
| 498 | |||
| 499 | /** | ||
| 500 | * Usage added by one `thread/tokenUsage/updated` notification. Codex reports a | ||
| 501 | * running `total` for the thread and `last`, the usage of the newest model | ||
| 502 | * response. Within a turn the growth of `total` equals `last`. Without a prior | ||
| 503 | * total (first update after resume or rollback), or when Codex reset the | ||
| 504 | * running total, `last` is the delta. | ||
| 505 | */ | ||
| 506 | function codexTurnTokenUsageDelta( | ||
| 507 | previous: CodexCumulativeTokenUsage | undefined, | ||
| 508 | current: CodexCumulativeTokenUsage, | ||
| 509 | last: CodexCumulativeTokenUsage, | ||
| 510 | ): CodexCumulativeTokenUsage { | ||
| 511 | if ( | ||
| 512 | previous === undefined || | ||
| 513 | current.inputTokens < previous.inputTokens || | ||
| 514 | current.cachedInputTokens < previous.cachedInputTokens || | ||
| 515 | current.outputTokens < previous.outputTokens || | ||
| 516 | current.reasoningTokens < previous.reasoningTokens | ||
| 517 | ) { | ||
| 518 | return last; | ||
| 519 | } | ||
| 520 | return { | ||
| 521 | inputTokens: current.inputTokens - previous.inputTokens, | ||
| 522 | cachedInputTokens: current.cachedInputTokens - previous.cachedInputTokens, | ||
| 523 | ...(current.cacheCreationTokens !== undefined && | ||
| 524 | previous.cacheCreationTokens !== undefined && | ||
| 525 | current.cacheCreationTokens >= previous.cacheCreationTokens | ||
| 526 | ? { cacheCreationTokens: current.cacheCreationTokens - previous.cacheCreationTokens } | ||
| 527 | : {}), | ||
| 528 | outputTokens: current.outputTokens - previous.outputTokens, | ||
| 529 | reasoningTokens: current.reasoningTokens - previous.reasoningTokens, | ||
| 530 | }; | ||
| 531 | } | ||
| 532 | |||
| 533 | function accumulateCodexTurnTokenUsage( | ||
| 534 | state: CodexTurnTokenUsageState, | ||
| 535 | turnId: string, | ||
| 536 | usage: EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification["tokenUsage"], | ||
| 537 | ): void { | ||
| 538 | const current = codexTokenUsageBreakdown(usage.total); | ||
| 539 | if (state.activeTurnId !== turnId) { | ||
| 540 | // The total is thread-wide, so every update moves the baseline. A late | ||
| 541 | // update for a finished turn is not counted toward the live turn. | ||
| 542 | state.baseline = current; | ||
| 543 | return; | ||
| 544 | } | ||
| 545 | |||
| 546 | const accumulator = getCodexTurnAccumulator(state, turnId); | ||
| 547 | const delta = codexTurnTokenUsageDelta( | ||
| 548 | state.baseline, | ||
| 549 | current, | ||
| 550 | codexTokenUsageBreakdown(usage.last), | ||
| 551 | ); | ||
| 552 | state.baseline = current; | ||
| 553 | |||
| 554 | if ( | ||
| 555 | delta.inputTokens > 0 || | ||
| 556 | delta.cachedInputTokens > 0 || | ||
| 557 | delta.outputTokens > 0 || | ||
| 558 | delta.reasoningTokens > 0 | ||
| 559 | ) { | ||
| 560 | accumulator.observed = true; | ||
| 561 | } | ||
| 562 | accumulator.inputTokens += delta.inputTokens; | ||
| 563 | accumulator.cachedInputTokens += delta.cachedInputTokens; | ||
| 564 | accumulator.outputTokens += delta.outputTokens; | ||
| 565 | accumulator.reasoningTokens += delta.reasoningTokens; | ||
| 566 | if (delta.cacheCreationTokens === undefined) { | ||
| 567 | accumulator.cacheCreationTokens = undefined; | ||
| 568 | } else if (accumulator.cacheCreationTokens !== undefined) { | ||
| 569 | accumulator.cacheCreationTokens += delta.cacheCreationTokens; | ||
| 570 | } | ||
| 571 | } | ||
| 572 | |||
| 573 | function completeCodexTurnTokenUsage( | ||
| 574 | state: CodexTurnTokenUsageState, | ||
| 575 | turnId: string, | ||
| 576 | completed: boolean, | ||
| 577 | ): TurnTokenUsage { | ||
| 578 | const usage = state.byTurnId.get(turnId); | ||
| 579 | state.byTurnId.delete(turnId); | ||
| 580 | if (state.activeTurnId === turnId) state.activeTurnId = undefined; | ||
| 581 | if (!usage) { | ||
| 582 | return { | ||
| 583 | usageStatus: "unavailable", | ||
| 584 | usageScope: "main_agent", | ||
| 585 | hasSubagents: false, | ||
| 586 | }; | ||
| 587 | } | ||
| 588 | |||
| 589 | if (!usage.observed) { | ||
| 590 | return { | ||
| 591 | usageStatus: "unavailable", | ||
| 592 | usageScope: "main_agent", | ||
| 593 | hasSubagents: usage.hasSubagents, | ||
| 594 | }; | ||
| 595 | } | ||
| 596 | |||
| 597 | // Codex counts cache reads and writes inside inputTokens. Clamp the | ||
| 598 | // subsets so the record keeps the documented relationships even if a | ||
| 599 | // counter drifts. | ||
| 600 | return { | ||
| 601 | usageStatus: completed ? "complete" : "partial", | ||
| 602 | usageScope: "main_agent", | ||
| 603 | inputTokens: usage.inputTokens, | ||
| 604 | cachedInputTokens: Math.min(usage.inputTokens, usage.cachedInputTokens), | ||
| 605 | ...(usage.cacheCreationTokens !== undefined | ||
| 606 | ? { cacheCreationTokens: Math.min(usage.inputTokens, usage.cacheCreationTokens) } | ||
| 607 | : {}), | ||
| 608 | outputTokens: usage.outputTokens, | ||
| 609 | reasoningTokens: Math.min(usage.outputTokens, usage.reasoningTokens), | ||
| 610 | hasSubagents: usage.hasSubagents, | ||
| 611 | }; | ||
| 612 | } | ||
| 613 | |||
| 432 | function toTurnStatus( | 614 | function toTurnStatus( |
| 433 | value: EffectCodexSchema.V2TurnCompletedNotification["turn"]["status"] | "cancelled", | 615 | value: EffectCodexSchema.V2TurnCompletedNotification["turn"]["status"] | "cancelled", |
| 434 | ): "completed" | "failed" | "cancelled" | "interrupted" { | 616 | ): "completed" | "failed" | "cancelled" | "interrupted" { |
| ... | @@ -2063,6 +2245,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2063,6 +2245,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2063 | } | 2245 | } |
| 2064 | : {}), | 2246 | : {}), |
| 2065 | }; | 2247 | }; |
| 2248 | const turnTokenUsage = makeCodexTurnTokenUsageState(); | ||
| 2066 | const sessionScope = yield* Scope.make("sequential"); | 2249 | const sessionScope = yield* Scope.make("sequential"); |
| 2067 | let sessionScopeTransferred = false; | 2250 | let sessionScopeTransferred = false; |
| 2068 | yield* Effect.addFinalizer(() => | 2251 | yield* Effect.addFinalizer(() => |
| ... | @@ -2091,7 +2274,64 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2091,7 +2274,64 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2091 | const eventFiber = yield* Stream.runForEach(runtime.events, (event) => | 2274 | const eventFiber = yield* Stream.runForEach(runtime.events, (event) => |
| 2092 | Effect.gen(function* () { | 2275 | Effect.gen(function* () { |
| 2093 | yield* writeNativeEvent(event); | 2276 | yield* writeNativeEvent(event); |
| 2094 | const runtimeEvents = mapToRuntimeEvents(event, event.threadId); | 2277 | if (event.method === "turn/started" && event.turnId) { |
| 2278 | if (turnTokenUsage.activeTurnId !== event.turnId) { | ||
| 2279 | turnTokenUsage.byTurnId.clear(); | ||
| 2280 | turnTokenUsage.activeTurnId = event.turnId; | ||
| 2281 | getCodexTurnAccumulator(turnTokenUsage, event.turnId); | ||
| 2282 | } | ||
| 2283 | } else if (event.method === "thread/tokenUsage/updated") { | ||
| 2284 | const payload = readPayload( | ||
| 2285 | EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification, | ||
| 2286 | event.payload, | ||
| 2287 | ); | ||
| 2288 | if (payload) { | ||
| 2289 | accumulateCodexTurnTokenUsage(turnTokenUsage, payload.turnId, payload.tokenUsage); | ||
| 2290 | } | ||
| 2291 | } else if (turnTokenUsage.activeTurnId) { | ||
| 2292 | const collabPayload = | ||
| 2293 | typeof event.payload === "object" && event.payload !== null | ||
| 2294 | ? (event.payload as Record<string, unknown>) | ||
| 2295 | : undefined; | ||
| 2296 | const isCollabSpawn = | ||
| 2297 | event.method === "collabAgent/started" || | ||
| 2298 | (event.method === "collabAgent/activity" && | ||
| 2299 | collabPayload?.activityKind === "started"); | ||
| 2300 | if (isCollabSpawn && event.turnId === turnTokenUsage.activeTurnId) { | ||
| 2301 | getCodexTurnAccumulator(turnTokenUsage, turnTokenUsage.activeTurnId).hasSubagents = | ||
| 2302 | true; | ||
| 2303 | } | ||
| 2304 | } | ||
| 2305 | |||
| 2306 | const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => { | ||
| 2307 | if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) { | ||
| 2308 | return { | ||
| 2309 | ...runtimeEvent, | ||
| 2310 | payload: { | ||
| 2311 | ...runtimeEvent.payload, | ||
| 2312 | tokenUsage: completeCodexTurnTokenUsage( | ||
| 2313 | turnTokenUsage, | ||
| 2314 | String(runtimeEvent.turnId), | ||
| 2315 | runtimeEvent.payload.state === "completed", | ||
| 2316 | ), | ||
| 2317 | }, | ||
| 2318 | } satisfies ProviderRuntimeEvent; | ||
| 2319 | } | ||
| 2320 | if (runtimeEvent.type === "turn.aborted" && runtimeEvent.turnId) { | ||
| 2321 | return { | ||
| 2322 | ...runtimeEvent, | ||
| 2323 | payload: { | ||
| 2324 | ...runtimeEvent.payload, | ||
| 2325 | tokenUsage: completeCodexTurnTokenUsage( | ||
| 2326 | turnTokenUsage, | ||
| 2327 | String(runtimeEvent.turnId), | ||
| 2328 | false, | ||
| 2329 | ), | ||
| 2330 | }, | ||
| 2331 | } satisfies ProviderRuntimeEvent; | ||
| 2332 | } | ||
| 2333 | return runtimeEvent; | ||
| 2334 | }); | ||
| 2095 | if (runtimeEvents.length === 0) { | 2335 | if (runtimeEvents.length === 0) { |
| 2096 | yield* Effect.logDebug("ignoring unhandled Codex provider event", { | 2336 | yield* Effect.logDebug("ignoring unhandled Codex provider event", { |
| 2097 | method: event.method, | 2337 | method: event.method, |
| ... | @@ -2129,6 +2369,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2129,6 +2369,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2129 | scope: sessionScope, | 2369 | scope: sessionScope, |
| 2130 | runtime, | 2370 | runtime, |
| 2131 | eventFiber, | 2371 | eventFiber, |
| 2372 | turnTokenUsage, | ||
| 2132 | stopped: false, | 2373 | stopped: false, |
| 2133 | }); | 2374 | }); |
| 2134 | sessionScopeTransferred = true; | 2375 | sessionScopeTransferred = true; |
| ... | @@ -2262,7 +2503,17 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2262,7 +2503,17 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2262 | } | 2503 | } |
| 2263 | 2504 | ||
| 2264 | return requireSession(threadId).pipe( | 2505 | return requireSession(threadId).pipe( |
| 2265 | Effect.flatMap((session) => session.runtime.rollbackThread(numTurns)), | 2506 | Effect.flatMap((session) => |
| 2507 | session.runtime.rollbackThread(numTurns).pipe( | ||
| 2508 | Effect.tap(() => | ||
| 2509 | Effect.sync(() => { | ||
| 2510 | session.turnTokenUsage.baseline = undefined; | ||
| 2511 | session.turnTokenUsage.activeTurnId = undefined; | ||
| 2512 | session.turnTokenUsage.byTurnId.clear(); | ||
| 2513 | }), | ||
| 2514 | ), | ||
| 2515 | ), | ||
| 2516 | ), | ||
| 2266 | Effect.mapError((cause) => | 2517 | Effect.mapError((cause) => |
| 2267 | cause._tag === "ProviderAdapterSessionNotFoundError" | 2518 | cause._tag === "ProviderAdapterSessionNotFoundError" |
| 2268 | ? cause | 2519 | ? cause |
apps/server/src/provider/Layers/OpenCodeAdapter.test.ts+745
| ... | @@ -3,6 +3,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; | ... | @@ -3,6 +3,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 3 | import { it } from "@effect/vitest"; | 3 | import { it } from "@effect/vitest"; |
| 4 | import * as Cause from "effect/Cause"; | 4 | import * as Cause from "effect/Cause"; |
| 5 | import * as Context from "effect/Context"; | 5 | import * as Context from "effect/Context"; |
| 6 | import * as Crypto from "effect/Crypto"; | ||
| 7 | import * as Deferred from "effect/Deferred"; | ||
| 6 | import * as Effect from "effect/Effect"; | 8 | import * as Effect from "effect/Effect"; |
| 7 | import * as Exit from "effect/Exit"; | 9 | import * as Exit from "effect/Exit"; |
| 8 | import * as Fiber from "effect/Fiber"; | 10 | import * as Fiber from "effect/Fiber"; |
| ... | @@ -2086,6 +2088,557 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { | ... | @@ -2086,6 +2088,557 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { |
| 2086 | }), | 2088 | }), |
| 2087 | ); | 2089 | ); |
| 2088 | 2090 | ||
| 2091 | it.effect("marks subagents when a child is proven related by ancestry lookup", () => | ||
| 2092 | Effect.gen(function* () { | ||
| 2093 | const adapter = yield* OpenCodeAdapter; | ||
| 2094 | const threadId = asThreadId("thread-child-ancestry-usage"); | ||
| 2095 | // The child's `session.created` event was missed. Only the ancestry | ||
| 2096 | // lookup can prove that `ses_child` belongs to this thread. | ||
| 2097 | runtimeMock.state.sessionParentById.set("ses_child", "http://127.0.0.1:9999/session"); | ||
| 2098 | runtimeMock.state.sessionStatus = "busy"; | ||
| 2099 | const busy = promiseWithResolvers<unknown>(); | ||
| 2100 | const childPermission = promiseWithResolvers<unknown>(); | ||
| 2101 | const idle = promiseWithResolvers<unknown>(); | ||
| 2102 | runtimeMock.state.subscribedEvents = [busy.promise, childPermission.promise, idle.promise]; | ||
| 2103 | |||
| 2104 | const eventsFiber = yield* adapter.streamEvents.pipe( | ||
| 2105 | Stream.filter( | ||
| 2106 | (event) => | ||
| 2107 | event.threadId === threadId && | ||
| 2108 | (event.type === "request.opened" || event.type === "turn.completed"), | ||
| 2109 | ), | ||
| 2110 | Stream.take(2), | ||
| 2111 | Stream.runCollect, | ||
| 2112 | Effect.forkChild, | ||
| 2113 | ); | ||
| 2114 | yield* adapter.startSession({ | ||
| 2115 | provider: ProviderDriverKind.make("opencode"), | ||
| 2116 | threadId, | ||
| 2117 | runtimeMode: "approval-required", | ||
| 2118 | }); | ||
| 2119 | const sendFiber = yield* adapter | ||
| 2120 | .sendTurn({ | ||
| 2121 | threadId, | ||
| 2122 | input: "Delegate to a child", | ||
| 2123 | modelSelection: createModelSelection( | ||
| 2124 | ProviderInstanceId.make("opencode"), | ||
| 2125 | "opencode/kimi-k3", | ||
| 2126 | ), | ||
| 2127 | }) | ||
| 2128 | .pipe(Effect.forkChild); | ||
| 2129 | busy.resolve({ | ||
| 2130 | id: "evt-child-ancestry-busy", | ||
| 2131 | type: "session.status", | ||
| 2132 | properties: { | ||
| 2133 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2134 | status: { type: "busy" }, | ||
| 2135 | }, | ||
| 2136 | }); | ||
| 2137 | yield* Fiber.join(sendFiber); | ||
| 2138 | |||
| 2139 | const requestOpened = promiseWithResolvers<void>(); | ||
| 2140 | runtimeMock.state.sessionGetObserved = (sessionID) => { | ||
| 2141 | if (sessionID === "ses_child") requestOpened.resolve(undefined); | ||
| 2142 | }; | ||
| 2143 | childPermission.resolve({ | ||
| 2144 | id: "evt-child-ancestry-permission", | ||
| 2145 | type: "permission.asked", | ||
| 2146 | properties: permissionRequest("per_child_ancestry", "ses_child"), | ||
| 2147 | }); | ||
| 2148 | yield* Effect.promise(() => requestOpened.promise); | ||
| 2149 | yield* Effect.yieldNow; | ||
| 2150 | runtimeMock.state.sessionStatus = "idle"; | ||
| 2151 | idle.resolve({ | ||
| 2152 | id: "evt-child-ancestry-idle", | ||
| 2153 | type: "session.status", | ||
| 2154 | properties: { | ||
| 2155 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2156 | status: { type: "idle" }, | ||
| 2157 | }, | ||
| 2158 | }); | ||
| 2159 | |||
| 2160 | const events = Array.from(yield* Fiber.join(eventsFiber).pipe(Effect.timeout("1 second"))); | ||
| 2161 | NodeAssert.deepEqual( | ||
| 2162 | events.map((event) => event.type), | ||
| 2163 | ["request.opened", "turn.completed"], | ||
| 2164 | ); | ||
| 2165 | const completed = events[1]; | ||
| 2166 | if (completed?.type === "turn.completed") { | ||
| 2167 | NodeAssert.deepEqual(completed.payload.tokenUsage, { | ||
| 2168 | usageStatus: "unavailable", | ||
| 2169 | usageScope: "main_agent", | ||
| 2170 | hasSubagents: true, | ||
| 2171 | }); | ||
| 2172 | } | ||
| 2173 | yield* adapter.stopSession(threadId); | ||
| 2174 | }), | ||
| 2175 | ); | ||
| 2176 | |||
| 2177 | it.effect("sums owned OpenCode step usage and marks unresolved usage partial", () => | ||
| 2178 | Effect.gen(function* () { | ||
| 2179 | const adapter = yield* OpenCodeAdapter; | ||
| 2180 | const threadId = asThreadId("thread-step-usage"); | ||
| 2181 | const busy = promiseWithResolvers<unknown>(); | ||
| 2182 | const firstStep = promiseWithResolvers<unknown>(); | ||
| 2183 | const assistantMessage = promiseWithResolvers<unknown>(); | ||
| 2184 | const duplicateStep = promiseWithResolvers<unknown>(); | ||
| 2185 | const secondStep = promiseWithResolvers<unknown>(); | ||
| 2186 | const unresolvedHeader = promiseWithResolvers<unknown>(); | ||
| 2187 | const unresolvedStep = promiseWithResolvers<unknown>(); | ||
| 2188 | const recoveredStep = promiseWithResolvers<unknown>(); | ||
| 2189 | const recoveredIncompleteHeader = promiseWithResolvers<unknown>(); | ||
| 2190 | const recoveredCompleteHeader = promiseWithResolvers<unknown>(); | ||
| 2191 | const childSession = promiseWithResolvers<unknown>(); | ||
| 2192 | const idle = promiseWithResolvers<unknown>(); | ||
| 2193 | runtimeMock.state.subscribedEvents = [ | ||
| 2194 | busy.promise, | ||
| 2195 | firstStep.promise, | ||
| 2196 | assistantMessage.promise, | ||
| 2197 | duplicateStep.promise, | ||
| 2198 | secondStep.promise, | ||
| 2199 | unresolvedHeader.promise, | ||
| 2200 | unresolvedStep.promise, | ||
| 2201 | recoveredStep.promise, | ||
| 2202 | recoveredIncompleteHeader.promise, | ||
| 2203 | recoveredCompleteHeader.promise, | ||
| 2204 | childSession.promise, | ||
| 2205 | idle.promise, | ||
| 2206 | ]; | ||
| 2207 | |||
| 2208 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 2209 | Stream.filter((event) => event.threadId === threadId && event.type === "turn.completed"), | ||
| 2210 | Stream.runHead, | ||
| 2211 | Effect.forkChild, | ||
| 2212 | ); | ||
| 2213 | yield* adapter.startSession({ | ||
| 2214 | provider: ProviderDriverKind.make("opencode"), | ||
| 2215 | threadId, | ||
| 2216 | runtimeMode: "full-access", | ||
| 2217 | }); | ||
| 2218 | const sendFiber = yield* adapter | ||
| 2219 | .sendTurn({ | ||
| 2220 | threadId, | ||
| 2221 | input: "Use two model steps", | ||
| 2222 | modelSelection: createModelSelection( | ||
| 2223 | ProviderInstanceId.make("opencode"), | ||
| 2224 | "opencode/kimi-k3", | ||
| 2225 | ), | ||
| 2226 | }) | ||
| 2227 | .pipe(Effect.forkChild); | ||
| 2228 | busy.resolve({ | ||
| 2229 | id: "evt-step-usage-busy", | ||
| 2230 | type: "session.status", | ||
| 2231 | properties: { | ||
| 2232 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2233 | status: { type: "busy" }, | ||
| 2234 | }, | ||
| 2235 | }); | ||
| 2236 | yield* Fiber.join(sendFiber); | ||
| 2237 | const promptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string }).messageID; | ||
| 2238 | |||
| 2239 | const stepPart = { | ||
| 2240 | id: "step-usage-1", | ||
| 2241 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2242 | messageID: "assistant-step-usage-1", | ||
| 2243 | type: "step-finish", | ||
| 2244 | reason: "tool-calls", | ||
| 2245 | cost: 0, | ||
| 2246 | tokens: { | ||
| 2247 | input: 100, | ||
| 2248 | output: 20, | ||
| 2249 | reasoning: 5, | ||
| 2250 | cache: { read: 40, write: 10 }, | ||
| 2251 | }, | ||
| 2252 | } as const; | ||
| 2253 | firstStep.resolve({ | ||
| 2254 | id: "evt-step-usage-1", | ||
| 2255 | type: "message.part.updated", | ||
| 2256 | properties: { | ||
| 2257 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2258 | part: stepPart, | ||
| 2259 | }, | ||
| 2260 | }); | ||
| 2261 | assistantMessage.resolve({ | ||
| 2262 | id: "evt-step-usage-assistant", | ||
| 2263 | type: "message.updated", | ||
| 2264 | properties: { | ||
| 2265 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2266 | info: { | ||
| 2267 | id: "assistant-step-usage-1", | ||
| 2268 | role: "assistant", | ||
| 2269 | parentID: promptMessageId, | ||
| 2270 | }, | ||
| 2271 | }, | ||
| 2272 | }); | ||
| 2273 | duplicateStep.resolve({ | ||
| 2274 | id: "evt-step-usage-1-duplicate", | ||
| 2275 | type: "message.part.updated", | ||
| 2276 | properties: { | ||
| 2277 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2278 | part: stepPart, | ||
| 2279 | }, | ||
| 2280 | }); | ||
| 2281 | secondStep.resolve({ | ||
| 2282 | id: "evt-step-usage-2", | ||
| 2283 | type: "message.part.updated", | ||
| 2284 | properties: { | ||
| 2285 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2286 | part: { | ||
| 2287 | ...stepPart, | ||
| 2288 | id: "step-usage-2", | ||
| 2289 | tokens: { | ||
| 2290 | input: 50, | ||
| 2291 | output: 10, | ||
| 2292 | reasoning: 2, | ||
| 2293 | cache: { read: 10, write: 0 }, | ||
| 2294 | }, | ||
| 2295 | }, | ||
| 2296 | }, | ||
| 2297 | }); | ||
| 2298 | unresolvedHeader.resolve({ | ||
| 2299 | id: "evt-step-usage-unresolved-header", | ||
| 2300 | type: "message.updated", | ||
| 2301 | properties: { | ||
| 2302 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2303 | info: { | ||
| 2304 | id: "assistant-step-usage-without-parent", | ||
| 2305 | role: "assistant", | ||
| 2306 | }, | ||
| 2307 | }, | ||
| 2308 | }); | ||
| 2309 | unresolvedStep.resolve({ | ||
| 2310 | id: "evt-step-usage-unresolved", | ||
| 2311 | type: "message.part.updated", | ||
| 2312 | properties: { | ||
| 2313 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2314 | part: { | ||
| 2315 | ...stepPart, | ||
| 2316 | id: "step-usage-unresolved", | ||
| 2317 | messageID: "assistant-step-usage-without-parent", | ||
| 2318 | tokens: { | ||
| 2319 | input: 1_000, | ||
| 2320 | output: 1_000, | ||
| 2321 | reasoning: 0, | ||
| 2322 | cache: { read: 0, write: 0 }, | ||
| 2323 | }, | ||
| 2324 | }, | ||
| 2325 | }, | ||
| 2326 | }); | ||
| 2327 | recoveredStep.resolve({ | ||
| 2328 | id: "evt-step-usage-recovered", | ||
| 2329 | type: "message.part.updated", | ||
| 2330 | properties: { | ||
| 2331 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2332 | part: { | ||
| 2333 | ...stepPart, | ||
| 2334 | id: "step-usage-recovered", | ||
| 2335 | messageID: "assistant-step-usage-recovered", | ||
| 2336 | tokens: { | ||
| 2337 | input: 30, | ||
| 2338 | output: 10, | ||
| 2339 | reasoning: 2, | ||
| 2340 | cache: { read: 5, write: 1 }, | ||
| 2341 | }, | ||
| 2342 | }, | ||
| 2343 | }, | ||
| 2344 | }); | ||
| 2345 | recoveredIncompleteHeader.resolve({ | ||
| 2346 | id: "evt-step-usage-recovered-incomplete-header", | ||
| 2347 | type: "message.updated", | ||
| 2348 | properties: { | ||
| 2349 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2350 | info: { | ||
| 2351 | id: "assistant-step-usage-recovered", | ||
| 2352 | role: "assistant", | ||
| 2353 | parentID: "", | ||
| 2354 | }, | ||
| 2355 | }, | ||
| 2356 | }); | ||
| 2357 | recoveredCompleteHeader.resolve({ | ||
| 2358 | id: "evt-step-usage-recovered-complete-header", | ||
| 2359 | type: "message.updated", | ||
| 2360 | properties: { | ||
| 2361 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2362 | info: { | ||
| 2363 | id: "assistant-step-usage-recovered", | ||
| 2364 | role: "assistant", | ||
| 2365 | parentID: promptMessageId, | ||
| 2366 | }, | ||
| 2367 | }, | ||
| 2368 | }); | ||
| 2369 | childSession.resolve({ | ||
| 2370 | id: "evt-step-usage-child", | ||
| 2371 | type: "session.created", | ||
| 2372 | properties: { | ||
| 2373 | info: { | ||
| 2374 | id: "child-step-usage", | ||
| 2375 | parentID: "http://127.0.0.1:9999/session", | ||
| 2376 | }, | ||
| 2377 | }, | ||
| 2378 | }); | ||
| 2379 | idle.resolve({ | ||
| 2380 | id: "evt-step-usage-idle", | ||
| 2381 | type: "session.status", | ||
| 2382 | properties: { | ||
| 2383 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2384 | status: { type: "idle" }, | ||
| 2385 | }, | ||
| 2386 | }); | ||
| 2387 | |||
| 2388 | const completed = yield* Fiber.join(completedFiber).pipe(Effect.timeout("1 second")); | ||
| 2389 | NodeAssert.equal(completed._tag, "Some"); | ||
| 2390 | if (completed._tag === "Some" && completed.value.type === "turn.completed") { | ||
| 2391 | NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, { | ||
| 2392 | usageStatus: "partial", | ||
| 2393 | usageScope: "main_agent", | ||
| 2394 | inputTokens: 246, | ||
| 2395 | cachedInputTokens: 55, | ||
| 2396 | cacheCreationTokens: 11, | ||
| 2397 | outputTokens: 49, | ||
| 2398 | reasoningTokens: 9, | ||
| 2399 | hasSubagents: true, | ||
| 2400 | }); | ||
| 2401 | } | ||
| 2402 | |||
| 2403 | yield* adapter.stopSession(threadId); | ||
| 2404 | }), | ||
| 2405 | ); | ||
| 2406 | |||
| 2407 | it.effect("keeps the next turn usage while the prior completion is delayed", () => | ||
| 2408 | Effect.gen(function* () { | ||
| 2409 | const threadId = asThreadId("thread-token-usage-terminal-handoff"); | ||
| 2410 | const firstBusy = promiseWithResolvers<unknown>(); | ||
| 2411 | const firstAssistantMessage = promiseWithResolvers<unknown>(); | ||
| 2412 | const firstStep = promiseWithResolvers<unknown>(); | ||
| 2413 | const firstIdle = promiseWithResolvers<unknown>(); | ||
| 2414 | const secondBusy = promiseWithResolvers<unknown>(); | ||
| 2415 | const secondAssistantMessage = promiseWithResolvers<unknown>(); | ||
| 2416 | const secondStep = promiseWithResolvers<unknown>(); | ||
| 2417 | const secondIdle = promiseWithResolvers<unknown>(); | ||
| 2418 | runtimeMock.state.subscribedEvents = [ | ||
| 2419 | firstBusy.promise, | ||
| 2420 | firstAssistantMessage.promise, | ||
| 2421 | firstStep.promise, | ||
| 2422 | firstIdle.promise, | ||
| 2423 | secondBusy.promise, | ||
| 2424 | secondAssistantMessage.promise, | ||
| 2425 | secondStep.promise, | ||
| 2426 | secondIdle.promise, | ||
| 2427 | ]; | ||
| 2428 | |||
| 2429 | const firstStepWriteStarted = yield* Deferred.make<void>(); | ||
| 2430 | const firstStepWriteRelease = yield* Deferred.make<void>(); | ||
| 2431 | const terminalUuidStarted = yield* Deferred.make<void>(); | ||
| 2432 | const terminalUuidRelease = yield* Deferred.make<void>(); | ||
| 2433 | let blockFirstStepWrite = true; | ||
| 2434 | let blockNextUuid = false; | ||
| 2435 | const nodeCrypto = yield* Crypto.Crypto; | ||
| 2436 | const gatedCrypto = { | ||
| 2437 | ...nodeCrypto, | ||
| 2438 | randomUUIDv4: Effect.suspend(() => { | ||
| 2439 | if (!blockNextUuid) return nodeCrypto.randomUUIDv4; | ||
| 2440 | blockNextUuid = false; | ||
| 2441 | return Deferred.succeed(terminalUuidStarted, undefined).pipe( | ||
| 2442 | Effect.andThen(Deferred.await(terminalUuidRelease)), | ||
| 2443 | Effect.andThen(nodeCrypto.randomUUIDv4), | ||
| 2444 | ); | ||
| 2445 | }), | ||
| 2446 | } satisfies Crypto.Crypto; | ||
| 2447 | const adapter = yield* makeOpenCodeAdapter(openCodeAdapterTestSettings, { | ||
| 2448 | nativeEventLogger: { | ||
| 2449 | filePath: "memory://opencode-token-usage-terminal-handoff", | ||
| 2450 | write: (record) => { | ||
| 2451 | const eventType = (record as { event?: { type?: unknown } }).event?.type; | ||
| 2452 | if (blockFirstStepWrite && eventType === "message.part.updated") { | ||
| 2453 | blockFirstStepWrite = false; | ||
| 2454 | return Deferred.succeed(firstStepWriteStarted, undefined).pipe( | ||
| 2455 | Effect.andThen(Deferred.await(firstStepWriteRelease)), | ||
| 2456 | ); | ||
| 2457 | } | ||
| 2458 | return Effect.void; | ||
| 2459 | }, | ||
| 2460 | close: () => Effect.void, | ||
| 2461 | }, | ||
| 2462 | }).pipe(Effect.provideService(Crypto.Crypto, gatedCrypto)); | ||
| 2463 | |||
| 2464 | const completedFiber = yield* adapter.streamEvents.pipe( | ||
| 2465 | Stream.filter((event) => event.threadId === threadId && event.type === "turn.completed"), | ||
| 2466 | Stream.take(2), | ||
| 2467 | Stream.runCollect, | ||
| 2468 | Effect.forkChild, | ||
| 2469 | ); | ||
| 2470 | yield* adapter.startSession({ | ||
| 2471 | provider: ProviderDriverKind.make("opencode"), | ||
| 2472 | threadId, | ||
| 2473 | runtimeMode: "full-access", | ||
| 2474 | }); | ||
| 2475 | |||
| 2476 | const firstSend = yield* adapter | ||
| 2477 | .sendTurn({ | ||
| 2478 | threadId, | ||
| 2479 | input: "Run the first token handoff turn", | ||
| 2480 | modelSelection: createModelSelection( | ||
| 2481 | ProviderInstanceId.make("opencode"), | ||
| 2482 | "opencode/kimi-k3", | ||
| 2483 | ), | ||
| 2484 | }) | ||
| 2485 | .pipe(Effect.forkChild); | ||
| 2486 | firstBusy.resolve({ | ||
| 2487 | id: "evt-token-handoff-first-busy", | ||
| 2488 | type: "session.status", | ||
| 2489 | properties: { | ||
| 2490 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2491 | status: { type: "busy" }, | ||
| 2492 | }, | ||
| 2493 | }); | ||
| 2494 | const firstTurn = yield* Fiber.join(firstSend); | ||
| 2495 | const firstPromptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string }) | ||
| 2496 | .messageID; | ||
| 2497 | firstAssistantMessage.resolve({ | ||
| 2498 | id: "evt-token-handoff-first-assistant", | ||
| 2499 | type: "message.updated", | ||
| 2500 | properties: { | ||
| 2501 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2502 | info: { | ||
| 2503 | id: "assistant-token-handoff-first", | ||
| 2504 | role: "assistant", | ||
| 2505 | parentID: firstPromptMessageId, | ||
| 2506 | }, | ||
| 2507 | }, | ||
| 2508 | }); | ||
| 2509 | firstStep.resolve({ | ||
| 2510 | id: "evt-token-handoff-first-step", | ||
| 2511 | type: "message.part.updated", | ||
| 2512 | properties: { | ||
| 2513 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2514 | part: { | ||
| 2515 | id: "step-token-handoff-first", | ||
| 2516 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2517 | messageID: "assistant-token-handoff-first", | ||
| 2518 | type: "step-finish", | ||
| 2519 | reason: "stop", | ||
| 2520 | cost: 0, | ||
| 2521 | tokens: { | ||
| 2522 | input: 100, | ||
| 2523 | output: 20, | ||
| 2524 | reasoning: 5, | ||
| 2525 | cache: { read: 40, write: 10 }, | ||
| 2526 | }, | ||
| 2527 | }, | ||
| 2528 | }, | ||
| 2529 | }); | ||
| 2530 | yield* Deferred.await(firstStepWriteStarted); | ||
| 2531 | blockNextUuid = true; | ||
| 2532 | firstIdle.resolve({ | ||
| 2533 | id: "evt-token-handoff-first-idle", | ||
| 2534 | type: "session.status", | ||
| 2535 | properties: { | ||
| 2536 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2537 | status: { type: "idle" }, | ||
| 2538 | }, | ||
| 2539 | }); | ||
| 2540 | yield* Deferred.succeed(firstStepWriteRelease, undefined); | ||
| 2541 | yield* Deferred.await(terminalUuidStarted); | ||
| 2542 | |||
| 2543 | runtimeMock.state.sessionStatus = "busy"; | ||
| 2544 | const secondTurn = yield* adapter.sendTurn({ | ||
| 2545 | threadId, | ||
| 2546 | input: "Run the second token handoff turn", | ||
| 2547 | modelSelection: createModelSelection( | ||
| 2548 | ProviderInstanceId.make("opencode"), | ||
| 2549 | "opencode/kimi-k3", | ||
| 2550 | ), | ||
| 2551 | }); | ||
| 2552 | NodeAssert.notEqual(secondTurn.turnId, firstTurn.turnId); | ||
| 2553 | const secondPromptMessageId = (runtimeMock.state.promptCalls[1] as { messageID: string }) | ||
| 2554 | .messageID; | ||
| 2555 | yield* Deferred.succeed(terminalUuidRelease, undefined); | ||
| 2556 | |||
| 2557 | secondBusy.resolve({ | ||
| 2558 | id: "evt-token-handoff-second-busy", | ||
| 2559 | type: "session.status", | ||
| 2560 | properties: { | ||
| 2561 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2562 | status: { type: "busy" }, | ||
| 2563 | }, | ||
| 2564 | }); | ||
| 2565 | secondAssistantMessage.resolve({ | ||
| 2566 | id: "evt-token-handoff-second-assistant", | ||
| 2567 | type: "message.updated", | ||
| 2568 | properties: { | ||
| 2569 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2570 | info: { | ||
| 2571 | id: "assistant-token-handoff-second", | ||
| 2572 | role: "assistant", | ||
| 2573 | parentID: secondPromptMessageId, | ||
| 2574 | }, | ||
| 2575 | }, | ||
| 2576 | }); | ||
| 2577 | secondStep.resolve({ | ||
| 2578 | id: "evt-token-handoff-second-step", | ||
| 2579 | type: "message.part.updated", | ||
| 2580 | properties: { | ||
| 2581 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2582 | part: { | ||
| 2583 | id: "step-token-handoff-second", | ||
| 2584 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2585 | messageID: "assistant-token-handoff-second", | ||
| 2586 | type: "step-finish", | ||
| 2587 | reason: "stop", | ||
| 2588 | cost: 0, | ||
| 2589 | tokens: { | ||
| 2590 | input: 90, | ||
| 2591 | output: 13, | ||
| 2592 | reasoning: 3, | ||
| 2593 | cache: { read: 20, write: 5 }, | ||
| 2594 | }, | ||
| 2595 | }, | ||
| 2596 | }, | ||
| 2597 | }); | ||
| 2598 | secondIdle.resolve({ | ||
| 2599 | id: "evt-token-handoff-second-idle", | ||
| 2600 | type: "session.status", | ||
| 2601 | properties: { | ||
| 2602 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2603 | status: { type: "idle" }, | ||
| 2604 | }, | ||
| 2605 | }); | ||
| 2606 | |||
| 2607 | const completed = Array.from( | ||
| 2608 | yield* Fiber.join(completedFiber).pipe(Effect.timeout("1 second")), | ||
| 2609 | ); | ||
| 2610 | NodeAssert.deepStrictEqual( | ||
| 2611 | completed.map((event) => | ||
| 2612 | event.type === "turn.completed" ? event.payload.tokenUsage : undefined, | ||
| 2613 | ), | ||
| 2614 | [ | ||
| 2615 | { | ||
| 2616 | usageStatus: "complete", | ||
| 2617 | usageScope: "main_agent", | ||
| 2618 | inputTokens: 150, | ||
| 2619 | cachedInputTokens: 40, | ||
| 2620 | cacheCreationTokens: 10, | ||
| 2621 | outputTokens: 25, | ||
| 2622 | reasoningTokens: 5, | ||
| 2623 | hasSubagents: false, | ||
| 2624 | }, | ||
| 2625 | { | ||
| 2626 | usageStatus: "complete", | ||
| 2627 | usageScope: "main_agent", | ||
| 2628 | inputTokens: 115, | ||
| 2629 | cachedInputTokens: 20, | ||
| 2630 | cacheCreationTokens: 5, | ||
| 2631 | outputTokens: 16, | ||
| 2632 | reasoningTokens: 3, | ||
| 2633 | hasSubagents: false, | ||
| 2634 | }, | ||
| 2635 | ], | ||
| 2636 | ); | ||
| 2637 | |||
| 2638 | yield* adapter.stopSession(threadId); | ||
| 2639 | }), | ||
| 2640 | ); | ||
| 2641 | |||
| 2089 | it.effect("ignores a stale admission status response after the next turn starts", () => | 2642 | it.effect("ignores a stale admission status response after the next turn starts", () => |
| 2090 | Effect.gen(function* () { | 2643 | Effect.gen(function* () { |
| 2091 | const adapter = yield* OpenCodeAdapter; | 2644 | const adapter = yield* OpenCodeAdapter; |
| ... | @@ -2203,6 +2756,188 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { | ... | @@ -2203,6 +2756,188 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { |
| 2203 | }), | 2756 | }), |
| 2204 | ); | 2757 | ); |
| 2205 | 2758 | ||
| 2759 | it.effect("ignores a late prior-turn step after the next prompt is admitted", () => | ||
| 2760 | Effect.gen(function* () { | ||
| 2761 | const adapter = yield* OpenCodeAdapter; | ||
| 2762 | const threadId = asThreadId("thread-token-usage-late-prior-step"); | ||
| 2763 | const firstBusy = promiseWithResolvers<unknown>(); | ||
| 2764 | const firstAssistant = promiseWithResolvers<unknown>(); | ||
| 2765 | const secondBusy = promiseWithResolvers<unknown>(); | ||
| 2766 | const secondAssistant = promiseWithResolvers<unknown>(); | ||
| 2767 | const lateFirstStep = promiseWithResolvers<unknown>(); | ||
| 2768 | const secondStep = promiseWithResolvers<unknown>(); | ||
| 2769 | const secondIdle = promiseWithResolvers<unknown>(); | ||
| 2770 | runtimeMock.state.subscribedEvents = [ | ||
| 2771 | firstBusy.promise, | ||
| 2772 | firstAssistant.promise, | ||
| 2773 | secondBusy.promise, | ||
| 2774 | secondAssistant.promise, | ||
| 2775 | lateFirstStep.promise, | ||
| 2776 | secondStep.promise, | ||
| 2777 | secondIdle.promise, | ||
| 2778 | ]; | ||
| 2779 | |||
| 2780 | const terminalsFiber = yield* adapter.streamEvents.pipe( | ||
| 2781 | Stream.filter( | ||
| 2782 | (event) => | ||
| 2783 | event.threadId === threadId && | ||
| 2784 | (event.type === "turn.aborted" || event.type === "turn.completed"), | ||
| 2785 | ), | ||
| 2786 | Stream.take(2), | ||
| 2787 | Stream.runCollect, | ||
| 2788 | Effect.forkChild, | ||
| 2789 | ); | ||
| 2790 | yield* adapter.startSession({ | ||
| 2791 | provider: ProviderDriverKind.make("opencode"), | ||
| 2792 | threadId, | ||
| 2793 | runtimeMode: "full-access", | ||
| 2794 | }); | ||
| 2795 | |||
| 2796 | const firstSend = yield* adapter | ||
| 2797 | .sendTurn({ | ||
| 2798 | threadId, | ||
| 2799 | input: "Start the interrupted turn", | ||
| 2800 | modelSelection: createModelSelection( | ||
| 2801 | ProviderInstanceId.make("opencode"), | ||
| 2802 | "opencode/kimi-k3", | ||
| 2803 | ), | ||
| 2804 | }) | ||
| 2805 | .pipe(Effect.forkChild); | ||
| 2806 | firstBusy.resolve({ | ||
| 2807 | id: "evt-token-late-first-busy", | ||
| 2808 | type: "session.status", | ||
| 2809 | properties: { | ||
| 2810 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2811 | status: { type: "busy" }, | ||
| 2812 | }, | ||
| 2813 | }); | ||
| 2814 | yield* Fiber.join(firstSend); | ||
| 2815 | const firstPromptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string }) | ||
| 2816 | .messageID; | ||
| 2817 | firstAssistant.resolve({ | ||
| 2818 | id: "evt-token-late-first-assistant", | ||
| 2819 | type: "message.updated", | ||
| 2820 | properties: { | ||
| 2821 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2822 | info: { | ||
| 2823 | id: "assistant-token-late-first", | ||
| 2824 | role: "assistant", | ||
| 2825 | parentID: firstPromptMessageId, | ||
| 2826 | }, | ||
| 2827 | }, | ||
| 2828 | }); | ||
| 2829 | yield* Effect.yieldNow; | ||
| 2830 | yield* adapter.interruptTurn(threadId); | ||
| 2831 | |||
| 2832 | const secondSend = yield* adapter | ||
| 2833 | .sendTurn({ | ||
| 2834 | threadId, | ||
| 2835 | input: "Start the next turn", | ||
| 2836 | modelSelection: createModelSelection( | ||
| 2837 | ProviderInstanceId.make("opencode"), | ||
| 2838 | "opencode/kimi-k3", | ||
| 2839 | ), | ||
| 2840 | }) | ||
| 2841 | .pipe(Effect.forkChild); | ||
| 2842 | while (runtimeMock.state.promptCalls.length < 2) { | ||
| 2843 | yield* Effect.yieldNow; | ||
| 2844 | } | ||
| 2845 | secondBusy.resolve({ | ||
| 2846 | id: "evt-token-late-second-busy", | ||
| 2847 | type: "session.status", | ||
| 2848 | properties: { | ||
| 2849 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2850 | status: { type: "busy" }, | ||
| 2851 | }, | ||
| 2852 | }); | ||
| 2853 | yield* Fiber.join(secondSend); | ||
| 2854 | const secondPromptMessageId = (runtimeMock.state.promptCalls[1] as { messageID: string }) | ||
| 2855 | .messageID; | ||
| 2856 | secondAssistant.resolve({ | ||
| 2857 | id: "evt-token-late-second-assistant", | ||
| 2858 | type: "message.updated", | ||
| 2859 | properties: { | ||
| 2860 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2861 | info: { | ||
| 2862 | id: "assistant-token-late-second", | ||
| 2863 | role: "assistant", | ||
| 2864 | parentID: secondPromptMessageId, | ||
| 2865 | }, | ||
| 2866 | }, | ||
| 2867 | }); | ||
| 2868 | lateFirstStep.resolve({ | ||
| 2869 | id: "evt-token-late-first-step", | ||
| 2870 | type: "message.part.updated", | ||
| 2871 | properties: { | ||
| 2872 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2873 | part: { | ||
| 2874 | id: "step-token-late-first", | ||
| 2875 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2876 | messageID: "assistant-token-late-first", | ||
| 2877 | type: "step-finish", | ||
| 2878 | reason: "stop", | ||
| 2879 | cost: 0, | ||
| 2880 | tokens: { | ||
| 2881 | input: 1_000, | ||
| 2882 | output: 1_000, | ||
| 2883 | reasoning: 0, | ||
| 2884 | cache: { read: 0, write: 0 }, | ||
| 2885 | }, | ||
| 2886 | }, | ||
| 2887 | }, | ||
| 2888 | }); | ||
| 2889 | secondStep.resolve({ | ||
| 2890 | id: "evt-token-late-second-step", | ||
| 2891 | type: "message.part.updated", | ||
| 2892 | properties: { | ||
| 2893 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2894 | part: { | ||
| 2895 | id: "step-token-late-second", | ||
| 2896 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2897 | messageID: "assistant-token-late-second", | ||
| 2898 | type: "step-finish", | ||
| 2899 | reason: "stop", | ||
| 2900 | cost: 0, | ||
| 2901 | tokens: { | ||
| 2902 | input: 40, | ||
| 2903 | output: 10, | ||
| 2904 | reasoning: 2, | ||
| 2905 | cache: { read: 5, write: 1 }, | ||
| 2906 | }, | ||
| 2907 | }, | ||
| 2908 | }, | ||
| 2909 | }); | ||
| 2910 | secondIdle.resolve({ | ||
| 2911 | id: "evt-token-late-second-idle", | ||
| 2912 | type: "session.status", | ||
| 2913 | properties: { | ||
| 2914 | sessionID: "http://127.0.0.1:9999/session", | ||
| 2915 | status: { type: "idle" }, | ||
| 2916 | }, | ||
| 2917 | }); | ||
| 2918 | |||
| 2919 | const terminals = Array.from( | ||
| 2920 | yield* Fiber.join(terminalsFiber).pipe(Effect.timeout("1 second")), | ||
| 2921 | ); | ||
| 2922 | const secondCompleted = terminals.find((event) => event.type === "turn.completed"); | ||
| 2923 | NodeAssert.equal(secondCompleted?.type, "turn.completed"); | ||
| 2924 | if (secondCompleted?.type === "turn.completed") { | ||
| 2925 | NodeAssert.deepStrictEqual(secondCompleted.payload.tokenUsage, { | ||
| 2926 | usageStatus: "complete", | ||
| 2927 | usageScope: "main_agent", | ||
| 2928 | inputTokens: 46, | ||
| 2929 | cachedInputTokens: 5, | ||
| 2930 | cacheCreationTokens: 1, | ||
| 2931 | outputTokens: 12, | ||
| 2932 | reasoningTokens: 2, | ||
| 2933 | hasSubagents: false, | ||
| 2934 | }); | ||
| 2935 | } | ||
| 2936 | |||
| 2937 | yield* adapter.stopSession(threadId); | ||
| 2938 | }), | ||
| 2939 | ); | ||
| 2940 | |||
| 2206 | it.effect("reconciles a sole idle when the matching prompt echo arrives later", () => | 2941 | it.effect("reconciles a sole idle when the matching prompt echo arrives later", () => |
| 2207 | Effect.gen(function* () { | 2942 | Effect.gen(function* () { |
| 2208 | const adapter = yield* OpenCodeAdapter; | 2943 | const adapter = yield* OpenCodeAdapter; |
| ... | @@ -5432,6 +6167,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { | ... | @@ -5432,6 +6167,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { |
| 5432 | 6167 | ||
| 5433 | return Effect.gen(function* () { | 6168 | return Effect.gen(function* () { |
| 5434 | const adapter = yield* OpenCodeAdapter; | 6169 | const adapter = yield* OpenCodeAdapter; |
| 6170 | const startedFiber = yield* adapter.streamEvents.pipe( | ||
| 6171 | Stream.filter((event) => event.type === "turn.started"), | ||
| 6172 | Stream.runHead, | ||
| 6173 | Effect.forkChild, | ||
| 6174 | ); | ||
| 5435 | yield* adapter.startSession({ | 6175 | yield* adapter.startSession({ |
| 5436 | provider: ProviderDriverKind.make("opencode"), | 6176 | provider: ProviderDriverKind.make("opencode"), |
| 5437 | threadId: asThreadId("thread-custom-instance"), | 6177 | threadId: asThreadId("thread-custom-instance"), |
| ... | @@ -5470,6 +6210,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { | ... | @@ -5470,6 +6210,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { |
| 5470 | }), | 6210 | }), |
| 5471 | parts: [{ type: "text", text: "Fix it" }], | 6211 | parts: [{ type: "text", text: "Fix it" }], |
| 5472 | }); | 6212 | }); |
| 6213 | const started = yield* Fiber.join(startedFiber); | ||
| 6214 | NodeAssert.equal(started._tag, "Some"); | ||
| 6215 | if (started._tag === "Some" && started.value.type === "turn.started") { | ||
| 6216 | NodeAssert.equal(started.value.payload.effort, undefined); | ||
| 6217 | } | ||
| 5473 | }).pipe(Effect.provide(adapterLayer)); | 6218 | }).pipe(Effect.provide(adapterLayer)); |
| 5474 | }); | 6219 | }); |
| 5475 | 6220 |
apps/server/src/provider/Layers/OpenCodeAdapter.ts+152-4
| ... | @@ -9,6 +9,7 @@ import { | ... | @@ -9,6 +9,7 @@ import { |
| 9 | RuntimeRequestId, | 9 | RuntimeRequestId, |
| 10 | ThreadId, | 10 | ThreadId, |
| 11 | type ToolLifecycleItemType, | 11 | type ToolLifecycleItemType, |
| 12 | type TurnTokenUsage, | ||
| 12 | TurnId, | 13 | TurnId, |
| 13 | type UserInputQuestion, | 14 | type UserInputQuestion, |
| 14 | } from "@t3tools/contracts"; | 15 | } from "@t3tools/contracts"; |
| ... | @@ -338,6 +339,7 @@ interface OpenCodeSessionContext { | ... | @@ -338,6 +339,7 @@ interface OpenCodeSessionContext { |
| 338 | readonly partById: Map<string, Part>; | 339 | readonly partById: Map<string, Part>; |
| 339 | readonly emittedTextByPartId: Map<string, string>; | 340 | readonly emittedTextByPartId: Map<string, string>; |
| 340 | readonly completedAssistantPartIds: Set<string>; | 341 | readonly completedAssistantPartIds: Set<string>; |
| 342 | turnTokenUsage: OpenCodeTurnTokenUsageAccumulator | undefined; | ||
| 341 | activeTurnId: TurnId | undefined; | 343 | activeTurnId: TurnId | undefined; |
| 342 | activeAgent: string | undefined; | 344 | activeAgent: string | undefined; |
| 343 | activeVariant: string | undefined; | 345 | activeVariant: string | undefined; |
| ... | @@ -368,6 +370,75 @@ interface OpenCodeSessionContext { | ... | @@ -368,6 +370,75 @@ interface OpenCodeSessionContext { |
| 368 | readonly sessionScope: Scope.Closeable; | 370 | readonly sessionScope: Scope.Closeable; |
| 369 | } | 371 | } |
| 370 | 372 | ||
| 373 | interface OpenCodeTurnTokenUsageAccumulator { | ||
| 374 | readonly partIds: Set<string>; | ||
| 375 | readonly promptMessageIds: Set<string>; | ||
| 376 | readonly assistantOwnershipByMessageId: Map<string, "owned" | "other" | "unknown">; | ||
| 377 | readonly unresolvedStepPartIds: Set<string>; | ||
| 378 | inputTokens: number; | ||
| 379 | cachedInputTokens: number; | ||
| 380 | cacheCreationTokens: number; | ||
| 381 | outputTokens: number; | ||
| 382 | reasoningTokens: number; | ||
| 383 | complete: boolean; | ||
| 384 | hasSubagents: boolean; | ||
| 385 | } | ||
| 386 | |||
| 387 | function makeOpenCodeTurnTokenUsageAccumulator(): OpenCodeTurnTokenUsageAccumulator { | ||
| 388 | return { | ||
| 389 | partIds: new Set(), | ||
| 390 | promptMessageIds: new Set(), | ||
| 391 | assistantOwnershipByMessageId: new Map(), | ||
| 392 | unresolvedStepPartIds: new Set(), | ||
| 393 | inputTokens: 0, | ||
| 394 | cachedInputTokens: 0, | ||
| 395 | cacheCreationTokens: 0, | ||
| 396 | outputTokens: 0, | ||
| 397 | reasoningTokens: 0, | ||
| 398 | complete: true, | ||
| 399 | hasSubagents: false, | ||
| 400 | }; | ||
| 401 | } | ||
| 402 | |||
| 403 | function accumulateOpenCodeStepUsage( | ||
| 404 | accumulator: OpenCodeTurnTokenUsageAccumulator, | ||
| 405 | part: Extract<Part, { readonly type: "step-finish" }>, | ||
| 406 | ): void { | ||
| 407 | if (accumulator.partIds.has(part.id)) return; | ||
| 408 | accumulator.partIds.add(part.id); | ||
| 409 | accumulator.inputTokens += part.tokens.input + part.tokens.cache.read + part.tokens.cache.write; | ||
| 410 | accumulator.cachedInputTokens += part.tokens.cache.read; | ||
| 411 | accumulator.cacheCreationTokens += part.tokens.cache.write; | ||
| 412 | accumulator.outputTokens += part.tokens.output + part.tokens.reasoning; | ||
| 413 | accumulator.reasoningTokens += part.tokens.reasoning; | ||
| 414 | } | ||
| 415 | |||
| 416 | function takeOpenCodeTurnTokenUsage( | ||
| 417 | context: OpenCodeSessionContext, | ||
| 418 | complete: boolean, | ||
| 419 | ): TurnTokenUsage { | ||
| 420 | const usage = context.turnTokenUsage; | ||
| 421 | context.turnTokenUsage = undefined; | ||
| 422 | if (!usage || usage.partIds.size === 0) { | ||
| 423 | return { | ||
| 424 | usageStatus: "unavailable", | ||
| 425 | usageScope: "main_agent", | ||
| 426 | hasSubagents: usage?.hasSubagents ?? false, | ||
| 427 | }; | ||
| 428 | } | ||
| 429 | return { | ||
| 430 | usageStatus: | ||
| 431 | complete && usage.complete && usage.unresolvedStepPartIds.size === 0 ? "complete" : "partial", | ||
| 432 | usageScope: "main_agent", | ||
| 433 | inputTokens: usage.inputTokens, | ||
| 434 | cachedInputTokens: usage.cachedInputTokens, | ||
| 435 | cacheCreationTokens: usage.cacheCreationTokens, | ||
| 436 | outputTokens: usage.outputTokens, | ||
| 437 | reasoningTokens: Math.min(usage.outputTokens, usage.reasoningTokens), | ||
| 438 | hasSubagents: usage.hasSubagents, | ||
| 439 | }; | ||
| 440 | } | ||
| 441 | |||
| 371 | export interface OpenCodeAdapterLiveOptions { | 442 | export interface OpenCodeAdapterLiveOptions { |
| 372 | readonly instanceId?: ProviderInstanceId; | 443 | readonly instanceId?: ProviderInstanceId; |
| 373 | readonly environment?: NodeJS.ProcessEnv; | 444 | readonly environment?: NodeJS.ProcessEnv; |
| ... | @@ -1032,6 +1103,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1032,6 +1103,7 @@ export function makeOpenCodeAdapter( |
| 1032 | ) { | 1103 | ) { |
| 1033 | context.pendingIdleReconciliation = undefined; | 1104 | context.pendingIdleReconciliation = undefined; |
| 1034 | } | 1105 | } |
| 1106 | const tokenUsage = takeOpenCodeTurnTokenUsage(context, true); | ||
| 1035 | context.activeTurnId = undefined; | 1107 | context.activeTurnId = undefined; |
| 1036 | context.activeAgent = undefined; | 1108 | context.activeAgent = undefined; |
| 1037 | context.activeVariant = undefined; | 1109 | context.activeVariant = undefined; |
| ... | @@ -1061,6 +1133,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1061,6 +1133,7 @@ export function makeOpenCodeAdapter( |
| 1061 | type: "turn.completed", | 1133 | type: "turn.completed", |
| 1062 | payload: { | 1134 | payload: { |
| 1063 | state: "completed", | 1135 | state: "completed", |
| 1136 | tokenUsage, | ||
| 1064 | }, | 1137 | }, |
| 1065 | }); | 1138 | }); |
| 1066 | }); | 1139 | }); |
| ... | @@ -1198,6 +1271,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1198,6 +1271,7 @@ export function makeOpenCodeAdapter( |
| 1198 | deleteContextIfCurrent(context); | 1271 | deleteContextIfCurrent(context); |
| 1199 | return; | 1272 | return; |
| 1200 | } | 1273 | } |
| 1274 | const tokenUsage = takeOpenCodeTurnTokenUsage(context, false); | ||
| 1201 | context.promptAdmission = undefined; | 1275 | context.promptAdmission = undefined; |
| 1202 | context.activeTurnId = undefined; | 1276 | context.activeTurnId = undefined; |
| 1203 | context.activeAgent = undefined; | 1277 | context.activeAgent = undefined; |
| ... | @@ -1219,6 +1293,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1219,6 +1293,7 @@ export function makeOpenCodeAdapter( |
| 1219 | payload: { | 1293 | payload: { |
| 1220 | state: "failed", | 1294 | state: "failed", |
| 1221 | errorMessage: detail, | 1295 | errorMessage: detail, |
| 1296 | tokenUsage, | ||
| 1222 | }, | 1297 | }, |
| 1223 | }); | 1298 | }); |
| 1224 | yield* emit({ | 1299 | yield* emit({ |
| ... | @@ -1396,7 +1471,13 @@ export function makeOpenCodeAdapter( | ... | @@ -1396,7 +1471,13 @@ export function makeOpenCodeAdapter( |
| 1396 | if (cancellation) { | 1471 | if (cancellation) { |
| 1397 | context.cancellation = undefined; | 1472 | context.cancellation = undefined; |
| 1398 | } | 1473 | } |
| 1474 | let tokenUsage: TurnTokenUsage = { | ||
| 1475 | usageStatus: "unavailable", | ||
| 1476 | usageScope: "main_agent", | ||
| 1477 | hasSubagents: false, | ||
| 1478 | }; | ||
| 1399 | if (context.activeTurnId === turnId) { | 1479 | if (context.activeTurnId === turnId) { |
| 1480 | tokenUsage = takeOpenCodeTurnTokenUsage(context, false); | ||
| 1400 | context.activeTurnId = undefined; | 1481 | context.activeTurnId = undefined; |
| 1401 | context.activeAgent = undefined; | 1482 | context.activeAgent = undefined; |
| 1402 | context.activeVariant = undefined; | 1483 | context.activeVariant = undefined; |
| ... | @@ -1416,6 +1497,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1416,6 +1497,7 @@ export function makeOpenCodeAdapter( |
| 1416 | type: "turn.aborted", | 1497 | type: "turn.aborted", |
| 1417 | payload: { | 1498 | payload: { |
| 1418 | reason: "Interrupted by user.", | 1499 | reason: "Interrupted by user.", |
| 1500 | tokenUsage, | ||
| 1419 | }, | 1501 | }, |
| 1420 | }); | 1502 | }); |
| 1421 | if (cancellation) { | 1503 | if (cancellation) { |
| ... | @@ -1550,6 +1632,16 @@ export function makeOpenCodeAdapter( | ... | @@ -1550,6 +1632,16 @@ export function makeOpenCodeAdapter( |
| 1550 | } | 1632 | } |
| 1551 | }); | 1633 | }); |
| 1552 | 1634 | ||
| 1635 | // Records a child session of this thread. A child seen during a live turn | ||
| 1636 | // means that turn used subagents, whether the relation came from a | ||
| 1637 | // `session.created` event or a later ancestry lookup after reconnect. | ||
| 1638 | const addRelatedOpenCodeSession = (context: OpenCodeSessionContext, sessionId: string) => { | ||
| 1639 | context.relatedSessionIds.add(sessionId); | ||
| 1640 | if (context.activeTurnId && context.turnTokenUsage) { | ||
| 1641 | context.turnTokenUsage.hasSubagents = true; | ||
| 1642 | } | ||
| 1643 | }; | ||
| 1644 | |||
| 1553 | const isRelatedOpenCodeSession = Effect.fn("isRelatedOpenCodeSession")(function* ( | 1645 | const isRelatedOpenCodeSession = Effect.fn("isRelatedOpenCodeSession")(function* ( |
| 1554 | context: OpenCodeSessionContext, | 1646 | context: OpenCodeSessionContext, |
| 1555 | candidateSessionId: string, | 1647 | candidateSessionId: string, |
| ... | @@ -1581,7 +1673,7 @@ export function makeOpenCodeAdapter( | ... | @@ -1581,7 +1673,7 @@ export function makeOpenCodeAdapter( |
| 1581 | let sessionId: string | undefined = candidateSessionId; | 1673 | let sessionId: string | undefined = candidateSessionId; |
| 1582 | for (let depth = 0; sessionId !== undefined && depth < 32; depth += 1) { | 1674 | for (let depth = 0; sessionId !== undefined && depth < 32; depth += 1) { |
| 1583 | if (context.relatedSessionIds.has(sessionId)) { | 1675 | if (context.relatedSessionIds.has(sessionId)) { |
| 1584 | context.relatedSessionIds.add(candidateSessionId); | 1676 | addRelatedOpenCodeSession(context, candidateSessionId); |
| 1585 | return true; | 1677 | return true; |
| 1586 | } | 1678 | } |
| 1587 | if (seen.has(sessionId)) { | 1679 | if (seen.has(sessionId)) { |
| ... | @@ -2078,6 +2170,9 @@ export function makeOpenCodeAdapter( | ... | @@ -2078,6 +2170,9 @@ export function makeOpenCodeAdapter( |
| 2078 | } | 2170 | } |
| 2079 | yield* schedulePendingRequestRecovery(context); | 2171 | yield* schedulePendingRequestRecovery(context); |
| 2080 | if (!isFirstConnection) { | 2172 | if (!isFirstConnection) { |
| 2173 | if (context.turnTokenUsage) { | ||
| 2174 | context.turnTokenUsage.complete = false; | ||
| 2175 | } | ||
| 2081 | yield* schedulePromptAdmissionRecovery(context, event); | 2176 | yield* schedulePromptAdmissionRecovery(context, event); |
| 2082 | if (context.activeTurnId !== undefined && context.promptAdmission === undefined) { | 2177 | if (context.activeTurnId !== undefined && context.promptAdmission === undefined) { |
| 2083 | yield* scheduleIdleReconciliation(context, context.activeTurnId, event); | 2178 | yield* scheduleIdleReconciliation(context, context.activeTurnId, event); |
| ... | @@ -2097,7 +2192,7 @@ export function makeOpenCodeAdapter( | ... | @@ -2097,7 +2192,7 @@ export function makeOpenCodeAdapter( |
| 2097 | if (event.type === "session.created" || event.type === "session.updated") { | 2192 | if (event.type === "session.created" || event.type === "session.updated") { |
| 2098 | const session = event.properties.info; | 2193 | const session = event.properties.info; |
| 2099 | if (session.parentID && context.relatedSessionIds.has(session.parentID)) { | 2194 | if (session.parentID && context.relatedSessionIds.has(session.parentID)) { |
| 2100 | context.relatedSessionIds.add(session.id); | 2195 | addRelatedOpenCodeSession(context, session.id); |
| 2101 | } | 2196 | } |
| 2102 | } else if (event.type === "session.deleted") { | 2197 | } else if (event.type === "session.deleted") { |
| 2103 | context.relatedSessionIds.delete(event.properties.info.id); | 2198 | context.relatedSessionIds.delete(event.properties.info.id); |
| ... | @@ -2221,10 +2316,36 @@ export function makeOpenCodeAdapter( | ... | @@ -2221,10 +2316,36 @@ export function makeOpenCodeAdapter( |
| 2221 | } | 2316 | } |
| 2222 | context.messageRoleById.set(event.properties.info.id, event.properties.info.role); | 2317 | context.messageRoleById.set(event.properties.info.id, event.properties.info.role); |
| 2223 | if (event.properties.info.role === "assistant") { | 2318 | if (event.properties.info.role === "assistant") { |
| 2319 | const usage = context.turnTokenUsage; | ||
| 2320 | const parentMessageId = | ||
| 2321 | typeof event.properties.info.parentID === "string" && | ||
| 2322 | event.properties.info.parentID.trim().length > 0 | ||
| 2323 | ? event.properties.info.parentID | ||
| 2324 | : undefined; | ||
| 2325 | const observedOwnership = | ||
| 2326 | parentMessageId === undefined | ||
| 2327 | ? "unknown" | ||
| 2328 | : usage?.promptMessageIds.has(parentMessageId) | ||
| 2329 | ? "owned" | ||
| 2330 | : "other"; | ||
| 2331 | const priorOwnership = usage?.assistantOwnershipByMessageId.get( | ||
| 2332 | event.properties.info.id, | ||
| 2333 | ); | ||
| 2334 | const ownership = | ||
| 2335 | priorOwnership === undefined || priorOwnership === "unknown" | ||
| 2336 | ? observedOwnership | ||
| 2337 | : priorOwnership; | ||
| 2338 | if (usage) { | ||
| 2339 | usage.assistantOwnershipByMessageId.set(event.properties.info.id, ownership); | ||
| 2340 | } | ||
| 2224 | for (const part of context.partById.values()) { | 2341 | for (const part of context.partById.values()) { |
| 2225 | if (part.messageID !== event.properties.info.id) { | 2342 | if (part.messageID !== event.properties.info.id) { |
| 2226 | continue; | 2343 | continue; |
| 2227 | } | 2344 | } |
| 2345 | if (usage && part.type === "step-finish") { | ||
| 2346 | if (ownership !== "unknown") usage.unresolvedStepPartIds.delete(part.id); | ||
| 2347 | if (ownership === "owned") accumulateOpenCodeStepUsage(usage, part); | ||
| 2348 | } | ||
| 2228 | yield* emitAssistantTextDelta(context, part, turnId, event); | 2349 | yield* emitAssistantTextDelta(context, part, turnId, event); |
| 2229 | } | 2350 | } |
| 2230 | } | 2351 | } |
| ... | @@ -2290,6 +2411,21 @@ export function makeOpenCodeAdapter( | ... | @@ -2290,6 +2411,21 @@ export function makeOpenCodeAdapter( |
| 2290 | context.partById.set(part.id, part); | 2411 | context.partById.set(part.id, part); |
| 2291 | const messageRole = messageRoleForPart(context, part); | 2412 | const messageRole = messageRoleForPart(context, part); |
| 2292 | 2413 | ||
| 2414 | if (turnId && part.type === "step-finish" && context.turnTokenUsage) { | ||
| 2415 | const ownership = context.turnTokenUsage.assistantOwnershipByMessageId.get( | ||
| 2416 | part.messageID, | ||
| 2417 | ); | ||
| 2418 | if (ownership === "owned") { | ||
| 2419 | accumulateOpenCodeStepUsage(context.turnTokenUsage, part); | ||
| 2420 | } else if ( | ||
| 2421 | ownership === "unknown" || | ||
| 2422 | (ownership === undefined && | ||
| 2423 | context.messageRoleById.get(part.messageID) !== "assistant") | ||
| 2424 | ) { | ||
| 2425 | context.turnTokenUsage.unresolvedStepPartIds.add(part.id); | ||
| 2426 | } | ||
| 2427 | } | ||
| 2428 | |||
| 2293 | if (messageRole === "assistant") { | 2429 | if (messageRole === "assistant") { |
| 2294 | yield* emitAssistantTextDelta(context, part, turnId, event); | 2430 | yield* emitAssistantTextDelta(context, part, turnId, event); |
| 2295 | } | 2431 | } |
| ... | @@ -2483,6 +2619,7 @@ export function makeOpenCodeAdapter( | ... | @@ -2483,6 +2619,7 @@ export function makeOpenCodeAdapter( |
| 2483 | terminalCancellation.turnSettled = true; | 2619 | terminalCancellation.turnSettled = true; |
| 2484 | terminalCancellation.acknowledged = true; | 2620 | terminalCancellation.acknowledged = true; |
| 2485 | } | 2621 | } |
| 2622 | const tokenUsage = activeTurnId ? takeOpenCodeTurnTokenUsage(context, false) : undefined; | ||
| 2486 | context.activeTurnId = undefined; | 2623 | context.activeTurnId = undefined; |
| 2487 | context.activeAgent = undefined; | 2624 | context.activeAgent = undefined; |
| 2488 | context.activeVariant = undefined; | 2625 | context.activeVariant = undefined; |
| ... | @@ -2507,6 +2644,7 @@ export function makeOpenCodeAdapter( | ... | @@ -2507,6 +2644,7 @@ export function makeOpenCodeAdapter( |
| 2507 | payload: { | 2644 | payload: { |
| 2508 | state: "failed", | 2645 | state: "failed", |
| 2509 | errorMessage: message, | 2646 | errorMessage: message, |
| 2647 | tokenUsage, | ||
| 2510 | }, | 2648 | }, |
| 2511 | }); | 2649 | }); |
| 2512 | } | 2650 | } |
| ... | @@ -2826,6 +2964,7 @@ export function makeOpenCodeAdapter( | ... | @@ -2826,6 +2964,7 @@ export function makeOpenCodeAdapter( |
| 2826 | emittedTextByPartId: new Map(), | 2964 | emittedTextByPartId: new Map(), |
| 2827 | messageRoleById: new Map(), | 2965 | messageRoleById: new Map(), |
| 2828 | completedAssistantPartIds: new Set(), | 2966 | completedAssistantPartIds: new Set(), |
| 2967 | turnTokenUsage: undefined, | ||
| 2829 | activeTurnId: undefined, | 2968 | activeTurnId: undefined, |
| 2830 | activeAgent: undefined, | 2969 | activeAgent: undefined, |
| 2831 | activeVariant: undefined, | 2970 | activeVariant: undefined, |
| ... | @@ -2998,6 +3137,10 @@ export function makeOpenCodeAdapter( | ... | @@ -2998,6 +3137,10 @@ export function makeOpenCodeAdapter( |
| 2998 | context.promptAdmission = promptAdmission; | 3137 | context.promptAdmission = promptAdmission; |
| 2999 | 3138 | ||
| 3000 | context.activeTurnId = turnId; | 3139 | context.activeTurnId = turnId; |
| 3140 | if (steeringTurnId === undefined) { | ||
| 3141 | context.turnTokenUsage = makeOpenCodeTurnTokenUsageAccumulator(); | ||
| 3142 | } | ||
| 3143 | context.turnTokenUsage?.promptMessageIds.add(messageId); | ||
| 3001 | context.activeAgent = agent ?? (input.interactionMode === "plan" ? "plan" : undefined); | 3144 | context.activeAgent = agent ?? (input.interactionMode === "plan" ? "plan" : undefined); |
| 3002 | context.activeVariant = variant; | 3145 | context.activeVariant = variant; |
| 3003 | if (steeringTurnId === undefined) { | 3146 | if (steeringTurnId === undefined) { |
| ... | @@ -3022,7 +3165,6 @@ export function makeOpenCodeAdapter( | ... | @@ -3022,7 +3165,6 @@ export function makeOpenCodeAdapter( |
| 3022 | type: "turn.started", | 3165 | type: "turn.started", |
| 3023 | payload: { | 3166 | payload: { |
| 3024 | model: modelSelection?.model ?? context.session.model, | 3167 | model: modelSelection?.model ?? context.session.model, |
| 3025 | ...(variant ? { effort: variant } : {}), | ||
| 3026 | }, | 3168 | }, |
| 3027 | }); | 3169 | }); |
| 3028 | } | 3170 | } |
| ... | @@ -3087,6 +3229,7 @@ export function makeOpenCodeAdapter( | ... | @@ -3087,6 +3229,7 @@ export function makeOpenCodeAdapter( |
| 3087 | } | 3229 | } |
| 3088 | return; | 3230 | return; |
| 3089 | } | 3231 | } |
| 3232 | const tokenUsage = takeOpenCodeTurnTokenUsage(context, false); | ||
| 3090 | context.promptAdmission = undefined; | 3233 | context.promptAdmission = undefined; |
| 3091 | context.activeTurnId = undefined; | 3234 | context.activeTurnId = undefined; |
| 3092 | context.activeAgent = undefined; | 3235 | context.activeAgent = undefined; |
| ... | @@ -3103,7 +3246,10 @@ export function makeOpenCodeAdapter( | ... | @@ -3103,7 +3246,10 @@ export function makeOpenCodeAdapter( |
| 3103 | yield* emit({ | 3246 | yield* emit({ |
| 3104 | ...(yield* buildEventBase({ threadId: input.threadId, turnId })), | 3247 | ...(yield* buildEventBase({ threadId: input.threadId, turnId })), |
| 3105 | type: "turn.aborted", | 3248 | type: "turn.aborted", |
| 3106 | payload: { reason: requestError.detail }, | 3249 | payload: { |
| 3250 | reason: requestError.detail, | ||
| 3251 | tokenUsage, | ||
| 3252 | }, | ||
| 3107 | }); | 3253 | }); |
| 3108 | return; | 3254 | return; |
| 3109 | } | 3255 | } |
| ... | @@ -3131,6 +3277,7 @@ export function makeOpenCodeAdapter( | ... | @@ -3131,6 +3277,7 @@ export function makeOpenCodeAdapter( |
| 3131 | }); | 3277 | }); |
| 3132 | return; | 3278 | return; |
| 3133 | } | 3279 | } |
| 3280 | const tokenUsage = takeOpenCodeTurnTokenUsage(context, false); | ||
| 3134 | context.promptAdmission = undefined; | 3281 | context.promptAdmission = undefined; |
| 3135 | context.activeTurnId = undefined; | 3282 | context.activeTurnId = undefined; |
| 3136 | context.activeAgent = undefined; | 3283 | context.activeAgent = undefined; |
| ... | @@ -3154,6 +3301,7 @@ export function makeOpenCodeAdapter( | ... | @@ -3154,6 +3301,7 @@ export function makeOpenCodeAdapter( |
| 3154 | type: "turn.aborted", | 3301 | type: "turn.aborted", |
| 3155 | payload: { | 3302 | payload: { |
| 3156 | reason: requestError.detail, | 3303 | reason: requestError.detail, |
| 3304 | tokenUsage, | ||
| 3157 | }, | 3305 | }, |
| 3158 | }); | 3306 | }); |
| 3159 | }), | 3307 | }), |
apps/server/src/provider/Layers/ProviderService.test.ts+1237-4
| ... | @@ -314,6 +314,69 @@ function makeFakeCodexAdapter( | ... | @@ -314,6 +314,69 @@ function makeFakeCodexAdapter( |
| 314 | }; | 314 | }; |
| 315 | } | 315 | } |
| 316 | 316 | ||
| 317 | interface RecordedAnalyticsEvent { | ||
| 318 | readonly event: string; | ||
| 319 | readonly properties?: Readonly<Record<string, unknown>>; | ||
| 320 | } | ||
| 321 | |||
| 322 | function makeRecordingAnalytics() { | ||
| 323 | const events: Array<RecordedAnalyticsEvent> = []; | ||
| 324 | const layer = Layer.succeed( | ||
| 325 | AnalyticsService.AnalyticsService, | ||
| 326 | AnalyticsService.AnalyticsService.of({ | ||
| 327 | record: (event, properties) => | ||
| 328 | Effect.sync(() => { | ||
| 329 | events.push({ event, ...(properties ? { properties } : {}) }); | ||
| 330 | }), | ||
| 331 | flush: Effect.void, | ||
| 332 | }), | ||
| 333 | ); | ||
| 334 | |||
| 335 | return { | ||
| 336 | layer, | ||
| 337 | reset: () => { | ||
| 338 | events.length = 0; | ||
| 339 | }, | ||
| 340 | eventsByName: (event: string) => events.filter((entry) => entry.event === event), | ||
| 341 | }; | ||
| 342 | } | ||
| 343 | |||
| 344 | function makeStaticInstanceRegistry( | ||
| 345 | entries: ReadonlyArray<readonly [ProviderInstanceId, ProviderAdapterShape<ProviderAdapterError>]>, | ||
| 346 | ): ProviderAdapterRegistry.ProviderAdapterRegistry["Service"] { | ||
| 347 | const adapters = new Map(entries); | ||
| 348 | const unsupported = (instanceId: ProviderInstanceId) => | ||
| 349 | new ProviderUnsupportedError({ | ||
| 350 | provider: ProviderDriverKind.make(instanceId), | ||
| 351 | }); | ||
| 352 | |||
| 353 | return { | ||
| 354 | getByInstance: (instanceId) => { | ||
| 355 | const adapter = adapters.get(instanceId); | ||
| 356 | return adapter ? Effect.succeed(adapter) : Effect.fail(unsupported(instanceId)); | ||
| 357 | }, | ||
| 358 | getInstanceInfo: (instanceId) => { | ||
| 359 | const adapter = adapters.get(instanceId); | ||
| 360 | return adapter | ||
| 361 | ? Effect.succeed({ | ||
| 362 | instanceId, | ||
| 363 | driverKind: adapter.provider, | ||
| 364 | displayName: undefined, | ||
| 365 | enabled: true, | ||
| 366 | continuationIdentity: { | ||
| 367 | driverKind: adapter.provider, | ||
| 368 | continuationKey: `${adapter.provider}:instance:${instanceId}`, | ||
| 369 | }, | ||
| 370 | }) | ||
| 371 | : Effect.fail(unsupported(instanceId)); | ||
| 372 | }, | ||
| 373 | listInstances: () => Effect.succeed(Array.from(adapters.keys())), | ||
| 374 | subscribeChanges: Effect.flatMap(PubSub.unbounded<void>(), (pubsub) => | ||
| 375 | PubSub.subscribe(pubsub), | ||
| 376 | ), | ||
| 377 | }; | ||
| 378 | } | ||
| 379 | |||
| 317 | const advanceTestClock = (ms: number) => | 380 | const advanceTestClock = (ms: number) => |
| 318 | TestClock.adjust(`${ms} millis`).pipe(Effect.andThen(Effect.yieldNow)); | 381 | TestClock.adjust(`${ms} millis`).pipe(Effect.andThen(Effect.yieldNow)); |
| 319 | 382 | ||
| ... | @@ -332,6 +395,7 @@ function makeProviderServiceLayer( | ... | @@ -332,6 +395,7 @@ function makeProviderServiceLayer( |
| 332 | input: { | 395 | input: { |
| 333 | readonly directory?: ProviderSessionDirectory.ProviderSessionDirectory["Service"]; | 396 | readonly directory?: ProviderSessionDirectory.ProviderSessionDirectory["Service"]; |
| 334 | readonly supportsConversationRollback?: boolean; | 397 | readonly supportsConversationRollback?: boolean; |
| 398 | readonly analyticsLayer?: Layer.Layer<AnalyticsService.AnalyticsService>; | ||
| 335 | readonly registry?: ProviderAdapterRegistry.ProviderAdapterRegistry["Service"]; | 399 | readonly registry?: ProviderAdapterRegistry.ProviderAdapterRegistry["Service"]; |
| 336 | } = {}, | 400 | } = {}, |
| 337 | ) { | 401 | ) { |
| ... | @@ -365,7 +429,7 @@ function makeProviderServiceLayer( | ... | @@ -365,7 +429,7 @@ function makeProviderServiceLayer( |
| 365 | Layer.provide(directoryLayer), | 429 | Layer.provide(directoryLayer), |
| 366 | Layer.provide(defaultServerSettingsLayer), | 430 | Layer.provide(defaultServerSettingsLayer), |
| 367 | Layer.provide(serverConfigTestLayer), | 431 | Layer.provide(serverConfigTestLayer), |
| 368 | Layer.provideMerge(AnalyticsService.layerTest), | 432 | Layer.provideMerge(input.analyticsLayer ?? AnalyticsService.layerTest), |
| 369 | Layer.provide( | 433 | Layer.provide( |
| 370 | Layer.succeed( | 434 | Layer.succeed( |
| 371 | ProviderEventLoggers.ProviderEventLoggers, | 435 | ProviderEventLoggers.ProviderEventLoggers, |
| ... | @@ -440,6 +504,124 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => | ... | @@ -440,6 +504,124 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => |
| 440 | }), | 504 | }), |
| 441 | ); | 505 | ); |
| 442 | 506 | ||
| 507 | it.effect("ProviderServiceLive flushes deferred completions during shutdown", () => | ||
| 508 | Effect.gen(function* () { | ||
| 509 | const recordedAnalytics = makeRecordingAnalytics(); | ||
| 510 | const codex = makeFakeCodexAdapter(); | ||
| 511 | const registry = makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]); | ||
| 512 | const providerAdapterLayer = Layer.succeed( | ||
| 513 | ProviderAdapterRegistry.ProviderAdapterRegistry, | ||
| 514 | registry, | ||
| 515 | ); | ||
| 516 | const runtimeRepositoryLayer = ProviderSessionRuntime.layer.pipe( | ||
| 517 | Layer.provide(SqlitePersistenceMemory), | ||
| 518 | ); | ||
| 519 | const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer)); | ||
| 520 | const providerLayer = Layer.mergeAll( | ||
| 521 | makeProviderServiceLive().pipe( | ||
| 522 | Layer.provide(providerAdapterLayer), | ||
| 523 | Layer.provide(directoryLayer), | ||
| 524 | Layer.provide(defaultServerSettingsLayer), | ||
| 525 | Layer.provide(serverConfigTestLayer), | ||
| 526 | Layer.provide(recordedAnalytics.layer), | ||
| 527 | Layer.provide( | ||
| 528 | Layer.succeed( | ||
| 529 | ProviderEventLoggers.ProviderEventLoggers, | ||
| 530 | ProviderEventLoggers.NoOpProviderEventLoggers, | ||
| 531 | ), | ||
| 532 | ), | ||
| 533 | ), | ||
| 534 | directoryLayer, | ||
| 535 | runtimeRepositoryLayer, | ||
| 536 | NodeServices.layer, | ||
| 537 | ); | ||
| 538 | const scope = yield* Scope.make(); | ||
| 539 | const runtimeServices = yield* Layer.build(providerLayer).pipe(Scope.provide(scope)); | ||
| 540 | const provider = yield* ProviderService.ProviderService.pipe(Effect.provide(runtimeServices)); | ||
| 541 | const threadId = asThreadId("thread-turn-analytics-stop-all-deferred"); | ||
| 542 | const firstStarted = yield* Deferred.make<void>(); | ||
| 543 | const secondStarted = yield* Deferred.make<void>(); | ||
| 544 | const sendRelease = yield* Deferred.make<void>(); | ||
| 545 | const turnId = asTurnId("turn-analytics-stop-all-deferred"); | ||
| 546 | yield* provider.startSession(threadId, { | ||
| 547 | provider: CODEX_DRIVER, | ||
| 548 | providerInstanceId: codexInstanceId, | ||
| 549 | threadId, | ||
| 550 | runtimeMode: "full-access", | ||
| 551 | }); | ||
| 552 | codex.sendTurn | ||
| 553 | .mockImplementationOnce(() => | ||
| 554 | Effect.gen(function* () { | ||
| 555 | yield* Deferred.succeed(firstStarted, undefined); | ||
| 556 | yield* Deferred.await(sendRelease); | ||
| 557 | return { threadId, turnId }; | ||
| 558 | }), | ||
| 559 | ) | ||
| 560 | .mockImplementationOnce(() => | ||
| 561 | Effect.gen(function* () { | ||
| 562 | yield* Deferred.succeed(secondStarted, undefined); | ||
| 563 | yield* Deferred.await(sendRelease); | ||
| 564 | return { threadId, turnId: asTurnId("turn-analytics-stop-all-other") }; | ||
| 565 | }), | ||
| 566 | ); | ||
| 567 | |||
| 568 | const firstSend = yield* provider | ||
| 569 | .sendTurn({ threadId, input: "first", attachments: [] }) | ||
| 570 | .pipe(Effect.forkChild); | ||
| 571 | yield* Deferred.await(firstStarted); | ||
| 572 | const secondSend = yield* provider | ||
| 573 | .sendTurn({ threadId, input: "second", attachments: [] }) | ||
| 574 | .pipe(Effect.forkChild); | ||
| 575 | yield* Deferred.await(secondStarted); | ||
| 576 | |||
| 577 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe( | ||
| 578 | Stream.runDrain, | ||
| 579 | Effect.forkChild, | ||
| 580 | ); | ||
| 581 | yield* Effect.yieldNow; | ||
| 582 | codex.emit({ | ||
| 583 | type: "turn.started", | ||
| 584 | eventId: asEventId("evt-turn-analytics-stop-all-deferred-start"), | ||
| 585 | provider: CODEX_DRIVER, | ||
| 586 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 587 | threadId, | ||
| 588 | turnId, | ||
| 589 | payload: { model: "native-stop-all" }, | ||
| 590 | }); | ||
| 591 | codex.emit({ | ||
| 592 | type: "turn.completed", | ||
| 593 | eventId: asEventId("evt-turn-analytics-stop-all-deferred-complete"), | ||
| 594 | provider: CODEX_DRIVER, | ||
| 595 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 596 | threadId, | ||
| 597 | turnId, | ||
| 598 | payload: { | ||
| 599 | state: "completed", | ||
| 600 | tokenUsage: { | ||
| 601 | usageStatus: "complete", | ||
| 602 | usageScope: "main_agent", | ||
| 603 | inputTokens: 1_200, | ||
| 604 | outputTokens: 300, | ||
| 605 | hasSubagents: false, | ||
| 606 | }, | ||
| 607 | }, | ||
| 608 | }); | ||
| 609 | yield* Fiber.join(runtimeEvents); | ||
| 610 | assert.equal(recordedAnalytics.eventsByName("provider.turn.completed").length, 0); | ||
| 611 | |||
| 612 | const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit); | ||
| 613 | const completed = recordedAnalytics.eventsByName("provider.turn.completed"); | ||
| 614 | assert.equal(Exit.isSuccess(closeExit), true); | ||
| 615 | assert.equal(completed.length, 1); | ||
| 616 | assert.equal(completed[0]?.properties?.model, "native-stop-all"); | ||
| 617 | assert.equal(completed[0]?.properties?.inputTokens, 1_200); | ||
| 618 | assert.equal(completed[0]?.properties?.outputTokens, 300); | ||
| 619 | yield* Fiber.interrupt(firstSend); | ||
| 620 | yield* Fiber.interrupt(secondSend); | ||
| 621 | assert.equal(recordedAnalytics.eventsByName("provider.turn.completed").length, 1); | ||
| 622 | }), | ||
| 623 | ); | ||
| 624 | |||
| 443 | it.effect("ProviderServiceLive rejects new sessions for disabled providers", () => | 625 | it.effect("ProviderServiceLive rejects new sessions for disabled providers", () => |
| 444 | Effect.gen(function* () { | 626 | Effect.gen(function* () { |
| 445 | const codex = makeFakeCodexAdapter(); | 627 | const codex = makeFakeCodexAdapter(); |
| ... | @@ -2282,7 +2464,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { | ... | @@ -2282,7 +2464,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { |
| 2282 | createdAt: "2026-01-01T00:00:00.000Z", | 2464 | createdAt: "2026-01-01T00:00:00.000Z", |
| 2283 | threadId: session.threadId, | 2465 | threadId: session.threadId, |
| 2284 | turnId: asTurnId("turn-1"), | 2466 | turnId: asTurnId("turn-1"), |
| 2285 | status: "completed", | 2467 | payload: { state: "completed" }, |
| 2286 | }; | 2468 | }; |
| 2287 | 2469 | ||
| 2288 | fanout.codex.emit(completedEvent); | 2470 | fanout.codex.emit(completedEvent); |
| ... | @@ -2349,7 +2531,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { | ... | @@ -2349,7 +2531,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { |
| 2349 | createdAt: "2026-01-01T00:00:00.000Z", | 2531 | createdAt: "2026-01-01T00:00:00.000Z", |
| 2350 | threadId: session.threadId, | 2532 | threadId: session.threadId, |
| 2351 | turnId: asTurnId("turn-1"), | 2533 | turnId: asTurnId("turn-1"), |
| 2352 | status: "completed", | 2534 | payload: { state: "completed" }, |
| 2353 | }); | 2535 | }); |
| 2354 | 2536 | ||
| 2355 | yield* Fiber.join(consumer); | 2537 | yield* Fiber.join(consumer); |
| ... | @@ -2415,7 +2597,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { | ... | @@ -2415,7 +2597,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { |
| 2415 | createdAt: "2026-01-01T00:00:00.000Z", | 2597 | createdAt: "2026-01-01T00:00:00.000Z", |
| 2416 | threadId: session.threadId, | 2598 | threadId: session.threadId, |
| 2417 | turnId: asTurnId("turn-1"), | 2599 | turnId: asTurnId("turn-1"), |
| 2418 | status: "completed", | 2600 | payload: { state: "completed" }, |
| 2419 | }, | 2601 | }, |
| 2420 | ]; | 2602 | ]; |
| 2421 | 2603 | ||
| ... | @@ -2659,6 +2841,1057 @@ citations.layer("ProviderServiceLive assistant citations", (it) => { | ... | @@ -2659,6 +2841,1057 @@ citations.layer("ProviderServiceLive assistant citations", (it) => { |
| 2659 | ); | 2841 | ); |
| 2660 | }); | 2842 | }); |
| 2661 | 2843 | ||
| 2844 | const recordedTurnAnalytics = makeRecordingAnalytics(); | ||
| 2845 | const secondaryCodexInstanceId = ProviderInstanceId.make("codex_work"); | ||
| 2846 | const primaryAnalyticsCodex = makeFakeCodexAdapter(); | ||
| 2847 | const secondaryAnalyticsCodex = makeFakeCodexAdapter(); | ||
| 2848 | const turnAnalytics = makeProviderServiceLayer({ | ||
| 2849 | analyticsLayer: recordedTurnAnalytics.layer, | ||
| 2850 | registry: makeStaticInstanceRegistry([ | ||
| 2851 | [codexInstanceId, primaryAnalyticsCodex.adapter], | ||
| 2852 | [secondaryCodexInstanceId, secondaryAnalyticsCodex.adapter], | ||
| 2853 | ]), | ||
| 2854 | }); | ||
| 2855 | |||
| 2856 | turnAnalytics.layer("ProviderServiceLive turn analytics", (it) => { | ||
| 2857 | it.effect("records one completed-turn event with the allowed properties", () => | ||
| 2858 | Effect.gen(function* () { | ||
| 2859 | recordedTurnAnalytics.reset(); | ||
| 2860 | const provider = yield* ProviderService.ProviderService; | ||
| 2861 | const threadId = asThreadId("thread-turn-analytics-complete"); | ||
| 2862 | yield* provider.startSession(threadId, { | ||
| 2863 | provider: CODEX_DRIVER, | ||
| 2864 | providerInstanceId: codexInstanceId, | ||
| 2865 | threadId, | ||
| 2866 | runtimeMode: "full-access", | ||
| 2867 | }); | ||
| 2868 | const turn = yield* provider.sendTurn({ | ||
| 2869 | threadId, | ||
| 2870 | input: "measure this turn", | ||
| 2871 | attachments: [], | ||
| 2872 | interactionMode: "plan", | ||
| 2873 | modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-sol", [ | ||
| 2874 | { id: "reasoningEffort", value: "high" }, | ||
| 2875 | ]), | ||
| 2876 | }); | ||
| 2877 | yield* advanceTestClock(40); | ||
| 2878 | |||
| 2879 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe( | ||
| 2880 | Stream.runDrain, | ||
| 2881 | Effect.forkChild, | ||
| 2882 | ); | ||
| 2883 | yield* Effect.yieldNow; | ||
| 2884 | const completedEvent: LegacyProviderRuntimeEvent = { | ||
| 2885 | type: "turn.completed", | ||
| 2886 | eventId: asEventId("evt-turn-analytics-complete"), | ||
| 2887 | provider: CODEX_DRIVER, | ||
| 2888 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 2889 | threadId, | ||
| 2890 | turnId: turn.turnId, | ||
| 2891 | payload: { | ||
| 2892 | state: "completed", | ||
| 2893 | tokenUsage: { | ||
| 2894 | usageStatus: "complete", | ||
| 2895 | usageScope: "main_agent", | ||
| 2896 | inputTokens: 1_200, | ||
| 2897 | cachedInputTokens: 800, | ||
| 2898 | cacheCreationTokens: 100, | ||
| 2899 | outputTokens: 300, | ||
| 2900 | reasoningTokens: 120, | ||
| 2901 | hasSubagents: false, | ||
| 2902 | }, | ||
| 2903 | }, | ||
| 2904 | }; | ||
| 2905 | primaryAnalyticsCodex.emit(completedEvent); | ||
| 2906 | primaryAnalyticsCodex.emit({ | ||
| 2907 | ...completedEvent, | ||
| 2908 | eventId: asEventId("evt-turn-analytics-complete-duplicate"), | ||
| 2909 | }); | ||
| 2910 | yield* Fiber.join(runtimeEvents); | ||
| 2911 | |||
| 2912 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 2913 | assert.equal(completed.length, 1); | ||
| 2914 | assert.deepEqual(completed[0]?.properties, { | ||
| 2915 | provider: CODEX_DRIVER, | ||
| 2916 | model: "gpt-5.6-sol", | ||
| 2917 | effort: "high", | ||
| 2918 | interactionMode: "plan", | ||
| 2919 | runtimeMode: "full-access", | ||
| 2920 | mixedModels: false, | ||
| 2921 | durationMs: 40, | ||
| 2922 | terminalStatus: "completed", | ||
| 2923 | usageStatus: "complete", | ||
| 2924 | usageScope: "main_agent", | ||
| 2925 | hasSubagents: false, | ||
| 2926 | inputTokens: 1_200, | ||
| 2927 | cachedInputTokens: 800, | ||
| 2928 | cacheCreationTokens: 100, | ||
| 2929 | outputTokens: 300, | ||
| 2930 | reasoningTokens: 120, | ||
| 2931 | }); | ||
| 2932 | }), | ||
| 2933 | ); | ||
| 2934 | |||
| 2935 | it.effect("does not report a generic model variant as reasoning effort", () => | ||
| 2936 | Effect.gen(function* () { | ||
| 2937 | recordedTurnAnalytics.reset(); | ||
| 2938 | const provider = yield* ProviderService.ProviderService; | ||
| 2939 | const threadId = asThreadId("thread-turn-analytics-generic-variant"); | ||
| 2940 | yield* provider.startSession(threadId, { | ||
| 2941 | provider: CODEX_DRIVER, | ||
| 2942 | providerInstanceId: codexInstanceId, | ||
| 2943 | threadId, | ||
| 2944 | runtimeMode: "full-access", | ||
| 2945 | }); | ||
| 2946 | const turn = yield* provider.sendTurn({ | ||
| 2947 | threadId, | ||
| 2948 | input: "use the provider preset", | ||
| 2949 | attachments: [], | ||
| 2950 | modelSelection: createModelSelection(codexInstanceId, "provider/model", [ | ||
| 2951 | { id: "variant", value: "high" }, | ||
| 2952 | ]), | ||
| 2953 | }); | ||
| 2954 | |||
| 2955 | const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe( | ||
| 2956 | Stream.runDrain, | ||
| 2957 | Effect.forkChild, | ||
| 2958 | ); | ||
| 2959 | yield* Effect.yieldNow; | ||
| 2960 | primaryAnalyticsCodex.emit({ | ||
| 2961 | type: "turn.completed", | ||
| 2962 | eventId: asEventId("evt-turn-analytics-generic-variant"), | ||
| 2963 | provider: CODEX_DRIVER, | ||
| 2964 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 2965 | threadId, | ||
| 2966 | turnId: turn.turnId, | ||
| 2967 | payload: { state: "completed" }, | ||
| 2968 | }); | ||
| 2969 | yield* Fiber.join(runtimeEvent); | ||
| 2970 | |||
| 2971 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 2972 | assert.equal(completed.length, 1); | ||
| 2973 | assert.notProperty(completed[0]?.properties ?? {}, "effort"); | ||
| 2974 | }), | ||
| 2975 | ); | ||
| 2976 | |||
| 2977 | it.effect("ignores model metadata bound to another provider instance", () => | ||
| 2978 | Effect.gen(function* () { | ||
| 2979 | recordedTurnAnalytics.reset(); | ||
| 2980 | const provider = yield* ProviderService.ProviderService; | ||
| 2981 | const threadId = asThreadId("thread-turn-analytics-mismatched-model-instance"); | ||
| 2982 | yield* provider.startSession(threadId, { | ||
| 2983 | provider: CODEX_DRIVER, | ||
| 2984 | providerInstanceId: codexInstanceId, | ||
| 2985 | threadId, | ||
| 2986 | runtimeMode: "full-access", | ||
| 2987 | }); | ||
| 2988 | const turn = yield* provider.sendTurn({ | ||
| 2989 | threadId, | ||
| 2990 | input: "ignore this mismatched selection", | ||
| 2991 | attachments: [], | ||
| 2992 | modelSelection: createModelSelection(secondaryCodexInstanceId, "wrong-model", [ | ||
| 2993 | { id: "reasoningEffort", value: "high" }, | ||
| 2994 | ]), | ||
| 2995 | }); | ||
| 2996 | |||
| 2997 | const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe( | ||
| 2998 | Stream.runDrain, | ||
| 2999 | Effect.forkChild, | ||
| 3000 | ); | ||
| 3001 | yield* Effect.yieldNow; | ||
| 3002 | primaryAnalyticsCodex.emit({ | ||
| 3003 | type: "turn.completed", | ||
| 3004 | eventId: asEventId("evt-turn-analytics-mismatched-model-instance"), | ||
| 3005 | provider: CODEX_DRIVER, | ||
| 3006 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3007 | threadId, | ||
| 3008 | turnId: turn.turnId, | ||
| 3009 | payload: { state: "completed" }, | ||
| 3010 | }); | ||
| 3011 | yield* Fiber.join(runtimeEvent); | ||
| 3012 | |||
| 3013 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3014 | assert.equal(completed.length, 1); | ||
| 3015 | assert.notProperty(completed[0]?.properties ?? {}, "model"); | ||
| 3016 | assert.notProperty(completed[0]?.properties ?? {}, "effort"); | ||
| 3017 | }), | ||
| 3018 | ); | ||
| 3019 | |||
| 3020 | it.effect("keeps overlapping request metadata with out-of-order adapter responses", () => | ||
| 3021 | Effect.gen(function* () { | ||
| 3022 | recordedTurnAnalytics.reset(); | ||
| 3023 | const provider = yield* ProviderService.ProviderService; | ||
| 3024 | const threadId = asThreadId("thread-turn-analytics-overlap"); | ||
| 3025 | yield* provider.startSession(threadId, { | ||
| 3026 | provider: CODEX_DRIVER, | ||
| 3027 | providerInstanceId: codexInstanceId, | ||
| 3028 | threadId, | ||
| 3029 | runtimeMode: "full-access", | ||
| 3030 | }); | ||
| 3031 | |||
| 3032 | const firstStarted = yield* Deferred.make<void>(); | ||
| 3033 | const secondStarted = yield* Deferred.make<void>(); | ||
| 3034 | const firstRelease = yield* Deferred.make<void>(); | ||
| 3035 | const secondRelease = yield* Deferred.make<void>(); | ||
| 3036 | const initialStartsObserved = yield* Deferred.make<void>(); | ||
| 3037 | let initialStartCount = 0; | ||
| 3038 | const firstTurnId = asTurnId("turn-analytics-overlap-first"); | ||
| 3039 | const secondTurnId = asTurnId("turn-analytics-overlap-second"); | ||
| 3040 | primaryAnalyticsCodex.sendTurn | ||
| 3041 | .mockImplementationOnce((input) => | ||
| 3042 | Effect.gen(function* () { | ||
| 3043 | yield* Deferred.succeed(firstStarted, undefined); | ||
| 3044 | yield* Deferred.await(firstRelease); | ||
| 3045 | return { threadId: input.threadId, turnId: firstTurnId }; | ||
| 3046 | }), | ||
| 3047 | ) | ||
| 3048 | .mockImplementationOnce((input) => | ||
| 3049 | Effect.gen(function* () { | ||
| 3050 | yield* Deferred.succeed(secondStarted, undefined); | ||
| 3051 | yield* Deferred.await(secondRelease); | ||
| 3052 | return { threadId: input.threadId, turnId: secondTurnId }; | ||
| 3053 | }), | ||
| 3054 | ); | ||
| 3055 | |||
| 3056 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 5).pipe( | ||
| 3057 | Stream.tap((event) => { | ||
| 3058 | if (event.type !== "turn.started" || initialStartCount >= 2) return Effect.void; | ||
| 3059 | initialStartCount += 1; | ||
| 3060 | return initialStartCount === 2 | ||
| 3061 | ? Deferred.succeed(initialStartsObserved, undefined).pipe(Effect.asVoid) | ||
| 3062 | : Effect.void; | ||
| 3063 | }), | ||
| 3064 | Stream.runDrain, | ||
| 3065 | Effect.forkChild, | ||
| 3066 | ); | ||
| 3067 | const firstSend = yield* provider | ||
| 3068 | .sendTurn({ | ||
| 3069 | threadId, | ||
| 3070 | input: "first", | ||
| 3071 | attachments: [], | ||
| 3072 | interactionMode: "default", | ||
| 3073 | modelSelection: createModelSelection(codexInstanceId, "requested-first"), | ||
| 3074 | }) | ||
| 3075 | .pipe(Effect.forkChild); | ||
| 3076 | yield* Deferred.await(firstStarted); | ||
| 3077 | const secondSend = yield* provider | ||
| 3078 | .sendTurn({ | ||
| 3079 | threadId, | ||
| 3080 | input: "second", | ||
| 3081 | attachments: [], | ||
| 3082 | interactionMode: "plan", | ||
| 3083 | modelSelection: createModelSelection(codexInstanceId, "requested-second"), | ||
| 3084 | }) | ||
| 3085 | .pipe(Effect.forkChild); | ||
| 3086 | yield* Deferred.await(secondStarted); | ||
| 3087 | |||
| 3088 | for (const [turnId, suffix] of [ | ||
| 3089 | [firstTurnId, "first"], | ||
| 3090 | [secondTurnId, "second"], | ||
| 3091 | ] as const) { | ||
| 3092 | primaryAnalyticsCodex.emit({ | ||
| 3093 | type: "turn.started", | ||
| 3094 | eventId: asEventId(`evt-turn-analytics-overlap-start-${suffix}`), | ||
| 3095 | provider: CODEX_DRIVER, | ||
| 3096 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3097 | threadId, | ||
| 3098 | turnId, | ||
| 3099 | payload: { model: `native-${suffix}`, effort: `native-effort-${suffix}` }, | ||
| 3100 | }); | ||
| 3101 | } | ||
| 3102 | yield* Deferred.await(initialStartsObserved); | ||
| 3103 | yield* Deferred.succeed(secondRelease, undefined); | ||
| 3104 | yield* Fiber.join(secondSend); | ||
| 3105 | primaryAnalyticsCodex.emit({ | ||
| 3106 | type: "turn.started", | ||
| 3107 | eventId: asEventId("evt-turn-analytics-overlap-start-second-duplicate"), | ||
| 3108 | provider: CODEX_DRIVER, | ||
| 3109 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3110 | threadId, | ||
| 3111 | turnId: secondTurnId, | ||
| 3112 | payload: { model: "native-second", effort: "native-effort-second" }, | ||
| 3113 | }); | ||
| 3114 | yield* Deferred.succeed(firstRelease, undefined); | ||
| 3115 | yield* Fiber.join(firstSend); | ||
| 3116 | for (const [turnId, suffix] of [ | ||
| 3117 | [secondTurnId, "second"], | ||
| 3118 | [firstTurnId, "first"], | ||
| 3119 | ] as const) { | ||
| 3120 | primaryAnalyticsCodex.emit({ | ||
| 3121 | type: "turn.completed", | ||
| 3122 | eventId: asEventId(`evt-turn-analytics-overlap-complete-${suffix}`), | ||
| 3123 | provider: CODEX_DRIVER, | ||
| 3124 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3125 | threadId, | ||
| 3126 | turnId, | ||
| 3127 | payload: { state: "completed" }, | ||
| 3128 | }); | ||
| 3129 | } | ||
| 3130 | yield* Fiber.join(runtimeEvents); | ||
| 3131 | |||
| 3132 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3133 | assert.equal(completed.length, 2); | ||
| 3134 | assert.deepInclude(completed[0]?.properties ?? {}, { | ||
| 3135 | model: "native-second", | ||
| 3136 | effort: "native-effort-second", | ||
| 3137 | interactionMode: "plan", | ||
| 3138 | }); | ||
| 3139 | assert.deepInclude(completed[1]?.properties ?? {}, { | ||
| 3140 | model: "native-first", | ||
| 3141 | effort: "native-effort-first", | ||
| 3142 | interactionMode: "default", | ||
| 3143 | }); | ||
| 3144 | }), | ||
| 3145 | ); | ||
| 3146 | |||
| 3147 | it.effect("waits for the adapter response when a turn completes before sendTurn returns", () => | ||
| 3148 | Effect.gen(function* () { | ||
| 3149 | recordedTurnAnalytics.reset(); | ||
| 3150 | const provider = yield* ProviderService.ProviderService; | ||
| 3151 | const threadId = asThreadId("thread-turn-analytics-fast-completion"); | ||
| 3152 | const turnId = asTurnId("turn-analytics-fast-completion"); | ||
| 3153 | const returnRelease = yield* Deferred.make<void>(); | ||
| 3154 | yield* provider.startSession(threadId, { | ||
| 3155 | provider: CODEX_DRIVER, | ||
| 3156 | providerInstanceId: codexInstanceId, | ||
| 3157 | threadId, | ||
| 3158 | runtimeMode: "full-access", | ||
| 3159 | }); | ||
| 3160 | yield* Effect.yieldNow; | ||
| 3161 | primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) => | ||
| 3162 | Effect.gen(function* () { | ||
| 3163 | primaryAnalyticsCodex.emit({ | ||
| 3164 | type: "turn.started", | ||
| 3165 | eventId: asEventId("evt-turn-analytics-fast-start"), | ||
| 3166 | provider: CODEX_DRIVER, | ||
| 3167 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3168 | threadId, | ||
| 3169 | turnId, | ||
| 3170 | payload: { model: "native-fast", effort: "high" }, | ||
| 3171 | }); | ||
| 3172 | primaryAnalyticsCodex.emit({ | ||
| 3173 | type: "turn.completed", | ||
| 3174 | eventId: asEventId("evt-turn-analytics-fast-complete"), | ||
| 3175 | provider: CODEX_DRIVER, | ||
| 3176 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3177 | threadId, | ||
| 3178 | turnId, | ||
| 3179 | payload: { state: "completed" }, | ||
| 3180 | }); | ||
| 3181 | yield* Deferred.await(returnRelease); | ||
| 3182 | return { threadId: input.threadId, turnId }; | ||
| 3183 | }), | ||
| 3184 | ); | ||
| 3185 | |||
| 3186 | const terminalReceipt = yield* provider.streamEvents.pipe( | ||
| 3187 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 3188 | Stream.runHead, | ||
| 3189 | Effect.forkChild, | ||
| 3190 | ); | ||
| 3191 | yield* Effect.yieldNow; | ||
| 3192 | const sendFiber = yield* provider | ||
| 3193 | .sendTurn({ | ||
| 3194 | threadId, | ||
| 3195 | input: "finish immediately", | ||
| 3196 | attachments: [], | ||
| 3197 | interactionMode: "plan", | ||
| 3198 | modelSelection: createModelSelection(codexInstanceId, "requested-fast"), | ||
| 3199 | }) | ||
| 3200 | .pipe(Effect.forkChild); | ||
| 3201 | const terminal = yield* Fiber.join(terminalReceipt); | ||
| 3202 | assert.equal(terminal._tag, "Some"); | ||
| 3203 | assert.equal(sendFiber.pollUnsafe(), undefined); | ||
| 3204 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0); | ||
| 3205 | |||
| 3206 | yield* Deferred.succeed(returnRelease, undefined); | ||
| 3207 | yield* Fiber.join(sendFiber); | ||
| 3208 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3209 | assert.equal(completed.length, 1); | ||
| 3210 | assert.deepInclude(completed[0]?.properties ?? {}, { | ||
| 3211 | model: "native-fast", | ||
| 3212 | effort: "high", | ||
| 3213 | interactionMode: "plan", | ||
| 3214 | }); | ||
| 3215 | }), | ||
| 3216 | ); | ||
| 3217 | |||
| 3218 | it.effect("does not give a synthetic turn the metadata of an in-flight send", () => | ||
| 3219 | Effect.gen(function* () { | ||
| 3220 | recordedTurnAnalytics.reset(); | ||
| 3221 | const provider = yield* ProviderService.ProviderService; | ||
| 3222 | const threadId = asThreadId("thread-turn-analytics-synthetic-start"); | ||
| 3223 | const syntheticTurnId = asTurnId("turn-analytics-synthetic"); | ||
| 3224 | const realTurnId = asTurnId("turn-analytics-real"); | ||
| 3225 | yield* provider.startSession(threadId, { | ||
| 3226 | provider: CODEX_DRIVER, | ||
| 3227 | providerInstanceId: codexInstanceId, | ||
| 3228 | threadId, | ||
| 3229 | runtimeMode: "full-access", | ||
| 3230 | }); | ||
| 3231 | yield* Effect.yieldNow; | ||
| 3232 | // Claude closes a leftover synthetic turn while it prepares the real | ||
| 3233 | // turn, so both events arrive before sendTurn returns the real turn ID. | ||
| 3234 | primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) => | ||
| 3235 | Effect.gen(function* () { | ||
| 3236 | primaryAnalyticsCodex.emit({ | ||
| 3237 | type: "turn.started", | ||
| 3238 | eventId: asEventId("evt-turn-analytics-synthetic-start"), | ||
| 3239 | provider: CODEX_DRIVER, | ||
| 3240 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3241 | threadId, | ||
| 3242 | turnId: syntheticTurnId, | ||
| 3243 | payload: {}, | ||
| 3244 | }); | ||
| 3245 | primaryAnalyticsCodex.emit({ | ||
| 3246 | type: "turn.completed", | ||
| 3247 | eventId: asEventId("evt-turn-analytics-synthetic-complete"), | ||
| 3248 | provider: CODEX_DRIVER, | ||
| 3249 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3250 | threadId, | ||
| 3251 | turnId: syntheticTurnId, | ||
| 3252 | payload: { state: "completed" }, | ||
| 3253 | }); | ||
| 3254 | primaryAnalyticsCodex.emit({ | ||
| 3255 | type: "turn.started", | ||
| 3256 | eventId: asEventId("evt-turn-analytics-real-start"), | ||
| 3257 | provider: CODEX_DRIVER, | ||
| 3258 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3259 | threadId, | ||
| 3260 | turnId: realTurnId, | ||
| 3261 | payload: { model: "native-real" }, | ||
| 3262 | }); | ||
| 3263 | yield* Effect.yieldNow; | ||
| 3264 | return { threadId: input.threadId, turnId: realTurnId }; | ||
| 3265 | }), | ||
| 3266 | ); | ||
| 3267 | |||
| 3268 | yield* provider.sendTurn({ | ||
| 3269 | threadId, | ||
| 3270 | input: "start the real turn", | ||
| 3271 | attachments: [], | ||
| 3272 | interactionMode: "plan", | ||
| 3273 | modelSelection: createModelSelection(codexInstanceId, "requested-real"), | ||
| 3274 | }); | ||
| 3275 | const realCompletion = yield* provider.streamEvents.pipe( | ||
| 3276 | Stream.filter((event) => event.type === "turn.completed" && event.turnId === realTurnId), | ||
| 3277 | Stream.runHead, | ||
| 3278 | Effect.forkChild, | ||
| 3279 | ); | ||
| 3280 | yield* Effect.yieldNow; | ||
| 3281 | primaryAnalyticsCodex.emit({ | ||
| 3282 | type: "turn.completed", | ||
| 3283 | eventId: asEventId("evt-turn-analytics-real-complete"), | ||
| 3284 | provider: CODEX_DRIVER, | ||
| 3285 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3286 | threadId, | ||
| 3287 | turnId: realTurnId, | ||
| 3288 | payload: { state: "completed" }, | ||
| 3289 | }); | ||
| 3290 | yield* Fiber.join(realCompletion); | ||
| 3291 | |||
| 3292 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3293 | assert.equal(completed.length, 2); | ||
| 3294 | assert.equal(completed[0]?.properties?.interactionMode, undefined); | ||
| 3295 | assert.equal(completed[0]?.properties?.model, undefined); | ||
| 3296 | assert.deepInclude(completed[1]?.properties ?? {}, { | ||
| 3297 | model: "native-real", | ||
| 3298 | interactionMode: "plan", | ||
| 3299 | }); | ||
| 3300 | }), | ||
| 3301 | ); | ||
| 3302 | |||
| 3303 | it.effect("defers overlapping terminal analytics until exact request association", () => | ||
| 3304 | Effect.gen(function* () { | ||
| 3305 | recordedTurnAnalytics.reset(); | ||
| 3306 | const provider = yield* ProviderService.ProviderService; | ||
| 3307 | const threadId = asThreadId("thread-turn-analytics-overlap-fast-completion"); | ||
| 3308 | const firstStarted = yield* Deferred.make<void>(); | ||
| 3309 | const secondStarted = yield* Deferred.make<void>(); | ||
| 3310 | const firstRelease = yield* Deferred.make<void>(); | ||
| 3311 | const secondRelease = yield* Deferred.make<void>(); | ||
| 3312 | const firstTurnId = asTurnId("turn-analytics-overlap-fast-first"); | ||
| 3313 | const secondTurnId = asTurnId("turn-analytics-overlap-fast-second"); | ||
| 3314 | yield* provider.startSession(threadId, { | ||
| 3315 | provider: CODEX_DRIVER, | ||
| 3316 | providerInstanceId: codexInstanceId, | ||
| 3317 | threadId, | ||
| 3318 | runtimeMode: "full-access", | ||
| 3319 | }); | ||
| 3320 | primaryAnalyticsCodex.sendTurn | ||
| 3321 | .mockImplementationOnce(() => | ||
| 3322 | Effect.gen(function* () { | ||
| 3323 | yield* Deferred.succeed(firstStarted, undefined); | ||
| 3324 | yield* Deferred.await(firstRelease); | ||
| 3325 | return { threadId, turnId: firstTurnId }; | ||
| 3326 | }), | ||
| 3327 | ) | ||
| 3328 | .mockImplementationOnce(() => | ||
| 3329 | Effect.gen(function* () { | ||
| 3330 | yield* Deferred.succeed(secondStarted, undefined); | ||
| 3331 | yield* Deferred.await(secondRelease); | ||
| 3332 | return { threadId, turnId: secondTurnId }; | ||
| 3333 | }), | ||
| 3334 | ); | ||
| 3335 | |||
| 3336 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 4).pipe( | ||
| 3337 | Stream.runDrain, | ||
| 3338 | Effect.forkChild, | ||
| 3339 | ); | ||
| 3340 | const firstSend = yield* provider | ||
| 3341 | .sendTurn({ | ||
| 3342 | threadId, | ||
| 3343 | input: "first fast completion", | ||
| 3344 | attachments: [], | ||
| 3345 | interactionMode: "default", | ||
| 3346 | modelSelection: createModelSelection(codexInstanceId, "requested-first"), | ||
| 3347 | }) | ||
| 3348 | .pipe(Effect.forkChild); | ||
| 3349 | yield* Deferred.await(firstStarted); | ||
| 3350 | yield* advanceTestClock(10); | ||
| 3351 | const secondSend = yield* provider | ||
| 3352 | .sendTurn({ | ||
| 3353 | threadId, | ||
| 3354 | input: "second fast completion", | ||
| 3355 | attachments: [], | ||
| 3356 | interactionMode: "plan", | ||
| 3357 | modelSelection: createModelSelection(codexInstanceId, "requested-second"), | ||
| 3358 | }) | ||
| 3359 | .pipe(Effect.forkChild); | ||
| 3360 | yield* Deferred.await(secondStarted); | ||
| 3361 | yield* advanceTestClock(20); | ||
| 3362 | |||
| 3363 | for (const [turnId, suffix] of [ | ||
| 3364 | [firstTurnId, "first"], | ||
| 3365 | [secondTurnId, "second"], | ||
| 3366 | ] as const) { | ||
| 3367 | primaryAnalyticsCodex.emit({ | ||
| 3368 | type: "turn.started", | ||
| 3369 | eventId: asEventId(`evt-turn-analytics-overlap-fast-start-${suffix}`), | ||
| 3370 | provider: CODEX_DRIVER, | ||
| 3371 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3372 | threadId, | ||
| 3373 | turnId, | ||
| 3374 | payload: { model: `native-${suffix}`, effort: `native-effort-${suffix}` }, | ||
| 3375 | }); | ||
| 3376 | primaryAnalyticsCodex.emit({ | ||
| 3377 | type: "turn.completed", | ||
| 3378 | eventId: asEventId(`evt-turn-analytics-overlap-fast-complete-${suffix}`), | ||
| 3379 | provider: CODEX_DRIVER, | ||
| 3380 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3381 | threadId, | ||
| 3382 | turnId, | ||
| 3383 | payload: { state: "completed" }, | ||
| 3384 | }); | ||
| 3385 | } | ||
| 3386 | yield* Fiber.join(runtimeEvents); | ||
| 3387 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0); | ||
| 3388 | yield* advanceTestClock(40); | ||
| 3389 | |||
| 3390 | yield* Deferred.succeed(secondRelease, undefined); | ||
| 3391 | yield* Fiber.join(secondSend); | ||
| 3392 | let completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3393 | assert.equal(completed.length, 1); | ||
| 3394 | assert.deepInclude(completed[0]?.properties ?? {}, { | ||
| 3395 | model: "native-second", | ||
| 3396 | effort: "native-effort-second", | ||
| 3397 | interactionMode: "plan", | ||
| 3398 | durationMs: 20, | ||
| 3399 | }); | ||
| 3400 | |||
| 3401 | yield* Deferred.succeed(firstRelease, undefined); | ||
| 3402 | yield* Fiber.join(firstSend); | ||
| 3403 | completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3404 | assert.equal(completed.length, 2); | ||
| 3405 | assert.deepInclude(completed[1]?.properties ?? {}, { | ||
| 3406 | model: "native-first", | ||
| 3407 | effort: "native-effort-first", | ||
| 3408 | interactionMode: "default", | ||
| 3409 | durationMs: 30, | ||
| 3410 | }); | ||
| 3411 | }), | ||
| 3412 | ); | ||
| 3413 | |||
| 3414 | it.effect("cleans pending metadata when a send is canceled", () => | ||
| 3415 | Effect.gen(function* () { | ||
| 3416 | recordedTurnAnalytics.reset(); | ||
| 3417 | const provider = yield* ProviderService.ProviderService; | ||
| 3418 | const threadId = asThreadId("thread-turn-analytics-canceled-send"); | ||
| 3419 | const canceledStarted = yield* Deferred.make<void>(); | ||
| 3420 | const canceledRelease = yield* Deferred.make<void>(); | ||
| 3421 | const nextReturnRelease = yield* Deferred.make<void>(); | ||
| 3422 | const nextTurnId = asTurnId("turn-analytics-after-canceled-send"); | ||
| 3423 | yield* provider.startSession(threadId, { | ||
| 3424 | provider: CODEX_DRIVER, | ||
| 3425 | providerInstanceId: codexInstanceId, | ||
| 3426 | threadId, | ||
| 3427 | runtimeMode: "full-access", | ||
| 3428 | }); | ||
| 3429 | primaryAnalyticsCodex.sendTurn | ||
| 3430 | .mockImplementationOnce(() => | ||
| 3431 | Effect.gen(function* () { | ||
| 3432 | yield* Deferred.succeed(canceledStarted, undefined); | ||
| 3433 | yield* Deferred.await(canceledRelease); | ||
| 3434 | return { threadId, turnId: asTurnId("turn-analytics-canceled") }; | ||
| 3435 | }), | ||
| 3436 | ) | ||
| 3437 | .mockImplementationOnce(() => | ||
| 3438 | Effect.gen(function* () { | ||
| 3439 | primaryAnalyticsCodex.emit({ | ||
| 3440 | type: "turn.started", | ||
| 3441 | eventId: asEventId("evt-turn-analytics-after-canceled-start"), | ||
| 3442 | provider: CODEX_DRIVER, | ||
| 3443 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3444 | threadId, | ||
| 3445 | turnId: nextTurnId, | ||
| 3446 | payload: { model: "native-next", effort: "high" }, | ||
| 3447 | }); | ||
| 3448 | primaryAnalyticsCodex.emit({ | ||
| 3449 | type: "turn.completed", | ||
| 3450 | eventId: asEventId("evt-turn-analytics-after-canceled-complete"), | ||
| 3451 | provider: CODEX_DRIVER, | ||
| 3452 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3453 | threadId, | ||
| 3454 | turnId: nextTurnId, | ||
| 3455 | payload: { state: "completed" }, | ||
| 3456 | }); | ||
| 3457 | yield* Deferred.await(nextReturnRelease); | ||
| 3458 | return { threadId, turnId: nextTurnId }; | ||
| 3459 | }), | ||
| 3460 | ); | ||
| 3461 | |||
| 3462 | const canceledSend = yield* provider | ||
| 3463 | .sendTurn({ | ||
| 3464 | threadId, | ||
| 3465 | input: "cancel this request", | ||
| 3466 | attachments: [], | ||
| 3467 | interactionMode: "default", | ||
| 3468 | }) | ||
| 3469 | .pipe(Effect.forkChild); | ||
| 3470 | yield* Deferred.await(canceledStarted); | ||
| 3471 | yield* Fiber.interrupt(canceledSend); | ||
| 3472 | |||
| 3473 | const terminalReceipt = yield* provider.streamEvents.pipe( | ||
| 3474 | Stream.filter((event) => event.type === "turn.completed"), | ||
| 3475 | Stream.runHead, | ||
| 3476 | Effect.forkChild, | ||
| 3477 | ); | ||
| 3478 | yield* Effect.yieldNow; | ||
| 3479 | const nextSend = yield* provider | ||
| 3480 | .sendTurn({ | ||
| 3481 | threadId, | ||
| 3482 | input: "measure the next request", | ||
| 3483 | attachments: [], | ||
| 3484 | interactionMode: "plan", | ||
| 3485 | }) | ||
| 3486 | .pipe(Effect.forkChild); | ||
| 3487 | const terminal = yield* Fiber.join(terminalReceipt); | ||
| 3488 | assert.equal(terminal._tag, "Some"); | ||
| 3489 | assert.equal(nextSend.pollUnsafe(), undefined); | ||
| 3490 | // The canceled request must not hold the completion. The live request | ||
| 3491 | // still does, until its adapter response links it to the turn. | ||
| 3492 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0); | ||
| 3493 | yield* Deferred.succeed(nextReturnRelease, undefined); | ||
| 3494 | const nextTurn = yield* Fiber.join(nextSend); | ||
| 3495 | assert.equal(nextTurn.turnId, nextTurnId); | ||
| 3496 | |||
| 3497 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3498 | assert.equal(completed.length, 1); | ||
| 3499 | assert.deepInclude(completed[0]?.properties ?? {}, { | ||
| 3500 | model: "native-next", | ||
| 3501 | effort: "high", | ||
| 3502 | interactionMode: "plan", | ||
| 3503 | }); | ||
| 3504 | }), | ||
| 3505 | ); | ||
| 3506 | |||
| 3507 | it.effect("bounds deferred completions and drains them after sends are canceled", () => | ||
| 3508 | Effect.gen(function* () { | ||
| 3509 | recordedTurnAnalytics.reset(); | ||
| 3510 | const provider = yield* ProviderService.ProviderService; | ||
| 3511 | const threadId = asThreadId("thread-turn-analytics-bounded-deferred"); | ||
| 3512 | const allStarted = yield* Deferred.make<void>(); | ||
| 3513 | const sendRelease = yield* Deferred.make<void>(); | ||
| 3514 | const turnIds = Array.from({ length: 9 }, (_, index) => | ||
| 3515 | asTurnId(`turn-analytics-bounded-deferred-${index + 1}`), | ||
| 3516 | ); | ||
| 3517 | let startedCount = 0; | ||
| 3518 | yield* provider.startSession(threadId, { | ||
| 3519 | provider: CODEX_DRIVER, | ||
| 3520 | providerInstanceId: codexInstanceId, | ||
| 3521 | threadId, | ||
| 3522 | runtimeMode: "full-access", | ||
| 3523 | }); | ||
| 3524 | for (const turnId of turnIds) { | ||
| 3525 | primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) => | ||
| 3526 | Effect.gen(function* () { | ||
| 3527 | startedCount += 1; | ||
| 3528 | if (startedCount === turnIds.length) { | ||
| 3529 | yield* Deferred.succeed(allStarted, undefined); | ||
| 3530 | } | ||
| 3531 | yield* Deferred.await(sendRelease); | ||
| 3532 | return { threadId: input.threadId, turnId }; | ||
| 3533 | }), | ||
| 3534 | ); | ||
| 3535 | } | ||
| 3536 | |||
| 3537 | const runtimeEvents = yield* Stream.take(provider.streamEvents, turnIds.length * 2).pipe( | ||
| 3538 | Stream.runDrain, | ||
| 3539 | Effect.forkChild, | ||
| 3540 | ); | ||
| 3541 | yield* Effect.yieldNow; | ||
| 3542 | const sends = []; | ||
| 3543 | for (let index = 0; index < turnIds.length; index += 1) { | ||
| 3544 | sends.push( | ||
| 3545 | yield* provider | ||
| 3546 | .sendTurn({ | ||
| 3547 | threadId, | ||
| 3548 | input: `bounded deferred ${index + 1}`, | ||
| 3549 | attachments: [], | ||
| 3550 | interactionMode: index % 2 === 0 ? "default" : "plan", | ||
| 3551 | }) | ||
| 3552 | .pipe(Effect.forkChild), | ||
| 3553 | ); | ||
| 3554 | } | ||
| 3555 | yield* Deferred.await(allStarted); | ||
| 3556 | |||
| 3557 | for (const [index, turnId] of turnIds.entries()) { | ||
| 3558 | primaryAnalyticsCodex.emit({ | ||
| 3559 | type: "turn.started", | ||
| 3560 | eventId: asEventId(`evt-turn-analytics-bounded-deferred-start-${index + 1}`), | ||
| 3561 | provider: CODEX_DRIVER, | ||
| 3562 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3563 | threadId, | ||
| 3564 | turnId, | ||
| 3565 | payload: { model: `native-bounded-${index + 1}` }, | ||
| 3566 | }); | ||
| 3567 | primaryAnalyticsCodex.emit({ | ||
| 3568 | type: "turn.completed", | ||
| 3569 | eventId: asEventId(`evt-turn-analytics-bounded-deferred-complete-${index + 1}`), | ||
| 3570 | provider: CODEX_DRIVER, | ||
| 3571 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3572 | threadId, | ||
| 3573 | turnId, | ||
| 3574 | payload: { state: "completed" }, | ||
| 3575 | }); | ||
| 3576 | } | ||
| 3577 | yield* Fiber.join(runtimeEvents); | ||
| 3578 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 1); | ||
| 3579 | |||
| 3580 | for (const send of sends) { | ||
| 3581 | yield* Fiber.interrupt(send); | ||
| 3582 | } | ||
| 3583 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 9); | ||
| 3584 | }), | ||
| 3585 | ); | ||
| 3586 | |||
| 3587 | it.effect("flushes a deferred completion when the session stops", () => | ||
| 3588 | Effect.gen(function* () { | ||
| 3589 | recordedTurnAnalytics.reset(); | ||
| 3590 | const provider = yield* ProviderService.ProviderService; | ||
| 3591 | const threadId = asThreadId("thread-turn-analytics-stop-deferred"); | ||
| 3592 | const firstStarted = yield* Deferred.make<void>(); | ||
| 3593 | const secondStarted = yield* Deferred.make<void>(); | ||
| 3594 | const sendRelease = yield* Deferred.make<void>(); | ||
| 3595 | const turnId = asTurnId("turn-analytics-stop-deferred"); | ||
| 3596 | yield* provider.startSession(threadId, { | ||
| 3597 | provider: CODEX_DRIVER, | ||
| 3598 | providerInstanceId: codexInstanceId, | ||
| 3599 | threadId, | ||
| 3600 | runtimeMode: "full-access", | ||
| 3601 | }); | ||
| 3602 | primaryAnalyticsCodex.sendTurn | ||
| 3603 | .mockImplementationOnce(() => | ||
| 3604 | Effect.gen(function* () { | ||
| 3605 | yield* Deferred.succeed(firstStarted, undefined); | ||
| 3606 | yield* Deferred.await(sendRelease); | ||
| 3607 | return { threadId, turnId }; | ||
| 3608 | }), | ||
| 3609 | ) | ||
| 3610 | .mockImplementationOnce(() => | ||
| 3611 | Effect.gen(function* () { | ||
| 3612 | yield* Deferred.succeed(secondStarted, undefined); | ||
| 3613 | yield* Deferred.await(sendRelease); | ||
| 3614 | return { threadId, turnId: asTurnId("turn-analytics-stop-other") }; | ||
| 3615 | }), | ||
| 3616 | ); | ||
| 3617 | |||
| 3618 | const firstSend = yield* provider | ||
| 3619 | .sendTurn({ threadId, input: "first", attachments: [] }) | ||
| 3620 | .pipe(Effect.forkChild); | ||
| 3621 | yield* Deferred.await(firstStarted); | ||
| 3622 | const secondSend = yield* provider | ||
| 3623 | .sendTurn({ threadId, input: "second", attachments: [] }) | ||
| 3624 | .pipe(Effect.forkChild); | ||
| 3625 | yield* Deferred.await(secondStarted); | ||
| 3626 | |||
| 3627 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe( | ||
| 3628 | Stream.runDrain, | ||
| 3629 | Effect.forkChild, | ||
| 3630 | ); | ||
| 3631 | yield* Effect.yieldNow; | ||
| 3632 | primaryAnalyticsCodex.emit({ | ||
| 3633 | type: "turn.started", | ||
| 3634 | eventId: asEventId("evt-turn-analytics-stop-deferred-start"), | ||
| 3635 | provider: CODEX_DRIVER, | ||
| 3636 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3637 | threadId, | ||
| 3638 | turnId, | ||
| 3639 | payload: { model: "native-stop" }, | ||
| 3640 | }); | ||
| 3641 | primaryAnalyticsCodex.emit({ | ||
| 3642 | type: "turn.completed", | ||
| 3643 | eventId: asEventId("evt-turn-analytics-stop-deferred-complete"), | ||
| 3644 | provider: CODEX_DRIVER, | ||
| 3645 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3646 | threadId, | ||
| 3647 | turnId, | ||
| 3648 | payload: { state: "completed" }, | ||
| 3649 | }); | ||
| 3650 | yield* Fiber.join(runtimeEvents); | ||
| 3651 | assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0); | ||
| 3652 | |||
| 3653 | yield* provider.stopSession({ threadId }); | ||
| 3654 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3655 | assert.equal(completed.length, 1); | ||
| 3656 | assert.equal(completed[0]?.properties?.model, "native-stop"); | ||
| 3657 | yield* Fiber.interrupt(firstSend); | ||
| 3658 | yield* Fiber.interrupt(secondSend); | ||
| 3659 | }), | ||
| 3660 | ); | ||
| 3661 | |||
| 3662 | it.effect("keeps the first metadata when steering reuses a rerouted turn", () => | ||
| 3663 | Effect.gen(function* () { | ||
| 3664 | recordedTurnAnalytics.reset(); | ||
| 3665 | const provider = yield* ProviderService.ProviderService; | ||
| 3666 | const threadId = asThreadId("thread-turn-analytics-steering"); | ||
| 3667 | yield* provider.startSession(threadId, { | ||
| 3668 | provider: CODEX_DRIVER, | ||
| 3669 | providerInstanceId: codexInstanceId, | ||
| 3670 | threadId, | ||
| 3671 | runtimeMode: "auto", | ||
| 3672 | }); | ||
| 3673 | const firstTurn = yield* provider.sendTurn({ | ||
| 3674 | threadId, | ||
| 3675 | input: "start", | ||
| 3676 | attachments: [], | ||
| 3677 | interactionMode: "default", | ||
| 3678 | modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-sol", [ | ||
| 3679 | { id: "reasoningEffort", value: "high" }, | ||
| 3680 | ]), | ||
| 3681 | }); | ||
| 3682 | yield* advanceTestClock(10); | ||
| 3683 | |||
| 3684 | const reroutedEvent = yield* Stream.take(provider.streamEvents, 1).pipe( | ||
| 3685 | Stream.runDrain, | ||
| 3686 | Effect.forkChild, | ||
| 3687 | ); | ||
| 3688 | yield* Effect.yieldNow; | ||
| 3689 | primaryAnalyticsCodex.emit({ | ||
| 3690 | type: "model.rerouted", | ||
| 3691 | eventId: asEventId("evt-turn-analytics-rerouted"), | ||
| 3692 | provider: CODEX_DRIVER, | ||
| 3693 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3694 | threadId, | ||
| 3695 | turnId: firstTurn.turnId, | ||
| 3696 | payload: { | ||
| 3697 | fromModel: "gpt-5.6-sol", | ||
| 3698 | toModel: "gpt-5.6-terra", | ||
| 3699 | reason: "capacity", | ||
| 3700 | }, | ||
| 3701 | }); | ||
| 3702 | yield* Fiber.join(reroutedEvent); | ||
| 3703 | yield* advanceTestClock(15); | ||
| 3704 | |||
| 3705 | const steeredTurn = yield* provider.sendTurn({ | ||
| 3706 | threadId, | ||
| 3707 | input: "steer", | ||
| 3708 | attachments: [], | ||
| 3709 | interactionMode: "plan", | ||
| 3710 | modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-terra", [ | ||
| 3711 | { id: "reasoningEffort", value: "low" }, | ||
| 3712 | ]), | ||
| 3713 | }); | ||
| 3714 | assert.equal(steeredTurn.turnId, firstTurn.turnId); | ||
| 3715 | yield* advanceTestClock(20); | ||
| 3716 | |||
| 3717 | const completedEvent = yield* Stream.take(provider.streamEvents, 1).pipe( | ||
| 3718 | Stream.runDrain, | ||
| 3719 | Effect.forkChild, | ||
| 3720 | ); | ||
| 3721 | yield* Effect.yieldNow; | ||
| 3722 | primaryAnalyticsCodex.emit({ | ||
| 3723 | type: "turn.completed", | ||
| 3724 | eventId: asEventId("evt-turn-analytics-steered-complete"), | ||
| 3725 | provider: CODEX_DRIVER, | ||
| 3726 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3727 | threadId, | ||
| 3728 | turnId: firstTurn.turnId, | ||
| 3729 | payload: { | ||
| 3730 | state: "completed", | ||
| 3731 | tokenUsage: { | ||
| 3732 | usageStatus: "complete", | ||
| 3733 | usageScope: "main_agent", | ||
| 3734 | inputTokens: 500, | ||
| 3735 | outputTokens: 100, | ||
| 3736 | hasSubagents: false, | ||
| 3737 | }, | ||
| 3738 | }, | ||
| 3739 | }); | ||
| 3740 | yield* Fiber.join(completedEvent); | ||
| 3741 | |||
| 3742 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3743 | assert.equal(completed.length, 1); | ||
| 3744 | assert.equal(completed[0]?.properties?.model, "gpt-5.6-sol"); | ||
| 3745 | assert.equal(completed[0]?.properties?.effort, "high"); | ||
| 3746 | assert.equal(completed[0]?.properties?.interactionMode, "default"); | ||
| 3747 | assert.equal(completed[0]?.properties?.mixedModels, true); | ||
| 3748 | assert.equal(completed[0]?.properties?.durationMs, 45); | ||
| 3749 | }), | ||
| 3750 | ); | ||
| 3751 | |||
| 3752 | it.effect("bounds active metadata while preserving recent delayed completions", () => | ||
| 3753 | Effect.gen(function* () { | ||
| 3754 | recordedTurnAnalytics.reset(); | ||
| 3755 | const provider = yield* ProviderService.ProviderService; | ||
| 3756 | const threadId = asThreadId("thread-turn-analytics-bounded-active"); | ||
| 3757 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 12).pipe( | ||
| 3758 | Stream.runDrain, | ||
| 3759 | Effect.forkChild, | ||
| 3760 | ); | ||
| 3761 | yield* Effect.yieldNow; | ||
| 3762 | |||
| 3763 | for (let index = 1; index <= 10; index += 1) { | ||
| 3764 | primaryAnalyticsCodex.emit({ | ||
| 3765 | type: "turn.started", | ||
| 3766 | eventId: asEventId(`evt-turn-analytics-bounded-start-${index}`), | ||
| 3767 | provider: CODEX_DRIVER, | ||
| 3768 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3769 | threadId, | ||
| 3770 | turnId: asTurnId(`turn-analytics-bounded-${index}`), | ||
| 3771 | payload: { model: `model-${index}` }, | ||
| 3772 | }); | ||
| 3773 | } | ||
| 3774 | for (const index of [3, 1]) { | ||
| 3775 | primaryAnalyticsCodex.emit({ | ||
| 3776 | type: "turn.completed", | ||
| 3777 | eventId: asEventId(`evt-turn-analytics-bounded-complete-${index}`), | ||
| 3778 | provider: CODEX_DRIVER, | ||
| 3779 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3780 | threadId, | ||
| 3781 | turnId: asTurnId(`turn-analytics-bounded-${index}`), | ||
| 3782 | payload: { state: "completed" }, | ||
| 3783 | }); | ||
| 3784 | } | ||
| 3785 | yield* Fiber.join(runtimeEvents); | ||
| 3786 | |||
| 3787 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3788 | assert.equal(completed.length, 2); | ||
| 3789 | assert.equal(completed[0]?.properties?.model, "model-3"); | ||
| 3790 | assert.notProperty(completed[1]?.properties ?? {}, "model"); | ||
| 3791 | }), | ||
| 3792 | ); | ||
| 3793 | |||
| 3794 | it.effect("separates provider instances and omits unavailable counts", () => | ||
| 3795 | Effect.gen(function* () { | ||
| 3796 | recordedTurnAnalytics.reset(); | ||
| 3797 | const provider = yield* ProviderService.ProviderService; | ||
| 3798 | const threadId = asThreadId("thread-turn-analytics-instances"); | ||
| 3799 | const turnId = asTurnId("turn-shared-between-instances"); | ||
| 3800 | const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe( | ||
| 3801 | Stream.runDrain, | ||
| 3802 | Effect.forkChild, | ||
| 3803 | ); | ||
| 3804 | yield* Effect.yieldNow; | ||
| 3805 | const event: LegacyProviderRuntimeEvent = { | ||
| 3806 | type: "turn.completed", | ||
| 3807 | eventId: asEventId("evt-turn-analytics-primary-instance"), | ||
| 3808 | provider: CODEX_DRIVER, | ||
| 3809 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3810 | threadId, | ||
| 3811 | turnId, | ||
| 3812 | payload: { | ||
| 3813 | state: "completed", | ||
| 3814 | tokenUsage: { | ||
| 3815 | usageStatus: "unavailable", | ||
| 3816 | usageScope: "main_agent", | ||
| 3817 | hasSubagents: false, | ||
| 3818 | }, | ||
| 3819 | }, | ||
| 3820 | }; | ||
| 3821 | primaryAnalyticsCodex.emit(event); | ||
| 3822 | secondaryAnalyticsCodex.emit({ | ||
| 3823 | ...event, | ||
| 3824 | eventId: asEventId("evt-turn-analytics-secondary-instance"), | ||
| 3825 | }); | ||
| 3826 | yield* Fiber.join(runtimeEvents); | ||
| 3827 | |||
| 3828 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3829 | assert.equal(completed.length, 2); | ||
| 3830 | for (const entry of completed) { | ||
| 3831 | assert.deepEqual(entry.properties, { | ||
| 3832 | provider: CODEX_DRIVER, | ||
| 3833 | terminalStatus: "completed", | ||
| 3834 | usageStatus: "unavailable", | ||
| 3835 | usageScope: "main_agent", | ||
| 3836 | hasSubagents: false, | ||
| 3837 | }); | ||
| 3838 | } | ||
| 3839 | }), | ||
| 3840 | ); | ||
| 3841 | |||
| 3842 | it.effect("records known token counts for an interrupted turn", () => | ||
| 3843 | Effect.gen(function* () { | ||
| 3844 | recordedTurnAnalytics.reset(); | ||
| 3845 | const provider = yield* ProviderService.ProviderService; | ||
| 3846 | const threadId = asThreadId("thread-turn-analytics-interrupted"); | ||
| 3847 | yield* provider.startSession(threadId, { | ||
| 3848 | provider: CODEX_DRIVER, | ||
| 3849 | providerInstanceId: codexInstanceId, | ||
| 3850 | threadId, | ||
| 3851 | runtimeMode: "approval-required", | ||
| 3852 | }); | ||
| 3853 | const turn = yield* provider.sendTurn({ | ||
| 3854 | threadId, | ||
| 3855 | input: "stop after some work", | ||
| 3856 | attachments: [], | ||
| 3857 | }); | ||
| 3858 | |||
| 3859 | const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe( | ||
| 3860 | Stream.runDrain, | ||
| 3861 | Effect.forkChild, | ||
| 3862 | ); | ||
| 3863 | yield* Effect.yieldNow; | ||
| 3864 | primaryAnalyticsCodex.emit({ | ||
| 3865 | type: "turn.aborted", | ||
| 3866 | eventId: asEventId("evt-turn-analytics-interrupted"), | ||
| 3867 | provider: CODEX_DRIVER, | ||
| 3868 | createdAt: "2026-01-01T00:00:00.000Z", | ||
| 3869 | threadId, | ||
| 3870 | turnId: turn.turnId, | ||
| 3871 | payload: { | ||
| 3872 | reason: "Interrupted by user", | ||
| 3873 | tokenUsage: { | ||
| 3874 | usageStatus: "partial", | ||
| 3875 | usageScope: "main_agent", | ||
| 3876 | inputTokens: 120, | ||
| 3877 | outputTokens: 30, | ||
| 3878 | hasSubagents: true, | ||
| 3879 | }, | ||
| 3880 | }, | ||
| 3881 | }); | ||
| 3882 | yield* Fiber.join(runtimeEvent); | ||
| 3883 | |||
| 3884 | const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed"); | ||
| 3885 | assert.equal(completed.length, 1); | ||
| 3886 | assert.equal(completed[0]?.properties?.terminalStatus, "interrupted"); | ||
| 3887 | assert.equal(completed[0]?.properties?.usageStatus, "partial"); | ||
| 3888 | assert.equal(completed[0]?.properties?.inputTokens, 120); | ||
| 3889 | assert.equal(completed[0]?.properties?.outputTokens, 30); | ||
| 3890 | assert.equal(completed[0]?.properties?.hasSubagents, true); | ||
| 3891 | }), | ||
| 3892 | ); | ||
| 3893 | }); | ||
| 3894 | |||
| 2662 | const validation = makeProviderServiceLayer(); | 3895 | const validation = makeProviderServiceLayer(); |
| 2663 | validation.layer("ProviderServiceLive validation", (it) => { | 3896 | validation.layer("ProviderServiceLive validation", (it) => { |
| 2664 | it.effect("rejects citation-expanded input over the provider character limit", () => | 3897 | it.effect("rejects citation-expanded input over the provider character limit", () => |
apps/server/src/provider/Layers/ProviderService.ts+484-1
| ... | @@ -31,6 +31,7 @@ import { | ... | @@ -31,6 +31,7 @@ import { |
| 31 | } from "@t3tools/contracts"; | 31 | } from "@t3tools/contracts"; |
| 32 | import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; | 32 | import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; |
| 33 | import { causeErrorTag } from "@t3tools/shared/observability"; | 33 | import { causeErrorTag } from "@t3tools/shared/observability"; |
| 34 | import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; | ||
| 34 | import * as DateTime from "effect/DateTime"; | 35 | import * as DateTime from "effect/DateTime"; |
| 35 | import * as Deferred from "effect/Deferred"; | 36 | import * as Deferred from "effect/Deferred"; |
| 36 | import * as Effect from "effect/Effect"; | 37 | import * as Effect from "effect/Effect"; |
| ... | @@ -96,6 +97,75 @@ export interface ProviderServiceLiveOptions { | ... | @@ -96,6 +97,75 @@ export interface ProviderServiceLiveOptions { |
| 96 | readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; | 97 | readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; |
| 97 | } | 98 | } |
| 98 | 99 | ||
| 100 | interface TurnAnalyticsMetadata { | ||
| 101 | readonly requestId: number; | ||
| 102 | readonly provider: ProviderDriverKind; | ||
| 103 | readonly startedAtMs: number; | ||
| 104 | readonly mixedModels: boolean; | ||
| 105 | readonly model?: string; | ||
| 106 | readonly effort?: string; | ||
| 107 | readonly interactionMode?: string; | ||
| 108 | readonly runtimeMode?: string; | ||
| 109 | } | ||
| 110 | |||
| 111 | interface ActiveTurnAnalytics { | ||
| 112 | readonly metadata: TurnAnalyticsMetadata; | ||
| 113 | readonly requestAssociated: boolean; | ||
| 114 | } | ||
| 115 | |||
| 116 | interface DeferredTurnAnalyticsCompletion { | ||
| 117 | readonly completionKey: string; | ||
| 118 | readonly completedAtMs: number; | ||
| 119 | readonly terminalProperties: Readonly<Record<string, unknown>>; | ||
| 120 | } | ||
| 121 | |||
| 122 | interface TurnAnalyticsSessionState { | ||
| 123 | readonly pendingByRequestId: Map<number, TurnAnalyticsMetadata>; | ||
| 124 | readonly activeByTurnId: Map<string, ActiveTurnAnalytics>; | ||
| 125 | readonly deferredCompletionsByTurnId: Map<string, DeferredTurnAnalyticsCompletion>; | ||
| 126 | } | ||
| 127 | |||
| 128 | interface TurnAnalyticsState { | ||
| 129 | readonly sessions: Map<string, TurnAnalyticsSessionState>; | ||
| 130 | readonly completedKeys: Set<string>; | ||
| 131 | readonly completedOrder: Array<string>; | ||
| 132 | } | ||
| 133 | |||
| 134 | const MAX_COMPLETED_TURN_ANALYTICS_KEYS = 512; | ||
| 135 | const MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION = 8; | ||
| 136 | |||
| 137 | function setActiveTurnAnalytics( | ||
| 138 | session: TurnAnalyticsSessionState, | ||
| 139 | turnId: string, | ||
| 140 | active: ActiveTurnAnalytics, | ||
| 141 | ): void { | ||
| 142 | session.activeByTurnId.set(turnId, active); | ||
| 143 | while (session.activeByTurnId.size > MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION) { | ||
| 144 | const oldestTurnId = session.activeByTurnId.keys().next().value; | ||
| 145 | if (oldestTurnId === undefined) return; | ||
| 146 | session.activeByTurnId.delete(oldestTurnId); | ||
| 147 | } | ||
| 148 | } | ||
| 149 | |||
| 150 | function turnAnalyticsSessionKey(instanceId: ProviderInstanceId, threadId: ThreadId): string { | ||
| 151 | return `${String(instanceId)}\u0000${String(threadId)}`; | ||
| 152 | } | ||
| 153 | |||
| 154 | function turnAnalyticsCompletionKey( | ||
| 155 | instanceId: ProviderInstanceId, | ||
| 156 | threadId: ThreadId, | ||
| 157 | turnId: string, | ||
| 158 | ): string { | ||
| 159 | return `${turnAnalyticsSessionKey(instanceId, threadId)}\u0000${turnId}`; | ||
| 160 | } | ||
| 161 | |||
| 162 | function turnEffort(modelSelection: ProviderSendTurnInput["modelSelection"]): string | undefined { | ||
| 163 | return ( | ||
| 164 | getModelSelectionStringOptionValue(modelSelection, "reasoningEffort") ?? | ||
| 165 | getModelSelectionStringOptionValue(modelSelection, "effort") | ||
| 166 | ); | ||
| 167 | } | ||
| 168 | |||
| 99 | type ProviderServiceMethod<Name extends keyof ProviderService.ProviderService["Service"]> = | 169 | type ProviderServiceMethod<Name extends keyof ProviderService.ProviderService["Service"]> = |
| 100 | ProviderService.ProviderService["Service"][Name]; | 170 | ProviderService.ProviderService["Service"][Name]; |
| 101 | 171 | ||
| ... | @@ -258,7 +328,366 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -258,7 +328,366 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 258 | yield* Deferred.succeed(pending.completion, terminal); | 328 | yield* Deferred.succeed(pending.completion, terminal); |
| 259 | return true; | 329 | return true; |
| 260 | }); | 330 | }); |
| 331 | const turnAnalytics = yield* Ref.make<TurnAnalyticsState>({ | ||
| 332 | sessions: new Map(), | ||
| 333 | completedKeys: new Set(), | ||
| 334 | completedOrder: [], | ||
| 335 | }); | ||
| 336 | let turnAnalyticsRequestId = 0; | ||
| 261 | const nowIso = Effect.map(DateTime.now, DateTime.formatIso); | 337 | const nowIso = Effect.map(DateTime.now, DateTime.formatIso); |
| 338 | |||
| 339 | const finishTurnAnalytics = ( | ||
| 340 | state: TurnAnalyticsState, | ||
| 341 | input: { | ||
| 342 | readonly sessionKey: string; | ||
| 343 | readonly turnId: string; | ||
| 344 | readonly completion: DeferredTurnAnalyticsCompletion; | ||
| 345 | }, | ||
| 346 | ): Readonly<Record<string, unknown>> | undefined => { | ||
| 347 | if (state.completedKeys.has(input.completion.completionKey)) return undefined; | ||
| 348 | state.completedKeys.add(input.completion.completionKey); | ||
| 349 | state.completedOrder.push(input.completion.completionKey); | ||
| 350 | while (state.completedOrder.length > MAX_COMPLETED_TURN_ANALYTICS_KEYS) { | ||
| 351 | const expired = state.completedOrder.shift(); | ||
| 352 | if (expired) state.completedKeys.delete(expired); | ||
| 353 | } | ||
| 354 | |||
| 355 | const session = state.sessions.get(input.sessionKey); | ||
| 356 | const metadata = session?.activeByTurnId.get(input.turnId)?.metadata; | ||
| 357 | session?.activeByTurnId.delete(input.turnId); | ||
| 358 | session?.deferredCompletionsByTurnId.delete(input.turnId); | ||
| 359 | if ( | ||
| 360 | session && | ||
| 361 | session.activeByTurnId.size === 0 && | ||
| 362 | session.pendingByRequestId.size === 0 && | ||
| 363 | session.deferredCompletionsByTurnId.size === 0 | ||
| 364 | ) { | ||
| 365 | state.sessions.delete(input.sessionKey); | ||
| 366 | } | ||
| 367 | |||
| 368 | return { | ||
| 369 | ...input.completion.terminalProperties, | ||
| 370 | ...(metadata?.model ? { model: metadata.model } : {}), | ||
| 371 | ...(metadata?.effort ? { effort: metadata.effort } : {}), | ||
| 372 | ...(metadata?.interactionMode ? { interactionMode: metadata.interactionMode } : {}), | ||
| 373 | ...(metadata?.runtimeMode ? { runtimeMode: metadata.runtimeMode } : {}), | ||
| 374 | ...(metadata ? { mixedModels: metadata.mixedModels } : {}), | ||
| 375 | ...(metadata | ||
| 376 | ? { durationMs: Math.max(0, input.completion.completedAtMs - metadata.startedAtMs) } | ||
| 377 | : {}), | ||
| 378 | }; | ||
| 379 | }; | ||
| 380 | |||
| 381 | const recordCompletedTurnProperties = ( | ||
| 382 | properties: ReadonlyArray<Readonly<Record<string, unknown>>>, | ||
| 383 | ) => | ||
| 384 | Effect.forEach(properties, (entry) => analytics.record("provider.turn.completed", entry), { | ||
| 385 | discard: true, | ||
| 386 | }); | ||
| 387 | |||
| 388 | const clearTurnAnalyticsSession = (providerInstanceId: ProviderInstanceId, threadId: ThreadId) => | ||
| 389 | Effect.gen(function* () { | ||
| 390 | const properties = yield* Ref.modify(turnAnalytics, (state) => { | ||
| 391 | const sessionKey = turnAnalyticsSessionKey(providerInstanceId, threadId); | ||
| 392 | const session = state.sessions.get(sessionKey); | ||
| 393 | const completed: Array<Readonly<Record<string, unknown>>> = []; | ||
| 394 | if (session) { | ||
| 395 | for (const [turnId, completion] of session.deferredCompletionsByTurnId) { | ||
| 396 | const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion }); | ||
| 397 | if (entry) completed.push(entry); | ||
| 398 | } | ||
| 399 | } | ||
| 400 | state.sessions.delete(sessionKey); | ||
| 401 | return [completed, state] as const; | ||
| 402 | }); | ||
| 403 | yield* recordCompletedTurnProperties(properties); | ||
| 404 | }); | ||
| 405 | |||
| 406 | const beginTurnAnalytics = Effect.fn("beginTurnAnalytics")(function* (input: { | ||
| 407 | readonly providerInstanceId: ProviderInstanceId; | ||
| 408 | readonly provider: ProviderDriverKind; | ||
| 409 | readonly threadId: ThreadId; | ||
| 410 | readonly modelSelection: ProviderSendTurnInput["modelSelection"]; | ||
| 411 | readonly interactionMode: ProviderSendTurnInput["interactionMode"]; | ||
| 412 | readonly runtimeMode: string | undefined; | ||
| 413 | }) { | ||
| 414 | const startedAtMs = DateTime.toEpochMillis(yield* DateTime.now); | ||
| 415 | turnAnalyticsRequestId += 1; | ||
| 416 | const requestId = turnAnalyticsRequestId; | ||
| 417 | const effort = turnEffort(input.modelSelection); | ||
| 418 | return yield* Ref.modify(turnAnalytics, (state) => { | ||
| 419 | const key = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId); | ||
| 420 | const session = state.sessions.get(key) ?? { | ||
| 421 | pendingByRequestId: new Map(), | ||
| 422 | activeByTurnId: new Map(), | ||
| 423 | deferredCompletionsByTurnId: new Map(), | ||
| 424 | }; | ||
| 425 | const metadata: TurnAnalyticsMetadata = { | ||
| 426 | provider: input.provider, | ||
| 427 | startedAtMs, | ||
| 428 | mixedModels: false, | ||
| 429 | requestId, | ||
| 430 | ...(input.modelSelection?.model ? { model: input.modelSelection.model } : {}), | ||
| 431 | ...(effort ? { effort } : {}), | ||
| 432 | ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}), | ||
| 433 | ...(input.runtimeMode ? { runtimeMode: input.runtimeMode } : {}), | ||
| 434 | }; | ||
| 435 | session.pendingByRequestId.set(requestId, metadata); | ||
| 436 | state.sessions.set(key, session); | ||
| 437 | return [metadata, state] as const; | ||
| 438 | }); | ||
| 439 | }); | ||
| 440 | |||
| 441 | const clearPendingTurnAnalytics = (input: { | ||
| 442 | readonly providerInstanceId: ProviderInstanceId; | ||
| 443 | readonly threadId: ThreadId; | ||
| 444 | readonly requestId: number; | ||
| 445 | }) => | ||
| 446 | Effect.gen(function* () { | ||
| 447 | const properties = yield* Ref.modify(turnAnalytics, (state) => { | ||
| 448 | const sessionKey = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId); | ||
| 449 | const session = state.sessions.get(sessionKey); | ||
| 450 | if (!session) | ||
| 451 | return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const; | ||
| 452 | session.pendingByRequestId.delete(input.requestId); | ||
| 453 | const completed: Array<Readonly<Record<string, unknown>>> = []; | ||
| 454 | if (session.pendingByRequestId.size === 0) { | ||
| 455 | for (const [turnId, completion] of session.deferredCompletionsByTurnId) { | ||
| 456 | const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion }); | ||
| 457 | if (entry) completed.push(entry); | ||
| 458 | } | ||
| 459 | } | ||
| 460 | if ( | ||
| 461 | session.activeByTurnId.size === 0 && | ||
| 462 | session.pendingByRequestId.size === 0 && | ||
| 463 | session.deferredCompletionsByTurnId.size === 0 | ||
| 464 | ) { | ||
| 465 | state.sessions.delete(sessionKey); | ||
| 466 | } | ||
| 467 | return [completed, state] as const; | ||
| 468 | }); | ||
| 469 | yield* recordCompletedTurnProperties(properties); | ||
| 470 | }); | ||
| 471 | |||
| 472 | const associateTurnAnalytics = (input: { | ||
| 473 | readonly providerInstanceId: ProviderInstanceId; | ||
| 474 | readonly threadId: ThreadId; | ||
| 475 | readonly turnId: string; | ||
| 476 | readonly metadata: TurnAnalyticsMetadata; | ||
| 477 | }) => | ||
| 478 | Effect.gen(function* () { | ||
| 479 | const properties = yield* Ref.modify(turnAnalytics, (state) => { | ||
| 480 | const completionKey = turnAnalyticsCompletionKey( | ||
| 481 | input.providerInstanceId, | ||
| 482 | input.threadId, | ||
| 483 | input.turnId, | ||
| 484 | ); | ||
| 485 | const sessionKey = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId); | ||
| 486 | const session = state.sessions.get(sessionKey); | ||
| 487 | if (!session || state.completedKeys.has(completionKey)) { | ||
| 488 | if (session) { | ||
| 489 | session.pendingByRequestId.delete(input.metadata.requestId); | ||
| 490 | if ( | ||
| 491 | session.activeByTurnId.size === 0 && | ||
| 492 | session.pendingByRequestId.size === 0 && | ||
| 493 | session.deferredCompletionsByTurnId.size === 0 | ||
| 494 | ) { | ||
| 495 | state.sessions.delete(sessionKey); | ||
| 496 | } | ||
| 497 | } | ||
| 498 | return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const; | ||
| 499 | } | ||
| 500 | const existing = session.activeByTurnId.get(input.turnId); | ||
| 501 | const existingMetadata = existing?.metadata; | ||
| 502 | const base = existing?.requestAssociated ? existing.metadata : input.metadata; | ||
| 503 | setActiveTurnAnalytics(session, input.turnId, { | ||
| 504 | requestAssociated: true, | ||
| 505 | metadata: { | ||
| 506 | ...base, | ||
| 507 | ...(existingMetadata?.model | ||
| 508 | ? { model: existingMetadata.model } | ||
| 509 | : input.metadata.model | ||
| 510 | ? { model: input.metadata.model } | ||
| 511 | : {}), | ||
| 512 | ...(existingMetadata?.effort | ||
| 513 | ? { effort: existingMetadata.effort } | ||
| 514 | : input.metadata.effort | ||
| 515 | ? { effort: input.metadata.effort } | ||
| 516 | : {}), | ||
| 517 | ...(base?.interactionMode | ||
| 518 | ? {} | ||
| 519 | : input.metadata.interactionMode | ||
| 520 | ? { interactionMode: input.metadata.interactionMode } | ||
| 521 | : {}), | ||
| 522 | ...(base?.runtimeMode | ||
| 523 | ? {} | ||
| 524 | : input.metadata.runtimeMode | ||
| 525 | ? { runtimeMode: input.metadata.runtimeMode } | ||
| 526 | : {}), | ||
| 527 | mixedModels: existingMetadata?.mixedModels ?? input.metadata.mixedModels, | ||
| 528 | }, | ||
| 529 | }); | ||
| 530 | session.pendingByRequestId.delete(input.metadata.requestId); | ||
| 531 | const completion = session.deferredCompletionsByTurnId.get(input.turnId); | ||
| 532 | const completed = completion | ||
| 533 | ? finishTurnAnalytics(state, { | ||
| 534 | sessionKey, | ||
| 535 | turnId: input.turnId, | ||
| 536 | completion, | ||
| 537 | }) | ||
| 538 | : undefined; | ||
| 539 | return [completed ? [completed] : [], state] as const; | ||
| 540 | }); | ||
| 541 | yield* recordCompletedTurnProperties(properties); | ||
| 542 | }); | ||
| 543 | |||
| 544 | const observeTurnStartedForAnalytics = Effect.fn("observeTurnStartedForAnalytics")(function* ( | ||
| 545 | source: { readonly instanceId: ProviderInstanceId; readonly provider: ProviderDriverKind }, | ||
| 546 | event: Extract<ProviderRuntimeEvent, { readonly type: "turn.started" }>, | ||
| 547 | ) { | ||
| 548 | if (!event.turnId) return; | ||
| 549 | const observedAtMs = DateTime.toEpochMillis(yield* DateTime.now); | ||
| 550 | yield* Ref.update(turnAnalytics, (state) => { | ||
| 551 | const completionKey = turnAnalyticsCompletionKey( | ||
| 552 | source.instanceId, | ||
| 553 | event.threadId, | ||
| 554 | String(event.turnId), | ||
| 555 | ); | ||
| 556 | if (state.completedKeys.has(completionKey)) return state; | ||
| 557 | const sessionKey = turnAnalyticsSessionKey(source.instanceId, event.threadId); | ||
| 558 | const session = state.sessions.get(sessionKey) ?? { | ||
| 559 | pendingByRequestId: new Map(), | ||
| 560 | activeByTurnId: new Map(), | ||
| 561 | deferredCompletionsByTurnId: new Map(), | ||
| 562 | }; | ||
| 563 | // A start never binds send metadata on its own. Claude can start a | ||
| 564 | // synthetic turn for leftover agent output while sendTurn is still | ||
| 565 | // preparing the real turn, so only the adapter's sendTurn response | ||
| 566 | // links a request to its turn. Completions that land before that | ||
| 567 | // response wait in deferredCompletionsByTurnId. | ||
| 568 | const current = session.activeByTurnId.get(String(event.turnId)); | ||
| 569 | const metadata: TurnAnalyticsMetadata = { | ||
| 570 | ...(current?.metadata ?? { | ||
| 571 | requestId: ++turnAnalyticsRequestId, | ||
| 572 | provider: source.provider, | ||
| 573 | startedAtMs: observedAtMs, | ||
| 574 | mixedModels: false, | ||
| 575 | }), | ||
| 576 | ...(event.payload.model ? { model: event.payload.model } : {}), | ||
| 577 | ...(event.payload.effort ? { effort: event.payload.effort } : {}), | ||
| 578 | }; | ||
| 579 | setActiveTurnAnalytics(session, String(event.turnId), { | ||
| 580 | metadata, | ||
| 581 | requestAssociated: current?.requestAssociated ?? false, | ||
| 582 | }); | ||
| 583 | state.sessions.set(sessionKey, session); | ||
| 584 | return state; | ||
| 585 | }); | ||
| 586 | }); | ||
| 587 | |||
| 588 | const observeModelReroutedForAnalytics = ( | ||
| 589 | source: { readonly instanceId: ProviderInstanceId }, | ||
| 590 | event: Extract<ProviderRuntimeEvent, { readonly type: "model.rerouted" }>, | ||
| 591 | ) => | ||
| 592 | Ref.update(turnAnalytics, (state) => { | ||
| 593 | const session = state.sessions.get( | ||
| 594 | turnAnalyticsSessionKey(source.instanceId, event.threadId), | ||
| 595 | ); | ||
| 596 | if (!session) return state; | ||
| 597 | if (event.turnId) { | ||
| 598 | const current = session.activeByTurnId.get(String(event.turnId)); | ||
| 599 | if (current) { | ||
| 600 | session.activeByTurnId.set(String(event.turnId), { | ||
| 601 | ...current, | ||
| 602 | metadata: { ...current.metadata, mixedModels: true }, | ||
| 603 | }); | ||
| 604 | } | ||
| 605 | } else { | ||
| 606 | for (const [turnId, current] of session.activeByTurnId) { | ||
| 607 | session.activeByTurnId.set(turnId, { | ||
| 608 | ...current, | ||
| 609 | metadata: { ...current.metadata, mixedModels: true }, | ||
| 610 | }); | ||
| 611 | } | ||
| 612 | } | ||
| 613 | return state; | ||
| 614 | }); | ||
| 615 | |||
| 616 | const recordTurnCompletedAnalytics = Effect.fn("recordTurnCompletedAnalytics")(function* ( | ||
| 617 | source: { readonly instanceId: ProviderInstanceId; readonly provider: ProviderDriverKind }, | ||
| 618 | event: Extract<ProviderRuntimeEvent, { readonly type: "turn.completed" | "turn.aborted" }>, | ||
| 619 | ) { | ||
| 620 | if (!event.turnId) return; | ||
| 621 | const completedAtMs = DateTime.toEpochMillis(yield* DateTime.now); | ||
| 622 | const tokenUsage = event.payload.tokenUsage; | ||
| 623 | const completion: DeferredTurnAnalyticsCompletion = { | ||
| 624 | completionKey: turnAnalyticsCompletionKey( | ||
| 625 | source.instanceId, | ||
| 626 | event.threadId, | ||
| 627 | String(event.turnId), | ||
| 628 | ), | ||
| 629 | completedAtMs, | ||
| 630 | terminalProperties: { | ||
| 631 | provider: source.provider, | ||
| 632 | terminalStatus: | ||
| 633 | event.type === "turn.completed" | ||
| 634 | ? event.payload.state | ||
| 635 | : event.payload.reason.toLowerCase().includes("interrupt") | ||
| 636 | ? "interrupted" | ||
| 637 | : "cancelled", | ||
| 638 | usageStatus: tokenUsage?.usageStatus ?? "unavailable", | ||
| 639 | usageScope: tokenUsage?.usageScope ?? "main_agent", | ||
| 640 | ...(tokenUsage ? { hasSubagents: tokenUsage.hasSubagents } : {}), | ||
| 641 | ...(tokenUsage?.inputTokens !== undefined ? { inputTokens: tokenUsage.inputTokens } : {}), | ||
| 642 | ...(tokenUsage?.cachedInputTokens !== undefined | ||
| 643 | ? { cachedInputTokens: tokenUsage.cachedInputTokens } | ||
| 644 | : {}), | ||
| 645 | ...(tokenUsage?.cacheCreationTokens !== undefined | ||
| 646 | ? { cacheCreationTokens: tokenUsage.cacheCreationTokens } | ||
| 647 | : {}), | ||
| 648 | ...(tokenUsage?.outputTokens !== undefined | ||
| 649 | ? { outputTokens: tokenUsage.outputTokens } | ||
| 650 | : {}), | ||
| 651 | ...(tokenUsage?.reasoningTokens !== undefined | ||
| 652 | ? { reasoningTokens: tokenUsage.reasoningTokens } | ||
| 653 | : {}), | ||
| 654 | }, | ||
| 655 | }; | ||
| 656 | const properties = yield* Ref.modify(turnAnalytics, (state) => { | ||
| 657 | if (state.completedKeys.has(completion.completionKey)) { | ||
| 658 | return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const; | ||
| 659 | } | ||
| 660 | const turnId = String(event.turnId); | ||
| 661 | const sessionKey = turnAnalyticsSessionKey(source.instanceId, event.threadId); | ||
| 662 | const session = state.sessions.get(sessionKey); | ||
| 663 | if (session?.deferredCompletionsByTurnId.has(turnId)) { | ||
| 664 | return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const; | ||
| 665 | } | ||
| 666 | const active = session?.activeByTurnId.get(turnId); | ||
| 667 | const needsAssociation = | ||
| 668 | (session?.pendingByRequestId.size ?? 0) > 0 && active?.requestAssociated !== true; | ||
| 669 | if (!session || !needsAssociation) { | ||
| 670 | const completed = finishTurnAnalytics(state, { sessionKey, turnId, completion }); | ||
| 671 | return [completed ? [completed] : [], state] as const; | ||
| 672 | } | ||
| 673 | |||
| 674 | session.deferredCompletionsByTurnId.set(turnId, completion); | ||
| 675 | const completed: Array<Readonly<Record<string, unknown>>> = []; | ||
| 676 | while (session.deferredCompletionsByTurnId.size > MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION) { | ||
| 677 | const oldest = session.deferredCompletionsByTurnId.entries().next().value; | ||
| 678 | if (!oldest) break; | ||
| 679 | const [oldestTurnId, oldestCompletion] = oldest; | ||
| 680 | const entry = finishTurnAnalytics(state, { | ||
| 681 | sessionKey, | ||
| 682 | turnId: oldestTurnId, | ||
| 683 | completion: oldestCompletion, | ||
| 684 | }); | ||
| 685 | if (entry) completed.push(entry); | ||
| 686 | } | ||
| 687 | return [completed, state] as const; | ||
| 688 | }); | ||
| 689 | yield* recordCompletedTurnProperties(properties); | ||
| 690 | }); | ||
| 262 | /** | 691 | /** |
| 263 | * Attach the `t3-code` MCP server to the session that is about to start. | 692 | * Attach the `t3-code` MCP server to the session that is about to start. |
| 264 | * | 693 | * |
| ... | @@ -437,6 +866,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -437,6 +866,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 437 | provider: canonicalEvent.provider, | 866 | provider: canonicalEvent.provider, |
| 438 | eventType: canonicalEvent.type, | 867 | eventType: canonicalEvent.type, |
| 439 | }); | 868 | }); |
| 869 | if (canonicalEvent.type === "turn.started") { | ||
| 870 | yield* observeTurnStartedForAnalytics(source, canonicalEvent); | ||
| 871 | } else if (canonicalEvent.type === "model.rerouted") { | ||
| 872 | yield* observeModelReroutedForAnalytics(source, canonicalEvent); | ||
| 873 | } else if ( | ||
| 874 | canonicalEvent.type === "turn.completed" || | ||
| 875 | canonicalEvent.type === "turn.aborted" | ||
| 876 | ) { | ||
| 877 | yield* recordTurnCompletedAnalytics(source, canonicalEvent); | ||
| 878 | } else if (canonicalEvent.type === "session.exited") { | ||
| 879 | yield* clearTurnAnalyticsSession(source.instanceId, canonicalEvent.threadId); | ||
| 880 | } | ||
| 440 | if ( | 881 | if ( |
| 441 | isCompactedEvent(canonicalEvent) && | 882 | isCompactedEvent(canonicalEvent) && |
| 442 | timedOutNativeCompactions.delete(canonicalEvent.threadId) | 883 | timedOutNativeCompactions.delete(canonicalEvent.threadId) |
| ... | @@ -790,6 +1231,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -790,6 +1231,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 790 | "provider.cwd.effective": effectiveCwd ?? "", | 1231 | "provider.cwd.effective": effectiveCwd ?? "", |
| 791 | }); | 1232 | }); |
| 792 | const adapter = yield* registry.getByInstance(resolvedInstanceId); | 1233 | const adapter = yield* registry.getByInstance(resolvedInstanceId); |
| 1234 | yield* clearTurnAnalyticsSession(resolvedInstanceId, threadId); | ||
| 793 | yield* prepareMcpSession(threadId, resolvedInstanceId); | 1235 | yield* prepareMcpSession(threadId, resolvedInstanceId); |
| 794 | const session = yield* adapter | 1236 | const session = yield* adapter |
| 795 | .startSession({ | 1237 | .startSession({ |
| ... | @@ -953,7 +1395,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -953,7 +1395,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 953 | // rather than issuing a new one: sessions that go a long time between | 1395 | // rather than issuing a new one: sessions that go a long time between |
| 954 | // browser tool calls used to lose the toolkit outright. | 1396 | // browser tool calls used to lose the toolkit outright. |
| 955 | yield* McpSessionRegistry.touchActiveMcpThread(input.threadId); | 1397 | yield* McpSessionRegistry.touchActiveMcpThread(input.threadId); |
| 956 | const turn = yield* routed.adapter.sendTurn(input); | 1398 | const analyticsModelSelection = |
| 1399 | input.modelSelection?.instanceId === routed.instanceId ? input.modelSelection : undefined; | ||
| 1400 | const turn = yield* Effect.acquireUseRelease( | ||
| 1401 | beginTurnAnalytics({ | ||
| 1402 | providerInstanceId: routed.instanceId, | ||
| 1403 | provider: routed.adapter.provider, | ||
| 1404 | threadId: input.threadId, | ||
| 1405 | modelSelection: analyticsModelSelection, | ||
| 1406 | interactionMode: input.interactionMode, | ||
| 1407 | runtimeMode: routed.runtimeMode, | ||
| 1408 | }), | ||
| 1409 | (turnMetadata) => | ||
| 1410 | Effect.gen(function* () { | ||
| 1411 | const turn = yield* routed.adapter.sendTurn(input); | ||
| 1412 | yield* associateTurnAnalytics({ | ||
| 1413 | providerInstanceId: routed.instanceId, | ||
| 1414 | threadId: input.threadId, | ||
| 1415 | turnId: String(turn.turnId), | ||
| 1416 | metadata: turnMetadata, | ||
| 1417 | }); | ||
| 1418 | return turn; | ||
| 1419 | }), | ||
| 1420 | (turnMetadata) => | ||
| 1421 | clearPendingTurnAnalytics({ | ||
| 1422 | providerInstanceId: routed.instanceId, | ||
| 1423 | threadId: input.threadId, | ||
| 1424 | requestId: turnMetadata.requestId, | ||
| 1425 | }), | ||
| 1426 | ); | ||
| 957 | yield* directory.upsert({ | 1427 | yield* directory.upsert({ |
| 958 | threadId: input.threadId, | 1428 | threadId: input.threadId, |
| 959 | provider: routed.adapter.provider, | 1429 | provider: routed.adapter.provider, |
| ... | @@ -1254,6 +1724,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -1254,6 +1724,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 1254 | yield* settleCompaction(input.threadId, pendingCompaction, "turn.aborted"); | 1724 | yield* settleCompaction(input.threadId, pendingCompaction, "turn.aborted"); |
| 1255 | } | 1725 | } |
| 1256 | timedOutNativeCompactions.delete(input.threadId); | 1726 | timedOutNativeCompactions.delete(input.threadId); |
| 1727 | yield* clearTurnAnalyticsSession(routed.instanceId, input.threadId); | ||
| 1257 | yield* clearMcpSession(input.threadId); | 1728 | yield* clearMcpSession(input.threadId); |
| 1258 | yield* directory.upsert({ | 1729 | yield* directory.upsert({ |
| 1259 | threadId: input.threadId, | 1730 | threadId: input.threadId, |
| ... | @@ -1471,6 +1942,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( | ... | @@ -1471,6 +1942,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( |
| 1471 | ); | 1942 | ); |
| 1472 | 1943 | ||
| 1473 | const runStopAll = Effect.fn("runStopAll")(function* () { | 1944 | const runStopAll = Effect.fn("runStopAll")(function* () { |
| 1945 | const properties = yield* Ref.modify(turnAnalytics, (state) => { | ||
| 1946 | const completed: Array<Readonly<Record<string, unknown>>> = []; | ||
| 1947 | for (const [sessionKey, session] of state.sessions) { | ||
| 1948 | for (const [turnId, completion] of session.deferredCompletionsByTurnId) { | ||
| 1949 | const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion }); | ||
| 1950 | if (entry) completed.push(entry); | ||
| 1951 | } | ||
| 1952 | } | ||
| 1953 | state.sessions.clear(); | ||
| 1954 | return [completed, state] as const; | ||
| 1955 | }); | ||
| 1956 | yield* recordCompletedTurnProperties(properties); | ||
| 1474 | const threadIds = yield* directory.listThreadIds(); | 1957 | const threadIds = yield* directory.listThreadIds(); |
| 1475 | const currentAdapters = yield* getAdapterEntries; | 1958 | const currentAdapters = yield* getAdapterEntries; |
| 1476 | const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => | 1959 | const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => |
apps/server/src/telemetry/AnalyticsService.test.ts+43
| ... | @@ -148,4 +148,47 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { | ... | @@ -148,4 +148,47 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { |
| 148 | ); | 148 | ); |
| 149 | }), | 149 | }), |
| 150 | ); | 150 | ); |
| 151 | |||
| 152 | it.effect("does not send batch requests when telemetry is disabled", () => | ||
| 153 | Effect.gen(function* () { | ||
| 154 | const capturedPaths: Array<string> = []; | ||
| 155 | const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), { | ||
| 156 | prefix: "t3-telemetry-disabled-", | ||
| 157 | }); | ||
| 158 | const telemetryLayer = AnalyticsService.layer.pipe(Layer.provideMerge(serverConfigLayer)); | ||
| 159 | const configLayer = ConfigProvider.layer( | ||
| 160 | ConfigProvider.fromUnknown({ | ||
| 161 | T3CODE_TELEMETRY_ENABLED: false, | ||
| 162 | T3CODE_POSTHOG_KEY: "phc_test_key", | ||
| 163 | T3CODE_POSTHOG_HOST: "http://localhost", | ||
| 164 | }), | ||
| 165 | ); | ||
| 166 | const batchServerLayer = HttpServer.serve( | ||
| 167 | Effect.gen(function* () { | ||
| 168 | const request = yield* HttpServerRequest.HttpServerRequest; | ||
| 169 | capturedPaths.push(request.url); | ||
| 170 | return HttpServerResponse.jsonUnsafe({}); | ||
| 171 | }), | ||
| 172 | ); | ||
| 173 | const runtimeLayer = telemetryLayer.pipe( | ||
| 174 | Layer.provide(configLayer), | ||
| 175 | Layer.provide( | ||
| 176 | Layer.mergeAll( | ||
| 177 | Layer.succeed(HostProcessPlatform, "linux"), | ||
| 178 | Layer.succeed(HostProcessArchitecture, "arm64"), | ||
| 179 | ), | ||
| 180 | ), | ||
| 181 | Layer.provideMerge(NodeHttpServer.layerTest), | ||
| 182 | ); | ||
| 183 | |||
| 184 | yield* Effect.gen(function* () { | ||
| 185 | yield* Layer.launch(batchServerLayer).pipe(Effect.forkScoped); | ||
| 186 | const analytics = yield* AnalyticsService.AnalyticsService; | ||
| 187 | yield* analytics.record("test.disabled", { index: 1 }); | ||
| 188 | yield* analytics.flush; | ||
| 189 | }).pipe(Effect.provide(runtimeLayer)); | ||
| 190 | |||
| 191 | assert.deepEqual(capturedPaths, []); | ||
| 192 | }), | ||
| 193 | ); | ||
| 151 | }); | 194 | }); |
docs/README.md+1
| ... | @@ -7,6 +7,7 @@ | ... | @@ -7,6 +7,7 @@ |
| 7 | - [Keyboard shortcuts](./user/keybindings.md) | 7 | - [Keyboard shortcuts](./user/keybindings.md) |
| 8 | - [Organizing threads](./user/thread-sidebar.md) | 8 | - [Organizing threads](./user/thread-sidebar.md) |
| 9 | - [Review usage](./user/usage.md) | 9 | - [Review usage](./user/usage.md) |
| 10 | - [Anonymous usage data](./user/telemetry.md) | ||
| 10 | - [Customize a project icon](./user/project-settings.md) | 11 | - [Customize a project icon](./user/project-settings.md) |
| 11 | - [Mobile appearance](./user/mobile-appearance.md) | 12 | - [Mobile appearance](./user/mobile-appearance.md) |
| 12 | - [Environment themes](./user/environment-theme.md) | 13 | - [Environment themes](./user/environment-theme.md) |
docs/internals/product-analytics.md+68-3
| ... | @@ -23,6 +23,54 @@ time. | ... | @@ -23,6 +23,54 @@ time. |
| 23 | client metadata because a provider turn can continue after the requesting | 23 | client metadata because a provider turn can continue after the requesting |
| 24 | client disconnects. | 24 | client disconnects. |
| 25 | 25 | ||
| 26 | ## Provider turn events | ||
| 27 | |||
| 28 | `provider.turn.completed` records one anonymous event when a provider emits a | ||
| 29 | canonical completed or aborted turn. The event reports normalized main-agent | ||
| 30 | token counts when the provider supplies enough data. | ||
| 31 | |||
| 32 | | Property | Values and meaning | | ||
| 33 | | --------------------- | ------------------------------------------------------------------------------------------------------------------------------- | | ||
| 34 | | `provider` | Provider driver, such as `codex`, `claudeAgent`, or `opencode`. Use this as the harness breakdown. | | ||
| 35 | | `model` | Selected or provider-reported model when known. This is metadata, not proof that one model produced every token. | | ||
| 36 | | `effort` | Selected or provider-reported reasoning effort when known. | | ||
| 37 | | `interactionMode` | Turn interaction mode, `default` or `plan`, when known. | | ||
| 38 | | `runtimeMode` | Session permission mode when known. | | ||
| 39 | | `mixedModels` | `true` when T3 Code observed `model.rerouted` during the turn. This does not split token counts by model. | | ||
| 40 | | `durationMs` | Time from the observed turn start or accepted send request to the terminal event. Omitted when start metadata is not available. | | ||
| 41 | | `terminalStatus` | `completed`, `failed`, `interrupted`, or `cancelled`. | | ||
| 42 | | `usageStatus` | `complete`, `partial`, or `unavailable`. | | ||
| 43 | | `usageScope` | `main_agent`. Child-agent token use is not included. | | ||
| 44 | | `hasSubagents` | Whether the adapter observed child agents during the turn. Omitted when the adapter does not provide a normalized usage report. | | ||
| 45 | | `inputTokens` | Total main-agent input tokens. This includes uncached input, cache reads, and cache creation or writes. | | ||
| 46 | | `cachedInputTokens` | Cache-read tokens. This is a subset of `inputTokens`. | | ||
| 47 | | `cacheCreationTokens` | Cache creation or write tokens. This is a subset of `inputTokens`. | | ||
| 48 | | `outputTokens` | Total main-agent output tokens, including reasoning tokens. | | ||
| 49 | | `reasoningTokens` | Reasoning tokens when the provider reports them separately. This is a subset of `outputTokens`. | | ||
| 50 | |||
| 51 | `complete` means the provider supplied full input and output totals for the | ||
| 52 | turn. `partial` means each included count is valid, but T3 Code did not observe | ||
| 53 | the full turn. This can happen after an interruption, provider failure, or | ||
| 54 | reconnect. `unavailable` means the provider did not supply trustworthy counts. | ||
| 55 | Numeric properties are omitted when their values are unknown. They are not set | ||
| 56 | to zero. | ||
| 57 | |||
| 58 | Codex uses deltas from its cumulative token totals, and falls back to the | ||
| 59 | newest response's `last` usage when no prior total exists or the total was | ||
| 60 | reset. Claude uses the final result's per-turn `usage` value and does not use | ||
| 61 | cumulative `modelUsage`. | ||
| 62 | OpenCode sums unique main-session step totals. Cursor, Grok, and Antigravity | ||
| 63 | usage stays `unavailable` until their ACP token fields and scope are verified. | ||
| 64 | |||
| 65 | The completed event count does not have to match `provider.turn.sent`. For | ||
| 66 | example, Claude can emit synthetic turns without a matching send request. | ||
| 67 | Duplicate terminal events for the same provider instance, thread, and turn | ||
| 68 | produce one analytics event. | ||
| 69 | |||
| 70 | Collection is best effort and starts with the first server release that | ||
| 71 | contains this event. T3 Code does not scan provider history or backfill older | ||
| 72 | turns. | ||
| 73 | |||
| 26 | ## Recommended properties | 74 | ## Recommended properties |
| 27 | 75 | ||
| 28 | Client properties appear on the three client events when the connected client | 76 | Client properties appear on the three client events when the connected client |
| ... | @@ -94,6 +142,20 @@ In PostHog Data management, use the event and property descriptions from this | ... | @@ -94,6 +142,20 @@ In PostHog Data management, use the event and property descriptions from this |
| 94 | document. Mark the recommended properties as verified. Keep `clientType` | 142 | document. Mark the recommended properties as verified. Keep `clientType` |
| 95 | visible with its legacy description so old reports remain understandable. | 143 | visible with its legacy description so old reports remain understandable. |
| 96 | 144 | ||
| 145 | Create a second saved dashboard named `Provider token efficiency`. Start each | ||
| 146 | token insight with these filters: | ||
| 147 | |||
| 148 | - `usageStatus = complete` | ||
| 149 | - `hasSubagents = false` | ||
| 150 | - `mixedModels = false` | ||
| 151 | |||
| 152 | Break down input tokens, output tokens, and their ratio by `provider`. Compare | ||
| 153 | the same `model`, `effort`, `interactionMode`, and `terminalStatus`. Without | ||
| 154 | these filters, model choice, reasoning level, plan mode, failures, and child | ||
| 155 | agents can look like harness differences. Keep separate views for median turn | ||
| 156 | usage and aggregate `sum(outputTokens) / sum(inputTokens)`. Do not average each | ||
| 157 | turn's ratio because small-input turns can dominate that result. | ||
| 158 | |||
| 97 | ## Collection and release boundary | 159 | ## Collection and release boundary |
| 98 | 160 | ||
| 99 | Client values are best effort. Invalid values are ignored and never reject a | 161 | Client values are best effort. Invalid values are ignored and never reject a |
| ... | @@ -101,9 +163,12 @@ connection. Browser clients use user-agent data for broad OS, browser, phone, | ... | @@ -101,9 +163,12 @@ connection. Browser clients use user-agent data for broad OS, browser, phone, |
| 101 | and tablet groups. They do not infer CPU architecture or an exact OS version | 163 | and tablet groups. They do not infer CPU architecture or an exact OS version |
| 102 | from `navigator.platform`. | 164 | from `navigator.platform`. |
| 103 | 165 | ||
| 104 | This change does not collect URLs, tokens, prompts, IP addresses, or | 166 | This change does not collect URLs, authentication tokens, prompts, IP |
| 105 | user-assigned device names. It measures authenticated product use. It does not | 167 | addresses, or user-assigned device names. The token event does not include |
| 106 | measure a person who visits the hosted app without connecting to a server. | 168 | thread IDs, turn IDs, provider instance IDs, event IDs, request IDs, raw |
| 169 | provider payloads, or child-agent output. It does not scan provider history on | ||
| 170 | disk or start external work. It measures authenticated product use. It does | ||
| 171 | not measure a person who visits the hosted app without connecting to a server. | ||
| 107 | 172 | ||
| 108 | The new fields start with the first client and server release that contains | 173 | The new fields start with the first client and server release that contains |
| 109 | this metadata path. Historical events cannot reliably identify the client OS, | 174 | this metadata path. Historical events cannot reliably identify the client OS, |
docs/user/telemetry.md created+15
| ... | @@ -0,0 +1,15 @@ | ||
| 1 | # Anonymous usage data | ||
| 2 | |||
| 3 | The T3 server sends anonymous product events to PostHog. These events include | ||
| 4 | the provider, model, reasoning effort, permission mode, turn result, duration, | ||
| 5 | and normalized main-agent token totals when the provider reports them. | ||
| 6 | |||
| 7 | T3 Code does not send prompts, responses, file contents, raw provider events, | ||
| 8 | thread IDs, turn IDs, provider instance IDs, authentication tokens, or child | ||
| 9 | agent output. Token totals can be complete, partial, or unavailable. Child | ||
| 10 | agent token use is not included. | ||
| 11 | |||
| 12 | Set `T3CODE_TELEMETRY_ENABLED=false` before you start the T3 server to stop | ||
| 13 | product events from being recorded or sent. The server still derives an | ||
| 14 | anonymous identifier during startup and can create its local fallback | ||
| 15 | identifier file. | ||
packages/contracts/src/providerRuntime.test.ts+46
| ... | @@ -6,6 +6,52 @@ import { classifyTaskAgentKind, ProviderRuntimeEvent } from "./providerRuntime.t | ... | @@ -6,6 +6,52 @@ import { classifyTaskAgentKind, ProviderRuntimeEvent } from "./providerRuntime.t |
| 6 | const decodeRuntimeEvent = Schema.decodeUnknownSync(ProviderRuntimeEvent); | 6 | const decodeRuntimeEvent = Schema.decodeUnknownSync(ProviderRuntimeEvent); |
| 7 | 7 | ||
| 8 | describe("ProviderRuntimeEvent", () => { | 8 | describe("ProviderRuntimeEvent", () => { |
| 9 | it("requires input and output totals for complete turn usage", () => { | ||
| 10 | const completeEvent = { | ||
| 11 | type: "turn.completed", | ||
| 12 | eventId: "event-complete-usage", | ||
| 13 | provider: "codex", | ||
| 14 | createdAt: "2026-02-28T00:00:00.000Z", | ||
| 15 | threadId: "thread-1", | ||
| 16 | turnId: "turn-1", | ||
| 17 | payload: { | ||
| 18 | state: "completed", | ||
| 19 | tokenUsage: { | ||
| 20 | usageStatus: "complete", | ||
| 21 | usageScope: "main_agent", | ||
| 22 | hasSubagents: false, | ||
| 23 | }, | ||
| 24 | }, | ||
| 25 | }; | ||
| 26 | |||
| 27 | expect(() => decodeRuntimeEvent(completeEvent)).toThrow(); | ||
| 28 | expect( | ||
| 29 | decodeRuntimeEvent({ | ||
| 30 | ...completeEvent, | ||
| 31 | payload: { | ||
| 32 | ...completeEvent.payload, | ||
| 33 | tokenUsage: { | ||
| 34 | ...completeEvent.payload.tokenUsage, | ||
| 35 | inputTokens: 10, | ||
| 36 | outputTokens: 2, | ||
| 37 | }, | ||
| 38 | }, | ||
| 39 | }).type, | ||
| 40 | ).toBe("turn.completed"); | ||
| 41 | expect( | ||
| 42 | decodeRuntimeEvent({ | ||
| 43 | ...completeEvent, | ||
| 44 | payload: { | ||
| 45 | ...completeEvent.payload, | ||
| 46 | tokenUsage: { | ||
| 47 | ...completeEvent.payload.tokenUsage, | ||
| 48 | usageStatus: "partial", | ||
| 49 | }, | ||
| 50 | }, | ||
| 51 | }).type, | ||
| 52 | ).toBe("turn.completed"); | ||
| 53 | }); | ||
| 54 | |||
| 9 | it("accepts fork-provided driver kinds as branded slugs", () => { | 55 | it("accepts fork-provided driver kinds as branded slugs", () => { |
| 10 | const parsed = decodeRuntimeEvent({ | 56 | const parsed = decodeRuntimeEvent({ |
| 11 | type: "session.started", | 57 | type: "session.started", |
packages/contracts/src/providerRuntime.ts+31
| ... | @@ -367,6 +367,35 @@ const TurnStartedPayload = Schema.Struct({ | ... | @@ -367,6 +367,35 @@ const TurnStartedPayload = Schema.Struct({ |
| 367 | }); | 367 | }); |
| 368 | export type TurnStartedPayload = typeof TurnStartedPayload.Type; | 368 | export type TurnStartedPayload = typeof TurnStartedPayload.Type; |
| 369 | 369 | ||
| 370 | /** | ||
| 371 | * Normalized main-agent usage for one turn. Input includes cache reads and | ||
| 372 | * writes. Output includes reasoning, and reasoningTokens is an optional subset. | ||
| 373 | * Complete means the provider supplied full input and output totals. Partial | ||
| 374 | * means every included count is valid, but the full turn total is not known. | ||
| 375 | */ | ||
| 376 | const TurnTokenUsageCommonFields = { | ||
| 377 | usageScope: Schema.Literal("main_agent"), | ||
| 378 | cachedInputTokens: Schema.optional(NonNegativeInt), | ||
| 379 | cacheCreationTokens: Schema.optional(NonNegativeInt), | ||
| 380 | reasoningTokens: Schema.optional(NonNegativeInt), | ||
| 381 | hasSubagents: Schema.Boolean, | ||
| 382 | }; | ||
| 383 | export const TurnTokenUsage = Schema.Union([ | ||
| 384 | Schema.Struct({ | ||
| 385 | ...TurnTokenUsageCommonFields, | ||
| 386 | usageStatus: Schema.Literal("complete"), | ||
| 387 | inputTokens: NonNegativeInt, | ||
| 388 | outputTokens: NonNegativeInt, | ||
| 389 | }), | ||
| 390 | Schema.Struct({ | ||
| 391 | ...TurnTokenUsageCommonFields, | ||
| 392 | usageStatus: Schema.Literals(["partial", "unavailable"]), | ||
| 393 | inputTokens: Schema.optional(NonNegativeInt), | ||
| 394 | outputTokens: Schema.optional(NonNegativeInt), | ||
| 395 | }), | ||
| 396 | ]); | ||
| 397 | export type TurnTokenUsage = typeof TurnTokenUsage.Type; | ||
| 398 | |||
| 370 | const TurnCompletedPayload = Schema.Struct({ | 399 | const TurnCompletedPayload = Schema.Struct({ |
| 371 | state: RuntimeTurnState, | 400 | state: RuntimeTurnState, |
| 372 | stopReason: Schema.optional(Schema.NullOr(TrimmedNonEmptyStringSchema)), | 401 | stopReason: Schema.optional(Schema.NullOr(TrimmedNonEmptyStringSchema)), |
| ... | @@ -374,11 +403,13 @@ const TurnCompletedPayload = Schema.Struct({ | ... | @@ -374,11 +403,13 @@ const TurnCompletedPayload = Schema.Struct({ |
| 374 | modelUsage: Schema.optional(UnknownRecordSchema), | 403 | modelUsage: Schema.optional(UnknownRecordSchema), |
| 375 | totalCostUsd: Schema.optional(Schema.Number), | 404 | totalCostUsd: Schema.optional(Schema.Number), |
| 376 | errorMessage: Schema.optional(TrimmedNonEmptyStringSchema), | 405 | errorMessage: Schema.optional(TrimmedNonEmptyStringSchema), |
| 406 | tokenUsage: Schema.optional(TurnTokenUsage), | ||
| 377 | }); | 407 | }); |
| 378 | export type TurnCompletedPayload = typeof TurnCompletedPayload.Type; | 408 | export type TurnCompletedPayload = typeof TurnCompletedPayload.Type; |
| 379 | 409 | ||
| 380 | const TurnAbortedPayload = Schema.Struct({ | 410 | const TurnAbortedPayload = Schema.Struct({ |
| 381 | reason: TrimmedNonEmptyStringSchema, | 411 | reason: TrimmedNonEmptyStringSchema, |
| 412 | tokenUsage: Schema.optional(TurnTokenUsage), | ||
| 382 | }); | 413 | }); |
| 383 | export type TurnAbortedPayload = typeof TurnAbortedPayload.Type; | 414 | export type TurnAbortedPayload = typeof TurnAbortedPayload.Type; |
| 384 | 415 |