authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-07 06:16:11-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-07 03:16:11-07:00
log7963cc70f3c9873768a5eaad7ca0a20a23d67aed
treef445773459cf2c3486edb09ec51551b278890e57
parentb792ed9f7b9c7c6386a14a1697c4393567812546
signature Signed by PGP key B5690EEEBB952194

feat(server): record runtime mode per turn and on mode changes (#5593)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

2 files changed, 129 insertions(+), 29 deletions(-)

apps/server/integration/providerService.integration.test.ts+109-29
......@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";
88import * as Layer from "effect/Layer";
99import * as Path from "effect/Path";
1010import * as Queue from "effect/Queue";
11import * as Ref from "effect/Ref";
1112import * as Stream from "effect/Stream";
1213
1314import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";
......@@ -54,35 +55,58 @@ interface IntegrationFixture {
5455 readonly layer: Layer.Layer<ProviderService, unknown, never>;
5556}
5657
57const makeIntegrationFixture = Effect.gen(function* () {
58 const cwd = yield* makeWorkspaceDirectory;
59 const harness = yield* makeTestProviderAdapterHarness();
60
61 const registry = makeAdapterRegistryMock({
62 [ProviderDriverKind.make("codex")]: harness.adapter,
63 });
58interface RecordedAnalyticsEvent {
59 readonly event: string;
60 readonly properties: Readonly<Record<string, unknown>> | undefined;
61}
6462
65 const directoryLayer = ProviderSessionDirectoryLive.pipe(
66 Layer.provide(ProviderSessionRuntime.layer),
63/**
64 * Analytics layer that keeps captured events in memory so tests can assert on
65 * telemetry payloads. `AnalyticsService.layerTest` discards them.
66 */
67const makeRecordingAnalytics = Effect.gen(function* () {
68 const recorded = yield* Ref.make<ReadonlyArray<RecordedAnalyticsEvent>>([]);
69 const layer = Layer.succeed(
70 AnalyticsService,
71 AnalyticsService.of({
72 record: (event, properties) =>
73 Ref.update(recorded, (current) => [...current, { event, properties }]),
74 flush: Effect.void,
75 }),
6776 );
68
69 const shared = Layer.mergeAll(
70 directoryLayer,
71 Layer.succeed(ProviderAdapterRegistry, registry),
72 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
73 AnalyticsService.layerTest,
74 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
75 ).pipe(Layer.provide(SqlitePersistenceMemory));
76
77 const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
78
79 return {
80 cwd,
81 harness,
82 layer,
83 } satisfies IntegrationFixture;
77 return { layer, get: Ref.get(recorded) } as const;
8478});
8579
80const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<AnalyticsService> }) =>
81 Effect.gen(function* () {
82 const cwd = yield* makeWorkspaceDirectory;
83 const harness = yield* makeTestProviderAdapterHarness();
84
85 const registry = makeAdapterRegistryMock({
86 [ProviderDriverKind.make("codex")]: harness.adapter,
87 });
88
89 const directoryLayer = ProviderSessionDirectoryLive.pipe(
90 Layer.provide(ProviderSessionRuntime.layer),
91 );
92
93 const shared = Layer.mergeAll(
94 directoryLayer,
95 Layer.succeed(ProviderAdapterRegistry, registry),
96 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
97 options?.analytics ?? AnalyticsService.layerTest,
98 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
99 ).pipe(Layer.provide(SqlitePersistenceMemory));
100
101 const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
102
103 return {
104 cwd,
105 harness,
106 layer,
107 } satisfies IntegrationFixture;
108 });
109
86110const collectEventsDuring = <A, E, R>(
87111 stream: Stream.Stream<ProviderRuntimeEvent>,
88112 count: number,
......@@ -126,7 +150,7 @@ const runTurn = (input: {
126150
127151it.live("replays typed runtime fixture events", () =>
128152 Effect.gen(function* () {
129 const fixture = yield* makeIntegrationFixture;
153 const fixture = yield* makeIntegrationFixture();
130154
131155 yield* Effect.gen(function* () {
132156 const provider = yield* ProviderService;
......@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>
161185
162186it.live("replays file-changing fixture turn events", () =>
163187 Effect.gen(function* () {
164 const fixture = yield* makeIntegrationFixture;
188 const fixture = yield* makeIntegrationFixture();
165189 const { join } = yield* Path.Path;
166190 const { writeFileString } = yield* FileSystem.FileSystem;
167191
......@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>
198222
199223it.live("runs multi-turn tool/approval flow", () =>
200224 Effect.gen(function* () {
201 const fixture = yield* makeIntegrationFixture;
225 const fixture = yield* makeIntegrationFixture();
202226 const { join } = yield* Path.Path;
203227 const { writeFileString } = yield* FileSystem.FileSystem;
204228
......@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>
250274
251275it.live("rolls back provider conversation state only", () =>
252276 Effect.gen(function* () {
253 const fixture = yield* makeIntegrationFixture;
277 const fixture = yield* makeIntegrationFixture();
254278 const { join } = yield* Path.Path;
255279 const { writeFileString, readFileString } = yield* FileSystem.FileSystem;
256280
......@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>
302326 }).pipe(Effect.provide(fixture.layer));
303327 }).pipe(Effect.provide(NodeServices.layer)),
304328);
329
330it.live("reports runtime mode per turn and on mode transitions", () =>
331 Effect.gen(function* () {
332 const analytics = yield* makeRecordingAnalytics;
333 const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer });
334 const threadId = ThreadId.make("thread-integration-runtime-mode");
335
336 yield* Effect.gen(function* () {
337 const provider = yield* ProviderService;
338 const startSession = (runtimeMode: "approval-required" | "full-access") =>
339 provider.startSession(threadId, {
340 threadId,
341 provider: ProviderDriverKind.make("codex"),
342 providerInstanceId: codexInstanceId,
343 cwd: fixture.cwd,
344 runtimeMode,
345 });
346
347 yield* startSession("approval-required");
348 yield* runTurn({
349 provider,
350 harness: fixture.harness,
351 threadId,
352 userText: "supervised turn",
353 response: { events: codexTurnTextFixture },
354 });
355
356 // Toggling the mode restarts the session, which is the only place the
357 // transition is observable.
358 yield* startSession("full-access");
359 yield* runTurn({
360 provider,
361 harness: fixture.harness,
362 threadId,
363 userText: "full access turn",
364 response: { events: codexTurnTextFixture },
365 });
366
367 const recorded = yield* analytics.get;
368
369 assert.deepEqual(
370 recorded
371 .filter((entry) => entry.event === "provider.turn.sent")
372 .map((entry) => entry.properties?.runtimeMode),
373 ["approval-required", "full-access"],
374 );
375
376 assert.deepEqual(
377 recorded
378 .filter((entry) => entry.event === "provider.runtime_mode.changed")
379 .map((entry) => [entry.properties?.from, entry.properties?.to]),
380 [["approval-required", "full-access"]],
381 );
382 }).pipe(Effect.provide(fixture.layer));
383 }).pipe(Effect.provide(NodeServices.layer)),
384);
apps/server/src/provider/Layers/ProviderService.ts+20
......@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
459459 adapter,
460460 instanceId,
461461 threadId: input.threadId,
462 runtimeMode: binding.runtimeMode,
462463 isActive: true,
463464 } as const;
464465 }
......@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
468469 adapter,
469470 instanceId,
470471 threadId: input.threadId,
472 runtimeMode: binding.runtimeMode,
471473 isActive: false,
472474 } as const;
473475 }
......@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
480482 adapter: recovered.adapter,
481483 instanceId,
482484 threadId: input.threadId,
485 runtimeMode: recovered.session.runtimeMode,
483486 isActive: true,
484487 } as const;
485488 });
......@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
629632 input.modelSelection.model.trim().length > 0,
630633 });
631634
635 // Changing runtime mode restarts the session, so the transition is only
636 // observable here, by diffing against the mode the previous session for
637 // this thread was bound to. Recording it separately is what makes the
638 // "started supervised, switched to full access" funnel answerable.
639 const previousRuntimeMode = persistedBinding?.runtimeMode;
640 if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) {
641 yield* analytics.record("provider.runtime_mode.changed", {
642 provider: sessionWithInstance.provider,
643 from: previousRuntimeMode,
644 to: input.runtimeMode,
645 });
646 }
647
632648 return sessionWithInstance;
633649 }).pipe(
634650 withMetrics({
......@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
703719 provider: routed.adapter.provider,
704720 model: input.modelSelection?.model,
705721 interactionMode: input.interactionMode,
722 // Session-start events alone skew runtime mode toward users who toggle
723 // often, since every toggle restarts the session. Recording it per turn
724 // gives a usage-weighted view and lets it cross with interactionMode.
725 runtimeMode: routed.runtimeMode,
706726 attachmentCount: input.attachments.length,
707727 hasInput: typeof input.input === "string" && input.input.trim().length > 0,
708728 });