authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-26 20:02:55-08:00
committergravatar for jmarminge@gmail.comJulius Marminge <jmarminge@gmail.com> 2026-02-26 20:43:22-08:00
logcc29ceea80d6f02181e4ce7dce2812c07190ebe8
tree6db5e20877ab56f77ba44f970311edab696f5037
parent32a17b91098edf78a3fab93f978e8b765fd112fb

Spill oversized buffered assistant deltas without losing final text

- cap buffered assistant text memory and spill older chunks as delta events - finalize buffered messages by emitting retained remainder before completion when spilled - add regression coverage for oversized buffered deltas and tighten type-safe test assertions

3 files changed, 137 insertions(+), 17 deletions(-)

apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts+53
...@@ -402,6 +402,59 @@ describe("ProviderRuntimeIngestion", () => {...@@ -402,6 +402,59 @@ describe("ProviderRuntimeIngestion", () => {
402 expect(finalMessage?.streaming).toBe(false);402 expect(finalMessage?.streaming).toBe(false);
403 });403 });
404404
405 it("spills oversized buffered deltas and still finalizes full assistant text", async () => {
406 const harness = await createHarness();
407 const now = new Date().toISOString();
408 const oversizedText = "x".repeat(40_000);
409
410 harness.emit({
411 type: "turn.started",
412 eventId: asEventId("evt-turn-started-buffer-spill"),
413 provider: "codex",
414 sessionId: asSessionId("sess-1"),
415 createdAt: now,
416 turnId: asProviderTurnId("turn-buffer-spill"),
417 });
418 await waitForThread(
419 harness.engine,
420 (thread) =>
421 thread.session?.status === "running" &&
422 thread.session?.activeTurnId === "turn-buffer-spill",
423 );
424
425 harness.emit({
426 type: "message.delta",
427 eventId: asEventId("evt-message-delta-buffer-spill"),
428 provider: "codex",
429 sessionId: asSessionId("sess-1"),
430 createdAt: now,
431 turnId: asProviderTurnId("turn-buffer-spill"),
432 itemId: asItemId("item-buffer-spill"),
433 delta: oversizedText,
434 });
435 harness.emit({
436 type: "message.completed",
437 eventId: asEventId("evt-message-completed-buffer-spill"),
438 provider: "codex",
439 sessionId: asSessionId("sess-1"),
440 createdAt: now,
441 turnId: asProviderTurnId("turn-buffer-spill"),
442 itemId: asItemId("item-buffer-spill"),
443 });
444
445 const thread = await waitForThread(
446 harness.engine,
447 (entry) =>
448 entry.messages.some(
449 (message) => message.id === "assistant:item-buffer-spill" && !message.streaming,
450 ),
451 );
452 const message = thread.messages.find((entry) => entry.id === "assistant:item-buffer-spill");
453 expect(message?.text.length).toBe(oversizedText.length);
454 expect(message?.text).toBe(oversizedText);
455 expect(message?.streaming).toBe(false);
456 });
457
405 it("does not duplicate assistant completion when message.completed is followed by turn.completed", async () => {458 it("does not duplicate assistant completion when message.completed is followed by turn.completed", async () => {
406 const harness = await createHarness();459 const harness = await createHarness();
407 const now = new Date().toISOString();460 const now = new Date().toISOString();
apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+77-12
...@@ -28,6 +28,10 @@ const TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY = 10_000;...@@ -28,6 +28,10 @@ const TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY = 10_000;
28const TURN_MESSAGE_IDS_BY_TURN_TTL = Duration.minutes(120);28const TURN_MESSAGE_IDS_BY_TURN_TTL = Duration.minutes(120);
29const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_CACHE_CAPACITY = 20_000;29const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_CACHE_CAPACITY = 20_000;
30const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_TTL = Duration.minutes(120);30const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_TTL = Duration.minutes(120);
31const BUFFERED_ASSISTANT_MESSAGE_SPILLED_CACHE_CAPACITY = 20_000;
32const BUFFERED_ASSISTANT_MESSAGE_SPILLED_TTL = Duration.minutes(120);
33const MAX_BUFFERED_ASSISTANT_CHARS = 24_000;
34const BUFFERED_ASSISTANT_KEEP_CHARS = 8_000;
3135
32type TurnStartRequestedDomainEvent = Extract<36type TurnStartRequestedDomainEvent = Extract<
33 OrchestrationEvent,37 OrchestrationEvent,
...@@ -169,6 +173,11 @@ const make = Effect.gen(function* () {...@@ -169,6 +173,11 @@ const make = Effect.gen(function* () {
169 timeToLive: BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_TTL,173 timeToLive: BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_TTL,
170 lookup: () => Effect.succeed(""),174 lookup: () => Effect.succeed(""),
171 });175 });
176 const bufferedAssistantMessageSpilledByMessageId = yield* Cache.make<MessageId, true>({
177 capacity: BUFFERED_ASSISTANT_MESSAGE_SPILLED_CACHE_CAPACITY,
178 timeToLive: BUFFERED_ASSISTANT_MESSAGE_SPILLED_TTL,
179 lookup: () => Effect.succeed(true),
180 });
172181
173 const rememberAssistantMessageId = (182 const rememberAssistantMessageId = (
174 sessionId: ProviderSessionId,183 sessionId: ProviderSessionId,
...@@ -226,14 +235,23 @@ const make = Effect.gen(function* () {...@@ -226,14 +235,23 @@ const make = Effect.gen(function* () {
226 const appendBufferedAssistantText = (messageId: MessageId, delta: string) =>235 const appendBufferedAssistantText = (messageId: MessageId, delta: string) =>
227 Cache.getOption(bufferedAssistantTextByMessageId, messageId).pipe(236 Cache.getOption(bufferedAssistantTextByMessageId, messageId).pipe(
228 Effect.flatMap((existingText) =>237 Effect.flatMap((existingText) =>
229 Cache.set(238 Effect.gen(function* () {
230 bufferedAssistantTextByMessageId,239 const nextText = Option.match(existingText, {
231 messageId,
232 Option.match(existingText, {
233 onNone: () => delta,240 onNone: () => delta,
234 onSome: (text) => `${text}${delta}`,241 onSome: (text) => `${text}${delta}`,
235 }),242 });
236 ),243 if (nextText.length <= MAX_BUFFERED_ASSISTANT_CHARS) {
244 yield* Cache.set(bufferedAssistantTextByMessageId, messageId, nextText);
245 return "";
246 }
247
248 // Safety valve: spill oldest text as a persisted assistant delta to cap memory.
249 const spillChars = Math.max(1, nextText.length - BUFFERED_ASSISTANT_KEEP_CHARS);
250 const spillChunk = nextText.slice(0, spillChars);
251 const retainedTail = nextText.slice(spillChars);
252 yield* Cache.set(bufferedAssistantTextByMessageId, messageId, retainedTail);
253 return spillChunk;
254 }),
237 ),255 ),
238 );256 );
239257
...@@ -249,18 +267,51 @@ const make = Effect.gen(function* () {...@@ -249,18 +267,51 @@ const make = Effect.gen(function* () {
249 const clearBufferedAssistantText = (messageId: MessageId) =>267 const clearBufferedAssistantText = (messageId: MessageId) =>
250 Cache.invalidate(bufferedAssistantTextByMessageId, messageId);268 Cache.invalidate(bufferedAssistantTextByMessageId, messageId);
251269
252 const clearAssistantMessageState = (messageId: MessageId) => clearBufferedAssistantText(messageId);270 const markBufferedAssistantMessageSpilled = (messageId: MessageId) =>
271 Cache.set(bufferedAssistantMessageSpilledByMessageId, messageId, true);
272
273 const takeBufferedAssistantMessageSpilled = (messageId: MessageId) =>
274 Cache.getOption(bufferedAssistantMessageSpilledByMessageId, messageId).pipe(
275 Effect.flatMap((spilled) =>
276 Cache.invalidate(bufferedAssistantMessageSpilledByMessageId, messageId).pipe(
277 Effect.as(Option.isSome(spilled)),
278 ),
279 ),
280 );
281
282 const clearBufferedAssistantMessageSpilled = (messageId: MessageId) =>
283 Cache.invalidate(bufferedAssistantMessageSpilledByMessageId, messageId);
284
285 const clearAssistantMessageState = (messageId: MessageId) =>
286 Effect.all(
287 [clearBufferedAssistantText(messageId), clearBufferedAssistantMessageSpilled(messageId)],
288 { concurrency: 1 },
289 ).pipe(Effect.asVoid);
253290
254 const dispatchAssistantCompletion = (input: {291 const finalizeAssistantMessage = (input: {
255 event: ProviderRuntimeEvent;292 event: ProviderRuntimeEvent;
256 threadId: ThreadId;293 threadId: ThreadId;
257 messageId: MessageId;294 messageId: MessageId;
258 turnId?: TurnId;295 turnId?: TurnId;
259 createdAt: string;296 createdAt: string;
260 commandTag: string;297 commandTag: string;
298 spillRemainderCommandTag: string;
261 }) =>299 }) =>
262 Effect.gen(function* () {300 Effect.gen(function* () {
263 const text = yield* takeBufferedAssistantText(input.messageId);301 const text = yield* takeBufferedAssistantText(input.messageId);
302 const spilled = yield* takeBufferedAssistantMessageSpilled(input.messageId);
303
304 if (spilled && text.length > 0) {
305 yield* orchestrationEngine.dispatch({
306 type: "thread.message.assistant.delta",
307 commandId: providerCommandId(input.event, input.spillRemainderCommandTag),
308 threadId: input.threadId,
309 messageId: input.messageId,
310 delta: text,
311 ...(input.turnId ? { turnId: input.turnId } : {}),
312 createdAt: input.createdAt,
313 });
314 }
264315
265 yield* orchestrationEngine.dispatch({316 yield* orchestrationEngine.dispatch({
266 type: "thread.message.assistant.complete",317 type: "thread.message.assistant.complete",
...@@ -268,7 +319,7 @@ const make = Effect.gen(function* () {...@@ -268,7 +319,7 @@ const make = Effect.gen(function* () {
268 threadId: input.threadId,319 threadId: input.threadId,
269 messageId: input.messageId,320 messageId: input.messageId,
270 ...(input.turnId ? { turnId: input.turnId } : {}),321 ...(input.turnId ? { turnId: input.turnId } : {}),
271 text,322 text: spilled ? "" : text,
272 createdAt: input.createdAt,323 createdAt: input.createdAt,
273 });324 });
274 yield* clearAssistantMessageState(input.messageId);325 yield* clearAssistantMessageState(input.messageId);
...@@ -369,7 +420,19 @@ const make = Effect.gen(function* () {...@@ -369,7 +420,19 @@ const make = Effect.gen(function* () {
369420
370 const assistantDeliveryMode = yield* Ref.get(assistantDeliveryModeRef);421 const assistantDeliveryMode = yield* Ref.get(assistantDeliveryModeRef);
371 if (assistantDeliveryMode === "buffered") {422 if (assistantDeliveryMode === "buffered") {
372 yield* appendBufferedAssistantText(assistantMessageId, event.delta);423 const spillChunk = yield* appendBufferedAssistantText(assistantMessageId, event.delta);
424 if (spillChunk.length > 0) {
425 yield* markBufferedAssistantMessageSpilled(assistantMessageId);
426 yield* orchestrationEngine.dispatch({
427 type: "thread.message.assistant.delta",
428 commandId: providerCommandId(event, "assistant-delta-buffer-spill"),
429 threadId: thread.id,
430 messageId: assistantMessageId,
431 delta: spillChunk,
432 ...(turnId ? { turnId } : {}),
433 createdAt: now,
434 });
435 }
373 } else {436 } else {
374 yield* orchestrationEngine.dispatch({437 yield* orchestrationEngine.dispatch({
375 type: "thread.message.assistant.delta",438 type: "thread.message.assistant.delta",
...@@ -390,13 +453,14 @@ const make = Effect.gen(function* () {...@@ -390,13 +453,14 @@ const make = Effect.gen(function* () {
390 yield* rememberAssistantMessageId(event.sessionId, turnId, assistantMessageId);453 yield* rememberAssistantMessageId(event.sessionId, turnId, assistantMessageId);
391 }454 }
392455
393 yield* dispatchAssistantCompletion({456 yield* finalizeAssistantMessage({
394 event,457 event,
395 threadId: thread.id,458 threadId: thread.id,
396 messageId: assistantMessageId,459 messageId: assistantMessageId,
397 ...(turnId ? { turnId } : {}),460 ...(turnId ? { turnId } : {}),
398 createdAt: now,461 createdAt: now,
399 commandTag: "assistant-complete",462 commandTag: "assistant-complete",
463 spillRemainderCommandTag: "assistant-delta-buffer-remainder",
400 });464 });
401465
402 if (turnId) {466 if (turnId) {
...@@ -412,13 +476,14 @@ const make = Effect.gen(function* () {...@@ -412,13 +476,14 @@ const make = Effect.gen(function* () {
412 assistantMessageIds,476 assistantMessageIds,
413 (assistantMessageId) =>477 (assistantMessageId) =>
414 Effect.gen(function* () {478 Effect.gen(function* () {
415 yield* dispatchAssistantCompletion({479 yield* finalizeAssistantMessage({
416 event,480 event,
417 threadId: thread.id,481 threadId: thread.id,
418 messageId: assistantMessageId,482 messageId: assistantMessageId,
419 turnId,483 turnId,
420 createdAt: now,484 createdAt: now,
421 commandTag: "assistant-complete-finalize",485 commandTag: "assistant-complete-finalize",
486 spillRemainderCommandTag: "assistant-delta-buffer-remainder-finalize",
422 });487 });
423 }),488 }),
424 { concurrency: 1 },489 { concurrency: 1 },
apps/server/src/orchestration/decider.projectScripts.test.ts+7-5
...@@ -162,10 +162,12 @@ describe("decider project scripts", () => {...@@ -162,10 +162,12 @@ describe("decider project scripts", () => {
162 const events = Array.isArray(result) ? result : [result];162 const events = Array.isArray(result) ? result : [result];
163 expect(events).toHaveLength(2);163 expect(events).toHaveLength(2);
164 expect(events[0]?.type).toBe("thread.message-sent");164 expect(events[0]?.type).toBe("thread.message-sent");
165 expect(events[1]?.type).toBe("thread.turn-start-requested");165 const turnStartEvent = events[1];
166 expect(events[1]?.causationEventId).toBe(events[0]?.eventId ?? null);166 expect(turnStartEvent?.type).toBe("thread.turn-start-requested");
167 expect((events[1]?.payload as { assistantDeliveryMode?: string }).assistantDeliveryMode).toBe(167 expect(turnStartEvent?.causationEventId).toBe(events[0]?.eventId ?? null);
168 "buffered",168 if (turnStartEvent?.type !== "thread.turn-start-requested") {
169 );169 return;
170 }
171 expect(turnStartEvent.payload.assistantDeliveryMode).toBe("buffered");
170 });172 });
171});173});