authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-04 13:37:03-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-04 13:37:03-07:00
log36caf34c68b7bc6c2e967fdec1812fd08f61cdc8
tree4bec12af023974bedf93972aca056fd880cb7378
parent90e3778660690e4f00bdd4e237fc541725dc1427
signature Signed by PGP key B5690EEEBB952194

fix(desktop,web): contain renderer memory growth and recover from renderer OOM crashes (#5148)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

4 files changed, 173 insertions(+), 8 deletions(-)

apps/desktop/src/window/DesktopWindow.ts+41-4
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import * as Clock from "effect/Clock";
1import * as Context from "effect/Context";2import * as Context from "effect/Context";
2import * as Effect from "effect/Effect";3import * as Effect from "effect/Effect";
3import * as Fiber from "effect/Fiber";4import * as Fiber from "effect/Fiber";
...@@ -25,6 +26,12 @@ const TITLEBAR_LIGHT_SYMBOL_COLOR = "#1f2937";...@@ -25,6 +26,12 @@ const TITLEBAR_LIGHT_SYMBOL_COLOR = "#1f2937";
25const TITLEBAR_DARK_SYMBOL_COLOR = "#f8fafc";26const TITLEBAR_DARK_SYMBOL_COLOR = "#f8fafc";
26const MAIN_WINDOW_BOUNDS_PERSIST_DEBOUNCE_MS = 500;27const MAIN_WINDOW_BOUNDS_PERSIST_DEBOUNCE_MS = 500;
27const DEVELOPMENT_LOAD_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;28const DEVELOPMENT_LOAD_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;
29// Renderer crash (usually V8 OOM on long sessions) recovery: reload after a
30// short delay, at most MAX_ATTEMPTS times per rolling WINDOW so a renderer
31// that dies on boot cannot reload-loop forever.
32const RENDERER_RECOVERY_RELOAD_DELAY_MS = 500;
33const RENDERER_RECOVERY_MAX_ATTEMPTS = 3;
34const RENDERER_RECOVERY_WINDOW_MS = 60_000;
28const DEVELOPMENT_RETRYABLE_LOAD_ERROR_CODES = new Set([35const DEVELOPMENT_RETRYABLE_LOAD_ERROR_CODES = new Set([
29 -2, // ERR_FAILED36 -2, // ERR_FAILED
30 -7, // ERR_TIMED_OUT37 -7, // ERR_TIMED_OUT
...@@ -549,6 +556,7 @@ export const make = Effect.gen(function* () {...@@ -549,6 +556,7 @@ export const make = Effect.gen(function* () {
549556
550 let developmentLoadRetryIndex = 0;557 let developmentLoadRetryIndex = 0;
551 let developmentLoadRetryFiber: Fiber.Fiber<void, never> | undefined;558 let developmentLoadRetryFiber: Fiber.Fiber<void, never> | undefined;
559 let rendererRecoveryTimestamps: number[] = [];
552 const clearDevelopmentLoadRetry = () => {560 const clearDevelopmentLoadRetry = () => {
553 if (developmentLoadRetryFiber === undefined) {561 if (developmentLoadRetryFiber === undefined) {
554 return;562 return;
...@@ -630,10 +638,39 @@ export const make = Effect.gen(function* () {...@@ -630,10 +638,39 @@ export const make = Effect.gen(function* () {
630 },638 },
631 );639 );
632 window.webContents.on("render-process-gone", (_event, details) => {640 window.webContents.on("render-process-gone", (_event, details) => {
633 void runPromise(641 const recoverable =
634 logWindowWarning("main window render process gone", {642 details.reason === "crashed" ||
635 reason: details.reason,643 details.reason === "oom" ||
636 exitCode: details.exitCode,644 details.reason === "abnormal-exit";
645 // Long sessions can OOM the renderer (V8 heap exhaustion from
646 // accumulated thread state). Without a reload the user is left staring
647 // at a dead white window while agents keep running invisibly, so
648 // recover by reloading — the renderer rehydrates from the backend,
649 // which is unaffected. Recovery attempts are bounded so a renderer
650 // that dies immediately on boot cannot reload-loop forever.
651 runFork(
652 Effect.gen(function* () {
653 const now = yield* Clock.currentTimeMillis;
654 rendererRecoveryTimestamps = rendererRecoveryTimestamps.filter(
655 (timestamp) => now - timestamp < RENDERER_RECOVERY_WINDOW_MS,
656 );
657 const shouldRecover =
658 recoverable &&
659 !window.isDestroyed() &&
660 rendererRecoveryTimestamps.length < RENDERER_RECOVERY_MAX_ATTEMPTS;
661 yield* logWindowWarning("main window render process gone", {
662 reason: details.reason,
663 exitCode: details.exitCode,
664 recovering: shouldRecover,
665 });
666 if (!shouldRecover) {
667 return;
668 }
669 rendererRecoveryTimestamps.push(now);
670 yield* Effect.sleep(RENDERER_RECOVERY_RELOAD_DELAY_MS);
671 if (!window.isDestroyed()) {
672 loadApplication();
673 }
637 }),674 }),
638 );675 );
639 });676 });
apps/web/src/components/Sidebar.logic.ts+6-2
...@@ -16,8 +16,12 @@ import { resolveServerBackedAppStageLabel } from "../branding.logic";...@@ -16,8 +16,12 @@ import { resolveServerBackedAppStageLabel } from "../branding.logic";
16export const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]";16export const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]";
17export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 100;17export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 100;
18// Visible sidebar rows are prewarmed into the thread-detail cache so opening a18// Visible sidebar rows are prewarmed into the thread-detail cache so opening a
19// nearby thread usually reuses an already-hot subscription.19// nearby thread usually reuses an already-hot subscription. Each prewarmed
20export const SIDEBAR_THREAD_PREWARM_LIMIT = 10;20// thread holds a live, fully hydrated detail subscription (all messages and
21// activities, growing as agents work) for as long as the row stays visible,
22// so this limit is a direct renderer-heap and server-load multiplier — keep
23// it small; cold opens still render instantly from the cached snapshot.
24export const SIDEBAR_THREAD_PREWARM_LIMIT = 3;
2125
22type SidebarProject = {26type SidebarProject = {
23 id: string;27 id: string;
packages/client-runtime/src/state/threadReducer.test.ts+88
...@@ -633,6 +633,94 @@ describe("applyThreadDetailEvent", () => {...@@ -633,6 +633,94 @@ describe("applyThreadDetailEvent", () => {
633 expect(result.thread.activities[0]?.id).toBe("activity-0");633 expect(result.thread.activities[0]?.id).toBe("activity-0");
634 }634 }
635 });635 });
636
637 it("replaces earlier resolvable context-window updates for the same turn", () => {
638 const contextWindowActivity = (id: string, sequence: number, usedTokens: unknown) => ({
639 id: EventId.make(id),
640 tone: "info" as const,
641 kind: "context-window.updated",
642 summary: "Context window updated",
643 payload: { usedTokens },
644 turnId: TurnId.make("turn-1"),
645 sequence,
646 createdAt: "2026-04-01T11:00:00.000Z",
647 });
648 const otherTurnActivity = contextWindowActivity("activity-other-turn", 2, 500);
649 const existingActivities = [
650 contextWindowActivity("activity-cw-1", 1, 1_000),
651 { ...otherTurnActivity, turnId: TurnId.make("turn-0") },
652 // Malformed row (no usedTokens): must survive, and must not be
653 // treated as the latest value by consumers.
654 contextWindowActivity("activity-cw-malformed", 3, undefined),
655 contextWindowActivity("activity-cw-2", 4, 2_000),
656 ];
657
658 const result = applyThreadDetailEvent(
659 { ...baseThread, activities: existingActivities },
660 {
661 ...baseEventFields,
662 sequence: 20,
663 occurredAt: "2026-04-01T11:02:00.000Z",
664 aggregateKind: "thread",
665 aggregateId: ThreadId.make("thread-1"),
666 type: "thread.activity-appended",
667 payload: {
668 threadId: ThreadId.make("thread-1"),
669 activity: contextWindowActivity("activity-cw-3", 5, 3_000),
670 },
671 },
672 );
673
674 expect(result.kind).toBe("updated");
675 if (result.kind === "updated") {
676 const ids = result.thread.activities.map((activity) => activity.id);
677 // Same-turn resolvable rows collapse to the newest; the other turn's
678 // row and the malformed row are untouched.
679 expect(ids).toEqual(["activity-other-turn", "activity-cw-malformed", "activity-cw-3"]);
680 }
681 });
682
683 it("does not collapse context-window history for a malformed update", () => {
684 const resolvable = {
685 id: EventId.make("activity-cw-resolvable"),
686 tone: "info" as const,
687 kind: "context-window.updated",
688 summary: "Context window updated",
689 payload: { usedTokens: 1_000 },
690 turnId: TurnId.make("turn-1"),
691 sequence: 1,
692 createdAt: "2026-04-01T11:00:00.000Z",
693 };
694
695 const result = applyThreadDetailEvent(
696 { ...baseThread, activities: [resolvable] },
697 {
698 ...baseEventFields,
699 sequence: 21,
700 occurredAt: "2026-04-01T11:03:00.000Z",
701 aggregateKind: "thread",
702 aggregateId: ThreadId.make("thread-1"),
703 type: "thread.activity-appended",
704 payload: {
705 threadId: ThreadId.make("thread-1"),
706 activity: {
707 ...resolvable,
708 id: EventId.make("activity-cw-broken"),
709 payload: { usedTokens: Number.NaN },
710 sequence: 2,
711 },
712 },
713 },
714 );
715
716 expect(result.kind).toBe("updated");
717 if (result.kind === "updated") {
718 // The resolvable row must survive so consumers can still derive a
719 // usage value by walking backwards past the malformed row.
720 const ids = result.thread.activities.map((activity) => activity.id);
721 expect(ids).toEqual(["activity-cw-resolvable", "activity-cw-broken"]);
722 }
723 });
636 });724 });
637725
638 describe("thread.turn-diff-completed", () => {726 describe("thread.turn-diff-completed", () => {
packages/client-runtime/src/state/threadReducer.ts+38-2
...@@ -35,6 +35,24 @@ const activityOrder = O.combineAll<OrchestrationThreadActivity>([...@@ -35,6 +35,24 @@ const activityOrder = O.combineAll<OrchestrationThreadActivity>([
35 O.mapInput(O.String, (a) => a.id),35 O.mapInput(O.String, (a) => a.id),
36]);36]);
3737
38/**
39 * Matches the validity rule in `deriveLatestContextWindowSnapshot` (and the
40 * server's snapshot-side `dropStaleContextWindowActivities`): rows without a
41 * finite, non-negative `usedTokens` are skipped during the consumer's backward
42 * walk, so they must not replace an earlier resolvable row here.
43 */
44function isResolvableContextWindowActivity(activity: OrchestrationThreadActivity): boolean {
45 if (activity.kind !== "context-window.updated") {
46 return false;
47 }
48 const payload =
49 activity.payload && typeof activity.payload === "object"
50 ? (activity.payload as Record<string, unknown>)
51 : null;
52 const usedTokens = payload?.usedTokens;
53 return typeof usedTokens === "number" && Number.isFinite(usedTokens) && usedTokens >= 0;
54}
55
38/**56/**
39 * Apply a single orchestration event to an `OrchestrationThread`, returning57 * Apply a single orchestration event to an `OrchestrationThread`, returning
40 * the updated thread, a deletion signal, or an "unchanged" marker when the58 * the updated thread, a deletion signal, or an "unchanged" marker when the
...@@ -509,10 +527,28 @@ export function applyThreadDetailEvent(...@@ -509,10 +527,28 @@ export function applyThreadDetailEvent(
509527
510 // ── Activities ──────────────────────────────────────────────────528 // ── Activities ──────────────────────────────────────────────────
511 case "thread.activity-appended": {529 case "thread.activity-appended": {
530 const activity = event.payload.activity;
531 // A resolvable context-window update supersedes earlier resolvable ones
532 // for the same turn: consumers only read the latest value (walking the
533 // array backwards), and providers stream these updates continuously, so
534 // retaining the history grows the thread by thousands of rows over a
535 // long session. Mirrors the server-side snapshot rule in
536 // dropStaleContextWindowActivities; retention stays per turn so a
537 // thread.reverted that discards turns can still resolve a value from
538 // the turns that survive.
539 const supersedesContextWindow = isResolvableContextWindowActivity(activity);
512 const activities = pipe(540 const activities = pipe(
513 thread.activities,541 thread.activities,
514 Arr.filter((activity) => activity.id !== event.payload.activity.id),542 Arr.filter(
515 Arr.append(event.payload.activity),543 (entry) =>
544 entry.id !== activity.id &&
545 !(
546 supersedesContextWindow &&
547 entry.turnId === activity.turnId &&
548 isResolvableContextWindowActivity(entry)
549 ),
550 ),
551 Arr.append(activity),
516 Arr.sort(activityOrder),552 Arr.sort(activityOrder),
517 );553 );
518554