Commit 08463e2c401ce87858aaaebcb70ed86fb002fb5f | the-knife

author	Guillermo Casanova <75276669+Gigioxx@users.noreply.github.com> 2026-09-08 23:27:05-03:00
committer	GitHub <noreply@github.com> 2026-09-08 19:27:05-07:00
log	08463e2c401ce87858aaaebcb70ed86fb002fb5f
tree	6900e14419e994cf95ab7229921fee6ac866c32e
parent	1862686f9e42bf1d77e7f2ad03e24d7b0c7908f1
signature	Signed by PGP key B5690EEEBB952194

fix(server): release consumed event replay pages (#10777)

2 files changed, 82 insertions(+), 45 deletions(-)

apps
- server
  - src
    persistence
    Layers
    OrchestrationEventStore.test.ts
    OrchestrationEventStore.ts

apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts+51

...	@@ -1,3 +1,5 @@	...	@@ -1,3 +1,5 @@
		1	import * as NodeV8 from "node:v8";
		2
1	import {	3	import {
2	CommandId,	4	CommandId,
3	EventId,	5	EventId,
...	@@ -303,6 +305,55 @@ layer("OrchestrationEventStore", (it) => {	...	@@ -303,6 +305,55 @@ layer("OrchestrationEventStore", (it) => {
303	replayed.map((event) => event.sequence),	305	replayed.map((event) => event.sequence),
304	persisted.map((event) => event.sequence),	306	persisted.map((event) => event.sequence),
305	);	307	);
		308	const limited = store.readFromSequence(persisted[0]!.sequence, 501.9);
		309	for (let run = 0; run < 2; run++) {
		310	assert.deepEqual(
		311	(yield* Stream.runCollect(limited)).map((event) => event.sequence),
		312	persisted.slice(1).map((event) => event.sequence),
		313	);
		314	}
		315	assert.deepEqual(yield* Stream.runCollect(store.readFromSequence(0, -1)), []);
306	}),	316	}),
307	);	317	);
308	});	318	});
		319
		320	for (const reader of ["all", "aggregate"] as const) {
		321	it.effect(`releases consumed pages during ${reader} replay`, () =>
		322	Effect.gen(function* () {
		323	const store = yield* OrchestrationEventStore;
		324	const threadId = ThreadId.make(`retention-${reader}`);
		325	yield* Effect.forEach(
		326	Array.from({ length: 1_501 }, (_, index) => index),
		327	(index) => store.append(messageEvent(threadId, `retention-${reader}-${index}`)),
		328	{ discard: true },
		329	);
		330	// oxlint-disable-next-line typescript/no-extraneous-class -- Identifies page markers for V8's heap query.
		331	class ReplayPage {}
		332	let count = 0;
		333	const replay =
		334	reader === "all"
		335	? store.readAll()
		336	: store.readAggregateRange({
		337	aggregateKind: "thread",
		338	aggregateId: threadId,
		339	fromSequenceExclusive: 0,
		340	toSequenceInclusive: 1_501,
		341	limit: 1_501,
		342	});
		343	yield* Stream.runForEach(replay, (event) =>
		344	Effect.sync(() => {
		345	assert.equal(event.sequence, count + 1);
		346	if (count % 500 === 0) {
		347	// Count live page markers after full GC, without timing or heap-size thresholds.
		348	Object.assign(event, { replayPage: new ReplayPage() });
		349	assert.isAtMost(NodeV8.queryObjects(ReplayPage, { format: "count" }), 1);
		350	}
		351	count++;
		352	}),
		353	);
		354	assert.equal(count, 1_501);
		355	}).pipe(
		356	Effect.provide(OrchestrationEventStoreLive.pipe(Layer.provide(SqlitePersistenceMemory))),
		357	),
		358	);
		359	}

apps/server/src/persistence/Layers/OrchestrationEventStore.ts+31-45

 @@ -287,11 +287,9 @@ const makeEventStore = Effect.gen(function* () {
     if (normalizedLimit === 0) {
       return Stream.empty;
     }
-    const readPage = (
+    return Stream.paginate(
-      cursor: number,
+      { cursor: sequenceExclusive, remaining: normalizedLimit },
-      remaining: number,
+      ({ cursor, remaining }) =>
-    ): Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError> =>
-      Stream.fromEffect(
         readEventRowsFromSequence({
           sequenceExclusive: cursor,
           limit: Math.min(remaining, READ_PAGE_SIZE),
 @@ -311,24 +309,18 @@ const makeEventStore = Effect.gen(function* () {
               ),
             ),
           ),
+          Effect.map((events) => {
+            const last = events.at(-1);
+            const nextRemaining = remaining - events.length;
+            return [
+              events,
+              last === undefined || nextRemaining <= 0
+                ? Option.none()
+                : Option.some({ cursor: last.sequence, remaining: nextRemaining }),
+            ] as const;
+          }),
         ),
-      ).pipe(
+    );
-        Stream.flatMap((events) => {
-          if (events.length === 0) {
-            return Stream.empty;
-          }
-          const nextRemaining = remaining - events.length;
-          if (nextRemaining <= 0) {
-            return Stream.fromIterable(events);
-          }
-          return Stream.concat(
-            Stream.fromIterable(events),
-            readPage(events[events.length - 1]!.sequence, nextRemaining),
-          );
-        }),
-      );
-    return readPage(sequenceExclusive, normalizedLimit);
   };
   const findEventAfter = SqlSchema.findOneOption({
 @@ -363,11 +355,9 @@ const makeEventStore = Effect.gen(function* () {
     if (limit === 0 || input.fromSequenceExclusive >= input.toSequenceInclusive) {
       return Stream.empty;
     }
-    const readPage = (
+    return Stream.paginate(
-      cursor: number,
+      { cursor: input.fromSequenceExclusive, remaining: limit },
-      remaining: number,
+      ({ cursor, remaining }) =>
-    ): Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError> =>
-      Stream.fromEffect(
         readAggregateEventRows({
           ...input,
           fromSequenceExclusive: cursor,
 @@ -388,25 +378,21 @@ const makeEventStore = Effect.gen(function* () {
               ),
             ),
           ),
+          Effect.map((events) => {
+            const last = events.at(-1);
+            const nextRemaining = remaining - events.length;
+            return [
+              events,
+              last === undefined ||
+              events.length < READ_PAGE_SIZE ||
+              nextRemaining === 0 ||
+              last.sequence >= input.toSequenceInclusive
+                ? Option.none()
+                : Option.some({ cursor: last.sequence, remaining: nextRemaining }),
+            ] as const;
+          }),
         ),
-      ).pipe(
+    );
-        Stream.flatMap((events) => {
-          const last = events.at(-1);
-          if (last === undefined) {
-            return Stream.empty;
-          }
-          const nextRemaining = remaining - events.length;
-          if (
-            events.length < READ_PAGE_SIZE ||
-            nextRemaining === 0 ||
-            last.sequence >= input.toSequenceInclusive
-          ) {
-            return Stream.fromIterable(events);
-          }
-          return Stream.concat(Stream.fromIterable(events), readPage(last.sequence, nextRemaining));
-        }),
-      );
-    return readPage(input.fromSequenceExclusive, limit);
   };
   const getAggregateReplayStats: OrchestrationEventStoreShape["getAggregateReplayStats"] = (