authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 02:01:46-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 02:01:46-07:00
log5fc1c5cb4cfa562552fb0a5fc8278b176a765256
tree044d39da7d87091b2c6fcb7c2f003c17283449f5
parent3a6ab583d2799c9c904d8d0943afba60ec2cc5f0
signature Signed by PGP key B5690EEEBB952194

perf(server): SQLite checkpoints no longer block the event loop (#19324)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

4 files changed, 216 insertions(+), 29 deletions(-)

apps/server/src/persistence/Sqlite.test.ts+6-24
......@@ -58,31 +58,13 @@ it.effect("waits out a concurrent writer instead of failing with SQLITE_BUSY", (
5858 );
5959});
6060
61it.effect("shrinks the WAL file back to the size limit after a large write", () => {
62 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-wal-"));
63 const dbPath = NodePath.join(tempDir, "state.sqlite");
64 const walFileSize = () => NodeFS.statSync(`${dbPath}-wal`).size;
65 // About 25% more 4 KB rows than the limit holds, in one transaction.
66 const rowCount = Math.ceil((WAL_SIZE_LIMIT_BYTES * 1.25) / 4000);
67
68 return Effect.gen(function* () {
61it.effect("limits the WAL file size in the shared persistence setup", () =>
62 Effect.gen(function* () {
6963 const sql = yield* SqlClient.SqlClient;
70 yield* sql`CREATE TABLE wal_probe(payload BLOB)`;
71 yield* sql`
72 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${rowCount})
73 INSERT INTO wal_probe(payload) SELECT randomblob(4000) FROM n
74 `;
75 assert.isAbove(walFileSize(), WAL_SIZE_LIMIT_BYTES);
76
77 // The auto-checkpoint after the large commit copied every frame into the
78 // database, so the next commit restarts the WAL and cuts the file back.
79 yield* sql`INSERT INTO wal_probe(payload) VALUES (x'00')`;
80 assert.isAtMost(walFileSize(), WAL_SIZE_LIMIT_BYTES);
81 }).pipe(
82 Effect.provide(SqlitePersistence.layerFromPath(dbPath).pipe(Layer.provide(NodeServices.layer))),
83 Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
84 );
85});
64 const rows = yield* sql<{ readonly journal_size_limit: number }>`PRAGMA journal_size_limit`;
65 assert.equal(rows[0]?.journal_size_limit, WAL_SIZE_LIMIT_BYTES);
66 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
67);
8668
8769it.effect("collects planner statistics for an existing database on open", () => {
8870 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-stats-"));
apps/server/src/persistence/Sqlite.ts+3
......@@ -50,6 +50,9 @@ export const layerFromPath = Effect.fn("makeSqlitePersistenceLive")(function* (d
5050 NodeSqliteClient.layer({
5151 filename: dbPath,
5252 readerWorker: true,
53 // Checkpoints sync the database, with F_FULLFSYNC on macOS. Run them off
54 // the event loop so a slow flush cannot stall clients.
55 checkpointWorker: true,
5356 spanAttributes: {
5457 "db.name": path.basename(dbPath),
5558 "service.name": "t3code-server",
packages/shared/src/nodeSqliteClient.test.ts+84
......@@ -678,3 +678,87 @@ it.effect("does not hand out a worker that reports ready after another read time
678678 }),
679679 ),
680680);
681
682const withCheckpointClient = <A, E>(
683 use: (
684 sql: SqlClient.SqlClient,
685 filename: string,
686 ) => Effect.Effect<A, E, SqlClient.SqlClient | FileSystem.FileSystem>,
687) =>
688 Effect.gen(function* () {
689 const fs = yield* FileSystem.FileSystem;
690 const path = yield* Path.Path;
691 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sqlite-checkpoint-" });
692 const filename = path.join(directory, "state.sqlite");
693 return yield* Effect.gen(function* () {
694 const sql = yield* SqlClient.SqlClient;
695 yield* sql`PRAGMA journal_mode = WAL`;
696 return yield* use(sql, filename);
697 }).pipe(Effect.provide(SqliteClient.layer({ filename, checkpointWorker: true })));
698 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer));
699
700// Resolves once the worker has handled a request sent now, after every
701// statement so far. Earlier requests from the writer reply with null.
702const checkpointWorkerReply = (worker: NodeWorkerThreads.Worker) =>
703 Effect.promise(
704 () =>
705 new Promise<void>((resolve) => {
706 const onReply = (reply: unknown) => {
707 if (reply !== "barrier") return;
708 worker.off("message", onReply);
709 resolve();
710 };
711 worker.on("message", onReply);
712 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
713 worker.postMessage("barrier");
714 }),
715 );
716
717it.effect("checkpoints the WAL on the worker instead of the writer", () =>
718 withCheckpointClient((sql, filename) =>
719 Effect.gen(function* () {
720 const fs = yield* FileSystem.FileSystem;
721 const walSize = Effect.map(fs.stat(`${filename}-wal`), (info) => Number(info.size));
722 const worker = startedWorkers.at(-1)!;
723 const limit = 1024 * 1024;
724 yield* sql.unsafe(`PRAGMA journal_size_limit = ${limit}`);
725 const autocheckpoint = yield* sql<{
726 readonly wal_autocheckpoint: number;
727 }>`PRAGMA wal_autocheckpoint`;
728 // The writer leaves a WAL of this size to the worker.
729 assert.equal(autocheckpoint[0]?.wal_autocheckpoint, 10_000);
730
731 // About 1200 frames, past the 1000 that start a checkpoint.
732 yield* sql`CREATE TABLE pages(payload BLOB)`;
733 yield* sql`
734 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 1200)
735 INSERT INTO pages(payload) SELECT randomblob(4000) FROM n
736 `;
737 yield* checkpointWorkerReply(worker);
738 const wal = yield* sql<{
739 readonly log: number;
740 readonly checkpointed: number;
741 }>`PRAGMA wal_checkpoint(NOOP)`;
742 assert.isAbove(wal[0]!.log, 1000);
743 assert.equal(wal[0]!.checkpointed, wal[0]!.log);
744
745 // Every frame is in the database, so the next commit restarts the WAL and
746 // cuts the file back to the size limit.
747 assert.isAbove(yield* walSize, limit);
748 yield* sql`INSERT INTO pages(payload) VALUES (x'00')`;
749 assert.isAtMost(yield* walSize, limit);
750 }),
751 ),
752);
753
754it.effect("hands checkpoints back to the writer when the worker stops", () =>
755 withCheckpointClient((sql) =>
756 Effect.gen(function* () {
757 yield* Effect.promise(() => startedWorkers.at(-1)!.terminate());
758 const autocheckpoint = yield* sql<{
759 readonly wal_autocheckpoint: number;
760 }>`PRAGMA wal_autocheckpoint`;
761 assert.equal(autocheckpoint[0]?.wal_autocheckpoint, 1000);
762 }),
763 ),
764);
packages/shared/src/nodeSqliteClient.ts+123-5
......@@ -47,6 +47,12 @@ export interface SqliteClientConfig {
4747 readonly readerWorker?: boolean | undefined;
4848 /** How long a reader worker may take to open the database before reads use the writer. Default 10 seconds. */
4949 readonly readerStartupTimeout?: Duration.Input | undefined;
50 /**
51 * Run WAL checkpoints on a read-write connection in a worker thread, so
52 * copying pages into the database and syncing it never block the event loop.
53 * Ignored for in-memory databases.
54 */
55 readonly checkpointWorker?: boolean | undefined;
5056}
5157
5258/**
......@@ -477,12 +483,115 @@ const makeReader = Effect.fnUntraced(function* (options: SqliteClientConfig) {
477483 });
478484});
479485
486// SQLite's default wal_autocheckpoint, in WAL frames (pages).
487const CHECKPOINT_FRAMES = 1000;
488// While the worker runs, the writer still checkpoints a WAL this long. Writes
489// with no pause leave the worker behind, and the WAL can only restart once
490// every frame is checkpointed, so this keeps the file from growing forever.
491const WRITER_BACKSTOP_FRAMES = 10 * CHECKPOINT_FRAMES;
492// How long writes must pause before the worker checkpoints the rest of a long WAL.
493const CHECKPOINT_IDLE_MS = 100;
494
495// Each message asks the worker to checkpoint if CHECKPOINT_FRAMES frames are
496// waiting, and it replies with the same message when done. Once writes pause, it also checkpoints the
497// frames that arrived during its last checkpoint, so the writer can restart the
498// WAL. NOOP only counts frames. SQLite before 3.51 reads NOOP as PASSIVE and
499// would checkpoint on every request, so the worker refuses to start there and
500// the writer keeps checkpointing.
501const checkpointWorkerSource = `
502const { parentPort, workerData } = require("node:worker_threads");
503const { DatabaseSync } = require("node:sqlite");
504const db = new DatabaseSync(workerData.filename);
505const version = db.prepare("SELECT sqlite_version() AS version").get().version;
506const [major, minor] = version.split(".").map(Number);
507if (major < 3 || (major === 3 && minor < 51)) throw new Error("SQLite " + version + " has no NOOP checkpoint");
508// macOS fsync() does not flush the drive cache; F_FULLFSYNC does. Other platforms ignore this.
509db.exec("PRAGMA checkpoint_fullfsync = ON;");
510const frames = db.prepare("PRAGMA wal_checkpoint(NOOP)");
511const checkpoint = db.prepare("PRAGMA wal_checkpoint(PASSIVE)");
512const finish = () => {
513 const { log, checkpointed } = frames.get();
514 if (log >= workerData.frames && checkpointed < log) checkpoint.get();
515};
516let idle;
517parentPort.on("message", (request) => {
518 const { log, checkpointed } = frames.get();
519 if (log - checkpointed >= workerData.frames) checkpoint.get();
520 clearTimeout(idle);
521 idle = setTimeout(finish, workerData.idleMs);
522 parentPort.postMessage(request);
523});
524`;
525
526/**
527 * Moves the writer's WAL checkpoints to a worker. Call the returned function
528 * after each statement on the writer. While the worker runs, the writer only
529 * checkpoints at WRITER_BACKSTOP_FRAMES. If the worker fails or exits, the
530 * writer goes back to SQLite's default.
531 */
532const startCheckpointWorker = Effect.fnUntraced(function* (
533 filename: string,
534 writer: NodeSqlite.DatabaseSync,
535) {
536 let worker: NodeWorkerThreads.Worker;
537 try {
538 worker = new NodeWorkerThreads.Worker(checkpointWorkerSource, {
539 eval: true,
540 workerData: { filename, frames: CHECKPOINT_FRAMES, idleMs: CHECKPOINT_IDLE_MS },
541 });
542 } catch {
543 // No worker threads here: the writer keeps checkpointing.
544 return undefined;
545 }
546 writer.exec(`PRAGMA wal_autocheckpoint = ${WRITER_BACKSTOP_FRAMES};`);
547 let stopped = false;
548 let checking = false;
549 let checkAgain = false;
550 const handBack = () => {
551 if (stopped) return;
552 stopped = true;
553 try {
554 writer.exec(`PRAGMA wal_autocheckpoint = ${CHECKPOINT_FRAMES};`);
555 } catch {
556 // The writer is already closed.
557 }
558 };
559 worker.on("error", handBack);
560 worker.on("exit", handBack);
561 yield* Effect.addFinalizer(() => Effect.promise(() => worker.terminate()));
562
563 // At most one request is in flight. A statement that runs during it asks for
564 // one more check, so the frames of a last commit before idle still get one.
565 const check = () => {
566 if (stopped) return;
567 if (checking) {
568 checkAgain = true;
569 return;
570 }
571 checking = true;
572 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
573 worker.postMessage(null);
574 };
575 worker.on("message", () => {
576 checking = false;
577 if (checkAgain) {
578 checkAgain = false;
579 check();
580 }
581 });
582 // After the listeners: adding a "message" listener refs the worker again.
583 // A checkpoint in progress must not keep the process alive.
584 worker.unref();
585 return check;
586});
587
480588const make = Effect.fn("makeWithDatabase")(function* (
481589 options: SqliteClientConfig,
482590): Effect.fn.Return<Client.SqlClient, SqlError, Scope.Scope | Reactivity.Reactivity> {
483591 yield* checkNodeSqliteCompat();
484592
485593 const compiler = Statement.makeCompilerSqlite(options.transformQueryNames);
594 const isFileDatabase = options.filename !== ":memory:" && options.filename !== "";
486595 const transformRows = options.transformResultNames
487596 ? Statement.defaultTransforms(options.transformResultNames).array
488597 : undefined;
......@@ -510,6 +619,11 @@ const make = Effect.fn("makeWithDatabase")(function* (
510619 }),
511620 }).pipe(Effect.orDie),
512621 );
622 // Started after the close finalizer, so the worker stops before the writer closes.
623 const checkpoint =
624 options.checkpointWorker && isFileDatabase
625 ? yield* startCheckpointWorker(options.filename, db)
626 : undefined;
513627
514628 const statementReaderCache = new WeakMap<NodeSqlite.StatementSync, boolean>();
515629 const hasRows = (statement: NodeSqlite.StatementSync): boolean => {
......@@ -550,9 +664,12 @@ const make = Effect.fn("makeWithDatabase")(function* (
550664 try {
551665 statement.setReadBigInts(Boolean(Context.get(fiber.context, Client.SafeIntegers)));
552666 if (hasRows(statement)) {
553 return Effect.succeed(statement.all(...(params as any)));
667 const rows = statement.all(...(params as any));
668 checkpoint?.();
669 return Effect.succeed(rows);
554670 }
555671 const result = statement.run(...(params as any));
672 checkpoint?.();
556673 return Effect.succeed(raw ? (result as unknown as ReadonlyArray<any>) : []);
557674 } catch (cause) {
558675 return Effect.fail(
......@@ -581,11 +698,14 @@ const make = Effect.fn("makeWithDatabase")(function* (
581698 if (hasRows(statement)) {
582699 statement.setReturnArrays(true);
583700 // Safe to cast to array after we've setReturnArrays(true)
584 return statement.all(...(params as any)) as unknown as ReadonlyArray<
701 const rows = statement.all(...(params as any)) as unknown as ReadonlyArray<
585702 ReadonlyArray<unknown>
586703 >;
704 checkpoint?.();
705 return rows;
587706 }
588707 statement.run(...(params as any));
708 checkpoint?.();
589709 return [];
590710 },
591711 catch: (cause) =>
......@@ -646,9 +766,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
646766 const semaphore = yield* Semaphore.make(1);
647767 const connection = yield* makeConnection;
648768 const acquireReader =
649 options.readerWorker && options.filename !== ":memory:" && options.filename !== ""
650 ? yield* makeReader(options)
651 : undefined;
769 options.readerWorker && isFileDatabase ? yield* makeReader(options) : undefined;
652770 const readerSemaphore = yield* Semaphore.make(1);
653771
654772 const writer = { lock: semaphore, connection };