authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-08-11 23:04:26-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
logbc5017b0e58626345037c3bd2e32e6a2a1b576b4
tree9e2a50c394352f533ee2dc8f3790836675a700ab
parent98be3e2bd40a40c6595de522d087092df63cdbb7
signature Commit is signed but in an unrecognized format.

feat: add sqlite database reloading

this is used by scan3 to notify active servers that there is new data to be aware of.

3 files changed, 66 insertions(+), 14 deletions(-)

framework/lib/sqlite.ts+29-11
......@@ -1,29 +1,32 @@
11// Guard against reloads and bundler duplication.
2type DbMap = Map<string, WrappedDatabase>;
23// @ts-ignore
3const map = globalThis[Symbol.for("clover.db")] ??= new Map<
4 string,
5 WrappedDatabase
6>();
4const map = (globalThis[Symbol.for("clover.db")] as DbMap) ??= new Map();
5for (const v of map.values()) {
6 v.node.close();
7}
8map.clear();
79
810export function getDb(file: string) {
9 let db: WrappedDatabase | null = map.get(file);
11 let db = map.get(file);
1012 if (db) return db;
1113 const fileWithExt = file.includes(".") ? file : file + ".sqlite";
1214 db = new WrappedDatabase(
13 new DatabaseSync(
14 path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
15 ),
15 path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
1616 );
1717 map.set(file, db);
1818 return db;
1919}
2020
2121export class WrappedDatabase {
22 file: string;
2223 node: DatabaseSync;
24 stmts: Stmt[];
2325 stmtTableMigrate: WeakRef<StatementSync> | null = null;
2426
25 constructor(node: DatabaseSync) {
26 this.node = node;
27 constructor(file: string) {
28 this.file = file;
29 this.node = new DatabaseSync(file);
2730 this.node.exec(`
2831 create table if not exists clover_migrations (
2932 key text not null primary key,
......@@ -66,7 +69,18 @@ export class WrappedDatabase {
6669 if (err) (err as { query: string }).query = query;
6770 throw err;
6871 }
69 return new Stmt(prepared);
72 const stmt = new Stmt<Args, Result>(prepared);
73 this.stmts.push(stmt);
74 return stmt;
75 }
76
77 reload() {
78 const newNode = new DatabaseSync(this.file);
79 this.node.close();
80 this.node = newNode;
81 for (const stmt of this.stmts) {
82 stmt["reload"](newNode);
83 }
7084 }
7185}
7286
......@@ -80,6 +94,10 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
8094 this.query = node.sourceSQL;
8195 }
8296
97 private reload(db: DatabaseSync) {
98 this.#node = db.prepare(this.query);
99 }
100
83101 /** Get one row */
84102 get(...args: Args): Row | null {
85103 return this.#wrap(args, () => {
src/file-viewer/bin/scan3.ts+26-1
......@@ -12,6 +12,7 @@
1212// quality cannot be written with AI.
1313const root = path.resolve("/Volumes/clover/Published");
1414const workDir = path.resolve(".clover/derived");
15const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
1516
1617export async function main() {
1718 const start = performance.now();
......@@ -337,13 +338,37 @@ export async function main() {
337338 console.info("No new derived assets");
338339 }
339340
341 MediaFile.db.prepare("VACUUM").run();
342 MediaFile.db.reload();
343 // TODO: reload prod web instance
344 await rsync.spawn({
345 args: [
346 MediaFile.db.file,
347 "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
348 ],
349 title: "Uploading Database",
350 cwd: process.cwd(),
351 });
352 {
353 const res = await fetch("https://db.paperclover.net/reload", {
354 method: "post",
355 headers: {
356 Authorization: sotToken,
357 },
358 });
359 if (!res.ok) {
360 console.warn(
361 `Failed to reload remote database ${res.status} ${res.statusText}`,
362 );
363 }
364 }
365
340366 console.info(
341367 "Updated file viewer index in \x1b[1m" +
342368 ((performance.now() - start) / 1000).toFixed(1) +
343369 "s\x1b[0m",
344370 );
345371
346 MediaFile.db.prepare("VACUUM").run();
347372 const { duration, count } = MediaFile.db
348373 .prepare<[], { count: number; duration: number }>(
349374 `
src/source-of-truth.ts+11-2
......@@ -20,7 +20,7 @@
2020const app = new Hono();
2121export default app;
2222
23const token = process.env.CLOVER_SOT_KEY;
23const token = UNWRAP(process.env.CLOVER_SOT_KEY);
2424
2525const nasRoot = process.platform === "win32"
2626 ? "\\\\zenith\\clover"
......@@ -48,12 +48,13 @@ const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();
4848app.get("/file/*", async (c) => {
4949 const fullQuery = c.req.path.slice("/file".length);
5050 const [filePath, derivedAsset, ...invalid] = fullQuery.split("$/");
51 ASSERT(filePath != null);
5152 if (invalid.length > 0) return c.notFound();
5253 if (filePath.length <= 1) return c.notFound();
5354 const permissions = FilePermissions.getByPrefix(filePath);
5455 if (permissions !== 0) {
5556 if (c.req.header("Authorization") !== token) {
56 return c.json({ error: "invalid authorization header" });
57 return c.json({ error: "invalid authorization header" }, 401);
5758 }
5859 }
5960 const file = MediaFile.getByPath(filePath);
......@@ -104,6 +105,14 @@ app.get("/file/*", async (c) => {
104105 return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream);
105106});
106107
108app.post("/reload", async (c) => {
109 if (c.req.header("Authorization") !== token) {
110 return c.json({ error: "invalid authorization header" }, 401);
111 }
112 MediaFile.db.reload();
113 return c.body(null, 204);
114});
115
107116const openFile = util.promisify(fsCallbacks.open);
108117const closeFile = util.promisify(fsCallbacks.close);
109118