Commit 12f1b96207363924cca0970b5e6cb515a6d7541a | sitegen

framework/lib/sqlite.ts+33-16

 @@ -26,7 +26,7 @@ export class WrappedDatabase {
   constructor(file: string) {
     this.file = file;
-    this.node = new DatabaseSync(file);
+    this.node = WrappedDatabase.open(file);
     this.node.exec(`
       create table if not exists clover_migrations (
         key text not null primary key,
 @@ -35,6 +35,16 @@ export class WrappedDatabase {
     `);
   }
+  // wal so the source of truth can serve reads + db snapshots while the
+  // indexer writes. foreign keys are per-connection and must be re-applied
+  // on reload.
+  private static open(file: string) {
+    const node = new DatabaseSync(file);
+    node.exec(`pragma journal_mode = wal;`);
+    node.exec(`pragma foreign_keys = on;`);
+    return node;
+  }
   // TODO: add migration support
   // the idea is you keep `schema` as the new schema but can add
   // migrations to the mix really easily.
 @@ -60,20 +70,13 @@ export class WrappedDatabase {
     );
     query = lines.map((x) => x.slice(trim)).join("\n");
-    let prepared;
+    const stmt = new Stmt<Args, Result>(this, query);
-    try {
-      prepared = this.node.prepare(query);
-    } catch (err) {
-      if (err) (err as { query: string }).query = query;
-      throw err;
-    }
-    const stmt = new Stmt<Args, Result>(prepared);
     this.stmts.push(stmt);
     return stmt;
   }
   reload() {
-    const newNode = new DatabaseSync(this.file);
+    const newNode = WrappedDatabase.open(this.file);
     this.node.close();
     this.node = newNode;
     for (const stmt of this.stmts) {
 @@ -83,17 +86,31 @@ export class WrappedDatabase {
 }
 export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
-  #node: StatementSync;
+  // statements prepare lazily on first use, so that importing a model
+  // module never requires its tables to exist yet (the migration scripts
+  // depend on this, and it makes `reload` cheap).
+  #db: WrappedDatabase;
+  #lazyNode: StatementSync | null = null;
   #class: any | null = null;
   query: string;
-  constructor(node: StatementSync) {
+  constructor(db: WrappedDatabase, query: string) {
-    this.#node = node;
+    this.#db = db;
-    this.query = node.sourceSQL;
+    this.query = query;
+  }
+  get #node(): StatementSync {
+    if (this.#lazyNode) return this.#lazyNode;
+    try {
+      return this.#lazyNode = this.#db.node.prepare(this.query);
+    } catch (err) {
+      if (err) (err as any).query = this.query;
+      throw err;
+    }
   }
-  private reload(db: DatabaseSync) {
+  private reload(_db: DatabaseSync) {
-    this.#node = db.prepare(this.query);
+    this.#lazyNode = null;
   }
   /** Get one row */

lib/log.ts+19-6

 @@ -457,11 +457,24 @@ export function createTerminalWidgetHost(
     timer = null;
     ASSERT(!rendering);
     rendering = true;
+    // the finally is load-bearing: if a render callback or terminal write
+    // throws, `rendering` must reset, or the exit path's cancel() asserts
+    // and masks the original error.
+    try {
+      redrawCallbackInner();
+    } finally {
+      rendering = false;
+    }
+  }
+  function redrawCallbackInner() {
     redrawTime = (lastFlush = now()) - 0.00001; // windows time precision workaround
     // trivial path when not using widgets
     if (!lines.length && !widgets.length) {
-      ASSERT(buffer);
+      // a redraw can get scheduled with nothing to write (e.g. widgets torn
+      // down before the timer fired); it is a no-op, not an error
+      if (!buffer) return;
       needsToRestoreCursor = false;
       needsToSaveCursor = false;
       if (writeOutputTemporaryLock) {
 @@ -482,7 +495,6 @@ export function createTerminalWidgetHost(
       }
       partialLineIndex = partialLineLength(buffer);
       buffer = "";
-      rendering = false;
       return;
     }
 @@ -544,7 +556,6 @@ export function createTerminalWidgetHost(
       if (buffer) terminal.writeOutput(buffer);
       buffer = "";
       if (hasSyncStart) terminal.writeInteractive(ansi.syncEnd);
-      rendering = false;
       return;
     }
 @@ -645,7 +656,6 @@ export function createTerminalWidgetHost(
     needsToSaveCursor = false;
     lines = newWidgetLines;
     buffer = "";
-    rendering = false;
   }
   function redrawSoon(ms: number) {
 @@ -770,8 +780,11 @@ export function createTerminalWidgetHost(
       };
     },
     cancel() {
-      ASSERT(!rendering, "cannot call cancel() during rendering");
+      // cancel may be reached from exit handlers while a redraw is on the
-      flushAndClear(false);
+      // stack (a crash inside a render callback unwinds through here); skip
+      // the flush in that case and just tear down, so the original error
+      // is the one that surfaces.
+      if (!rendering) flushAndClear(false);
       widgets.splice(0, widgets.length);
       internals.splice(0, internals.length);
     },

lib/progress.test.ts+25

...	@@ -248,6 +248,30 @@ test("event encoding round trip cases", async () => {	...	@@ -248,6 +248,30 @@ test("event encoding round trip cases", async () => {
248	},	248	},
249	[9],	249	[9],
250	]);	250	]);
		251
		252	// log messages: a non-info level without scope/stack/custom previously
		253	// desynced the whole stream (the flag tests used `&&` instead of `&`),
		254	// and the message/stack-frame loops never decremented their counters
		255	await roundTrip([
		256	0,
		257	{
		258	k: 1 as EncodedKey,
		259	t: "encode av1",
		260	l: [
		261	{ level: "warn", text: "deprecated pixel format", time: 1768210850000 },
		262	{ level: "info", text: "hello", time: 1768210851000 },
		263	{
		264	level: "error",
		265	text: "scoped + stacked",
		266	time: 1768210852000,
		267	scope: "ffmpeg",
		268	stack: [{ fn: "spawn", file: "ffmpeg.ts", line: 12, col: 3 }],
		269	},
		270	],
		271	[progress.internals.kNode]: fakeNode,
		272	},
		273	[1],
		274	]);
251	});	275	});
252		276
253	function cleanProgressEvent(event: progress.StreamEvent) {	277	function cleanProgressEvent(event: progress.StreamEvent) {
...	@@ -255,6 +279,7 @@ function cleanProgressEvent(event: progress.StreamEvent) {	...	@@ -255,6 +279,7 @@ function cleanProgressEvent(event: progress.StreamEvent) {
255	typeof x === "object" && !Array.isArray(x)	279	typeof x === "object" && !Array.isArray(x)
256	? testing.removeUndefinedKeys({	280	? testing.removeUndefinedKeys({
257	...x,	281	...x,
		282	l: x.l?.map((m) => testing.removeUndefinedKeys({ ...m })),
258	E: undefined,	283	E: undefined,
259	p: undefined,	284	p: undefined,
260	s: undefined,	285	s: undefined,

lib/progress.ts+36-11

 @@ -1007,6 +1007,7 @@ export function decodeEventStream<
       }
       ASSERT(hasEmittedEnd, "Stream terminated early.");
     } catch (err) {
+      console.error(err);
       reader.cancel(err);
       if (!signal.aborted) reject(err);
     } finally {
 @@ -1251,6 +1252,9 @@ class Decoder<
           t: text,
           v: value,
           e: total,
+          s: showTotal,
+          p: passive,
+          h: hidden,
           l: messages,
           c: children,
           E: estimatedTime,
 @@ -1265,6 +1269,9 @@ class Decoder<
           if (text) node.text = text;
           if (value) node.value = value;
           if (total) node.total = total;
+          if (showTotal != null) node.showTotal = showTotal;
+          if (passive != null) node.passive = passive;
+          if (hidden != null) node.hidden = hidden;
           for (const message of messages ?? []) node.log.writeMessage(message);
           if (estimatedTime) node.estimatedTime = estimatedTime;
           if (formattedValue) node.valueFormatter = () => formattedValue;
 @@ -1275,6 +1282,9 @@ class Decoder<
             text,
             value,
             total,
+            showTotal,
+            passive,
+            hidden,
             messages,
             estimatedTime,
             formattedValue,
 @@ -1301,7 +1311,9 @@ class Decoder<
   startRecursive(key: EncodedKey, opts: PendingStart): Ref {
     ASSERT(this.pendingStart.delete(key));
-    const parentId = UNWRAP(this.pendingParents.get(key));
+    // a node whose parent linkage is missing (malformed stream) attaches at
+    // the root rather than killing the whole decoder
+    const parentId = this.pendingParents.get(key) ?? (0 as EncodedKey);
     let ref: Ref | null = parentId === 0
       ? this.target
       : this.active.get(parentId) ?? null;
 @@ -1311,13 +1323,14 @@ class Decoder<
         UNWRAP(this.pendingStart.get(parentId)),
       );
     }
-    const { text, estimatedTime, ...options } = opts;
+    const { text, estimatedTime, formattedValue, ...options } = opts;
     const node = ref.start(text, {
       ...options,
       estimateCompletion: false,
     });
     this.active.set(key, node);
     node.estimatedTime = estimatedTime;
+    if (formattedValue) node.valueFormatter = () => formattedValue;
     this.pendingParents.delete(key);
     return node;
   }
 @@ -1327,6 +1340,9 @@ interface PendingStart {
   text: string;
   value?: number;
   total?: number;
+  showTotal?: boolean;
+  passive?: boolean;
+  hidden?: boolean;
   messages?: log.Message[];
   estimatedTime?: number | null;
   formattedValue?: string;
 @@ -1504,21 +1520,27 @@ export function decodeByteStream<
   target: Ref,
 ): async.Cancelable<Result> & Events<EventMap> {
   let cancelled = false;
+  let reader: stream.BufferedReader | null = null;
   return decodeEventStream<Result, EventMap>(
     new ReadableStream<StreamEvent>({
       async start(controller) {
-        using reader = new stream.BufferedReader(encoded.getReader());
+        reader = new stream.BufferedReader(encoded.getReader());
         try {
-          while (true) controller.enqueue(await readStreamEvent(reader));
+          while (reader) controller.enqueue(await readStreamEvent(reader));
         } catch (e) {
           if (!cancelled) {
             reader.cancel(e);
             throw e;
           }
+        } finally {
+          reader?.releaseLock();
+          reader = null;
         }
       },
       cancel(reason) {
         cancelled = true;
+        reader?.releaseLock();
+        reader = null;
         encoded.cancel(reason);
       },
     }),
 @@ -1560,20 +1582,23 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
     if (flags.changedLogs) {
       l = [];
       let len = await r.varUint();
-      while (len > 0) {
+      while (len-- > 0) {
         const msgFlags = await r.u8();
         const level = logLevelSerialize[msgFlags & 0b1111] ?? "info";
-        const hasScope = msgFlags && (1 << 5) > 0;
+        // bitwise tests: `msgFlags && (1 << 5) > 0` parsed as
-        const hasStack = msgFlags && (1 << 6) > 0;
+        // `msgFlags && true`, which fabricated scope/stack/custom reads for
-        const hasCustom = msgFlags && (1 << 7) > 0;
+        // any message with a non-zero level and desynced the entire stream
+        const hasScope = (msgFlags & (1 << 5)) !== 0;
+        const hasStack = (msgFlags & (1 << 6)) !== 0;
+        const hasCustom = (msgFlags & (1 << 7)) !== 0;
         const text = await r.stringWithLength();
         const time = await r.varUint();
         const scope = hasScope ? await r.stringWithLength() : undefined;
         let stack: stack.Frame[] | undefined = undefined;
         if (hasStack) {
           stack = [];
-          let len = await r.varUint();
+          let frames = await r.varUint();
-          while (len > 0) {
+          while (frames-- > 0) {
             const fn = await r.stringWithLength();
             const file = await r.stringWithLength();
             const line = await r.varUint();
 @@ -1756,7 +1781,7 @@ const global: Ref = /** @__PURE__ */ ((root = new Root()) => (attachToScreen(roo
  * a {@linkcode Ref} to the global progress root. unlike referencing the
  * namespace import, this value is tree-shakable.
  */
-export const globalRoot: Ref = { start: global.start };
+export const globalRoot: Ref = { start: (text, opts) => global.start(text, opts) };
 /**
  * for testing. not covered by semver

lib/queue.ts+8-1

...	@@ -176,4 +176,11 @@ function insertSorted<T extends { priority: number }>(arr: T[], item: T) {	...	@@ -176,4 +176,11 @@ function insertSorted<T extends { priority: number }>(arr: T[], item: T) {
176	declare var navigator: { hardwareConcurrency: number };	176	declare var navigator: { hardwareConcurrency: number };
177	const global = new PriorityQueue(navigator.hardwareConcurrency);	177	const global = new PriorityQueue(navigator.hardwareConcurrency);
178		178
179	import { UNWRAP } from "./assert.ts";	179	/** adjust the global queue's concurrency */
		180	export function setConcurrency(n: number) {
		181	ASSERT(Number.isInteger(n) && n > 0, `setConcurrency(${n})`);
		182	global.coresRemain += n - global.concurrency;
		183	global.concurrency = n;
		184	}
		185
		186	import { ASSERT, UNWRAP } from "./assert.ts";

lib/subprocess/ffmpeg.ts+3

...	@@ -11,6 +11,8 @@ export interface SpawnOptions {	...	@@ -11,6 +11,8 @@ export interface SpawnOptions {
11	ffmpeg?: string;	11	ffmpeg?: string;
12	progress: progress.Node;	12	progress: progress.Node;
13	cwd?: string; // TODO: Path	13	cwd?: string; // TODO: Path
		14	/** kills the process when aborted */
		15	signal?: AbortSignal;
14	}	16	}
15		17
16	/**	18	/**
...	@@ -34,6 +36,7 @@ export async function spawn(options: SpawnOptions) {	...	@@ -34,6 +36,7 @@ export async function spawn(options: SpawnOptions) {
34	stdio: ["ignore", "inherit", "pipe"],	36	stdio: ["ignore", "inherit", "pipe"],
35	env: { ...process.env, SVT_LOG: "2" },	37	env: { ...process.env, SVT_LOG: "2" },
36	cwd: cwd?.toString(),	38	cwd: cwd?.toString(),
		39	signal: options.signal,
37	});	40	});
38	const parser = new Parse();	41	const parser = new Parse();
39	let running = true;	42	let running = true;

run.js+1

...	@@ -81,6 +81,7 @@ console["log"] = log.log;	...	@@ -81,6 +81,7 @@ console["log"] = log.log;
81	process.on("uncaughtException", (error) => {	81	process.on("uncaughtException", (error) => {
82	console.error("Uncaught Exception");	82	console.error("Uncaught Exception");
83	console.error(error);	83	console.error(error);
		84	log.getDrawLock("long");
84	process.exit(1);	85	process.exit(1);
85	});	86	});
86		87

src/backend.ts+1

...	@@ -65,3 +65,4 @@ import { type Context, Hono, type Next } from "hono";	...	@@ -65,3 +65,4 @@ import { type Context, Hono, type Next } from "hono";
65	import { logger } from "hono/logger";	65	import { logger } from "hono/logger";
66	import { trimTrailingSlash } from "hono/trailing-slash";	66	import { trimTrailingSlash } from "hono/trailing-slash";
67	import * as admin from "./admin.ts";	67	import * as admin from "./admin.ts";
		68	import "./file-viewer/sync.ts";

src/bin/download-db.ts+5-10

 @@ -1,14 +1,8 @@
+// Pull production state for local development. cache.sqlite comes from the
+// source of truth's /db route (no ssh needed); questions.sqlite is still
+// rsync'd off the web node since the source of truth does not own it yet.
 export async function main() {
-  await rsync.spawn({
+  await sync.revalidate();
-    cwd: Path.resolve("."),
-    args: [
-      "-a",
-      "--progress",
-      "clo@paperclover.net:~/paperclover.net/.clover/cache.sqlite",
-      ".clover/cache.sqlite",
-    ],
-    progress: progress.start("download file cache"),
-  });
   await rsync.spawn({
     cwd: Path.resolve("."),
     args: [
 @@ -24,3 +18,4 @@ export async function main() {
 import { Path } from "#sitegen/path";
 import * as progress from "@clo/lib/progress";
 import * as rsync from "../file-viewer/rsync.ts";
+import * as sync from "../file-viewer/sync.ts";

src/bin/tail-progress.ts created+50

...	@@ -0,0 +1,50 @@
1	// Tail the source of truth's indexing progress from anywhere:
2	//
3	// node run tail-progress # production
4	// node run tail-progress http://zenith:43201/progress # staging
5	//
6	// Opens a fetch to the /progress route and renders the live progress tree
7	// in this terminal. Reconnects automatically; each (re)connect resumes from
8	// the server's current state, since the stream encoder snapshots on attach.
9	const defaultUrl = "https://db.paperclover.net/progress";
10
11	export async function main() {
12	const url = process.argv[2] ?? defaultUrl;
13	const token = process.env.CLOVER_SOT_KEY;
14	if (!token) {
15	console.warn("CLOVER_SOT_KEY is not set; the server will likely 401");
16	}
17
18	let delay = 1000;
19	while (true) {
20	const connectedAt = Date.now();
21	try {
22	const res = await fetch(url, {
23	headers: token ? { Authorization: token } : {},
24	});
25	if (res.status === 401) {
26	console.error("unauthorized; set CLOVER_SOT_KEY");
27	process.exit(1);
28	}
29	if (!res.ok \|\| !res.body) {
30	throw new Error(`server responded ${res.status} ${res.statusText}`);
31	}
32	console.info(`connected to ${url}`);
33	// resolves only when the stream ends; the indexer root never ends, so
34	// this await effectively lasts until disconnect.
35	await progress.decodeByteStream(
36	res.body as ReadableStream<Uint8Array>,
37	progress.globalRoot,
38	);
39	} catch (err: any) {
40	console.warn(`disconnected: ${err?.message ?? err}`);
41	}
42	if (Date.now() - connectedAt > 15_000) delay = 1000;
43	else delay = Math.min(delay * 2, 30_000);
44	console.info(`reconnecting in ${Math.round(delay / 1000)}s...`);
45	await async.delay(delay);
46	}
47	}
48
49	import * as async from "@clo/lib/async";
50	import * as progress from "@clo/lib/progress";

src/file-viewer/backend.ts+3-1

 @@ -83,7 +83,9 @@ app.get("/file/*", async (c, next) => {
   // - Old browsers like Internet Explorer act as `?view=dl`
   let viewMode = c.req.query("view");
   if (c.req.query("dl") != null) viewMode = "download";
-  if (lofi) viewMode = file.extension === ".html" ? "embed" : "download";
+  if (lofi) {
+    viewMode = file.extension.toLowerCase() === ".html" ? "embed" : "download";
+  }
   if (
     viewMode == null && !derivedKey

src/file-viewer/bin/file-scan.ts+36-999

...	@@ -1,408 +1,32 @@	...	@@ -1,408 +1,32 @@
1	// The file scanner incrementally updates an sqlite database with file	1	// Run one full scan of the file store from this machine, rendering progress
2	// stats. Additionally, it runs "processors" on files, which precompute	2	// in the terminal. In production the indexer service inside the source of
3	// expensive data such as running `ffprobe` on all media to get the	3	// truth server does this continuously; this wrapper exists for development
4	// duration.	4	// (and the nostalgia of watching it go).
5	//	5	//
6	// Processors are also used to derive compressed and optimized assets,	6	// CLOVER_FILE_RAW=/tmp/store CLOVER_FILE_DERIVED=/tmp/derived \
7	// which is how automatic JXL / AV1 encoding is done. Derived files are	7	// CLOVER_DB=.clover node run file-scan
8	// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9	const sotToken = process.env.CLOVER_SOT_KEY;
10
11	export async function main() {	8	export async function main() {
12	const start = performance.now();	9	const start = performance.now();
13	using _ = log.startWidget({	10	using _ = log.startWidget({
14	format: ({ now }) => `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,	11	format: ({ now }) => `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
15	});	12	});
16		13
17	const promises = new async.PromiseAggregator();	14	const scanner = new Scanner({
18		15	root: Path.resolve(rawFileRoot),
19	const walkQueue = new queue.PriorityQueue(10);	16	progress: progress.globalRoot,
20		17	settleMs: Number(process.env.CLOVER_SETTLE_MS ?? 10_000),
21	const dirsNode = progress.start("Walk Tree", { total: 1 });
22	dirsNode.passive = true;
23	const fileNode = progress.start("Process File", { total: 0 });
24	fileNode.sortChildren = (a, b) => {
25	const ac = a.children.length > 0 ? 1 : 0;
26	const bc = b.children.length > 0 ? 1 : 0;
27	if (ac !== bc) return bc - ac;
28	return a.text.localeCompare(b.text);
29	};
30
31	// Read a directory or file stat and queue up changed files.
32	const scanDirectory = walkQueue.wrap(
33	async (path: Path) => {
34	const publicPath = toPublicPath(path);
35	using node = dirsNode.start(publicPath + " - stat");
36	using _ = ts.defer(() => dirsNode.inc());
37
38	const stat = await path.stat();
39
40	const mediaFile = MediaFile.getByPath(publicPath);
41
42	if (stat.isDirectory()) {
43	node.text = publicPath + " - reading";
44	const items = (await path.readDir())
45	.filter((child) => !skipBasename(child.base))
46	.map((child) => (promises.push(scanDirectory(child)), child.base));
47	dirsNode.total += items.length;
48
49	for (const child of mediaFile?.getChildren() ?? []) {
50	if (items.includes(child.basename)) continue;
51	const recursive = child.kind === MediaFileKind.directory
52	? [child, ...child.getRecursiveFileChildren()]
53	: [child];
54	for (const deletion of recursive) deletion.delete();
55	}
56
57	return;
58	}
59
60	if (
61	!mediaFile // All processes must be performed if there is no file.
62	// Rerun all processors if it changed
63	\|\| stat.size !== mediaFile.size
64	\|\| stat.mtime.getTime() !== mediaFile.date.getTime()
65	) {
66	promises.push(updateMetadata({ path, publicPath, stat, mediaFile }));
67	} else {
68	// If the scanners changed, it may mean more processes should be run.
69	await queueProcessors({
70	path,
71	stat,
72	mediaFile,
73	node: fileNode.start(publicPath.slice(1)),
74	});
75	}
76	},
77	);
78	const updateMetadata = queue.wrap(
79	async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
80	using errorHandler = new DisposableStack();
81	const label = publicPath.slice(1);
82	const node = errorHandler.use(fileNode.start(label));
83
84	await scrubLocationMetadata(path, stat, node);
85
86	node.text = `${label} - hashing`;
87	const hash = await new Promise<string>((resolve, reject) => {
88	const reader = fs.createReadStream(path.toString());
89	reader.on("error", reject);
90
91	const hasher = crypto.createHash("sha1").setEncoding("hex");
92	hasher.on("error", reject);
93	hasher.on("readable", () => resolve(hasher.read()));
94
95	reader.pipe(hasher);
96	});
97	let date = stat.mtime;
98	if (
99	mediaFile
100	&& mediaFile.date.getTime() < stat.mtime.getTime()
101	&& Date.now() - stat.mtime.getTime() < monthMilliseconds
102	) {
103	date = mediaFile.date;
104	console.warn(
105	`M-time on ${publicPath} was likely corrupted. ${
106	formatDate(
107	mediaFile.date,
108	)
109	} -> ${formatDate(stat.mtime)}`,
110	);
111	}
112	mediaFile = MediaFile.createFile({
113	path: publicPath,
114	date,
115	hash,
116	size: stat.size,
117	duration: mediaFile?.duration ?? 0,
118	dimensions: mediaFile?.dimensions ?? "",
119	contents: mediaFile?.contents ?? "",
120	});
121	const parent = mediaFile.getParent();
122	if (parent) parent.setProcessed(0);
123
124	node.text = `${label}`;
125	await queueProcessors({
126	path,
127	stat,
128	mediaFile,
129	node,
130	});
131	errorHandler.move();
132	},
133	() => ({ priority: -1 }),
134	);
135	const queueFileProcessor = queue.wrap(async function({
136	path,
137	stat,
138	mediaFile,
139	processor,
140	index,
141	after,
142	fileNode: innerNode,
143	}: ProcessJob) {
144	using node = innerNode.start(processor.name);
145	await processor.run({
146	path,
147	stat,
148	mediaFile,
149	node,
150	});
151	fileNode.value += 1;
152	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
153	for (const dependantJob of after) {
154	ASSERT(
155	dependantJob.needs > 0,
156	`dependantJob.needs > 0, ${dependantJob.needs}`,
157	);
158	dependantJob.needs -= 1;
159	if (dependantJob.needs == 0) {
160	promises.push(queueFileProcessor(dependantJob));
161	}
162	}
163	}, (job) => ({ cores: job.processor.cores ?? 0 }));
164
165	function decodeProcessors(input: string) {
166	return input
167	.split(";")
168	.filter(Boolean)
169	.map(([a, b, c]) => ({
170	id: a,
171	hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
172	}));
173	}
174
175	async function queueProcessors({
176	path,
177	stat,
178	mediaFile,
179	node,
180	}: Omit<ProcessFileArgs, "spin">) {
181	using errorHandler = new DisposableStack();
182	errorHandler.use(node);
183
184	node.showTotal = false;
185	node.passive = true;
186
187	const ext = mediaFile.extensionNonEmpty.toLowerCase();
188	let possible = processors.filter((p) => p.include ? p.include.has(ext) : !p.exclude?.has(ext));
189	if (possible.length === 0) return;
190	ASSERT(possible.length < 16, "too many bits");
191
192	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
193	ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
194	let processed = mediaFile.processed;
195
196	// If the hash has changed, migrate the bitfield over.
197	// This also runs when the processor hash is in it's initial 0 state.
198	let order: ReturnType<typeof decodeProcessors>;
199	try {
200	order = decodeProcessors(mediaFile.processors);
201	} catch {
202	// this function sucks and this system sucks i hate it.
203	order = [];
204	}
205	if ((processed & 0xffff) !== hash) {
206	const previous = order.filter(
207	(_, i) => (processed & (1 << (16 + i))) !== 0,
208	);
209	processed = hash;
210	for (const { id, hash } of previous) {
211	const p = processors.find((p) => p.id === id);
212	if (!p) continue;
213	const index = possible.indexOf(p);
214	if (index !== -1 && p.hash === hash) processed \|= 1 << (16 + index);
215	}
216	mediaFile.setProcessors(
217	processed,
218	possible
219	.map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
220	.join(";"),
221	);
222	} else {
223	possible = order.map(({ id }) => UNWRAP(possible.find((p) => p.id === id)));
224	}
225
226	// Queue needed processors.
227	const jobs: ProcessJob[] = [];
228	for (let i = 0, { length } = possible; i < length; i += 1) {
229	if ((processed & (1 << (16 + i))) === 0) {
230	const processor = UNWRAP(possible[i]);
231	const job: ProcessJob = {
232	path,
233	stat,
234	mediaFile,
235	processor,
236	index: i,
237	after: [],
238	needs: processor.depends.length,
239	fileNode: node,
240	};
241	jobs.push(job);
242	if (job.needs === 0) promises.push(queueFileProcessor(job));
243	}
244	}
245	node.total = jobs.length;
246	fileNode.total += jobs.length;
247	for (const job of jobs) {
248	for (const dependId of job.processor.depends) {
249	const dependJob = jobs.find((j) => j.processor.id === dependId);
250	if (dependJob) {
251	dependJob.after.push(job);
252	} else {
253	ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
254	job.needs -= 1;
255	if (job.needs === 0) promises.push(queueFileProcessor(job));
256	}
257	}
258	}
259	if (node.total > 0) {
260	errorHandler.move();
261
262	const parent = mediaFile.getParent();
263	if (parent) parent.setProcessed(0);
264	}
265	}
266
267	// Add the root & recursively iterate!
268	const rootPath = Path.resolve(root);
269	if (!rootPath.ifExistsSync()) {
270	throw new Error(`file store ${rootPath} is not mounted`);
271	}
272	promises.push(scanDirectory(rootPath));
273
274	await promises.all();
275	fileNode.end();
276	dirsNode.end();
277
278	// Update directory metadata
279	using dirMetaNode = progress.start("update directory metadata");
280	const dirs = MediaFile.getDirectoriesToReindex()
281	.sort((a, b) => b.path.length - a.path.length);
282
283	for (const dir of dirs) {
284	using _ = dirMetaNode.start(dir.path);
285	const children = dir.getChildren();
286
287	// readme.txt
288	const readmeContent = children.find((x) => x.basename === "readme.txt")?.contents ?? "";
289
290	// dirsort
291	let dirsort: string[] \| null = null;
292	const dirSortRaw = children.find((x) => x.basename === ".dirsort")?.contents ?? "";
293	if (dirSortRaw) {
294	dirsort = dirSortRaw
295	.split("\n")
296	.map((x) => x.trim())
297	.filter(Boolean);
298	}
299
300	// Permissions
301	if (children.some((x) => x.basename === ".friends")) {
302	FilePermissions.setPermissions(dir.path, 1);
303	} else {
304	FilePermissions.setPermissions(dir.path, 0);
305	}
306
307	// Recursive stats.
308	let totalSize = 0;
309	let newestDate = new Date(0);
310	let allHashes = "";
311	for (const child of children) {
312	totalSize += child.size;
313	allHashes += child.hash;
314
315	if (child.basename !== "/readme.txt" && child.date > newestDate) {
316	newestDate = child.date;
317	}
318	}
319
320	// Project Date
321	const dateFile = children.find((x) => x.basename === ".date");
322	if (dateFile) {
323	const date = new Date(dateFile.contents);
324	newestDate = date;
325	dir.setProcessors(
326	0,
327	JSON.stringify({ hideChildrenDates: true }),
328	);
329	} else {
330	dir.setProcessors(0, "");
331	}
332
333	const dirHash = crypto
334	.createHash("sha1")
335	.update(dir.path + allHashes)
336	.digest("hex");
337
338	MediaFile.markDirectoryProcessed({
339	id: dir.id,
340	timestamp: newestDate,
341	contents: readmeContent,
342	size: totalSize,
343	hash: dirHash,
344	dirsort,
345	});
346	}
347	dirMetaNode.end();
348
349	// Sync to remote
350	if (
351	((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
352	) {
353	await rsync.spawn({
354	args: [
355	"--links",
356	"--recursive",
357	"--times",
358	"--partial",
359	"--progress",
360	// "--remove-source-files",
361	"--delay-updates",
362	"--exclude=tmp.*",
363	derived.workDir.toString() + "/",
364	"clo@file.paperclover.net:/mnt/storage1/clover/Documents/Config/paperclover/derived/",
365	],
366	progress: progress.start("upload derived assets"),
367	cwd: process.cwd(),
368	});
369
370	await fs.removeEmptyDirectories(derived.workDir.toString());
371	} else {
372	console.info("No new derived assets");
373	}
374
375	MediaFile.db.prepare("VACUUM").run();
376	MediaFile.db.reload();
377
378	await rsync.spawn({
379	args: [
380	MediaFile.db.file,
381	"clo@file.paperclover.net:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
382	],
383	progress: progress.start("Uploading database (source of truth)"),
384	cwd: process.cwd(),
385	});	18	});
386	await rsync.spawn({	19	await scanner.sweep();
387	args: [	20	await scanner.waitIdle();
388	MediaFile.db.file,	21	dirmeta.run(progress.globalRoot);
389	"clo@paperclover.net:paperclover.net/.clover/cache.sqlite",	22
390	],	23	const orphaned = derived.findOrphanedRoots();
391	progress: progress.start("Uploading database (web node)"),	24	for (const orphan of orphaned) {
392	});	25	console.info("delete orphaned " + orphan.key);
393	if (sotToken) {	26	await derived.deleteRootFiles(orphan);
394	const res = await fetch("https://db.paperclover.net/reload", {	27	derived.deleteRoot(orphan);
395	method: "POST",	28	}
396	headers: {	29	await derived.cleanAbandonedTmp();
397	Authorization: sotToken,
398	},
399	});
400	if (!res.ok) {
401	console.warn(
402	`Failed to reload remote database ${res.status} ${res.statusText}`,
403	);
404	}
405	} else console.warn("Missing SOT token");
406		30
407	console.info(	31	console.info(
408	"Updated file viewer index in \x1b[1m"	32	"Updated file viewer index in \x1b[1m"
...	@@ -426,6 +50,11 @@ export async function main() {	...	@@ -426,6 +50,11 @@ export async function main() {
426	from derived_files	50	from derived_files
427	`)	51	`)
428	.getNonNull();	52	.getNonNull();
		53	const { failed } = MediaFile.db
		54	.prepare<[], { failed: number }>(`
		55	select count(*) as failed from file_processors where status = 2
		56	`)
		57	.getNonNull();
429	const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;	58	const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
430		59
431	console.info();	60	console.info();
...	@@ -435,614 +64,22 @@ export async function main() {	...	@@ -435,614 +64,22 @@ export async function main() {
435	+ `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n`	64	+ `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n`
436	+ `- Media Duration: \x1b[1m${string.formatDurationLetters(duration)}\x1b[0m\n`	65	+ `- Media Duration: \x1b[1m${string.formatDurationLetters(duration)}\x1b[0m\n`
437	+ `- Canonical Size: \x1b[1m${string.formatByteSize(canonicalSize)}\x1b[0m\n`	66	+ `- Canonical Size: \x1b[1m${string.formatByteSize(canonicalSize)}\x1b[0m\n`
438	+ `- Derived Size: \x1b[1m${string.formatByteSize(derivedSize)}\x1b[0m\n`,	67	+ `- Derived Size: \x1b[1m${string.formatByteSize(derivedSize)}\x1b[0m\n`
439	);	68	+ (failed > 0
440	}	69	? `- \x1b[31mFailed Processors: ${failed}\x1b[0m (select * from file_processors where status = 2)\n`
441		70	: ""),
442	interface Process {
443	name: string;
444	cores?: number;
445	enable?: boolean;
446	include?: Set<string>;
447	exclude?: Set<string>;
448	depends?: string[];
449	version?: number;
450	/* Perform an action. */
451	run(args: ProcessFileArgs): Promise<void>;
452	}
453
454	const ffprobeBin = testProgram("ffprobe", "--help");
455	const ffmpegBin = testProgram("ffmpeg", "--help");
456
457	const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
458
459	// NOTE: Never re-order the processors. Add new ones at the end.
460	const procDuration: Process = {
461	name: "calculate duration",
462	enable: ffprobeBin !== null,
463	include: rules.extsDuration,
464	cores: 1,
465	async run({ path, mediaFile }) {
466	const { stdout } = await subprocess.exec(ffprobeBin!, [
467	"-v",
468	"error",
469	"-show_entries",
470	"format=duration",
471	"-of",
472	"default=noprint_wrappers=1:nokey=1",
473	path.toString(),
474	]);
475
476	const duration = parseFloat(stdout.trim());
477	if (Number.isNaN(duration)) {
478	throw new Error("Could not extract duration from " + stdout);
479	}
480	mediaFile.setDuration(Math.ceil(duration));
481	},
482	};
483
484	const procDimensions: Process = {
485	name: "calculate dimensions",
486	enable: ffprobeBin != null,
487	include: rules.extsDimensions,
488	cores: 1,
489	async run({ path, mediaFile }) {
490	const { ext } = path;
491
492	let dimensions;
493
494	if (ext === ".svg") {
495	// Parse out of text data
496	const content = await path.read("utf-8");
497	const widthMatch = content.match(/width="(\d+)"/);
498	const heightMatch = content.match(/height="(\d+)"/);
499
500	if (widthMatch && heightMatch) {
501	dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
502	}
503	} else if (rules.extsImage.has(ext)) {
504	// Use magick to observe streams
505	const { stdout } = await subprocess.exec("magick", [
506	"identify",
507	"-auto-orient",
508	"-format",
509	"%w %h",
510	path.toString(),
511	]);
512	const [w, h] = stdout.split(" ").map((x) => Number(x));
513	if (w && h) {
514	dimensions = w + "x" + h;
515	}
516	} else {
517	// Use ffprobe to observe streams
518	const { stdout } = await subprocess.exec("ffprobe", [
519	"-v",
520	"error",
521	"-select_streams",
522	"v:0",
523	"-show_entries",
524	"stream=width,height",
525	"-of",
526	"json",
527	path.toString(),
528	]);
529	const result = JSON.parse(stdout);
530	const stream = result.streams[0];
531	if (stream) {
532	dimensions = UNWRAP(stream.width) + "x" + UNWRAP(stream.height);
533	}
534	}
535
536	mediaFile.setDimensions(dimensions ?? "");
537	},
538	};
539
540	const procLoadTextContents: Process = {
541	name: "load text content",
542	include: rules.extsReadContents,
543	cores: 1,
544	version: 2,
545	async run({ path, mediaFile, stat }) {
546	if (stat.size > 1_000_000) return;
547	const text = await path.read("utf-8");
548	mediaFile.setContents(text);
549	},
550	};
551
552	const procHighlightCode: Process = {
553	name: "highlight source code",
554	include: new Set(rules.extsCode.keys()),
555	cores: 1,
556	version: 2,
557	async run({ path, mediaFile, stat }) {
558	const language = UNWRAP(
559	rules.extsCode.get(path.ext.toLowerCase()),
560	);
561	// An issue is that .ts is an overloaded extension, shared between
562	// 'transport stream' and 'typescript'.
563	//
564	// Filter used here is:
565	// - more than 1mb
566	// - invalid UTF-8
567	if (stat.size > 1_000_000) return;
568	let code;
569	const buf = await path.read();
570	try {
571	code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
572	} catch (error) {
573	mediaFile.setContents("");
574	return;
575	}
576	const content = await highlight.highlightCode(code, language);
577	mediaFile.setContents(content);
578	},
579	};
580
581	const procImageSubsets: Process = {
582	name: "encode image subsets",
583	include: rules.extsImage,
584	depends: [procDimensions.name],
585	version: 3,
586	async run({ path, mediaFile, node }) {
587	const { width, height } = UNWRAP(mediaFile.parseDimensions());
588	const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
589	const baseStatus = node.text;
590
591	using stack = new DisposableStack();
592	for (const size of targetSizes) {
593	const { w, h } = resizeDimensions(width, height, size);
594	for (const { ext, args } of transcodeRules.imagePresets) {
595	node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
596
597	stack.use(
598	await derived.produce({
599	mediaFile,
600	node,
601	cores: 2,
602	subkey: `${size}${ext}`,
603	async producer(dir) {
604	await subprocess.exec(ffmpegBin!, [
605	...ffmpegOptions,
606	"-i",
607	path.toString(),
608	"-vf",
609	`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
610	...args,
611	dir.join(`${size}${ext}`).toString(),
612	]);
613	},
614	}),
615	);
616	}
617	}
618
619	stack.move();
620	},
621	};
622
623	const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
624
625	const qualityMap: Record<string, string> = {
626	u: "ultra-high",
627	h: "high",
628	m: "medium",
629	l: "low",
630	d: "data-saving",
631	};
632	const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
633	name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
634	include: rules.extsVideo,
635	enable: ffmpegBin != null,
636	depends: [procDuration.name, procDimensions.name],
637	version: 4,
638	async run({ path, mediaFile, node }) {
639	if ((mediaFile.duration ?? 0) < 5) return;
640	if (!mediaFile.dimensions) return;
641
642	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
643
644	await derived.produce({
645	mediaFile,
646	node,
647	cores: 4,
648	subkey: `av1-${preset.id}`,
649	async producer(dir) {
650	const input = await videoArgsCache.getOrRun(path);
651	const dimensions = mediaFile.parseDimensions();
652	ASSERT(dimensions);
653	const args = transcodeRules.getAv1VideoArgs(
654	preset,
655	dimensions,
656	UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
657	dir,
658	);
659
660	await ffmpeg.spawn({
661	ffmpeg: ffmpegBin!,
662	progress: node,
663	args,
664	cwd: dir.toString(),
665	});
666	},
667	});
668	},
669	}));
670	const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
671	name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
672	include: rules.extsVideo,
673	enable: ffmpegBin != null,
674	depends: [procDuration.name, procDimensions.name],
675	version: 3,
676	async run({ path, mediaFile, node }) {
677	if ((mediaFile.duration ?? 0) < 5) return;
678	if (!mediaFile.dimensions) return;
679	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
680	await derived.produce({
681	mediaFile,
682	node,
683	cores: 4,
684	subkey: `opus-${preset.id}`,
685	async producer(dir) {
686	const input = await videoArgsCache.getOrRun(path);
687	ASSERT(input.video);
688	if (!input.audio) return;
689	const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
690
691	await ffmpeg.spawn({
692	ffmpeg: ffmpegBin!,
693	progress: node,
694	args,
695	cwd: dir.toString(),
696	});
697	},
698	});
699	},
700	}));
701	const procDash: Process = {
702	name: `encode mpeg-dash`,
703	include: rules.extsVideo,
704	enable: ffmpegBin != null,
705	depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
706	version: 7,
707	async run({ path, mediaFile, node }) {
708	if ((mediaFile.duration ?? 0) < 5) return;
709	if (!mediaFile.dimensions) return;
710
711	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
712	await derived.produce({
713	mediaFile,
714	node,
715	cores: 1,
716	subkey: `dash-av1`,
717	async producer(dir) {
718	const input = await videoArgsCache.getOrRun(path);
719	const videos = transcodeRules.videoFormats.map(
720	(preset) =>
721	UNWRAP(dir.parent).join(
722	`av1-${preset.id}`,
723	transcodeRules.av1FileName,
724	),
725	);
726	const audios = input.audio
727	? transcodeRules.audioFormats.map(
728	(preset) =>
729	UNWRAP(dir.parent).join(
730	`opus-${preset.id}`,
731	transcodeRules.opusFileName,
732	),
733	)
734	: [];
735	const args = transcodeRules.getMpegDashArgs(
736	videos,
737	audios,
738	dir,
739	);
740
741	await dir.join("d").makeDir();
742
743	await ffmpeg.spawn({
744	ffmpeg: ffmpegBin!,
745	progress: node,
746	args,
747	cwd: dir.toString(),
748	});
749	},
750	});
751	},
752	};
753	const procH264Hls: Process = {
754	name: `encode h.264 hls`,
755	include: rules.extsVideo,
756	enable: ffmpegBin != null,
757	depends: [procDuration.name, procDimensions.name],
758	async run({ path, mediaFile, node }) {
759	if ((mediaFile.duration ?? 0) < 5) return;
760	if (!mediaFile.dimensions) return;
761
762	await derived.produce({
763	mediaFile,
764	node,
765	cores: 8,
766	subkey: `hls`,
767	async producer(dir) {
768	const input = await videoArgsCache.getOrRun(path);
769	const args = transcodeRules.getH264HlsArgs(input, dir);
770
771	await ffmpeg.spawn({
772	ffmpeg: ffmpegBin!,
773	progress: node,
774	args,
775	cwd: dir.toString(),
776	});
777	},
778	});
779	},
780	};
781
782	const procCompression = [
783	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
784	{ name: "zstd", fn: () => zlib.createZstdCompress() },
785	].map<Process>(
786	({ name, fn }) => ({
787	name: `compress ${name}`,
788	exclude: rules.extsPreCompressed,
789	async run({ path, mediaFile, node }) {
790	await derived.produce({
791	mediaFile,
792	node,
793	cores: 1,
794	subkey: name,
795	async producer(dir) {
796	await stream.promises.pipeline(
797	fs.createReadStream(path.toString()),
798	fn(),
799	fs.createWriteStream(dir.join(name).toString()),
800	);
801	},
802	});
803	},
804	}),
805	);
806
807	const processors = [
808	procDimensions,
809	procDuration,
810	procLoadTextContents,
811	procHighlightCode,
812	procImageSubsets,
813	...procVideos,
814	...procVideoAudios,
815	procDash,
816	...procCompression,
817	procH264Hls,
818	].map((process, id, all) => {
819	const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
820	return {
821	...(process as Process),
822	id: strIndex(id),
823	// Create a unique key.
824	hash: new Uint16Array(
825	crypto
826	.createHash("sha1")
827	.update(
828	process.run.toString()
829	+ (process.version ? String(process.version) : ""),
830	)
831	.digest().buffer,
832	).reduce((a, b) => a ^ b),
833	depends: (process.depends ?? []).map((depend) => {
834	const index = all.findIndex((p) => p.name === depend);
835	if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
836	if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
837	return strIndex(index);
838	}),
839	};
840	});
841
842	function resizeDimensions(w: number, h: number, desiredWidth: number) {
843	ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
844	return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
845	}
846
847	interface UpdateMetadataArgs {
848	path: Path;
849	publicPath: string;
850	stat: fs.Stats;
851	mediaFile: MediaFile \| null;
852	}
853
854	interface ProcessFileArgs {
855	path: Path;
856	stat: fs.Stats;
857	mediaFile: MediaFile;
858	node: progress.Node;
859	}
860
861	interface ProcessJob {
862	path: Path;
863	stat: fs.Stats;
864	mediaFile: MediaFile;
865	processor: (typeof processors)[0];
866	index: number;
867	after: ProcessJob[];
868	needs: number;
869	fileNode: progress.Node;
870	}
871
872	function skipBasename(basename: string): boolean {
873	// dot files must be incrementally tracked
874	if (basename === ".dirsort") return false;
875	if (basename === ".friends") return false;
876	if (basename === ".date") return false;
877
878	return (
879	basename.startsWith(".")
880	// basename.startsWith("._") \|\|
881	// basename.startsWith(".tmp") \|\|
882	// basename === ".DS_Store" \|\|
883	\|\| basename.toLowerCase() === "thumbs.db"
884	\|\| basename.toLowerCase() === "desktop.ini"
885	);	71	);
886	}	72	}
887		73
888	function toPublicPath(diskPath: Path) {
889	if (diskPath.toString() === root) return "/";
890	return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
891	}
892
893	function testProgram(name: string, helpArgument: string) {
894	try {
895	child_process.spawnSync(name, [helpArgument]);
896	return name;
897	} catch (err) {
898	console.warn(`Missing or corrupt executable '${name}'`);
899	}
900	return null;
901	}
902
903	// Helper function to check and remove location metadata
904	async function scrubLocationMetadata(
905	path: Path,
906	stats: fs.Stats,
907	progress: progress.Ref,
908	): Promise<boolean> {
909	using _ = progress.start("scrub exif metadata");
910	const ext = path.ext.toLowerCase();
911	if (!rules.extsScrubExif.has(ext)) return false;
912
913	let hasLocation = false;
914	let args: string[] = [];
915
916	// Check for location metadata based on file type
917	const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
918	switch (ext) {
919	case ".jpg":
920	case ".jpeg":
921	case ".png":
922	const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
923	"-gps:all",
924	path.toString(),
925	]);
926	hasLocation = gpsCheck.trim().length > 0;
927	args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
928	break;
929	case ".mov":
930	case ".mp4":
931	const { stdout: videoCheck } = await subprocess.exec("exiftool", [
932	"-ee",
933	"-G3",
934	"-s",
935	path.toString(),
936	]);
937	hasLocation = videoCheck.includes("GPS")
938	\|\| videoCheck.includes("Location");
939	args = [
940	"-gps:all=",
941	"-xmp:all=",
942	path.toString(),
943	"-o",
944	tempOutput.toString(),
945	];
946	break;
947	case ".m4a":
948	const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
949	"-ee",
950	"-G3",
951	"-s",
952	path.toString(),
953	]);
954	hasLocation = m4aCheck.includes("GPS")
955	\|\| m4aCheck.includes("Location")
956	\|\| m4aCheck.includes("Filename")
957	\|\| m4aCheck.includes("Title");
958
959	if (hasLocation) {
960	args = [
961	"-gps:all=",
962	"-location:all=",
963	"-filename:all=",
964	"-title=",
965	"-m4a:all=",
966	path.toString(),
967	"-o",
968	tempOutput.toString(),
969	];
970	}
971	break;
972	}
973
974	const accessTime = stats.atime;
975	const modTime = stats.mtime;
976
977	let backup: Path \| null = null;
978	try {
979	if (hasLocation) {
980	// Prepare a backup
981	const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
982	await fsp.copyFile(path.toString(), tmp.toString());
983	await fsp.utimes(tmp.toString(), accessTime, modTime);
984	backup = tmp;
985
986	// Remove metadata
987	await subprocess.exec("exiftool", args);
988	if (!tempOutput.ifExistsSync()) {
989	throw new Error(`Failed to create output file: ${tempOutput}`);
990	}
991
992	// Restore original timestamps
993	await fsp.rename(tempOutput.toString(), path.toString());
994	await fsp.utimes(path.toString(), accessTime, modTime);
995
996	// Backup is no longer needed
997	await fsp.unlink(backup.toString());
998
999	console.info(
1000	`Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
1001	);
1002	return true;
1003	}
1004	} catch (error) {
1005	if (backup) {
1006	await fsp.rename(backup.toString(), path.toString());
1007	}
1008	if (fs.existsSync(tempOutput.toString())) {
1009	await fsp.unlink(tempOutput.toString());
1010	}
1011	throw error;
1012	}
1013
1014	return false;
1015	}
1016
1017	const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
1018
1019	import * as fs from "#sitegen/fs";
1020	import { Path } from "#sitegen/path";
1021
1022	import * as async from "@clo/lib/async";
1023	import * as log from "@clo/lib/log";	74	import * as log from "@clo/lib/log";
1024	import * as progress from "@clo/lib/progress";	75	import * as progress from "@clo/lib/progress";
1025	import * as queue from "@clo/lib/queue";
1026	import * as string from "@clo/lib/string";	76	import * as string from "@clo/lib/string";
1027	import * as subprocess from "@clo/lib/subprocess";
1028	import * as ts from "@clo/lib/ts";
1029		77
1030	import * as child_process from "node:child_process";	78	import { Path } from "#sitegen/path";
1031	import * as crypto from "node:crypto";	79	import { UNWRAP } from "@clo/lib/assert";
1032	import * as fsp from "node:fs/promises";
1033	import * as path from "node:path";
1034	import * as stream from "node:stream";
1035	import * as zlib from "node:zlib";
1036		80
1037	import { formatDate } from "#src/file-viewer/format.ts";	81	import * as dirmeta from "#src/file-viewer/indexer/dirmeta.ts";
1038	import * as highlight from "#src/file-viewer/highlight.ts";	82	import { Scanner } from "#src/file-viewer/indexer/scan.ts";
1039	import * as derived from "#src/file-viewer/models/derived.ts";	83	import * as derived from "#src/file-viewer/models/derived.ts";
1040	import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";	84	import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
1041	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";	85	import { rawFileRoot } from "../paths.ts";
1042	import * as rsync from "#src/file-viewer/rsync.ts";
1043	import * as rules from "#src/file-viewer/rules.ts";
1044	import * as transcodeRules from "#src/file-viewer/transcode-rules.ts";
1045	import * as ffmpeg from "@clo/lib/subprocess/ffmpeg";
1046
1047	import { ASSERT, UNWRAP } from "@clo/lib/assert";
1048	import { rawFileRoot as root } from "../paths.ts";

src/file-viewer/bin/file-trim.ts deleted-17

...	@@ -1,17 +0,0 @@
1	export async function main() {
2	const start = performance.now();
3	using _ = log.startWidget({
4	format: (now) => `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
5	});
6
7	const orphaned = derived.findOrphanedRoots();
8	for (const root of orphaned) {
9	console.info("delete " + root.key);
10	derived.deleteRoot(root);
11	}
12
13	// TODO: delete unreferenced files
14	}
15
16	import * as log from "@clo/lib/log";
17	import * as derived from "../models/derived.ts";

src/file-viewer/bin/migrate-db.ts created+331

...	@@ -0,0 +1,331 @@
1	// One-shot migration of cache.sqlite from the old bitfield processor
2	// tracking ("v1") to the processors/file_processors tables ("v2").
3	//
4	// CLOVER_DB=.clover node run migrate-db
5	//
6	// There is exactly one real copy of this database; migrate it once locally,
7	// verify the site works, then place the migrated file on all machines
8	// alongside the new code. The old `processed` column packed a 16-bit hash of
9	// the applicable processor set plus per-processor "ran" bits indexed into
10	// the `processors` string, whose entries were [letter id][2-char hash of
11	// the processor's source code]. Completions are carried over as
12	// done-at-current-version: the first sweep after migration must re-run
13	// NOTHING (re-encoding the entire store would take weeks). Force re-runs
14	// later by bumping a processor's version.
15	//
16	// This file intentionally avoids importing the models (their prepared
17	// statements require the new schema) and instead uses raw SQL. It imports
18	// the registry only for processor names, versions, and applicability.
19
20	// the old positional letter ids, frozen. a..q matched the old array order.
21	const letterMap: Record<string, string> = {
22	a: "dimensions",
23	b: "duration",
24	c: "text-contents",
25	d: "highlight-code",
26	e: "image-subsets",
27	f: "av1-au",
28	g: "av1-ah",
29	h: "av1-am",
30	i: "av1-al",
31	j: "av1-ad",
32	k: "opus-h",
33	l: "opus-m",
34	m: "opus-d",
35	n: "dash",
36	o: "gzip",
37	p: "zstd",
38	q: "h264-hls",
39	};
40
41	export async function main() {
42	const db = getDb("cache.sqlite");
43	const raw = db.node;
44	console.info(`migrating ${db.file}`);
45
46	// -- preflight --
47	const columns = raw.prepare(`pragma table_info(media_files)`).all() as {
48	name: string;
49	}[];
50	if (columns.length === 0) {
51	console.error("media_files does not exist; nothing to migrate");
52	process.exit(1);
53	}
54	if (!columns.some((c) => c.name === "processed")) {
55	console.info("already migrated (no `processed` column); nothing to do");
56	return;
57	}
58
59	// -- backup --
60	raw.exec(`pragma wal_checkpoint(truncate);`);
61	const backup = db.file + ".pre-v2";
62	if (fs.existsSync(backup) && !process.argv.includes("--force")) {
63	console.error(`backup ${backup} already exists; pass --force to continue`);
64	process.exit(1);
65	}
66	// content-only copy: fs.copyFile's metadata preservation gets EPERM'd on
67	// the NAS datasets (restrictive ACL mode), plain writes do not.
68	await stream.promises.pipeline(
69	fs.createReadStream(db.file),
70	fs.createWriteStream(backup),
71	);
72	console.info(`backed up to ${backup}`);
73
74	const now = Date.now();
75	const summary = new Map<string, { done: number; pending: number }>();
76	for (const p of registry.processors) {
77	summary.set(p.name, { done: 0, pending: 0 });
78	}
79
80	raw.exec(`pragma foreign_keys = off;`);
81	raw.exec(`begin;`);
82	try {
83	// -- new tables (kept in sync with models/ProcessorState.ts) --
84	raw.exec(/* SQL */ `
85	create table if not exists processors (
86	id integer primary key autoincrement,
87	name text not null unique,
88	version integer not null
89	);
90	create table if not exists file_processors (
91	file integer not null references media_files(id) on delete cascade,
92	processor integer not null references processors(id) on delete cascade,
93	version integer not null,
94	status integer not null,
95	updated integer not null,
96	error text,
97	primary key (file, processor)
98	);
99	create index if not exists file_processors_processor
100	on file_processors (processor);
101	`);
102	// mark the table-creation key so models/ProcessorState.ts skips its DDL
103	raw.prepare(
104	`insert or ignore into clover_migrations (key, version) values (?, ?);`,
105	).run("processor_state", 1);
106
107	const ids = new Map<string, number>();
108	const insertProcessor = raw.prepare(
109	`insert into processors (name, version) values (?, ?)
110	on conflict(name) do update set version = excluded.version
111	returning id;`,
112	);
113	for (const p of registry.processors) {
114	const { id } = insertProcessor.get(p.name, p.version) as { id: number };
115	ids.set(p.name, id);
116	}
117
118	// -- carry over completions from the bitfield --
119	const files = raw.prepare(
120	`select id, path, processed, processors from media_files where kind = 1;`,
121	).all() as {
122	id: number;
123	path: string;
124	processed: number;
125	processors: string;
126	}[];
127	const insertState = raw.prepare(
128	`insert or replace into file_processors
129	(file, processor, version, status, updated, error)
130	values (?, ?, ?, 1, ?, null);`,
131	);
132	let doneRows = 0;
133	let undecodable = 0;
134	for (const file of files) {
135	let entries: string[];
136	try {
137	entries = decodeProcessorLetters(file.processors);
138	} catch {
139	undecodable += 1;
140	continue;
141	}
142	for (let i = 0; i < entries.length; i += 1) {
143	if ((file.processed & (1 << (16 + i))) === 0) continue;
144	const name = letterMap[UNWRAP(entries[i])];
145	if (!name) continue; // processor no longer exists
146	const proc = registry.byName(name);
147	if (!proc) continue;
148	insertState.run(file.id, UNWRAP(ids.get(name)), proc.version, now);
149	UNWRAP(summary.get(name)).done += 1;
150	doneRows += 1;
151	}
152	}
153
154	// -- rebuild media_files without the bitfield columns --
155	raw.exec(/* SQL */ `
156	create table media_files_new (
157	id integer primary key autoincrement,
158	parent_id integer,
159	path text,
160	kind integer not null,
161	timestamp integer not null,
162	timestamp_updated integer not null default current_timestamp,
163	hash text not null,
164	size integer not null,
165	duration integer not null default 0,
166	dimensions text not null default "",
167	contents text not null,
168	dirsort text,
169	config text not null default "",
170	dir_reindex integer not null default 0,
171	pending integer not null default 0,
172	foreign key (parent_id) references media_files(id) on delete cascade
173	);
174	insert into media_files_new (
175	id, parent_id, path, kind, timestamp, timestamp_updated, hash, size,
176	duration, dimensions, contents, dirsort, config, dir_reindex, pending)
177	select
178	id, parent_id, path, kind, timestamp, timestamp_updated, hash, size,
179	duration, dimensions, contents, dirsort,
180	case when kind = 0 and processors like '{%' then processors else '' end,
181	case when kind = 0 and processed = 0 then 1 else 0 end,
182	0
183	from media_files;
184	drop table media_files;
185	alter table media_files_new rename to media_files;
186	`);
187
188	// -- collapse case duplicates --
189	// the file stores are case-insensitive, so two case spellings of one
190	// path are the same physical file. the old scanner could record both;
191	// keep the newest row (matching the new unique nocase index) and move
192	// any children over.
193	const dupeGroups = raw.prepare(
194	`select group_concat(id) ids, max(id) keep, lower(path) lp
195	from media_files group by lower(path) having count(*) > 1;`,
196	).all() as { ids: string; keep: number; lp: string }[];
197	for (const group of dupeGroups) {
198	const drop = group.ids.split(",").map(Number)
199	.filter((id) => id !== group.keep);
200	console.warn(
201	`case-duplicate rows for ${group.lp}: keeping ${group.keep}, dropping ${drop.join(", ")}`,
202	);
203	const dropList = drop.join(",");
204	raw.exec(/* SQL */ `
205	update media_files set parent_id = ${group.keep}
206	where parent_id in (${dropList});
207	delete from file_processors where file in (${dropList});
208	delete from derived_refs where file in (${dropList});
209	delete from media_files where id in (${dropList});
210	`);
211	}
212
213	raw.exec(/* SQL */ `
214	create unique index media_files_path
215	on media_files (path collate nocase);
216	create index media_files_parent_id on media_files (parent_id);
217	create index media_files_file_children on media_files (kind, path);
218	create index media_files_dir_reindex on media_files (kind, dir_reindex);
219	`);
220
221	// -- compute `pending` from applicability minus completions --
222	const states = raw.prepare(
223	`select processor, version from file_processors where file = ?;`,
224	);
225	const setPending = raw.prepare(
226	`update media_files set pending = ? where id = ?;`,
227	);
228	const pendingPaths: string[] = [];
229	for (const file of files) {
230	const ext = extensionNonEmpty(file.path).toLowerCase();
231	const applicable = registry.applicableFor(ext);
232	if (applicable.length === 0) continue;
233	const done = new Map(
234	(states.all(file.id) as { processor: number; version: number }[])
235	.map((row) => [row.processor, row.version]),
236	);
237	const missing = applicable.filter(
238	(p) => done.get(UNWRAP(ids.get(p.name))) !== p.version,
239	);
240	if (missing.length === 0) continue;
241	setPending.run(missing.length, file.id);
242	for (const p of missing) UNWRAP(summary.get(p.name)).pending += 1;
243	if (pendingPaths.length < 32) {
244	pendingPaths.push(
245	`${file.path} (${missing.map((p) => p.name).join(", ")})`,
246	);
247	}
248	}
249
250	raw.exec(`commit;`);
251	raw.exec(`pragma foreign_keys = on;`);
252
253	const violations = raw.prepare(`pragma foreign_key_check;`).all();
254	if (violations.length > 0) {
255	console.error("foreign key violations after migration:", violations);
256	process.exit(1);
257	}
258	raw.exec(`vacuum;`);
259	raw.exec(`pragma wal_checkpoint(truncate);`);
260
261	// -- summary --
262	console.info("");
263	console.info(`migrated ${files.length} files, ${doneRows} completions`);
264	if (undecodable) {
265	console.warn(`${undecodable} files had undecodable processor strings`);
266	}
267	console.info("per-processor state (done / pending):");
268	for (const [name, { done, pending }] of summary) {
269	const warn = pending > 0 && heavyProcessors.has(name) ? " <-- WILL RUN" : "";
270	console.info(
271	` ${name.padEnd(16)} ${String(done).padStart(6)} / ${String(pending).padStart(4)}${warn}`,
272	);
273	}
274	if (pendingPaths.length > 0) {
275	console.info("");
276	console.info("files with pending work (first 32):");
277	for (const line of pendingPaths) console.info(" " + line);
278	} else {
279	console.info("no files have pending work; first sweep will be a no-op");
280	}
281	} catch (err) {
282	raw.exec(`rollback;`);
283	raw.exec(`pragma foreign_keys = on;`);
284	console.error("migration failed and was rolled back");
285	throw err;
286	}
287	}
288
289	/** decode the old `processors` column into its letter ids */
290	function decodeProcessorLetters(input: string): string[] {
291	return input
292	.split(";")
293	.filter(Boolean)
294	.map(([a, b, c]) => {
295	UNWRAP(b);
296	UNWRAP(c);
297	return UNWRAP(a);
298	});
299	}
300
301	/** mirror of MediaFile.extensionNonEmpty for a raw path string */
302	function extensionNonEmpty(filePath: string) {
303	const basename = path.basename(filePath);
304	const ext = path.extname(basename);
305	if (ext === "") return basename;
306	return ext;
307	}
308
309	// processors expensive enough that an accidental re-run is a disaster
310	const heavyProcessors = new Set([
311	"image-subsets",
312	"av1-au",
313	"av1-ah",
314	"av1-am",
315	"av1-al",
316	"av1-ad",
317	"opus-h",
318	"opus-m",
319	"opus-d",
320	"dash",
321	"h264-hls",
322	]);
323
324	import * as fs from "node:fs";
325	import * as path from "node:path";
326	import * as stream from "node:stream";
327
328	import { getDb } from "#sitegen/sqlite";
329	import { UNWRAP } from "@clo/lib/assert";
330
331	import * as registry from "#src/file-viewer/indexer/registry.ts";

src/file-viewer/indexer/dirmeta.ts created+108

...	@@ -0,0 +1,108 @@
1	// Directory metadata pass: readme contents, explicit sort order, friend
2	// permissions, recursive size/date/hash aggregates, and the `.date`
3	// override. Driven by the `dir_reindex` flag, which the scanner sets on
4	// parents whenever children change. Ported from the tail of file-scan.ts.
5	const console = log.scoped("indexer");
6
7	export function run(p: progress.Ref): boolean {
8	const dirs = MediaFile.getDirectoriesToReindex()
9	.sort((a, b) => b.path.length - a.path.length);
10	if (dirs.length === 0) return false;
11	using node = p.start("update directory metadata", { total: dirs.length });
12
13	for (const dir of dirs) {
14	using _ = node.start(dir.path);
15	try {
16	processDir(dir);
17	} catch (err) {
18	// one broken directory must not starve the rest of the pass; the
19	// dir_reindex flag stays set, so it retries on the next trigger
20	console.error(`directory metadata for ${dir.path} failed:`, err);
21	}
22	node.inc();
23	}
24	console.info(`updated metadata for ${dirs.length} directories`);
25	return true;
26	}
27
28	function processDir(dir: MediaFile) {
29	const children = dir.getChildren();
30
31	// readme.txt
32	const readmeContent = children.find((x) => x.basename === "readme.txt")?.contents ?? "";
33
34	// dirsort
35	let dirsort: string[] \| null = null;
36	const dirSortRaw = children.find((x) => x.basename === ".dirsort")?.contents ?? "";
37	if (dirSortRaw) {
38	dirsort = dirSortRaw
39	.split("\n")
40	.map((x) => x.trim())
41	.filter(Boolean);
42	}
43
44	// Permissions
45	if (children.some((x) => x.basename === ".friends")) {
46	FilePermissions.setPermissions(dir.path, 1);
47	} else {
48	FilePermissions.setPermissions(dir.path, 0);
49	}
50
51	// Recursive stats.
52	let totalSize = 0;
53	let newestDate = new Date(0);
54	let allHashes = "";
55	for (const child of children) {
56	totalSize += child.size;
57	allHashes += child.hash;
58
59	// readme.txt and hidden files don't render a date in the UI, so they
60	// must not contribute to the directory's date either.
61	const dateExempt = child.basename === "readme.txt" \|\| child.basename.startsWith(".");
62	if (!dateExempt && child.date > newestDate) {
63	newestDate = child.date;
64	}
65	}
66
67	// Project Date
68	const dateFile = children.find((x) => x.basename === ".date");
69	if (dateFile) {
70	const date = new Date(dateFile.contents);
71	if (Number.isNaN(date.getTime())) {
72	// a fresh .date file has no extracted contents until the
73	// text-contents processor lands; leave dir_reindex set and let a
74	// later pass pick this directory back up. a genuinely malformed
75	// .date keeps warning here until it is fixed on disk.
76	console.warn(
77	`${dir.path}/.date is empty or unparseable; deferring date override`,
78	);
79	return;
80	}
81	newestDate = date;
82	dir.setConfig(JSON.stringify({ hideChildrenDates: true }));
83	} else {
84	dir.setConfig("");
85	}
86
87	const dirHash = crypto
88	.createHash("sha1")
89	.update(dir.path + allHashes)
90	.digest("hex");
91
92	MediaFile.markDirectoryProcessed({
93	id: dir.id,
94	timestamp: newestDate,
95	contents: readmeContent,
96	size: totalSize,
97	hash: dirHash,
98	dirsort,
99	});
100	}
101
102	import * as crypto from "node:crypto";
103
104	import * as log from "@clo/lib/log";
105	import * as progress from "@clo/lib/progress";
106
107	import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";
108	import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";

src/file-viewer/indexer/processors/compress.ts created+35

...	@@ -0,0 +1,35 @@
1	// pre-compressed variants for static serving. applies to everything that is
2	// not already compressed (media containers, archives, ...).
3	export const processors: Processor[] = [
4	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
5	{ name: "zstd", fn: () => zlib.createZstdCompress() },
6	].map(({ name, fn }) => ({
7	name,
8	version: 1,
9	title: `compress ${name}`,
10	exclude: rules.extsPreCompressed,
11	async run({ path, mediaFile, node, signal }) {
12	await derived.produce({
13	mediaFile,
14	node,
15	cores: 1,
16	subkey: name,
17	async producer(dir) {
18	await stream.promises.pipeline(
19	fs.createReadStream(path.toString()),
20	fn(),
21	fs.createWriteStream(dir.join(name).toString()),
22	{ signal },
23	);
24	},
25	});
26	},
27	}));
28
29	import * as fs from "node:fs";
30	import * as stream from "node:stream";
31	import * as zlib from "node:zlib";
32
33	import * as derived from "#src/file-viewer/models/derived.ts";
34	import * as rules from "#src/file-viewer/rules.ts";
35	import type { Processor } from "../registry.ts";

src/file-viewer/indexer/processors/media.ts created+352

...	@@ -0,0 +1,352 @@
1	// ffmpeg/ffprobe/magick-based processors: duration, dimensions, optimized
2	// image subsets, av1+opus+dash streaming variants, and h264 hls. bodies are
3	// ported unchanged from the old file-scan.ts.
4	const ffprobeBin = testProgram("ffprobe", "--help");
5	const ffmpegBin = testProgram("ffmpeg", "--help");
6
7	const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
8
9	const procDuration: Processor = {
10	name: "duration",
11	version: 1,
12	title: "calculate duration",
13	enable: ffprobeBin !== null,
14	include: rules.extsDuration,
15	cores: 1,
16	async run({ path, mediaFile, signal }) {
17	const { stdout } = await subprocess.exec(ffprobeBin!, [
18	"-v",
19	"error",
20	"-show_entries",
21	"format=duration",
22	"-of",
23	"default=noprint_wrappers=1:nokey=1",
24	path.toString(),
25	], { signal });
26
27	const duration = parseFloat(stdout.trim());
28	if (Number.isNaN(duration)) {
29	throw new Error("Could not extract duration from " + stdout);
30	}
31	mediaFile.setDuration(Math.ceil(duration));
32	},
33	};
34
35	const procDimensions: Processor = {
36	name: "dimensions",
37	version: 2,
38	title: "calculate dimensions",
39	enable: ffprobeBin != null,
40	include: rules.extsDimensions,
41	cores: 1,
42	async run({ path, mediaFile, signal }) {
43	const { ext } = path;
44
45	let dimensions;
46
47	if (ext === ".svg") {
48	// Parse out of text data
49	const content = await path.read("utf-8");
50	const widthMatch = content.match(/width="(\d+)"/);
51	const heightMatch = content.match(/height="(\d+)"/);
52
53	if (widthMatch && heightMatch) {
54	dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
55	}
56	} else if (rules.extsImage.has(ext)) {
57	// Use magick to observe streams
58	const { stdout } = await subprocess.exec("magick", [
59	"identify",
60	"-auto-orient",
61	"-format",
62	"%w %h",
63	path.toString(),
64	], { signal });
65	const [w, h] = stdout.split(" ").map((x) => Number(x));
66	if (w && h) {
67	dimensions = w + "x" + h;
68	}
69	} else {
70	// Use ffprobe to observe streams
71	const { stdout } = await subprocess.exec("ffprobe", [
72	"-v",
73	"error",
74	"-select_streams",
75	"v:0",
76	"-show_entries",
77	"stream=width,height:stream_side_data=rotation",
78	"-of",
79	"json",
80	path.toString(),
81	], { signal });
82	const result = JSON.parse(stdout);
83	const stream = result.streams[0];
84	if (stream) {
85	let width = UNWRAP(stream.width);
86	let height = UNWRAP(stream.height);
87	// phone videos store the sensor's landscape frame plus a display
88	// matrix; everything downstream (scale filters, aspect-ratio css)
89	// wants the rotated display dimensions.
90	const rotation = (stream.side_data_list ?? [])
91	.find((s: any) => typeof s.rotation === "number")?.rotation ?? 0;
92	if (Math.abs(rotation) % 180 === 90) {
93	[width, height] = [height, width];
94	}
95	dimensions = width + "x" + height;
96	}
97	}
98
99	mediaFile.setDimensions(dimensions ?? "");
100	},
101	};
102
103	const procImageSubsets: Processor = {
104	name: "image-subsets",
105	version: 3,
106	title: "encode image subsets",
107	include: rules.extsImage,
108	depends: [procDimensions.name],
109	async run({ path, mediaFile, node, signal }) {
110	const dims = mediaFile.parseDimensions();
111	// dimensions could not be extracted; nothing to scale.
112	if (!dims) return;
113	const { width, height } = dims;
114	const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
115	const baseStatus = node.text;
116
117	using stack = new DisposableStack();
118	for (const size of targetSizes) {
119	const { w, h } = resizeDimensions(width, height, size);
120	for (const { ext, args } of transcodeRules.imagePresets) {
121	node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
122
123	stack.use(
124	await derived.produce({
125	mediaFile,
126	node,
127	cores: 2,
128	subkey: `${size}${ext}`,
129	async producer(dir) {
130	await subprocess.exec(ffmpegBin!, [
131	...ffmpegOptions,
132	"-i",
133	path.toString(),
134	"-vf",
135	`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
136	...args,
137	dir.join(`${size}${ext}`).toString(),
138	], { signal });
139	},
140	}),
141	);
142	}
143	}
144
145	stack.move();
146	},
147	};
148
149	const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
150
151	const qualityMap: Record<string, string> = {
152	u: "ultra-high",
153	h: "high",
154	m: "medium",
155	l: "low",
156	d: "data-saving",
157	};
158
159	function skipVideo(mediaFile: MediaFile) {
160	if ((mediaFile.duration ?? 0) < 5) return true;
161	if (!mediaFile.dimensions) return true;
162	if (rules.processDenyList.has(mediaFile.path)) return true;
163	return false;
164	}
165
166	const procVideos = transcodeRules.videoFormats.map<Processor>((preset) => ({
167	name: `av1-${preset.id}`,
168	version: 4,
169	title: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
170	include: rules.extsVideo,
171	enable: ffmpegBin != null,
172	depends: [procDuration.name, procDimensions.name],
173	async run({ path, mediaFile, node, signal }) {
174	if (skipVideo(mediaFile)) return;
175
176	await derived.produce({
177	mediaFile,
178	node,
179	cores: 4,
180	subkey: `av1-${preset.id}`,
181	async producer(dir) {
182	const input = await videoArgsCache.getOrRun(path);
183	const dimensions = mediaFile.parseDimensions();
184	ASSERT(dimensions);
185	const args = transcodeRules.getAv1VideoArgs(
186	preset,
187	dimensions,
188	UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
189	dir,
190	);
191
192	await ffmpeg.spawn({
193	ffmpeg: ffmpegBin!,
194	progress: node,
195	args,
196	cwd: dir.toString(),
197	signal,
198	});
199	},
200	});
201	},
202	}));
203
204	const procVideoAudios = transcodeRules.audioFormats.map<Processor>((preset) => ({
205	name: `opus-${preset.id}`,
206	version: 3,
207	title: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
208	include: rules.extsVideo,
209	enable: ffmpegBin != null,
210	depends: [procDuration.name, procDimensions.name],
211	async run({ path, mediaFile, node, signal }) {
212	if (skipVideo(mediaFile)) return;
213	await derived.produce({
214	mediaFile,
215	node,
216	cores: 4,
217	subkey: `opus-${preset.id}`,
218	async producer(dir) {
219	const input = await videoArgsCache.getOrRun(path);
220	ASSERT(input.video);
221	if (!input.audio) return;
222	const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
223
224	await ffmpeg.spawn({
225	ffmpeg: ffmpegBin!,
226	progress: node,
227	args,
228	cwd: dir.toString(),
229	signal,
230	});
231	},
232	});
233	},
234	}));
235
236	const procDash: Processor = {
237	name: "dash",
238	version: 7,
239	title: "encode mpeg-dash",
240	include: rules.extsVideo,
241	enable: ffmpegBin != null,
242	depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
243	async run({ path, mediaFile, node, signal }) {
244	if (skipVideo(mediaFile)) return;
245	await derived.produce({
246	mediaFile,
247	node,
248	cores: 1,
249	subkey: `dash-av1`,
250	async producer(dir) {
251	const input = await videoArgsCache.getOrRun(path);
252	const videos = transcodeRules.videoFormats.map(
253	(preset) =>
254	UNWRAP(dir.parent).join(
255	`av1-${preset.id}`,
256	transcodeRules.av1FileName,
257	),
258	);
259	const audios = input.audio
260	? transcodeRules.audioFormats.map(
261	(preset) =>
262	UNWRAP(dir.parent).join(
263	`opus-${preset.id}`,
264	transcodeRules.opusFileName,
265	),
266	)
267	: [];
268	const args = transcodeRules.getMpegDashArgs(
269	videos,
270	audios,
271	dir,
272	);
273
274	await dir.join("d").makeDir();
275
276	await ffmpeg.spawn({
277	ffmpeg: ffmpegBin!,
278	progress: node,
279	args,
280	cwd: dir.toString(),
281	signal,
282	});
283	},
284	});
285	},
286	};
287
288	const procH264Hls: Processor = {
289	name: "h264-hls",
290	version: 1,
291	title: "encode h.264 hls",
292	include: rules.extsVideo,
293	enable: ffmpegBin != null,
294	depends: [procDuration.name, procDimensions.name],
295	async run({ path, mediaFile, node, signal }) {
296	if (skipVideo(mediaFile)) return;
297
298	await derived.produce({
299	mediaFile,
300	node,
301	cores: 8,
302	subkey: `hls`,
303	async producer(dir) {
304	const input = await videoArgsCache.getOrRun(path);
305	const args = transcodeRules.getH264HlsArgs(input, dir);
306
307	await ffmpeg.spawn({
308	ffmpeg: ffmpegBin!,
309	progress: node,
310	args,
311	cwd: dir.toString(),
312	signal,
313	});
314	},
315	});
316	},
317	};
318
319	function resizeDimensions(w: number, h: number, desiredWidth: number) {
320	ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
321	return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
322	}
323
324	function testProgram(name: string, helpArgument: string) {
325	// spawnSync does not throw on a missing binary; it reports `error`
326	const result = child_process.spawnSync(name, [helpArgument]);
327	if (!result.error) return name;
328	console.warn(`Missing or corrupt executable '${name}'`);
329	return null;
330	}
331
332	export const processors: Processor[] = [
333	procDimensions,
334	procDuration,
335	procImageSubsets,
336	...procVideos,
337	...procVideoAudios,
338	procDash,
339	procH264Hls,
340	];
341
342	import { ASSERT, UNWRAP } from "@clo/lib/assert";
343	import * as async from "@clo/lib/async";
344	import * as subprocess from "@clo/lib/subprocess";
345	import * as ffmpeg from "@clo/lib/subprocess/ffmpeg";
346	import * as child_process from "node:child_process";
347
348	import * as derived from "#src/file-viewer/models/derived.ts";
349	import type { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
350	import * as rules from "#src/file-viewer/rules.ts";
351	import * as transcodeRules from "#src/file-viewer/transcode-rules.ts";
352	import type { Processor } from "../registry.ts";

src/file-viewer/indexer/processors/text.ts created+56

...	@@ -0,0 +1,56 @@
1	// text-content extraction processors. `text-contents` fills
2	// `media_files.contents` for plain text files; `highlight-code` fills it
3	// with pre-rendered syntax highlighting html for source code.
4	const procLoadTextContents: Processor = {
5	name: "text-contents",
6	version: 2,
7	title: "load text content",
8	include: rules.extsReadContents,
9	cores: 1,
10	async run({ path, mediaFile, stat }) {
11	if (stat.size > 1_000_000) return;
12	const text = await path.read("utf-8");
13	mediaFile.setContents(text);
14	},
15	};
16
17	const procHighlightCode: Processor = {
18	name: "highlight-code",
19	version: 2,
20	title: "highlight source code",
21	include: new Set(rules.extsCode.keys()),
22	cores: 1,
23	async run({ path, mediaFile, stat }) {
24	const language = UNWRAP(
25	rules.extsCode.get(path.ext.toLowerCase()),
26	);
27	// An issue is that .ts is an overloaded extension, shared between
28	// 'transport stream' and 'typescript'.
29	//
30	// Filter used here is:
31	// - more than 1mb
32	// - invalid UTF-8
33	if (stat.size > 1_000_000) return;
34	let code;
35	const buf = await path.read();
36	try {
37	code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
38	} catch (error) {
39	mediaFile.setContents("");
40	return;
41	}
42	const content = await highlight.highlightCode(code, language);
43	mediaFile.setContents(content);
44	},
45	};
46
47	export const processors: Processor[] = [
48	procLoadTextContents,
49	procHighlightCode,
50	];
51
52	import { UNWRAP } from "@clo/lib/assert";
53
54	import * as highlight from "#src/file-viewer/highlight.ts";
55	import * as rules from "#src/file-viewer/rules.ts";
56	import type { Processor } from "../registry.ts";

src/file-viewer/indexer/registry.ts created+100

...	@@ -0,0 +1,100 @@
1	// The processor registry. Each processor has a stable string `name` and an
2	// integer `version`; bumping the version makes every applicable file re-run
3	// that processor on the next sweep. This replaces the old scheme of hashing
4	// `run.toString()` into a bitfield (rest in peace).
5	//
6	// This module is intentionally free of database imports so that the one-shot
7	// database migration can reuse the definitions and applicability rules
8	// before the new schema exists.
9	export interface Processor {
10	/** stable identifier, stored in the `processors` table. never rename. */
11	name: string;
12	/** bump to re-run this processor on all applicable files */
13	version: number;
14	/** human-readable label for progress display */
15	title: string;
16	/** approximate cores used while running, for the scheduler */
17	cores?: number;
18	/** false when a required tool (ffmpeg, ...) is missing on this host */
19	enable?: boolean;
20	/** if set, only these extensions apply; otherwise all but `exclude` */
21	include?: Set<string>;
22	/** extensions that do not apply (only when `include` is unset) */
23	exclude?: Set<string>;
24	/** processor names that must complete before this one runs */
25	depends?: string[];
26	run(args: ProcessorRunArgs): Promise<void>;
27	}
28
29	export interface ProcessorRunArgs {
30	path: Path;
31	stat: fs.Stats;
32	mediaFile: MediaFile;
33	node: progress.Node;
34	/** aborts when the file is deleted mid-run; pass to subprocesses */
35	signal: AbortSignal;
36	}
37
38	export const processors: readonly Processor[] = [
39	...mediaProcessors,
40	...textProcessors,
41	...compressProcessors,
42	];
43
44	// integrity checks, evaluated once at import
45	{
46	const names = new Set<string>();
47	for (const p of processors) {
48	if (names.has(p.name)) throw new Error(`duplicate processor: ${p.name}`);
49	names.add(p.name);
50	ASSERT(Number.isInteger(p.version) && p.version >= 1, p.name);
51	for (const depend of p.depends ?? []) {
52	if (depend === p.name) throw new Error(`${p.name} depends on itself`);
53	if (!processors.some((o) => o.name === depend)) {
54	throw new Error(`${p.name} depends on unknown '${depend}'`);
55	}
56	}
57	}
58	}
59
60	/**
61	* which processors apply to a file extension. extension must come from
62	* `MediaFile.extensionNonEmpty`, lowercased. deterministic across hosts:
63	* `enable` and the disable list do not affect applicability, only execution,
64	* so `media_files.pending` means the same thing everywhere.
65	*/
66	export function applicableFor(ext: string): Processor[] {
67	return processors.filter((p) => p.include ? p.include.has(ext) : !p.exclude?.has(ext));
68	}
69
70	const disabledList = new Set(
71	(process.env.CLOVER_PROCESSORS_DISABLE ?? "")
72	.split(",")
73	.map((x) => x.trim())
74	.filter(Boolean),
75	);
76	for (const name of disabledList) {
77	if (!processors.some((p) => p.name === name)) {
78	console.warn(`CLOVER_PROCESSORS_DISABLE: unknown processor '${name}'`);
79	}
80	}
81
82	/** false when the host cannot or should not execute this processor */
83	export function canExecute(p: Processor): boolean {
84	return p.enable !== false && !disabledList.has(p.name);
85	}
86
87	export function byName(name: string): Processor \| null {
88	return processors.find((p) => p.name === name) ?? null;
89	}
90
91	import type * as progress from "@clo/lib/progress";
92	import type * as fs from "node:fs";
93
94	import type { Path } from "#sitegen/path";
95	import type { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
96
97	import { ASSERT } from "@clo/lib/assert";
98	import { processors as compressProcessors } from "./processors/compress.ts";
99	import { processors as mediaProcessors } from "./processors/media.ts";
100	import { processors as textProcessors } from "./processors/text.ts";

src/file-viewer/indexer/scan.ts created+559

...	@@ -0,0 +1,559 @@
1	// The scanner keeps `media_files` in sync with the raw file store and
2	// schedules processors. It is used three ways:
3	// - a full sweep over the entire store (boot + periodic)
4	// - a targeted scan of one path (file watcher events)
5	// - the `file-scan` cli wrapper for development machines
6	//
7	// Files become visible in the database the moment their metadata row is
8	// written; processors run afterwards and trickle their outputs in. The
9	// `pending` column on `media_files` tells the UI that more data is coming.
10	// `scanPath` resolves once metadata is settled; processor jobs continue in
11	// the background on the global queue (await `waitIdle` to block on them).
12	//
13	// A file that is still being written (a large SMB upload, for example) is
14	// not touched until it has been stable for `settleMs`: no hashing, no exif
15	// scrubbing, no processors. The scanner just waits and re-stats.
16	const console = log.scoped("indexer");
17
18	export interface ScannerOptions {
19	/** root of the raw file store */
20	root: Path;
21	/** where the two persistent progress nodes live */
22	progress: progress.Ref;
23	/** notified when database contents change, for the web-node pinger */
24	onChange?: (kind: ChangeKind) => void;
25	/** called whenever all queued work (walks + processors) finishes */
26	onIdle?: () => void;
27	/** a file must be unmodified for this long before it is indexed */
28	settleMs?: number;
29	}
30
31	/** `metadata` is urgent (new/removed files); `processed` is lazy. */
32	export type ChangeKind = "metadata" \| "processed";
33
34	export class Scanner {
35	root: Path;
36	onChange: (kind: ChangeKind) => void;
37	onIdle: () => void;
38	settleMs: number;
39	/** processor name -> `processors` table row id */
40	ids: Map<string, number>;
41
42	walkQueue = new queue.PriorityQueue(10);
43	// scrub+hash runs on its own small pool, never the global queue: encode
44	// jobs hold global cores for minutes to hours, and a saturated pool would
45	// starve metadata updates — files must become visible immediately even
46	// mid-encode-storm. two slots bound disk thrash while staying responsive.
47	hashQueue = new queue.PriorityQueue(2);
48	#inFlight = 0;
49	#idleWaiters: (() => void)[] = [];
50
51	// the entire progress display is two permanent top-level groups: what is
52	// being indexed right now, and which processors are running. passive
53	// nodes vanish when idle. no per-batch wrapper nodes; it does not matter
54	// whether work came from the boot sweep, the watcher, or /scan.
55	indexNode: progress.Node;
56	processNode: progress.Node;
57
58	constructor(options: ScannerOptions) {
59	this.root = options.root;
60	this.onChange = options.onChange ?? (() => {});
61	this.onIdle = options.onIdle ?? (() => {});
62	this.settleMs = options.settleMs ?? 10_000;
63	this.ids = ProcessorState.syncRegistry(registry.processors);
64	if (!this.root.ifExistsSync()) {
65	throw new Error(`file store ${this.root} is not mounted`);
66	}
67	this.indexNode = options.progress.start("indexing files", {
68	passive: true,
69	showTotal: false,
70	});
71	this.processNode = options.progress.start("running processors", {
72	passive: true,
73	showTotal: false,
74	});
75	this.processNode.sortChildren = (a, b) => {
76	const ac = a.children.length > 0 ? 1 : 0;
77	const bc = b.children.length > 0 ? 1 : 0;
78	if (ac !== bc) return bc - ac;
79	return a.text.localeCompare(b.text);
80	};
81	}
82
83	/** scan the whole store. resolves when all metadata rows are updated. */
84	sweep(): Promise<void> {
85	return this.scanPath(this.root);
86	}
87
88	/**
89	* scan one absolute path (file or directory). resolves when the subtree's
90	* metadata is settled; processor jobs continue in the background.
91	*/
92	async scanPath(path: Path): Promise<void> {
93	const ctx: ScanContext = {
94	promises: new async.PromiseAggregator(),
95	};
96	ctx.promises.push(this.#visit(path, ctx));
97	await ctx.promises.all();
98	}
99
100	/** wait for all background processor jobs to finish */
101	async waitIdle(): Promise<void> {
102	if (this.#inFlight === 0) return;
103	await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
104	}
105
106	#beginJob() {
107	this.#inFlight += 1;
108	}
109	#endJob() {
110	ASSERT(this.#inFlight > 0);
111	this.#inFlight -= 1;
112	if (this.#inFlight === 0) {
113	const waiters = this.#idleWaiters;
114	this.#idleWaiters = [];
115	for (const w of waiters) w();
116	this.onIdle();
117	}
118	}
119
120	#visit = this.walkQueue.wrap(async (path: Path, ctx: ScanContext) => {
121	const publicPath = toPublicPath(this.root, path);
122	using node = this.indexNode.start(publicPath + " - stat");
123
124	let stat: fs.Stats;
125	try {
126	stat = await path.stat();
127	} catch (err) {
128	if (error.code(err) !== "ENOENT") throw err;
129	// deleted; watcher events often describe files already gone
130	this.#removePath(publicPath);
131	return;
132	}
133
134	const mediaFile = MediaFile.getByPath(publicPath);
135
136	// the row may carry a different spelling of the same path (case-only
137	// rename: same inode, same mtime, so no metadata pass runs). adopt the
138	// on-disk spelling — only the parent's listing knows it; both the event
139	// path and the stored path can be stale.
140	if (mediaFile && mediaFile.id !== 0 && mediaFile.path !== publicPath) {
141	await this.#reconcileCase(path, mediaFile);
142	}
143
144	if (stat.isDirectory()) {
145	node.text = publicPath + " - reading";
146	const items = (await path.readDir())
147	.filter((child) => !skipBasename(child.base))
148	.map((child) => (
149	ctx.promises.push(this.#visit(child, ctx)), child.base
150	));
151
152	// reconcile deletions against the database. the store is
153	// case-insensitive, so compare names folded; a case-only rename is the
154	// same file, not a delete + create.
155	const names = new Set(items.map((name) => name.toLowerCase()));
156	for (const child of mediaFile?.getChildren() ?? []) {
157	if (names.has(child.basename.toLowerCase())) continue;
158	this.#removeFile(child);
159	}
160	return;
161	}
162
163	if (
164	!mediaFile
165	\|\| stat.size !== mediaFile.size
166	\|\| stat.mtime.getTime() !== mediaFile.date.getTime()
167	) {
168	// do not hold a walk queue slot while settling/hashing. concurrent
169	// visits of the same path (watch events racing a sweep, or two case
170	// spellings of one physical file) share one metadata update instead
171	// of hashing — or worse, scrubbing — the file twice. one broken file
172	// logs and moves on; it must not abort the surrounding scan.
173	const flightKey = publicPath.toLowerCase();
174	let job = this.#metaInFlight.get(flightKey);
175	if (!job) {
176	job = this.#updateMetadata({ path, publicPath, stat, mediaFile })
177	.catch((err) => console.error(`indexing ${publicPath} failed:`, err))
178	.finally(() => this.#metaInFlight.delete(flightKey));
179	this.#metaInFlight.set(flightKey, job);
180	}
181	ctx.promises.push(job);
182	} else {
183	this.#queueProcessors({ path, stat, mediaFile });
184	}
185	});
186
187	#metaInFlight = new Map<string, Promise<void>>();
188
189	async #reconcileCase(path: Path, mediaFile: MediaFile) {
190	const parent = path.parent;
191	if (!parent) return;
192	let names: string[];
193	try {
194	names = (await parent.readDir()).map((entry) => entry.base);
195	} catch {
196	return;
197	}
198	const folded = path.base.toLowerCase();
199	const trueName = names.find((name) => name.toLowerCase() === folded);
200	if (!trueName) return;
201	const truePath = toPublicPath(this.root, parent.join(trueName));
202	if (truePath === mediaFile.path) return;
203	console.info(`case rename ${mediaFile.path} -> ${truePath}`);
204	mediaFile.updatePath(truePath);
205	mediaFile.getParent()?.markDirReindex();
206	this.onChange("metadata");
207	}
208
209	#removePath(publicPath: string) {
210	const row = MediaFile.getByPath(publicPath);
211	if (!row \|\| row.id === 0) return;
212	this.#removeFile(row);
213	}
214
215	#removeFile(file: MediaFile) {
216	const recursive = file.kind === MediaFileKind.directory
217	? [file, ...file.getRecursiveFileChildren()]
218	: [file];
219	for (const deletion of recursive) {
220	deletion.delete();
221	// kill in-flight processors; a deleted file must not keep encoding
222	this.#fileAborts.get(deletion.id)?.abort();
223	}
224	console.info(`deleted ${file.path}`);
225	file.getParent()?.markDirReindex();
226	this.onChange("metadata");
227	}
228
229	async #updateMetadata(
230	{ path, publicPath, stat, mediaFile }: {
231	path: Path;
232	publicPath: string;
233	stat: fs.Stats;
234	mediaFile: MediaFile \| null;
235	},
236	) {
237	const label = publicPath.slice(1);
238	using node = this.indexNode.start(label);
239
240	// hold off on everything until the file has stopped changing.
241	const settled = await this.#waitUntilSettled(path, stat, node);
242	if (settled === null) {
243	this.#removePath(publicPath);
244	return;
245	}
246	stat = settled;
247
248	node.text = `${label} - waiting to hash`;
249	const hash = await this.hashQueue.run({
250	cores: 1,
251	run: async () => {
252	if (await scrub.scrubLocationMetadata(path, stat, node)) {
253	stat = await path.stat();
254	}
255	node.text = `${label} - hashing`;
256	return await hashFile(path);
257	},
258	});
259
260	let date = stat.mtime;
261	if (
262	mediaFile
263	&& mediaFile.date.getTime() < stat.mtime.getTime()
264	&& Date.now() - stat.mtime.getTime() < monthMilliseconds
265	) {
266	date = mediaFile.date;
267	console.warn(
268	`M-time on ${publicPath} was likely corrupted. ${formatDate(mediaFile.date)} -> ${formatDate(stat.mtime)}`,
269	);
270	}
271
272	const contentChanged = !mediaFile \|\| mediaFile.hash !== hash;
273	mediaFile = MediaFile.createFile({
274	path: publicPath,
275	date,
276	hash,
277	size: stat.size,
278	duration: mediaFile?.duration ?? 0,
279	dimensions: mediaFile?.dimensions ?? "",
280	contents: mediaFile?.contents ?? "",
281	});
282	if (contentChanged) ProcessorState.invalidateFile(mediaFile.id);
283
284	mediaFile.getParent()?.markDirReindex();
285	this.onChange("metadata");
286
287	node.text = label;
288	this.#queueProcessors({ path, stat, mediaFile });
289	}
290
291	async #waitUntilSettled(
292	path: Path,
293	stat: fs.Stats,
294	node: progress.Node,
295	): Promise<fs.Stats \| null> {
296	const baseText = node.text;
297	let polls = 0;
298	while (true) {
299	const age = Date.now() - stat.mtime.getTime();
300	if (age >= this.settleMs) break;
301	// mtimes in the future cannot settle; the corruption guard dates them
302	if (age < -60_000) break;
303	node.text = `${baseText} - waiting for upload to finish`;
304	await async.delay(Math.min(this.settleMs, 2500));
305	if ((polls += 1) === 240) { // roughly ten minutes
306	console.warn(`${path} has been unstable for a long time`);
307	}
308	try {
309	stat = await path.stat();
310	} catch (err) {
311	if (error.code(err) === "ENOENT") return null;
312	throw err;
313	}
314	}
315	node.text = baseText;
316	return stat;
317	}
318
319	// files whose processor pipeline is currently queued or running. a second
320	// scan of the file (sweep racing the watcher) must not double-queue jobs
321	// or reset `pending` mid-flight. the abort controller cancels the
322	// pipeline's subprocesses when the file is deleted.
323	#activeProcessing = new Set<number>();
324	#fileAborts = new Map<number, AbortController>();
325
326	#queueProcessors(
327	args: {
328	path: Path;
329	stat: fs.Stats;
330	mediaFile: MediaFile;
331	},
332	) {
333	const { mediaFile } = args;
334	if (this.#activeProcessing.has(mediaFile.id)) return;
335	const ext = mediaFile.extensionNonEmpty.toLowerCase();
336	const applicable = registry.applicableFor(ext);
337	if (applicable.length === 0) {
338	if (mediaFile.pending !== 0) mediaFile.setPending(0);
339	return;
340	}
341
342	const states = ProcessorState.getStates(mediaFile.id);
343	const needed = applicable.filter((p) => {
344	const state = states.get(UNWRAP(this.ids.get(p.name)));
345	return !state \|\| state.version !== p.version
346	\|\| state.status === ProcessorState.ProcessorStatus.failed;
347	});
348	if (mediaFile.pending !== needed.length) {
349	mediaFile.setPending(needed.length);
350	}
351	if (needed.length === 0) return;
352
353	// a processor can only run when the host has its tools and all of its
354	// dependencies are either previously-done or also runnable now.
355	const runnable = new Set<registry.Processor>();
356	let grew = true;
357	while (grew) {
358	grew = false;
359	for (const p of needed) {
360	if (runnable.has(p) \|\| !registry.canExecute(p)) continue;
361	const ok = (p.depends ?? []).every((depend) => {
362	const dep = needed.find((o) => o.name === depend);
363	return !dep \|\| runnable.has(dep);
364	});
365	if (ok) {
366	runnable.add(p);
367	grew = true;
368	}
369	}
370	}
371	if (runnable.size === 0) return;
372
373	this.#activeProcessing.add(mediaFile.id);
374	const abort = new AbortController();
375	this.#fileAborts.set(mediaFile.id, abort);
376	const node = this.processNode.start(mediaFile.path.slice(1), {
377	passive: true,
378	showTotal: false,
379	total: runnable.size,
380	});
381	// the whole pipeline is accounted upfront: counting per-started-job
382	// would let the in-flight count transiently hit zero between a
383	// dependency finishing and its dependants starting.
384	for (let i = 0; i < runnable.size; i += 1) this.#beginJob();
385	let remaining = runnable.size;
386	const settleJob = () => {
387	node.value += 1;
388	this.#endJob();
389	if ((remaining -= 1) === 0) {
390	node.end();
391	this.#activeProcessing.delete(mediaFile.id);
392	this.#fileAborts.delete(mediaFile.id);
393	}
394	};
395
396	const jobs = [...runnable].map<ProcessJob>((processor) => ({
397	processor,
398	after: [],
399	needs: 0,
400	done: false,
401	}));
402	for (const job of jobs) {
403	for (const depend of job.processor.depends ?? []) {
404	const dependJob = jobs.find((j) => j.processor.name === depend);
405	if (dependJob) {
406	dependJob.after.push(job);
407	job.needs += 1;
408	}
409	}
410	}
411
412	// when a job fails, everything transitively depending on it is
413	// abandoned: still pending, retried together on the next sweep.
414	const abandon = (job: ProcessJob) => {
415	for (const dependant of job.after) {
416	if (dependant.done) continue;
417	dependant.done = true;
418	settleJob();
419	mediaFile.decPending();
420	abandon(dependant);
421	}
422	};
423
424	const start = (job: ProcessJob) => {
425	queue.run({
426	cores: job.processor.cores ?? 0,
427	run: () => this.#executeJob(job.processor, args, node, abort.signal),
428	}).then(() => {
429	job.done = true;
430	settleJob();
431	for (const dependant of job.after) {
432	ASSERT(dependant.needs > 0);
433	dependant.needs -= 1;
434	if (dependant.needs === 0 && !dependant.done) start(dependant);
435	}
436	}, () => {
437	job.done = true;
438	settleJob();
439	abandon(job);
440	});
441	};
442	for (const job of jobs) if (job.needs === 0) start(job);
443	}
444
445	async #executeJob(
446	processor: registry.Processor,
447	{ path, stat, mediaFile }: {
448	path: Path;
449	stat: fs.Stats;
450	mediaFile: MediaFile;
451	},
452	parent: progress.Node,
453	signal: AbortSignal,
454	) {
455	// deleted while this job sat in the queue: skip silently. the delete
456	// already cleaned up `file_processors` and `pending`.
457	const rowExists = () => MediaFile.getByPath(mediaFile.path)?.id === mediaFile.id;
458	if (signal.aborted \|\| !rowExists()) return;
459
460	using node = parent.start(processor.title);
461	const id = UNWRAP(this.ids.get(processor.name));
462	try {
463	await processor.run({ path, stat, mediaFile, node, signal });
464	if (signal.aborted \|\| !rowExists()) return;
465	ProcessorState.recordResult(
466	mediaFile.id,
467	id,
468	processor.version,
469	ProcessorState.ProcessorStatus.done,
470	);
471	mediaFile.decPending();
472	this.onChange("processed");
473	} catch (err: any) {
474	if (signal.aborted) {
475	console.info(`${processor.name} aborted on ${mediaFile.path} (deleted)`);
476	throw err;
477	}
478	if (rowExists()) {
479	const message = String(err?.stack ?? err).slice(0, 4000);
480	ProcessorState.recordResult(
481	mediaFile.id,
482	id,
483	processor.version,
484	ProcessorState.ProcessorStatus.failed,
485	message,
486	);
487	mediaFile.decPending();
488	this.onChange("processed");
489	}
490	console.error(`${processor.name} failed on ${mediaFile.path}:`, err);
491	throw err;
492	}
493	}
494	}
495
496	interface ScanContext {
497	promises: async.PromiseAggregator;
498	}
499
500	interface ProcessJob {
501	processor: registry.Processor;
502	after: ProcessJob[];
503	needs: number;
504	done: boolean;
505	}
506
507	export function hashFile(path: Path): Promise<string> {
508	return new Promise<string>((resolve, reject) => {
509	const reader = fs.createReadStream(path.toString());
510	reader.on("error", reject);
511
512	const hasher = crypto.createHash("sha1").setEncoding("hex");
513	hasher.on("error", reject);
514	hasher.on("readable", () => resolve(hasher.read()));
515
516	reader.pipe(hasher);
517	});
518	}
519
520	export function skipBasename(basename: string): boolean {
521	// dot files must be incrementally tracked
522	if (basename === ".dirsort") return false;
523	if (basename === ".friends") return false;
524	if (basename === ".date") return false;
525
526	return (
527	basename.startsWith(".")
528	\|\| basename.startsWith("tmp.")
529	\|\| basename.toLowerCase() === "thumbs.db"
530	\|\| basename.toLowerCase() === "desktop.ini"
531	);
532	}
533
534	export function toPublicPath(root: Path, diskPath: Path) {
535	if (diskPath.toString() === root.toString()) return "/";
536	return "/"
537	+ path.relative(root.toString(), diskPath.toString()).replaceAll("\\", "/");
538	}
539
540	const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
541
542	import * as crypto from "node:crypto";
543	import * as fs from "node:fs";
544	import * as path from "node:path";
545
546	import { Path } from "#sitegen/path";
547	import { ASSERT, UNWRAP } from "@clo/lib/assert";
548	import * as async from "@clo/lib/async";
549	import * as error from "@clo/lib/error";
550	import * as log from "@clo/lib/log";
551	import * as progress from "@clo/lib/progress";
552	import * as queue from "@clo/lib/queue";
553	import * as ts from "@clo/lib/ts";
554
555	import { formatDate } from "#src/file-viewer/format.ts";
556	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";
557	import * as ProcessorState from "#src/file-viewer/models/ProcessorState.ts";
558	import * as registry from "./registry.ts";
559	import * as scrub from "./scrub.ts";

src/file-viewer/indexer/scrub.ts created+160

...	@@ -0,0 +1,160 @@
1	// gps/location metadata removal, ported from the old file-scan.ts. runs
2	// before hashing so the stored hash always reflects the scrubbed file.
3	// requires the file to be fully written (the scanner's stability gate runs
4	// first); modifies the file in place while preserving timestamps.
5	const exiftoolBin = testProgram("exiftool");
6
7	// `-ee` on an iphone video dumps per-frame embedded metadata, which easily
8	// exceeds execFile's default 1MB stdout buffer
9	const execOptions = { maxBuffer: 64 * 1024 * 1024 };
10
11	export async function scrubLocationMetadata(
12	path: Path,
13	stats: fs.Stats,
14	progress: progress.Ref,
15	): Promise<boolean> {
16	using _ = progress.start("scrub exif metadata");
17	const ext = path.ext.toLowerCase();
18	if (!rules.extsScrubExif.has(ext)) return false;
19	if (!exiftoolBin) {
20	warnMissingExiftool();
21	return false;
22	}
23
24	let hasLocation = false;
25	let args: string[] = [];
26
27	// Check for location metadata based on file type
28	const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
29	switch (ext) {
30	case ".jpg":
31	case ".jpeg":
32	case ".png":
33	const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
34	"-gps:all",
35	path.toString(),
36	], execOptions);
37	hasLocation = gpsCheck.trim().length > 0;
38	args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
39	break;
40	case ".mov":
41	case ".mp4":
42	const { stdout: videoCheck } = await subprocess.exec("exiftool", [
43	"-ee",
44	"-G3",
45	"-s",
46	path.toString(),
47	], execOptions);
48	hasLocation = videoCheck.includes("GPS")
49	\|\| videoCheck.includes("Location");
50	args = [
51	"-gps:all=",
52	"-xmp:all=",
53	path.toString(),
54	"-o",
55	tempOutput.toString(),
56	];
57	break;
58	case ".m4a":
59	const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
60	"-ee",
61	"-G3",
62	"-s",
63	path.toString(),
64	], execOptions);
65	hasLocation = m4aCheck.includes("GPS")
66	\|\| m4aCheck.includes("Location")
67	\|\| m4aCheck.includes("Filename")
68	\|\| m4aCheck.includes("Title");
69
70	if (hasLocation) {
71	args = [
72	"-gps:all=",
73	"-location:all=",
74	"-filename:all=",
75	"-title=",
76	"-m4a:all=",
77	path.toString(),
78	"-o",
79	tempOutput.toString(),
80	];
81	}
82	break;
83	}
84
85	const accessTime = stats.atime;
86	const modTime = stats.mtime;
87
88	let backup: Path \| null = null;
89	try {
90	if (hasLocation) {
91	// Prepare a backup. content-only copy: fs.copyFile's metadata
92	// preservation gets EPERM'd on the NAS datasets, plain writes do not.
93	const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
94	await stream.promises.pipeline(
95	fs.createReadStream(path.toString()),
96	fs.createWriteStream(tmp.toString()),
97	);
98	await fsp.utimes(tmp.toString(), accessTime, modTime);
99	backup = tmp;
100
101	// a leftover temp from a crashed run makes exiftool refuse to write
102	await tempOutput.delete({ force: true });
103
104	// Remove metadata
105	await subprocess.exec("exiftool", args, execOptions);
106	if (!tempOutput.ifExistsSync()) {
107	throw new Error(`Failed to create output file: ${tempOutput}`);
108	}
109
110	// Restore original timestamps
111	await fsp.rename(tempOutput.toString(), path.toString());
112	await fsp.utimes(path.toString(), accessTime, modTime);
113
114	// Backup is no longer needed
115	await fsp.unlink(backup.toString());
116
117	console.info(`Scrubbed location metadata in ${path}`);
118	return true;
119	}
120	} catch (error) {
121	// restore is best-effort: a concurrent scrub of the same physical file
122	// (case-insensitive store) may have already consumed the backup
123	if (backup && fs.existsSync(backup.toString())) {
124	await fsp.rename(backup.toString(), path.toString());
125	}
126	if (fs.existsSync(tempOutput.toString())) {
127	await fsp.unlink(tempOutput.toString());
128	}
129	throw error;
130	}
131
132	return false;
133	}
134
135	let warnedMissingExiftool = false;
136	function warnMissingExiftool() {
137	if (warnedMissingExiftool) return;
138	warnedMissingExiftool = true;
139	console.warn(
140	"exiftool is not installed; skipping gps metadata scrubbing entirely",
141	);
142	}
143
144	function testProgram(name: string) {
145	// spawnSync does not throw on a missing binary; it reports `error`
146	const result = child_process.spawnSync(name, ["-ver"]);
147	return result.error ? null : name;
148	}
149
150	import * as child_process from "node:child_process";
151	import * as fs from "node:fs";
152	import * as fsp from "node:fs/promises";
153	import * as stream from "node:stream";
154
155	import { Path } from "#sitegen/path";
156	import { UNWRAP } from "@clo/lib/assert";
157	import * as progress from "@clo/lib/progress";
158	import * as subprocess from "@clo/lib/subprocess";
159
160	import * as rules from "#src/file-viewer/rules.ts";

src/file-viewer/indexer/service.ts created+297

...	@@ -0,0 +1,297 @@
1	// The indexer service: owns the singleton progress root that all scanning
2	// and processing operations report into, watches the file store, runs
3	// periodic full sweeps, and pings web nodes when the database changes.
4	//
5	// One progress root for the lifetime of the process; it is never ended.
6	// `GET /progress` on the source of truth attaches encoders to it, which
7	// snapshot current state on connect and then stream deltas.
8	const console = log.scoped("indexer");
9
10	/** the singleton progress root. */
11	export const root = new progress.Root();
12
13	/** bumped on every database mutation; `GET /db` uses this to invalidate */
14	export let generation = 0;
15
16	// -- database change events --
17	// consumers (the /db/events sse route) subscribe here; the indexer emits a
18	// debounced beacon after changes. anything else that swaps the database
19	// (e.g. the /reload route) can emit through `emitDbChange`.
20	const dbSubscribers = new Set<(generation: number) => void>();
21
22	/** subscribe to database-changed beacons. dispose to unsubscribe. */
23	export function subscribeDbChange(fn: (generation: number) => void): ts.Dispose {
24	dbSubscribers.add(fn);
25	return ts.defer(() => void dbSubscribers.delete(fn));
26	}
27
28	/** notify all subscribers that the database changed */
29	export function emitDbChange() {
30	generation += 1;
31	for (const fn of dbSubscribers) fn(generation);
32	}
33
34	export interface ServiceOptions {
35	/** raw file store root. default: paths.rawFileRoot */
36	root?: string;
37	/** full sweep interval. default: CLOVER_SCAN_INTERVAL or 24h */
38	sweepIntervalMs?: number;
39	/** stability window before indexing a changed file */
40	settleMs?: number;
41	/** disable the file watcher (CLOVER_WATCH=0) */
42	watch?: boolean;
43	}
44
45	let started = false;
46
47	export function start(options: ServiceOptions = {}) {
48	ASSERT(!started, "indexer service started twice");
49	started = true;
50
51	const cores = Number(process.env.CLOVER_INDEX_CORES ?? 0);
52	if (cores > 0) queue.setConcurrency(cores);
53
54	const service = new Service(options);
55	service.init();
56	return service;
57	}
58
59	export class Service {
60	scanner: Scanner;
61	sweepIntervalMs: number;
62	watchEnabled: boolean;
63	watcher: fs.FSWatcher \| null = null;
64
65	// paths that changed according to the watcher, waiting to be scanned
66	#dirty = new Map<string, number>();
67	#dirtyTimer: NodeJS.Timeout \| null = null;
68	#sweeping = false;
69
70	constructor(options: ServiceOptions) {
71	this.scanner = new Scanner({
72	root: Path.resolve(options.root ?? paths.rawFileRoot),
73	progress: root,
74	settleMs: options.settleMs
75	?? Number(process.env.CLOVER_SETTLE_MS ?? 10_000),
76	onChange: (kind) => this.#onDbChange(kind),
77	onIdle: () => this.#onIdle(),
78	});
79	this.sweepIntervalMs = options.sweepIntervalMs
80	?? parseDuration(process.env.CLOVER_SCAN_INTERVAL ?? "24h");
81	this.watchEnabled = options.watch ?? process.env.CLOVER_WATCH !== "0";
82	}
83
84	init() {
85	void this.sweep("boot");
86	setInterval(() => void this.sweep("periodic"), this.sweepIntervalMs)
87	.unref();
88	if (this.watchEnabled) this.#startWatcher();
89	console.info(
90	`indexer service started (root: ${this.scanner.root}, `
91	+ `sweep every ${string.formatDurationLetters(this.sweepIntervalMs / 1000)}, `
92	+ `watch: ${this.watchEnabled})`,
93	);
94	}
95
96	/** run a full sweep (or a subtree scan when `subPath` is given) */
97	async sweep(reason: string, subPath?: string) {
98	if (this.#sweeping && !subPath) {
99	console.warn(`skipping ${reason} sweep; one is already running`);
100	return;
101	}
102	try {
103	if (!subPath) this.#sweeping = true;
104	const target = subPath
105	? this.scanner.root.join("." + path.posix.normalize("/" + subPath))
106	: this.scanner.root;
107	await this.scanner.scanPath(target);
108	this.#runDirMeta();
109	if (!subPath) await this.#maintenance();
110	} catch (err) {
111	console.error(`${reason} scan failed:`, err);
112	} finally {
113	if (!subPath) this.#sweeping = false;
114	}
115	}
116
117	// delete derived assets whose last referencing file is gone, plus tmp
118	// directories left behind by crashed producers. folded in from the old
119	// `file-trim` binary.
120	async #maintenance() {
121	using node = this.scanner.indexNode.start("maintenance");
122	const orphaned = derived.findOrphanedRoots();
123	for (const orphan of orphaned) {
124	node.text = `delete orphaned ${orphan.key}`;
125	await derived.deleteRootFiles(orphan);
126	derived.deleteRoot(orphan);
127	}
128	const cleaned = await derived.cleanAbandonedTmp();
129	if (orphaned.length + cleaned > 0) {
130	console.info(
131	`maintenance: ${orphaned.length} orphaned roots, ${cleaned} stale tmp dirs`,
132	);
133	this.#onDbChange("processed");
134	}
135	}
136
137	// -- file watcher --
138
139	#startWatcher() {
140	try {
141	this.watcher = fs.watch(
142	this.scanner.root.toString(),
143	{ recursive: true },
144	(_event, subPath) => subPath && this.#markDirty(subPath),
145	);
146	this.watcher.on("error", (err) => {
147	console.error("file watcher died, relying on periodic sweeps:", err);
148	this.watcher = null;
149	});
150	} catch (err) {
151	console.error("file watcher unavailable, relying on sweeps:", err);
152	this.watcher = null;
153	}
154	}
155
156	#markDirty(subPath: string) {
157	subPath = subPath.replaceAll("\\", "/");
158	// ignore events for paths the scanner would skip anyway (.DS_Store and
159	// friends), but also for anything inside a skipped directory.
160	if (subPath.split("/").some((part) => skipBasename(part))) return;
161	this.#dirty.set(subPath, Date.now());
162	this.#dirtyTimer ??= setTimeout(() => {
163	this.#dirtyTimer = null;
164	void this.#flushDirty();
165	}, watchDebounceMs);
166	}
167
168	// targets currently being scanned. a path that keeps emitting events (a
169	// large upload sitting in the settle gate) must not pile up concurrent
170	// scans of itself; it is re-checked once the in-flight scan completes.
171	#scanning = new Map<string, Promise<void>>();
172
173	async #flushDirty() {
174	if (this.#dirty.size === 0) return;
175	const all = [...this.#dirty.keys()];
176	this.#dirty.clear();
177
178	// coalesce: if a parent path is queued, skip its children. scanning is
179	// recursive for directories, so the parent covers them. comparisons are
180	// case-insensitive to match the file store's semantics; rename events
181	// can spell the same physical path two ways.
182	const sorted = all.sort();
183	const targets: string[] = [];
184	for (const p of sorted) {
185	const lp = p.toLowerCase();
186	const covered = targets.some((t) => {
187	const lt = t.toLowerCase();
188	return lp === lt \|\| lp.startsWith(lt + "/");
189	});
190	if (!covered) targets.push(p);
191	}
192
193	const fresh: string[] = [];
194	for (const target of targets) {
195	// defer paths already being scanned; the tail of the flush that owns
196	// them re-arms the timer, which picks these back up.
197	if (this.#scanning.has(target.toLowerCase())) {
198	this.#dirty.set(target, Date.now());
199	} else fresh.push(target);
200	}
201	if (fresh.length === 0) return;
202
203	// parallel: one file mid-upload (settling) must not block the others
204	await Promise.all(fresh.map((target) => {
205	const job = this.scanner
206	.scanPath(this.scanner.root.join(target))
207	.catch((err) => console.error(`watch scan of ${target} failed:`, err))
208	.finally(() => {
209	this.#scanning.delete(target.toLowerCase());
210	});
211	this.#scanning.set(target.toLowerCase(), job);
212	return job;
213	}));
214	this.#runDirMeta();
215
216	// events that arrived during the scans (including deferred re-checks of
217	// the paths scanned just now) get their own flush.
218	if (this.#dirty.size > 0) {
219	this.#dirtyTimer ??= setTimeout(() => {
220	this.#dirtyTimer = null;
221	void this.#flushDirty();
222	}, watchDebounceMs);
223	}
224	}
225
226	// -- directory metadata --
227
228	// the dir meta pass also runs after processors finish, since readme.txt
229	// contents arrive via the text-contents processor.
230	#onIdle() {
231	if (this.#dirMetaTimer) return;
232	this.#dirMetaTimer = setTimeout(() => {
233	this.#dirMetaTimer = null;
234	this.#runDirMeta();
235	}, 1000);
236	this.#dirMetaTimer.unref();
237	}
238	#dirMetaTimer: NodeJS.Timeout \| null = null;
239
240	#runDirMeta() {
241	try {
242	if (dirmeta.run(this.scanner.indexNode)) this.#onDbChange("metadata");
243	} catch (err) {
244	console.error("directory metadata pass failed:", err);
245	}
246	}
247
248	// -- database change beacons --
249	// consumers subscribe via /db/events and pull /db when beaconed. metadata
250	// changes (new files) flush fast; processor completions are batched
251	// coarsely since they arrive in bursts during encodes.
252
253	#beaconTimer: NodeJS.Timeout \| null = null;
254	#beaconDeadline = Infinity;
255
256	#onDbChange(kind: ChangeKind) {
257	const delay = kind === "metadata" ? metadataBeaconMs : processedBeaconMs;
258	const deadline = Date.now() + delay;
259	if (deadline < this.#beaconDeadline) {
260	this.#beaconDeadline = deadline;
261	if (this.#beaconTimer) clearTimeout(this.#beaconTimer);
262	this.#beaconTimer = setTimeout(() => {
263	this.#beaconTimer = null;
264	this.#beaconDeadline = Infinity;
265	emitDbChange();
266	}, delay);
267	this.#beaconTimer.unref();
268	}
269	}
270	}
271
272	export function parseDuration(text: string): number {
273	const match = text.match(/^(\d+(?:\.\d+)?)\s*(ms\|s\|m\|h\|d)?$/);
274	if (!match) throw new Error(`cannot parse duration: ${JSON.stringify(text)}`);
275	const scale = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 };
276	return Number(match[1]) * scale[(match[2] ?? "ms") as keyof typeof scale];
277	}
278
279	const watchDebounceMs = 1000;
280	const metadataBeaconMs = 2_000;
281	const processedBeaconMs = 30_000;
282
283	import * as fs from "node:fs";
284	import * as path from "node:path";
285
286	import { Path } from "#sitegen/path";
287	import { ASSERT } from "@clo/lib/assert";
288	import * as log from "@clo/lib/log";
289	import * as progress from "@clo/lib/progress";
290	import * as queue from "@clo/lib/queue";
291	import * as string from "@clo/lib/string";
292	import * as ts from "@clo/lib/ts";
293
294	import * as derived from "#src/file-viewer/models/derived.ts";
295	import * as paths from "#src/file-viewer/paths.ts";
296	import * as dirmeta from "./dirmeta.ts";
297	import { type ChangeKind, Scanner, skipBasename } from "./scan.ts";

src/file-viewer/models/FilePermissions.ts+7-5

 @@ -32,21 +32,23 @@ export class FilePermissions {
   }
 }
+// comparisons fold case to match the file store: a permission on
+// "/2026/friends" must also gate a request spelled "/2026/Friends"
 const getByPrefixQuery = db.prepare<
   [prefix: string],
   Pick<FilePermissions, "allow">
 >(/* SQL */ `
   select allow
   from permissions
-  where ? glob prefix || '*'
+  where lower(?) glob lower(prefix) || '*'
   order by length(prefix) desc
   limit 1;
 `);
 const getExactQuery = db.prepare<
   [file: string],
   Pick<FilePermissions, "allow">
 >(/* SQL */ `
-  select allow from permissions where ? == prefix
+  select allow from permissions where ? == prefix collate nocase
 `);
 const insertQuery = db.prepare<[{ prefix: string; allow: number }]>(/* SQL */ `

src/file-viewer/models/MediaFile.ts+86-53

 @@ -5,7 +5,7 @@ db.table(
   create table media_files (
     id integer primary key autoincrement,
     parent_id integer,
-    path text unique,
+    path text,
     kind integer not null,
     timestamp integer not null,
     timestamp_updated integer not null default current_timestamp,
 @@ -15,18 +15,20 @@ db.table(
     dimensions text not null default "",
     contents text not null,
     dirsort text,
-    processed integer not null,
+    config text not null default "",
-    processors text not null default "",
+    dir_reindex integer not null default 0,
-    foreign key (parent_id) references media_files(id)
+    pending integer not null default 0,
+    foreign key (parent_id) references media_files(id) on delete cascade
   );
-  -- index for quickly looking up files by path
+  -- path lookups fold case: the underlying file stores (zfs smb datasets,
-  create index media_files_path on media_files (path);
+  -- apfs) are case-insensitive, so two case spellings are one file
+  create unique index media_files_path on media_files (path collate nocase);
   -- index for quickly looking up children
   create index media_files_parent_id on media_files (parent_id);
   -- index for quickly looking up recursive file children
   create index media_files_file_children on media_files (kind, path);
-  -- index for finding directories that need to be processed
+  -- index for finding directories that need re-indexing
-  create index media_files_directory_processed on media_files (kind, processed);
+  create index media_files_dir_reindex on media_files (kind, dir_reindex);
 `,
 );
 @@ -73,14 +75,18 @@ export class MediaFile {
   /** in bytes */
   size!: number;
   /**
-   * 0 - not processed
+   * For directories, a JSON-encoded object derived from special files
-   * non-zero - processed
+   * (`.date` sets hideChildrenDates). Empty string otherwise.
-   *
+   */
-   * file: a bit-field of the processors.
+  config!: string;
-   * directory: this is for re-indexing contents
+  /** for directories: 1 when the metadata pass must revisit this dir */
+  dir_reindex!: number;
+  /**
+   * number of processors queued or running for this file. when zero, all
+   * derived data (duration, dimensions, contents, derived assets) is as
+   * complete as it will get. the UI uses this to show "still processing".
    */
-  processed!: number;
+  pending!: number;
-  processors!: string;
   // -- instance ops --
   get date() {
 @@ -134,14 +140,20 @@ export class MediaFile {
     ASSERT(result.kind === MediaFileKind.directory);
     return result;
   }
-  setProcessed(processed: number) {
+  setConfig(config: string) {
-    setProcessedQuery.run({ id: this.id, processed });
+    setConfigQuery.run({ id: this.id, config });
-    this.processed = processed;
+    this.config = config;
+  }
+  markDirReindex() {
+    markDirReindexQuery.run(this.id);
+    this.dir_reindex = 1;
+  }
+  setPending(pending: number) {
+    setPendingQuery.run({ id: this.id, pending });
+    this.pending = pending;
   }
-  setProcessors(processed: number, processors: string) {
+  decPending() {
-    setProcessorsQuery.run({ id: this.id, processed, processors });
+    this.pending = decPendingQuery.getNonNull(this.id).pending;
-    this.processed = processed;
-    this.processors = processors;
   }
   setDuration(duration: number) {
     setDurationQuery.run({ id: this.id, duration });
 @@ -162,6 +174,14 @@ export class MediaFile {
   delete() {
     deleteCascadeQuery.run({ id: this.id });
   }
+  /** adopt a new spelling of the same path (the file stores fold case) */
+  updatePath(newPath: string) {
+    if (this.kind === MediaFileKind.directory) {
+      updatePathPrefixQuery.run({ old: this.path + "/", new: newPath + "/" });
+    }
+    updatePathQuery.run({ id: this.id, path: newPath });
+    this.path = newPath;
+  }
   // -- static ops --
   static getByPath(filePath: string): MediaFile | null {
 @@ -179,7 +199,9 @@ export class MediaFile {
         contents: "the file scanner has not been run yet",
         dirsort: null,
         size: 0,
-        processed: 1,
+        config: "",
+        dir_reindex: 0,
+        pending: 0,
       });
     }
     return null;
 @@ -269,9 +291,6 @@ export class MediaFile {
       size,
     });
   }
-  static setProcessed(id: number, processed: number) {
-    setProcessedQuery.run({ id, processed });
-  }
   static createOrUpdateDirectory(dirPath: string) {
     const id = MediaFile.getOrPutDirectoryId(dirPath);
     return updateDirectoryQuery.get(id);
 @@ -323,15 +342,16 @@ const createDirectoryQuery = db.prepare<
   /* SQL */ `
   insert into media_files (
     path, parent_id, kind, timestamp, timestamp_updated, hash,
-    size, duration, dimensions, contents, dirsort, processed)
+    size, duration, dimensions, contents, dirsort, dir_reindex)
   values (
     $path, $parentId, ${MediaFileKind.directory}, 0, $time, '',
-    0, 0, '', '', '', 0)
+    0, 0, '', '', '', 1)
   returning id;
 `,
 );
 const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ `
-  SELECT id FROM media_files WHERE path = ? AND kind = ${MediaFileKind.directory};
+  SELECT id FROM media_files
+  WHERE path = ? collate nocase AND kind = ${MediaFileKind.directory};
 `);
 const createFileQuery = db.prepare<[{
   path: string;
 @@ -346,38 +366,41 @@ const createFileQuery = db.prepare<[{
 }], void>(/* SQL */ `
   insert into media_files (
     path, parent_id, kind, timestamp, timestamp_updated, hash,
-    size, duration, dimensions, contents, processed)
+    size, duration, dimensions, contents)
   values (
     $path, $parentId, ${MediaFileKind.file}, $timestamp, $timestampUpdated,
-    $hash, $size, $duration, $dimensions, $contents, 0)
+    $hash, $size, $duration, $dimensions, $contents)
-  on conflict(path) do update set
+  on conflict(path collate nocase) do update set
+    path = excluded.path,
     timestamp = excluded.timestamp,
     timestamp_updated = excluded.timestamp_updated,
+    hash = excluded.hash,
     duration = excluded.duration,
     size = excluded.size,
-    contents = excluded.contents,
+    contents = excluded.contents
-    processed = case
-        when media_files.hash != excluded.hash then 0
-        else media_files.processed
-    end
   returning *;
 `).as(MediaFile);
-const setProcessedQuery = db.prepare<[{
+const setConfigQuery = db.prepare<[{
   id: number;
-  processed: number;
+  config: string;
 }]>(/* SQL */ `
-  update media_files set processed = $processed where id = $id;
+  update media_files set config = $config where id = $id;
+`);
+const markDirReindexQuery = db.prepare<[id: number]>(/* SQL */ `
+  update media_files set dir_reindex = 1 where id = ?;
 `);
-const setProcessorsQuery = db.prepare<[{
+const setPendingQuery = db.prepare<[{
   id: number;
-  processed: number;
+  pending: number;
-  processors: string;
 }]>(/* SQL */ `
-  update media_files set
+  update media_files set pending = $pending where id = $id;
-    processed = $processed,
-    processors = $processors
-  where id = $id;
 `);
+const decPendingQuery = db.prepare<[id: number], { pending: number }>(
+  /* SQL */ `
+  update media_files set pending = max(0, pending - 1) where id = ?
+  returning pending;
+`,
+);
 const setDurationQuery = db.prepare<[{
   id: number;
   duration: number;
 @@ -397,8 +420,18 @@ const setContentsQuery = db.prepare<[{
   update media_files set contents = $contents where id = $id;
 `);
 const getByPathQuery = db.prepare<[string]>(/* SQL */ `
-  select * from media_files where path = ?;
+  select * from media_files where path = ? collate nocase;
 `).as(MediaFile);
+const updatePathQuery = db.prepare<[{ id: number; path: string }]>(/* SQL */ `
+  update media_files set path = $path where id = $id;
+`);
+const updatePathPrefixQuery = db.prepare<[{ old: string; new: string }]>(
+  /* SQL */ `
+  update media_files
+  set path = $new || substr(path, length($old) + 1)
+  where lower(substr(path, 1, length($old))) = lower($old);
+`,
+);
 const markDirectoryProcessedQuery = db.prepare<[{
   timestamp: number;
   contents: string;
 @@ -408,7 +441,7 @@ const markDirectoryProcessedQuery = db.prepare<[{
   id: number;
 }]>(/* SQL */ `
   update media_files set
-    processed = 1,
+    dir_reindex = 0,
     timestamp = $timestamp,
     contents = $contents,
     dirsort = $dirsort,
 @@ -417,7 +450,7 @@ const markDirectoryProcessedQuery = db.prepare<[{
   where id = $id;
 `);
 const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ `
-  update media_files set processed = 0 where id = ?;
+  update media_files set dir_reindex = 1 where id = ?;
 `);
 const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ `
 @@ -448,8 +481,8 @@ const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
 const getDirectoriesToReindexQuery = db.prepare(`
   with recursive directory_chain as (
     -- base case
     select id, parent_id, path from media_files
-    where kind = 0 and processed = 0
+    where kind = 0 and dir_reindex = 1
     -- recurse to find all parents so that size/hash can be updated
     union
     select m.id, m.parent_id, m.path

src/file-viewer/models/ProcessorState.ts created+146

...	@@ -0,0 +1,146 @@
1	// Tracks which processors have run on which files. Replaces the old
2	// `processed` bitfield + `processors` string columns on `media_files`.
3	//
4	// - `processors` is the registry: one row per known processor, with the
5	// version that the current code declares. rows for processors removed
6	// from the code are pruned (cascading their file state).
7	// - `file_processors` holds completions only. "pending" is the absence of
8	// a row, or a row with a stale version. failures are recorded with
9	// status=2 + the error text, and retried on the next sweep.
10	const db = getDb("cache.sqlite");
11	db.table(
12	"processor_state",
13	/* SQL */ `
14	create table if not exists processors (
15	id integer primary key autoincrement,
16	name text not null unique,
17	version integer not null
18	);
19	create table if not exists file_processors (
20	file integer not null references media_files(id) on delete cascade,
21	processor integer not null references processors(id) on delete cascade,
22	version integer not null,
23	status integer not null,
24	updated integer not null,
25	error text,
26	primary key (file, processor)
27	);
28	create index file_processors_processor on file_processors (processor);
29	`,
30	);
31
32	export enum ProcessorStatus {
33	done = 1,
34	failed = 2,
35	}
36
37	export interface FileProcessorRow {
38	file: number;
39	processor: number;
40	version: number;
41	status: ProcessorStatus;
42	updated: number;
43	error: string \| null;
44	}
45
46	/**
47	* upsert the registry and prune processors that no longer exist in code.
48	* returns a map of processor name to its row id, used by the scanner.
49	*/
50	export function syncRegistry(
51	defs: readonly { name: string; version: number }[],
52	): Map<string, number> {
53	const ids = new Map<string, number>();
54	const tx = db.node;
55	tx.exec("begin");
56	try {
57	for (const { name, version } of defs) {
58	const { id } = upsertProcessorQuery.getNonNull({ name, version });
59	ids.set(name, id);
60	}
61	const known = new Set(defs.map((d) => d.name));
62	for (const { id, name } of allProcessorsQuery.array()) {
63	if (!known.has(name)) deleteProcessorQuery.run(id);
64	}
65	tx.exec("commit");
66	} catch (err) {
67	tx.exec("rollback");
68	throw err;
69	}
70	return ids;
71	}
72
73	/** completion state for one file, keyed by processor row id */
74	export function getStates(
75	fileId: number,
76	): Map<number, Pick<FileProcessorRow, "version" \| "status">> {
77	const map = new Map<number, { version: number; status: ProcessorStatus }>();
78	for (const row of getStatesQuery.array(fileId)) {
79	map.set(row.processor, { version: row.version, status: row.status });
80	}
81	return map;
82	}
83
84	export function recordResult(
85	fileId: number,
86	processorId: number,
87	version: number,
88	status: ProcessorStatus,
89	error: string \| null = null,
90	) {
91	recordResultQuery.run({
92	file: fileId,
93	processor: processorId,
94	version,
95	status,
96	updated: Date.now(),
97	error,
98	});
99	}
100
101	/** the file's contents changed (hash mismatch); all processors must re-run */
102	export function invalidateFile(fileId: number) {
103	invalidateFileQuery.run(fileId);
104	}
105
106	// -- queries --
107	const upsertProcessorQuery = db.prepare<
108	[{ name: string; version: number }],
109	{ id: number }
110	>(/* SQL */ `
111	insert into processors (name, version) values ($name, $version)
112	on conflict(name) do update set version = excluded.version
113	returning id;
114	`);
115	const allProcessorsQuery = db.prepare<[], { id: number; name: string }>(
116	/* SQL */ `
117	select id, name from processors;
118	`,
119	);
120	const deleteProcessorQuery = db.prepare<[id: number]>(/* SQL */ `
121	delete from processors where id = ?;
122	`);
123	const getStatesQuery = db.prepare<[file: number], FileProcessorRow>(/* SQL */ `
124	select * from file_processors where file = ?;
125	`);
126	const recordResultQuery = db.prepare<[{
127	file: number;
128	processor: number;
129	version: number;
130	status: number;
131	updated: number;
132	error: string \| null;
133	}]>(/* SQL */ `
134	insert into file_processors (file, processor, version, status, updated, error)
135	values ($file, $processor, $version, $status, $updated, $error)
136	on conflict(file, processor) do update set
137	version = excluded.version,
138	status = excluded.status,
139	updated = excluded.updated,
140	error = excluded.error;
141	`);
142	const invalidateFileQuery = db.prepare<[file: number]>(/* SQL */ `
143	delete from file_processors where file = ?;
144	`);
145
146	import { getDb } from "#sitegen/sqlite";

src/file-viewer/models/derived.ts+59-3

...	@@ -39,7 +39,10 @@ db.table(	...	@@ -39,7 +39,10 @@ db.table(
39	`,	39	`,
40	);	40	);
41		41
42	export const workDir = Path.resolve(".clover/derived");	42	// derived assets are written directly into the derived file store. files are
		43	// produced into a `tmp.`-prefixed sibling directory and renamed into place,
		44	// so a crash never leaves a partially-written asset at its final path.
		45	export const workDir = Path.resolve(derivedFileRoot);
43		46
44	let ongoing = new Map<string, Promise<number>>();	47	let ongoing = new Map<string, Promise<number>>();
45		48
...	@@ -63,8 +66,13 @@ export async function produce(	...	@@ -63,8 +66,13 @@ export async function produce(
63	if (root) break brk;	66	if (root) break brk;
64	const { promise, resolve, reject } = Promise.withResolvers<number>();	67	const { promise, resolve, reject } = Promise.withResolvers<number>();
65	ongoing.set(key, promise);	68	ongoing.set(key, promise);
		69	// the rejection is rethrown to this caller; concurrent callers await
		70	// through `ongoing`, but when there are none the rejection would
		71	// otherwise be unobserved
		72	promise.catch(() => {});
		73	const finalDir = workDir.join(key);
		74	const tmp = workDir.join(`${file.hash}/tmp.${subkey}`);
66	try {	75	try {
67	const tmp = workDir.join(key);
68	node.hidden = true;	76	node.hidden = true;
69	const filesWithStats = await queue.run({	77	const filesWithStats = await queue.run({
70	cores,	78	cores,
...	@@ -106,6 +114,11 @@ export async function produce(	...	@@ -106,6 +114,11 @@ export async function produce(
106	);	114	);
107	},	115	},
108	});	116	});
		117	// move into the final location before recording rows; readers only
		118	// discover the asset through the database, so this is safe.
		119	await finalDir.delete({ recursive: true, force: true });
		120	await fsp.rename(tmp.toString(), finalDir.toString());
		121
109	db.node.exec("BEGIN");	122	db.node.exec("BEGIN");
110	try {	123	try {
111	root = insertRootQuery.getNonNull({ key, date: Date.now() }).id;	124	root = insertRootQuery.getNonNull({ key, date: Date.now() }).id;
...	@@ -126,6 +139,11 @@ export async function produce(	...	@@ -126,6 +139,11 @@ export async function produce(
126	resolve(root);	139	resolve(root);
127	} catch (e) {	140	} catch (e) {
128	if (root) deleteRootQuery.run({ root });	141	if (root) deleteRootQuery.run({ root });
		142	// remove partial output from disk: the tmp dir if the producer died,
		143	// or the renamed final dir if the database writes failed (e.g. the
		144	// source file was deleted while this asset was being produced)
		145	await tmp.delete({ recursive: true, force: true }).catch(() => {});
		146	await finalDir.delete({ recursive: true, force: true }).catch(() => {});
129	reject(e);	147	reject(e);
130	throw e;	148	throw e;
131	} finally {	149	} finally {
...	@@ -173,6 +191,42 @@ export function deleteRoot(root: { id: number }) {	...	@@ -173,6 +191,42 @@ export function deleteRoot(root: { id: number }) {
173	return deleteRootQuery.run({ root: root.id });	191	return deleteRootQuery.run({ root: root.id });
174	}	192	}
175		193
		194	/** remove an orphaned root's files from the derived store */
		195	export async function deleteRootFiles(root: { key: string }) {
		196	const dir = workDir.join(root.key);
		197	await dir.delete({ recursive: true, force: true });
		198	// reclaim the per-hash parent directory once its last root is gone
		199	// (rmdir refuses non-empty directories, which is exactly what we want)
		200	await fsp.rmdir(UNWRAP(dir.parent).toString()).catch(() => {});
		201	}
		202
		203	/**
		204	* delete `tmp.*` directories left behind by crashed producers. only removes
		205	* directories untouched for a day, to never race an ongoing producer.
		206	*
		207	* `tmp.*` FILES are never touched: encode intermediates inside completed
		208	* roots (`av1-au/tmp.av1.mp4`, ...) are intentionally named that way to be
		209	* excluded from `derived_files`, but the dash producer reads them across
		210	* root directories, so they must persist.
		211	*/
		212	export async function cleanAbandonedTmp(): Promise<number> {
		213	let cleaned = 0;
		214	const dayAgo = Date.now() - 24 * 60 * 60 * 1000;
		215	for (const hashDir of await workDir.readDir().catch(() => [])) {
		216	for (const sub of await hashDir.readDir().catch(() => [])) {
		217	if (!sub.base.startsWith("tmp.")) continue;
		218	try {
		219	const stat = await sub.stat();
		220	if (!stat.isDirectory()) continue;
		221	if (stat.mtime.getTime() > dayAgo) continue;
		222	await sub.delete({ recursive: true, force: true });
		223	cleaned += 1;
		224	} catch {}
		225	}
		226	}
		227	return cleaned;
		228	}
		229
176	const insertRootQuery = db.prepare<	230	const insertRootQuery = db.prepare<
177	[{ key: string; date: number }],	231	[{ key: string; date: number }],
178	{ id: number }	232	{ id: number }
...	@@ -214,9 +268,11 @@ const findOrphanedRootsQuery = db.prepare<	...	@@ -214,9 +268,11 @@ const findOrphanedRootsQuery = db.prepare<
214		268
215	import { Path } from "#sitegen/path";	269	import { Path } from "#sitegen/path";
216	import { getDb } from "#sitegen/sqlite";	270	import { getDb } from "#sitegen/sqlite";
217	import { ASSERT } from "@clo/lib/assert";	271	import { ASSERT, UNWRAP } from "@clo/lib/assert";
218	import * as progress from "@clo/lib/progress";	272	import * as progress from "@clo/lib/progress";
219	import * as queue from "@clo/lib/queue";	273	import * as queue from "@clo/lib/queue";
220	import * as crypto from "node:crypto";	274	import * as crypto from "node:crypto";
221	import * as fs from "node:fs";	275	import * as fs from "node:fs";
		276	import * as fsp from "node:fs/promises";
		277	import { derivedFileRoot } from "../paths.ts";
222	import type { MediaFile } from "./MediaFile.ts";	278	import type { MediaFile } from "./MediaFile.ts";

src/file-viewer/rules.ts+6

...	@@ -107,6 +107,12 @@ export const extsPreCompressed = new Set([	...	@@ -107,6 +107,12 @@ export const extsPreCompressed = new Set([
107	]);	107	]);
108	extsPreCompressed.delete(".svg");	108	extsPreCompressed.delete(".svg");
109		109
		110	// files that should never have media processors (transcodes) run on them,
		111	// usually because the file is intentionally cursed.
		112	export const processDenyList = new Set([
		113	"/2021/top-10000-bread/output.mp4",
		114	]);
		115
110	export function fileIcon(	116	export function fileIcon(
111	file: Pick<MediaFile, "kind" \| "basename" \| "path">,	117	file: Pick<MediaFile, "kind" \| "basename" \| "path">,
112	dirOpen?: boolean,	118	dirOpen?: boolean,

src/file-viewer/sync.ts created+160

...	@@ -0,0 +1,160 @@
1	// Keeps the web node's copy of cache.sqlite in sync with the source of
2	// truth. The SOT serves a server-sent-events feed at /db/events; this
3	// module holds it open and pulls /db on every beacon. The server emits a
4	// beacon on connect, so boot, reconnect, and missed-while-disconnected all
5	// resolve through the same path — there is no polling and no TTL.
6	//
7	// Pulls are ETag-guarded (no-change is a tiny 304; the etag is persisted so
8	// a reboot does not re-download an unchanged database), land in a temp
9	// file, and atomically rename over .clover/cache.sqlite before hot-swapping
10	// the open database handle.
11	const console = log.scoped("sync");
12
13	const token = process.env.CLOVER_SOT_KEY ?? null;
14	const enabled = process.env.CLOVER_DB_SYNC !== "0" && token != null;
15	const sotUrl = new URL(process.env.CLOVER_SOT_URL ?? "https://db.paperclover.net")
16	.toString().replace(/\/+$/g, "");
17	if (!enabled) {
18	console.warn(
19	"database sync disabled"
20	+ (token ? " (CLOVER_DB_SYNC=0)" : " (no CLOVER_SOT_KEY)"),
21	);
22	}
23
24	let inFlight: Promise<void> \| null = null;
25
26	export function revalidate(): Promise<void> {
27	return inFlight ??= revalidateInner().finally(() => {
28	inFlight = null;
29	});
30	}
31
32	async function revalidateInner(): Promise<void> {
33	const db = getDb("cache.sqlite");
34	const res = await fetch(`${sotUrl}/db`, {
35	headers: {
36	Authorization: token!,
37	...etag() ? { "If-None-Match": etag()! } : null,
38	},
39	signal: AbortSignal.timeout(120_000),
40	});
41	if (res.status === 304) return;
42	if (!res.ok \|\| !res.body) {
43	throw new Error(`source of truth responded ${res.status} ${res.statusText}`);
44	}
45
46	const tmp = db.file + ".tmp";
47	await stream.promises.pipeline(
48	stream.Readable.fromWeb(res.body as any),
49	fs.createWriteStream(tmp),
50	);
51	// fsync before the rename so a power cut cannot leave a torn file
52	const handle = await fsp.open(tmp, "r+");
53	await handle.sync();
54	await handle.close();
55	await fsp.rename(tmp, db.file);
56	db.reload();
57	saveEtag(res.headers.get("ETag"));
58	console.info(`database updated (${string.formatByteSize(Number(res.headers.get("Content-Length") ?? "0"))})`);
59	}
60
61	// -- the persisted etag, stored beside the database --
62	let cachedEtag: string \| null \| undefined;
63	function etagPath() {
64	return getDb("cache.sqlite").file + ".etag";
65	}
66	function etag(): string \| null {
67	if (cachedEtag === undefined) {
68	try {
69	cachedEtag = fs.readFileSync(etagPath(), "utf-8").trim() \|\| null;
70	} catch {
71	cachedEtag = null;
72	}
73	}
74	return cachedEtag;
75	}
76	function saveEtag(value: string \| null) {
77	cachedEtag = value;
78	try {
79	fs.writeFileSync(etagPath(), value ?? "");
80	} catch (err) {
81	console.warn("could not persist database etag:", err);
82	}
83	}
84
85	// -- the subscription --
86	// a hand-rolled sse reader over fetch (EventSource cannot send the
87	// Authorization header). any event block triggers a revalidate; a failed
88	// revalidate aborts the connection so the reconnect path retries it.
89	async function subscribeLoop() {
90	let backoff = 1000;
91	while (true) {
92	const ctrl = new AbortController();
93	let watchdog: NodeJS.Timeout \| null = null;
94	try {
95	const res = await fetch(`${sotUrl}/db/events`, {
96	headers: { Authorization: token! },
97	signal: ctrl.signal,
98	});
99	if (!res.ok \|\| !res.body) {
100	throw new Error(`${res.status} ${res.statusText}`);
101	}
102	console.info(`subscribed to database changes from ${sotUrl}`);
103	backoff = 1000;
104
105	// the server keepalives every 25s; 90s of silence means the
106	// connection died without a FIN
107	const armWatchdog = () => {
108	if (watchdog) clearTimeout(watchdog);
109	watchdog = setTimeout(
110	() => ctrl.abort(new Error("no keepalive")),
111	90_000,
112	);
113	watchdog.unref();
114	};
115	armWatchdog();
116	const reader = res.body.getReader();
117	const decoder = new TextDecoder();
118	let buffer = "";
119	while (true) {
120	const { value, done } = await reader.read();
121	if (done) break;
122	armWatchdog();
123	buffer += decoder.decode(value, { stream: true });
124	const blocks = buffer.split("\n\n");
125	buffer = blocks.pop()!;
126	const changed = blocks.some((block) =>
127	block.split("\n").some((line) => line.startsWith("event:") \|\| line.startsWith("data:"))
128	);
129	if (changed) {
130	void revalidate().catch((err) => {
131	console.error("revalidate failed:", err);
132	ctrl.abort(new Error("revalidate failed"));
133	});
134	}
135	}
136	throw new Error("stream ended");
137	} catch (err: any) {
138	console.warn(
139	`database subscription lost (${err?.message ?? err}); `
140	+ `retrying in ${Math.round(backoff / 1000)}s`,
141	);
142	} finally {
143	if (watchdog) clearTimeout(watchdog);
144	ctrl.abort();
145	}
146	await new Promise<void>((resolve) => setTimeout(resolve, backoff).unref());
147	backoff = Math.min(backoff * 2, 60_000);
148	}
149	}
150
151	if (enabled) void subscribeLoop();
152
153	import * as fs from "node:fs";
154	import * as fsp from "node:fs/promises";
155	import * as stream from "node:stream";
156
157	import { getDb } from "#sitegen/sqlite";
158
159	import * as log from "@clo/lib/log";
160	import * as string from "@clo/lib/string";

src/file-viewer/tags/media-dir.marko+1-1

...	@@ -27,7 +27,7 @@ static interface DirConfig {	...	@@ -27,7 +27,7 @@ static interface DirConfig {
27	return { sorted, activeFilename, readme };	27	return { sorted, activeFilename, readme };
28	})()>	28	})()>
29		29
30	<const/extraConfig: DirConfig=dir.processors ? JSON.parse(dir.processors) : {}>	30	<const/extraConfig: DirConfig=dir.config ? JSON.parse(dir.config) : {}>
31		31
32	<define/Inner>	32	<define/Inner>
33	<div.content.primary>	33	<div.content.primary>

src/file-viewer/tags/media-panel.marko+1-1

...	@@ -100,7 +100,7 @@ static const cotyledonEndingFile = "/2024/for everone";	...	@@ -100,7 +100,7 @@ static const cotyledonEndingFile = "/2024/for everone";
100	</div>	100	</div>
101		101
102	// content	102	// content
103	<const/ext=path.extname(file.path)>	103	<const/ext=path.extname(file.path).toLowerCase()>
104	<if=file.path === "/">	104	<if=file.path === "/">
105	<media-root-dir dir=file ...{ activeFilename, isLast, hasCotyledonCookie }/>	105	<media-root-dir dir=file ...{ activeFilename, isLast, hasCotyledonCookie }/>
106	</if>	106	</if>

src/file-viewer/tags/view-audio.marko+1-1

...	@@ -25,7 +25,7 @@ export interface Input {	...	@@ -25,7 +25,7 @@ export interface Input {
25	<audio controls preload="none" style="width: 100%">	25	<audio controls preload="none" style="width: 100%">
26	<source	26	<source
27	src="/file" + file.path	27	src="/file" + file.path
28	type=`audio/${path.extname(file.path).substring(1)}`	28	type=`audio/${path.extname(file.path).substring(1).toLowerCase()}`
29	>	29	>
30	</audio>	30	</audio>
31	//<if=lyricsFile>	31	//<if=lyricsFile>

src/file-viewer/tags/view-code.marko+1-2

...	@@ -9,8 +9,7 @@ export interface Input {	...	@@ -9,8 +9,7 @@ export interface Input {
9	<if=contents>	9	<if=contents>
10	<pre style="white-space:pre-wrap">$!{contents}</pre>	10	<pre style="white-space:pre-wrap">$!{contents}</pre>
11	</if>	11	</if>
12	<else if=size>	12	<else if=(size > 1_000_000)>
13	${" "}1_000_000>
14	<view-download-too-big ...input/>	13	<view-download-too-big ...input/>
15	</else>	14	</else>
16	<else>	15	<else>

src/file-viewer/tags/view-download.marko+1-1

...	@@ -28,7 +28,7 @@ export interface Input {	...	@@ -28,7 +28,7 @@ export interface Input {
28	<a href="/file" + file.path + "?dl" aria-label="download">${file.basename}</a>	28	<a href="/file" + file.path + "?dl" aria-label="download">${file.basename}</a>
29	</define>	29	</define>
30		30
31	<const/ext=path.extname(file.basename)>	31	<const/ext=path.extname(file.basename).toLowerCase()>
32	<if=ext === ".blend">	32	<if=ext === ".blend">
33	<p>	33	<p>
34	<code>.blend</code> files can be open in	34	<code>.blend</code> files can be open in

src/file-viewer/tags/view-text.marko+16-11

...	@@ -145,18 +145,23 @@ static function highlightHashComments(text: string) {	...	@@ -145,18 +145,23 @@ static function highlightHashComments(text: string) {
145	file: { path, basename, contents },	145	file: { path, basename, contents },
146	siblings,	146	siblings,
147	}=input>	147	}=input>
148	<pre style={ "white-space": "pre-wrap", "max-width": "70ch" }>	148	<if=!contents && input.file.pending>
		149	<p>this file was just added and is still being processed. come back soon!</p>
		150	</if>
		151	<else>
		152	<pre style={ "white-space": "pre-wrap", "max-width": "70ch" }>
149	$!{	153	$!{
150	path.startsWith("/2021/phoenix-write/maps") && basename === "map.txt"	154	path.startsWith("/2021/phoenix-write/maps") && basename === "map.txt"
151	? // special cased for phoenix write maps	155	? // special cased for phoenix write maps
152	highlightHashComments(contents)	156	highlightHashComments(contents)
153	: // normal	157	: // normal
154	highlightLinksInTextView(	158	highlightLinksInTextView(
155	contents,	159	contents,
156	siblings.filter((f) => f.kind === MediaFileKind.file),	160	siblings.filter((f) => f.kind === MediaFileKind.file),
157	)	161	)
158	}	162	}
159	</pre>	163	</pre>
		164	</else>
160		165
161	import * as string from "@clo/lib/string";	166	import * as string from "@clo/lib/string";
162	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";	167	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";

src/source-of-truth.dockerfile+22-22

...	@@ -1,31 +1,31 @@	...	@@ -1,31 +1,31 @@
1	from node:24 as builder	1	# The source of truth server: file serving + the auto-indexer service.
		2	# No bundling step; the whole source tree plus node_modules ship in the
		3	# image and `node run` executes it the same way development does.
		4	#
		5	# trixie rather than bookworm: the dimensions processor needs the `magick`
		6	# entry point, which is ImageMagick 7 (bookworm only packages v6).
		7	from node:24-trixie
2		8
3	run apt install -y git	9	# media processors shell out to these. exiftool is `libimage-exiftool-perl`
4		10	# on debian; curl is for the compose healthcheck.
5	workdir /paperclover.net	11	run apt-get update && apt-get install -y --no-install-recommends \
6	copy package*.json ./	12	ffmpeg \
7	run npm ci	13	imagemagick \
8	copy . ./	14	libimage-exiftool-perl \
9	run npx esbuild --bundle \	15	curl \
10	framework/backend/entry-node.ts \	16	&& rm -rf /var/lib/apt/lists/*
11	--platform=node \
12	--format=esm \
13	--define:globalThis.CLOVER_SERVER_ENTRY='"#src/source-of-truth.ts"' \
14	--minify \
15	--sourcemap=linked \
16	--entry-names=index \
17	--outdir=/app
18
19	from node:24 as runtime
20		17
21	workdir /app	18	workdir /app
22	copy --from=builder /app/ ./	19	copy package.json package-lock.json ./
		20	run npm ci --no-audit --no-fund
		21	copy . .
		22	# prime run.js's dependency hash so container boot never re-runs npm
		23	run node run \|\| true
23		24
24	env PORT=43200	25	env PORT=80
25	# must be configured	26	# must be configured
26	env CLOVER_DB=/dev/null	27	env CLOVER_DB=/dev/null
27	env CLOVER_FILE_RAW=/dev/null	28	env CLOVER_FILE_RAW=/dev/null
28	env CLOVER_FILE_DERIVED=/dev/null	29	env CLOVER_FILE_DERIVED=/dev/null
29		30
30	cmd ["node", "--enable-source-maps", "/app/index.js"]	31	cmd ["node", "run", "source-of-truth"]
31

src/source-of-truth.ts+165-3

...	@@ -22,7 +22,7 @@	...	@@ -22,7 +22,7 @@
22	*/	22	*/
23	const app = new Hono();	23	const app = new Hono();
24	export default app;	24	export default app;
25	export const port = 4000;	25	export const port = Number(process.env.PORT ?? 4000);
26		26
27	const token = UNWRAP(process.env.CLOVER_SOT_KEY);	27	const token = UNWRAP(process.env.CLOVER_SOT_KEY);
28		28
...	@@ -33,6 +33,26 @@ if (!fs.existsSync(derivedFileRoot)) {	...	@@ -33,6 +33,26 @@ if (!fs.existsSync(derivedFileRoot)) {
33	throw new Error(`${derivedFileRoot} does not exist`);	33	throw new Error(`${derivedFileRoot} does not exist`);
34	}	34	}
35		35
		36	/**
		37	* `node run source-of-truth` boots the full source of truth: the indexer
		38	* service (file watcher, periodic sweeps, processors) plus this http server.
		39	* set CLOVER_INDEXER=0 for a serve-only process.
		40	*/
		41	export async function main() {
		42	if (process.env.CLOVER_INDEXER !== "0") {
		43	service = indexer.start();
		44	}
		45	const server = await http.serve({
		46	respond: async (request) => app.fetch(request),
		47	port,
		48	});
		49	console.info(`source of truth live at ${server.url}`);
		50	const close = () => void server.close().then(() => process.exit(0));
		51	process.on("SIGINT", close);
		52	process.on("SIGTERM", close);
		53	}
		54	let service: indexer.Service \| null = null;
		55
36	// Re-use file descriptors if the same file is being read twice.	56	// Re-use file descriptors if the same file is being read twice.
37	const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();	57	const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();
38		58
...	@@ -52,6 +72,13 @@ app.get("/file/*", async (c) => {	...	@@ -52,6 +72,13 @@ app.get("/file/*", async (c) => {
52	}	72	}
53	const file = MediaFile.getByPath(filePath);	73	const file = MediaFile.getByPath(filePath);
54	if (!file \|\| file.kind === MediaFileKind.directory) return c.notFound();	74	if (!file \|\| file.kind === MediaFileKind.directory) return c.notFound();
		75	// derived asset names are plain relative paths under the hash directory
		76	if (
		77	derivedAsset
		78	&& derivedAsset.split("/").some((part) => part === "" \|\| part === "." \|\| part === "..")
		79	) {
		80	return c.notFound();
		81	}
55	const fullPath = derivedAsset	82	const fullPath = derivedAsset
56	? path.join(derivedFileRoot, file.hash, derivedAsset)	83	? path.join(derivedFileRoot, file.hash, derivedAsset)
57	: path.join(rawFileRoot, file.path);	84	: path.join(rawFileRoot, file.path);
...	@@ -65,8 +92,8 @@ app.get("/file/*", async (c) => {	...	@@ -65,8 +92,8 @@ app.get("/file/*", async (c) => {
65	fds.delete(fullPath);	92	fds.delete(fullPath);
66	throw err;	93	throw err;
67	});	94	});
68	fds.set(file.path, promise);	95	fds.set(fullPath, promise);
69	fds.set(file.path, handle = await promise);	96	fds.set(fullPath, handle = await promise);
70	}	97	}
71	handle.refs += 1;	98	handle.refs += 1;
72	} catch (err: any) {	99	} catch (err: any) {
...	@@ -97,18 +124,153 @@ app.post("/reload", async (c) => {	...	@@ -97,18 +124,153 @@ app.post("/reload", async (c) => {
97	return c.json({ error: "invalid authorization header" }, 401);	124	return c.json({ error: "invalid authorization header" }, 401);
98	}	125	}
99	MediaFile.db.reload();	126	MediaFile.db.reload();
		127	indexer.emitDbChange();
100	return c.body(null, 204);	128	return c.body(null, 204);
101	});	129	});
102		130
		131	// a tiny server-sent-events feed of database changes. consumers (web nodes,
		132	// dev servers) hold this open and pull /db whenever a beacon arrives; the
		133	// connection itself is the subscription, so the source of truth never needs
		134	// a list of its consumers. events carry the generation, but clients can
		135	// treat any event as "something changed" — /db pulls are ETag-guarded.
		136	app.get("/db/events", (c) => {
		137	if (c.req.header("Authorization") !== token) {
		138	return c.json({ error: "invalid authorization header" }, 401);
		139	}
		140	const encoder = new TextEncoder();
		141	let cleanup: (() => void) \| null = null;
		142	const stream = new ReadableStream<Uint8Array>({
		143	start(controller) {
		144	const send = (text: string) => controller.enqueue(encoder.encode(text));
		145	// an event on connect makes clients revalidate immediately, covering
		146	// anything they missed while disconnected
		147	send(`retry: 5000\n\n`);
		148	send(`event: change\ndata: ${indexer.generation}\n\n`);
		149	const unsubscribe = indexer.subscribeDbChange((generation) => {
		150	send(`event: change\ndata: ${generation}\n\n`);
		151	});
		152	// keepalive comments so proxies and clients hold the connection open
		153	const keepalive = setInterval(() => send(`: keepalive\n\n`), 25_000);
		154	keepalive.unref();
		155	cleanup = () => {
		156	unsubscribe[Symbol.dispose]();
		157	clearInterval(keepalive);
		158	};
		159	},
		160	cancel() {
		161	cleanup?.();
		162	},
		163	});
		164	c.header("Content-Type", "text/event-stream");
		165	c.header("Cache-Control", "no-store");
		166	return c.body(stream);
		167	});
		168
		169	// live progress of all indexing operations, as a clover progress stream.
		170	// each request snapshots the current tree state and then follows along;
		171	// connect any time with `node run tail-progress`.
		172	app.get("/progress", (c) => {
		173	if (c.req.header("Authorization") !== token) {
		174	return c.json({ error: "invalid authorization header" }, 401);
		175	}
		176	c.header("Content-Type", progress.contentType);
		177	return c.body(progress.encodeByteStream(indexer.root) as ReadableStream);
		178	});
		179
		180	// download a consistent snapshot of cache.sqlite. web nodes call this with
		181	// `If-None-Match` on a stale-while-revalidate schedule, and immediately
		182	// when pinged at /internal/db-changed.
		183	app.get("/db", async (c) => {
		184	if (c.req.header("Authorization") !== token) {
		185	return c.json({ error: "invalid authorization header" }, 401);
		186	}
		187	const snap = await getDbSnapshot();
		188	if (c.req.header("If-None-Match") === snap.etag) {
		189	return c.body(null, 304);
		190	}
		191	c.header("Content-Type", "application/vnd.sqlite3");
		192	c.header("Content-Length", String(snap.size));
		193	c.header("ETag", snap.etag);
		194	return c.body(
		195	stream.Readable.toWeb(fs.createReadStream(snap.path)) as ReadableStream,
		196	);
		197	});
		198
		199	// trigger a sweep (or a subtree scan with {"path": "/2025"}). progress is
		200	// visible on /progress; this returns immediately.
		201	app.post("/scan", async (c) => {
		202	if (c.req.header("Authorization") !== token) {
		203	return c.json({ error: "invalid authorization header" }, 401);
		204	}
		205	if (!service) return c.json({ error: "indexer is disabled" }, 409);
		206	const body = await c.req.json().catch(() => ({}));
		207	const subPath = typeof body.path === "string" ? body.path : undefined;
		208	void service.sweep("manual", subPath);
		209	return c.json({ ok: true }, 202);
		210	});
		211
		212	// -- database snapshotting --
		213	// `VACUUM INTO` writes a compact, transactionally-consistent copy that does
		214	// not depend on -wal sidecar files. regenerated lazily when the indexer
		215	// reports changes, at most every 30 seconds.
		216	interface DbSnapshot {
		217	gen: number;
		218	time: number;
		219	path: string;
		220	etag: string;
		221	size: number;
		222	}
		223	let snapshot: DbSnapshot \| null = null;
		224	let snapshotInFlight: Promise<DbSnapshot> \| null = null;
		225
		226	function getDbSnapshot() {
		227	const gen = indexer.generation;
		228	// a stale-generation snapshot is never served: subscribers pull exactly
		229	// once per beacon, so a 304 here would leave them out of date until the
		230	// ttl backstop. beacons are debounced upstream, which bounds how often
		231	// the vacuum can actually run.
		232	if (snapshot && snapshot.gen === gen && fs.existsSync(snapshot.path)) {
		233	return Promise.resolve(snapshot);
		234	}
		235	return snapshotInFlight ??= (async () => {
		236	try {
		237	const dir = process.env.CLOVER_DB ?? ".clover";
		238	const file = path.join(dir, "db-snapshot.sqlite");
		239	await fs.rm(file, { force: true });
		240	MediaFile.db.node.exec(
		241	`vacuum into '${file.replaceAll("'", "''")}';`,
		242	);
		243	const [hash, { size }] = await Promise.all([
		244	hashFile(new Path(file)),
		245	fs.stat(file),
		246	]);
		247	return snapshot = {
		248	gen,
		249	time: Date.now(),
		250	path: file,
		251	etag: `"${hash}"`,
		252	size,
		253	};
		254	} finally {
		255	snapshotInFlight = null;
		256	}
		257	})();
		258	}
		259
103	const openFile = util.promisify(fsCallbacks.open);	260	const openFile = util.promisify(fsCallbacks.open);
104	const closeFile = util.promisify(fsCallbacks.close);	261	const closeFile = util.promisify(fsCallbacks.close);
105	const fstat = util.promisify(fsCallbacks.fstat);	262	const fstat = util.promisify(fsCallbacks.fstat);
106		263
107	import * as fs from "#sitegen/fs";	264	import * as fs from "#sitegen/fs";
		265	import { Path } from "#sitegen/path";
		266	import { hashFile } from "#src/file-viewer/indexer/scan.ts";
		267	import * as indexer from "#src/file-viewer/indexer/service.ts";
108	import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";	268	import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";
109	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";	269	import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";
110	import { ASSERT, UNWRAP } from "@clo/lib/assert";	270	import { ASSERT, UNWRAP } from "@clo/lib/assert";
		271	import * as http from "@clo/lib/http";
111	import * as mime from "@clo/lib/mime";	272	import * as mime from "@clo/lib/mime";
		273	import * as progress from "@clo/lib/progress";
112	import { Hono } from "hono";	274	import { Hono } from "hono";
113	import * as fsCallbacks from "node:fs";	275	import * as fsCallbacks from "node:fs";
114	import * as path from "node:path";	276	import * as path from "node:path";

src/tags/clover-media.marko+32-20

 @@ -14,28 +14,13 @@ export interface Input {
 <else if=extsVideo.has(file.extension.toLowerCase())>
   <clover-video minimal noFigure ...{ file }/>
 </else>
-<else if=file.extension === ".gif">
+<else if=file.extension.toLowerCase() === ".gif">
   <img src=`/file${escapeUri(file.path)}` ...rest>
 </else>
 <else>
   <const/base=`/file${escapeUri(file.path)}`>
-  <const/{ width }=UNWRAP(file.parseDimensions())>
+  <const/dims=file.parseDimensions()>
-  <const/targetSizes=transcodeRules.imageSizes.filter((w) => w < width)>
+  <if=!dims || file.pending>
-  <const/sizes=(
-    displayWidthEstimate
-      ? typeof displayWidthEstimate === "number"
-        ? `${displayWidthEstimate}px`
-        : displayWidthEstimate
-      : `100vw`
-  )>
-  <picture>
-    <for|ext| of=["jxl", "webp"]>
-      <source
-        type=`image/${ext}`
-        srcset=targetSizes.map((s) => `${base}:/${s}.${ext} ${s}w`)
-        ...{ sizes }
-      >
-    </for>
     <img
       src=base
       style=(
 @@ -45,11 +30,38 @@ export interface Input {
       )
       ...rest
     >
-  </picture>
+  </if>
+  <else>
+    <const/targetSizes=transcodeRules.imageSizes.filter((w) => w < dims.width)>
+    <const/sizes=(
+      displayWidthEstimate
+        ? typeof displayWidthEstimate === "number"
+          ? `${displayWidthEstimate}px`
+          : displayWidthEstimate
+        : `100vw`
+    )>
+    <picture>
+      <for|ext| of=["jxl", "webp"]>
+        <source
+          type=`image/${ext}`
+          srcset=targetSizes.map((s) => `${base}:/${s}.${ext} ${s}w`).join(",")
+          ...{ sizes }
+        >
+      </for>
+      <img
+        src=base
+        style=(
+          align
+            ? { "object-position": align, width: "100%", height: "100%" }
+            : undefined
+        )
+        ...rest
+      >
+    </picture>
+  </else>
 </else>
 import * as transcodeRules from "#src/file-viewer/transcode-rules.ts";
 import { extsVideo } from "#src/file-viewer/rules.ts";
 import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
-import { UNWRAP } from "@clo/lib/assert";
 import { escapeUri } from "#src/file-viewer/format.ts";

src/tags/clover-video.client.ts+55-15

...	@@ -65,21 +65,7 @@ function decidePlaybackMode(): Promise<PlaybackMode> {	...	@@ -65,21 +65,7 @@ function decidePlaybackMode(): Promise<PlaybackMode> {
65	console.warn(`does not implement av1+opus`);	65	console.warn(`does not implement av1+opus`);
66	}	66	}
67		67
68	return canNativelyPlayHls().then((nativeHls): PlaybackMode => {	68	return decideHlsMode();
69	if (nativeHls) return "hls-native";
70	// @ts-expect-error
71	return (hls ??= import("/js/scripts/vendor/hls.js"))
72	.then((hls): PlaybackMode => {
73	// Polyfill HLS (Chrome 23, Firefox 42, IE11 on Win8)
74	// TODO: ES modules have a minimum of Chrome 63 or Firefox 60
75	if (hls?.default?.isSupported?.() \|\| hls?.isSupported()) {
76	return "hls-polyfill";
77	}
78	console.warn("does not support hls.js");
79	// Other browsers
80	return "none";
81	});
82	});
83	})	69	})
84	.catch((e): PlaybackMode => {	70	.catch((e): PlaybackMode => {
85	console.warn(e);	71	console.warn(e);
...	@@ -92,6 +78,28 @@ function decidePlaybackMode(): Promise<PlaybackMode> {	...	@@ -92,6 +78,28 @@ function decidePlaybackMode(): Promise<PlaybackMode> {
92	});	78	});
93	}	79	}
94		80
		81	// the hls fallback decision is also needed by av1-capable browsers when a
		82	// video has an hls manifest but no dash one (dash encodes can fail or lag)
		83	function decideHlsMode(): Promise<PlaybackMode> {
		84	return canNativelyPlayHls().then(
		85	(nativeHls): PlaybackMode \| Promise<PlaybackMode> => {
		86	if (nativeHls) return "hls-native";
		87	// @ts-expect-error
		88	return (hls ??= import("/js/scripts/vendor/hls.js"))
		89	.then((hls): PlaybackMode => {
		90	// Polyfill HLS (Chrome 23, Firefox 42, IE11 on Win8)
		91	// TODO: ES modules have a minimum of Chrome 63 or Firefox 60
		92	if (hls?.default?.isSupported?.() \|\| hls?.isSupported()) {
		93	return "hls-polyfill";
		94	}
		95	console.warn("does not support hls.js");
		96	// Other browsers
		97	return "none";
		98	});
		99	},
		100	);
		101	}
		102
95	function hydrateVideoInner(	103	function hydrateVideoInner(
96	container: HTMLElement,	104	container: HTMLElement,
97	mode: PlaybackMode,	105	mode: PlaybackMode,
...	@@ -109,6 +117,38 @@ function hydrateVideoInner(	...	@@ -109,6 +117,38 @@ function hydrateVideoInner(
109	const dashFile = `${src}:/dash.mpd`;	117	const dashFile = `${src}:/dash.mpd`;
110	const hlsFile = `${src}:/master.m3u8`;	118	const hlsFile = `${src}:/master.m3u8`;
111		119
		120	// data-streams lists which manifests exist ("dash", "hls", or both). a
		121	// manifest may be missing because the indexer has not finished the file
		122	// yet ("pending"), it skipped it (short/tiny videos, "none"), or that
		123	// encode failed. never pick a backend whose manifest would 404; fall back
		124	// to playing the original file directly.
		125	const streams = (video.getAttribute("data-streams") ?? "").split(",");
		126	const hasDash = streams.includes("dash");
		127	const hasHls = streams.includes("hls");
		128	if (!hasDash && !hasHls) {
		129	if (streams.includes("pending")) {
		130	const note = document.createElement("p");
		131	note.className = "clover-video-pending";
		132	note.textContent = "this video is still being processed, so quality selection is "
		133	+ "unavailable. if it does not play, come back later :(";
		134	(container.querySelector("figcaption") ?? video.parentElement)
		135	?.appendChild(note);
		136	}
		137	video.src = src;
		138	video.controls = true;
		139	onCloverVideoInit?.(id, video);
		140	return;
		141	}
		142	if (mode === "av1" && !hasDash) {
		143	// av1-capable browser, but only an hls manifest exists for this video
		144	decideHlsMode().then((m) => hydrateVideoInner(container, m));
		145	return;
		146	}
		147	if ((mode === "hls-native" \|\| mode === "hls-polyfill") && !hasHls) {
		148	// dash-only video on a browser that cannot decode av1: raw playback
		149	mode = "none";
		150	}
		151
112	if (forceMode) {	152	if (forceMode) {
113	const figcaption = container.querySelector("figcaption");	153	const figcaption = container.querySelector("figcaption");
114	if (figcaption) {	154	if (figcaption) {

src/tags/clover-video.marko+18-1

...	@@ -29,13 +29,29 @@ export interface Input {	...	@@ -29,13 +29,29 @@ export interface Input {
29	: null	29	: null
30	)>	30	)>
31		31
32	<const/{ width, height }=file.parseDimensions()>	32	<const/{ width, height }=file.parseDimensions() ?? {
		33	width: 1920,
		34	height: 1080,
		35	}>
		36	<!-- which streaming manifests exist ("dash", "hls", or both). the player
		37	must only pick a backend whose manifest is actually present: dash can fail
		38	or lag behind hls for the same file. "pending" tells it to show a
		39	come-back-later note instead of silently failing -->
		40	<const/streams=(
		41	[
		42	derived.get(file, "dash.mpd") && "dash",
		43	derived.get(file, "master.m3u8") && "hls",
		44	]
		45	.filter(Boolean)
		46	.join(",") \|\| (file.pending ? "pending" : "none")
		47	)>
33		48
34	<define/Inner>	49	<define/Inner>
35	<video	50	<video
36	...{ id }	51	...{ id }
37	class={ "clover-video": noFigure, minimal: noFigure && minimal }	52	class={ "clover-video": noFigure, minimal: noFigure && minimal }
38	data-src=file.path	53	data-src=file.path
		54	data-streams=streams
39	poster=poster ? `/file${poster.path}` : null	55	poster=poster ? `/file${poster.path}` : null
40	width=width	56	width=width
41	height=height	57	height=height
...	@@ -83,4 +99,5 @@ export interface Input {	...	@@ -83,4 +99,5 @@ export interface Input {
83	client import "./clover-video.client.ts";	99	client import "./clover-video.client.ts";
84		100
85	import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";	101	import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
		102	import * as derived from "#src/file-viewer/models/derived.ts";
86	import { UNWRAP, ASSERT } from "@clo/lib/assert";	103	import { UNWRAP, ASSERT } from "@clo/lib/assert";

author	clover caruso <git@paperclover.net> 2026-06-09 23:34:10-07:00
committer	clover caruso <git@paperclover.net> 2026-06-12 18:10:53-07:00
log	12f1b96207363924cca0970b5e6cb515a6d7541a
tree	6a7082de6ca703b06e9f72831b15fe16276e7c24
parent	b6380f3a43e39e9571af27722d728fc17327d472
signature	Signed by SSH key SHA256:cOKiuRFOeSRxne6EWgHtdQQSlBxjOXm2hOCFnCdLQbQ

feat: watcher-based file indexing