1// Run one full scan of the file store from this machine, rendering progress
2// in the terminal. In production the indexer service inside the source of
3// truth server does this continuously; this wrapper exists for development
4// (and the nostalgia of watching it go).
5//
6// CLOVER_FILE_RAW=/tmp/store CLOVER_FILE_DERIVED=/tmp/derived \
7// CLOVER_DB=.clover node run file-scan
8export async function main() {
9 const start = performance.now();
10 using _ = log.startWidget({
11 format: ({ now }) => `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
12 });
13
14 const scanner = new Scanner({
15 root: Path.resolve(rawFileRoot),
16 progress: progress.globalRoot,
17 settleMs: Number(process.env.CLOVER_SETTLE_MS ?? 10_000),
18 });
19 await scanner.sweep();
20 await scanner.waitIdle();
21 dirmeta.run(progress.globalRoot);
22
23 const orphaned = derived.findOrphanedRoots();
24 for (const orphan of orphaned) {
25 console.info("delete orphaned " + orphan.key);
26 await derived.deleteRootFiles(orphan);
27 derived.deleteRoot(orphan);
28 }
29 await derived.cleanAbandonedTmp();
30
31 console.info(
32 "Updated file viewer index in \x1b[1m"
33 + ((performance.now() - start) / 1000).toFixed(1)
34 + "s\x1b[0m",
35 );
36
37 const { duration, count } = MediaFile.db
38 .prepare<[], { count: number; duration: number }>(`
39 select
40 count(*) as count,
41 sum(duration) as duration
42 from media_files
43 `)
44 .getNonNull();
45 const { derivedSize, derivedCount } = MediaFile.db
46 .prepare<[], { derivedCount: number; derivedSize: number }>(`
47 select
48 count(*) as derivedCount,
49 sum(size) as derivedSize
50 from derived_files
51 `)
52 .getNonNull();
53 const { failed } = MediaFile.db
54 .prepare<[], { failed: number }>(`
55 select count(*) as failed from file_processors where status = 2
56 `)
57 .getNonNull();
58 const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
59
60 console.info();
61 console.info(
62 "Global Stats:\n"
63 + `- File Count: \x1b[1m${count}\x1b[0m\n`
64 + `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n`
65 + `- Media Duration: \x1b[1m${string.formatDurationLetters(duration)}\x1b[0m\n`
66 + `- Canonical Size: \x1b[1m${string.formatByteSize(canonicalSize)}\x1b[0m\n`
67 + `- Derived Size: \x1b[1m${string.formatByteSize(derivedSize)}\x1b[0m\n`
68 + (failed > 0
69 ? `- \x1b[31mFailed Processors: ${failed}\x1b[0m (select * from file_processors where status = 2)\n`
70 : ""),
71 );
72}
73
74import * as log from "@clo/lib/log";
75import * as progress from "@clo/lib/progress";
76import * as string from "@clo/lib/string";
77
78import { Path } from "#sitegen/path";
79import { UNWRAP } from "@clo/lib/assert";
80
81import * as dirmeta from "#src/file-viewer/indexer/dirmeta.ts";
82import { Scanner } from "#src/file-viewer/indexer/scan.ts";
83import * as derived from "#src/file-viewer/models/derived.ts";
84import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
85import { rawFileRoot } from "../paths.ts";