authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-11-07 00:28:14-08:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-11-16 14:39:58-08:00
log2bed05bdbc03b105f930feed4573a5b0768bab07
treea7be464b8d6a5fcaed1d91cb6fba1180cbce8276
parent29751666e4e62269bc3f48767bf726ac5789ced7
signature Commit is signed but in an unrecognized format.

feat(site): work to release october personal blog

18 files changed, 1289 insertions(+), 1147 deletions(-)

flake.nix+2
...@@ -13,9 +13,11 @@...@@ -13,9 +13,11 @@
13 pkgs.nodejs_24 # runtime13 pkgs.nodejs_24 # runtime
14 pkgs.deno # formatter14 pkgs.deno # formatter
15 pkgs.python3 # for font subsetting15 pkgs.python3 # for font subsetting
16 pkgs.python3Packages.pip # for font subsetting
1617
17 # paperclover.net18 # paperclover.net
18 pkgs.exiftool19 pkgs.exiftool
20 pkgs.imagemagick
19 pkgs.ffmpeg-full21 pkgs.ffmpeg-full
20 pkgs.rsync22 pkgs.rsync
21 ];23 ];
lib/progress.ts+4-4
...@@ -154,9 +154,9 @@ function newNode(...@@ -154,9 +154,9 @@ function newNode(
154 parent?.sortChildren && parent.children.sort(parent.sortChildren);154 parent?.sortChildren && parent.children.sort(parent.sortChildren);
155 state.signal("draw");155 state.signal("draw");
156 }156 }
157 const scope = log.headlessScope(() => {157 // const scope = log.headlessScope((msg) => {
158 console.log("TODO");158 // console.log("TODO");
159 });159 // });
160 const binding: Node = {160 const binding: Node = {
161 start(text, opts) {161 start(text, opts) {
162 const [child, node] = newNode(text, opts);162 const [child, node] = newNode(text, opts);
...@@ -233,7 +233,7 @@ function newNode(...@@ -233,7 +233,7 @@ function newNode(
233 mutate();233 mutate();
234 }234 }
235 },235 },
236 log: scope,236 log: log.scoped(""),
237 end: () => void end(),237 end: () => void end(),
238 [Symbol.dispose]: () => void end(),238 [Symbol.dispose]: () => void end(),
239 };239 };
src/bin/download-db.ts created+26
...@@ -0,0 +1,26 @@
1export async function main() {
2 await rsync.spawn({
3 cwd: Path.resolve(import.meta.dirname).parent!.toString(),
4 args: [
5 "-a",
6 "--progress",
7 "clo@paperclover.net:~/paperclover.net/.clover/cache.sqlite",
8 ".clover/",
9 ],
10 progress: progress.start("download file cache"),
11 });
12 await rsync.spawn({
13 cwd: Path.resolve(import.meta.dirname).parent!.toString(),
14 args: [
15 "-a",
16 "--progress",
17 "clo@paperclover.net:~/paperclover.net/.clover/questions.sqlite",
18 ".clover/",
19 ],
20 progress: progress.start("download questions state"),
21 });
22}
23
24import * as progress from "lib/progress.ts";
25import * as rsync from "../file-viewer/rsync.ts";
26import { Path } from "#sitegen/path";
src/file-viewer/backend.tsx+3-2
...@@ -45,7 +45,8 @@ app.get("/file/*", async (c, next) => {...@@ -45,7 +45,8 @@ app.get("/file/*", async (c, next) => {
45 // The permissions system is currently binary, using the friend auth.45 // The permissions system is currently binary, using the friend auth.
46 const permissions = FilePermissions.getByPrefix(rawFilePath);46 const permissions = FilePermissions.getByPrefix(rawFilePath);
47 if (permissions !== 0) {47 if (permissions !== 0) {
48 const friendAuthChallenge = requireFriendAuth(c);48 console.log(rawFilePath, getForFile(rawFilePath));
49 const friendAuthChallenge = requireFriendAuth(c, getForFile(rawFilePath));
49 if (friendAuthChallenge) return friendAuthChallenge;50 if (friendAuthChallenge) return friendAuthChallenge;
50 }51 }
5152
...@@ -292,7 +293,7 @@ import { etagMatches, serveAsset } from "#sitegen/assets";...@@ -292,7 +293,7 @@ import { etagMatches, serveAsset } from "#sitegen/assets";
292import * as view from "#sitegen/view";293import * as view from "#sitegen/view";
293import * as mime from "lib/mime.ts";294import * as mime from "lib/mime.ts";
294295
295import { requireFriendAuth } from "@/friend-auth.ts";296import { getForFile, requireFriendAuth } from "@/friend-auth.ts";
296import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";297import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
297import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";298import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
298import { MediaPanel } from "@/file-viewer/views/clofi.tsx";299import { MediaPanel } from "@/file-viewer/views/clofi.tsx";
src/file-viewer/bin/file-scan.ts created+1058
...@@ -0,0 +1,1058 @@
1// The file scanner incrementally updates an sqlite database with file
2// stats. Additionally, it runs "processors" on files, which precompute
3// expensive data such as running `ffprobe` on all media to get the
4// duration.
5//
6// Processors are also used to derive compressed and optimized assets,
7// which is how automatic JXL / AV1 encoding is done. Derived files are
8// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9//
10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.
13const sotToken = process.env.CLOVER_SOT_KEY;
14
15export async function main() {
16 const start = performance.now();
17 using _ = log.startWidget({
18 format: (now) =>
19 `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
20 });
21
22 const promises = new async.PromiseAggregator();
23
24 const walkQueue = new queue.PriorityQueue(10);
25
26 const dirsNode = progress.start("Walk Tree", { estimate: 1 });
27 dirsNode.passive = true;
28 const fileNode = progress.start("Process File", { estimate: 0 });
29 fileNode.sortChildren = (a, b) => {
30 const ac = a.children.length > 0 ? 1 : 0;
31 const bc = b.children.length > 0 ? 1 : 0;
32 if (ac !== bc) return bc - ac;
33 return a.text.localeCompare(b.text);
34 };
35
36 // Read a directory or file stat and queue up changed files.
37 const scanDirectory = walkQueue.wrap(
38 async (path: Path) => {
39 const publicPath = toPublicPath(path);
40 using node = dirsNode.start(publicPath + " - stat");
41 using _ = ts.defer(() => dirsNode.inc());
42
43 const stat = await path.stat();
44
45 const mediaFile = MediaFile.getByPath(publicPath);
46
47 if (stat.isDirectory()) {
48 node.text = publicPath + " - reading";
49 const items = (await path.readDir())
50 .filter((child) => !skipBasename(child.base))
51 .map((child) => (promises.push(scanDirectory(child)), child.base));
52 dirsNode.estimate += items.length;
53
54 for (const child of mediaFile?.getChildren() ?? []) {
55 if (items.includes(child.basename)) continue;
56 const recursive = child.kind === MediaFileKind.directory
57 ? [child, ...child.getRecursiveFileChildren()]
58 : [child];
59 for (const deletion of recursive) deletion.delete();
60 }
61
62 return;
63 }
64
65 if (
66 !mediaFile || // All processes must be performed if there is no file.
67 // Rerun all processors if it changed
68 stat.size !== mediaFile.size ||
69 stat.mtime.getTime() !== mediaFile.date.getTime()
70 ) {
71 promises.push(updateMetadata({ path, publicPath, stat, mediaFile }));
72 } else {
73 // If the scanners changed, it may mean more processes should be run.
74 await queueProcessors({
75 path,
76 stat,
77 mediaFile,
78 node: fileNode.start(publicPath.slice(1)),
79 });
80 }
81 },
82 );
83 const updateMetadata = queue.wrap(
84 async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
85 using errorHandler = new DisposableStack();
86 const label = publicPath.slice(1);
87 const node = errorHandler.use(fileNode.start(label));
88
89 await scrubLocationMetadata(path, stat, node);
90
91 node.text = `${label} - hashing`;
92 const hash = await new Promise<string>((resolve, reject) => {
93 const reader = fs.createReadStream(path.toString());
94 reader.on("error", reject);
95
96 const hasher = crypto.createHash("sha1").setEncoding("hex");
97 hasher.on("error", reject);
98 hasher.on("readable", () => resolve(hasher.read()));
99
100 reader.pipe(hasher);
101 });
102 let date = stat.mtime;
103 if (
104 mediaFile &&
105 mediaFile.date.getTime() < stat.mtime.getTime() &&
106 Date.now() - stat.mtime.getTime() < monthMilliseconds
107 ) {
108 date = mediaFile.date;
109 console.warn(
110 `M-time on ${publicPath} was likely corrupted. ${
111 formatDate(
112 mediaFile.date,
113 )
114 } -> ${formatDate(stat.mtime)}`,
115 );
116 }
117 mediaFile = MediaFile.createFile({
118 path: publicPath,
119 date,
120 hash,
121 size: stat.size,
122 duration: mediaFile?.duration ?? 0,
123 dimensions: mediaFile?.dimensions ?? "",
124 contents: mediaFile?.contents ?? "",
125 });
126 const parent = mediaFile.getParent();
127 if (parent) parent.setProcessed(0);
128
129 node.text = `${label}`;
130 await queueProcessors({
131 path,
132 stat,
133 mediaFile,
134 node,
135 });
136 errorHandler.move();
137 },
138 () => ({ priority: -1 }),
139 );
140 const queueFileProcessor = queue.wrap(async function ({
141 path,
142 stat,
143 mediaFile,
144 processor,
145 index,
146 after,
147 fileNode: innerNode,
148 }: ProcessJob) {
149 using node = innerNode.start(processor.name);
150 await processor.run({
151 path,
152 stat,
153 mediaFile,
154 node,
155 });
156 fileNode.value += 1;
157 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
158 for (const dependantJob of after) {
159 ASSERT(
160 dependantJob.needs > 0,
161 `dependantJob.needs > 0, ${dependantJob.needs}`,
162 );
163 dependantJob.needs -= 1;
164 if (dependantJob.needs == 0) {
165 promises.push(queueFileProcessor(dependantJob));
166 }
167 }
168 }, (job) => ({ cores: job.processor.cores ?? 0 }));
169
170 function decodeProcessors(input: string) {
171 return input
172 .split(";")
173 .filter(Boolean)
174 .map(([a, b, c]) => ({
175 id: a,
176 hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
177 }));
178 }
179
180 async function queueProcessors({
181 path,
182 stat,
183 mediaFile,
184 node,
185 }: Omit<ProcessFileArgs, "spin">) {
186 using errorHandler = new DisposableStack();
187 errorHandler.use(node);
188
189 node.showEstimate = false;
190 node.passive = true;
191
192 const ext = mediaFile.extensionNonEmpty.toLowerCase();
193 let possible = processors.filter((p) =>
194 p.include ? p.include.has(ext) : !p.exclude?.has(ext)
195 );
196 if (possible.length === 0) return;
197 ASSERT(possible.length < 16, "too many bits");
198
199 const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
200 ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
201 let processed = mediaFile.processed;
202
203 // If the hash has changed, migrate the bitfield over.
204 // This also runs when the processor hash is in it's initial 0 state.
205 let order: ReturnType<typeof decodeProcessors>;
206 try {
207 order = decodeProcessors(mediaFile.processors);
208 } catch {
209 // this function sucks and this system sucks i hate it.
210 order = [];
211 }
212 if ((processed & 0xffff) !== hash) {
213 const previous = order.filter(
214 (_, i) => (processed & (1 << (16 + i))) !== 0,
215 );
216 processed = hash;
217 for (const { id, hash } of previous) {
218 const p = processors.find((p) => p.id === id);
219 if (!p) continue;
220 const index = possible.indexOf(p);
221 if (index !== -1 && p.hash === hash) processed |= 1 << (16 + index);
222 }
223 mediaFile.setProcessors(
224 processed,
225 possible
226 .map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
227 .join(";"),
228 );
229 } else {
230 possible = order.map(({ id }) =>
231 UNWRAP(possible.find((p) => p.id === id))
232 );
233 }
234
235 // Queue needed processors.
236 const jobs: ProcessJob[] = [];
237 for (let i = 0, { length } = possible; i < length; i += 1) {
238 if ((processed & (1 << (16 + i))) === 0) {
239 const processor = UNWRAP(possible[i]);
240 const job: ProcessJob = {
241 path,
242 stat,
243 mediaFile,
244 processor,
245 index: i,
246 after: [],
247 needs: processor.depends.length,
248 fileNode: node,
249 };
250 jobs.push(job);
251 if (job.needs === 0) promises.push(queueFileProcessor(job));
252 }
253 }
254 node.estimate = jobs.length;
255 fileNode.estimate += jobs.length;
256 for (const job of jobs) {
257 for (const dependId of job.processor.depends) {
258 const dependJob = jobs.find((j) => j.processor.id === dependId);
259 if (dependJob) {
260 dependJob.after.push(job);
261 } else {
262 ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
263 job.needs -= 1;
264 if (job.needs === 0) promises.push(queueFileProcessor(job));
265 }
266 }
267 }
268 if (node.estimate > 0) {
269 errorHandler.move();
270
271 const parent = mediaFile.getParent();
272 if (parent) parent.setProcessed(0);
273 }
274 }
275
276 // Add the root & recursively iterate!
277 const rootPath = Path.resolve(root);
278 if (!rootPath.ifExistsSync()) {
279 throw new Error(`file store ${rootPath} is not mounted`);
280 }
281 promises.push(scanDirectory(rootPath));
282
283 await promises.all();
284 fileNode.end();
285 dirsNode.end();
286
287 // Update directory metadata
288 using dirMetaNode = progress.start("update directory metadata");
289 const dirs = MediaFile.getDirectoriesToReindex()
290 .sort((a, b) => b.path.length - a.path.length);
291
292 for (const dir of dirs) {
293 using _ = dirMetaNode.start(dir.path);
294 const children = dir.getChildren();
295
296 // readme.txt
297 const readmeContent = children.find((x) =>
298 x.basename === "readme.txt"
299 )?.contents ?? "";
300
301 // dirsort
302 let dirsort: string[] | null = null;
303 const dirSortRaw =
304 children.find((x) => x.basename === ".dirsort")?.contents ?? "";
305 if (dirSortRaw) {
306 dirsort = dirSortRaw
307 .split("\n")
308 .map((x) => x.trim())
309 .filter(Boolean);
310 }
311
312 // Permissions
313 if (children.some((x) => x.basename === ".friends")) {
314 FilePermissions.setPermissions(dir.path, 1);
315 } else {
316 FilePermissions.setPermissions(dir.path, 0);
317 }
318
319 // Recursive stats.
320 let totalSize = 0;
321 let newestDate = new Date(0);
322 let allHashes = "";
323 for (const child of children) {
324 totalSize += child.size;
325 allHashes += child.hash;
326
327 if (child.basename !== "/readme.txt" && child.date > newestDate) {
328 newestDate = child.date;
329 }
330 }
331
332 // Project Date
333 const dateFile = children.find((x) => x.basename === ".date");
334 if (dateFile) {
335 const date = new Date(dateFile.contents);
336 newestDate = date;
337 dir.setProcessors(
338 0,
339 JSON.stringify({ hideChildrenDates: true }),
340 );
341 } else {
342 dir.setProcessors(0, "");
343 }
344
345 const dirHash = crypto
346 .createHash("sha1")
347 .update(dir.path + allHashes)
348 .digest("hex");
349
350 MediaFile.markDirectoryProcessed({
351 id: dir.id,
352 timestamp: newestDate,
353 contents: readmeContent,
354 size: totalSize,
355 hash: dirHash,
356 dirsort,
357 });
358 }
359 dirMetaNode.end();
360
361 // Sync to remote
362 if (
363 ((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
364 ) {
365 await rsync.spawn({
366 args: [
367 "--links",
368 "--recursive",
369 "--times",
370 "--partial",
371 "--progress",
372 // "--remove-source-files",
373 "--delay-updates",
374 "--exclude=tmp.*",
375 derived.workDir.toString() + "/",
376 "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/derived/",
377 ],
378 progress: progress.start("upload derived assets"),
379 cwd: process.cwd(),
380 });
381
382 await fs.removeEmptyDirectories(derived.workDir.toString());
383 } else {
384 console.info("No new derived assets");
385 }
386
387 MediaFile.db.prepare("VACUUM").run();
388 MediaFile.db.reload();
389
390 await rsync.spawn({
391 args: [
392 MediaFile.db.file,
393 "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
394 ],
395 progress: progress.start("Uploading database (source of truth)"),
396 cwd: process.cwd(),
397 });
398 await rsync.spawn({
399 args: [
400 MediaFile.db.file,
401 "clo@paperclover.net:paperclover.net/.clover/cache.sqlite",
402 ],
403 progress: progress.start("Uploading database (web node)"),
404 });
405 if (sotToken) {
406 const res = await fetch("https://db.paperclover.net/reload", {
407 method: "POST",
408 headers: {
409 Authorization: sotToken,
410 },
411 });
412 if (!res.ok) {
413 console.warn(
414 `Failed to reload remote database ${res.status} ${res.statusText}`,
415 );
416 }
417 } else console.warn("Missing SOT token");
418
419 console.info(
420 "Updated file viewer index in \x1b[1m" +
421 ((performance.now() - start) / 1000).toFixed(1) +
422 "s\x1b[0m",
423 );
424
425 const { duration, count } = MediaFile.db
426 .prepare<[], { count: number; duration: number }>(`
427 select
428 count(*) as count,
429 sum(duration) as duration
430 from media_files
431 `)
432 .getNonNull();
433 const { derivedSize, derivedCount } = MediaFile.db
434 .prepare<[], { derivedCount: number; derivedSize: number }>(`
435 select
436 count(*) as derivedCount,
437 sum(size) as derivedSize
438 from derived_files
439 `)
440 .getNonNull();
441 const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
442
443 console.info();
444 console.info(
445 "Global Stats:\n" +
446 `- File Count: \x1b[1m${count}\x1b[0m\n` +
447 `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n` +
448 `- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n` +
449 `- Canonical Size: \x1b[1m${formatSize(canonicalSize)}\x1b[0m\n` +
450 `- Derived Size: \x1b[1m${formatSize(derivedSize)}\x1b[0m\n`,
451 );
452}
453
454interface Process {
455 name: string;
456 cores?: number;
457 enable?: boolean;
458 include?: Set<string>;
459 exclude?: Set<string>;
460 depends?: string[];
461 version?: number;
462 /* Perform an action. */
463 run(args: ProcessFileArgs): Promise<void>;
464}
465
466const ffprobeBin = testProgram("ffprobe", "--help");
467const ffmpegBin = testProgram("ffmpeg", "--help");
468
469const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
470
471// NOTE: Never re-order the processors. Add new ones at the end.
472const procDuration: Process = {
473 name: "calculate duration",
474 enable: ffprobeBin !== null,
475 include: rules.extsDuration,
476 cores: 1,
477 async run({ path, mediaFile }) {
478 const { stdout } = await subprocess.exec(ffprobeBin!, [
479 "-v",
480 "error",
481 "-show_entries",
482 "format=duration",
483 "-of",
484 "default=noprint_wrappers=1:nokey=1",
485 path.toString(),
486 ]);
487
488 const duration = parseFloat(stdout.trim());
489 if (Number.isNaN(duration)) {
490 throw new Error("Could not extract duration from " + stdout);
491 }
492 mediaFile.setDuration(Math.ceil(duration));
493 },
494};
495
496const procDimensions: Process = {
497 name: "calculate dimensions",
498 enable: ffprobeBin != null,
499 include: rules.extsDimensions,
500 cores: 1,
501 async run({ path, mediaFile }) {
502 const { ext } = path;
503
504 let dimensions;
505
506 if (ext === ".svg") {
507 // Parse out of text data
508 const content = await path.read("utf-8");
509 const widthMatch = content.match(/width="(\d+)"/);
510 const heightMatch = content.match(/height="(\d+)"/);
511
512 if (widthMatch && heightMatch) {
513 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
514 }
515 } else if (rules.extsImage.has(ext)) {
516 // Use magick to observe streams
517 const { stdout } = await subprocess.exec("magick", [
518 "identify",
519 "-auto-orient",
520 "-format",
521 "%w %h",
522 path.toString(),
523 ]);
524 const [w, h] = stdout.split(" ").map((x) => Number(x));
525 if (w && h) {
526 dimensions = w + "x" + h;
527 }
528 } else {
529 // Use ffprobe to observe streams
530 const { stdout } = await subprocess.exec("ffprobe", [
531 "-v",
532 "error",
533 "-select_streams",
534 "v:0",
535 "-show_entries",
536 "stream=width,height",
537 "-of",
538 "json",
539 path.toString(),
540 ]);
541 const result = JSON.parse(stdout);
542 const stream = result.streams[0];
543 if (stream) {
544 dimensions = UNWRAP(stream.width) + "x" + UNWRAP(stream.height);
545 }
546 }
547
548 mediaFile.setDimensions(dimensions ?? "");
549 },
550};
551
552const procLoadTextContents: Process = {
553 name: "load text content",
554 include: rules.extsReadContents,
555 cores: 1,
556 version: 2,
557 async run({ path, mediaFile, stat }) {
558 if (stat.size > 1_000_000) return;
559 const text = await path.read("utf-8");
560 mediaFile.setContents(text);
561 },
562};
563
564const procHighlightCode: Process = {
565 name: "highlight source code",
566 include: new Set(rules.extsCode.keys()),
567 cores: 1,
568 version: 2,
569 async run({ path, mediaFile, stat }) {
570 const language = UNWRAP(
571 rules.extsCode.get(path.ext.toLowerCase()),
572 );
573 // An issue is that .ts is an overloaded extension, shared between
574 // 'transport stream' and 'typescript'.
575 //
576 // Filter used here is:
577 // - more than 1mb
578 // - invalid UTF-8
579 if (stat.size > 1_000_000) return;
580 let code;
581 const buf = await path.read();
582 try {
583 code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
584 } catch (error) {
585 mediaFile.setContents("");
586 return;
587 }
588 const content = await highlight.highlightCode(code, language);
589 mediaFile.setContents(content);
590 },
591};
592
593const procImageSubsets: Process = {
594 name: "encode image subsets",
595 include: rules.extsImage,
596 depends: [procDimensions.name],
597 version: 2,
598 async run({ path, mediaFile, node }) {
599 const { width, height } = UNWRAP(mediaFile.parseDimensions());
600 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
601 const baseStatus = node.text;
602
603 using stack = new DisposableStack();
604 for (const size of targetSizes) {
605 const { w, h } = resizeDimensions(width, height, size);
606 for (const { ext, args } of transcodeRules.imagePresets) {
607 node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
608
609 stack.use(
610 await derived.produce({
611 mediaFile,
612 node,
613 cores: 2,
614 subkey: `${size}${ext}`,
615 async producer(dir) {
616 await subprocess.exec(ffmpegBin!, [
617 ...ffmpegOptions,
618 "-i",
619 path.toString(),
620 "-vf",
621 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
622 ...args,
623 dir.join(`${size}${ext}`).toString(),
624 ]);
625 },
626 }),
627 );
628 }
629 }
630
631 stack.move();
632 },
633};
634
635const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
636
637const qualityMap: Record<string, string> = {
638 u: "ultra-high",
639 h: "high",
640 m: "medium",
641 l: "low",
642 d: "data-saving",
643};
644const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
645 name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
646 include: rules.extsVideo,
647 enable: ffmpegBin != null,
648 depends: [procDuration.name, procDimensions.name],
649 async run({ path, mediaFile, node }) {
650 if ((mediaFile.duration ?? 0) < 5) return;
651 if (!mediaFile.dimensions) return;
652
653 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
654
655 await derived.produce({
656 mediaFile,
657 node,
658 cores: 4,
659 subkey: `av1-${preset.id}`,
660 async producer(dir) {
661 const input = await videoArgsCache.getOrRun(path);
662 const args = transcodeRules.getAv1VideoArgs(
663 preset,
664 UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
665 dir,
666 );
667
668 await ffmpeg.spawn({
669 ffmpeg: ffmpegBin!,
670 progress: node,
671 args,
672 cwd: dir,
673 });
674 },
675 });
676 },
677}));
678const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
679 name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
680 include: rules.extsVideo,
681 enable: ffmpegBin != null,
682 depends: [procDuration.name, procDimensions.name],
683 version: 2,
684 async run({ path, mediaFile, node }) {
685 if ((mediaFile.duration ?? 0) < 5) return;
686 if (!mediaFile.dimensions) return;
687 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
688 await derived.produce({
689 mediaFile,
690 node,
691 cores: 4,
692 subkey: `opus-${preset.id}`,
693 async producer(dir) {
694 const input = await videoArgsCache.getOrRun(path);
695 ASSERT(input.video);
696 if (!input.audio) return;
697 const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
698
699 await ffmpeg.spawn({
700 ffmpeg: ffmpegBin!,
701 progress: node,
702 args,
703 cwd: dir,
704 });
705 },
706 });
707 },
708}));
709const procDash: Process = {
710 name: `encode mpeg-dash`,
711 include: rules.extsVideo,
712 enable: ffmpegBin != null,
713 depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
714 version: 4,
715 async run({ path, mediaFile, node }) {
716 if ((mediaFile.duration ?? 0) < 5) return;
717 if (!mediaFile.dimensions) return;
718
719 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
720 await derived.produce({
721 mediaFile,
722 node,
723 cores: 1,
724 subkey: `dash-av1`,
725 async producer(dir) {
726 const input = await videoArgsCache.getOrRun(path);
727 const videos = transcodeRules.videoFormats.map(
728 (preset) =>
729 UNWRAP(dir.parent).join(
730 `av1-${preset.id}`,
731 transcodeRules.av1FileName,
732 ),
733 );
734 const audios = input.audio
735 ? transcodeRules.audioFormats.map(
736 (preset) =>
737 UNWRAP(dir.parent).join(
738 `opus-${preset.id}`,
739 transcodeRules.opusFileName,
740 ),
741 )
742 : [];
743 const args = transcodeRules.getMpegDashArgs(
744 videos,
745 audios,
746 dir,
747 );
748
749 await dir.join("d").makeDir();
750
751 await ffmpeg.spawn({
752 ffmpeg: ffmpegBin!,
753 progress: node,
754 args,
755 cwd: dir,
756 });
757 },
758 });
759 },
760};
761const procH264Hls: Process = {
762 name: `encode h.264 hls`,
763 include: rules.extsVideo,
764 enable: ffmpegBin != null,
765 depends: [procDuration.name, procDimensions.name],
766 async run({ path, mediaFile, node }) {
767 if ((mediaFile.duration ?? 0) < 5) return;
768 if (!mediaFile.dimensions) return;
769
770 await derived.produce({
771 mediaFile,
772 node,
773 cores: 8,
774 subkey: `hls`,
775 async producer(dir) {
776 const input = await videoArgsCache.getOrRun(path);
777 const args = transcodeRules.getH264HlsArgs(input, dir);
778
779 await ffmpeg.spawn({
780 ffmpeg: ffmpegBin!,
781 progress: node,
782 args,
783 cwd: dir,
784 });
785 },
786 });
787 },
788};
789
790const procCompression = [
791 { name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
792 { name: "zstd", fn: () => zlib.createZstdCompress() },
793].map<Process>(
794 ({ name, fn }) => ({
795 name: `compress ${name}`,
796 exclude: rules.extsPreCompressed,
797 async run({ path, mediaFile, node }) {
798 await derived.produce({
799 mediaFile,
800 node,
801 cores: 1,
802 subkey: name,
803 async producer(dir) {
804 await stream.promises.pipeline(
805 fs.createReadStream(path.toString()),
806 fn(),
807 fs.createWriteStream(dir.join(name).toString()),
808 );
809 },
810 });
811 },
812 }),
813);
814
815const processors = [
816 procDimensions,
817 procDuration,
818 procLoadTextContents,
819 procHighlightCode,
820 procImageSubsets,
821 ...procVideos,
822 ...procVideoAudios,
823 procDash,
824 ...procCompression,
825 procH264Hls,
826].map((process, id, all) => {
827 const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
828 return {
829 ...(process as Process),
830 id: strIndex(id),
831 // Create a unique key.
832 hash: new Uint16Array(
833 crypto
834 .createHash("sha1")
835 .update(
836 process.run.toString() +
837 (process.version ? String(process.version) : ""),
838 )
839 .digest().buffer,
840 ).reduce((a, b) => a ^ b),
841 depends: (process.depends ?? []).map((depend) => {
842 const index = all.findIndex((p) => p.name === depend);
843 if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
844 if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
845 return strIndex(index);
846 }),
847 };
848});
849
850function resizeDimensions(w: number, h: number, desiredWidth: number) {
851 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
852 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
853}
854
855interface UpdateMetadataArgs {
856 path: Path;
857 publicPath: string;
858 stat: fs.Stats;
859 mediaFile: MediaFile | null;
860}
861
862interface ProcessFileArgs {
863 path: Path;
864 stat: fs.Stats;
865 mediaFile: MediaFile;
866 node: progress.Node;
867}
868
869interface ProcessJob {
870 path: Path;
871 stat: fs.Stats;
872 mediaFile: MediaFile;
873 processor: (typeof processors)[0];
874 index: number;
875 after: ProcessJob[];
876 needs: number;
877 fileNode: progress.Node;
878}
879
880function skipBasename(basename: string): boolean {
881 // dot files must be incrementally tracked
882 if (basename === ".dirsort") return false;
883 if (basename === ".friends") return false;
884 if (basename === ".date") return false;
885
886 return (
887 basename.startsWith(".") ||
888 // basename.startsWith("._") ||
889 // basename.startsWith(".tmp") ||
890 // basename === ".DS_Store" ||
891 basename.toLowerCase() === "thumbs.db" ||
892 basename.toLowerCase() === "desktop.ini"
893 );
894}
895
896function toPublicPath(diskPath: Path) {
897 if (diskPath.toString() === root) return "/";
898 return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
899}
900
901function testProgram(name: string, helpArgument: string) {
902 try {
903 child_process.spawnSync(name, [helpArgument]);
904 return name;
905 } catch (err) {
906 console.warn(`Missing or corrupt executable '${name}'`);
907 }
908 return null;
909}
910
911// Helper function to check and remove location metadata
912async function scrubLocationMetadata(
913 path: Path,
914 stats: fs.Stats,
915 progress: progress.Ref,
916): Promise<boolean> {
917 using _ = progress.start("scrub exif metadata");
918 const ext = path.ext.toLowerCase();
919 if (!rules.extsScrubExif.has(ext)) return false;
920
921 let hasLocation = false;
922 let args: string[] = [];
923
924 // Check for location metadata based on file type
925 const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
926 switch (ext) {
927 case ".jpg":
928 case ".jpeg":
929 case ".png":
930 const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
931 "-gps:all",
932 path.toString(),
933 ]);
934 hasLocation = gpsCheck.trim().length > 0;
935 args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
936 break;
937 case ".mov":
938 case ".mp4":
939 const { stdout: videoCheck } = await subprocess.exec("exiftool", [
940 "-ee",
941 "-G3",
942 "-s",
943 path.toString(),
944 ]);
945 hasLocation = videoCheck.includes("GPS") ||
946 videoCheck.includes("Location");
947 args = [
948 "-gps:all=",
949 "-xmp:all=",
950 path.toString(),
951 "-o",
952 tempOutput.toString(),
953 ];
954 break;
955 case ".m4a":
956 const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
957 "-ee",
958 "-G3",
959 "-s",
960 path.toString(),
961 ]);
962 hasLocation = m4aCheck.includes("GPS") ||
963 m4aCheck.includes("Location") ||
964 m4aCheck.includes("Filename") ||
965 m4aCheck.includes("Title");
966
967 if (hasLocation) {
968 args = [
969 "-gps:all=",
970 "-location:all=",
971 "-filename:all=",
972 "-title=",
973 "-m4a:all=",
974 path.toString(),
975 "-o",
976 tempOutput.toString(),
977 ];
978 }
979 break;
980 }
981
982 const accessTime = stats.atime;
983 const modTime = stats.mtime;
984
985 let backup: Path | null = null;
986 try {
987 if (hasLocation) {
988 // Prepare a backup
989 const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
990 await fsp.copyFile(path.toString(), tmp.toString());
991 await fsp.utimes(tmp.toString(), accessTime, modTime);
992 backup = tmp;
993
994 // Remove metadata
995 await subprocess.exec("exiftool", args);
996 if (!tempOutput.ifExistsSync()) {
997 throw new Error(`Failed to create output file: ${tempOutput}`);
998 }
999
1000 // Restore original timestamps
1001 await fsp.rename(tempOutput.toString(), path.toString());
1002 await fsp.utimes(path.toString(), accessTime, modTime);
1003
1004 // Backup is no longer needed
1005 await fsp.unlink(backup.toString());
1006
1007 console.info(
1008 `Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
1009 );
1010 return true;
1011 }
1012 } catch (error) {
1013 if (backup) {
1014 await fsp.rename(backup.toString(), path.toString());
1015 }
1016 if (fs.existsSync(tempOutput.toString())) {
1017 await fsp.unlink(tempOutput.toString());
1018 }
1019 throw error;
1020 }
1021
1022 return false;
1023}
1024
1025const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
1026
1027import * as async from "lib/async.ts";
1028import * as fs from "#sitegen/fs";
1029import * as subprocess from "lib/subprocess.ts";
1030import { Path } from "#sitegen/path";
1031import * as queue from "../../../lib/queue.ts";
1032import * as log from "../../../lib/log.ts";
1033import * as progress from "lib/progress.ts";
1034import * as ts from "../../../lib/ts.ts";
1035
1036import * as child_process from "node:child_process";
1037import * as crypto from "node:crypto";
1038import * as fsp from "node:fs/promises";
1039import * as path from "node:path";
1040import * as stream from "node:stream";
1041import * as zlib from "node:zlib";
1042
1043import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
1044import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
1045import {
1046 formatDate,
1047 formatDurationLong,
1048 formatSize,
1049} from "@/file-viewer/format.ts";
1050import * as rules from "@/file-viewer/rules.ts";
1051import * as highlight from "@/file-viewer/highlight.ts";
1052import * as ffmpeg from "lib/subprocess/ffmpeg.ts";
1053import * as rsync from "@/file-viewer/rsync.ts";
1054import * as derived from "@/file-viewer/models/derived.ts";
1055import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
1056
1057import { rawFileRoot as root } from "../paths.ts";
1058import { ASSERT, UNWRAP } from "lib/assert.ts";
src/file-viewer/bin/file-trim.ts created+18
...@@ -0,0 +1,18 @@
1export async function main() {
2 const start = performance.now();
3 using _ = log.startWidget({
4 format: (now) =>
5 `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
6 });
7
8 const orphaned = derived.findOrphanedRoots();
9 for (const root of orphaned) {
10 console.info("delete " + root.key);
11 derived.deleteRoot(root);
12 }
13
14 // TODO: delete unreferenced files
15}
16
17import * as log from "lib/log.ts";
18import * as derived from "../models/derived.ts";
src/file-viewer/bin/scan3.ts deleted-1043
...@@ -1,1043 +0,0 @@
1// The file scanner incrementally updates an sqlite database with file
2// stats. Additionally, it runs "processors" on files, which precompute
3// expensive data such as running `ffprobe` on all media to get the
4// duration.
5//
6// Processors are also used to derive compressed and optimized assets,
7// which is how automatic JXL / AV1 encoding is done. Derived files are
8// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9//
10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.
13const sotToken = process.env.CLOVER_SOT_KEY;
14
15export async function main() {
16 const start = performance.now();
17 using _ = log.startWidget({
18 format: (now) =>
19 `paper clover's scan3 [${((now - start) / 1000).toFixed(1)}s]`,
20 });
21
22 const promises = new async.PromiseAggregator();
23
24 const walkQueue = new queue.PriorityQueue(10);
25
26 const dirsNode = progress.start("Walk Tree", { estimate: 1 });
27 dirsNode.passive = true;
28 const fileNode = progress.start("Process File", { estimate: 0 });
29 fileNode.sortChildren = (a, b) => {
30 const ac = a.children.length > 0 ? 1 : 0;
31 const bc = b.children.length > 0 ? 1 : 0;
32 if (ac !== bc) return bc - ac;
33 return a.text.localeCompare(b.text);
34 };
35
36 // Read a directory or file stat and queue up changed files.
37 const scanDirectory = walkQueue.wrap(
38 async (path: Path) => {
39 const publicPath = toPublicPath(path);
40 using node = dirsNode.start(publicPath + " - stat");
41 using _ = ts.defer(() => dirsNode.inc());
42
43 const stat = await path.stat();
44
45 const mediaFile = MediaFile.getByPath(publicPath);
46
47 if (stat.isDirectory()) {
48 node.text = publicPath + " - reading";
49 const items = (await path.readDir())
50 .filter((child) => !skipBasename(child.base))
51 .map((child) => (promises.push(scanDirectory(child)), child.base));
52 dirsNode.estimate += items.length;
53
54 for (const child of mediaFile?.getChildren() ?? []) {
55 if (items.includes(child.basename)) continue;
56 const recursive = child.kind === MediaFileKind.directory
57 ? [child, ...child.getRecursiveFileChildren()]
58 : [child];
59 for (const deletion of recursive) deletion.delete();
60 }
61
62 return;
63 }
64
65 if (
66 !mediaFile || // All processes must be performed if there is no file.
67 // Rerun all processors if it changed
68 stat.size !== mediaFile.size ||
69 stat.mtime.getTime() !== mediaFile.date.getTime()
70 ) {
71 promises.push(updateMetadata({ path, publicPath, stat, mediaFile }));
72 } else {
73 // If the scanners changed, it may mean more processes should be run.
74 await queueProcessors({
75 path,
76 stat,
77 mediaFile,
78 node: fileNode.start(publicPath.slice(1)),
79 });
80 }
81 },
82 );
83 const updateMetadata = queue.wrap(
84 async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
85 using errorHandler = new DisposableStack();
86 const label = publicPath.slice(1);
87 const node = errorHandler.use(fileNode.start(label));
88
89 await scrubLocationMetadata(path, stat, node);
90
91 node.text = `${label} - hashing`;
92 const hash = await new Promise<string>((resolve, reject) => {
93 const reader = fs.createReadStream(path.toString());
94 reader.on("error", reject);
95
96 const hasher = crypto.createHash("sha1").setEncoding("hex");
97 hasher.on("error", reject);
98 hasher.on("readable", () => resolve(hasher.read()));
99
100 reader.pipe(hasher);
101 });
102 let date = stat.mtime;
103 if (
104 mediaFile &&
105 mediaFile.date.getTime() < stat.mtime.getTime() &&
106 Date.now() - stat.mtime.getTime() < monthMilliseconds
107 ) {
108 date = mediaFile.date;
109 console.warn(
110 `M-time on ${publicPath} was likely corrupted. ${
111 formatDate(
112 mediaFile.date,
113 )
114 } -> ${formatDate(stat.mtime)}`,
115 );
116 }
117 mediaFile = MediaFile.createFile({
118 path: publicPath,
119 date,
120 hash,
121 size: stat.size,
122 duration: mediaFile?.duration ?? 0,
123 dimensions: mediaFile?.dimensions ?? "",
124 contents: mediaFile?.contents ?? "",
125 });
126 const parent = mediaFile.getParent();
127 if (parent) parent.setProcessed(0);
128
129 node.text = `${label}`;
130 await queueProcessors({
131 path,
132 stat,
133 mediaFile,
134 node,
135 });
136 errorHandler.move();
137 },
138 () => ({ priority: -1 }),
139 );
140 const queueFileProcessor = queue.wrap(async function ({
141 path,
142 stat,
143 mediaFile,
144 processor,
145 index,
146 after,
147 fileNode: innerNode,
148 }: ProcessJob) {
149 using node = innerNode.start(processor.name);
150 await processor.run({
151 path,
152 stat,
153 mediaFile,
154 node,
155 });
156 fileNode.value += 1;
157 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
158 for (const dependantJob of after) {
159 ASSERT(
160 dependantJob.needs > 0,
161 `dependantJob.needs > 0, ${dependantJob.needs}`,
162 );
163 dependantJob.needs -= 1;
164 if (dependantJob.needs == 0) {
165 promises.push(queueFileProcessor(dependantJob));
166 }
167 }
168 }, (job) => ({ cores: job.processor.cores ?? 0 }));
169
170 function decodeProcessors(input: string) {
171 return input
172 .split(";")
173 .filter(Boolean)
174 .map(([a, b, c]) => ({
175 id: a,
176 hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
177 }));
178 }
179
180 async function queueProcessors({
181 path,
182 stat,
183 mediaFile,
184 node,
185 }: Omit<ProcessFileArgs, "spin">) {
186 using errorHandler = new DisposableStack();
187 errorHandler.use(node);
188
189 node.showEstimate = false;
190 node.passive = true;
191
192 const ext = mediaFile.extensionNonEmpty.toLowerCase();
193 let possible = processors.filter((p) =>
194 p.include ? p.include.has(ext) : !p.exclude?.has(ext)
195 );
196 if (possible.length === 0) return;
197 ASSERT(possible.length < 16, "too many bits");
198
199 const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
200 ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
201 let processed = mediaFile.processed;
202
203 // If the hash has changed, migrate the bitfield over.
204 // This also runs when the processor hash is in it's initial 0 state.
205 let order: ReturnType<typeof decodeProcessors>;
206 try {
207 order = decodeProcessors(mediaFile.processors);
208 } catch {
209 // this function sucks and this system sucks i hate it.
210 order = [];
211 }
212 if ((processed & 0xffff) !== hash) {
213 const previous = order.filter(
214 (_, i) => (processed & (1 << (16 + i))) !== 0,
215 );
216 processed = hash;
217 for (const { id, hash } of previous) {
218 const p = processors.find((p) => p.id === id);
219 if (!p) continue;
220 const index = possible.indexOf(p);
221 if (index !== -1 && p.hash === hash) processed |= 1 << (16 + index);
222 }
223 mediaFile.setProcessors(
224 processed,
225 possible
226 .map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
227 .join(";"),
228 );
229 } else {
230 possible = order.map(({ id }) =>
231 UNWRAP(possible.find((p) => p.id === id))
232 );
233 }
234
235 // Queue needed processors.
236 const jobs: ProcessJob[] = [];
237 for (let i = 0, { length } = possible; i < length; i += 1) {
238 if ((processed & (1 << (16 + i))) === 0) {
239 const processor = UNWRAP(possible[i]);
240 const job: ProcessJob = {
241 path,
242 stat,
243 mediaFile,
244 processor,
245 index: i,
246 after: [],
247 needs: processor.depends.length,
248 fileNode: node,
249 };
250 jobs.push(job);
251 if (job.needs === 0) promises.push(queueFileProcessor(job));
252 }
253 }
254 node.estimate = jobs.length;
255 fileNode.estimate += jobs.length;
256 for (const job of jobs) {
257 for (const dependId of job.processor.depends) {
258 const dependJob = jobs.find((j) => j.processor.id === dependId);
259 if (dependJob) {
260 dependJob.after.push(job);
261 } else {
262 ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
263 job.needs -= 1;
264 if (job.needs === 0) promises.push(queueFileProcessor(job));
265 }
266 }
267 }
268 if (node.estimate > 0) {
269 errorHandler.move();
270
271 const parent = mediaFile.getParent();
272 if (parent) parent.setProcessed(0);
273 }
274 }
275
276 // Add the root & recursively iterate!
277 const rootPath = Path.resolve(root);
278 if (!rootPath.ifExistsSync()) {
279 throw new Error(`file store ${rootPath} is not mounted`);
280 }
281 promises.push(scanDirectory(rootPath));
282
283 await promises.all();
284 fileNode.end();
285 dirsNode.end();
286
287 // Update directory metadata
288 using dirMetaNode = progress.start("update directory metadata");
289 const dirs = MediaFile.getDirectoriesToReindex()
290 .sort((a, b) => b.path.length - a.path.length);
291
292 for (const dir of dirs) {
293 using _ = dirMetaNode.start(dir.path);
294 const children = dir.getChildren();
295
296 // readme.txt
297 const readmeContent = children.find((x) =>
298 x.basename === "readme.txt"
299 )?.contents ?? "";
300
301 // dirsort
302 let dirsort: string[] | null = null;
303 const dirSortRaw =
304 children.find((x) => x.basename === ".dirsort")?.contents ?? "";
305 if (dirSortRaw) {
306 dirsort = dirSortRaw
307 .split("\n")
308 .map((x) => x.trim())
309 .filter(Boolean);
310 }
311
312 // Permissions
313 if (children.some((x) => x.basename === ".friends")) {
314 FilePermissions.setPermissions(dir.path, 1);
315 } else {
316 FilePermissions.setPermissions(dir.path, 0);
317 }
318
319 // Recursive stats.
320 let totalSize = 0;
321 let newestDate = new Date(0);
322 let allHashes = "";
323 for (const child of children) {
324 totalSize += child.size;
325 allHashes += child.hash;
326
327 if (child.basename !== "/readme.txt" && child.date > newestDate) {
328 newestDate = child.date;
329 }
330 }
331
332 // Project Date
333 const dateFile = children.find((x) => x.basename === ".date");
334 if (dateFile) {
335 const date = new Date(dateFile.contents);
336 newestDate = date;
337 dir.setProcessors(
338 0,
339 JSON.stringify({ hideChildrenDates: true }),
340 );
341 } else {
342 dir.setProcessors(0, "");
343 }
344
345 const dirHash = crypto
346 .createHash("sha1")
347 .update(dir.path + allHashes)
348 .digest("hex");
349
350 MediaFile.markDirectoryProcessed({
351 id: dir.id,
352 timestamp: newestDate,
353 contents: readmeContent,
354 size: totalSize,
355 hash: dirHash,
356 dirsort,
357 });
358 }
359 dirMetaNode.end();
360
361 // Sync to remote
362 if (
363 ((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
364 ) {
365 await rsync.spawn({
366 args: [
367 "--links",
368 "--recursive",
369 "--times",
370 "--partial",
371 "--progress",
372 // "--remove-source-files",
373 "--delay-updates",
374 "--exclude=tmp.*",
375 derived.workDir.toString() + "/",
376 "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/derived/",
377 ],
378 progress: progress.start("upload derived assets"),
379 cwd: process.cwd(),
380 });
381
382 await fs.removeEmptyDirectories(derived.workDir.toString());
383 } else {
384 console.info("No new derived assets");
385 }
386
387 MediaFile.db.prepare("VACUUM").run();
388 MediaFile.db.reload();
389
390 await rsync.spawn({
391 args: [
392 MediaFile.db.file,
393 "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
394 ],
395 progress: progress.start("Uploading database (source of truth)"),
396 cwd: process.cwd(),
397 });
398 await rsync.spawn({
399 args: [
400 MediaFile.db.file,
401 "clo@paperclover.net:paperclover.net/.clover/cache.sqlite",
402 ],
403 progress: progress.start("Uploading database (web node)"),
404 });
405 if (sotToken) {
406 const res = await fetch("https://db.paperclover.net/reload", {
407 method: "POST",
408 headers: {
409 Authorization: sotToken,
410 },
411 });
412 if (!res.ok) {
413 console.warn(
414 `Failed to reload remote database ${res.status} ${res.statusText}`,
415 );
416 }
417 } else console.warn("Missing SOT token");
418
419 console.info(
420 "Updated file viewer index in \x1b[1m" +
421 ((performance.now() - start) / 1000).toFixed(1) +
422 "s\x1b[0m",
423 );
424
425 const { duration, count } = MediaFile.db
426 .prepare<[], { count: number; duration: number }>(`
427 select
428 count(*) as count,
429 sum(duration) as duration
430 from media_files
431 `)
432 .getNonNull();
433 const { derivedSize, derivedCount } = MediaFile.db
434 .prepare<[], { derivedCount: number; derivedSize: number }>(`
435 select
436 count(*) as derivedCount,
437 sum(size) as derivedSize
438 from derived_files
439 `)
440 .getNonNull();
441 const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
442
443 console.info();
444 console.info(
445 "Global Stats:\n" +
446 `- File Count: \x1b[1m${count}\x1b[0m\n` +
447 `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n` +
448 `- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n` +
449 `- Canonical Size: \x1b[1m${formatSize(canonicalSize)}\x1b[0m\n` +
450 `- Derived Size: \x1b[1m${formatSize(derivedSize)}\x1b[0m\n`,
451 );
452}
453
454interface Process {
455 name: string;
456 cores?: number;
457 enable?: boolean;
458 include?: Set<string>;
459 exclude?: Set<string>;
460 depends?: string[];
461 version?: number;
462 /* Perform an action. */
463 run(args: ProcessFileArgs): Promise<void>;
464}
465
466const ffprobeBin = testProgram("ffprobe", "--help");
467const ffmpegBin = testProgram("ffmpeg", "--help");
468
469const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
470
471// NOTE: Never re-order the processors. Add new ones at the end.
472const procDuration: Process = {
473 name: "calculate duration",
474 enable: ffprobeBin !== null,
475 include: rules.extsDuration,
476 cores: 1,
477 async run({ path, mediaFile }) {
478 const { stdout } = await subprocess.exec(ffprobeBin!, [
479 "-v",
480 "error",
481 "-show_entries",
482 "format=duration",
483 "-of",
484 "default=noprint_wrappers=1:nokey=1",
485 path.toString(),
486 ]);
487
488 const duration = parseFloat(stdout.trim());
489 if (Number.isNaN(duration)) {
490 throw new Error("Could not extract duration from " + stdout);
491 }
492 mediaFile.setDuration(Math.ceil(duration));
493 },
494};
495
496const procDimensions: Process = {
497 name: "calculate dimensions",
498 enable: ffprobeBin != null,
499 include: rules.extsDimensions,
500 cores: 1,
501 async run({ path, mediaFile }) {
502 const { ext } = path;
503
504 let dimensions;
505
506 if (ext === ".svg") {
507 // Parse out of text data
508 const content = await path.read("utf-8");
509 const widthMatch = content.match(/width="(\d+)"/);
510 const heightMatch = content.match(/height="(\d+)"/);
511
512 if (widthMatch && heightMatch) {
513 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
514 }
515 } else {
516 // Use ffprobe to observe streams
517 const { stdout } = await subprocess.exec("ffprobe", [
518 "-v",
519 "error",
520 "-select_streams",
521 "v:0",
522 "-show_entries",
523 "stream=width,height",
524 "-of",
525 "csv=s=x:p=0",
526 path.toString(),
527 ]);
528 if (stdout.includes("x")) {
529 dimensions = stdout.trim();
530 }
531 }
532
533 mediaFile.setDimensions(dimensions ?? "");
534 },
535};
536
537const procLoadTextContents: Process = {
538 name: "load text content",
539 include: rules.extsReadContents,
540 cores: 1,
541 version: 2,
542 async run({ path, mediaFile, stat }) {
543 if (stat.size > 1_000_000) return;
544 const text = await path.read("utf-8");
545 mediaFile.setContents(text);
546 },
547};
548
549const procHighlightCode: Process = {
550 name: "highlight source code",
551 include: new Set(rules.extsCode.keys()),
552 cores: 1,
553 version: 2,
554 async run({ path, mediaFile, stat }) {
555 const language = UNWRAP(
556 rules.extsCode.get(path.ext.toLowerCase()),
557 );
558 // An issue is that .ts is an overloaded extension, shared between
559 // 'transport stream' and 'typescript'.
560 //
561 // Filter used here is:
562 // - more than 1mb
563 // - invalid UTF-8
564 if (stat.size > 1_000_000) return;
565 let code;
566 const buf = await path.read();
567 try {
568 code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
569 } catch (error) {
570 mediaFile.setContents("");
571 return;
572 }
573 const content = await highlight.highlightCode(code, language);
574 mediaFile.setContents(content);
575 },
576};
577
578const procImageSubsets: Process = {
579 name: "encode image subsets",
580 include: rules.extsImage,
581 depends: [procDimensions.name],
582 version: 2,
583 async run({ path, mediaFile, node }) {
584 const { width, height } = UNWRAP(mediaFile.parseDimensions());
585 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
586 const baseStatus = node.text;
587
588 using stack = new DisposableStack();
589 for (const size of targetSizes) {
590 const { w, h } = resizeDimensions(width, height, size);
591 for (const { ext, args } of transcodeRules.imagePresets) {
592 node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
593
594 stack.use(
595 await derived.produce({
596 mediaFile,
597 node,
598 cores: 2,
599 subkey: `${size}${ext}`,
600 async producer(dir) {
601 await subprocess.exec(ffmpegBin!, [
602 ...ffmpegOptions,
603 "-i",
604 path.toString(),
605 "-vf",
606 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
607 ...args,
608 dir.join(`${size}${ext}`).toString(),
609 ]);
610 },
611 }),
612 );
613 }
614 }
615
616 stack.move();
617 },
618};
619
620const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
621
622const qualityMap: Record<string, string> = {
623 u: "ultra-high",
624 h: "high",
625 m: "medium",
626 l: "low",
627 d: "data-saving",
628};
629const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
630 name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
631 include: rules.extsVideo,
632 enable: ffmpegBin != null,
633 depends: [procDuration.name, procDimensions.name],
634 async run({ path, mediaFile, node }) {
635 if ((mediaFile.duration ?? 0) < 5) return;
636 if (!mediaFile.dimensions) return;
637
638 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
639
640 await derived.produce({
641 mediaFile,
642 node,
643 cores: 4,
644 subkey: `av1-${preset.id}`,
645 async producer(dir) {
646 const input = await videoArgsCache.getOrRun(path);
647 const args = transcodeRules.getAv1VideoArgs(
648 preset,
649 UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
650 dir,
651 );
652
653 await ffmpeg.spawn({
654 ffmpeg: ffmpegBin!,
655 progress: node,
656 args,
657 cwd: dir,
658 });
659 },
660 });
661 },
662}));
663const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
664 name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
665 include: rules.extsVideo,
666 enable: ffmpegBin != null,
667 depends: [procDuration.name, procDimensions.name],
668 version: 2,
669 async run({ path, mediaFile, node }) {
670 if ((mediaFile.duration ?? 0) < 5) return;
671 if (!mediaFile.dimensions) return;
672 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
673 await derived.produce({
674 mediaFile,
675 node,
676 cores: 4,
677 subkey: `opus-${preset.id}`,
678 async producer(dir) {
679 const input = await videoArgsCache.getOrRun(path);
680 ASSERT(input.video);
681 if (!input.audio) return;
682 const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
683
684 await ffmpeg.spawn({
685 ffmpeg: ffmpegBin!,
686 progress: node,
687 args,
688 cwd: dir,
689 });
690 },
691 });
692 },
693}));
694const procDash: Process = {
695 name: `encode mpeg-dash`,
696 include: rules.extsVideo,
697 enable: ffmpegBin != null,
698 depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
699 version: 4,
700 async run({ path, mediaFile, node }) {
701 if ((mediaFile.duration ?? 0) < 5) return;
702 if (!mediaFile.dimensions) return;
703
704 if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
705 await derived.produce({
706 mediaFile,
707 node,
708 cores: 1,
709 subkey: `dash-av1`,
710 async producer(dir) {
711 const input = await videoArgsCache.getOrRun(path);
712 const videos = transcodeRules.videoFormats.map(
713 (preset) =>
714 UNWRAP(dir.parent).join(
715 `av1-${preset.id}`,
716 transcodeRules.av1FileName,
717 ),
718 );
719 const audios = input.audio
720 ? transcodeRules.audioFormats.map(
721 (preset) =>
722 UNWRAP(dir.parent).join(
723 `opus-${preset.id}`,
724 transcodeRules.opusFileName,
725 ),
726 )
727 : [];
728 const args = transcodeRules.getMpegDashArgs(
729 videos,
730 audios,
731 dir,
732 );
733
734 await dir.join("d").makeDir();
735
736 await ffmpeg.spawn({
737 ffmpeg: ffmpegBin!,
738 progress: node,
739 args,
740 cwd: dir,
741 });
742 },
743 });
744 },
745};
746const procH264Hls: Process = {
747 name: `encode h.264 hls`,
748 include: rules.extsVideo,
749 enable: ffmpegBin != null,
750 depends: [procDuration.name, procDimensions.name],
751 async run({ path, mediaFile, node }) {
752 if ((mediaFile.duration ?? 0) < 5) return;
753 if (!mediaFile.dimensions) return;
754
755 await derived.produce({
756 mediaFile,
757 node,
758 cores: 8,
759 subkey: `hls`,
760 async producer(dir) {
761 const input = await videoArgsCache.getOrRun(path);
762 const args = transcodeRules.getH264HlsArgs(input, dir);
763
764 await ffmpeg.spawn({
765 ffmpeg: ffmpegBin!,
766 progress: node,
767 args,
768 cwd: dir,
769 });
770 },
771 });
772 },
773};
774
775const procCompression = [
776 { name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
777 { name: "zstd", fn: () => zlib.createZstdCompress() },
778].map<Process>(
779 ({ name, fn }) => ({
780 name: `compress ${name}`,
781 exclude: rules.extsPreCompressed,
782 async run({ path, mediaFile, node }) {
783 await derived.produce({
784 mediaFile,
785 node,
786 cores: 1,
787 subkey: name,
788 async producer(dir) {
789 await stream.promises.pipeline(
790 fs.createReadStream(path.toString()),
791 fn(),
792 fs.createWriteStream(dir.join(name).toString()),
793 );
794 },
795 });
796 },
797 }),
798);
799
800const processors = [
801 procDimensions,
802 procDuration,
803 procLoadTextContents,
804 procHighlightCode,
805 procImageSubsets,
806 ...procVideos,
807 ...procVideoAudios,
808 procDash,
809 ...procCompression,
810 procH264Hls,
811].map((process, id, all) => {
812 const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
813 return {
814 ...(process as Process),
815 id: strIndex(id),
816 // Create a unique key.
817 hash: new Uint16Array(
818 crypto
819 .createHash("sha1")
820 .update(
821 process.run.toString() +
822 (process.version ? String(process.version) : ""),
823 )
824 .digest().buffer,
825 ).reduce((a, b) => a ^ b),
826 depends: (process.depends ?? []).map((depend) => {
827 const index = all.findIndex((p) => p.name === depend);
828 if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
829 if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
830 return strIndex(index);
831 }),
832 };
833});
834
835function resizeDimensions(w: number, h: number, desiredWidth: number) {
836 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
837 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
838}
839
840interface UpdateMetadataArgs {
841 path: Path;
842 publicPath: string;
843 stat: fs.Stats;
844 mediaFile: MediaFile | null;
845}
846
847interface ProcessFileArgs {
848 path: Path;
849 stat: fs.Stats;
850 mediaFile: MediaFile;
851 node: progress.Node;
852}
853
854interface ProcessJob {
855 path: Path;
856 stat: fs.Stats;
857 mediaFile: MediaFile;
858 processor: (typeof processors)[0];
859 index: number;
860 after: ProcessJob[];
861 needs: number;
862 fileNode: progress.Node;
863}
864
865function skipBasename(basename: string): boolean {
866 // dot files must be incrementally tracked
867 if (basename === ".dirsort") return false;
868 if (basename === ".friends") return false;
869 if (basename === ".date") return false;
870
871 return (
872 basename.startsWith(".") ||
873 // basename.startsWith("._") ||
874 // basename.startsWith(".tmp") ||
875 // basename === ".DS_Store" ||
876 basename.toLowerCase() === "thumbs.db" ||
877 basename.toLowerCase() === "desktop.ini"
878 );
879}
880
881function toPublicPath(diskPath: Path) {
882 if (diskPath.toString() === root) return "/";
883 return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
884}
885
886function testProgram(name: string, helpArgument: string) {
887 try {
888 child_process.spawnSync(name, [helpArgument]);
889 return name;
890 } catch (err) {
891 console.warn(`Missing or corrupt executable '${name}'`);
892 }
893 return null;
894}
895
896// Helper function to check and remove location metadata
897async function scrubLocationMetadata(
898 path: Path,
899 stats: fs.Stats,
900 progress: progress.Ref,
901): Promise<boolean> {
902 using _ = progress.start("scrub exif metadata");
903 const ext = path.ext.toLowerCase();
904 if (!rules.extsScrubExif.has(ext)) return false;
905
906 let hasLocation = false;
907 let args: string[] = [];
908
909 // Check for location metadata based on file type
910 const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
911 switch (ext) {
912 case ".jpg":
913 case ".jpeg":
914 case ".png":
915 const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
916 "-gps:all",
917 path.toString(),
918 ]);
919 hasLocation = gpsCheck.trim().length > 0;
920 args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
921 break;
922 case ".mov":
923 case ".mp4":
924 const { stdout: videoCheck } = await subprocess.exec("exiftool", [
925 "-ee",
926 "-G3",
927 "-s",
928 path.toString(),
929 ]);
930 hasLocation = videoCheck.includes("GPS") ||
931 videoCheck.includes("Location");
932 args = [
933 "-gps:all=",
934 "-xmp:all=",
935 path.toString(),
936 "-o",
937 tempOutput.toString(),
938 ];
939 break;
940 case ".m4a":
941 const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
942 "-ee",
943 "-G3",
944 "-s",
945 path.toString(),
946 ]);
947 hasLocation = m4aCheck.includes("GPS") ||
948 m4aCheck.includes("Location") ||
949 m4aCheck.includes("Filename") ||
950 m4aCheck.includes("Title");
951
952 if (hasLocation) {
953 args = [
954 "-gps:all=",
955 "-location:all=",
956 "-filename:all=",
957 "-title=",
958 "-m4a:all=",
959 path.toString(),
960 "-o",
961 tempOutput.toString(),
962 ];
963 }
964 break;
965 }
966
967 const accessTime = stats.atime;
968 const modTime = stats.mtime;
969
970 let backup: Path | null = null;
971 try {
972 if (hasLocation) {
973 // Prepare a backup
974 const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
975 await fsp.copyFile(path.toString(), tmp.toString());
976 await fsp.utimes(tmp.toString(), accessTime, modTime);
977 backup = tmp;
978
979 // Remove metadata
980 await subprocess.exec("exiftool", args);
981 if (!tempOutput.ifExistsSync()) {
982 throw new Error(`Failed to create output file: ${tempOutput}`);
983 }
984
985 // Restore original timestamps
986 await fsp.rename(tempOutput.toString(), path.toString());
987 await fsp.utimes(path.toString(), accessTime, modTime);
988
989 // Backup is no longer needed
990 await fsp.unlink(backup.toString());
991
992 console.info(
993 `Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
994 );
995 return true;
996 }
997 } catch (error) {
998 if (backup) {
999 await fsp.rename(backup.toString(), path.toString());
1000 }
1001 if (fs.existsSync(tempOutput.toString())) {
1002 await fsp.unlink(tempOutput.toString());
1003 }
1004 throw error;
1005 }
1006
1007 return false;
1008}
1009
1010const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
1011
1012import * as async from "lib/async.ts";
1013import * as fs from "#sitegen/fs";
1014import * as subprocess from "lib/subprocess.ts";
1015import { Path } from "#sitegen/path";
1016import * as queue from "../../../lib/queue.ts";
1017import * as log from "../../../lib/log.ts";
1018import * as progress from "lib/progress.ts";
1019import * as ts from "../../../lib/ts.ts";
1020
1021import * as child_process from "node:child_process";
1022import * as crypto from "node:crypto";
1023import * as fsp from "node:fs/promises";
1024import * as path from "node:path";
1025import * as stream from "node:stream";
1026import * as zlib from "node:zlib";
1027
1028import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
1029import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
1030import {
1031 formatDate,
1032 formatDurationLong,
1033 formatSize,
1034} from "@/file-viewer/format.ts";
1035import * as rules from "@/file-viewer/rules.ts";
1036import * as highlight from "@/file-viewer/highlight.ts";
1037import * as ffmpeg from "lib/subprocess/ffmpeg.ts";
1038import * as rsync from "@/file-viewer/rsync.ts";
1039import * as derived from "@/file-viewer/models/derived.ts";
1040import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
1041
1042import { rawFileRoot as root } from "../paths.ts";
1043import { ASSERT, UNWRAP } from "lib/assert.ts";
src/file-viewer/models/MediaFile.ts+8-5
...@@ -224,6 +224,7 @@ export class MediaFile {...@@ -224,6 +224,7 @@ export class MediaFile {
224 return createDirectoryQuery.getNonNull({224 return createDirectoryQuery.getNonNull({
225 path: filePath,225 path: filePath,
226 parentId,226 parentId,
227 time: Date.now(),
227 }).id;228 }).id;
228 }229 }
229 // walk up the path until we find a directory that exists230 // walk up the path until we find a directory that exists
...@@ -236,6 +237,7 @@ export class MediaFile {...@@ -236,6 +237,7 @@ export class MediaFile {
236 parentId = createDirectoryQuery.getNonNull({237 parentId = createDirectoryQuery.getNonNull({
237 path: current,238 path: current,
238 parentId,239 parentId,
240 time: Date.now(),
239 }).id;241 }).id;
240 }242 }
241 // walk back down the path, creating directories as needed243 // walk back down the path, creating directories as needed
...@@ -245,6 +247,7 @@ export class MediaFile {...@@ -245,6 +247,7 @@ export class MediaFile {
245 parentId = createDirectoryQuery.getNonNull({247 parentId = createDirectoryQuery.getNonNull({
246 path: current,248 path: current,
247 parentId,249 parentId,
250 time: Date.now(),
248 }).id;251 }).id;
249 }252 }
250 return parentId;253 return parentId;
...@@ -314,16 +317,16 @@ export interface DirConfig {...@@ -314,16 +317,16 @@ export interface DirConfig {
314317
315// Get a directory ID by path, creating it if it doesn't exist318// Get a directory ID by path, creating it if it doesn't exist
316const createDirectoryQuery = db.prepare<319const createDirectoryQuery = db.prepare<
317 [{ path: string; parentId: number | null }],320 [{ path: string; parentId: number | null; time: number }],
318 { id: number }321 { id: number }
319>(322>(
320 /* SQL */ `323 /* SQL */ `
321 insert into media_files (324 insert into media_files (
322 path, parent_id, kind, timestamp, hash, size,325 path, parent_id, kind, timestamp, timestamp_updated, hash,
323 duration, dimensions, contents, dirsort, processed)326 size, duration, dimensions, contents, dirsort, processed)
324 values (327 values (
325 $path, $parentId, ${MediaFileKind.directory}, 0, '', 0,328 $path, $parentId, ${MediaFileKind.directory}, 0, $time, '',
326 0, '', '', '', 0)329 0, 0, '', '', '', 0)
327 returning id;330 returning id;
328`,331`,
329);332);
src/file-viewer/models/derived.ts+18
...@@ -165,6 +165,14 @@ export function get(file: MediaFile, subPath: string) {...@@ -165,6 +165,14 @@ export function get(file: MediaFile, subPath: string) {
165 return null;165 return null;
166}166}
167167
168export function findOrphanedRoots() {
169 return findOrphanedRootsQuery.array();
170}
171
172export function deleteRoot(root: { id: number }) {
173 return deleteRootQuery.run({ root: root.id });
174}
175
168const insertRootQuery = db.prepare<176const insertRootQuery = db.prepare<
169 [{ key: string; date: number }],177 [{ key: string; date: number }],
170 { id: number }178 { id: number }
...@@ -194,6 +202,16 @@ const deleteRefQuery = db.prepare<[{ root: number; file: number }]>(/* SQL */ `...@@ -194,6 +202,16 @@ const deleteRefQuery = db.prepare<[{ root: number; file: number }]>(/* SQL */ `
194 delete from derived_refs where root = $root and file = $file;202 delete from derived_refs where root = $root and file = $file;
195`);203`);
196204
205const findOrphanedRootsQuery = db.prepare<
206 [],
207 { id: number; key: string; date: number }
208>(/* SQL */ `
209 select dr.id, dr.key, dr.date
210 from derived_roots dr
211 left join derived_refs refs on refs.root = dr.id
212 where refs.id is null
213`);
214
197import { getDb } from "#sitegen/sqlite";215import { getDb } from "#sitegen/sqlite";
198import type { MediaFile } from "./MediaFile.ts";216import type { MediaFile } from "./MediaFile.ts";
199import { Path } from "#sitegen/path";217import { Path } from "#sitegen/path";
src/file-viewer/rsync.ts+5-4
...@@ -10,7 +10,7 @@ export type Line =...@@ -10,7 +10,7 @@ export type Line =
10 currentFile: string;10 currentFile: string;
11 bytesTransferred: number;11 bytesTransferred: number;
12 percentage: number;12 percentage: number;
13 timeElapsed: string;13 timeElapsed: string | null;
14 transferNumber: number;14 transferNumber: number;
15 filesToCheck: number;15 filesToCheck: number;
16 totalFiles: number;16 totalFiles: number;
...@@ -91,7 +91,7 @@ export async function spawn(options: SpawnOptions) {...@@ -91,7 +91,7 @@ export async function spawn(options: SpawnOptions) {
91 e.args = [rsync, ...args].join(" ");91 e.args = [rsync, ...args].join(" ");
92 e.code = code;92 e.code = code;
93 e.signal = signal;93 e.signal = signal;
94 return e;94 throw e;
95 }95 }
96}96}
9797
...@@ -126,9 +126,9 @@ export class Parse {...@@ -126,9 +126,9 @@ export class Parse {
126 return {126 return {
127 kind: "progress",127 kind: "progress",
128 currentFile: this.lastSeenFile || "",128 currentFile: this.lastSeenFile || "",
129 bytesTransferred: Number(bytesStr.replaceAll(",", "")),129 bytesTransferred: Number(UNWRAP(bytesStr).replaceAll(",", "")),
130 percentage: Number(percentageStr),130 percentage: Number(percentageStr),
131 timeElapsed,131 timeElapsed: timeElapsed ?? null,
132 transferNumber: this.currentTransfer,132 transferNumber: this.currentTransfer,
133 filesToCheck: toCheckStr133 filesToCheck: toCheckStr
134 ? this.toCheck = Number(toCheckStr)134 ? this.toCheck = Number(toCheckStr)
...@@ -185,3 +185,4 @@ import * as readline from "node:readline";...@@ -185,3 +185,4 @@ import * as readline from "node:readline";
185import events from "node:events";185import events from "node:events";
186import * as progress from "lib/progress.ts";186import * as progress from "lib/progress.ts";
187import { formatSize } from "@/file-viewer/format.ts";187import { formatSize } from "@/file-viewer/format.ts";
188import { UNWRAP } from "lib/assert.ts";
src/friend-auth.ts+57-36
...@@ -1,75 +1,96 @@...@@ -1,75 +1,96 @@
1let friendPassword = "";1let hardcoded = {
2 friendPassword: "",
3 perPage: {} as Record<string, string>,
4 getForFile: (file: string) => [] as string[],
5};
2try {6try {
3 friendPassword = require("./friends/hardcoded-password.ts").friendPassword;7 hardcoded = require("./friends/hardcoded-password.ts");
4} catch {}8} catch {}
59
6export const app = new Hono();10export const app = new Hono();
711
8const cookieAge = 60 * 60 * 24 * 30; // 1 month12const cookieAge = 60 * 60 * 24 * 30; // 1 month
913
10function checkFriendsCookie(c: Context) {14export const getForFile = hardcoded.getForFile;
15
16function checkFriendsCookie(c: Context, passwords: string[]) {
11 const cookie = c.req.header("Cookie");17 const cookie = c.req.header("Cookie");
12 if (!cookie) return false;18 if (!cookie) return false;
13 const cookies = cookie.split("; ").map((x) => x.split("="));19 const cookies = cookie.split("; ").map((x) => x.split("="));
14 return cookies.some(20 return cookies.some(
15 (kv) =>21 (kv) =>
16 kv[0].trim() === "friends_password" &&22 kv[0]!.trim() === "friends_password" &&
17 kv[1].trim() &&23 kv[1]!.trim() &&
18 kv[1].trim() === friendPassword,24 passwords.includes(kv[1]!.trim()),
19 );25 );
20}26}
2127
22export function requireFriendAuth(c: Context) {28export function requireFriendAuth(
29 c: Context,
30 passwords = [hardcoded.friendPassword],
31) {
32 if (passwords.length === 0 || passwords[0]!.length === 0) {
33 return serveAsset(c, "/friends/misconfigure", 501);
34 }
23 const k = c.req.query("password") || c.req.query("k");35 const k = c.req.query("password") || c.req.query("k");
24 if (k) {36 if (k) {
25 if (k === friendPassword) {37 if (passwords.includes(k)) {
26 return c.body(null, 303, {38 return c.body(null, 303, {
27 Location: "/friends",39 Location: c.req.path,
28 "Set-Cookie":40 "Set-Cookie":
29 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,41 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
30 });42 });
31 } else {43 } else {
32 return c.body(null, 303, {44 return c.body(null, 303, {
33 Location: "/friends",45 Location: c.req.path,
34 });46 });
35 }47 }
36 }48 }
37 if (checkFriendsCookie(c)) {49 if (checkFriendsCookie(c, passwords)) {
38 return undefined;50 return undefined;
39 } else {51 } else {
40 return serveAsset(c, "/friends/auth", 403);52 return serveAsset(c, "/friends/auth", 403);
41 }53 }
42}54}
4355
44app.get("/friends", (c) => {
45 const friendAuthChallenge = requireFriendAuth(c);
46 if (friendAuthChallenge) return friendAuthChallenge;
47 return serveAsset(c, "/friends", 200);
48});
49
50let incorrectMap: Record<string, boolean> = {};56let incorrectMap: Record<string, boolean> = {};
51app.post("/friends", async (c) => {57
52 const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??58function friendPage(route: assets.Key) {
53 "unknown";59 const expected = hardcoded.perPage[route] ?? hardcoded.friendPassword;
54 if (incorrectMap[ip]) {60
61 app.get(route, (c) => {
62 const friendAuthChallenge = requireFriendAuth(c, [expected]);
63 if (friendAuthChallenge) return friendAuthChallenge;
64 return serveAsset(c, route, 200);
65 });
66
67 app.post(route, async (c) => {
68 const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??
69 "unknown";
70 if (incorrectMap[ip]) {
71 return serveAsset(c, "/friends/auth/fail", 403);
72 }
73 const data = await c.req.formData();
74 const k = data.get("password");
75 if (k === expected) {
76 return c.body(null, 303, {
77 Location: c.req.path,
78 "Set-Cookie":
79 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
80 });
81 }
82 incorrectMap[ip] = true;
83 await setTimeout(2500);
84 incorrectMap[ip] = false;
55 return serveAsset(c, "/friends/auth/fail", 403);85 return serveAsset(c, "/friends/auth/fail", 403);
56 }86 });
57 const data = await c.req.formData();87}
58 const k = data.get("password");88
59 if (k === friendPassword) {89friendPage("/friends");
60 return c.body(null, 303, {90friendPage("/friends/oct25");
61 Location: "/friends",
62 "Set-Cookie":
63 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
64 });
65 }
66 incorrectMap[ip] = true;
67 await setTimeout(2500);
68 incorrectMap[ip] = false;
69 return serveAsset(c, "/friends/auth/fail", 403);
70});
7191
72import { type Context, Hono } from "hono";92import { type Context, Hono } from "hono";
73import { serveAsset } from "#sitegen/assets";93import { serveAsset } from "#sitegen/assets";
74import { setTimeout } from "node:timers/promises";94import { setTimeout } from "node:timers/promises";
75import { getConnInfo } from "#hono/conninfo";95import { getConnInfo } from "#hono/conninfo";
96import * as assets from "#sitegen/assets";
src/pages/friends/auth.fail.marko+2-2
...@@ -3,9 +3,9 @@ export const meta = { title: "password required" };...@@ -3,9 +3,9 @@ export const meta = { title: "password required" };
3<main>3<main>
4 <p>incorrect or outdated password</p>4 <p>incorrect or outdated password</p>
5 <p>please contact clover</p>5 <p>please contact clover</p>
6 <form method="post" action="/friends">6 <form method="post">
7 <h1>try again</h1>7 <h1>try again</h1>
8 <input type="password" name="password"/>8 <input type="password" name="password" placeholder="password" />
9 <button type="submit">enter</button>9 <button type="submit">enter</button>
10 </form>10 </form>
11</main>11</main>
src/pages/friends/auth.marko+2-2
...@@ -1,9 +1,9 @@...@@ -1,9 +1,9 @@
1export const meta = { title: "password required" };1export const meta = { title: "password required" };
22
3<main>3<main>
4 <form method="post" action="/friends">4 <form method="post">
5 <h1>what's the password</h1>5 <h1>what's the password</h1>
6 <input type="password" name="password"/>6 <input type="password" name="password" placeholder="password"/>
7 <button type="submit">enter</button>7 <button type="submit">enter</button>
8 <p>(clover will give the password to you if you're a friend)</p>8 <p>(clover will give the password to you if you're a friend)</p>
9 </form>9 </form>
src/pages/friends/misconfigure.marko created+10
...@@ -0,0 +1,10 @@
1export const meta = { title: "password required" };
2
3<main>
4 <form method="post" action="/friends">
5 <h1>what's the password</h1>
6 <input type="password" name="password" placeholder="password" disabled />
7 <button type="submit" disabled>enter</button>
8 <p style="color:red">this page's password is not configured</p>
9 </form>
10</main>
src/tags/clover-video.client.ts+3-1
...@@ -67,7 +67,9 @@ function hydrateVideoInner(...@@ -67,7 +67,9 @@ function hydrateVideoInner(
67 container: HTMLElement,67 container: HTMLElement,
68 mode: PlaybackMode,68 mode: PlaybackMode,
69) {69) {
70 const video = container.querySelector("video")!;70 const video = container instanceof HTMLVideoElement
71 ? container
72 : container.querySelector("video")!;
71 if (video.hasAttribute("controls")) return;73 if (video.hasAttribute("controls")) return;
72 let src = video.getAttribute("data-src");74 let src = video.getAttribute("data-src");
73 if (!src) return;75 if (!src) return;
src/tags/clover-video.marko+46-35
...@@ -5,9 +5,10 @@ export interface Input {...@@ -5,9 +5,10 @@ export interface Input {
5 poster?: MediaFile | string;5 poster?: MediaFile | string;
6 header?: Marko.AttrTag<{ content: Marko.Body }>;6 header?: Marko.AttrTag<{ content: Marko.Body }>;
7 minimal?: boolean;7 minimal?: boolean;
8 noFigure?: boolean
8}9}
910
10<const/{ header, minimal } = input />11<const/{ header, minimal, noFigure } = input />
11<const/file = typeof UNWRAP(input.file) === 'object'12<const/file = typeof UNWRAP(input.file) === 'object'
12 ? input.file13 ? input.file
13 : UNWRAP(MediaFile.getByPath(input.file), `No file ${JSON.stringify(input.file)}`) />14 : UNWRAP(MediaFile.getByPath(input.file), `No file ${JSON.stringify(input.file)}`) />
...@@ -19,40 +20,50 @@ export interface Input {...@@ -19,40 +20,50 @@ export interface Input {
1920
20<const/{ width, height } = file.parseDimensions() />21<const/{ width, height } = file.parseDimensions() />
2122
22<figure.clover-video class={ minimal }>23<define/Inner>
23 <if=!minimal>24 <video
24 <figcaption>25 class={ 'clover-video': noFigure }
25 <if=header><${header.content}/></><else>26 data-src=file.path
26 ${file.basename}27 poster=poster ? `/file${poster.path}` : null
27 </else>28 width=width
28 </figcaption>29 height=height
29 </>30 />
30 <div.stack style=`aspect-ratio:${width}/${height}`>31</define>
31 <video32
32 data-src=file.path33<if=!noFigure>
33 width=192034 <figure.clover-video class={ minimal }>
34 height=108035 <if=!minimal>
35 poster=poster ? `/file${poster.path}` : null36 <figcaption>
36 />37 <if=header><${header.content}/></><else>
37 <noscript.fallback>38 ${file.basename}
38 <h2>scripting disabled!</h2>39 </else>
39 <p>40 </figcaption>
40 to play this video, JavaScript must be enabled in your41 </>
41 browser. the default video playback behavior is not enabled here42 <div.stack style=`aspect-ratio:${width}/${height}`>
42 as it results in a worse experience on slower internet43 <Inner />
43 connections, and <a href="http://caniuse.com/mpeg-dash">MPEG-DASH44 <noscript.fallback>
44 streaming is not natively supported in any browser</a> without JavaScript.45 <h2>scripting disabled!</h2>
45 </p>46 <p>
46 <p>47 to play this video, JavaScript must be enabled in your
47 since disabling JavaScript is not the default in any browser,48 browser. the default video playback behavior is not enabled here
48 you likely wanted to download the raw video file anyways.49 as it results in a worse experience on slower internet
49 those links are available below, but note that you will need50 connections, and <a href="http://caniuse.com/mpeg-dash">MPEG-DASH
50 an AV1-compatible video player, such as VLC Media Player or51 streaming is not natively supported in any browser</a> without JavaScript.
51 QuickTime Player (M3 chip or newer)52 </p>
52 </p>53 <p>
53 </noscript>54 since disabling JavaScript is not the default in any browser,
54 </div>55 you likely wanted to download the raw video file anyways.
55</figure>56 those links are available below, but note that you will need
57 an AV1-compatible video player, such as VLC Media Player or
58 QuickTime Player (M3 chip or newer)
59 </p>
60 </noscript>
61 </div>
62 </figure>
63</>
64<else>
65 <Inner />
66</>
5667
57client import "./clover-video.client.ts";68client import "./clover-video.client.ts";
5869
src/tags/image.tsx+6
...@@ -16,6 +16,10 @@ export default function Image(...@@ -16,6 +16,10 @@ export default function Image(
16 ? src16 ? src
17 : UNWRAP(MediaFile.getByPath(src), `No file ${JSON.stringify(src)}`);17 : UNWRAP(MediaFile.getByPath(src), `No file ${JSON.stringify(src)}`);
1818
19 if (extsVideo.has(src.extension.toLowerCase())) {
20 return <CloverVideo noFigure file={src} />;
21 }
22
19 const base = `/file${escapeUri(src.path)}`;23 const base = `/file${escapeUri(src.path)}`;
20 if (src.extension === ".gif") {24 if (src.extension === ".gif") {
21 return <img src={base} class={className} style={style} {...rest} />;25 return <img src={base} class={className} style={style} {...rest} />;
...@@ -57,3 +61,5 @@ import * as transcodeRules from "@/file-viewer/transcode-rules.ts";...@@ -57,3 +61,5 @@ import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
57import { MediaFile } from "@/file-viewer/models/MediaFile.ts";61import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
58import { escapeUri } from "@/file-viewer/format.ts";62import { escapeUri } from "@/file-viewer/format.ts";
59import { UNWRAP } from "lib/assert.ts";63import { UNWRAP } from "lib/assert.ts";
64import { extsVideo } from "@/file-viewer/rules.ts";
65import CloverVideo from "./clover-video.marko";
src/tags/photo-grid.css+21-13
...@@ -1,17 +1,25 @@...@@ -1,17 +1,25 @@
1photo-grid {1photo-grid {
2 display: grid;2 display: grid;
3 gap: 11px;3 gap: 11px;
4 div {4}
5 position: relative;5photo-grid > div {
6 width: 100%;6 position: relative;
7 height: 100%;7 width: 100%;
8 border: 2px solid black;8 height: 100%;
9 outline: 4px solid var(--primary, #000);9 border: 2px solid black;
10 }10 outline: 4px solid var(--primary, #000);
11 img {11}
12 position: absolute;12photo-grid img {
13 width: 100%;13 position: absolute;
14 height: 100%;14 width: 100%;
15 object-fit: cover;15 height: 100%;
16 }16 object-fit: cover;
17}
18photo-grid video {
19 position: absolute;
20 top: 0;
21 left: 0;
22 width: 100% !important;
23 height: 100% !important;
24 object-fit: cover;
17}25}