Commit 2bed05bdbc03b105f930feed4573a5b0768bab07 | sitegen

author	clover caruso <git@paperclover.net> 2025-11-07 00:28:14-08:00
committer	clover caruso <git@paperclover.net> 2025-11-16 14:39:58-08:00
log	2bed05bdbc03b105f930feed4573a5b0768bab07
tree	a7be464b8d6a5fcaed1d91cb6fba1180cbce8276
parent	29751666e4e62269bc3f48767bf726ac5789ced7
signature	Commit is signed but in an unrecognized format.

feat(site): work to release october personal blog

18 files changed, 1289 insertions(+), 1147 deletions(-)

flake.nix
lib
- progress.ts
src
- bin
  - download-db.ts
- file-viewer
  - backend.tsx
  - bin
    file-scan.ts
    file-trim.ts
    scan3.ts
  - models
    MediaFile.ts
    derived.ts
  - rsync.ts
- friend-auth.ts
- pages
  - friends
    auth.fail.marko
    auth.marko
    misconfigure.marko
- tags

flake.nix+2

...	...	@@ -13,9 +13,11 @@
13	13	pkgs.nodejs_24 # runtime
14	14	pkgs.deno # formatter
15	15	pkgs.python3 # for font subsetting
	16	pkgs.python3Packages.pip # for font subsetting
16	17
17	18	# paperclover.net
18	19	pkgs.exiftool
	20	pkgs.imagemagick
19	21	pkgs.ffmpeg-full
20	22	pkgs.rsync
21	23	];

lib/progress.ts+4-4

...	...	@@ -154,9 +154,9 @@ function newNode(
154	154	parent?.sortChildren && parent.children.sort(parent.sortChildren);
155	155	state.signal("draw");
156	156	}
157		const scope = log.headlessScope(() => {
158		console.log("TODO");
159		});
	157	// const scope = log.headlessScope((msg) => {
	158	// console.log("TODO");
	159	// });
160	160	const binding: Node = {
161	161	start(text, opts) {
162	162	const [child, node] = newNode(text, opts);
...	...	@@ -233,7 +233,7 @@ function newNode(
233	233	mutate();
234	234	}
235	235	},
236		log: scope,
	236	log: log.scoped(""),
237	237	end: () => void end(),
238	238	[Symbol.dispose]: () => void end(),
239	239	};

src/bin/download-db.ts created+26

...	...	@@ -0,0 +1,26 @@
	1	export async function main() {
	2	await rsync.spawn({
	3	cwd: Path.resolve(import.meta.dirname).parent!.toString(),
	4	args: [
	5	"-a",
	6	"--progress",
	7	"clo@paperclover.net:~/paperclover.net/.clover/cache.sqlite",
	8	".clover/",
	9	],
	10	progress: progress.start("download file cache"),
	11	});
	12	await rsync.spawn({
	13	cwd: Path.resolve(import.meta.dirname).parent!.toString(),
	14	args: [
	15	"-a",
	16	"--progress",
	17	"clo@paperclover.net:~/paperclover.net/.clover/questions.sqlite",
	18	".clover/",
	19	],
	20	progress: progress.start("download questions state"),
	21	});
	22	}
	23
	24	import * as progress from "lib/progress.ts";
	25	import * as rsync from "../file-viewer/rsync.ts";
	26	import { Path } from "#sitegen/path";

src/file-viewer/backend.tsx+3-2

...	...	@@ -45,7 +45,8 @@ app.get("/file/*", async (c, next) => {
45	45	// The permissions system is currently binary, using the friend auth.
46	46	const permissions = FilePermissions.getByPrefix(rawFilePath);
47	47	if (permissions !== 0) {
48		const friendAuthChallenge = requireFriendAuth(c);
	48	console.log(rawFilePath, getForFile(rawFilePath));
	49	const friendAuthChallenge = requireFriendAuth(c, getForFile(rawFilePath));
49	50	if (friendAuthChallenge) return friendAuthChallenge;
50	51	}
51	52
...	...	@@ -292,7 +293,7 @@ import { etagMatches, serveAsset } from "#sitegen/assets";
292	293	import * as view from "#sitegen/view";
293	294	import * as mime from "lib/mime.ts";
294	295
295		import { requireFriendAuth } from "@/friend-auth.ts";
	296	import { getForFile, requireFriendAuth } from "@/friend-auth.ts";
296	297	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
297	298	import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
298	299	import { MediaPanel } from "@/file-viewer/views/clofi.tsx";

src/file-viewer/bin/file-scan.ts created+1058

...	...	@@ -0,0 +1,1058 @@
	1	// The file scanner incrementally updates an sqlite database with file
	2	// stats. Additionally, it runs "processors" on files, which precompute
	3	// expensive data such as running `ffprobe` on all media to get the
	4	// duration.
	5	//
	6	// Processors are also used to derive compressed and optimized assets,
	7	// which is how automatic JXL / AV1 encoding is done. Derived files are
	8	// uploaded to the clover NAS to be pulled by VPS instances for hosting.
	9	//
	10	// This is the third iteration of the scanner, hence its name "scan3";
	11	// Remember that any software you want to be maintainable and high
	12	// quality cannot be written with AI.
	13	const sotToken = process.env.CLOVER_SOT_KEY;
	14
	15	export async function main() {
	16	const start = performance.now();
	17	using _ = log.startWidget({
	18	format: (now) =>
	19	`paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
	20	});
	21
	22	const promises = new async.PromiseAggregator();
	23
	24	const walkQueue = new queue.PriorityQueue(10);
	25
	26	const dirsNode = progress.start("Walk Tree", { estimate: 1 });
	27	dirsNode.passive = true;
	28	const fileNode = progress.start("Process File", { estimate: 0 });
	29	fileNode.sortChildren = (a, b) => {
	30	const ac = a.children.length > 0 ? 1 : 0;
	31	const bc = b.children.length > 0 ? 1 : 0;
	32	if (ac !== bc) return bc - ac;
	33	return a.text.localeCompare(b.text);
	34	};
	35
	36	// Read a directory or file stat and queue up changed files.
	37	const scanDirectory = walkQueue.wrap(
	38	async (path: Path) => {
	39	const publicPath = toPublicPath(path);
	40	using node = dirsNode.start(publicPath + " - stat");
	41	using _ = ts.defer(() => dirsNode.inc());
	42
	43	const stat = await path.stat();
	44
	45	const mediaFile = MediaFile.getByPath(publicPath);
	46
	47	if (stat.isDirectory()) {
	48	node.text = publicPath + " - reading";
	49	const items = (await path.readDir())
	50	.filter((child) => !skipBasename(child.base))
	51	.map((child) => (promises.push(scanDirectory(child)), child.base));
	52	dirsNode.estimate += items.length;
	53
	54	for (const child of mediaFile?.getChildren() ?? []) {
	55	if (items.includes(child.basename)) continue;
	56	const recursive = child.kind === MediaFileKind.directory
	57	? [child, ...child.getRecursiveFileChildren()]
	58	: [child];
	59	for (const deletion of recursive) deletion.delete();
	60	}
	61
	62	return;
	63	}
	64
	65	if (
	66	!mediaFile \|\| // All processes must be performed if there is no file.
	67	// Rerun all processors if it changed
	68	stat.size !== mediaFile.size \|\|
	69	stat.mtime.getTime() !== mediaFile.date.getTime()
	70	) {
	71	promises.push(updateMetadata({ path, publicPath, stat, mediaFile }));
	72	} else {
	73	// If the scanners changed, it may mean more processes should be run.
	74	await queueProcessors({
	75	path,
	76	stat,
	77	mediaFile,
	78	node: fileNode.start(publicPath.slice(1)),
	79	});
	80	}
	81	},
	82	);
	83	const updateMetadata = queue.wrap(
	84	async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
	85	using errorHandler = new DisposableStack();
	86	const label = publicPath.slice(1);
	87	const node = errorHandler.use(fileNode.start(label));
	88
	89	await scrubLocationMetadata(path, stat, node);
	90
	91	node.text = `${label} - hashing`;
	92	const hash = await new Promise<string>((resolve, reject) => {
	93	const reader = fs.createReadStream(path.toString());
	94	reader.on("error", reject);
	95
	96	const hasher = crypto.createHash("sha1").setEncoding("hex");
	97	hasher.on("error", reject);
	98	hasher.on("readable", () => resolve(hasher.read()));
	99
	100	reader.pipe(hasher);
	101	});
	102	let date = stat.mtime;
	103	if (
	104	mediaFile &&
	105	mediaFile.date.getTime() < stat.mtime.getTime() &&
	106	Date.now() - stat.mtime.getTime() < monthMilliseconds
	107	) {
	108	date = mediaFile.date;
	109	console.warn(
	110	`M-time on ${publicPath} was likely corrupted. ${
	111	formatDate(
	112	mediaFile.date,
	113	)
	114	} -> ${formatDate(stat.mtime)}`,
	115	);
	116	}
	117	mediaFile = MediaFile.createFile({
	118	path: publicPath,
	119	date,
	120	hash,
	121	size: stat.size,
	122	duration: mediaFile?.duration ?? 0,
	123	dimensions: mediaFile?.dimensions ?? "",
	124	contents: mediaFile?.contents ?? "",
	125	});
	126	const parent = mediaFile.getParent();
	127	if (parent) parent.setProcessed(0);
	128
	129	node.text = `${label}`;
	130	await queueProcessors({
	131	path,
	132	stat,
	133	mediaFile,
	134	node,
	135	});
	136	errorHandler.move();
	137	},
	138	() => ({ priority: -1 }),
	139	);
	140	const queueFileProcessor = queue.wrap(async function ({
	141	path,
	142	stat,
	143	mediaFile,
	144	processor,
	145	index,
	146	after,
	147	fileNode: innerNode,
	148	}: ProcessJob) {
	149	using node = innerNode.start(processor.name);
	150	await processor.run({
	151	path,
	152	stat,
	153	mediaFile,
	154	node,
	155	});
	156	fileNode.value += 1;
	157	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
	158	for (const dependantJob of after) {
	159	ASSERT(
	160	dependantJob.needs > 0,
	161	`dependantJob.needs > 0, ${dependantJob.needs}`,
	162	);
	163	dependantJob.needs -= 1;
	164	if (dependantJob.needs == 0) {
	165	promises.push(queueFileProcessor(dependantJob));
	166	}
	167	}
	168	}, (job) => ({ cores: job.processor.cores ?? 0 }));
	169
	170	function decodeProcessors(input: string) {
	171	return input
	172	.split(";")
	173	.filter(Boolean)
	174	.map(([a, b, c]) => ({
	175	id: a,
	176	hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
	177	}));
	178	}
	179
	180	async function queueProcessors({
	181	path,
	182	stat,
	183	mediaFile,
	184	node,
	185	}: Omit<ProcessFileArgs, "spin">) {
	186	using errorHandler = new DisposableStack();
	187	errorHandler.use(node);
	188
	189	node.showEstimate = false;
	190	node.passive = true;
	191
	192	const ext = mediaFile.extensionNonEmpty.toLowerCase();
	193	let possible = processors.filter((p) =>
	194	p.include ? p.include.has(ext) : !p.exclude?.has(ext)
	195	);
	196	if (possible.length === 0) return;
	197	ASSERT(possible.length < 16, "too many bits");
	198
	199	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
	200	ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
	201	let processed = mediaFile.processed;
	202
	203	// If the hash has changed, migrate the bitfield over.
	204	// This also runs when the processor hash is in it's initial 0 state.
	205	let order: ReturnType<typeof decodeProcessors>;
	206	try {
	207	order = decodeProcessors(mediaFile.processors);
	208	} catch {
	209	// this function sucks and this system sucks i hate it.
	210	order = [];
	211	}
	212	if ((processed & 0xffff) !== hash) {
	213	const previous = order.filter(
	214	(_, i) => (processed & (1 << (16 + i))) !== 0,
	215	);
	216	processed = hash;
	217	for (const { id, hash } of previous) {
	218	const p = processors.find((p) => p.id === id);
	219	if (!p) continue;
	220	const index = possible.indexOf(p);
	221	if (index !== -1 && p.hash === hash) processed \|= 1 << (16 + index);
	222	}
	223	mediaFile.setProcessors(
	224	processed,
	225	possible
	226	.map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
	227	.join(";"),
	228	);
	229	} else {
	230	possible = order.map(({ id }) =>
	231	UNWRAP(possible.find((p) => p.id === id))
	232	);
	233	}
	234
	235	// Queue needed processors.
	236	const jobs: ProcessJob[] = [];
	237	for (let i = 0, { length } = possible; i < length; i += 1) {
	238	if ((processed & (1 << (16 + i))) === 0) {
	239	const processor = UNWRAP(possible[i]);
	240	const job: ProcessJob = {
	241	path,
	242	stat,
	243	mediaFile,
	244	processor,
	245	index: i,
	246	after: [],
	247	needs: processor.depends.length,
	248	fileNode: node,
	249	};
	250	jobs.push(job);
	251	if (job.needs === 0) promises.push(queueFileProcessor(job));
	252	}
	253	}
	254	node.estimate = jobs.length;
	255	fileNode.estimate += jobs.length;
	256	for (const job of jobs) {
	257	for (const dependId of job.processor.depends) {
	258	const dependJob = jobs.find((j) => j.processor.id === dependId);
	259	if (dependJob) {
	260	dependJob.after.push(job);
	261	} else {
	262	ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
	263	job.needs -= 1;
	264	if (job.needs === 0) promises.push(queueFileProcessor(job));
	265	}
	266	}
	267	}
	268	if (node.estimate > 0) {
	269	errorHandler.move();
	270
	271	const parent = mediaFile.getParent();
	272	if (parent) parent.setProcessed(0);
	273	}
	274	}
	275
	276	// Add the root & recursively iterate!
	277	const rootPath = Path.resolve(root);
	278	if (!rootPath.ifExistsSync()) {
	279	throw new Error(`file store ${rootPath} is not mounted`);
	280	}
	281	promises.push(scanDirectory(rootPath));
	282
	283	await promises.all();
	284	fileNode.end();
	285	dirsNode.end();
	286
	287	// Update directory metadata
	288	using dirMetaNode = progress.start("update directory metadata");
	289	const dirs = MediaFile.getDirectoriesToReindex()
	290	.sort((a, b) => b.path.length - a.path.length);
	291
	292	for (const dir of dirs) {
	293	using _ = dirMetaNode.start(dir.path);
	294	const children = dir.getChildren();
	295
	296	// readme.txt
	297	const readmeContent = children.find((x) =>
	298	x.basename === "readme.txt"
	299	)?.contents ?? "";
	300
	301	// dirsort
	302	let dirsort: string[] \| null = null;
	303	const dirSortRaw =
	304	children.find((x) => x.basename === ".dirsort")?.contents ?? "";
	305	if (dirSortRaw) {
	306	dirsort = dirSortRaw
	307	.split("\n")
	308	.map((x) => x.trim())
	309	.filter(Boolean);
	310	}
	311
	312	// Permissions
	313	if (children.some((x) => x.basename === ".friends")) {
	314	FilePermissions.setPermissions(dir.path, 1);
	315	} else {
	316	FilePermissions.setPermissions(dir.path, 0);
	317	}
	318
	319	// Recursive stats.
	320	let totalSize = 0;
	321	let newestDate = new Date(0);
	322	let allHashes = "";
	323	for (const child of children) {
	324	totalSize += child.size;
	325	allHashes += child.hash;
	326
	327	if (child.basename !== "/readme.txt" && child.date > newestDate) {
	328	newestDate = child.date;
	329	}
	330	}
	331
	332	// Project Date
	333	const dateFile = children.find((x) => x.basename === ".date");
	334	if (dateFile) {
	335	const date = new Date(dateFile.contents);
	336	newestDate = date;
	337	dir.setProcessors(
	338	0,
	339	JSON.stringify({ hideChildrenDates: true }),
	340	);
	341	} else {
	342	dir.setProcessors(0, "");
	343	}
	344
	345	const dirHash = crypto
	346	.createHash("sha1")
	347	.update(dir.path + allHashes)
	348	.digest("hex");
	349
	350	MediaFile.markDirectoryProcessed({
	351	id: dir.id,
	352	timestamp: newestDate,
	353	contents: readmeContent,
	354	size: totalSize,
	355	hash: dirHash,
	356	dirsort,
	357	});
	358	}
	359	dirMetaNode.end();
	360
	361	// Sync to remote
	362	if (
	363	((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
	364	) {
	365	await rsync.spawn({
	366	args: [
	367	"--links",
	368	"--recursive",
	369	"--times",
	370	"--partial",
	371	"--progress",
	372	// "--remove-source-files",
	373	"--delay-updates",
	374	"--exclude=tmp.*",
	375	derived.workDir.toString() + "/",
	376	"clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/derived/",
	377	],
	378	progress: progress.start("upload derived assets"),
	379	cwd: process.cwd(),
	380	});
	381
	382	await fs.removeEmptyDirectories(derived.workDir.toString());
	383	} else {
	384	console.info("No new derived assets");
	385	}
	386
	387	MediaFile.db.prepare("VACUUM").run();
	388	MediaFile.db.reload();
	389
	390	await rsync.spawn({
	391	args: [
	392	MediaFile.db.file,
	393	"clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
	394	],
	395	progress: progress.start("Uploading database (source of truth)"),
	396	cwd: process.cwd(),
	397	});
	398	await rsync.spawn({
	399	args: [
	400	MediaFile.db.file,
	401	"clo@paperclover.net:paperclover.net/.clover/cache.sqlite",
	402	],
	403	progress: progress.start("Uploading database (web node)"),
	404	});
	405	if (sotToken) {
	406	const res = await fetch("https://db.paperclover.net/reload", {
	407	method: "POST",
	408	headers: {
	409	Authorization: sotToken,
	410	},
	411	});
	412	if (!res.ok) {
	413	console.warn(
	414	`Failed to reload remote database ${res.status} ${res.statusText}`,
	415	);
	416	}
	417	} else console.warn("Missing SOT token");
	418
	419	console.info(
	420	"Updated file viewer index in \x1b[1m" +
	421	((performance.now() - start) / 1000).toFixed(1) +
	422	"s\x1b[0m",
	423	);
	424
	425	const { duration, count } = MediaFile.db
	426	.prepare<[], { count: number; duration: number }>(`
	427	select
	428	count(*) as count,
	429	sum(duration) as duration
	430	from media_files
	431	`)
	432	.getNonNull();
	433	const { derivedSize, derivedCount } = MediaFile.db
	434	.prepare<[], { derivedCount: number; derivedSize: number }>(`
	435	select
	436	count(*) as derivedCount,
	437	sum(size) as derivedSize
	438	from derived_files
	439	`)
	440	.getNonNull();
	441	const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
	442
	443	console.info();
	444	console.info(
	445	"Global Stats:\n" +
	446	`- File Count: \x1b[1m${count}\x1b[0m\n` +
	447	`- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n` +
	448	`- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n` +
	449	`- Canonical Size: \x1b[1m${formatSize(canonicalSize)}\x1b[0m\n` +
	450	`- Derived Size: \x1b[1m${formatSize(derivedSize)}\x1b[0m\n`,
	451	);
	452	}
	453
	454	interface Process {
	455	name: string;
	456	cores?: number;
	457	enable?: boolean;
	458	include?: Set<string>;
	459	exclude?: Set<string>;
	460	depends?: string[];
	461	version?: number;
	462	/* Perform an action. */
	463	run(args: ProcessFileArgs): Promise<void>;
	464	}
	465
	466	const ffprobeBin = testProgram("ffprobe", "--help");
	467	const ffmpegBin = testProgram("ffmpeg", "--help");
	468
	469	const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
	470
	471	// NOTE: Never re-order the processors. Add new ones at the end.
	472	const procDuration: Process = {
	473	name: "calculate duration",
	474	enable: ffprobeBin !== null,
	475	include: rules.extsDuration,
	476	cores: 1,
	477	async run({ path, mediaFile }) {
	478	const { stdout } = await subprocess.exec(ffprobeBin!, [
	479	"-v",
	480	"error",
	481	"-show_entries",
	482	"format=duration",
	483	"-of",
	484	"default=noprint_wrappers=1:nokey=1",
	485	path.toString(),
	486	]);
	487
	488	const duration = parseFloat(stdout.trim());
	489	if (Number.isNaN(duration)) {
	490	throw new Error("Could not extract duration from " + stdout);
	491	}
	492	mediaFile.setDuration(Math.ceil(duration));
	493	},
	494	};
	495
	496	const procDimensions: Process = {
	497	name: "calculate dimensions",
	498	enable: ffprobeBin != null,
	499	include: rules.extsDimensions,
	500	cores: 1,
	501	async run({ path, mediaFile }) {
	502	const { ext } = path;
	503
	504	let dimensions;
	505
	506	if (ext === ".svg") {
	507	// Parse out of text data
	508	const content = await path.read("utf-8");
	509	const widthMatch = content.match(/width="(\d+)"/);
	510	const heightMatch = content.match(/height="(\d+)"/);
	511
	512	if (widthMatch && heightMatch) {
	513	dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
	514	}
	515	} else if (rules.extsImage.has(ext)) {
	516	// Use magick to observe streams
	517	const { stdout } = await subprocess.exec("magick", [
	518	"identify",
	519	"-auto-orient",
	520	"-format",
	521	"%w %h",
	522	path.toString(),
	523	]);
	524	const [w, h] = stdout.split(" ").map((x) => Number(x));
	525	if (w && h) {
	526	dimensions = w + "x" + h;
	527	}
	528	} else {
	529	// Use ffprobe to observe streams
	530	const { stdout } = await subprocess.exec("ffprobe", [
	531	"-v",
	532	"error",
	533	"-select_streams",
	534	"v:0",
	535	"-show_entries",
	536	"stream=width,height",
	537	"-of",
	538	"json",
	539	path.toString(),
	540	]);
	541	const result = JSON.parse(stdout);
	542	const stream = result.streams[0];
	543	if (stream) {
	544	dimensions = UNWRAP(stream.width) + "x" + UNWRAP(stream.height);
	545	}
	546	}
	547
	548	mediaFile.setDimensions(dimensions ?? "");
	549	},
	550	};
	551
	552	const procLoadTextContents: Process = {
	553	name: "load text content",
	554	include: rules.extsReadContents,
	555	cores: 1,
	556	version: 2,
	557	async run({ path, mediaFile, stat }) {
	558	if (stat.size > 1_000_000) return;
	559	const text = await path.read("utf-8");
	560	mediaFile.setContents(text);
	561	},
	562	};
	563
	564	const procHighlightCode: Process = {
	565	name: "highlight source code",
	566	include: new Set(rules.extsCode.keys()),
	567	cores: 1,
	568	version: 2,
	569	async run({ path, mediaFile, stat }) {
	570	const language = UNWRAP(
	571	rules.extsCode.get(path.ext.toLowerCase()),
	572	);
	573	// An issue is that .ts is an overloaded extension, shared between
	574	// 'transport stream' and 'typescript'.
	575	//
	576	// Filter used here is:
	577	// - more than 1mb
	578	// - invalid UTF-8
	579	if (stat.size > 1_000_000) return;
	580	let code;
	581	const buf = await path.read();
	582	try {
	583	code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
	584	} catch (error) {
	585	mediaFile.setContents("");
	586	return;
	587	}
	588	const content = await highlight.highlightCode(code, language);
	589	mediaFile.setContents(content);
	590	},
	591	};
	592
	593	const procImageSubsets: Process = {
	594	name: "encode image subsets",
	595	include: rules.extsImage,
	596	depends: [procDimensions.name],
	597	version: 2,
	598	async run({ path, mediaFile, node }) {
	599	const { width, height } = UNWRAP(mediaFile.parseDimensions());
	600	const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
	601	const baseStatus = node.text;
	602
	603	using stack = new DisposableStack();
	604	for (const size of targetSizes) {
	605	const { w, h } = resizeDimensions(width, height, size);
	606	for (const { ext, args } of transcodeRules.imagePresets) {
	607	node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
	608
	609	stack.use(
	610	await derived.produce({
	611	mediaFile,
	612	node,
	613	cores: 2,
	614	subkey: `${size}${ext}`,
	615	async producer(dir) {
	616	await subprocess.exec(ffmpegBin!, [
	617	...ffmpegOptions,
	618	"-i",
	619	path.toString(),
	620	"-vf",
	621	`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
	622	...args,
	623	dir.join(`${size}${ext}`).toString(),
	624	]);
	625	},
	626	}),
	627	);
	628	}
	629	}
	630
	631	stack.move();
	632	},
	633	};
	634
	635	const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
	636
	637	const qualityMap: Record<string, string> = {
	638	u: "ultra-high",
	639	h: "high",
	640	m: "medium",
	641	l: "low",
	642	d: "data-saving",
	643	};
	644	const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
	645	name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
	646	include: rules.extsVideo,
	647	enable: ffmpegBin != null,
	648	depends: [procDuration.name, procDimensions.name],
	649	async run({ path, mediaFile, node }) {
	650	if ((mediaFile.duration ?? 0) < 5) return;
	651	if (!mediaFile.dimensions) return;
	652
	653	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
	654
	655	await derived.produce({
	656	mediaFile,
	657	node,
	658	cores: 4,
	659	subkey: `av1-${preset.id}`,
	660	async producer(dir) {
	661	const input = await videoArgsCache.getOrRun(path);
	662	const args = transcodeRules.getAv1VideoArgs(
	663	preset,
	664	UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
	665	dir,
	666	);
	667
	668	await ffmpeg.spawn({
	669	ffmpeg: ffmpegBin!,
	670	progress: node,
	671	args,
	672	cwd: dir,
	673	});
	674	},
	675	});
	676	},
	677	}));
	678	const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
	679	name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
	680	include: rules.extsVideo,
	681	enable: ffmpegBin != null,
	682	depends: [procDuration.name, procDimensions.name],
	683	version: 2,
	684	async run({ path, mediaFile, node }) {
	685	if ((mediaFile.duration ?? 0) < 5) return;
	686	if (!mediaFile.dimensions) return;
	687	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
	688	await derived.produce({
	689	mediaFile,
	690	node,
	691	cores: 4,
	692	subkey: `opus-${preset.id}`,
	693	async producer(dir) {
	694	const input = await videoArgsCache.getOrRun(path);
	695	ASSERT(input.video);
	696	if (!input.audio) return;
	697	const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
	698
	699	await ffmpeg.spawn({
	700	ffmpeg: ffmpegBin!,
	701	progress: node,
	702	args,
	703	cwd: dir,
	704	});
	705	},
	706	});
	707	},
	708	}));
	709	const procDash: Process = {
	710	name: `encode mpeg-dash`,
	711	include: rules.extsVideo,
	712	enable: ffmpegBin != null,
	713	depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
	714	version: 4,
	715	async run({ path, mediaFile, node }) {
	716	if ((mediaFile.duration ?? 0) < 5) return;
	717	if (!mediaFile.dimensions) return;
	718
	719	if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
	720	await derived.produce({
	721	mediaFile,
	722	node,
	723	cores: 1,
	724	subkey: `dash-av1`,
	725	async producer(dir) {
	726	const input = await videoArgsCache.getOrRun(path);
	727	const videos = transcodeRules.videoFormats.map(
	728	(preset) =>
	729	UNWRAP(dir.parent).join(
	730	`av1-${preset.id}`,
	731	transcodeRules.av1FileName,
	732	),
	733	);
	734	const audios = input.audio
	735	? transcodeRules.audioFormats.map(
	736	(preset) =>
	737	UNWRAP(dir.parent).join(
	738	`opus-${preset.id}`,
	739	transcodeRules.opusFileName,
	740	),
	741	)
	742	: [];
	743	const args = transcodeRules.getMpegDashArgs(
	744	videos,
	745	audios,
	746	dir,
	747	);
	748
	749	await dir.join("d").makeDir();
	750
	751	await ffmpeg.spawn({
	752	ffmpeg: ffmpegBin!,
	753	progress: node,
	754	args,
	755	cwd: dir,
	756	});
	757	},
	758	});
	759	},
	760	};
	761	const procH264Hls: Process = {
	762	name: `encode h.264 hls`,
	763	include: rules.extsVideo,
	764	enable: ffmpegBin != null,
	765	depends: [procDuration.name, procDimensions.name],
	766	async run({ path, mediaFile, node }) {
	767	if ((mediaFile.duration ?? 0) < 5) return;
	768	if (!mediaFile.dimensions) return;
	769
	770	await derived.produce({
	771	mediaFile,
	772	node,
	773	cores: 8,
	774	subkey: `hls`,
	775	async producer(dir) {
	776	const input = await videoArgsCache.getOrRun(path);
	777	const args = transcodeRules.getH264HlsArgs(input, dir);
	778
	779	await ffmpeg.spawn({
	780	ffmpeg: ffmpegBin!,
	781	progress: node,
	782	args,
	783	cwd: dir,
	784	});
	785	},
	786	});
	787	},
	788	};
	789
	790	const procCompression = [
	791	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
	792	{ name: "zstd", fn: () => zlib.createZstdCompress() },
	793	].map<Process>(
	794	({ name, fn }) => ({
	795	name: `compress ${name}`,
	796	exclude: rules.extsPreCompressed,
	797	async run({ path, mediaFile, node }) {
	798	await derived.produce({
	799	mediaFile,
	800	node,
	801	cores: 1,
	802	subkey: name,
	803	async producer(dir) {
	804	await stream.promises.pipeline(
	805	fs.createReadStream(path.toString()),
	806	fn(),
	807	fs.createWriteStream(dir.join(name).toString()),
	808	);
	809	},
	810	});
	811	},
	812	}),
	813	);
	814
	815	const processors = [
	816	procDimensions,
	817	procDuration,
	818	procLoadTextContents,
	819	procHighlightCode,
	820	procImageSubsets,
	821	...procVideos,
	822	...procVideoAudios,
	823	procDash,
	824	...procCompression,
	825	procH264Hls,
	826	].map((process, id, all) => {
	827	const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
	828	return {
	829	...(process as Process),
	830	id: strIndex(id),
	831	// Create a unique key.
	832	hash: new Uint16Array(
	833	crypto
	834	.createHash("sha1")
	835	.update(
	836	process.run.toString() +
	837	(process.version ? String(process.version) : ""),
	838	)
	839	.digest().buffer,
	840	).reduce((a, b) => a ^ b),
	841	depends: (process.depends ?? []).map((depend) => {
	842	const index = all.findIndex((p) => p.name === depend);
	843	if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
	844	if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
	845	return strIndex(index);
	846	}),
	847	};
	848	});
	849
	850	function resizeDimensions(w: number, h: number, desiredWidth: number) {
	851	ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
	852	return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
	853	}
	854
	855	interface UpdateMetadataArgs {
	856	path: Path;
	857	publicPath: string;
	858	stat: fs.Stats;
	859	mediaFile: MediaFile \| null;
	860	}
	861
	862	interface ProcessFileArgs {
	863	path: Path;
	864	stat: fs.Stats;
	865	mediaFile: MediaFile;
	866	node: progress.Node;
	867	}
	868
	869	interface ProcessJob {
	870	path: Path;
	871	stat: fs.Stats;
	872	mediaFile: MediaFile;
	873	processor: (typeof processors)[0];
	874	index: number;
	875	after: ProcessJob[];
	876	needs: number;
	877	fileNode: progress.Node;
	878	}
	879
	880	function skipBasename(basename: string): boolean {
	881	// dot files must be incrementally tracked
	882	if (basename === ".dirsort") return false;
	883	if (basename === ".friends") return false;
	884	if (basename === ".date") return false;
	885
	886	return (
	887	basename.startsWith(".") \|\|
	888	// basename.startsWith("._") \|\|
	889	// basename.startsWith(".tmp") \|\|
	890	// basename === ".DS_Store" \|\|
	891	basename.toLowerCase() === "thumbs.db" \|\|
	892	basename.toLowerCase() === "desktop.ini"
	893	);
	894	}
	895
	896	function toPublicPath(diskPath: Path) {
	897	if (diskPath.toString() === root) return "/";
	898	return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
	899	}
	900
	901	function testProgram(name: string, helpArgument: string) {
	902	try {
	903	child_process.spawnSync(name, [helpArgument]);
	904	return name;
	905	} catch (err) {
	906	console.warn(`Missing or corrupt executable '${name}'`);
	907	}
	908	return null;
	909	}
	910
	911	// Helper function to check and remove location metadata
	912	async function scrubLocationMetadata(
	913	path: Path,
	914	stats: fs.Stats,
	915	progress: progress.Ref,
	916	): Promise<boolean> {
	917	using _ = progress.start("scrub exif metadata");
	918	const ext = path.ext.toLowerCase();
	919	if (!rules.extsScrubExif.has(ext)) return false;
	920
	921	let hasLocation = false;
	922	let args: string[] = [];
	923
	924	// Check for location metadata based on file type
	925	const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
	926	switch (ext) {
	927	case ".jpg":
	928	case ".jpeg":
	929	case ".png":
	930	const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
	931	"-gps:all",
	932	path.toString(),
	933	]);
	934	hasLocation = gpsCheck.trim().length > 0;
	935	args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
	936	break;
	937	case ".mov":
	938	case ".mp4":
	939	const { stdout: videoCheck } = await subprocess.exec("exiftool", [
	940	"-ee",
	941	"-G3",
	942	"-s",
	943	path.toString(),
	944	]);
	945	hasLocation = videoCheck.includes("GPS") \|\|
	946	videoCheck.includes("Location");
	947	args = [
	948	"-gps:all=",
	949	"-xmp:all=",
	950	path.toString(),
	951	"-o",
	952	tempOutput.toString(),
	953	];
	954	break;
	955	case ".m4a":
	956	const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
	957	"-ee",
	958	"-G3",
	959	"-s",
	960	path.toString(),
	961	]);
	962	hasLocation = m4aCheck.includes("GPS") \|\|
	963	m4aCheck.includes("Location") \|\|
	964	m4aCheck.includes("Filename") \|\|
	965	m4aCheck.includes("Title");
	966
	967	if (hasLocation) {
	968	args = [
	969	"-gps:all=",
	970	"-location:all=",
	971	"-filename:all=",
	972	"-title=",
	973	"-m4a:all=",
	974	path.toString(),
	975	"-o",
	976	tempOutput.toString(),
	977	];
	978	}
	979	break;
	980	}
	981
	982	const accessTime = stats.atime;
	983	const modTime = stats.mtime;
	984
	985	let backup: Path \| null = null;
	986	try {
	987	if (hasLocation) {
	988	// Prepare a backup
	989	const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
	990	await fsp.copyFile(path.toString(), tmp.toString());
	991	await fsp.utimes(tmp.toString(), accessTime, modTime);
	992	backup = tmp;
	993
	994	// Remove metadata
	995	await subprocess.exec("exiftool", args);
	996	if (!tempOutput.ifExistsSync()) {
	997	throw new Error(`Failed to create output file: ${tempOutput}`);
	998	}
	999
	1000	// Restore original timestamps
	1001	await fsp.rename(tempOutput.toString(), path.toString());
	1002	await fsp.utimes(path.toString(), accessTime, modTime);
	1003
	1004	// Backup is no longer needed
	1005	await fsp.unlink(backup.toString());
	1006
	1007	console.info(
	1008	`Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
	1009	);
	1010	return true;
	1011	}
	1012	} catch (error) {
	1013	if (backup) {
	1014	await fsp.rename(backup.toString(), path.toString());
	1015	}
	1016	if (fs.existsSync(tempOutput.toString())) {
	1017	await fsp.unlink(tempOutput.toString());
	1018	}
	1019	throw error;
	1020	}
	1021
	1022	return false;
	1023	}
	1024
	1025	const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
	1026
	1027	import * as async from "lib/async.ts";
	1028	import * as fs from "#sitegen/fs";
	1029	import * as subprocess from "lib/subprocess.ts";
	1030	import { Path } from "#sitegen/path";
	1031	import * as queue from "../../../lib/queue.ts";
	1032	import * as log from "../../../lib/log.ts";
	1033	import * as progress from "lib/progress.ts";
	1034	import * as ts from "../../../lib/ts.ts";
	1035
	1036	import * as child_process from "node:child_process";
	1037	import * as crypto from "node:crypto";
	1038	import * as fsp from "node:fs/promises";
	1039	import * as path from "node:path";
	1040	import * as stream from "node:stream";
	1041	import * as zlib from "node:zlib";
	1042
	1043	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
	1044	import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
	1045	import {
	1046	formatDate,
	1047	formatDurationLong,
	1048	formatSize,
	1049	} from "@/file-viewer/format.ts";
	1050	import * as rules from "@/file-viewer/rules.ts";
	1051	import * as highlight from "@/file-viewer/highlight.ts";
	1052	import * as ffmpeg from "lib/subprocess/ffmpeg.ts";
	1053	import * as rsync from "@/file-viewer/rsync.ts";
	1054	import * as derived from "@/file-viewer/models/derived.ts";
	1055	import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
	1056
	1057	import { rawFileRoot as root } from "../paths.ts";
	1058	import { ASSERT, UNWRAP } from "lib/assert.ts";

src/file-viewer/bin/file-trim.ts created+18

...	...	@@ -0,0 +1,18 @@
	1	export async function main() {
	2	const start = performance.now();
	3	using _ = log.startWidget({
	4	format: (now) =>
	5	`paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
	6	});
	7
	8	const orphaned = derived.findOrphanedRoots();
	9	for (const root of orphaned) {
	10	console.info("delete " + root.key);
	11	derived.deleteRoot(root);
	12	}
	13
	14	// TODO: delete unreferenced files
	15	}
	16
	17	import * as log from "lib/log.ts";
	18	import * as derived from "../models/derived.ts";

src/file-viewer/bin/scan3.ts deleted-1043

...	...	@@ -1,1043 +0,0 @@
1		// The file scanner incrementally updates an sqlite database with file
2		// stats. Additionally, it runs "processors" on files, which precompute
3		// expensive data such as running `ffprobe` on all media to get the
4		// duration.
5		//
6		// Processors are also used to derive compressed and optimized assets,
7		// which is how automatic JXL / AV1 encoding is done. Derived files are
8		// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9		//
10		// This is the third iteration of the scanner, hence its name "scan3";
11		// Remember that any software you want to be maintainable and high
12		// quality cannot be written with AI.
13		const sotToken = process.env.CLOVER_SOT_KEY;
14
15		export async function main() {
16		const start = performance.now();
17		using _ = log.startWidget({
18		format: (now) =>
19		`paper clover's scan3 [${((now - start) / 1000).toFixed(1)}s]`,
20		});
21
22		const promises = new async.PromiseAggregator();
23
24		const walkQueue = new queue.PriorityQueue(10);
25
26		const dirsNode = progress.start("Walk Tree", { estimate: 1 });
27		dirsNode.passive = true;
28		const fileNode = progress.start("Process File", { estimate: 0 });
29		fileNode.sortChildren = (a, b) => {
30		const ac = a.children.length > 0 ? 1 : 0;
31		const bc = b.children.length > 0 ? 1 : 0;
32		if (ac !== bc) return bc - ac;
33		return a.text.localeCompare(b.text);
34		};
35
36		// Read a directory or file stat and queue up changed files.
37		const scanDirectory = walkQueue.wrap(
38		async (path: Path) => {
39		const publicPath = toPublicPath(path);
40		using node = dirsNode.start(publicPath + " - stat");
41		using _ = ts.defer(() => dirsNode.inc());
42
43		const stat = await path.stat();
44
45		const mediaFile = MediaFile.getByPath(publicPath);
46
47		if (stat.isDirectory()) {
48		node.text = publicPath + " - reading";
49		const items = (await path.readDir())
50		.filter((child) => !skipBasename(child.base))
51		.map((child) => (promises.push(scanDirectory(child)), child.base));
52		dirsNode.estimate += items.length;
53
54		for (const child of mediaFile?.getChildren() ?? []) {
55		if (items.includes(child.basename)) continue;
56		const recursive = child.kind === MediaFileKind.directory
57		? [child, ...child.getRecursiveFileChildren()]
58		: [child];
59		for (const deletion of recursive) deletion.delete();
60		}
61
62		return;
63		}
64
65		if (
66		!mediaFile \|\| // All processes must be performed if there is no file.
67		// Rerun all processors if it changed
68		stat.size !== mediaFile.size \|\|
69		stat.mtime.getTime() !== mediaFile.date.getTime()
70		) {
71		promises.push(updateMetadata({ path, publicPath, stat, mediaFile }));
72		} else {
73		// If the scanners changed, it may mean more processes should be run.
74		await queueProcessors({
75		path,
76		stat,
77		mediaFile,
78		node: fileNode.start(publicPath.slice(1)),
79		});
80		}
81		},
82		);
83		const updateMetadata = queue.wrap(
84		async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
85		using errorHandler = new DisposableStack();
86		const label = publicPath.slice(1);
87		const node = errorHandler.use(fileNode.start(label));
88
89		await scrubLocationMetadata(path, stat, node);
90
91		node.text = `${label} - hashing`;
92		const hash = await new Promise<string>((resolve, reject) => {
93		const reader = fs.createReadStream(path.toString());
94		reader.on("error", reject);
95
96		const hasher = crypto.createHash("sha1").setEncoding("hex");
97		hasher.on("error", reject);
98		hasher.on("readable", () => resolve(hasher.read()));
99
100		reader.pipe(hasher);
101		});
102		let date = stat.mtime;
103		if (
104		mediaFile &&
105		mediaFile.date.getTime() < stat.mtime.getTime() &&
106		Date.now() - stat.mtime.getTime() < monthMilliseconds
107		) {
108		date = mediaFile.date;
109		console.warn(
110		`M-time on ${publicPath} was likely corrupted. ${
111		formatDate(
112		mediaFile.date,
113		)
114		} -> ${formatDate(stat.mtime)}`,
115		);
116		}
117		mediaFile = MediaFile.createFile({
118		path: publicPath,
119		date,
120		hash,
121		size: stat.size,
122		duration: mediaFile?.duration ?? 0,
123		dimensions: mediaFile?.dimensions ?? "",
124		contents: mediaFile?.contents ?? "",
125		});
126		const parent = mediaFile.getParent();
127		if (parent) parent.setProcessed(0);
128
129		node.text = `${label}`;
130		await queueProcessors({
131		path,
132		stat,
133		mediaFile,
134		node,
135		});
136		errorHandler.move();
137		},
138		() => ({ priority: -1 }),
139		);
140		const queueFileProcessor = queue.wrap(async function ({
141		path,
142		stat,
143		mediaFile,
144		processor,
145		index,
146		after,
147		fileNode: innerNode,
148		}: ProcessJob) {
149		using node = innerNode.start(processor.name);
150		await processor.run({
151		path,
152		stat,
153		mediaFile,
154		node,
155		});
156		fileNode.value += 1;
157		mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
158		for (const dependantJob of after) {
159		ASSERT(
160		dependantJob.needs > 0,
161		`dependantJob.needs > 0, ${dependantJob.needs}`,
162		);
163		dependantJob.needs -= 1;
164		if (dependantJob.needs == 0) {
165		promises.push(queueFileProcessor(dependantJob));
166		}
167		}
168		}, (job) => ({ cores: job.processor.cores ?? 0 }));
169
170		function decodeProcessors(input: string) {
171		return input
172		.split(";")
173		.filter(Boolean)
174		.map(([a, b, c]) => ({
175		id: a,
176		hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
177		}));
178		}
179
180		async function queueProcessors({
181		path,
182		stat,
183		mediaFile,
184		node,
185		}: Omit<ProcessFileArgs, "spin">) {
186		using errorHandler = new DisposableStack();
187		errorHandler.use(node);
188
189		node.showEstimate = false;
190		node.passive = true;
191
192		const ext = mediaFile.extensionNonEmpty.toLowerCase();
193		let possible = processors.filter((p) =>
194		p.include ? p.include.has(ext) : !p.exclude?.has(ext)
195		);
196		if (possible.length === 0) return;
197		ASSERT(possible.length < 16, "too many bits");
198
199		const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
200		ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
201		let processed = mediaFile.processed;
202
203		// If the hash has changed, migrate the bitfield over.
204		// This also runs when the processor hash is in it's initial 0 state.
205		let order: ReturnType<typeof decodeProcessors>;
206		try {
207		order = decodeProcessors(mediaFile.processors);
208		} catch {
209		// this function sucks and this system sucks i hate it.
210		order = [];
211		}
212		if ((processed & 0xffff) !== hash) {
213		const previous = order.filter(
214		(_, i) => (processed & (1 << (16 + i))) !== 0,
215		);
216		processed = hash;
217		for (const { id, hash } of previous) {
218		const p = processors.find((p) => p.id === id);
219		if (!p) continue;
220		const index = possible.indexOf(p);
221		if (index !== -1 && p.hash === hash) processed \|= 1 << (16 + index);
222		}
223		mediaFile.setProcessors(
224		processed,
225		possible
226		.map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
227		.join(";"),
228		);
229		} else {
230		possible = order.map(({ id }) =>
231		UNWRAP(possible.find((p) => p.id === id))
232		);
233		}
234
235		// Queue needed processors.
236		const jobs: ProcessJob[] = [];
237		for (let i = 0, { length } = possible; i < length; i += 1) {
238		if ((processed & (1 << (16 + i))) === 0) {
239		const processor = UNWRAP(possible[i]);
240		const job: ProcessJob = {
241		path,
242		stat,
243		mediaFile,
244		processor,
245		index: i,
246		after: [],
247		needs: processor.depends.length,
248		fileNode: node,
249		};
250		jobs.push(job);
251		if (job.needs === 0) promises.push(queueFileProcessor(job));
252		}
253		}
254		node.estimate = jobs.length;
255		fileNode.estimate += jobs.length;
256		for (const job of jobs) {
257		for (const dependId of job.processor.depends) {
258		const dependJob = jobs.find((j) => j.processor.id === dependId);
259		if (dependJob) {
260		dependJob.after.push(job);
261		} else {
262		ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
263		job.needs -= 1;
264		if (job.needs === 0) promises.push(queueFileProcessor(job));
265		}
266		}
267		}
268		if (node.estimate > 0) {
269		errorHandler.move();
270
271		const parent = mediaFile.getParent();
272		if (parent) parent.setProcessed(0);
273		}
274		}
275
276		// Add the root & recursively iterate!
277		const rootPath = Path.resolve(root);
278		if (!rootPath.ifExistsSync()) {
279		throw new Error(`file store ${rootPath} is not mounted`);
280		}
281		promises.push(scanDirectory(rootPath));
282
283		await promises.all();
284		fileNode.end();
285		dirsNode.end();
286
287		// Update directory metadata
288		using dirMetaNode = progress.start("update directory metadata");
289		const dirs = MediaFile.getDirectoriesToReindex()
290		.sort((a, b) => b.path.length - a.path.length);
291
292		for (const dir of dirs) {
293		using _ = dirMetaNode.start(dir.path);
294		const children = dir.getChildren();
295
296		// readme.txt
297		const readmeContent = children.find((x) =>
298		x.basename === "readme.txt"
299		)?.contents ?? "";
300
301		// dirsort
302		let dirsort: string[] \| null = null;
303		const dirSortRaw =
304		children.find((x) => x.basename === ".dirsort")?.contents ?? "";
305		if (dirSortRaw) {
306		dirsort = dirSortRaw
307		.split("\n")
308		.map((x) => x.trim())
309		.filter(Boolean);
310		}
311
312		// Permissions
313		if (children.some((x) => x.basename === ".friends")) {
314		FilePermissions.setPermissions(dir.path, 1);
315		} else {
316		FilePermissions.setPermissions(dir.path, 0);
317		}
318
319		// Recursive stats.
320		let totalSize = 0;
321		let newestDate = new Date(0);
322		let allHashes = "";
323		for (const child of children) {
324		totalSize += child.size;
325		allHashes += child.hash;
326
327		if (child.basename !== "/readme.txt" && child.date > newestDate) {
328		newestDate = child.date;
329		}
330		}
331
332		// Project Date
333		const dateFile = children.find((x) => x.basename === ".date");
334		if (dateFile) {
335		const date = new Date(dateFile.contents);
336		newestDate = date;
337		dir.setProcessors(
338		0,
339		JSON.stringify({ hideChildrenDates: true }),
340		);
341		} else {
342		dir.setProcessors(0, "");
343		}
344
345		const dirHash = crypto
346		.createHash("sha1")
347		.update(dir.path + allHashes)
348		.digest("hex");
349
350		MediaFile.markDirectoryProcessed({
351		id: dir.id,
352		timestamp: newestDate,
353		contents: readmeContent,
354		size: totalSize,
355		hash: dirHash,
356		dirsort,
357		});
358		}
359		dirMetaNode.end();
360
361		// Sync to remote
362		if (
363		((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
364		) {
365		await rsync.spawn({
366		args: [
367		"--links",
368		"--recursive",
369		"--times",
370		"--partial",
371		"--progress",
372		// "--remove-source-files",
373		"--delay-updates",
374		"--exclude=tmp.*",
375		derived.workDir.toString() + "/",
376		"clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/derived/",
377		],
378		progress: progress.start("upload derived assets"),
379		cwd: process.cwd(),
380		});
381
382		await fs.removeEmptyDirectories(derived.workDir.toString());
383		} else {
384		console.info("No new derived assets");
385		}
386
387		MediaFile.db.prepare("VACUUM").run();
388		MediaFile.db.reload();
389
390		await rsync.spawn({
391		args: [
392		MediaFile.db.file,
393		"clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
394		],
395		progress: progress.start("Uploading database (source of truth)"),
396		cwd: process.cwd(),
397		});
398		await rsync.spawn({
399		args: [
400		MediaFile.db.file,
401		"clo@paperclover.net:paperclover.net/.clover/cache.sqlite",
402		],
403		progress: progress.start("Uploading database (web node)"),
404		});
405		if (sotToken) {
406		const res = await fetch("https://db.paperclover.net/reload", {
407		method: "POST",
408		headers: {
409		Authorization: sotToken,
410		},
411		});
412		if (!res.ok) {
413		console.warn(
414		`Failed to reload remote database ${res.status} ${res.statusText}`,
415		);
416		}
417		} else console.warn("Missing SOT token");
418
419		console.info(
420		"Updated file viewer index in \x1b[1m" +
421		((performance.now() - start) / 1000).toFixed(1) +
422		"s\x1b[0m",
423		);
424
425		const { duration, count } = MediaFile.db
426		.prepare<[], { count: number; duration: number }>(`
427		select
428		count(*) as count,
429		sum(duration) as duration
430		from media_files
431		`)
432		.getNonNull();
433		const { derivedSize, derivedCount } = MediaFile.db
434		.prepare<[], { derivedCount: number; derivedSize: number }>(`
435		select
436		count(*) as derivedCount,
437		sum(size) as derivedSize
438		from derived_files
439		`)
440		.getNonNull();
441		const canonicalSize = UNWRAP(MediaFile.getByPath("/")).size;
442
443		console.info();
444		console.info(
445		"Global Stats:\n" +
446		`- File Count: \x1b[1m${count}\x1b[0m\n` +
447		`- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n` +
448		`- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n` +
449		`- Canonical Size: \x1b[1m${formatSize(canonicalSize)}\x1b[0m\n` +
450		`- Derived Size: \x1b[1m${formatSize(derivedSize)}\x1b[0m\n`,
451		);
452		}
453
454		interface Process {
455		name: string;
456		cores?: number;
457		enable?: boolean;
458		include?: Set<string>;
459		exclude?: Set<string>;
460		depends?: string[];
461		version?: number;
462		/* Perform an action. */
463		run(args: ProcessFileArgs): Promise<void>;
464		}
465
466		const ffprobeBin = testProgram("ffprobe", "--help");
467		const ffmpegBin = testProgram("ffmpeg", "--help");
468
469		const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
470
471		// NOTE: Never re-order the processors. Add new ones at the end.
472		const procDuration: Process = {
473		name: "calculate duration",
474		enable: ffprobeBin !== null,
475		include: rules.extsDuration,
476		cores: 1,
477		async run({ path, mediaFile }) {
478		const { stdout } = await subprocess.exec(ffprobeBin!, [
479		"-v",
480		"error",
481		"-show_entries",
482		"format=duration",
483		"-of",
484		"default=noprint_wrappers=1:nokey=1",
485		path.toString(),
486		]);
487
488		const duration = parseFloat(stdout.trim());
489		if (Number.isNaN(duration)) {
490		throw new Error("Could not extract duration from " + stdout);
491		}
492		mediaFile.setDuration(Math.ceil(duration));
493		},
494		};
495
496		const procDimensions: Process = {
497		name: "calculate dimensions",
498		enable: ffprobeBin != null,
499		include: rules.extsDimensions,
500		cores: 1,
501		async run({ path, mediaFile }) {
502		const { ext } = path;
503
504		let dimensions;
505
506		if (ext === ".svg") {
507		// Parse out of text data
508		const content = await path.read("utf-8");
509		const widthMatch = content.match(/width="(\d+)"/);
510		const heightMatch = content.match(/height="(\d+)"/);
511
512		if (widthMatch && heightMatch) {
513		dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
514		}
515		} else {
516		// Use ffprobe to observe streams
517		const { stdout } = await subprocess.exec("ffprobe", [
518		"-v",
519		"error",
520		"-select_streams",
521		"v:0",
522		"-show_entries",
523		"stream=width,height",
524		"-of",
525		"csv=s=x:p=0",
526		path.toString(),
527		]);
528		if (stdout.includes("x")) {
529		dimensions = stdout.trim();
530		}
531		}
532
533		mediaFile.setDimensions(dimensions ?? "");
534		},
535		};
536
537		const procLoadTextContents: Process = {
538		name: "load text content",
539		include: rules.extsReadContents,
540		cores: 1,
541		version: 2,
542		async run({ path, mediaFile, stat }) {
543		if (stat.size > 1_000_000) return;
544		const text = await path.read("utf-8");
545		mediaFile.setContents(text);
546		},
547		};
548
549		const procHighlightCode: Process = {
550		name: "highlight source code",
551		include: new Set(rules.extsCode.keys()),
552		cores: 1,
553		version: 2,
554		async run({ path, mediaFile, stat }) {
555		const language = UNWRAP(
556		rules.extsCode.get(path.ext.toLowerCase()),
557		);
558		// An issue is that .ts is an overloaded extension, shared between
559		// 'transport stream' and 'typescript'.
560		//
561		// Filter used here is:
562		// - more than 1mb
563		// - invalid UTF-8
564		if (stat.size > 1_000_000) return;
565		let code;
566		const buf = await path.read();
567		try {
568		code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
569		} catch (error) {
570		mediaFile.setContents("");
571		return;
572		}
573		const content = await highlight.highlightCode(code, language);
574		mediaFile.setContents(content);
575		},
576		};
577
578		const procImageSubsets: Process = {
579		name: "encode image subsets",
580		include: rules.extsImage,
581		depends: [procDimensions.name],
582		version: 2,
583		async run({ path, mediaFile, node }) {
584		const { width, height } = UNWRAP(mediaFile.parseDimensions());
585		const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
586		const baseStatus = node.text;
587
588		using stack = new DisposableStack();
589		for (const size of targetSizes) {
590		const { w, h } = resizeDimensions(width, height, size);
591		for (const { ext, args } of transcodeRules.imagePresets) {
592		node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
593
594		stack.use(
595		await derived.produce({
596		mediaFile,
597		node,
598		cores: 2,
599		subkey: `${size}${ext}`,
600		async producer(dir) {
601		await subprocess.exec(ffmpegBin!, [
602		...ffmpegOptions,
603		"-i",
604		path.toString(),
605		"-vf",
606		`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
607		...args,
608		dir.join(`${size}${ext}`).toString(),
609		]);
610		},
611		}),
612		);
613		}
614		}
615
616		stack.move();
617		},
618		};
619
620		const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
621
622		const qualityMap: Record<string, string> = {
623		u: "ultra-high",
624		h: "high",
625		m: "medium",
626		l: "low",
627		d: "data-saving",
628		};
629		const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
630		name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
631		include: rules.extsVideo,
632		enable: ffmpegBin != null,
633		depends: [procDuration.name, procDimensions.name],
634		async run({ path, mediaFile, node }) {
635		if ((mediaFile.duration ?? 0) < 5) return;
636		if (!mediaFile.dimensions) return;
637
638		if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
639
640		await derived.produce({
641		mediaFile,
642		node,
643		cores: 4,
644		subkey: `av1-${preset.id}`,
645		async producer(dir) {
646		const input = await videoArgsCache.getOrRun(path);
647		const args = transcodeRules.getAv1VideoArgs(
648		preset,
649		UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
650		dir,
651		);
652
653		await ffmpeg.spawn({
654		ffmpeg: ffmpegBin!,
655		progress: node,
656		args,
657		cwd: dir,
658		});
659		},
660		});
661		},
662		}));
663		const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
664		name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
665		include: rules.extsVideo,
666		enable: ffmpegBin != null,
667		depends: [procDuration.name, procDimensions.name],
668		version: 2,
669		async run({ path, mediaFile, node }) {
670		if ((mediaFile.duration ?? 0) < 5) return;
671		if (!mediaFile.dimensions) return;
672		if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
673		await derived.produce({
674		mediaFile,
675		node,
676		cores: 4,
677		subkey: `opus-${preset.id}`,
678		async producer(dir) {
679		const input = await videoArgsCache.getOrRun(path);
680		ASSERT(input.video);
681		if (!input.audio) return;
682		const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
683
684		await ffmpeg.spawn({
685		ffmpeg: ffmpegBin!,
686		progress: node,
687		args,
688		cwd: dir,
689		});
690		},
691		});
692		},
693		}));
694		const procDash: Process = {
695		name: `encode mpeg-dash`,
696		include: rules.extsVideo,
697		enable: ffmpegBin != null,
698		depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
699		version: 4,
700		async run({ path, mediaFile, node }) {
701		if ((mediaFile.duration ?? 0) < 5) return;
702		if (!mediaFile.dimensions) return;
703
704		if (mediaFile.path === "/2021/top-10000-bread/output.mp4") return;
705		await derived.produce({
706		mediaFile,
707		node,
708		cores: 1,
709		subkey: `dash-av1`,
710		async producer(dir) {
711		const input = await videoArgsCache.getOrRun(path);
712		const videos = transcodeRules.videoFormats.map(
713		(preset) =>
714		UNWRAP(dir.parent).join(
715		`av1-${preset.id}`,
716		transcodeRules.av1FileName,
717		),
718		);
719		const audios = input.audio
720		? transcodeRules.audioFormats.map(
721		(preset) =>
722		UNWRAP(dir.parent).join(
723		`opus-${preset.id}`,
724		transcodeRules.opusFileName,
725		),
726		)
727		: [];
728		const args = transcodeRules.getMpegDashArgs(
729		videos,
730		audios,
731		dir,
732		);
733
734		await dir.join("d").makeDir();
735
736		await ffmpeg.spawn({
737		ffmpeg: ffmpegBin!,
738		progress: node,
739		args,
740		cwd: dir,
741		});
742		},
743		});
744		},
745		};
746		const procH264Hls: Process = {
747		name: `encode h.264 hls`,
748		include: rules.extsVideo,
749		enable: ffmpegBin != null,
750		depends: [procDuration.name, procDimensions.name],
751		async run({ path, mediaFile, node }) {
752		if ((mediaFile.duration ?? 0) < 5) return;
753		if (!mediaFile.dimensions) return;
754
755		await derived.produce({
756		mediaFile,
757		node,
758		cores: 8,
759		subkey: `hls`,
760		async producer(dir) {
761		const input = await videoArgsCache.getOrRun(path);
762		const args = transcodeRules.getH264HlsArgs(input, dir);
763
764		await ffmpeg.spawn({
765		ffmpeg: ffmpegBin!,
766		progress: node,
767		args,
768		cwd: dir,
769		});
770		},
771		});
772		},
773		};
774
775		const procCompression = [
776		{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
777		{ name: "zstd", fn: () => zlib.createZstdCompress() },
778		].map<Process>(
779		({ name, fn }) => ({
780		name: `compress ${name}`,
781		exclude: rules.extsPreCompressed,
782		async run({ path, mediaFile, node }) {
783		await derived.produce({
784		mediaFile,
785		node,
786		cores: 1,
787		subkey: name,
788		async producer(dir) {
789		await stream.promises.pipeline(
790		fs.createReadStream(path.toString()),
791		fn(),
792		fs.createWriteStream(dir.join(name).toString()),
793		);
794		},
795		});
796		},
797		}),
798		);
799
800		const processors = [
801		procDimensions,
802		procDuration,
803		procLoadTextContents,
804		procHighlightCode,
805		procImageSubsets,
806		...procVideos,
807		...procVideoAudios,
808		procDash,
809		...procCompression,
810		procH264Hls,
811		].map((process, id, all) => {
812		const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
813		return {
814		...(process as Process),
815		id: strIndex(id),
816		// Create a unique key.
817		hash: new Uint16Array(
818		crypto
819		.createHash("sha1")
820		.update(
821		process.run.toString() +
822		(process.version ? String(process.version) : ""),
823		)
824		.digest().buffer,
825		).reduce((a, b) => a ^ b),
826		depends: (process.depends ?? []).map((depend) => {
827		const index = all.findIndex((p) => p.name === depend);
828		if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
829		if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
830		return strIndex(index);
831		}),
832		};
833		});
834
835		function resizeDimensions(w: number, h: number, desiredWidth: number) {
836		ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
837		return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
838		}
839
840		interface UpdateMetadataArgs {
841		path: Path;
842		publicPath: string;
843		stat: fs.Stats;
844		mediaFile: MediaFile \| null;
845		}
846
847		interface ProcessFileArgs {
848		path: Path;
849		stat: fs.Stats;
850		mediaFile: MediaFile;
851		node: progress.Node;
852		}
853
854		interface ProcessJob {
855		path: Path;
856		stat: fs.Stats;
857		mediaFile: MediaFile;
858		processor: (typeof processors)[0];
859		index: number;
860		after: ProcessJob[];
861		needs: number;
862		fileNode: progress.Node;
863		}
864
865		function skipBasename(basename: string): boolean {
866		// dot files must be incrementally tracked
867		if (basename === ".dirsort") return false;
868		if (basename === ".friends") return false;
869		if (basename === ".date") return false;
870
871		return (
872		basename.startsWith(".") \|\|
873		// basename.startsWith("._") \|\|
874		// basename.startsWith(".tmp") \|\|
875		// basename === ".DS_Store" \|\|
876		basename.toLowerCase() === "thumbs.db" \|\|
877		basename.toLowerCase() === "desktop.ini"
878		);
879		}
880
881		function toPublicPath(diskPath: Path) {
882		if (diskPath.toString() === root) return "/";
883		return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
884		}
885
886		function testProgram(name: string, helpArgument: string) {
887		try {
888		child_process.spawnSync(name, [helpArgument]);
889		return name;
890		} catch (err) {
891		console.warn(`Missing or corrupt executable '${name}'`);
892		}
893		return null;
894		}
895
896		// Helper function to check and remove location metadata
897		async function scrubLocationMetadata(
898		path: Path,
899		stats: fs.Stats,
900		progress: progress.Ref,
901		): Promise<boolean> {
902		using _ = progress.start("scrub exif metadata");
903		const ext = path.ext.toLowerCase();
904		if (!rules.extsScrubExif.has(ext)) return false;
905
906		let hasLocation = false;
907		let args: string[] = [];
908
909		// Check for location metadata based on file type
910		const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
911		switch (ext) {
912		case ".jpg":
913		case ".jpeg":
914		case ".png":
915		const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
916		"-gps:all",
917		path.toString(),
918		]);
919		hasLocation = gpsCheck.trim().length > 0;
920		args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
921		break;
922		case ".mov":
923		case ".mp4":
924		const { stdout: videoCheck } = await subprocess.exec("exiftool", [
925		"-ee",
926		"-G3",
927		"-s",
928		path.toString(),
929		]);
930		hasLocation = videoCheck.includes("GPS") \|\|
931		videoCheck.includes("Location");
932		args = [
933		"-gps:all=",
934		"-xmp:all=",
935		path.toString(),
936		"-o",
937		tempOutput.toString(),
938		];
939		break;
940		case ".m4a":
941		const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
942		"-ee",
943		"-G3",
944		"-s",
945		path.toString(),
946		]);
947		hasLocation = m4aCheck.includes("GPS") \|\|
948		m4aCheck.includes("Location") \|\|
949		m4aCheck.includes("Filename") \|\|
950		m4aCheck.includes("Title");
951
952		if (hasLocation) {
953		args = [
954		"-gps:all=",
955		"-location:all=",
956		"-filename:all=",
957		"-title=",
958		"-m4a:all=",
959		path.toString(),
960		"-o",
961		tempOutput.toString(),
962		];
963		}
964		break;
965		}
966
967		const accessTime = stats.atime;
968		const modTime = stats.mtime;
969
970		let backup: Path \| null = null;
971		try {
972		if (hasLocation) {
973		// Prepare a backup
974		const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
975		await fsp.copyFile(path.toString(), tmp.toString());
976		await fsp.utimes(tmp.toString(), accessTime, modTime);
977		backup = tmp;
978
979		// Remove metadata
980		await subprocess.exec("exiftool", args);
981		if (!tempOutput.ifExistsSync()) {
982		throw new Error(`Failed to create output file: ${tempOutput}`);
983		}
984
985		// Restore original timestamps
986		await fsp.rename(tempOutput.toString(), path.toString());
987		await fsp.utimes(path.toString(), accessTime, modTime);
988
989		// Backup is no longer needed
990		await fsp.unlink(backup.toString());
991
992		console.info(
993		`Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
994		);
995		return true;
996		}
997		} catch (error) {
998		if (backup) {
999		await fsp.rename(backup.toString(), path.toString());
1000		}
1001		if (fs.existsSync(tempOutput.toString())) {
1002		await fsp.unlink(tempOutput.toString());
1003		}
1004		throw error;
1005		}
1006
1007		return false;
1008		}
1009
1010		const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
1011
1012		import * as async from "lib/async.ts";
1013		import * as fs from "#sitegen/fs";
1014		import * as subprocess from "lib/subprocess.ts";
1015		import { Path } from "#sitegen/path";
1016		import * as queue from "../../../lib/queue.ts";
1017		import * as log from "../../../lib/log.ts";
1018		import * as progress from "lib/progress.ts";
1019		import * as ts from "../../../lib/ts.ts";
1020
1021		import * as child_process from "node:child_process";
1022		import * as crypto from "node:crypto";
1023		import * as fsp from "node:fs/promises";
1024		import * as path from "node:path";
1025		import * as stream from "node:stream";
1026		import * as zlib from "node:zlib";
1027
1028		import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
1029		import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
1030		import {
1031		formatDate,
1032		formatDurationLong,
1033		formatSize,
1034		} from "@/file-viewer/format.ts";
1035		import * as rules from "@/file-viewer/rules.ts";
1036		import * as highlight from "@/file-viewer/highlight.ts";
1037		import * as ffmpeg from "lib/subprocess/ffmpeg.ts";
1038		import * as rsync from "@/file-viewer/rsync.ts";
1039		import * as derived from "@/file-viewer/models/derived.ts";
1040		import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
1041
1042		import { rawFileRoot as root } from "../paths.ts";
1043		import { ASSERT, UNWRAP } from "lib/assert.ts";

src/file-viewer/models/MediaFile.ts+8-5

...	...	@@ -224,6 +224,7 @@ export class MediaFile {
224	224	return createDirectoryQuery.getNonNull({
225	225	path: filePath,
226	226	parentId,
	227	time: Date.now(),
227	228	}).id;
228	229	}
229	230	// walk up the path until we find a directory that exists
...	...	@@ -236,6 +237,7 @@ export class MediaFile {
236	237	parentId = createDirectoryQuery.getNonNull({
237	238	path: current,
238	239	parentId,
	240	time: Date.now(),
239	241	}).id;
240	242	}
241	243	// walk back down the path, creating directories as needed
...	...	@@ -245,6 +247,7 @@ export class MediaFile {
245	247	parentId = createDirectoryQuery.getNonNull({
246	248	path: current,
247	249	parentId,
	250	time: Date.now(),
248	251	}).id;
249	252	}
250	253	return parentId;
...	...	@@ -314,16 +317,16 @@ export interface DirConfig {
314	317
315	318	// Get a directory ID by path, creating it if it doesn't exist
316	319	const createDirectoryQuery = db.prepare<
317		[{ path: string; parentId: number \| null }],
	320	[{ path: string; parentId: number \| null; time: number }],
318	321	{ id: number }
319	322	>(
320	323	/* SQL */ `
321	324	insert into media_files (
322		path, parent_id, kind, timestamp, hash, size,
323		duration, dimensions, contents, dirsort, processed)
	325	path, parent_id, kind, timestamp, timestamp_updated, hash,
	326	size, duration, dimensions, contents, dirsort, processed)
324	327	values (
325		$path, $parentId, ${MediaFileKind.directory}, 0, '', 0,
326		0, '', '', '', 0)
	328	$path, $parentId, ${MediaFileKind.directory}, 0, $time, '',
	329	0, 0, '', '', '', 0)
327	330	returning id;
328	331	`,
329	332	);

src/file-viewer/models/derived.ts+18

...	...	@@ -165,6 +165,14 @@ export function get(file: MediaFile, subPath: string) {
165	165	return null;
166	166	}
167	167
	168	export function findOrphanedRoots() {
	169	return findOrphanedRootsQuery.array();
	170	}
	171
	172	export function deleteRoot(root: { id: number }) {
	173	return deleteRootQuery.run({ root: root.id });
	174	}
	175
168	176	const insertRootQuery = db.prepare<
169	177	[{ key: string; date: number }],
170	178	{ id: number }
...	...	@@ -194,6 +202,16 @@ const deleteRefQuery = db.prepare<[{ root: number; file: number }]>(/* SQL */ `
194	202	delete from derived_refs where root = $root and file = $file;
195	203	`);
196	204
	205	const findOrphanedRootsQuery = db.prepare<
	206	[],
	207	{ id: number; key: string; date: number }
	208	>(/* SQL */ `
	209	select dr.id, dr.key, dr.date
	210	from derived_roots dr
	211	left join derived_refs refs on refs.root = dr.id
	212	where refs.id is null
	213	`);
	214
197	215	import { getDb } from "#sitegen/sqlite";
198	216	import type { MediaFile } from "./MediaFile.ts";
199	217	import { Path } from "#sitegen/path";

src/file-viewer/rsync.ts+5-4

...	...	@@ -10,7 +10,7 @@ export type Line =
10	10	currentFile: string;
11	11	bytesTransferred: number;
12	12	percentage: number;
13		timeElapsed: string;
	13	timeElapsed: string \| null;
14	14	transferNumber: number;
15	15	filesToCheck: number;
16	16	totalFiles: number;
...	...	@@ -91,7 +91,7 @@ export async function spawn(options: SpawnOptions) {
91	91	e.args = [rsync, ...args].join(" ");
92	92	e.code = code;
93	93	e.signal = signal;
94		return e;
	94	throw e;
95	95	}
96	96	}
97	97
...	...	@@ -126,9 +126,9 @@ export class Parse {
126	126	return {
127	127	kind: "progress",
128	128	currentFile: this.lastSeenFile \|\| "",
129		bytesTransferred: Number(bytesStr.replaceAll(",", "")),
	129	bytesTransferred: Number(UNWRAP(bytesStr).replaceAll(",", "")),
130	130	percentage: Number(percentageStr),
131		timeElapsed,
	131	timeElapsed: timeElapsed ?? null,
132	132	transferNumber: this.currentTransfer,
133	133	filesToCheck: toCheckStr
134	134	? this.toCheck = Number(toCheckStr)
...	...	@@ -185,3 +185,4 @@ import * as readline from "node:readline";
185	185	import events from "node:events";
186	186	import * as progress from "lib/progress.ts";
187	187	import { formatSize } from "@/file-viewer/format.ts";
	188	import { UNWRAP } from "lib/assert.ts";

src/friend-auth.ts+57-36

...	...	@@ -1,75 +1,96 @@
1		let friendPassword = "";
	1	let hardcoded = {
	2	friendPassword: "",
	3	perPage: {} as Record<string, string>,
	4	getForFile: (file: string) => [] as string[],
	5	};
2	6	try {
3		friendPassword = require("./friends/hardcoded-password.ts").friendPassword;
	7	hardcoded = require("./friends/hardcoded-password.ts");
4	8	} catch {}
5	9
6	10	export const app = new Hono();
7	11
8	12	const cookieAge = 60 * 60 * 24 * 30; // 1 month
9	13
10		function checkFriendsCookie(c: Context) {
	14	export const getForFile = hardcoded.getForFile;
	15
	16	function checkFriendsCookie(c: Context, passwords: string[]) {
11	17	const cookie = c.req.header("Cookie");
12	18	if (!cookie) return false;
13	19	const cookies = cookie.split("; ").map((x) => x.split("="));
14	20	return cookies.some(
15	21	(kv) =>
16		kv[0].trim() === "friends_password" &&
17		kv[1].trim() &&
18		kv[1].trim() === friendPassword,
	22	kv[0]!.trim() === "friends_password" &&
	23	kv[1]!.trim() &&
	24	passwords.includes(kv[1]!.trim()),
19	25	);
20	26	}
21	27
22		export function requireFriendAuth(c: Context) {
	28	export function requireFriendAuth(
	29	c: Context,
	30	passwords = [hardcoded.friendPassword],
	31	) {
	32	if (passwords.length === 0 \|\| passwords[0]!.length === 0) {
	33	return serveAsset(c, "/friends/misconfigure", 501);
	34	}
23	35	const k = c.req.query("password") \|\| c.req.query("k");
24	36	if (k) {
25		if (k === friendPassword) {
	37	if (passwords.includes(k)) {
26	38	return c.body(null, 303, {
27		Location: "/friends",
	39	Location: c.req.path,
28	40	"Set-Cookie":
29	41	`friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
30	42	});
31	43	} else {
32	44	return c.body(null, 303, {
33		Location: "/friends",
	45	Location: c.req.path,
34	46	});
35	47	}
36	48	}
37		if (checkFriendsCookie(c)) {
	49	if (checkFriendsCookie(c, passwords)) {
38	50	return undefined;
39	51	} else {
40	52	return serveAsset(c, "/friends/auth", 403);
41	53	}
42	54	}
43	55
44		app.get("/friends", (c) => {
45		const friendAuthChallenge = requireFriendAuth(c);
46		if (friendAuthChallenge) return friendAuthChallenge;
47		return serveAsset(c, "/friends", 200);
48		});
49
50	56	let incorrectMap: Record<string, boolean> = {};
51		app.post("/friends", async (c) => {
52		const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??
53		"unknown";
54		if (incorrectMap[ip]) {
	57
	58	function friendPage(route: assets.Key) {
	59	const expected = hardcoded.perPage[route] ?? hardcoded.friendPassword;
	60
	61	app.get(route, (c) => {
	62	const friendAuthChallenge = requireFriendAuth(c, [expected]);
	63	if (friendAuthChallenge) return friendAuthChallenge;
	64	return serveAsset(c, route, 200);
	65	});
	66
	67	app.post(route, async (c) => {
	68	const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??
	69	"unknown";
	70	if (incorrectMap[ip]) {
	71	return serveAsset(c, "/friends/auth/fail", 403);
	72	}
	73	const data = await c.req.formData();
	74	const k = data.get("password");
	75	if (k === expected) {
	76	return c.body(null, 303, {
	77	Location: c.req.path,
	78	"Set-Cookie":
	79	`friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
	80	});
	81	}
	82	incorrectMap[ip] = true;
	83	await setTimeout(2500);
	84	incorrectMap[ip] = false;
55	85	return serveAsset(c, "/friends/auth/fail", 403);
56		}
57		const data = await c.req.formData();
58		const k = data.get("password");
59		if (k === friendPassword) {
60		return c.body(null, 303, {
61		Location: "/friends",
62		"Set-Cookie":
63		`friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
64		});
65		}
66		incorrectMap[ip] = true;
67		await setTimeout(2500);
68		incorrectMap[ip] = false;
69		return serveAsset(c, "/friends/auth/fail", 403);
70		});
	86	});
	87	}
	88
	89	friendPage("/friends");
	90	friendPage("/friends/oct25");
71	91
72	92	import { type Context, Hono } from "hono";
73	93	import { serveAsset } from "#sitegen/assets";
74	94	import { setTimeout } from "node:timers/promises";
75	95	import { getConnInfo } from "#hono/conninfo";
	96	import * as assets from "#sitegen/assets";

src/pages/friends/auth.fail.marko+2-2

...	...	@@ -3,9 +3,9 @@ export const meta = { title: "password required" };
3	3	<main>
4	4	<p>incorrect or outdated password</p>
5	5	<p>please contact clover</p>
6		<form method="post" action="/friends">
	6	<form method="post">
7	7	<h1>try again</h1>
8		<input type="password" name="password"/>
	8	<input type="password" name="password" placeholder="password" />
9	9	<button type="submit">enter</button>
10	10	</form>
11	11	</main>

src/pages/friends/auth.marko+2-2

...	...	@@ -1,9 +1,9 @@
1	1	export const meta = { title: "password required" };
2	2
3	3	<main>
4		<form method="post" action="/friends">
	4	<form method="post">
5	5	<h1>what's the password</h1>
6		<input type="password" name="password"/>
	6	<input type="password" name="password" placeholder="password"/>
7	7	<button type="submit">enter</button>
8	8	<p>(clover will give the password to you if you're a friend)</p>
9	9	</form>

src/pages/friends/misconfigure.marko created+10

...	...	@@ -0,0 +1,10 @@
	1	export const meta = { title: "password required" };
	2
	3	<main>
	4	<form method="post" action="/friends">
	5	<h1>what's the password</h1>
	6	<input type="password" name="password" placeholder="password" disabled />
	7	<button type="submit" disabled>enter</button>
	8	<p style="color:red">this page's password is not configured</p>
	9	</form>
	10	</main>

src/tags/clover-video.client.ts+3-1

...	...	@@ -67,7 +67,9 @@ function hydrateVideoInner(
67	67	container: HTMLElement,
68	68	mode: PlaybackMode,
69	69	) {
70		const video = container.querySelector("video")!;
	70	const video = container instanceof HTMLVideoElement
	71	? container
	72	: container.querySelector("video")!;
71	73	if (video.hasAttribute("controls")) return;
72	74	let src = video.getAttribute("data-src");
73	75	if (!src) return;

src/tags/clover-video.marko+46-35

...	...	@@ -5,9 +5,10 @@ export interface Input {
5	5	poster?: MediaFile \| string;
6	6	header?: Marko.AttrTag<{ content: Marko.Body }>;
7	7	minimal?: boolean;
	8	noFigure?: boolean
8	9	}
9	10
10		<const/{ header, minimal } = input />
	11	<const/{ header, minimal, noFigure } = input />
11	12	<const/file = typeof UNWRAP(input.file) === 'object'
12	13	? input.file
13	14	: UNWRAP(MediaFile.getByPath(input.file), `No file ${JSON.stringify(input.file)}`) />
...	...	@@ -19,40 +20,50 @@ export interface Input {
19	20
20	21	<const/{ width, height } = file.parseDimensions() />
21	22
22		<figure.clover-video class={ minimal }>
23		<if=!minimal>
24		<figcaption>
25		<if=header><${header.content}/></><else>
26		${file.basename}
27		</else>
28		</figcaption>
29		</>
30		<div.stack style=`aspect-ratio:${width}/${height}`>
31		<video
32		data-src=file.path
33		width=1920
34		height=1080
35		poster=poster ? `/file${poster.path}` : null
36		/>
37		<noscript.fallback>
38		<h2>scripting disabled!</h2>
39		<p>
40		to play this video, JavaScript must be enabled in your
41		browser. the default video playback behavior is not enabled here
42		as it results in a worse experience on slower internet
43		connections, and <a href="http://caniuse.com/mpeg-dash">MPEG-DASH
44		streaming is not natively supported in any browser</a> without JavaScript.
45		</p>
46		<p>
47		since disabling JavaScript is not the default in any browser,
48		you likely wanted to download the raw video file anyways.
49		those links are available below, but note that you will need
50		an AV1-compatible video player, such as VLC Media Player or
51		QuickTime Player (M3 chip or newer)
52		</p>
53		</noscript>
54		</div>
55		</figure>
	23	<define/Inner>
	24	<video
	25	class={ 'clover-video': noFigure }
	26	data-src=file.path
	27	poster=poster ? `/file${poster.path}` : null
	28	width=width
	29	height=height
	30	/>
	31	</define>
	32
	33	<if=!noFigure>
	34	<figure.clover-video class={ minimal }>
	35	<if=!minimal>
	36	<figcaption>
	37	<if=header><${header.content}/></><else>
	38	${file.basename}
	39	</else>
	40	</figcaption>
	41	</>
	42	<div.stack style=`aspect-ratio:${width}/${height}`>
	43	<Inner />
	44	<noscript.fallback>
	45	<h2>scripting disabled!</h2>
	46	<p>
	47	to play this video, JavaScript must be enabled in your
	48	browser. the default video playback behavior is not enabled here
	49	as it results in a worse experience on slower internet
	50	connections, and <a href="http://caniuse.com/mpeg-dash">MPEG-DASH
	51	streaming is not natively supported in any browser</a> without JavaScript.
	52	</p>
	53	<p>
	54	since disabling JavaScript is not the default in any browser,
	55	you likely wanted to download the raw video file anyways.
	56	those links are available below, but note that you will need
	57	an AV1-compatible video player, such as VLC Media Player or
	58	QuickTime Player (M3 chip or newer)
	59	</p>
	60	</noscript>
	61	</div>
	62	</figure>
	63	</>
	64	<else>
	65	<Inner />
	66	</>
56	67
57	68	client import "./clover-video.client.ts";
58	69

src/tags/image.tsx+6

...	...	@@ -16,6 +16,10 @@ export default function Image(
16	16	? src
17	17	: UNWRAP(MediaFile.getByPath(src), `No file ${JSON.stringify(src)}`);
18	18
	19	if (extsVideo.has(src.extension.toLowerCase())) {
	20	return <CloverVideo noFigure file={src} />;
	21	}
	22
19	23	const base = `/file${escapeUri(src.path)}`;
20	24	if (src.extension === ".gif") {
21	25	return <img src={base} class={className} style={style} {...rest} />;
...	...	@@ -57,3 +61,5 @@ import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
57	61	import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
58	62	import { escapeUri } from "@/file-viewer/format.ts";
59	63	import { UNWRAP } from "lib/assert.ts";
	64	import { extsVideo } from "@/file-viewer/rules.ts";
	65	import CloverVideo from "./clover-video.marko";

src/tags/photo-grid.css+21-13

...	...	@@ -1,17 +1,25 @@
1	1	photo-grid {
2	2	display: grid;
3	3	gap: 11px;
4		div {
5		position: relative;
6		width: 100%;
7		height: 100%;
8		border: 2px solid black;
9		outline: 4px solid var(--primary, #000);
10		}
11		img {
12		position: absolute;
13		width: 100%;
14		height: 100%;
15		object-fit: cover;
16		}
	4	}
	5	photo-grid > div {
	6	position: relative;
	7	width: 100%;
	8	height: 100%;
	9	border: 2px solid black;
	10	outline: 4px solid var(--primary, #000);
	11	}
	12	photo-grid img {
	13	position: absolute;
	14	width: 100%;
	15	height: 100%;
	16	object-fit: cover;
	17	}
	18	photo-grid video {
	19	position: absolute;
	20	top: 0;
	21	left: 0;
	22	width: 100% !important;
	23	height: 100% !important;
	24	object-fit: cover;
17	25	}