Commit c8b7e2ef01711a96e294ade99c4c478a1a8f0be8 | sitegen

author	clover caruso <git@paperclover.net> 2025-09-06 03:52:38-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:47-07:00
log	c8b7e2ef01711a96e294ade99c4c478a1a8f0be8
tree	e0abbf048b25deb28226d3effc86e7bdad602763
parent	3f7ee731b61bebfb8d577c2cc80b97211691b31d
signature	Commit is signed but in an unrecognized format.

feat(file-viewer): rework scan3, add mpeg-dash encoding

i swore i already wrote this code. but it was not there. i'm much
happier with this pass of the 'AV1+Opus via MPEG-Dash' encoding -- the
code is a lot cleaner. this also reworks the table schema for asset refs
to store derived assets as first class entries. table joins and
everything make `derived.get` really nice (needed to fufill HEAD and
"If-None-Match" requests, which i was stuck on for the cache layer).

not everything is done here, but just enough to actually start running
it on the production store.

13 files changed, 1076 insertions(+), 407 deletions(-)

flake.lock
flake.nix
framework
- lib
src
- file-viewer
  - bin
    scan3.ts
  - ffmpeg.ts
  - models
    AssetRef.ts
    MediaFile.ts
    derived.ts
  - rules.ts
  - transcode-rules.ts

flake.lock+4-4

...	...	@@ -2,16 +2,16 @@
2	2	"nodes": {
3	3	"nixpkgs": {
4	4	"locked": {
5		"lastModified": 1751271578,
6		"narHash": "sha256-P/SQmKDu06x8yv7i0s8bvnnuJYkxVGBWLWHaU+tt4YY=",
	5	"lastModified": 1758763312,
	6	"narHash": "sha256-puBMviZhYlqOdUUgEmMVJpXqC/ToEqSvkyZ30qQ09xM=",
7	7	"owner": "NixOS",
8	8	"repo": "nixpkgs",
9		"rev": "3016b4b15d13f3089db8a41ef937b13a9e33a8df",
	9	"rev": "e57b3b16ad8758fd681511a078f35c416a8cc939",
10	10	"type": "github"
11	11	},
12	12	"original": {
13	13	"owner": "NixOS",
14		"ref": "nixos-unstable",
	14	"ref": "nixpkgs-unstable",
15	15	"repo": "nixpkgs",
16	16	"type": "github"
17	17	}

flake.nix+3-7

...	...	@@ -1,6 +1,6 @@
1	1	{
2	2	inputs = {
3		nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
	3	nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
4	4	utils.url = "github:numtide/flake-utils";
5	5	};
6	6	outputs =
...	...	@@ -15,12 +15,8 @@
15	15	pkgs.python3 # for font subsetting
16	16
17	17	# paperclover.net
18		# (pkgs.ffmpeg.override {
19		# withOpus = true;
20		# withSvtav1 = true;
21		# withJxl = true;
22		# withWebp = true;
23		# })
	18	pkgs.exiftool
	19	pkgs.ffmpeg-full
24	20	pkgs.rsync
25	21	];
26	22	};

framework/lib/async.ts+129

...	...	@@ -213,6 +213,7 @@ function defaultGetItemText(item: unknown) {
213	213	return itemText;
214	214	}
215	215
	216	/** @deprecated use OnceMap2 */
216	217	export class OnceMap<T> {
217	218	private ongoing = new Map<string, Promise<T>>();
218	219
...	...	@@ -300,6 +301,134 @@ export function once<T>(fn: () => Promise<T>): () => Promise<T> {
300	301	};
301	302	}
302	303
	304	export class Watch<T> {
	305	#value: T;
	306	#observers = new Set<WatchCallback<T>>();
	307	constructor(value: T) {
	308	this.#value = value;
	309	}
	310
	311	on(cb: WatchCallback<T>) {
	312	this.#observers.add(cb);
	313	return () => void this.#observers.delete(cb);
	314	}
	315	once(cb: WatchCallback<T>) {
	316	const once = (n: T, p: T) => (cb(n, p), this.#observers.delete(once));
	317	this.#observers.add(once);
	318	}
	319
	320	next(): Promise<T> {
	321	return new Promise((resolve) => this.once(resolve));
	322	}
	323
	324	until(condition: (v: T) => boolean): Promise<T> {
	325	if (condition(this.#value)) return Promise.resolve(this.#value);
	326	return new Promise((resolve) => {
	327	function check(next: T) {
	328	if (condition(next)) resolve(next), release();
	329	}
	330	const release = this.on(check);
	331	});
	332	}
	333
	334	get value() {
	335	return this.#value;
	336	}
	337	set value(next: T) {
	338	const prev = this.#value;
	339	if (prev !== next) {
	340	this.#value = next;
	341	this.#observers.forEach((cb) => cb(next, prev));
	342	}
	343	}
	344	}
	345	type WatchCallback<T> = (next: T, prev: T) => void;
	346
	347	/**
	348	* When two requests with the same args come in at the same time, the response
	349	* from them is shared. After returning, the data is no longer stored.
	350	*/
	351	export class DedupeConcurrent<Args extends unknown[], Ret> {
	352	pending = new Map<unknown, Promise<Ret>>();
	353	constructor(
	354	public fn: (...args: Args) => Promise<Ret>,
	355	public keyFn: (args: Args) => unknown = JSON.stringify,
	356	) {}
	357
	358	isPending(...args: Args) {
	359	return this.pending.has(this.keyFn(args));
	360	}
	361
	362	/** Unbound to allow easily exporting this callback */
	363	get = (...args: Args): Promise<Ret> => {
	364	const k = this.keyFn(args);
	365	let promise = this.pending.get(k);
	366	if (promise) return promise;
	367	promise = this.fn(...args);
	368	this.pending.set(k, promise);
	369	void promise.finally(() => void this.pending.delete(k));
	370	return promise;
	371	};
	372	}
	373
	374	/**
	375	* When two requests with the same args come, the result of the first is memoized.
	376	*/
	377	export class OnceMap2<Args extends unknown[], Ret> {
	378	cache = new Map<unknown, Promise<Ret>>();
	379	constructor(
	380	public fn: (...args: Args) => Promise<Ret>,
	381	public keyFn: (args: Args) => unknown = JSON.stringify,
	382	) {}
	383
	384	has(...args: Args) {
	385	return this.cache.has(this.keyFn(args));
	386	}
	387
	388	/** Unbound to allow easily exporting this callback */
	389	getOrRun = (...args: Args): Promise<Ret> => {
	390	const k = this.keyFn(args);
	391	let promise = this.cache.get(k);
	392	if (promise) return promise;
	393	promise = this.fn(...args);
	394	this.cache.set(k, promise);
	395	return promise;
	396	};
	397	}
	398
	399	export class PromiseAggregator {
	400	failures: unknown[] = [];
	401	promises = new Set<Promise<unknown>>();
	402
	403	push(p: Promise<unknown>) {
	404	p = p.then(
	405	() => {
	406	this.promises.delete(p);
	407	},
	408	(err) => {
	409	this.failures.push(err);
	410	},
	411	);
	412	this.promises.add(p);
	413	}
	414
	415	async all() {
	416	while (this.promises.size > 0) {
	417	await Promise.all(this.promises);
	418	this.promises.clear();
	419	}
	420	if (this.failures.length > 0) {
	421	const agg = new AggregateError(this.failures);
	422	this.failures = [];
	423	throw agg;
	424	}
	425	}
	426
	427	[Symbol.asyncDispose]() {
	428	return this.all();
	429	}
	430	}
	431
303	432	import { Progress } from "@paperclover/console/Progress";
304	433	import { Spinner } from "@paperclover/console/Spinner";
305	434	import * as path from "node:path";

framework/lib/lru.ts-2

...	...	@@ -155,8 +155,6 @@ export class Lru<K, V> extends Map<K, V> {
155	155	this.#used -= needed - remain;
156	156	}
157	157
158		hasEvictableCapacity() {}
159
160	158	/** Unlink an entry without removing it. */
161	159	#evict(key: K) {
162	160	const entry = this.#order.get(key);

framework/lib/path.ts+137-12

...	...	@@ -2,19 +2,25 @@ const unique = new Map<string, WeakRef<Path>>();
2	2	const final = new FinalizationRegistry((raw: string) => unique.delete(raw));
3	3
4	4	export class Path {
5		raw: string;
	5	private raw: string;
6	6
7		static from(...parts: (string \| Path)[]) {
8		return new Path(...parts.map((x) => x instanceof Path ? x.raw : x));
	7	static resolve(base: Path \| string, ...more: string[]): Path {
	8	if (more.length === 0) {
	9	if (typeof base === "string") {
	10	return new Path(path.resolve(base));
	11	}
	12	return base;
	13	}
	14	return new Path(path.resolve(
	15	typeof base === "string" ? base : base.raw,
	16	...more,
	17	));
9	18	}
10	19
11	20	/** For convenience, paths are deduplicated. */
12		constructor(...parts: string[]) {
13		const raw = this.raw =
14		parts.length === 1 && parts[0] && path.isAbsolute(parts[0])
15		? parts[0]
16		: path.resolve(...parts);
17		const existing = unique.get(raw)?.deref();
	21	constructor(raw: string) {
	22	ASSERT(path.isAbsolute(raw));
	23	const existing = unique.get(this.raw = raw)?.deref();
18	24	if (existing) return existing;
19	25	unique.set(raw, new WeakRef(this));
20	26	final.register(this, raw);
...	...	@@ -31,8 +37,8 @@ export class Path {
31	37	return new Path(parent);
32	38	}
33	39
34		get relative() {
35		return path.relative(".", this.raw);
	40	relative(from: Path \| "." = ".") {
	41	return path.relative(from.toString(), this.raw);
36	42	}
37	43
38	44	get ext() {
...	...	@@ -47,14 +53,131 @@ export class Path {
47	53	return path.basename(this.raw, this.ext);
48	54	}
49	55
	56	replaceExt(ext: string) {
	57	if (ext.length > 0 && ext[0] !== ".") ext = `.${ext}`;
	58	const { ext: previous } = this;
	59	return new Path(this.raw.slice(0, previous.length) + ext);
	60	}
	61
	62	// -- filesystem --
	63
50	64	join(subPath: string, ...more: string[]) {
51		return new Path(this.raw, subPath, ...more);
	65	return new Path(path.join(this.raw, subPath, ...more));
52	66	}
53	67
	68	/** @deprecated ifExistsSync */
54	69	existsSync() {
55	70	return fs.existsSync(this.raw);
56	71	}
57	72
	73	/**
	74	* Be aware that checking for a file's existence before performing an action
	75	* can always be a race condition. Prefer reading the file directly and
	76	* handling the ENOENT error.
	77	*/
	78	ifExistsSync() {
	79	return fs.existsSync(this.raw) ? this : null;
	80	}
	81
	82	async delete(o?: RmOptions) {
	83	await fs.rm(this.raw, o);
	84	}
	85
	86	deleteSync(o?: RmOptions) {
	87	fs.rmSync(this.raw, o);
	88	}
	89
	90	async stat() {
	91	return await fs.stat(this.raw);
	92	}
	93
	94	statSync() {
	95	return fs.statSync(this.raw);
	96	}
	97
	98	async makeDir() {
	99	return await fs.mkdir(this.raw);
	100	}
	101
	102	makeDirSync() {
	103	return fs.mkdirSync(this.raw);
	104	}
	105
	106	async makeOrEmptyDir() {
	107	// TODO: remove contents instead of node
	108	await fs.rm(this.raw, { force: true, recursive: true });
	109	return await fs.mkdir(this.raw);
	110	}
	111
	112	makeOrEmptyDirSync() {
	113	// TODO: remove contents instead of node
	114	fs.rmSync(this.raw, { force: true, recursive: true });
	115	return fs.mkdirSync(this.raw);
	116	}
	117
	118	readIfExistsSync(encoding?: "buffer"): Buffer \| null;
	119	readIfExistsSync<T = unknown>(encoding: "json"): T \| null;
	120	readIfExistsSync(encoding: "utf-8"): string \| null;
	121	readIfExistsSync(encoding?: "utf-8" \| "buffer" \| "json"): unknown;
	122	readIfExistsSync(encoding?: "utf-8" \| "buffer" \| "json"): unknown {
	123	try {
	124	return this.readSync(encoding);
	125	} catch (err) {
	126	if (error.code(err) === "ENOENT") return null;
	127	throw err;
	128	}
	129	}
	130
	131	readSync(encoding?: "buffer"): Buffer;
	132	readSync<T = unknown>(encoding: "json"): T;
	133	readSync(encoding: "utf-8"): string;
	134	readSync(encoding?: "utf-8" \| "buffer" \| "json"): unknown;
	135	readSync(encoding?: "utf-8" \| "buffer" \| "json"): unknown {
	136	if (encoding === "json") {
	137	return JSON.parse(fs.readFileSync(this.raw, "utf-8"));
	138	}
	139	return fs.readFileSync(this.raw, encoding === "buffer" ? {} : { encoding });
	140	}
	141
	142	async readIfExists(encoding?: "buffer"): Promise<Buffer \| null>;
	143	async readIfExists<T = unknown>(encoding: "json"): Promise<T \| null>;
	144	async readIfExists(encoding: "utf-8"): Promise<string \| null>;
	145	async readIfExists(
	146	encoding?: "utf-8" \| "buffer" \| "json",
	147	): Promise<unknown>;
	148	async readIfExists(
	149	encoding?: "utf-8" \| "buffer" \| "json",
	150	): Promise<unknown> {
	151	try {
	152	return await this.read(encoding);
	153	} catch (err) {
	154	if (error.code(err) === "ENOENT") return null;
	155	throw err;
	156	}
	157	}
	158
	159	async read(encoding?: "buffer"): Promise<Buffer>;
	160	async read<T = unknown>(encoding: "json"): Promise<T>;
	161	async read(encoding: "utf-8"): Promise<string>;
	162	async read(encoding?: "utf-8" \| "buffer" \| "json"): Promise<unknown>;
	163	async read(encoding?: "utf-8" \| "buffer" \| "json"): Promise<unknown> {
	164	if (encoding === "json") {
	165	return JSON.parse(await fs.readFile(this.raw, "utf-8"));
	166	}
	167	return await fs.readFile(
	168	this.raw,
	169	encoding === "buffer" ? {} : { encoding },
	170	);
	171	}
	172
	173	readDirSync(): Array<Path> {
	174	return fs.readdirSync(this.raw).map((sub) => this.join(sub));
	175	}
	176
	177	async readDir(): Promise<Array<Path>> {
	178	return (await fs.readdir(this.raw)).map((sub) => this.join(sub));
	179	}
	180
58	181	static sep = path.sep;
59	182
60	183	[Symbol.for("nodejs.util.inspect.custom")](
...	...	@@ -69,3 +192,5 @@ export class Path {
69	192	import * as path from "node:path";
70	193	import * as fs from "./fs.ts";
71	194	import * as util from "node:util";
	195	import * as error from "./error.ts";
	196	import type { RmOptions } from "node:fs";

framework/lib/subprocess.ts+43

...	...	@@ -11,5 +11,48 @@ export const exec: typeof execFileRaw = ((
11	11	throw e;
12	12	})) as any;
13	13
	14	const activePids = new Set<number>();
	15	export interface SpawnOptions extends child_process.SpawnOptions {
	16	cmd: string[];
	17	}
	18	export function spawn({ cmd, ...rest }: SpawnOptions) {
	19	rest.detached ??= true;
	20	const process = child_process.spawn(cmd[0]!, cmd.slice(1), rest);
	21	const pid = rest.detached && process.pid ? -process.pid : process.pid;
	22	if (pid) {
	23	activePids.add(pid);
	24	process.on("exit", () => void activePids.delete(pid));
	25	}
	26	return process;
	27	}
	28
	29	export async function spawnAndWait(opts: SpawnOptions) {
	30	opts.stdio ??= ["ignore", "inherit", "inherit"];
	31	const process = spawn(opts);
	32	let [code, signal] = await events.EventEmitter.once(process, "exit") as [
	33	number \| null,
	34	string \| null,
	35	];
	36	if (code !== 0) {
	37	if (code !== null && code > 2 ** 31) code \|= 0;
	38	const exit = signal ? `signal ${signal}` : `code ${code}`;
	39	throw new Error(`${opts.cmd[0]} failed with ${exit}`);
	40	}
	41	const [stdout, stderr] = await Promise.all([
	42	process.stdout ? Array.fromAsync(process.stdout).then(Buffer.concat) : null,
	43	process.stderr ? Array.fromAsync(process.stderr).then(Buffer.concat) : null,
	44	]);
	45	return { stdout, stderr };
	46	}
	47
	48	process.on("exit", () => {
	49	for (const pid of activePids) {
	50	console.info(`Kill ${Math.abs(pid)}`);
	51	process.kill(pid, 9);
	52	}
	53	});
	54
	55	import * as process from "node:process";
	56	import * as events from "node:events";
14	57	import * as util from "node:util";
15	58	import * as child_process from "node:child_process";

src/file-viewer/bin/scan3.ts+371-224

...	...	@@ -10,8 +10,7 @@
10	10	// This is the third iteration of the scanner, hence its name "scan3";
11	11	// Remember that any software you want to be maintainable and high
12	12	// quality cannot be written with AI.
13		const workDir = path.resolve(".clover/derived");
14		const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
	13	const sotToken = process.env.CLOVER_SOT_KEY;
15	14
16	15	export async function main() {
17	16	const start = performance.now();
...	...	@@ -32,11 +31,12 @@ export async function main() {
32	31	async fn(absPath: string) {
33	32	const stat = await fs.stat(absPath);
34	33
35		const publicPath = toPublicPath(absPath);
	34	const publicPath = toPublicPath(Path.resolve(absPath));
36	35	const mediaFile = MediaFile.getByPath(publicPath);
37	36
38	37	if (stat.isDirectory()) {
39		const items = await fs.readdir(absPath);
	38	const items = (await fs.readdir(absPath))
	39	.filter((basename) => !skipBasename(basename));
40	40	qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
41	41
42	42	if (mediaFile) {
...	...	@@ -51,7 +51,7 @@ export async function main() {
51	51
52	52	qMeta.addMany(
53	53	deleted.map((mediaFile) => ({
54		absPath: path.join(root, mediaFile.path),
	54	path: Path.resolve(root, mediaFile.path),
55	55	publicPath: mediaFile.path,
56	56	stat: null,
57	57	mediaFile,
...	...	@@ -68,27 +68,27 @@ export async function main() {
68	68	stat.size !== mediaFile.size \|\|
69	69	stat.mtime.getTime() !== mediaFile.date.getTime()
70	70	) {
71		qMeta.add({ absPath, publicPath, stat, mediaFile });
	71	qMeta.add({ path: new Path(absPath), publicPath, stat, mediaFile });
72	72	return;
73	73	}
74	74
75	75	// If the scanners changed, it may mean more processes should be run.
76		queueProcessors({ absPath, stat, mediaFile });
	76	queueProcessors({ path: new Path(absPath), stat, mediaFile });
77	77	},
78	78	maxJobs: 24,
79	79	});
80	80	using qMeta = new async.Queue({
81	81	name: "Update Metadata",
82		async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) {
	82	async fn({ path, publicPath, stat, mediaFile }: UpdateMetadataJob) {
83	83	if (!stat) {
84	84	// File was deleted.
85	85	await runUndoProcessors(UNWRAP(mediaFile));
86	86	return;
87	87	}
88		// TODO: run scrubLocationMetadata first
	88	await scrubLocationMetadata(path, stat);
89	89
90	90	const hash = await new Promise<string>((resolve, reject) => {
91		const reader = fs.createReadStream(absPath);
	91	const reader = fs.createReadStream(path.toString());
92	92	reader.on("error", reject);
93	93
94	94	const hasher = crypto.createHash("sha1").setEncoding("hex");
...	...	@@ -121,7 +121,11 @@ export async function main() {
121	121	dimensions: mediaFile?.dimensions ?? "",
122	122	contents: mediaFile?.contents ?? "",
123	123	});
124		await queueProcessors({ absPath, stat, mediaFile });
	124	await queueProcessors({
	125	path,
	126	stat,
	127	mediaFile,
	128	});
125	129	},
126	130	getItemText: (job) =>
127	131	job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"),
...	...	@@ -130,10 +134,15 @@ export async function main() {
130	134	using qProcess = new async.Queue({
131	135	name: "Process Contents",
132	136	async fn(
133		{ absPath, stat, mediaFile, processor, index, after }: ProcessJob,
	137	{ path, stat, mediaFile, processor, index, after }: ProcessJob,
134	138	spin,
135	139	) {
136		await processor.run({ absPath, stat, mediaFile, spin });
	140	await processor.run({
	141	path,
	142	stat,
	143	mediaFile,
	144	spin,
	145	});
137	146	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
138	147	for (const dependantJob of after) {
139	148	ASSERT(
...	...	@@ -155,12 +164,12 @@ export async function main() {
155	164	.filter(Boolean)
156	165	.map(([a, b, c]) => ({
157	166	id: a,
158		hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0),
	167	hash: (UNWRAP(b).charCodeAt(0) << 8) + UNWRAP(c).charCodeAt(0),
159	168	}));
160	169	}
161	170
162	171	async function queueProcessors({
163		absPath,
	172	path,
164	173	stat,
165	174	mediaFile,
166	175	}: Omit<ProcessFileArgs, "spin">) {
...	...	@@ -169,6 +178,7 @@ export async function main() {
169	178	p.include ? p.include.has(ext) : !p.exclude?.has(ext)
170	179	);
171	180	if (possible.length === 0) return;
	181	ASSERT(possible.length < 16, "too many bits");
172	182
173	183	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
174	184	ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
...	...	@@ -209,13 +219,13 @@ export async function main() {
209	219	for (let i = 0, { length } = possible; i < length; i += 1) {
210	220	if ((processed & (1 << (16 + i))) === 0) {
211	221	const job: ProcessJob = {
212		absPath,
	222	path,
213	223	stat,
214	224	mediaFile,
215		processor: possible[i],
	225	processor: UNWRAP(possible[i]),
216	226	index: i,
217	227	after: [],
218		needs: possible[i].depends.length,
	228	needs: UNWRAP(possible[i]).depends.length,
219	229	};
220	230	jobs.push(job);
221	231	if (job.needs === 0) qProcess.add(job);
...	...	@@ -257,9 +267,8 @@ export async function main() {
257	267	await qProcess.done();
258	268
259	269	// Update directory metadata
260		const dirs = MediaFile.getDirectoriesToReindex().sort(
261		(a, b) => b.path.length - a.path.length,
262		);
	270	const dirs = MediaFile.getDirectoriesToReindex()
	271	.sort((a, b) => b.path.length - a.path.length);
263	272	for (const dir of dirs) {
264	273	const children = dir.getChildren();
265	274
...	...	@@ -315,52 +324,54 @@ export async function main() {
315	324	}
316	325
317	326	// Sync to remote
318		if ((await fs.readdir(workDir)).length > 0) {
319		await rsync.spawn({
320		args: [
321		"--links",
322		"--recursive",
323		"--times",
324		"--partial",
325		"--progress",
326		"--remove-source-files",
327		"--delay-updates",
328		workDir + "/",
329		"clo@zenith:/mnt/storage1/clover/Documents/Config/clover_file/derived/",
330		],
331		title: "Uploading Derived Assets",
332		cwd: process.cwd(),
333		});
334
335		await fs.removeEmptyDirectories(workDir);
336		} else {
337		console.info("No new derived assets");
338		}
	327	// if (
	328	// ((await derived.workDir.ifExistsSync()?.readDir())?.length ?? 0) > 0
	329	// ) {
	330	// await rsync.spawn({
	331	// args: [
	332	// "--links",
	333	// "--recursive",
	334	// "--times",
	335	// "--partial",
	336	// "--progress",
	337	// "--remove-source-files",
	338	// "--delay-updates",
	339	// derived.workDir.toString() + "/",
	340	// "clo@zenith:/mnt/storage1/clover/Documents/Config/clover_file/derived/",
	341	// ],
	342	// title: "Uploading Derived Assets",
	343	// cwd: process.cwd(),
	344	// });
	345	//
	346	// await fs.removeEmptyDirectories(derived.workDir.toString());
	347	// } else {
	348	// console.info("No new derived assets");
	349	// }
339	350
340	351	MediaFile.db.prepare("VACUUM").run();
341	352	MediaFile.db.reload();
342		// TODO: reload prod web instance
343		await rsync.spawn({
344		args: [
345		MediaFile.db.file,
346		"clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
347		],
348		title: "Uploading Database",
349		cwd: process.cwd(),
350		});
351		{
352		const res = await fetch("https://db.paperclover.net/reload", {
353		method: "post",
354		headers: {
355		Authorization: sotToken,
356		},
357		});
358		if (!res.ok) {
359		console.warn(
360		`Failed to reload remote database ${res.status} ${res.statusText}`,
361		);
362		}
363		}
	353
	354	// await rsync.spawn({
	355	// args: [
	356	// MediaFile.db.file,
	357	// "clo@zenith:/mnt/storage1/clover/Documents/Config/paperclover/cache.sqlite",
	358	// ],
	359	// title: "Uploading Database",
	360	// cwd: process.cwd(),
	361	// });
	362	// if (sotToken) {
	363	// const res = await fetch("https://db.paperclover.net/reload", {
	364	// method: "post",
	365	// headers: {
	366	// Authorization: sotToken,
	367	// },
	368	// });
	369	// if (!res.ok) {
	370	// console.warn(
	371	// `Failed to reload remote database ${res.status} ${res.statusText}`,
	372	// );
	373	// }
	374	// } else console.warn("Missing SOT token");
364	375
365	376	console.info(
366	377	"Updated file viewer index in \x1b[1m" +
...	...	@@ -369,14 +380,12 @@ export async function main() {
369	380	);
370	381
371	382	const { duration, count } = MediaFile.db
372		.prepare<[], { count: number; duration: number }>(
373		`
374		select
375		count(*) as count,
376		sum(duration) as duration
377		from media_files
378		`,
379		)
	383	.prepare<[], { count: number; duration: number }>(`
	384	select
	385	count(*) as count,
	386	sum(duration) as duration
	387	from media_files
	388	`)
380	389	.getNonNull();
381	390
382	391	console.info();
...	...	@@ -408,11 +417,12 @@ const ffmpegBin = testProgram("ffmpeg", "--help");
408	417
409	418	const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
410	419
	420	// NOTE: Never re-order the processors. Add new ones at the end.
411	421	const procDuration: Process = {
412	422	name: "calculate duration",
413	423	enable: ffprobeBin !== null,
414	424	include: rules.extsDuration,
415		async run({ absPath, mediaFile }) {
	425	async run({ path, mediaFile }) {
416	426	const { stdout } = await subprocess.exec(ffprobeBin!, [
417	427	"-v",
418	428	"error",
...	...	@@ -420,7 +430,7 @@ const procDuration: Process = {
420	430	"format=duration",
421	431	"-of",
422	432	"default=noprint_wrappers=1:nokey=1",
423		absPath,
	433	path.toString(),
424	434	]);
425	435
426	436	const duration = parseFloat(stdout.trim());
...	...	@@ -431,19 +441,18 @@ const procDuration: Process = {
431	441	},
432	442	};
433	443
434		// NOTE: Never re-order the processors. Add new ones at the end.
435	444	const procDimensions: Process = {
436	445	name: "calculate dimensions",
437	446	enable: ffprobeBin != null,
438	447	include: rules.extsDimensions,
439		async run({ absPath, mediaFile }) {
440		const ext = path.extname(absPath);
	448	async run({ path, mediaFile }) {
	449	const { ext } = path;
441	450
442	451	let dimensions;
443	452
444	453	if (ext === ".svg") {
445	454	// Parse out of text data
446		const content = await fs.readFile(absPath, "utf8");
	455	const content = await path.read("utf-8");
447	456	const widthMatch = content.match(/width="(\d+)"/);
448	457	const heightMatch = content.match(/height="(\d+)"/);
449	458
...	...	@@ -452,7 +461,7 @@ const procDimensions: Process = {
452	461	}
453	462	} else {
454	463	// Use ffprobe to observe streams
455		const { stdout } = await execFile("ffprobe", [
	464	const { stdout } = await subprocess.exec("ffprobe", [
456	465	"-v",
457	466	"error",
458	467	"-select_streams",
...	...	@@ -461,7 +470,7 @@ const procDimensions: Process = {
461	470	"stream=width,height",
462	471	"-of",
463	472	"csv=s=x:p=0",
464		absPath,
	473	path.toString(),
465	474	]);
466	475	if (stdout.includes("x")) {
467	476	dimensions = stdout.trim();
...	...	@@ -475,9 +484,9 @@ const procDimensions: Process = {
475	484	const procLoadTextContents: Process = {
476	485	name: "load text content",
477	486	include: rules.extsReadContents,
478		async run({ absPath, mediaFile, stat }) {
	487	async run({ path, mediaFile, stat }) {
479	488	if (stat.size > 1_000_000) return;
480		const text = await fs.readFile(absPath, "utf-8");
	489	const text = await path.read("utf-8");
481	490	mediaFile.setContents(text);
482	491	},
483	492	};
...	...	@@ -485,9 +494,9 @@ const procLoadTextContents: Process = {
485	494	const procHighlightCode: Process = {
486	495	name: "highlight source code",
487	496	include: new Set(rules.extsCode.keys()),
488		async run({ absPath, mediaFile, stat }) {
	497	async run({ path, mediaFile, stat }) {
489	498	const language = UNWRAP(
490		rules.extsCode.get(path.extname(absPath).toLowerCase()),
	499	rules.extsCode.get(path.ext.toLowerCase()),
491	500	);
492	501	// An issue is that .ts is an overloaded extension, shared between
493	502	// 'transport stream' and 'typescript'.
...	...	@@ -497,7 +506,7 @@ const procHighlightCode: Process = {
497	506	// - invalid UTF-8
498	507	if (stat.size > 1_000_000) return;
499	508	let code;
500		const buf = await fs.readFile(absPath);
	509	const buf = await path.read();
501	510	try {
502	511	code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
503	512	} catch (error) {
...	...	@@ -512,9 +521,9 @@ const procHighlightCode: Process = {
512	521	const procImageSubsets: Process = {
513	522	name: "encode image subsets",
514	523	include: rules.extsImage,
515		depends: ["calculate dimensions"],
	524	depends: [procDimensions.name],
516	525	version: 2,
517		async run({ absPath, mediaFile, spin }) {
	526	async run({ path, mediaFile, spin }) {
518	527	const { width, height } = UNWRAP(mediaFile.parseDimensions());
519	528	const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
520	529	const baseStatus = spin.text;
...	...	@@ -526,19 +535,16 @@ const procImageSubsets: Process = {
526	535	spin.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
527	536
528	537	stack.use(
529		await produceAsset(`${mediaFile.hash}/${size}${ext}`, async (out) => {
530		await fs.mkdir(path.dirname(out));
531		await fs.rm(out, { force: true });
532		await execFile(ffmpegBin!, [
	538	await derived.produce(mediaFile, `${size}${ext}`, async (dir) => {
	539	await subprocess.exec(ffmpegBin!, [
533	540	...ffmpegOptions,
534	541	"-i",
535		absPath,
	542	path.toString(),
536	543	"-vf",
537	544	`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
538	545	...args,
539		out,
	546	dir.join(`${size}${ext}`).toString(),
540	547	]);
541		return [out];
542	548	}),
543	549	);
544	550	}
...	...	@@ -546,16 +552,12 @@ const procImageSubsets: Process = {
546	552
547	553	stack.move();
548	554	},
549		async undo(mediaFile) {
550		const { width } = UNWRAP(mediaFile.parseDimensions());
551		const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
552		for (const size of targetSizes) {
553		for (const { ext } of transcodeRules.imagePresets) {
554		unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
555		}
556		}
557		},
558	555	};
	556
	557	const videoInputArgsCache = new async.OnceMap2(
	558	transcodeRules.getVideoInputArgs,
	559	);
	560
559	561	const qualityMap: Record<string, string> = {
560	562	u: "ultra-high",
561	563	h: "high",
...	...	@@ -564,91 +566,150 @@ const qualityMap: Record<string, string> = {
564	566	d: "data-saving",
565	567	};
566	568	const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
567		name: `encode ${preset.codec} ${UNWRAP(qualityMap[preset.id[1]])}`,
	569	name: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
568	570	include: rules.extsVideo,
569	571	enable: ffmpegBin != null,
570		async run({ absPath, mediaFile, spin }) {
571		if ((mediaFile.duration ?? 0) < 10) return;
572		await produceAsset(`${mediaFile.hash}/${preset.id}`, async (base) => {
573		base = path.dirname(base);
574		await fs.mkdir(base);
575
576		let inputArgs = ["-i", absPath];
577		try {
578		const config = await fs.readJson<any>(
579		path.join(
580		path.dirname(absPath),
581		path.basename(absPath, path.extname(absPath)) + ".json",
582		),
583		);
584		if (config.encoder && typeof config.encoder.videoSrc === "string") {
585		const { videoSrc, audioSrc, rate } = config.encoder;
586		inputArgs = [
587		...(rate ? ["-r", String(rate)] : []),
588		"-i",
589		videoSrc,
590		...(audioSrc ? ["-i", audioSrc] : []),
591		];
592		}
593		} catch (err: any) {
594		if (err?.code !== "ENOENT") throw err;
595		}
596
597		const args = transcodeRules.getVideoArgs(preset, base, inputArgs);
598		try {
599		const fakeProgress = new Progress({ text: spin.text, spinner: null });
600		fakeProgress.stop();
601		spin.format = (now: number) => fakeProgress.format(now);
602		// @ts-expect-error
603		fakeProgress.redraw = () => spin.redraw();
604
605		await ffmpeg.spawn({
606		ffmpeg: ffmpegBin!,
607		title: fakeProgress.text,
608		progress: fakeProgress,
609		args,
610		cwd: base,
611		});
612		return await collectFiles();
613		} catch (err) {
614		for (const file of await collectFiles()) {
615		try {
616		fs.rm(file);
617		} catch {}
618		}
619		throw err;
620		}
621
622		async function collectFiles(): Promise<string[]> {
623		return (await fs.readdir(base))
624		.filter((basename) => basename.startsWith(preset.id))
625		.map((basename) => path.join(base, basename));
626		}
	572	depends: [procDuration.name, procDimensions.name],
	573	async run({ path, mediaFile, spin }) {
	574	if ((mediaFile.duration ?? 0) < 5) return;
	575	await derived.produce(mediaFile, `av1-${preset.id}`, async (dir) => {
	576	const input = await videoInputArgsCache.getOrRun(path);
	577	const args = transcodeRules.getAv1VideoArgs(preset, input.video, dir);
	578
	579	const fakeProgress = new Progress({ text: spin.text, spinner: null });
	580	fakeProgress.stop();
	581	spin.format = (now: number) => fakeProgress.format(now);
	582	// @ts-expect-error
	583	fakeProgress.redraw = () => spin.redraw();
	584
	585	await ffmpeg.spawn({
	586	ffmpeg: ffmpegBin!,
	587	title: fakeProgress.text,
	588	progress: fakeProgress,
	589	args,
	590	cwd: dir,
	591	});
627	592	});
628	593	},
629	594	}));
	595	const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
	596	name: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
	597	include: rules.extsVideo,
	598	enable: ffmpegBin != null,
	599	depends: [procDuration.name, procDimensions.name],
	600	async run({ path, mediaFile, spin }) {
	601	if ((mediaFile.duration ?? 0) < 5) return;
	602	await derived.produce(mediaFile, `opus-${preset.id}`, async (dir) => {
	603	const input = await videoInputArgsCache.getOrRun(path);
	604	if (!input.audio) return;
	605	const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
	606
	607	const fakeProgress = new Progress({ text: spin.text, spinner: null });
	608	fakeProgress.stop();
	609	spin.format = (now: number) => fakeProgress.format(now);
	610	// @ts-expect-error
	611	fakeProgress.redraw = () => spin.redraw();
	612
	613	await ffmpeg.spawn({
	614	ffmpeg: ffmpegBin!,
	615	title: fakeProgress.text,
	616	progress: fakeProgress,
	617	args,
	618	cwd: dir,
	619	});
	620	});
	621	},
	622	}));
	623	const procDash: Process = {
	624	name: `encode mpeg-dash`,
	625	include: rules.extsVideo,
	626	enable: ffmpegBin != null,
	627	depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
	628	async run({ path, mediaFile, spin }) {
	629	if ((mediaFile.duration ?? 0) < 5) return;
	630	await derived.produce(mediaFile, `dash-av1`, async (dir) => {
	631	const input = await videoInputArgsCache.getOrRun(path);
	632	const videos = transcodeRules.videoFormats.map(
	633	(preset) =>
	634	UNWRAP(dir.parent).join(
	635	`av1-${preset.id}`,
	636	transcodeRules.av1FileName,
	637	),
	638	);
	639	const audios = input.audio
	640	? transcodeRules.videoFormats.map(
	641	(preset) =>
	642	UNWRAP(dir.parent).join(
	643	`av1-${preset.id}`,
	644	transcodeRules.av1FileName,
	645	),
	646	)
	647	: [];
	648	const args = transcodeRules.getMpegDashArgs(videos, audios, dir);
	649
	650	await dir.join("d").makeDir();
	651
	652	const fakeProgress = new Progress({ text: spin.text, spinner: null });
	653	fakeProgress.stop();
	654	spin.format = (now: number) => fakeProgress.format(now);
	655	// @ts-expect-error
	656	fakeProgress.redraw = () => spin.redraw();
	657
	658	await ffmpeg.spawn({
	659	ffmpeg: ffmpegBin!,
	660	title: fakeProgress.text,
	661	progress: fakeProgress,
	662	args,
	663	cwd: dir,
	664	});
	665	});
	666	},
	667	};
	668	const procH264Hls: Process = {
	669	name: `encode h.264 hls`,
	670	include: rules.extsVideo,
	671	enable: ffmpegBin != null,
	672	depends: [procDuration.name, procDimensions.name],
	673	async run({ path, mediaFile, spin }) {
	674	if ((mediaFile.duration ?? 0) < 5) return;
	675	await derived.produce(mediaFile, `hls`, async (dir) => {
	676	const input = await videoInputArgsCache.getOrRun(path);
	677	const args = transcodeRules.getH264HlsArgs(input, dir);
	678
	679	const fakeProgress = new Progress({ text: spin.text, spinner: null });
	680	fakeProgress.stop();
	681	spin.format = (now: number) => fakeProgress.format(now);
	682	// @ts-expect-error
	683	fakeProgress.redraw = () => spin.redraw();
	684
	685	await ffmpeg.spawn({
	686	ffmpeg: ffmpegBin!,
	687	title: fakeProgress.text,
	688	progress: fakeProgress,
	689	args,
	690	cwd: dir,
	691	});
	692	});
	693	},
	694	};
630	695
631	696	const procCompression = [
632	697	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
633	698	{ name: "zstd", fn: () => zlib.createZstdCompress() },
634		].map(
635		({ name, fn }) =>
636		({
637		name: `compress ${name}`,
638		exclude: rules.extsPreCompressed,
639		async run({ absPath, mediaFile }) {
640		if ((mediaFile.size ?? 0) < 10) return;
641		await produceAsset(`${mediaFile.hash}/${name}`, async (base) => {
642		fs.mkdirSync(path.dirname(base));
643		await stream.promises.pipeline(
644		fs.createReadStream(absPath),
645		fn(),
646		fs.createWriteStream(base),
647		);
648		return [base];
649		});
650		},
651		}) satisfies Process as Process,
	699	].map<Process>(
	700	({ name, fn }) => ({
	701	name: `compress ${name}`,
	702	exclude: rules.extsPreCompressed,
	703	async run({ path, mediaFile }) {
	704	await derived.produce(mediaFile, name, async (dir) => {
	705	await stream.promises.pipeline(
	706	fs.createReadStream(path.toString()),
	707	fn(),
	708	fs.createWriteStream(dir.join(name).toString()),
	709	);
	710	});
	711	},
	712	}),
652	713	);
653	714
654	715	const processors = [
...	...	@@ -658,7 +719,10 @@ const processors = [
658	719	procHighlightCode,
659	720	procImageSubsets,
660	721	...procVideos,
	722	...procVideoAudios,
	723	procDash,
661	724	...procCompression,
	725	procH264Hls,
662	726	].map((process, id, all) => {
663	727	const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
664	728	return {
...	...	@@ -688,53 +752,22 @@ function resizeDimensions(w: number, h: number, desiredWidth: number) {
688	752	return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
689	753	}
690	754
691		async function produceAsset(
692		key: string,
693		builder: (prefix: string) => Promise<string[]>,
694		) {
695		const asset = AssetRef.putOrIncrement(key);
696		try {
697		if (asset.refs === 1) {
698		const paths = await builder(path.join(workDir, key));
699		asset.addFiles(
700		paths.map((file) => path.relative(workDir, file).replaceAll("\\", "/")),
701		);
702		}
703		return {
704		[Symbol.dispose]: () => asset.unref(),
705		};
706		} catch (err: any) {
707		if (err && typeof err === "object") err.assetKey = key;
708		asset.unref();
709		throw err;
710		}
711		}
712
713		async function unproduceAsset(key: string) {
714		const ref = AssetRef.get(key);
715		if (ref) {
716		ref.unref();
717		console.warn(`TODO: unref ${key}`);
718		// TODO: remove associated files from target
719		}
720		}
721
722	755	interface UpdateMetadataJob {
723		absPath: string;
	756	path: Path;
724	757	publicPath: string;
725	758	stat: fs.Stats \| null;
726	759	mediaFile: MediaFile \| null;
727	760	}
728	761
729	762	interface ProcessFileArgs {
730		absPath: string;
	763	path: Path;
731	764	stat: fs.Stats;
732	765	mediaFile: MediaFile;
733	766	spin: Spinner;
734	767	}
735	768
736	769	interface ProcessJob {
737		absPath: string;
	770	path: Path;
738	771	stat: fs.Stats;
739	772	mediaFile: MediaFile;
740	773	processor: (typeof processors)[0];
...	...	@@ -743,10 +776,10 @@ interface ProcessJob {
743	776	needs: number;
744	777	}
745	778
746		export function skipBasename(basename: string): boolean {
	779	function skipBasename(basename: string): boolean {
747	780	// dot files must be incrementally tracked
748		if (basename === ".dirsort") return true;
749		if (basename === ".friends") return true;
	781	if (basename === ".dirsort") return false;
	782	if (basename === ".friends") return false;
750	783
751	784	return (
752	785	basename.startsWith(".") \|\|
...	...	@@ -758,13 +791,12 @@ export function skipBasename(basename: string): boolean {
758	791	);
759	792	}
760	793
761		export function toPublicPath(absPath: string) {
762		ASSERT(path.isAbsolute(absPath), "non-absolute " + absPath);
763		if (absPath === root) return "/";
764		return "/" + path.relative(root, absPath).replaceAll("\\", "/");
	794	function toPublicPath(diskPath: Path) {
	795	if (diskPath.toString() === root) return "/";
	796	return "/" + path.relative(root, diskPath.toString()).replaceAll("\\", "/");
765	797	}
766	798
767		export function testProgram(name: string, helpArgument: string) {
	799	function testProgram(name: string, helpArgument: string) {
768	800	try {
769	801	child_process.spawnSync(name, [helpArgument]);
770	802	return name;
...	...	@@ -774,6 +806,118 @@ export function testProgram(name: string, helpArgument: string) {
774	806	return null;
775	807	}
776	808
	809	// Helper function to check and remove location metadata
	810	async function scrubLocationMetadata(
	811	path: Path,
	812	stats: fs.Stats,
	813	): Promise<boolean> {
	814	const ext = path.ext.toLowerCase();
	815	if (!rules.extsScrubExif.has(ext)) return false;
	816
	817	let hasLocation = false;
	818	let args: string[] = [];
	819
	820	// Check for location metadata based on file type
	821	const tempOutput = UNWRAP(path.parent).join(`.tmp.${path.base}`);
	822	switch (ext) {
	823	case ".jpg":
	824	case ".jpeg":
	825	case ".png":
	826	const { stdout: gpsCheck } = await subprocess.exec("exiftool", [
	827	"-gps:all",
	828	path.toString(),
	829	]);
	830	hasLocation = gpsCheck.trim().length > 0;
	831	args = ["-gps:all=", path.toString(), "-o", tempOutput.toString()];
	832	break;
	833	case ".mov":
	834	case ".mp4":
	835	const { stdout: videoCheck } = await subprocess.exec("exiftool", [
	836	"-ee",
	837	"-G3",
	838	"-s",
	839	path.toString(),
	840	]);
	841	hasLocation = videoCheck.includes("GPS") \|\|
	842	videoCheck.includes("Location");
	843	args = [
	844	"-gps:all=",
	845	"-xmp:all=",
	846	path.toString(),
	847	"-o",
	848	tempOutput.toString(),
	849	];
	850	break;
	851	case ".m4a":
	852	const { stdout: m4aCheck } = await subprocess.exec("exiftool", [
	853	"-ee",
	854	"-G3",
	855	"-s",
	856	path.toString(),
	857	]);
	858	hasLocation = m4aCheck.includes("GPS") \|\|
	859	m4aCheck.includes("Location") \|\|
	860	m4aCheck.includes("Filename") \|\|
	861	m4aCheck.includes("Title");
	862
	863	if (hasLocation) {
	864	args = [
	865	"-gps:all=",
	866	"-location:all=",
	867	"-filename:all=",
	868	"-title=",
	869	"-m4a:all=",
	870	path.toString(),
	871	"-o",
	872	tempOutput.toString(),
	873	];
	874	}
	875	break;
	876	}
	877
	878	const accessTime = stats.atime;
	879	const modTime = stats.mtime;
	880
	881	let backup: Path \| null = null;
	882	try {
	883	if (hasLocation) {
	884	// Prepare a backup
	885	const tmp = UNWRAP(path.parent).join(`.tmp.backup.${path.base}`);
	886	await fsp.copyFile(path.toString(), tmp.toString());
	887	await fsp.utimes(tmp.toString(), accessTime, modTime);
	888	backup = tmp;
	889
	890	// Remove metadata
	891	await subprocess.exec("exiftool", args);
	892	if (!tempOutput.ifExistsSync()) {
	893	throw new Error(`Failed to create output file: ${tempOutput}`);
	894	}
	895
	896	// Restore original timestamps
	897	await fsp.rename(tempOutput.toString(), path);
	898	await fsp.utimes(path.toString(), accessTime, modTime);
	899
	900	// Backup is no longer needed
	901	await fsp.unlink(backup.toString());
	902
	903	console.info(
	904	`Scrubbed location metadata in ${path.relative(Path.resolve(root))}`,
	905	);
	906	return true;
	907	}
	908	} catch (error) {
	909	if (backup) {
	910	await fsp.rename(backup.toString(), path.toString());
	911	}
	912	if (fs.existsSync(tempOutput.toString())) {
	913	await fsp.unlink(tempOutput.toString());
	914	}
	915	throw error;
	916	}
	917
	918	return false;
	919	}
	920
777	921	const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
778	922
779	923	import { Progress } from "@paperclover/console/Progress";
...	...	@@ -781,15 +925,16 @@ import { Spinner } from "@paperclover/console/Spinner";
781	925	import * as async from "#sitegen/async";
782	926	import * as fs from "#sitegen/fs";
783	927	import * as subprocess from "#sitegen/subprocess";
	928	import { Path } from "#sitegen/path";
784	929
785		import * as path from "node:path";
786		import * as zlib from "node:zlib";
787	930	import * as child_process from "node:child_process";
788	931	import * as crypto from "node:crypto";
	932	import * as fsp from "node:fs/promises";
	933	import * as path from "node:path";
789	934	import * as stream from "node:stream";
	935	import * as zlib from "node:zlib";
790	936
791	937	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
792		import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
793	938	import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
794	939	import {
795	940	formatDate,
...	...	@@ -800,5 +945,7 @@ import * as rules from "@/file-viewer/rules.ts";
800	945	import * as highlight from "@/file-viewer/highlight.ts";
801	946	import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
802	947	import * as rsync from "@/file-viewer/rsync.ts";
	948	import * as derived from "@/file-viewer/models/derived.ts";
803	949	import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
	950
804	951	import { rawFileRoot as root } from "../paths.ts";

src/file-viewer/ffmpeg.ts+24-4

...	...	@@ -23,7 +23,7 @@ export interface SpawnOptions {
23	23	title: string;
24	24	ffmpeg?: string;
25	25	progress?: Progress;
26		cwd: string;
	26	cwd: Path;
27	27	}
28	28
29	29	export async function spawn(options: SpawnOptions) {
...	...	@@ -31,7 +31,7 @@ export async function spawn(options: SpawnOptions) {
31	31	const proc = child_process.spawn(ffmpeg, [...defaultExtraOptions, ...args], {
32	32	stdio: ["ignore", "inherit", "pipe"],
33	33	env: { ...process.env, SVT_LOG: "2" },
34		cwd,
	34	cwd: cwd.toString(),
35	35	});
36	36	const parser = new Parse();
37	37	const bar = options.progress ?? new Progress({ text: title });
...	...	@@ -66,12 +66,30 @@ export async function spawn(options: SpawnOptions) {
66	66	e.args = [ffmpeg, ...args].join(" ");
67	67	e.code = code;
68	68	e.signal = signal;
69		bar.error(e.message);
70		return e;
	69	bar.stop();
	70	throw e;
71	71	}
72	72	bar.success(title);
73	73	}
74	74
	75	// this is janky
	76	export async function probeAudioStreams(path: Path) {
	77	const { stdout } = await subprocess.spawnAndWait({
	78	cmd: [
	79	"ffprobe",
	80	"-v",
	81	"quiet",
	82	"-select_streams",
	83	"a",
	84	"-show_entries",
	85	"stream=index",
	86	path.toString(),
	87	],
	88	stdio: "pipe",
	89	});
	90	return stdout?.byteLength ?? 0 > 0;
	91	}
	92
75	93	export class Parse {
76	94	parsingStart = true;
77	95	inIndentedIgnore: null \| "out" \| "inp" \| "other" = null;
...	...	@@ -158,8 +176,10 @@ export class Parse {
158	176	}
159	177
160	178	import * as child_process from "node:child_process";
	179	import * as subprocess from "#sitegen/subprocess";
161	180	import * as readline from "node:readline";
162	181	import * as process from "node:process";
163	182	import events from "node:events";
164	183	import * as path from "node:path";
165	184	import { Progress } from "@paperclover/console/Progress";
	185	import type { Path } from "#sitegen/path";

src/file-viewer/models/AssetRef.ts deleted-73

...	...	@@ -1,73 +0,0 @@
1		const db = getDb("cache.sqlite");
2		db.table(
3		"asset_refs",
4		/* SQL */ `
5		create table if not exists asset_refs (
6		id integer primary key autoincrement,
7		key text not null UNIQUE,
8		refs integer not null
9		);
10		create table if not exists asset_ref_files (
11		file text not null,
12		id integer not null,
13		foreign key (id) references asset_refs(id) ON DELETE CASCADE
14		);
15		create index asset_ref_files_id on asset_ref_files(id);
16		`,
17		);
18
19		/**
20		* Uncompressed files are read directly from the media store root. Derivied
21		* assets like compressed files, optimized images, and streamable video are
22		* stored in the `derived` folder. After scanning, the derived assets are
23		* uploaded into the store (storage1/clofi-derived dataset on NAS). Since
24		* multiple files can share the same hash, the number of references is
25		* tracked, and the derived content is only produced once. This means if a
26		* file is deleted, it should only decrement a reference count; deleting it
27		* once all references are removed.
28		*/
29		export class AssetRef {
30		/** Key which aws referenced */
31		id!: number;
32		key!: string;
33		refs!: number;
34
35		unref() {
36		decrementQuery.run(this.key);
37		deleteUnreferencedQuery.run().changes > 0;
38		}
39
40		addFiles(files: string[]) {
41		for (const file of files) {
42		addFileQuery.run({ id: this.id, file });
43		}
44		}
45
46		static get(key: string) {
47		return getQuery.get(key);
48		}
49
50		static putOrIncrement(key: string) {
51		putOrIncrementQuery.get(key);
52		return UNWRAP(AssetRef.get(key));
53		}
54		}
55
56		const getQuery = db.prepare<[key: string]>(/* SQL */ `
57		select * from asset_refs where key = ?;
58		`).as(AssetRef);
59		const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ `
60		insert into asset_refs (key, refs) values (?, 1)
61		on conflict(key) do update set refs = refs + 1;
62		`);
63		const decrementQuery = db.prepare<[key: string]>(/* SQL */ `
64		update asset_refs set refs = refs - 1 where key = ? and refs > 0;
65		`);
66		const deleteUnreferencedQuery = db.prepare(/* SQL */ `
67		delete from asset_refs where refs <= 0;
68		`);
69		const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ `
70		insert into asset_ref_files (id, file) values ($id, $file);
71		`);
72
73		import { getDb } from "#sitegen/sqlite";

src/file-viewer/models/MediaFile.ts+2

...	...	@@ -93,6 +93,8 @@ export class MediaFile {
93	93	const dimensions = this.dimensions;
94	94	if (!dimensions) return null;
95	95	const [width, height] = dimensions.split("x").map(Number);
	96	ASSERT(width);
	97	ASSERT(height);
96	98	return { width, height };
97	99	}
98	100	get basename() {

src/file-viewer/models/derived.ts created+157

...	...	@@ -0,0 +1,157 @@
	1	// Uncompressed files are read directly from the media store root. Derivied
	2	// assets like compressed files, optimized images, and streamable video are
	3	// stored in the `derived` folder. After scanning, the derived assets are
	4	// uploaded into the store (storage1/clofi-derived dataset on NAS). Since
	5	// multiple files can share the same hash, the number of references is
	6	// tracked, and the derived content is only produced once. This means if a
	7	// file is deleted, it should only decrement a reference count; deleting it
	8	// once all references are removed.
	9
	10	const db = getDb("cache.sqlite");
	11	db.table(
	12	"asset_refs",
	13	/* SQL */ `
	14	create table if not exists derived_roots (
	15	id integer primary key autoincrement,
	16	date integer not null, -- milliseconds
	17	key text not null unique
	18	);
	19	create table if not exists derived_files (
	20	id integer primary key autoincrement,
	21	file text not null,
	22	root integer not null,
	23	size integer not null, -- bytes
	24	foreign key(root) references derived_roots(id) on delete cascade
	25	);
	26	create table if not exists derived_refs (
	27	id integer primary key autoincrement,
	28	file integer not null,
	29	root integer not null,
	30	foreign key(root) references derived_roots(id) on delete cascade,
	31	foreign key(file) references media_files(id) on delete cascade,
	32	unique(file, root) on conflict replace
	33	);
	34	create index derived_roots_key on derived_roots(key);
	35	create index derived_files_root on derived_files(root);
	36	create index derived_refs_file on derived_refs(file);
	37	create index derived_refs_root on derived_refs(root);
	38	`,
	39	);
	40
	41	export const workDir = Path.resolve(".clover/derived");
	42
	43	let ongoing = new Map<string, Promise<number>>();
	44
	45	/** produce a derived */
	46	export async function produce(
	47	file: MediaFile,
	48	subkey: string,
	49	producer: (root: Path) => Promise<void>,
	50	) {
	51	const key = `${file.hash}/${subkey}`;
	52	const current = ongoing.get(key);
	53	let root: number \| null = null;
	54	if (current) {
	55	await current;
	56	} else {brk: {
	57	root = getRootQuery.get({ key })?.id ?? null;
	58	if (root) break brk;
	59	const { promise, resolve, reject } = Promise.withResolvers<number>();
	60	ongoing.set(key, promise);
	61	try {
	62	const tmp = workDir.join(key);
	63	await tmp.makeOrEmptyDir();
	64	await producer(tmp);
	65	const filesWithStats = await Promise.all(
	66	(await tmp.readDir())
	67	.filter((x) => !x.base.startsWith("tmp."))
	68	.map(async (x) => [x, await x.stat()] as const),
	69	);
	70	db.node.exec("BEGIN");
	71	try {
	72	root = insertRootQuery.getNonNull({ key, date: Date.now() }).id;
	73	for (const [path, stats] of filesWithStats) {
	74	insertFileQuery.run({
	75	root,
	76	file: path.relative(tmp),
	77	size: stats.size,
	78	});
	79	}
	80	} catch (e) {
	81	db.node.exec("ROLLBACK");
	82	throw e;
	83	}
	84	db.node.exec("COMMIT");
	85	resolve(root);
	86	} catch (e) {
	87	if (root) deleteRootQuery.run({ root });
	88	reject(e);
	89	throw e;
	90	} finally {
	91	ongoing.delete(key);
	92	}
	93	}}
	94	ASSERT(root);
	95	insertRefQuery.run({ root, file: file.id });
	96	return {
	97	[Symbol.dispose]: () => {
	98	deleteRefQuery.run({ root, file: file.id });
	99	},
	100	};
	101	}
	102
	103	const getDerivedAssetQuery = db.prepare<
	104	[number, string],
	105	{ size: number; key: string }
	106	>(/* SQL */ `
	107	select df.size, rt.key
	108	from derived_refs dr
	109	join derived_files df on df.root = dr.root
	110	join derived_roots rt on df.root = rt.id
	111	where dr.file = ? and df.file like ?
	112	`);
	113
	114	export function get(file: MediaFile, subPath: string) {
	115	const row = getDerivedAssetQuery.get(file.id, subPath) ?? null;
	116	if (row) {
	117	return {
	118	path: Path.resolve(derivedFileRoot).join(row.key, subPath),
	119	size: row.size,
	120	};
	121	}
	122	return null;
	123	}
	124
	125	const insertRootQuery = db.prepare<
	126	[{ key: string; date: number }],
	127	{ id: number }
	128	>(
	129	/* SQL */ `insert into derived_roots (key, date) values ($key, $date) returning id;`,
	130	);
	131
	132	const getRootQuery = db.prepare<[{ key: string }], { id: number }>(/* SQL */ `
	133	select id from derived_roots where key = $key;
	134	`);
	135
	136	const insertFileQuery = db.prepare<
	137	[{ file: string; root: number; size: number }]
	138	>(/* SQL */ `
	139	insert into derived_files (file, root, size) values ($file, $root, $size);
	140	`);
	141
	142	const insertRefQuery = db.prepare<[{ root: number; file: number }]>(/* SQL */ `
	143	insert into derived_refs (file, root) values ($file, $root);
	144	`);
	145
	146	const deleteRootQuery = db.prepare<[{ root: number }]>(/* SQL */ `
	147	delete from derived_roots where id = $root;
	148	`);
	149
	150	const deleteRefQuery = db.prepare<[{ root: number; file: number }]>(/* SQL */ `
	151	delete from derived_refs where root = $root and file = $file;
	152	`);
	153
	154	import { getDb } from "#sitegen/sqlite";
	155	import type { MediaFile } from "./MediaFile.ts";
	156	import { Path } from "#sitegen/path";
	157	import { derivedFileRoot } from "../paths.ts";

src/file-viewer/rules.ts+1-1

...	...	@@ -1,7 +1,7 @@
1	1	// -- file extension rules --
2	2
3	3	/** Extensions that must have EXIF/etc data stripped */
4		export const extScrubExif = new Set([
	4	export const extsScrubExif = new Set([
5	5	".jpg",
6	6	".jpeg",
7	7	".png",

src/file-viewer/transcode-rules.ts+205-80

...	...	@@ -1,38 +1,27 @@
1		type VideoEncodePreset = {
	1	interface Av1Preset {
2	2	id: string;
3		codec: "av1";
4	3	preset: 0 \| 1 \| 2 \| 3 \| 4 \| 5 \| 6 \| 7 \| 8;
5	4	mbitMax?: number;
6	5	crf: number;
7	6	maxHeight?: number;
8		audioKbit?: number;
9	7	depth?: 8 \| 10;
10		} \| {
	8	}
	9	interface OpusPreset {
11	10	id: string;
12		codec: "vp9";
13		crf: number;
14		mbitMax?: number;
15		maxHeight?: number;
16		audioKbit?: number;
17		};
	11	audioKbit: number;
	12	}
18	13
19		export const videoFormats = [
	14	export const videoFormats: Av1Preset[] = [
20	15	{
21	16	id: "au", // AV1 Ultra-High
22		codec: "av1",
23	17	preset: 1,
24	18	crf: 28,
25	19	depth: 10,
26		},
27		{
28		id: "vu", // VP9 Ultra-High
29		codec: "vp9",
30		crf: 30,
	20	mbitMax: 30,
31	21	},
32	22	{
33	23	id: "ah", // AV1 High
34	24	preset: 2,
35		codec: "av1",
36	25	mbitMax: 5,
37	26	crf: 35,
38	27	depth: 10,
...	...	@@ -40,49 +29,30 @@ export const videoFormats = [
40	29	{
41	30	id: "am", // AV1 Medium
42	31	preset: 2,
43		codec: "av1",
44	32	mbitMax: 2,
45	33	crf: 40,
46	34	maxHeight: 900,
47	35	},
48		{
49		id: "vm", // VP9 Medium
50		codec: "vp9",
51		crf: 35,
52		mbitMax: 4,
53		maxHeight: 1080,
54		},
55	36	{
56	37	id: "al", // AV1 Low
57	38	preset: 2,
58		codec: "av1",
59	39	mbitMax: 1.25,
60	40	crf: 40,
61	41	maxHeight: 600,
62	42	},
63		{
64		id: "vl", // VP9 Low
65		codec: "vp9",
66		crf: 45,
67		mbitMax: 2,
68		maxHeight: 600,
69		},
70	43	{
71	44	id: "ad", // AV1 Data-saving
72		codec: "av1",
73	45	preset: 1,
74	46	mbitMax: 0.5,
75		crf: 10,
	47	crf: 10, // highest quality clamping to 0.5
76	48	maxHeight: 360,
77	49	},
78		{
79		id: "vl", // VP9 Low
80		codec: "vp9",
81		crf: 10,
82		mbitMax: 0.75,
83		maxHeight: 360,
84		},
85		] satisfies VideoEncodePreset[] as VideoEncodePreset[];
	50	];
	51	export const audioFormats: OpusPreset[] = [
	52	{ id: "h", audioKbit: 192 },
	53	{ id: "m", audioKbit: 128 },
	54	{ id: "d", audioKbit: 64 },
	55	];
86	56
87	57	export const imageSizes = [64, 128, 256, 512, 1024, 2048];
88	58	export const imagePresets = [
...	...	@@ -127,53 +97,208 @@ export const imagePresets = [
127	97	},
128	98	];
129	99
130		export function getVideoArgs(
131		preset: VideoEncodePreset,
132		outbase: string,
133		input: string[],
	100	export async function getVideoInputArgs(path: Path): Promise<InputArgs> {
	101	const config = await path.replaceExt(".json").readIfExists<any>("json");
	102	if (config) {
	103	if (config.encoder && typeof config.encoder.videoSrc === "string") {
	104	const { videoSrc, audioSrc, rate } = config.encoder;
	105	return {
	106	both: [
	107	...(rate ? ["-r", String(rate)] : []),
	108	"-i",
	109	String(videoSrc),
	110	...audioSrc ? ["-i", path.toString()] : [],
	111	],
	112	video: [
	113	...(rate ? ["-r", String(rate)] : []),
	114	"-i",
	115	String(videoSrc),
	116	],
	117	audio: audioSrc
	118	? ["-i", String(audioSrc)]
	119	: (await ffmpeg.probeAudioStreams(path))
	120	? ["-i", path.toString()]
	121	: null,
	122	};
	123	}
	124	}
	125	return {
	126	both: ["-i", path.toString()],
	127	video: ["-i", path.toString()],
	128	audio: (await ffmpeg.probeAudioStreams(path))
	129	? ["-i", path.toString()]
	130	: null,
	131	};
	132	}
	133
	134	export interface InputArgs {
	135	video: string[];
	136	both: string[];
	137	audio: string[] \| null;
	138	}
	139
	140	export const opusFileName = "tmp.opus.webm";
	141	export const av1FileName = "tmp.av1.mp4";
	142
	143	export function getAv1VideoArgs(
	144	av1: Av1Preset,
	145	videoInputArgs: string[],
	146	outDir: Path,
134	147	) {
135		const cmd = [...input];
	148	const { preset, mbitMax, crf, maxHeight, depth } = av1;
	149	const cmd = [...videoInputArgs];
136	150
137		if (preset.codec === "av1") {
138		cmd.push("-c:v", "libsvtav1");
139		cmd.push(
140		"-svtav1-params",
141		[
142		`preset=${preset.preset}`,
143		"keyint=2s",
144		preset.depth && `input-depth=${preset.depth}`, // EncoderBitDepth
145		`crf=${preset.crf}`, // ConstantRateFactor
146		preset.mbitMax && `mbr=${preset.mbitMax}m`, // MaxBitRate
147		"tune=1", // Tune, PSNR
148		"enable-overlays=1", // EnableOverlays
149		"fast-decode=1", // FastDecode
150		"scm=2", // ScreenContentMode, adaptive
151		].filter(Boolean).join(":"),
152		);
153		} else if (preset.codec === "vp9") {
154		// Not much research has gone into this, since it is only going to be used on old Safari browsers.
155		cmd.push("-c:v", "libvpx-vp9");
156		cmd.push("-crf", String(preset.crf));
157		cmd.push("-b:v", preset.mbitMax ? `${preset.mbitMax * 1000}k` : "0");
158		} else preset satisfies never;
159
160		if (preset.maxHeight != null) {
161		cmd.push("-vf", `scale=-2:min(${preset.maxHeight}\\,ih)`);
	151	cmd.push("-c:v", "libsvtav1");
	152	cmd.push(
	153	"-svtav1-params",
	154	[
	155	`preset=${preset}`,
	156	"keyint=2s",
	157	depth && `input-depth=${depth}`, // EncoderBitDepth
	158	`crf=${crf}`, // ConstantRateFactor
	159	mbitMax && `mbr=${mbitMax}m`, // MaxBitRate
	160	"tune=1", // Tune, PSNR
	161	"enable-overlays=1", // EnableOverlays
	162	"fast-decode=1", // FastDecode
	163	"scm=2", // ScreenContentMode, adaptive
	164	].filter(Boolean).join(":"),
	165	);
	166	if (maxHeight != null) {
	167	cmd.push("-vf", `scale=-2:min(${maxHeight}\\,ih)`);
162	168	}
163	169
	170	cmd.push("-y");
	171	cmd.push(outDir.join(av1FileName).toString());
	172
	173	return cmd;
	174	}
	175
	176	export function getOpusAudioArgs(
	177	opus: OpusPreset,
	178	audioInputArgs: string[],
	179	outDir: Path,
	180	) {
	181	const { audioKbit } = opus;
	182	const cmd = [...audioInputArgs];
	183
164	184	cmd.push("-c:a", "libopus");
165		cmd.push("-b:a", (preset.audioKbit ?? 192) + "k");
	185	cmd.push("-b:a", audioKbit + "k");
166	186
167	187	cmd.push("-y");
	188	cmd.push(outDir.join(opusFileName).toString());
	189
	190	return cmd;
	191	}
	192
	193	export function getMpegDashArgs(
	194	videos: Array<Path>,
	195	audios: Array<Path>,
	196	outDir: Path,
	197	) {
	198	const cmd = [];
	199
	200	// inputs
	201	videos.forEach((file) => cmd.push("-i", file.toString()));
	202	audios.forEach((file) => cmd.push("-i", file.toString()));
	203
	204	// map streams
	205	videos.forEach((_, i) => {
	206	cmd.push("-map", `${i}:0`);
	207	});
	208	audios.forEach((_, i) => {
	209	cmd.push("-map", `${videos.length + i}:0`);
	210	});
	211
	212	// copy codecs
	213	cmd.push("-c", "copy");
	214
	215	cmd.push("-f", "dash");
	216	cmd.push("-seg_duration", "4");
	217	cmd.push("-init_seg_name", "d/$RepresentationID$.init.$ext$");
	218	cmd.push("-media_seg_name", "d/$RepresentationID$.$Number$.$ext$");
	219	cmd.push("-use_timeline", "1");
	220	cmd.push("-use_template", "1");
	221
	222	// build adaptation sets string
	223	const videoIndices = videos.map((_, i) => i).join(",");
	224	const audioIndices = audios.map((_, i) => videos.length + i).join(",");
	225	cmd.push(
	226	"-adaptation_sets",
	227	`id=0,streams=${videoIndices} id=1,streams=${audioIndices}`,
	228	);
	229
	230	cmd.push("-y");
	231	cmd.push(outDir.join("dash.mpd").toString());
	232
	233	return cmd;
	234	}
	235
	236	interface HlsPreset {
	237	id: string;
	238	videoBitrate: string;
	239	audioBitrate: string;
	240	maxHeight: number;
	241	}
	242
	243	const hlsPresets: HlsPreset[] = [
	244	{ id: "0", videoBitrate: "3000k", audioBitrate: "192k", maxHeight: 1080 },
	245	{ id: "1", videoBitrate: "1200k", audioBitrate: "128k", maxHeight: 720 },
	246	{ id: "2", videoBitrate: "400k", audioBitrate: "96k", maxHeight: 580 },
	247	];
	248
	249	export function getH264HlsArgs(
	250	inputArgs: InputArgs,
	251	outDir: Path,
	252	) {
	253	const cmd = [...inputArgs.both];
	254
	255	// map streams
	256	hlsPresets.forEach(() => {
	257	cmd.push("-map", "0:v:0");
	258	if (inputArgs.audio) {
	259	cmd.push("-map", "0:a:0");
	260	}
	261	});
	262
	263	// encode each variant
	264	hlsPresets.forEach((preset, i) => {
	265	if (inputArgs.audio) {
	266	const vIdx = i * 2;
	267	const aIdx = i * 2 + 1;
	268	cmd.push(`-c:v:${vIdx}`, "libx264");
	269	cmd.push(`-b:v:${vIdx}`, preset.videoBitrate);
	270	cmd.push(`-c:a:${aIdx}`, "aac");
	271	cmd.push(`-b:a:${aIdx}`, preset.audioBitrate);
	272	cmd.push(`-filter:v:${vIdx}`, `scale=-2:min(${preset.maxHeight}\\,ih)`);
	273	} else {
	274	cmd.push(`-c:v:${i}`, "libx264");
	275	cmd.push(`-b:v:${i}`, preset.videoBitrate);
	276	cmd.push(`-filter:v:${i}`, `scale=-2:min(${preset.maxHeight}\\,ih)`);
	277	}
	278	});
	279
168	280	cmd.push("-f", "hls");
	281	cmd.push("-hls_time", "2");
169	282	cmd.push("-hls_list_size", "0");
170	283	cmd.push("-hls_segment_type", "fmp4");
171		cmd.push("-hls_time", "2");
172	284	cmd.push("-hls_allow_cache", "1");
173		cmd.push("-hls_fmp4_init_filename", preset.id + ".mp4");
174		cmd.push(path.join(outbase, preset.id + ".m3u8"));
	285
	286	if (inputArgs.audio) {
	287	cmd.push(
	288	"-var_stream_map",
	289	hlsPresets.map((_, i) => `v:${i * 2},a:${i * 2 + 1}`).join(" "),
	290	);
	291	} else {
	292	cmd.push("-var_stream_map", hlsPresets.map((_, i) => `v:${i}`).join(" "));
	293	}
	294
	295	cmd.push("-master_pl_name", "master.m3u8");
	296	cmd.push("-hls_segment_filename", "hls.%v.%03d.ts");
	297
	298	cmd.push("-y");
	299	cmd.push(outDir.join("hls.%v.m3u8").toString());
175	300
176	301	return cmd;
177	302	}
178
179		import * as path from "node:path";
	303	import type { Path } from "#sitegen/path";
	304	import * as ffmpeg from "./ffmpeg.ts";