Commit 82d1470abd4b45708f0e4f91a4c05d954a09f1e4 | sitegen

author	clover caruso <git@paperclover.net> 2025-07-08 20:48:30-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:45-07:00
log	82d1470abd4b45708f0e4f91a4c05d954a09f1e4
tree	2f3000e9121b9f9b2fb65e91c2a14deec481c3ab
parent	ee1ec05cc2a55234d1cc51bb285ea247d9144bf8
signature	Commit is signed but in an unrecognized format.

finish scan3

11 files changed, 457 insertions(+), 47 deletions(-)

flake.nix
framework
- lib
readme.md
src
- file-viewer
  - bin
    scan3.ts
  - cotyledon.tsx
  - format.ts
  - models
    MediaFile.ts
  - rsync.ts
  - rules.ts

flake.nix+1

...	...	@@ -21,6 +21,7 @@
21	21	withJxl = true;
22	22	withWebp = true;
23	23	})
	24	pkgs.rsync
24	25	];
25	26	};
26	27	}

framework/lib/fs.ts+27

...	...	@@ -58,6 +58,32 @@ export function readJsonSync<T>(file: string) {
58	58	return JSON.parse(readFileSync(file, "utf-8")) as T;
59	59	}
60	60
	61	export async function removeEmptyDirectories(dir: string, removeRoot = false) {
	62	try {
	63	const entries = await readdir(dir, { withFileTypes: true });
	64	let len = entries.length;
	65	for (const entry of entries) {
	66	if (entry.isDirectory()) {
	67	const subDirPath = path.join(dir, entry.name);
	68	if (await removeEmptyDirectories(subDirPath, true)) len -= 1;
	69	}
	70	}
	71	if (len === 0) {
	72	if (removeRoot) {
	73	await rmdir(dir);
	74	}
	75	return true;
	76	}
	77	} catch (error: any) {
	78	if (error.code === "ENOENT") {
	79	// Directory doesn't exist, ignore
	80	return;
	81	}
	82	throw error;
	83	}
	84	return false;
	85	}
	86
61	87	import * as path from "node:path";
62	88	import {
63	89	createReadStream,
...	...	@@ -76,6 +102,7 @@ import {
76	102	readdir,
77	103	readFile,
78	104	rm,
	105	rmdir,
79	106	stat,
80	107	writeFile,
81	108	} from "node:fs/promises";

framework/lib/markdown.tsx+35-10

...	...	@@ -6,13 +6,14 @@
6	6	* components, and other features are gained for free here.
7	7	*/
8	8	function parse(src: string, options: Partial<ParseOpts> = {}) {
9
10	9	}
11	10
12	11	/* Render markdown content. Same function as 'parse', but JSX components
13	12	* only take one argument and must start with a capital letter. */
14		export function Markdown({ src, ...options }: { src: string } & Partial<ParseOpts>) {
15		return parse(src, options)
	13	export function Markdown(
	14	{ src, ...options }: { src: string } & Partial<ParseOpts>,
	15	) {
	16	return parse(src, options);
16	17	}
17	18
18	19	function parseInline(src: string, options: Partial<InlineOpts> = {}) {
...	...	@@ -20,7 +21,7 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {
20	21	const opts: InlineOpts = { rules, links };
21	22	const parts: engine.Node[] = [];
22	23	const ruleList = Object.values(rules);
23		parse: while(true) {
	24	parse: while (true) {
24	25	for (const rule of ruleList) {
25	26	if (!rule.match) continue;
26	27	const match = src.match(rule.match);
...	...	@@ -29,18 +30,20 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {
29	30	const after = src.slice(index + match[0].length);
30	31	const parse = rule.parse({ after, match: match[0], opts });
31	32	if (!parse) continue;
	33	// parse before
32	34	parts.push(src.slice(0, index), parse.result);
33	35	src = parse.rest ?? after;
34	36	continue parse;
35	37	}
36	38	break;
37		}
	39	}
38	40	parts.push(src);
39	41	return parts;
40	42	}
41	43
42	44	// -- interfaces --
43	45	interface ParseOpts {
	46	gfm: boolean;
44	47	blockRules: Record<string, BlockRule>;
45	48	inlineRules: Record<string, InlineRule>;
46	49	}
...	...	@@ -84,9 +87,9 @@ export const inlineRules: Record<string, InlineRule> = {
84	87	return { result: <code>{inner}</code>, rest };
85	88	},
86	89	},
87		emphasis: {},
88	90	link: {
89	91	match: /(?<!!)\[/,
	92	// 6.3 - links
90	93	parse({ after, opts }) {
91	94	// Match '[' to let the inner-most link win.
92	95	const splitText = splitFirst(after, /[[\]]/);
...	...	@@ -127,7 +130,29 @@ export const inlineRules: Record<string, InlineRule> = {
127	130	};
128	131	},
129	132	},
130		image: {},
	133	image: {
	134	match: /!\[/,
	135	// 6.4 - images
	136	parse({ after, opts }) {
	137	// Match '[' to let the inner-most link win.
	138	const splitText = splitFirst(after, /[[\]]/);
	139	if (!splitText) return null;
	140	if (splitText.delim !== "]") return null;
	141	const { first: textSrc, rest: afterText } = splitText;
	142	},
	143	},
	144	emphasis: {
	145	// detect left-flanking delimiter runs, but this expression does not
	146	// consider preceding escapes. instead, those are programatically
	147	// checked inside the parse function.
	148	match: /(?:\*+\|(?<!\p{P})_+)(?!\s\|\p{P}\|$)/u,
	149	// 6.2 - emphasis and strong emphasis
	150	parse({ before, match, after, opts }) {
	151	// find out how long the delim sequence is
	152
	153	// look for 'ends'
	154	},
	155	},
131	156	autolink: {},
132	157	html: {},
133	158	br: {
...	...	@@ -141,7 +166,7 @@ export const inlineRules: Record<string, InlineRule> = {
141	166	function parseLinkTarget(src: string) {
142	167	let href: string, title: string \| null = null;
143	168	href = src;
144		return { href, title };
	169	return { href, title };
145	170	}
146	171
147	172	/* Find a delimiter while considering backslash escapes. */
...	...	@@ -163,5 +188,5 @@ function splitFirst(text: string, match: RegExp) {
163	188
164	189	console.log(engine.ssrSync(parseInline("meow `bwaa` `` ` `` `` `z``")));
165	190
166		import * as engine from "#ssr";import type { ParseOptions } from "node:querystring";
167
	191	import * as engine from "#ssr";
	192	import type { ParseOptions } from "node:querystring";

framework/lib/sqlite.ts+1-1

...	...	@@ -95,7 +95,7 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
95	95	}
96	96	return item;
97	97	}
98		iter(...args: Args): Iterator<Row> {
	98	iter(...args: Args): IterableIterator<Row> {
99	99	return this.#wrap(args, () => this.array(...args)[Symbol.iterator]());
100	100	}
101	101	/** Get all rows */

readme.md+14-12

...	...	@@ -1,12 +1,12 @@
1	1	# clover sitegen framework
2	2
3	3	this repository contains clover's "sitegen" framework, which is a set of tools
4		that assist building websites. these tools power https://paperclover.net.
	4	that assist building websites. these tools power <https://paperclover.net>.
5	5
6	6	- HTML "Server Side Rendering") engine written from scratch. (~500 lines)
7	7	- A more practical JSX runtime (`class` instead of `className`, built-in
8	8	`clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc).
9		- Integration with [Marko][1] for concisely written components.
	9	- Integration with [Marko] for concisely written components.
10	10	- TODO: MDX-like compiler for content-heavy pages like blogs.
11	11	- Different languages can be used at the same time. Supports `async function`
12	12	components, `<Suspense />`, and custom extensions.
...	...	@@ -27,12 +27,16 @@ that assist building websites. these tools power https://paperclover.net.
27	27	- TODO: Font subsetting tools to reduce bytes downloaded by fonts.
28	28	- Built on the battle-tested Node.js runtime.
29	29
30		[1]: https://next.markojs.com
	30	None of these tools are complex or revolutionary. Rather, this project is the
	31	sum of many years of experience on managing content heavy websites, and an
	32	example on how other over-complicate other frameworks.
	33
	34	[Marko]: https://next.markojs.com
31	35
32	36	Included is `src`, which contains `paperclover.net`. Website highlights:
33	37
34	38	- [Question/Answer board, custom markdown parser and components][q+a].
35		- [File viewer with prefetching + client-side navigation][file].
	39	- [File viewer with fast ui/ux + optimized media streaming][file].
36	40	- [Personal, friends-only blog with password protection][friends].
37	41
38	42	[q+a]: https://paperclover.net/q+a
...	...	@@ -49,16 +53,14 @@ minimum system requirements:
49	53
50	54	my development machine, for example, is Dell Inspiron 7348 with Core i7
51	55
52		```
53		npm install
	56	npm install
54	57
55		# production generation
56		node run generate
57		node .clover/out/server
	58	# production generation
	59	node run generate
	60	node .clover/out/server
58	61
59		# "development" watch mode
60		node run watch
61		```
	62	# "development" watch mode
	63	node run watch
62	64
63	65	<!-- `repl.js` will open a read-eval-print-loop where plugin state is cached (on my -->
64	66	<!-- 2014 dev laptop, startup time is 600-1000ms). every file in `framework` and -->

src/file-viewer/bin/scan3.ts+142-12

...	...	@@ -11,7 +11,7 @@
11	11	// Remember that any software you want to be maintainable and high
12	12	// quality cannot be written with AI.
13	13	const root = path.resolve("/Volumes/clover/Published");
14		const workDir = path.resolve(".clover/file-assets");
	14	const workDir = path.resolve(".clover/derived");
15	15
16	16	export async function main() {
17	17	const start = performance.now();
...	...	@@ -27,7 +27,7 @@ export async function main() {
27	27	// Read a directory or file stat and queue up changed files.
28	28	using qList = new async.Queue({
29	29	name: "Discover Tree",
30		async fn(absPath: string, spin) {
	30	async fn(absPath: string) {
31	31	const stat = await fs.stat(absPath);
32	32
33	33	const publicPath = toPublicPath(absPath);
...	...	@@ -129,7 +129,7 @@ export async function main() {
129	129	await processor.run({ absPath, stat, mediaFile, spin });
130	130	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
131	131	for (const dependantJob of after) {
132		ASSERT(dependantJob.needs > 0);
	132	ASSERT(dependantJob.needs > 0, `dependantJob.needs > 0, ${dependantJob.needs}`);
133	133	dependantJob.needs -= 1;
134	134	if (dependantJob.needs == 0) qProcess.add(dependantJob);
135	135	}
...	...	@@ -152,12 +152,14 @@ export async function main() {
152	152	async function queueProcessors(
153	153	{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
154	154	) {
155		const ext = mediaFile.extension.toLowerCase();
156		let possible = processors.filter((p) => p.include.has(ext));
	155	const ext = mediaFile.extensionNonEmpty.toLowerCase();
	156	let possible = processors.filter((p) =>
	157	p.include ? p.include.has(ext) : !p.exclude?.has(ext)
	158	);
157	159	if (possible.length === 0) return;
158	160
159	161	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
160		ASSERT(hash <= 0xFFFF);
	162	ASSERT(hash <= 0xFFFF, `${hash.toString(16)} has no bits above 16 set`);
161	163	let processed = mediaFile.processed;
162	164
163	165	// If the hash has changed, migrate the bitfield over.
...	...	@@ -213,7 +215,7 @@ export async function main() {
213	215	if (dependJob) {
214	216	dependJob.after.push(job);
215	217	} else {
216		ASSERT(job.needs > 0);
	218	ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
217	219	job.needs -= 1;
218	220	if (job.needs === 0) qProcess.add(job);
219	221	}
...	...	@@ -241,16 +243,114 @@ export async function main() {
241	243	await qMeta.done();
242	244	await qProcess.done();
243	245
	246	// Update directory metadata
	247	const dirs = MediaFile.getDirectoriesToReindex()
	248	.sort((a, b) => b.path.length - a.path.length);
	249	for (const dir of dirs) {
	250	const children = dir.getChildren();
	251
	252	// readme.txt
	253	const readmeContent = children.find((x) =>
	254	x.basename === "readme.txt"
	255	)?.contents ?? "";
	256
	257	// dirsort
	258	let dirsort: string[] \| null = null;
	259	const dirSortRaw =
	260	children.find((x) => x.basename === ".dirsort")?.contents ?? "";
	261	if (dirSortRaw) {
	262	dirsort = dirSortRaw.split("\n")
	263	.map((x) => x.trim())
	264	.filter(Boolean);
	265	}
	266
	267	// Permissions
	268	if (children.some((x) => x.basename === ".friends")) {
	269	FilePermissions.setPermissions(dir.path, 1);
	270	} else {
	271	FilePermissions.setPermissions(dir.path, 0);
	272	}
	273
	274	// Recursive stats.
	275	let totalSize = 0;
	276	let newestDate = new Date(0);
	277	let allHashes = "";
	278	for (const child of children) {
	279	totalSize += child.size;
	280	allHashes += child.hash;
	281
	282	if (child.basename !== "/readme.txt" && child.date > newestDate) {
	283	newestDate = child.date;
	284	}
	285	}
	286
	287	const dirHash = crypto.createHash("sha1")
	288	.update(dir.path + allHashes)
	289	.digest("hex");
	290
	291	MediaFile.markDirectoryProcessed({
	292	id: dir.id,
	293	timestamp: newestDate,
	294	contents: readmeContent,
	295	size: totalSize,
	296	hash: dirHash,
	297	dirsort,
	298	});
	299	}
	300
	301	// Sync to remote
	302	if ((await fs.readdir(workDir)).length > 0) {
	303	await rsync.spawn({
	304	args: [
	305	"--links",
	306	"--recursive",
	307	"--times",
	308	"--partial",
	309	"--progress",
	310	"--remove-source-files",
	311	"--delay-updates",
	312	workDir + "/",
	313	"clo@zenith:/mnt/storage1/clover/Documents/Config/clover_file/derived/",
	314	],
	315	title: "Uploading Derived Assets",
	316	cwd: process.cwd(),
	317	});
	318
	319	await fs.removeEmptyDirectories(workDir);
	320	} else {
	321	console.info("No new derived assets");
	322	}
	323
	324	console.info(
	325	"Updated file viewer index in \x1b[1m" +
	326	((performance.now() - start) / 1000).toFixed(1) + "s\x1b[0m",
	327	);
	328
	329	MediaFile.db.prepare("VACUUM").run();
	330	const { duration, count } = MediaFile.db.prepare<
	331	[],
	332	{ count: number; duration: number }
	333	>(`
	334	select
	335	count(*) as count,
	336	sum(duration) as duration
	337	from media_files
	338	`).getNonNull();
	339
	340	console.info();
244	341	console.info(
245		"Updated file viewer index in " +
246		((performance.now() - start) / 1000).toFixed(1) + "s",
	342	"Global Stats:\n" +
	343	`- File Count: \x1b[1m${count}\x1b[0m\n` +
	344	`- Canonical Size: \x1b[1m${formatSize(MediaFile.getByPath("/")!.size)}\x1b[0m\n` +
	345	`- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n`,
247	346	);
248	347	}
249	348
250	349	interface Process {
251	350	name: string;
252	351	enable?: boolean;
253		include: Set<string>;
	352	include?: Set<string>;
	353	exclude?: Set<string>;
254	354	depends?: string[];
255	355	version?: number;
256	356	/* Perform an action. */
...	...	@@ -350,6 +450,7 @@ const procLoadTextContents: Process = {
350	450	async run({ absPath, mediaFile, stat }) {
351	451	if (stat.size > 1_000_000) return;
352	452	const text = await fs.readFile(absPath, "utf-8");
	453	console.log({ text });
353	454	mediaFile.setContents(text);
354	455	},
355	456	};
...	...	@@ -508,6 +609,26 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
508	609	},
509	610	}));
510	611
	612	const procCompression = [
	613	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
	614	{ name: "zstd", fn: () => zlib.createZstdCompress() },
	615	].map(({ name, fn }) => ({
	616	name: `compress ${name}`,
	617	exclude: rules.extsPreCompressed,
	618	async run({ absPath, mediaFile }) {
	619	if ((mediaFile.size ?? 0) < 10) return;
	620	await produceAsset(`${mediaFile.hash}/${name}`, async (base) => {
	621	fs.mkdirSync(path.dirname(base));
	622	await stream.promises.pipeline(
	623	fs.createReadStream(absPath),
	624	fn(),
	625	fs.createWriteStream(base),
	626	);
	627	return [base];
	628	});
	629	},
	630	} satisfies Process as Process));
	631
511	632	const processors = [
512	633	procDimensions,
513	634	procDuration,
...	...	@@ -515,6 +636,7 @@ const processors = [
515	636	procHighlightCode,
516	637	procImageSubsets,
517	638	...procVideos,
	639	...procCompression,
518	640	]
519	641	.map((process, id, all) => {
520	642	const strIndex = (id: number) =>
...	...	@@ -619,7 +741,7 @@ export function skipBasename(basename: string): boolean {
619	741	}
620	742
621	743	export function toPublicPath(absPath: string) {
622		ASSERT(path.isAbsolute(absPath));
	744	ASSERT(path.isAbsolute(absPath), "non-absolute " + absPath);
623	745	if (absPath === root) return "/";
624	746	return "/" + path.relative(root, absPath).replaceAll("\\", "/");
625	747	}
...	...	@@ -642,14 +764,22 @@ import * as async from "#sitegen/async";
642	764	import * as fs from "#sitegen/fs";
643	765
644	766	import * as path from "node:path";
	767	import * as zlib from "node:zlib";
645	768	import * as child_process from "node:child_process";
646	769	import * as util from "node:util";
647	770	import * as crypto from "node:crypto";
	771	import * as stream from "node:stream";
648	772
649	773	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
650	774	import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
651		import { formatDate } from "@/file-viewer/format.ts";
	775	import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
	776	import {
	777	formatDate,
	778	formatDurationLong,
	779	formatSize,
	780	} from "@/file-viewer/format.ts";
652	781	import * as rules from "@/file-viewer/rules.ts";
653	782	import * as highlight from "@/file-viewer/highlight.ts";
654	783	import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
	784	import * as rsync from "@/file-viewer/rsync.ts";
655	785	import * as transcodeRules from "@/file-viewer/transcode-rules.ts";

src/file-viewer/cotyledon.tsx+9-9

...	...	@@ -70,31 +70,31 @@ export function Speedbump() {
70	70	<div className="enter-container">
71	71	<div className="image-grid">
72	72	<button>
73		<img src="/captcha/image/1.jpeg" alt="a four-leaf clover" />
	73	<img src="/captcha/1.jpeg" alt="a four-leaf clover" />
74	74	</button>
75	75	<button>
76		<img src="/captcha/image/2.jpeg" alt="a four-leaf clover" />
	76	<img src="/captcha/2.jpeg" alt="a four-leaf clover" />
77	77	</button>
78	78	<button>
79		<img src="/captcha/image/3.jpeg" alt="a four-leaf clover" />
	79	<img src="/captcha/3.jpeg" alt="a four-leaf clover" />
80	80	</button>
81	81	<button>
82		<img src="/captcha/image/4.jpeg" alt="a four-leaf clover" />
	82	<img src="/captcha/4.jpeg" alt="a four-leaf clover" />
83	83	</button>
84	84	<button>
85		<img src="/captcha/image/5.jpeg" alt="a four-leaf clover" />
	85	<img src="/captcha/5.jpeg" alt="a four-leaf clover" />
86	86	</button>
87	87	<button>
88		<img src="/captcha/image/6.jpeg" alt="a four-leaf clover" />
	88	<img src="/captcha/6.jpeg" alt="a four-leaf clover" />
89	89	</button>
90	90	<button>
91		<img src="/captcha/image/7.jpeg" alt="a four-leaf clover" />
	91	<img src="/captcha/7.jpeg" alt="a four-leaf clover" />
92	92	</button>
93	93	<button>
94		<img src="/captcha/image/8.jpeg" alt="a four-leaf clover" />
	94	<img src="/captcha/8.jpeg" alt="a four-leaf clover" />
95	95	</button>
96	96	<button>
97		<img src="/captcha/image/9.jpeg" alt="a four-leaf clover" />
	97	<img src="/captcha/9.jpeg" alt="a four-leaf clover" />
98	98	</button>
99	99	</div>
100	100	</div>

src/file-viewer/format.ts+7

...	...	@@ -25,6 +25,13 @@ export function formatDuration(seconds: number) {
25	25	return `${minutes}:${remainingSeconds.toString().padStart(2, "0")}`;
26	26	}
27	27
	28	export function formatDurationLong(seconds: number) {
	29	const hours = Math.floor(seconds / 3600);
	30	const minutes = Math.floor((seconds % 3600) / 60);
	31	const remainingSeconds = seconds % 60;
	32	return `${hours}:${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`;
	33	}
	34
28	35	export function escapeUri(uri: string) {
29	36	return encodeURIComponent(uri)
30	37	.replace(/%2F/gi, "/")

src/file-viewer/models/MediaFile.ts+24

...	...	@@ -104,6 +104,12 @@ export class MediaFile {
104	104	get extension() {
105	105	return path.extname(this.path);
106	106	}
	107	get extensionNonEmpty() {
	108	const { basename } = this;
	109	const ext = path.extname(basename);
	110	if (ext === "") return basename;
	111	return ext;
	112	}
107	113	getChildren() {
108	114	return MediaFile.getChildren(this.id)
109	115	.filter((file) => !file.basename.startsWith("."));
...	...	@@ -267,6 +273,9 @@ export class MediaFile {
267	273	static getChildren(id: number) {
268	274	return getChildrenQuery.array(id);
269	275	}
	276	static getDirectoriesToReindex() {
	277	return getDirectoriesToReindexQuery.array();
	278	}
270	279	static db = db;
271	280	}
272	281
...	...	@@ -430,6 +439,21 @@ const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
430	439	delete from media_files
431	440	where id in (select id from items)
432	441	`);
	442	const getDirectoriesToReindexQuery = db.prepare(`
	443	with recursive directory_chain as (
	444	-- base case
	445	select id, parent_id, path from media_files
	446	where kind = 0 and processed = 0
	447	-- recurse to find all parents so that size/hash can be updated
	448	union
	449	select m.id, m.parent_id, m.path
	450	from media_files m
	451	inner join directory_chain d on m.id = d.parent_id
	452	)
	453	select distinct id, parent_id, path
	454	from directory_chain
	455	order by path;
	456	`).as(MediaFile);
433	457
434	458	import { getDb } from "#sitegen/sqlite";
435	459	import * as path from "node:path/posix";

src/file-viewer/rsync.ts created+184

...	...	@@ -0,0 +1,184 @@
	1	// Utilities for spawning rsync and consuming its output as a `Progress`
	2	// A headless parser is available with `Parse`
	3
	4	export type Line =
	5	\| { kind: "ignore" }
	6	\| { kind: "log"; level: "info" \| "warn" \| "error"; message: string }
	7	\| { kind: "count"; files: number }
	8	\| {
	9	kind: "progress";
	10	currentFile: string;
	11	bytesTransferred: number;
	12	percentage: number;
	13	timeElapsed: string;
	14	transferNumber: number;
	15	filesToCheck: number;
	16	totalFiles: number;
	17	speed: string \| null;
	18	};
	19
	20	export const defaultExtraOptions = [
	21	"--progress",
	22	];
	23
	24	export interface SpawnOptions {
	25	args: string[];
	26	title: string;
	27	rsync?: string;
	28	progress?: Progress;
	29	cwd: string;
	30	}
	31
	32	export async function spawn(options: SpawnOptions) {
	33	const { rsync = "rsync", args, title, cwd } = options;
	34	const proc = child_process.spawn(rsync, [...defaultExtraOptions, ...args], {
	35	stdio: ["ignore", "pipe", "pipe"],
	36	cwd,
	37	});
	38	const parser = new Parse();
	39	const bar = options.progress ?? new Progress({ text: title });
	40	let running = true;
	41
	42	const stdoutSplitter = readline.createInterface({ input: proc.stdout });
	43	const stderrSplitter = readline.createInterface({ input: proc.stderr });
	44
	45	const handleLine = (line: string) => {
	46	const result = parser.onLine(line);
	47	if (result.kind === "ignore") {
	48	return;
	49	} else if (result.kind === "log") {
	50	console[result.level](result.message);
	51	} else if (result.kind === "count") {
	52	if (!running) return;
	53	bar.text = `${result.files} files...`;
	54	} else if (result.kind === "progress") {
	55	if (!running) return;
	56	const {
	57	transferNumber,
	58	bytesTransferred,
	59	totalFiles,
	60	filesToCheck,
	61	currentFile,
	62	speed,
	63	} = result;
	64	bar.value = transferNumber;
	65	bar.total = totalFiles;
	66	const extras = [
	67	formatSize(bytesTransferred),
	68	(totalFiles > filesToCheck)
	69	? `${totalFiles - filesToCheck} unchecked`
	70	: null,
	71	speed,
	72	].filter(Boolean).join(", ");
	73	const fileName = currentFile.length > 20
	74	? `${currentFile.slice(0, 3)}..${currentFile.slice(-15)}`
	75	: currentFile;
	76	bar.text = `[${transferNumber}/${totalFiles}] ${fileName} ${
	77	extras.length > 0 ? `(${extras})` : ""
	78	}`;
	79	} else result satisfies never;
	80	};
	81
	82	stdoutSplitter.on("line", handleLine);
	83	stderrSplitter.on("line", handleLine);
	84
	85	const [code, signal] = await events.once(proc, "close");
	86	running = false;
	87	if (code !== 0) {
	88	const fmt = code ? `code ${code}` : `signal ${signal}`;
	89	const e: any = new Error(`rsync failed with ${fmt}`);
	90	e.args = [rsync, ...args].join(" ");
	91	e.code = code;
	92	e.signal = signal;
	93	bar.error(e.message);
	94	return e;
	95	}
	96	bar.success(title);
	97	}
	98
	99	export class Parse {
	100	totalFiles = 0;
	101	currentTransfer = 0;
	102	toCheck = 0;
	103
	104	onLine(line: string): Line {
	105	line = line.trimEnd();
	106
	107	// Parse progress lines like:
	108	// 20c83c16735608fc3de4aac61e36770d7774e0c6/au26.m4s
	109	// 238,377 100% 460.06kB/s 0:00:00 (xfr#557, to-chk=194111/194690)
	110	const progressMatch = line.match(
	111	/^\s+([\d,]+)\s+(\d+)%\s+(\S+)\s+(?:(\S+)\s+)?(?:$xfr#(\d+), to-chk=(\d+)\/(\d+)$)?/,
	112	);
	113	if (progressMatch) {
	114	const [
	115	,
	116	bytesStr,
	117	percentageStr,
	118	speed,
	119	timeElapsed,
	120	transferStr,
	121	toCheckStr,
	122	totalStr,
	123	] = progressMatch;
	124
	125	this.currentTransfer = Number(transferStr);
	126
	127	return {
	128	kind: "progress",
	129	currentFile: this.lastSeenFile \|\| "",
	130	bytesTransferred: Number(bytesStr.replaceAll(",", "")),
	131	percentage: Number(percentageStr),
	132	timeElapsed,
	133	transferNumber: this.currentTransfer,
	134	filesToCheck: toCheckStr ? this.toCheck = Number(toCheckStr) : this.toCheck,
	135	totalFiles: totalStr ? this.totalFiles = Number(totalStr) : this.totalFiles,
	136	speed: speed \|\| null,
	137	};
	138	}
	139
	140	// Skip common rsync info lines
	141	if (!line.startsWith(" ") && !line.startsWith("rsync")) {
	142	if (
	143	line.startsWith("sending incremental file list") \|\|
	144	line.startsWith("sent ") \|\|
	145	line.startsWith("total size is ") \|\|
	146	line.includes("speedup is ") \|\|
	147	line.startsWith("building file list")
	148	) {
	149	return { kind: "ignore" };
	150	}
	151	if (line.trim().length > 0) {
	152	this.lastSeenFile = line;
	153	}
	154	return { kind: "ignore" };
	155	}
	156	if (line.startsWith(" ")) {
	157	const match = line.match(/ (\d+) files.../);
	158	if (match) {
	159	return { kind: "count", files: Number(match[1]) };
	160	}
	161	}
	162	if (
	163	line.toLowerCase().includes("error") \|\|
	164	line.toLowerCase().includes("failed")
	165	) {
	166	return { kind: "log", level: "error", message: line };
	167	}
	168	if (
	169	line.toLowerCase().includes("warning") \|\|
	170	line.toLowerCase().includes("skipping")
	171	) {
	172	return { kind: "log", level: "warn", message: line };
	173	}
	174	return { kind: "log", level: "info", message: line };
	175	}
	176
	177	private lastSeenFile: string \| null = null;
	178	}
	179
	180	import * as child_process from "node:child_process";
	181	import * as readline from "node:readline";
	182	import events from "node:events";
	183	import { Progress } from "@paperclover/console/Progress";
	184	import { formatSize } from "@/file-viewer/format.ts";

src/file-viewer/rules.ts+13-3

...	...	@@ -73,7 +73,7 @@ export const extsDuration = new Set([...extsAudio, ...extsVideo]);
73	73	export const extsDimensions = new Set([...extsImage, ...extsVideo]);
74	74
75	75	/** These files read file contents into `contents`, as-is */
76		export const extsReadContents = new Set([".txt", ".chat"]);
	76	export const extsReadContents = new Set([".txt", ".chat", ".dirsort"]);
77	77
78	78	export const extsArchive = new Set([
79	79	".zip",
...	...	@@ -92,10 +92,20 @@ export const extsArchive = new Set([
92	92	export const extsPreCompressed = new Set([
93	93	...extsAudio,
94	94	...extsVideo,
95		...extsImage,
	95	...extsImageLike,
96	96	...extsArchive,
97		// TODO: are any of these NOT good for compression
	97	".docx",
	98	".xlsx",
	99	".pptx",
	100	".psd",
	101	".sketch",
	102	".ai",
	103	".3ds",
	104	".fbx",
	105	".blend",
	106	".dng",
98	107	]);
	108	extsPreCompressed.delete(".svg");
99	109
100	110	export function fileIcon(
101	111	file: Pick<MediaFile, "kind" \| "basename" \| "path">,