Commit 82d1470abd4b45708f0e4f91a4c05d954a09f1e4 | sitegen

flake.nix+1

...	@@ -21,6 +21,7 @@	...	@@ -21,6 +21,7 @@
21	withJxl = true;	21	withJxl = true;
22	withWebp = true;	22	withWebp = true;
23	})	23	})
		24	pkgs.rsync
24	];	25	];
25	};	26	};
26	}	27	}

framework/lib/fs.ts+27

...	@@ -58,6 +58,32 @@ export function readJsonSync<T>(file: string) {	...	@@ -58,6 +58,32 @@ export function readJsonSync<T>(file: string) {
58	return JSON.parse(readFileSync(file, "utf-8")) as T;	58	return JSON.parse(readFileSync(file, "utf-8")) as T;
59	}	59	}
60		60
		61	export async function removeEmptyDirectories(dir: string, removeRoot = false) {
		62	try {
		63	const entries = await readdir(dir, { withFileTypes: true });
		64	let len = entries.length;
		65	for (const entry of entries) {
		66	if (entry.isDirectory()) {
		67	const subDirPath = path.join(dir, entry.name);
		68	if (await removeEmptyDirectories(subDirPath, true)) len -= 1;
		69	}
		70	}
		71	if (len === 0) {
		72	if (removeRoot) {
		73	await rmdir(dir);
		74	}
		75	return true;
		76	}
		77	} catch (error: any) {
		78	if (error.code === "ENOENT") {
		79	// Directory doesn't exist, ignore
		80	return;
		81	}
		82	throw error;
		83	}
		84	return false;
		85	}
		86
61	import * as path from "node:path";	87	import * as path from "node:path";
62	import {	88	import {
63	createReadStream,	89	createReadStream,
...	@@ -76,6 +102,7 @@ import {	...	@@ -76,6 +102,7 @@ import {
76	readdir,	102	readdir,
77	readFile,	103	readFile,
78	rm,	104	rm,
		105	rmdir,
79	stat,	106	stat,
80	writeFile,	107	writeFile,
81	} from "node:fs/promises";	108	} from "node:fs/promises";

framework/lib/markdown.tsx+35-10

...	@@ -6,13 +6,14 @@	...	@@ -6,13 +6,14 @@
6	* components, and other features are gained for free here.	6	* components, and other features are gained for free here.
7	*/	7	*/
8	function parse(src: string, options: Partial<ParseOpts> = {}) {	8	function parse(src: string, options: Partial<ParseOpts> = {}) {
9
10	}	9	}
11		10
12	/* Render markdown content. Same function as 'parse', but JSX components	11	/* Render markdown content. Same function as 'parse', but JSX components
13	* only take one argument and must start with a capital letter. */	12	* only take one argument and must start with a capital letter. */
14	export function Markdown({ src, ...options }: { src: string } & Partial<ParseOpts>) {	13	export function Markdown(
15	return parse(src, options)	14	{ src, ...options }: { src: string } & Partial<ParseOpts>,
		15	) {
		16	return parse(src, options);
16	}	17	}
17		18
18	function parseInline(src: string, options: Partial<InlineOpts> = {}) {	19	function parseInline(src: string, options: Partial<InlineOpts> = {}) {
...	@@ -20,7 +21,7 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {	...	@@ -20,7 +21,7 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {
20	const opts: InlineOpts = { rules, links };	21	const opts: InlineOpts = { rules, links };
21	const parts: engine.Node[] = [];	22	const parts: engine.Node[] = [];
22	const ruleList = Object.values(rules);	23	const ruleList = Object.values(rules);
23	parse: while(true) {	24	parse: while (true) {
24	for (const rule of ruleList) {	25	for (const rule of ruleList) {
25	if (!rule.match) continue;	26	if (!rule.match) continue;
26	const match = src.match(rule.match);	27	const match = src.match(rule.match);
...	@@ -29,18 +30,20 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {	...	@@ -29,18 +30,20 @@ function parseInline(src: string, options: Partial<InlineOpts> = {}) {
29	const after = src.slice(index + match[0].length);	30	const after = src.slice(index + match[0].length);
30	const parse = rule.parse({ after, match: match[0], opts });	31	const parse = rule.parse({ after, match: match[0], opts });
31	if (!parse) continue;	32	if (!parse) continue;
		33	// parse before
32	parts.push(src.slice(0, index), parse.result);	34	parts.push(src.slice(0, index), parse.result);
33	src = parse.rest ?? after;	35	src = parse.rest ?? after;
34	continue parse;	36	continue parse;
35	}	37	}
36	break;	38	break;
37	}	39	}
38	parts.push(src);	40	parts.push(src);
39	return parts;	41	return parts;
40	}	42	}
41		43
42	// -- interfaces --	44	// -- interfaces --
43	interface ParseOpts {	45	interface ParseOpts {
		46	gfm: boolean;
44	blockRules: Record<string, BlockRule>;	47	blockRules: Record<string, BlockRule>;
45	inlineRules: Record<string, InlineRule>;	48	inlineRules: Record<string, InlineRule>;
46	}	49	}
...	@@ -84,9 +87,9 @@ export const inlineRules: Record<string, InlineRule> = {	...	@@ -84,9 +87,9 @@ export const inlineRules: Record<string, InlineRule> = {
84	return { result: <code>{inner}</code>, rest };	87	return { result: <code>{inner}</code>, rest };
85	},	88	},
86	},	89	},
87	emphasis: {},
88	link: {	90	link: {
89	match: /(?<!!)\[/,	91	match: /(?<!!)\[/,
		92	// 6.3 - links
90	parse({ after, opts }) {	93	parse({ after, opts }) {
91	// Match '[' to let the inner-most link win.	94	// Match '[' to let the inner-most link win.
92	const splitText = splitFirst(after, /[[\]]/);	95	const splitText = splitFirst(after, /[[\]]/);
...	@@ -127,7 +130,29 @@ export const inlineRules: Record<string, InlineRule> = {	...	@@ -127,7 +130,29 @@ export const inlineRules: Record<string, InlineRule> = {
127	};	130	};
128	},	131	},
129	},	132	},
130	image: {},	133	image: {
		134	match: /!\[/,
		135	// 6.4 - images
		136	parse({ after, opts }) {
		137	// Match '[' to let the inner-most link win.
		138	const splitText = splitFirst(after, /[[\]]/);
		139	if (!splitText) return null;
		140	if (splitText.delim !== "]") return null;
		141	const { first: textSrc, rest: afterText } = splitText;
		142	},
		143	},
		144	emphasis: {
		145	// detect left-flanking delimiter runs, but this expression does not
		146	// consider preceding escapes. instead, those are programatically
		147	// checked inside the parse function.
		148	match: /(?:\*+\|(?<!\p{P})_+)(?!\s\|\p{P}\|$)/u,
		149	// 6.2 - emphasis and strong emphasis
		150	parse({ before, match, after, opts }) {
		151	// find out how long the delim sequence is
		152
		153	// look for 'ends'
		154	},
		155	},
131	autolink: {},	156	autolink: {},
132	html: {},	157	html: {},
133	br: {	158	br: {
...	@@ -141,7 +166,7 @@ export const inlineRules: Record<string, InlineRule> = {	...	@@ -141,7 +166,7 @@ export const inlineRules: Record<string, InlineRule> = {
141	function parseLinkTarget(src: string) {	166	function parseLinkTarget(src: string) {
142	let href: string, title: string \| null = null;	167	let href: string, title: string \| null = null;
143	href = src;	168	href = src;
144	return { href, title };	169	return { href, title };
145	}	170	}
146		171
147	/* Find a delimiter while considering backslash escapes. */	172	/* Find a delimiter while considering backslash escapes. */
...	@@ -163,5 +188,5 @@ function splitFirst(text: string, match: RegExp) {	...	@@ -163,5 +188,5 @@ function splitFirst(text: string, match: RegExp) {
163		188
164	console.log(engine.ssrSync(parseInline("meow `bwaa` `` ` `` `` `z``")));	189	console.log(engine.ssrSync(parseInline("meow `bwaa` `` ` `` `` `z``")));
165		190
166	import * as engine from "#ssr";import type { ParseOptions } from "node:querystring";	191	import * as engine from "#ssr";
167		192	import type { ParseOptions } from "node:querystring";

framework/lib/sqlite.ts+1-1

...	@@ -95,7 +95,7 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {	...	@@ -95,7 +95,7 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
95	}	95	}
96	return item;	96	return item;
97	}	97	}
98	iter(...args: Args): Iterator<Row> {	98	iter(...args: Args): IterableIterator<Row> {
99	return this.#wrap(args, () => this.array(...args)[Symbol.iterator]());	99	return this.#wrap(args, () => this.array(...args)[Symbol.iterator]());
100	}	100	}
101	/** Get all rows */	101	/** Get all rows */

readme.md+14-12

 @@ -1,12 +1,12 @@
 # clover sitegen framework
 this repository contains clover's "sitegen" framework, which is a set of tools
-that assist building websites. these tools power https://paperclover.net.
+that assist building websites. these tools power <https://paperclover.net>.
 - **HTML "Server Side Rendering") engine written from scratch.** (~500 lines)
   - A more practical JSX runtime (`class` instead of `className`, built-in
     `clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc).
-  - Integration with [Marko][1] for concisely written components.
+  - Integration with [Marko] for concisely written components.
   - TODO: MDX-like compiler for content-heavy pages like blogs.
   - Different languages can be used at the same time. Supports `async function`
     components, `<Suspense />`, and custom extensions.
 @@ -27,12 +27,16 @@ that assist building websites. these tools power https://paperclover.net.
   - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
 - **Built on the battle-tested Node.js runtime.**
-[1]: https://next.markojs.com
+None of these tools are complex or revolutionary. Rather, this project is the
+sum of many years of experience on managing content heavy websites, and an
+example on how other over-complicate other frameworks.
+[Marko]: https://next.markojs.com
 Included is `src`, which contains `paperclover.net`. Website highlights:
 - [Question/Answer board, custom markdown parser and components][q+a].
-- [File viewer with prefetching + client-side navigation][file].
+- [File viewer with fast ui/ux + optimized media streaming][file].
 - [Personal, friends-only blog with password protection][friends].
 [q+a]: https://paperclover.net/q+a
 @@ -49,16 +53,14 @@ minimum system requirements:
 my development machine, for example, is Dell Inspiron 7348 with Core i7
-```
+    npm install
-npm install
-# production generation
+    # production generation
-node run generate
+    node run generate
-node .clover/out/server
+    node .clover/out/server
-# "development" watch mode
+    # "development" watch mode
-node run watch
+    node run watch
-```
 <!-- `repl.js` will open a read-eval-print-loop where plugin state is cached (on my -->
 <!-- 2014 dev laptop, startup time is 600-1000ms). every file in `framework` and -->

src/file-viewer/bin/scan3.ts+142-12

...	@@ -11,7 +11,7 @@	...	@@ -11,7 +11,7 @@
11	// Remember that any software you want to be maintainable and high	11	// Remember that any software you want to be maintainable and high
12	// quality cannot be written with AI.	12	// quality cannot be written with AI.
13	const root = path.resolve("/Volumes/clover/Published");	13	const root = path.resolve("/Volumes/clover/Published");
14	const workDir = path.resolve(".clover/file-assets");	14	const workDir = path.resolve(".clover/derived");
15		15
16	export async function main() {	16	export async function main() {
17	const start = performance.now();	17	const start = performance.now();
...	@@ -27,7 +27,7 @@ export async function main() {	...	@@ -27,7 +27,7 @@ export async function main() {
27	// Read a directory or file stat and queue up changed files.	27	// Read a directory or file stat and queue up changed files.
28	using qList = new async.Queue({	28	using qList = new async.Queue({
29	name: "Discover Tree",	29	name: "Discover Tree",
30	async fn(absPath: string, spin) {	30	async fn(absPath: string) {
31	const stat = await fs.stat(absPath);	31	const stat = await fs.stat(absPath);
32		32
33	const publicPath = toPublicPath(absPath);	33	const publicPath = toPublicPath(absPath);
...	@@ -129,7 +129,7 @@ export async function main() {	...	@@ -129,7 +129,7 @@ export async function main() {
129	await processor.run({ absPath, stat, mediaFile, spin });	129	await processor.run({ absPath, stat, mediaFile, spin });
130	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));	130	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
131	for (const dependantJob of after) {	131	for (const dependantJob of after) {
132	ASSERT(dependantJob.needs > 0);	132	ASSERT(dependantJob.needs > 0, `dependantJob.needs > 0, ${dependantJob.needs}`);
133	dependantJob.needs -= 1;	133	dependantJob.needs -= 1;
134	if (dependantJob.needs == 0) qProcess.add(dependantJob);	134	if (dependantJob.needs == 0) qProcess.add(dependantJob);
135	}	135	}
...	@@ -152,12 +152,14 @@ export async function main() {	...	@@ -152,12 +152,14 @@ export async function main() {
152	async function queueProcessors(	152	async function queueProcessors(
153	{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,	153	{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
154	) {	154	) {
155	const ext = mediaFile.extension.toLowerCase();	155	const ext = mediaFile.extensionNonEmpty.toLowerCase();
156	let possible = processors.filter((p) => p.include.has(ext));	156	let possible = processors.filter((p) =>
		157	p.include ? p.include.has(ext) : !p.exclude?.has(ext)
		158	);
157	if (possible.length === 0) return;	159	if (possible.length === 0) return;
158		160
159	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;	161	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
160	ASSERT(hash <= 0xFFFF);	162	ASSERT(hash <= 0xFFFF, `${hash.toString(16)} has no bits above 16 set`);
161	let processed = mediaFile.processed;	163	let processed = mediaFile.processed;
162		164
163	// If the hash has changed, migrate the bitfield over.	165	// If the hash has changed, migrate the bitfield over.
...	@@ -213,7 +215,7 @@ export async function main() {	...	@@ -213,7 +215,7 @@ export async function main() {
213	if (dependJob) {	215	if (dependJob) {
214	dependJob.after.push(job);	216	dependJob.after.push(job);
215	} else {	217	} else {
216	ASSERT(job.needs > 0);	218	ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
217	job.needs -= 1;	219	job.needs -= 1;
218	if (job.needs === 0) qProcess.add(job);	220	if (job.needs === 0) qProcess.add(job);
219	}	221	}
...	@@ -241,16 +243,114 @@ export async function main() {	...	@@ -241,16 +243,114 @@ export async function main() {
241	await qMeta.done();	243	await qMeta.done();
242	await qProcess.done();	244	await qProcess.done();
243		245
		246	// Update directory metadata
		247	const dirs = MediaFile.getDirectoriesToReindex()
		248	.sort((a, b) => b.path.length - a.path.length);
		249	for (const dir of dirs) {
		250	const children = dir.getChildren();
		251
		252	// readme.txt
		253	const readmeContent = children.find((x) =>
		254	x.basename === "readme.txt"
		255	)?.contents ?? "";
		256
		257	// dirsort
		258	let dirsort: string[] \| null = null;
		259	const dirSortRaw =
		260	children.find((x) => x.basename === ".dirsort")?.contents ?? "";
		261	if (dirSortRaw) {
		262	dirsort = dirSortRaw.split("\n")
		263	.map((x) => x.trim())
		264	.filter(Boolean);
		265	}
		266
		267	// Permissions
		268	if (children.some((x) => x.basename === ".friends")) {
		269	FilePermissions.setPermissions(dir.path, 1);
		270	} else {
		271	FilePermissions.setPermissions(dir.path, 0);
		272	}
		273
		274	// Recursive stats.
		275	let totalSize = 0;
		276	let newestDate = new Date(0);
		277	let allHashes = "";
		278	for (const child of children) {
		279	totalSize += child.size;
		280	allHashes += child.hash;
		281
		282	if (child.basename !== "/readme.txt" && child.date > newestDate) {
		283	newestDate = child.date;
		284	}
		285	}
		286
		287	const dirHash = crypto.createHash("sha1")
		288	.update(dir.path + allHashes)
		289	.digest("hex");
		290
		291	MediaFile.markDirectoryProcessed({
		292	id: dir.id,
		293	timestamp: newestDate,
		294	contents: readmeContent,
		295	size: totalSize,
		296	hash: dirHash,
		297	dirsort,
		298	});
		299	}
		300
		301	// Sync to remote
		302	if ((await fs.readdir(workDir)).length > 0) {
		303	await rsync.spawn({
		304	args: [
		305	"--links",
		306	"--recursive",
		307	"--times",
		308	"--partial",
		309	"--progress",
		310	"--remove-source-files",
		311	"--delay-updates",
		312	workDir + "/",
		313	"clo@zenith:/mnt/storage1/clover/Documents/Config/clover_file/derived/",
		314	],
		315	title: "Uploading Derived Assets",
		316	cwd: process.cwd(),
		317	});
		318
		319	await fs.removeEmptyDirectories(workDir);
		320	} else {
		321	console.info("No new derived assets");
		322	}
		323
		324	console.info(
		325	"Updated file viewer index in \x1b[1m" +
		326	((performance.now() - start) / 1000).toFixed(1) + "s\x1b[0m",
		327	);
		328
		329	MediaFile.db.prepare("VACUUM").run();
		330	const { duration, count } = MediaFile.db.prepare<
		331	[],
		332	{ count: number; duration: number }
		333	>(`
		334	select
		335	count(*) as count,
		336	sum(duration) as duration
		337	from media_files
		338	`).getNonNull();
		339
		340	console.info();
244	console.info(	341	console.info(
245	"Updated file viewer index in " +	342	"Global Stats:\n" +
246	((performance.now() - start) / 1000).toFixed(1) + "s",	343	`- File Count: \x1b[1m${count}\x1b[0m\n` +
		344	`- Canonical Size: \x1b[1m${formatSize(MediaFile.getByPath("/")!.size)}\x1b[0m\n` +
		345	`- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n`,
247	);	346	);
248	}	347	}
249		348
250	interface Process {	349	interface Process {
251	name: string;	350	name: string;
252	enable?: boolean;	351	enable?: boolean;
253	include: Set<string>;	352	include?: Set<string>;
		353	exclude?: Set<string>;
254	depends?: string[];	354	depends?: string[];
255	version?: number;	355	version?: number;
256	/* Perform an action. */	356	/* Perform an action. */
...	@@ -350,6 +450,7 @@ const procLoadTextContents: Process = {	...	@@ -350,6 +450,7 @@ const procLoadTextContents: Process = {
350	async run({ absPath, mediaFile, stat }) {	450	async run({ absPath, mediaFile, stat }) {
351	if (stat.size > 1_000_000) return;	451	if (stat.size > 1_000_000) return;
352	const text = await fs.readFile(absPath, "utf-8");	452	const text = await fs.readFile(absPath, "utf-8");
		453	console.log({ text });
353	mediaFile.setContents(text);	454	mediaFile.setContents(text);
354	},	455	},
355	};	456	};
...	@@ -508,6 +609,26 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({	...	@@ -508,6 +609,26 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
508	},	609	},
509	}));	610	}));
510		611
		612	const procCompression = [
		613	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
		614	{ name: "zstd", fn: () => zlib.createZstdCompress() },
		615	].map(({ name, fn }) => ({
		616	name: `compress ${name}`,
		617	exclude: rules.extsPreCompressed,
		618	async run({ absPath, mediaFile }) {
		619	if ((mediaFile.size ?? 0) < 10) return;
		620	await produceAsset(`${mediaFile.hash}/${name}`, async (base) => {
		621	fs.mkdirSync(path.dirname(base));
		622	await stream.promises.pipeline(
		623	fs.createReadStream(absPath),
		624	fn(),
		625	fs.createWriteStream(base),
		626	);
		627	return [base];
		628	});
		629	},
		630	} satisfies Process as Process));
		631
511	const processors = [	632	const processors = [
512	procDimensions,	633	procDimensions,
513	procDuration,	634	procDuration,
...	@@ -515,6 +636,7 @@ const processors = [	...	@@ -515,6 +636,7 @@ const processors = [
515	procHighlightCode,	636	procHighlightCode,
516	procImageSubsets,	637	procImageSubsets,
517	...procVideos,	638	...procVideos,
		639	...procCompression,
518	]	640	]
519	.map((process, id, all) => {	641	.map((process, id, all) => {
520	const strIndex = (id: number) =>	642	const strIndex = (id: number) =>
...	@@ -619,7 +741,7 @@ export function skipBasename(basename: string): boolean {	...	@@ -619,7 +741,7 @@ export function skipBasename(basename: string): boolean {
619	}	741	}
620		742
621	export function toPublicPath(absPath: string) {	743	export function toPublicPath(absPath: string) {
622	ASSERT(path.isAbsolute(absPath));	744	ASSERT(path.isAbsolute(absPath), "non-absolute " + absPath);
623	if (absPath === root) return "/";	745	if (absPath === root) return "/";
624	return "/" + path.relative(root, absPath).replaceAll("\\", "/");	746	return "/" + path.relative(root, absPath).replaceAll("\\", "/");
625	}	747	}
...	@@ -642,14 +764,22 @@ import * as async from "#sitegen/async";	...	@@ -642,14 +764,22 @@ import * as async from "#sitegen/async";
642	import * as fs from "#sitegen/fs";	764	import * as fs from "#sitegen/fs";
643		765
644	import * as path from "node:path";	766	import * as path from "node:path";
		767	import * as zlib from "node:zlib";
645	import * as child_process from "node:child_process";	768	import * as child_process from "node:child_process";
646	import * as util from "node:util";	769	import * as util from "node:util";
647	import * as crypto from "node:crypto";	770	import * as crypto from "node:crypto";
		771	import * as stream from "node:stream";
648		772
649	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";	773	import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
650	import { AssetRef } from "@/file-viewer/models/AssetRef.ts";	774	import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
651	import { formatDate } from "@/file-viewer/format.ts";	775	import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
		776	import {
		777	formatDate,
		778	formatDurationLong,
		779	formatSize,
		780	} from "@/file-viewer/format.ts";
652	import * as rules from "@/file-viewer/rules.ts";	781	import * as rules from "@/file-viewer/rules.ts";
653	import * as highlight from "@/file-viewer/highlight.ts";	782	import * as highlight from "@/file-viewer/highlight.ts";
654	import * as ffmpeg from "@/file-viewer/ffmpeg.ts";	783	import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
		784	import * as rsync from "@/file-viewer/rsync.ts";
655	import * as transcodeRules from "@/file-viewer/transcode-rules.ts";	785	import * as transcodeRules from "@/file-viewer/transcode-rules.ts";

src/file-viewer/cotyledon.tsx+9-9

...	@@ -70,31 +70,31 @@ export function Speedbump() {	...	@@ -70,31 +70,31 @@ export function Speedbump() {
70	<div className="enter-container">	70	<div className="enter-container">
71	<div className="image-grid">	71	<div className="image-grid">
72	<button>	72	<button>
73	<img src="/captcha/image/1.jpeg" alt="a four-leaf clover" />	73	<img src="/captcha/1.jpeg" alt="a four-leaf clover" />
74	</button>	74	</button>
75	<button>	75	<button>
76	<img src="/captcha/image/2.jpeg" alt="a four-leaf clover" />	76	<img src="/captcha/2.jpeg" alt="a four-leaf clover" />
77	</button>	77	</button>
78	<button>	78	<button>
79	<img src="/captcha/image/3.jpeg" alt="a four-leaf clover" />	79	<img src="/captcha/3.jpeg" alt="a four-leaf clover" />
80	</button>	80	</button>
81	<button>	81	<button>
82	<img src="/captcha/image/4.jpeg" alt="a four-leaf clover" />	82	<img src="/captcha/4.jpeg" alt="a four-leaf clover" />
83	</button>	83	</button>
84	<button>	84	<button>
85	<img src="/captcha/image/5.jpeg" alt="a four-leaf clover" />	85	<img src="/captcha/5.jpeg" alt="a four-leaf clover" />
86	</button>	86	</button>
87	<button>	87	<button>
88	<img src="/captcha/image/6.jpeg" alt="a four-leaf clover" />	88	<img src="/captcha/6.jpeg" alt="a four-leaf clover" />
89	</button>	89	</button>
90	<button>	90	<button>
91	<img src="/captcha/image/7.jpeg" alt="a four-leaf clover" />	91	<img src="/captcha/7.jpeg" alt="a four-leaf clover" />
92	</button>	92	</button>
93	<button>	93	<button>
94	<img src="/captcha/image/8.jpeg" alt="a four-leaf clover" />	94	<img src="/captcha/8.jpeg" alt="a four-leaf clover" />
95	</button>	95	</button>
96	<button>	96	<button>
97	<img src="/captcha/image/9.jpeg" alt="a four-leaf clover" />	97	<img src="/captcha/9.jpeg" alt="a four-leaf clover" />
98	</button>	98	</button>
99	</div>	99	</div>
100	</div>	100	</div>

src/file-viewer/format.ts+7

...	@@ -25,6 +25,13 @@ export function formatDuration(seconds: number) {	...	@@ -25,6 +25,13 @@ export function formatDuration(seconds: number) {
25	return `${minutes}:${remainingSeconds.toString().padStart(2, "0")}`;	25	return `${minutes}:${remainingSeconds.toString().padStart(2, "0")}`;
26	}	26	}
27		27
		28	export function formatDurationLong(seconds: number) {
		29	const hours = Math.floor(seconds / 3600);
		30	const minutes = Math.floor((seconds % 3600) / 60);
		31	const remainingSeconds = seconds % 60;
		32	return `${hours}:${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`;
		33	}
		34
28	export function escapeUri(uri: string) {	35	export function escapeUri(uri: string) {
29	return encodeURIComponent(uri)	36	return encodeURIComponent(uri)
30	.replace(/%2F/gi, "/")	37	.replace(/%2F/gi, "/")

src/file-viewer/models/MediaFile.ts+24

...	@@ -104,6 +104,12 @@ export class MediaFile {	...	@@ -104,6 +104,12 @@ export class MediaFile {
104	get extension() {	104	get extension() {
105	return path.extname(this.path);	105	return path.extname(this.path);
106	}	106	}
		107	get extensionNonEmpty() {
		108	const { basename } = this;
		109	const ext = path.extname(basename);
		110	if (ext === "") return basename;
		111	return ext;
		112	}
107	getChildren() {	113	getChildren() {
108	return MediaFile.getChildren(this.id)	114	return MediaFile.getChildren(this.id)
109	.filter((file) => !file.basename.startsWith("."));	115	.filter((file) => !file.basename.startsWith("."));
...	@@ -267,6 +273,9 @@ export class MediaFile {	...	@@ -267,6 +273,9 @@ export class MediaFile {
267	static getChildren(id: number) {	273	static getChildren(id: number) {
268	return getChildrenQuery.array(id);	274	return getChildrenQuery.array(id);
269	}	275	}
		276	static getDirectoriesToReindex() {
		277	return getDirectoriesToReindexQuery.array();
		278	}
270	static db = db;	279	static db = db;
271	}	280	}
272		281
...	@@ -430,6 +439,21 @@ const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `	...	@@ -430,6 +439,21 @@ const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
430	delete from media_files	439	delete from media_files
431	where id in (select id from items)	440	where id in (select id from items)
432	`);	441	`);
		442	const getDirectoriesToReindexQuery = db.prepare(`
		443	with recursive directory_chain as (
		444	-- base case
		445	select id, parent_id, path from media_files
		446	where kind = 0 and processed = 0
		447	-- recurse to find all parents so that size/hash can be updated
		448	union
		449	select m.id, m.parent_id, m.path
		450	from media_files m
		451	inner join directory_chain d on m.id = d.parent_id
		452	)
		453	select distinct id, parent_id, path
		454	from directory_chain
		455	order by path;
		456	`).as(MediaFile);
433		457
434	import { getDb } from "#sitegen/sqlite";	458	import { getDb } from "#sitegen/sqlite";
435	import * as path from "node:path/posix";	459	import * as path from "node:path/posix";

src/file-viewer/rsync.ts created+184

...	@@ -0,0 +1,184 @@
1	// Utilities for spawning rsync and consuming its output as a `Progress`
2	// A headless parser is available with `Parse`
3
4	export type Line =
5	\| { kind: "ignore" }
6	\| { kind: "log"; level: "info" \| "warn" \| "error"; message: string }
7	\| { kind: "count"; files: number }
8	\| {
9	kind: "progress";
10	currentFile: string;
11	bytesTransferred: number;
12	percentage: number;
13	timeElapsed: string;
14	transferNumber: number;
15	filesToCheck: number;
16	totalFiles: number;
17	speed: string \| null;
18	};
19
20	export const defaultExtraOptions = [
21	"--progress",
22	];
23
24	export interface SpawnOptions {
25	args: string[];
26	title: string;
27	rsync?: string;
28	progress?: Progress;
29	cwd: string;
30	}
31
32	export async function spawn(options: SpawnOptions) {
33	const { rsync = "rsync", args, title, cwd } = options;
34	const proc = child_process.spawn(rsync, [...defaultExtraOptions, ...args], {
35	stdio: ["ignore", "pipe", "pipe"],
36	cwd,
37	});
38	const parser = new Parse();
39	const bar = options.progress ?? new Progress({ text: title });
40	let running = true;
41
42	const stdoutSplitter = readline.createInterface({ input: proc.stdout });
43	const stderrSplitter = readline.createInterface({ input: proc.stderr });
44
45	const handleLine = (line: string) => {
46	const result = parser.onLine(line);
47	if (result.kind === "ignore") {
48	return;
49	} else if (result.kind === "log") {
50	console[result.level](result.message);
51	} else if (result.kind === "count") {
52	if (!running) return;
53	bar.text = `${result.files} files...`;
54	} else if (result.kind === "progress") {
55	if (!running) return;
56	const {
57	transferNumber,
58	bytesTransferred,
59	totalFiles,
60	filesToCheck,
61	currentFile,
62	speed,
63	} = result;
64	bar.value = transferNumber;
65	bar.total = totalFiles;
66	const extras = [
67	formatSize(bytesTransferred),
68	(totalFiles > filesToCheck)
69	? `${totalFiles - filesToCheck} unchecked`
70	: null,
71	speed,
72	].filter(Boolean).join(", ");
73	const fileName = currentFile.length > 20
74	? `${currentFile.slice(0, 3)}..${currentFile.slice(-15)}`
75	: currentFile;
76	bar.text = `[${transferNumber}/${totalFiles}] ${fileName} ${
77	extras.length > 0 ? `(${extras})` : ""
78	}`;
79	} else result satisfies never;
80	};
81
82	stdoutSplitter.on("line", handleLine);
83	stderrSplitter.on("line", handleLine);
84
85	const [code, signal] = await events.once(proc, "close");
86	running = false;
87	if (code !== 0) {
88	const fmt = code ? `code ${code}` : `signal ${signal}`;
89	const e: any = new Error(`rsync failed with ${fmt}`);
90	e.args = [rsync, ...args].join(" ");
91	e.code = code;
92	e.signal = signal;
93	bar.error(e.message);
94	return e;
95	}
96	bar.success(title);
97	}
98
99	export class Parse {
100	totalFiles = 0;
101	currentTransfer = 0;
102	toCheck = 0;
103
104	onLine(line: string): Line {
105	line = line.trimEnd();
106
107	// Parse progress lines like:
108	// 20c83c16735608fc3de4aac61e36770d7774e0c6/au26.m4s
109	// 238,377 100% 460.06kB/s 0:00:00 (xfr#557, to-chk=194111/194690)
110	const progressMatch = line.match(
111	/^\s+([\d,]+)\s+(\d+)%\s+(\S+)\s+(?:(\S+)\s+)?(?:$xfr#(\d+), to-chk=(\d+)\/(\d+)$)?/,
112	);
113	if (progressMatch) {
114	const [
115	,
116	bytesStr,
117	percentageStr,
118	speed,
119	timeElapsed,
120	transferStr,
121	toCheckStr,
122	totalStr,
123	] = progressMatch;
124
125	this.currentTransfer = Number(transferStr);
126
127	return {
128	kind: "progress",
129	currentFile: this.lastSeenFile \|\| "",
130	bytesTransferred: Number(bytesStr.replaceAll(",", "")),
131	percentage: Number(percentageStr),
132	timeElapsed,
133	transferNumber: this.currentTransfer,
134	filesToCheck: toCheckStr ? this.toCheck = Number(toCheckStr) : this.toCheck,
135	totalFiles: totalStr ? this.totalFiles = Number(totalStr) : this.totalFiles,
136	speed: speed \|\| null,
137	};
138	}
139
140	// Skip common rsync info lines
141	if (!line.startsWith(" ") && !line.startsWith("rsync")) {
142	if (
143	line.startsWith("sending incremental file list") \|\|
144	line.startsWith("sent ") \|\|
145	line.startsWith("total size is ") \|\|
146	line.includes("speedup is ") \|\|
147	line.startsWith("building file list")
148	) {
149	return { kind: "ignore" };
150	}
151	if (line.trim().length > 0) {
152	this.lastSeenFile = line;
153	}
154	return { kind: "ignore" };
155	}
156	if (line.startsWith(" ")) {
157	const match = line.match(/ (\d+) files.../);
158	if (match) {
159	return { kind: "count", files: Number(match[1]) };
160	}
161	}
162	if (
163	line.toLowerCase().includes("error") \|\|
164	line.toLowerCase().includes("failed")
165	) {
166	return { kind: "log", level: "error", message: line };
167	}
168	if (
169	line.toLowerCase().includes("warning") \|\|
170	line.toLowerCase().includes("skipping")
171	) {
172	return { kind: "log", level: "warn", message: line };
173	}
174	return { kind: "log", level: "info", message: line };
175	}
176
177	private lastSeenFile: string \| null = null;
178	}
179
180	import * as child_process from "node:child_process";
181	import * as readline from "node:readline";
182	import events from "node:events";
183	import { Progress } from "@paperclover/console/Progress";
184	import { formatSize } from "@/file-viewer/format.ts";

src/file-viewer/rules.ts+13-3

 @@ -73,7 +73,7 @@ export const extsDuration = new Set([...extsAudio, ...extsVideo]);
 export const extsDimensions = new Set([...extsImage, ...extsVideo]);
 /** These files read file contents into `contents`, as-is */
-export const extsReadContents = new Set([".txt", ".chat"]);
+export const extsReadContents = new Set([".txt", ".chat", ".dirsort"]);
 export const extsArchive = new Set([
   ".zip",
 @@ -92,10 +92,20 @@ export const extsArchive = new Set([
 export const extsPreCompressed = new Set([
   ...extsAudio,
   ...extsVideo,
-  ...extsImage,
+  ...extsImageLike,
   ...extsArchive,
-  // TODO: are any of these NOT good for compression
+  ".docx",
+  ".xlsx",
+  ".pptx",
+  ".psd",
+  ".sketch",
+  ".ai",
+  ".3ds",
+  ".fbx",
+  ".blend",
+  ".dng",
 ]);
+extsPreCompressed.delete(".svg");
 export function fileIcon(
   file: Pick<MediaFile, "kind" | "basename" | "path">,

author	clover caruso <git@paperclover.net> 2025-07-08 20:48:30-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:45-07:00
log	82d1470abd4b45708f0e4f91a4c05d954a09f1e4
tree	2f3000e9121b9f9b2fb65e91c2a14deec481c3ab
parent	ee1ec05cc2a55234d1cc51bb285ea247d9144bf8
signature	Commit is signed but in an unrecognized format.

finish scan3