Commit 27b17df57b1d672f6fee971407b0e4d9916bbf11 | sitegen

author	clover caruso <git@paperclover.net> 2025-09-27 18:13:11-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:47-07:00
log	27b17df57b1d672f6fee971407b0e4d9916bbf11
tree	c7c0531a642031b5bcd5bec15db73ca59af17c8a
parent	7773a878828806850398f80dfa514df399b2deed
signature	Commit is signed but in an unrecognized format.

feat(framework): use progress in sitegen / revive sitegen

fixes #30
fixes #32

15 files changed, 213 insertions(+), 106 deletions(-)

framework
lib
- log.ts
- log
  - progress.ts
- packaging.ts
- string.ts
readme.md
run.js
src
- file-viewer

framework/bundle.ts+17-5

...	...	@@ -41,6 +41,8 @@ export async function bundleClientJavaScript(
41	41	markoViaBuildCache(),
42	42	];
43	43
	44	using _ = io.start(`bundle ${entryPoints.length} files`);
	45
44	46	const bundle = await esbuild
45	47	.build({
46	48	assetNames: "/asset/[hash]",
...	...	@@ -108,6 +110,7 @@ export async function bundleClientJavaScript(
108	110
109	111	export type ServerPlatform = "node" \| "passthru";
110	112	export interface ServerSideOptions {
	113	progress: progress.Ref;
111	114	entries: string[];
112	115	viewItems: sg.FileItem[];
113	116	viewRefs: incr.Ref<PageOrView>[];
...	...	@@ -117,6 +120,7 @@ export interface ServerSideOptions {
117	120	metaTemplate: meta.Template;
118	121	}
119	122	export async function bundleServerJavaScript({
	123	progress,
120	124	viewItems,
121	125	viewRefs,
122	126	styleMap,
...	...	@@ -143,6 +147,7 @@ export async function bundleServerJavaScript({
143	147	}
144	148
145	149	const wViewSource = incr.work(
	150	progress,
146	151	async (
147	152	_,
148	153	{ viewItems, regenKeys, regenTtls }: {
...	...	@@ -188,6 +193,7 @@ export async function bundleServerJavaScript({
188	193	);
189	194
190	195	await incr.work(
	196	progress,
191	197	async (io, { regenKeys, viewItems }) => {
192	198	io.writeFile(
193	199	"../ts/view.d.ts",
...	...	@@ -212,8 +218,11 @@ export async function bundleServerJavaScript({
212	218	{ regenKeys: Object.keys(regenKeys), viewItems },
213	219	);
214	220
215		const wBundles = entries.map((entry) =>
216		incr.work(async (io, entry) => {
	221	const wBundles = entries.map((entry) => {
	222	const node = progress.start(incr.toRel(entry), { passive: true });
	223	return incr.work(node, async (io, entry) => {
	224	using _ = io.start("esbuild");
	225
217	226	const pkg = await io.readJson<{
218	227	dependencies: Record<string, string>;
219	228	}>("package.json");
...	...	@@ -313,12 +322,14 @@ export async function bundleServerJavaScript({
313	322	}
314	323	}
315	324	return fileWithMagicWord;
316		}, entry)
317		);
	325	}, entry);
	326	});
318	327
319	328	const wProcessed = wBundles.map(async (wBundle) => {
320	329	if (!(await wBundle)) return;
321		await incr.work(async (io) => {
	330	await incr.work(wBundle.progress, async (io) => {
	331	using _ = io.start("postprocess");
	332
322	333	// Only the reachable resources need to be read and inserted into the bundle.
323	334	// This is what Map<string, incr.Ref> is for
324	335	const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));
...	...	@@ -380,3 +391,4 @@ import * as sg from "#sitegen";
380	391	import type { PageOrView } from "./generate.ts";
381	392	import type * as view from "#sitegen/view";
382	393	import * as meta from "#sitegen/meta";
	394	import * as progress from "lib/log/progress.ts";

framework/css.ts+1

...	...	@@ -62,6 +62,7 @@ export async function bundleCssFiles(
62	62	dev: boolean;
63	63	},
64	64	) {
	65	using _ = io.start(styleKey(cssImports, theme));
65	66	cssImports = await Promise.all(
66	67	cssImports.map((file) => io.trackFile("src/" + file)),
67	68	);

framework/font.ts+36-19

...	...	@@ -1,14 +1,17 @@
1		export async function buildFonts(fonts: sg.Font[]) {
	1	export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) {
2	2	if (fonts.length === 0) return;
	3	using node = p.start("build fonts");
3	4
4	5	const temp = path.resolve(".clover/font");
5	6	const venv = path.join(temp, "venv");
6	7	const bin = path.join(venv, "bin");
7	8
8	9	if (!fs.existsSync(venv)) {
	10	using _ = node.start("setup python env");
9	11	await subprocess.exec("python", ["-m", "venv", venv]);
10	12	}
11	13	if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
	14	using _ = node.start("install pyftsubset");
12	15	await subprocess.exec(path.join(bin, "pip"), [
13	16	"install",
14	17	"fonttools==4.38.0",
...	...	@@ -17,19 +20,27 @@ export async function buildFonts(fonts: sg.Font[]) {
17	20	}
18	21
19	22	const instances = new async.OnceMap<void>();
20		async function makeInstance(input: string, vars: sg.FontVars) {
21		const args = Object.entries(vars).map((x) => {
	23	function getArgs(vars: sg.FontVars) {
	24	return Object.entries(vars).map((x) => {
22	25	const lower = x[0].toLowerCase();
23	26	const k = lower === "wght" \|\| lower === "slnt" ? lower : x[0];
24	27	const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
25	28	return `${k}=${v}`;
26	29	}).sort();
27		const hash = crypto
	30	}
	31
	32	function getHash(input: string, args: string[]) {
	33	return crypto
28	34	.createHash("sha256")
29	35	.update(`${input}${args.join(" ")}`)
30	36	.digest("hex")
31	37	.slice(0, 16)
32	38	.toLowerCase();
	39	}
	40
	41	async function makeInstance(input: string, vars: sg.FontVars) {
	42	const args = getArgs(vars);
	43	const hash = getHash(input, args);
33	44	const outfile = path.join(temp, `${hash}${path.extname(input)}`);
34	45	await instances.get(outfile, async () => {
35	46	await fs.rm(outfile, { force: true });
...	...	@@ -45,32 +56,37 @@ export async function buildFonts(fonts: sg.Font[]) {
45	56	}
46	57
47	58	await Promise.all(fonts.map((font) =>
48		incr.work(async (io, font) => {
	59	incr.work(node, async (io, font) => {
	60	using node = io.start(font.name);
49	61	const baseFile = await fetchFont(font.name, font.sources);
50	62	await Promise.all(font.subsets.map(async (subset) => {
51		let file = baseFile;
52		if (subset.vars) {
53		file = await makeInstance(baseFile, subset.vars);
54		}
	63	const args = getArgs(subset.vars ?? {});
55	64	const unicodes = fontRangesToString(
56	65	Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
57	66	);
58	67	const hash = crypto
59	68	.createHash("sha256")
60		.update(`${file}${unicodes}`)
	69	.update(`${baseFile}${args.join()}${unicodes}`)
61	70	.digest("hex")
62	71	.slice(0, 16)
63	72	.toLowerCase();
64	73	const woff = path.join(temp, hash + ".woff2");
65		await subprocess.exec(path.join(bin, "pyftsubset"), [
66		file,
67		"--flavor=woff2",
68		`--output-file=${woff}`,
69		...subset.layoutFeatures
70		? [`--layout-features=${subset.layoutFeatures.join(",")}`]
71		: [],
72		`--unicodes=${unicodes}`,
73		]);
	74	if (!fs.existsSync(woff)) {
	75	using _ = node.start(hash);
	76	let file = baseFile;
	77	if (subset.vars) {
	78	file = await makeInstance(baseFile, subset.vars);
	79	}
	80	await subprocess.exec(path.join(bin, "pyftsubset"), [
	81	file,
	82	"--flavor=woff2",
	83	`--output-file=${woff}`,
	84	...subset.layoutFeatures
	85	? [`--layout-features=${subset.layoutFeatures.join(",")}`]
	86	: [],
	87	`--unicodes=${unicodes}`,
	88	]);
	89	}
74	90	await io.writeAsset({
75	91	pathname: subset.asset.replace("[ext]", "woff2"),
76	92	buffer: await fs.readFile(woff),
...	...	@@ -195,3 +211,4 @@ import * as incr from "./incremental.ts";
195	211	import * as crypto from "node:crypto";
196	212	import * as async from "#sitegen/async";
197	213	import * as error from "#sitegen/error";
	214	import * as progress from "../lib/log/progress.ts";

framework/generate.ts+68-26

...	...	@@ -11,44 +11,70 @@ export async function main() {
11	11	await incr.compile(generate);
12	12	}
13	13
14		export async function generate() {
	14	export async function generate(p: progress.Ref) {
15	15	// -- read config and discover files --
16		const siteConfig = await incr.work(readManifest);
	16	const siteConfig = await incr.work(p, readManifest);
17	17	const metaTemplate: meta.Template = {
18	18	...siteConfig.meta,
19	19	base: new URL(siteConfig.meta.base),
20	20	};
	21	// -- start font work --
21	22	const {
22	23	staticFiles,
23	24	scripts,
24	25	views,
25	26	pages,
26		} = await discoverAllFiles(siteConfig);
	27	} = await discoverAllFiles(p, siteConfig);
	28
	29	const aliasingInputs = Array.from(
	30	Map.groupBy([...staticFiles, ...pages], (x) => x.id),
	31	).filter((x) => x[1].length > 1);
	32	if (
	33	aliasingInputs.length > 0
	34	) {
	35	throw new Error(
	36	`source files alias:\n` +
	37	aliasingInputs.map(([id, files]) =>
	38	` - ${id}\n` + files.map((x) => " - " + x.file).join("\n")
	39	).join("\n"),
	40	);
	41	}
27	42
28		// TODO: make sure that `static` and `pages` does not overlap
	43	const builtFonts = fonts.buildFonts(p, siteConfig.fonts);
29	44
30	45	// TODO: loadMarkoCache
31	46
32		// -- start font work --
33		const builtFonts = fonts.buildFonts(siteConfig.fonts);
	47	const pageNode = p.start("generate static pages", {});
	48	const viewNode = p.start("compile dynamic views", {});
	49	const staticNode = p.start("compress static asset", {});
34	50
35	51	// -- perform build-time rendering --
36	52	const builtPages = pages.map((item) =>
37		incr.work(preparePage, { item, metaTemplate })
	53	incr.work(
	54	pageNode.start(item.file, { passive: true }),
	55	preparePage,
	56	{ item, metaTemplate },
	57	)
38	58	);
39		const builtViews = views.map((item) => incr.work(prepareView, item));
	59	const builtViews = views.map((item) =>
	60	incr.work(
	61	viewNode.start(item.file, { passive: true }),
	62	prepareView,
	63	item,
	64	)
	65	);
	66	Promise.allSettled(builtViews).then(() => viewNode.end());
40	67	const builtStaticFiles = Promise.all(
41	68	staticFiles.map((item) =>
42		incr.work(
43		async (io, { id, file }) =>
44		void (await io.writeAsset({
45		pathname: id,
46		buffer: await io.readFile(file),
47		})),
48		item,
49		)
	69	incr.work(staticNode, async (io, { id, file }) => {
	70	using _ = io.start(file);
	71	return void (await io.writeAsset({
	72	pathname: id,
	73	buffer: await io.readFile(file),
	74	}));
	75	}, item)
50	76	),
51		);
	77	).then(() => staticNode.end());
52	78	const routes = await Promise.all([...builtViews, ...builtPages]);
53	79
54	80	const viewsAndDynPages: incr.Ref<PageOrView>[] = [
...	...	@@ -57,15 +83,16 @@ export async function generate() {
57	83	];
58	84
59	85	// -- page resources --
60		const scriptMap = incr.work(bundle.bundleClientJavaScript, {
	86	const scriptMap = incr.work(p, bundle.bundleClientJavaScript, {
61	87	clientRefs: routes.flatMap((x) => x.clientRefs),
62	88	extraPublicScripts: scripts.map((entry) => entry.file),
63	89	dev: false,
64	90	});
65		const styleMap = prepareInlineCss(routes);
	91	const styleMap = prepareInlineCss(p, routes);
66	92
67	93	// -- backend --
68	94	const builtBackend = bundle.bundleServerJavaScript({
	95	progress: p,
69	96	entries: siteConfig.backends,
70	97	platform: "node",
71	98	styleMap,
...	...	@@ -79,19 +106,20 @@ export async function generate() {
79	106	});
80	107
81	108	// -- assemble page assets --
82		const pAssemblePages = builtPages.map((page) =>
83		assembleAndWritePage(page, styleMap, scriptMap)
84		);
	109	const pAssemblePages = Promise.all(
	110	builtPages.map((page) => assembleAndWritePage(page, styleMap, scriptMap)),
	111	).then(() => pageNode.end());
85	112
86	113	await Promise.all([
87	114	builtBackend,
88	115	builtStaticFiles,
89		...pAssemblePages,
	116	pAssemblePages,
90	117	builtFonts,
91	118	]);
92	119	}
93	120
94	121	export async function readManifest(io: Io) {
	122	using _ = io.start("read site manifest");
95	123	const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
96	124	return {
97	125	siteSections: cfg.siteSections.map((section) => ({
...	...	@@ -107,12 +135,14 @@ export async function readManifest(io: Io) {
107	135	}
108	136
109	137	export async function discoverAllFiles(
	138	p: progress.Ref,
110	139	siteConfig: Awaited<ReturnType<typeof readManifest>>,
111	140	) {
	141	using node = p.start("discover files");
112	142	return (
113	143	await Promise.all(
114	144	siteConfig.siteSections.map(({ root: sectionRoot }) =>
115		incr.work(scanSiteSection, toAbs(sectionRoot))
	145	incr.work(node, scanSiteSection, toAbs(sectionRoot))
116	146	),
117	147	)
118	148	).reduce((acc, next) => ({
...	...	@@ -124,6 +154,7 @@ export async function discoverAllFiles(
124	154	}
125	155
126	156	export async function scanSiteSection(io: Io, sectionRoot: string) {
	157	using _ = io.start(toRel(sectionRoot));
127	158	// Static files are compressed and served as-is.
128	159	// - "{section}/static/*.png"
129	160	let staticFiles: FileItem[] = [];
...	...	@@ -274,7 +305,7 @@ export async function preparePage(
274	305	cssImports,
275	306	theme: theme ?? null,
276	307	styleKey,
277		clientRefs: Array.from(addon[sg.userData.key].scripts),
	308	clientRefs: Array.from(UNWRAP(addon[sg.userData.key]).scripts),
278	309	} as const;
279	310	}
280	311
...	...	@@ -303,16 +334,19 @@ export async function prepareView(io: Io, item: sg.FileItem) {
303	334	export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
304	335
305	336	export function prepareInlineCss(
	337	p: progress.Ref,
306	338	items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,
307	339	) {
	340	const node = p.start("bundle css");
308	341	const map = new Map<string, incr.Ref<string>>();
309	342	for (const { styleKey, cssImports, theme } of items) {
310	343	if (map.has(styleKey)) continue;
311	344	map.set(
312	345	styleKey,
313		incr.work(css.bundleCssFiles, { cssImports, theme, dev: false }),
	346	incr.work(node, css.bundleCssFiles, { cssImports, theme, dev: false }),
314	347	);
315	348	}
	349	Promise.allSettled([...map.values()]).then(() => node.end());
316	350	return map;
317	351	}
318	352
...	...	@@ -323,12 +357,18 @@ export async function assembleAndWritePage(
323	357	scriptWork: incr.Ref<Record<string, string>>,
324	358	) {
325	359	const page = await pageWork;
	360	ASSERT("passive" in pageWork.progress);
	361	const pageNode = pageWork.progress;
326	362	return incr.work(
	363	pageWork.progress,
327	364	async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
328	365	const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
329	366
330	367	const scriptIds = clientRefs.map(hot.getScriptId);
331	368	const scriptMap = await io.readWork(scriptWork);
	369
	370	pageNode.passive = false;
	371
332	372	const scripts = scriptIds
333	373	.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
334	374	.map((x) => `{${x}}`)
...	...	@@ -348,6 +388,7 @@ export async function assembleAndWritePage(
348	388	},
349	389	regenerative: !!regenerate,
350	390	});
	391	pageNode.passive = true;
351	392	},
352	393	page,
353	394	);
...	...	@@ -362,6 +403,7 @@ import * as meta from "#sitegen/meta";
362	403	import * as render from "#engine/render";
363	404	import * as sg from "#sitegen";
364	405	import type { FileItem } from "#sitegen";
	406	import * as progress from "../lib/log/progress.ts";
365	407
366	408	import * as bundle from "./bundle.ts";
367	409	import * as css from "./css.ts";

framework/incremental.ts+40-26

...	...	@@ -18,6 +18,8 @@ export interface Ref<T> {
18	18	/** This method is compatible with `await` syntax */
19	19	then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20	20	get value(): T \| null;
	21	/** The progress node passed into the input */
	22	progress: progress.Node \| progress.Ref;
21	23	}
22	24	type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
23	25
...	...	@@ -29,9 +31,13 @@ type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
29	31	*
30	32	* Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
31	33	*/
32		export function work<O>(job: Job<void, O>): Ref<O>;
33		export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;
34		export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
	34	export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
	35	export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
	36	export function work<I, O>(
	37	p: progress.Ref,
	38	job: Job<I, O>,
	39	input?: I,
	40	): Ref<O> {
35	41	const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1]));
36	42	const keySource = [source, util.inspect(input)].join(":");
37	43	const key = crypto.createHash("sha1").update(keySource).digest("base64url");
...	...	@@ -45,15 +51,21 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
45	51
46	52	const prev = works.get(key) as Work<O> \| null;
47	53	if (prev) {
48		return { key, then: (done) => done(prev.value), value: prev.value };
	54	return {
	55	key,
	56	then: (done) => done(prev.value),
	57	value: prev.value,
	58	progress: p,
	59	};
49	60	}
50	61
51	62	async function perform() {
52		const io = new Io(key);
	63	const io = new Io(key, p);
53	64	jobs += 1;
54	65	newKeys += 1;
	66	await timers.setTimeout(200);
55	67	try {
56		const value = await job(io, input);
	68	const value = await job(io, input as I);
57	69	validateSerializable(value, "");
58	70	const { reads, writes } = io;
59	71	works.set(key, { value, affects: [], reads, writes });
...	...	@@ -82,27 +94,24 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
82	94	get value() {
83	95	return (works.get(this.key)?.value as O) ?? null;
84	96	},
	97	progress: p,
85	98	};
86	99	}
87	100
88		export async function compile<T>(compiler: () => Promise<T>) {
	101	export async function compile<T>(
	102	compiler: (node: progress.Ref) => Promise<T>,
	103	p: progress.Ref = progress,
	104	) {
89	105	ASSERT(!running, `Cannot run twice at the same time.`);
	106	const node = p.start("sitegen!");
90	107	try {
91	108	running = true;
92	109	ASSERT(jobs === 0);
93	110	const start = performance.now();
94		const timerSpinner = new Spinner({
95		text: () =>
96		`sitegen! [${((performance.now() - start) / 1000).toFixed(1)}s]`,
97		fps: 10,
98		});
99		using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
100	111
101		const value = await compiler();
	112	const value = await compiler(node);
102	113	ASSERT(jobs === 0);
103		timerSpinner.text = "incremental flush";
104		await flush(start);
105		timerSpinner.stop();
	114	await flush(node, start);
106	115	return {
107	116	value,
108	117	watchFiles: new Set(files.keys()),
...	...	@@ -112,6 +121,7 @@ export async function compile<T>(compiler: () => Promise<T>) {
112	121	newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
113	122	};
114	123	} finally {
	124	node.end();
115	125	running = false;
116	126	newKeys = 0;
117	127	seenWrites.clear();
...	...	@@ -119,7 +129,8 @@ export async function compile<T>(compiler: () => Promise<T>) {
119	129	}
120	130	}
121	131
122		export async function flush(start: number) {
	132	export async function flush(progress: progress.Node, start: number) {
	133	using _ = progress.start("incremental flush");
123	134	// Trim
124	135	const detachedFiles = new Set<string>();
125	136	const referencedAssets = new Set<string>();
...	...	@@ -163,11 +174,11 @@ export async function flush(start: number) {
163	174	const serialized = msgpackr.pack(serialize());
164	175	await fs.writeMkdir(".clover/incr.state", serialized);
165	176	const time = (performance.now() - start).toFixed(0);
166		console.success(`sitegen! in ${time} ms`);
167		console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);
168		console.writeLine(` - ${assets.size} static assets`);
169		console.writeLine(
170		` - dist: ${formatSize(dist)},` +
	177	progress.log.info(
	178	`sitegen! in ${time} ms\n` +
	179	` - ${works.size} keys (${works.size - newKeys} cached)\n` +
	180	` - ${assets.size} static assets\n` +
	181	` - dist: ${formatSize(dist)},` +
171	182	` incremental: ${formatSize(serialized.byteLength)}`,
172	183	);
173	184	}
...	...	@@ -453,10 +464,14 @@ export function getAssetManifest() {
453	464	* - Outputs written with Io are deleted when abandoned.
454	465	*/
455	466	export class Io {
456		constructor(public key: string) {}
	467	constructor(public key: string, private node: progress.Ref) {}
457	468	reads: Reads = { files: new Set(), works: new Set() };
458	469	writes: Writes = { files: new Set(), assets: new Map() };
459	470
	471	start(text: string, options?: progress.StartOptions) {
	472	return this.node.start(text, options);
	473	}
	474
460	475	#trackFs(file: string) {
461	476	const resolved = toAbs(file);
462	477	const key = toRel(resolved);
...	...	@@ -720,8 +735,7 @@ import * as util from "node:util";
720	735	import * as crypto from "node:crypto";
721	736	import * as zlib from "node:zlib";
722	737
723		import * as console from "@paperclover/console";
724		import { Spinner } from "@paperclover/console/Spinner";
	738	import * as progress from "../lib/log/progress.ts";
725	739	import { formatSize } from "@/file-viewer/format.ts";
726	740
727	741	import * as msgpackr from "msgpackr";

lib/log.ts+1-4

...	...	@@ -66,10 +66,7 @@ export interface Widget {
66	66	* may get called more often than the specified `fps`.
67	67	* supports color codes but not ansi cursor movements.
68	68	*/
69		format(
70		now: ReturnType<typeof performance.now>,
71		focus: boolean,
72		): string \| null;
	69	format(now: ReturnType<typeof performance.now>): string \| null;
73	70	/** 'null' to never update (use 'onChange'). defaults to 12 fps */
74	71	fps?: number \| null;
75	72	/** Subscribe to manual widget updates. Call `rerender` when needed. */

lib/log/progress.ts+8-4

...	...	@@ -233,7 +233,7 @@ function newNode(
233	233	function formatProgress(now: number, list: Internal[]): string {
234	234	let out = "";
235	235	for (const top of list) {
236		if (top.passive && top.children.length === 0) continue;
	236	if (top.passive && !hasChildren(top)) continue;
237	237	out += renderMainLine(top, now, 0) + "\n";
238	238	out += renderChildren(top, now, []);
239	239	}
...	...	@@ -283,15 +283,19 @@ function renderMainLine(state: Internal, now: number, depth: number) {
283	283	(state.value > 0 ? `[${state.value}] ` : "") + state.text;
284	284	}
285	285
	286	function hasChildren(states: Internal): boolean {
	287	return states.children.some((x) => !x.passive \|\| hasChildren(x));
	288	}
	289
286	290	function renderChildren(state: Internal, now: number, depth: boolean[]) {
287		let maxHeight = 25; // TODO:
	291	let maxHeight = 50; // TODO:
288	292	let truncated = 0;
289	293	let out = "";
290		const { children } = state;
	294	let { children } = state;
	295	children = children.filter((x) => !x.passive \|\| hasChildren(x));
291	296	if (children.length === 0) return "";
292	297	for (let i = 0, { length } = children; i < length; i += 1) {
293	298	const child = UNWRAP(children[i]);
294		if (child.passive && child.children.length === 0) continue;
295	299	let item = "";
296	300	const left = depth.map((x) => x ? box.line : " ").join("");
297	301	item += left + (i === length - 1 && !truncated ? box.langle : box.tee);

lib/packaging.ts+3-4

...	...	@@ -33,12 +33,11 @@ export async function transpileForNpm(
33	33	packageJson: Record<string, unknown>,
34	34	tsToJsdoc?: TsToJsdoc,
35	35	) {
	36	const libName = await Promise.resolve("ts-to-jsdoc");
36	37	try {
37		// @ts-expect-error optional dependency
38		tsToJsdoc ??= await import("ts-to-jsdoc") as TsToJsdoc;
	38	tsToJsdoc ??= await import(libName) as TsToJsdoc;
39	39	} catch {
40		// @ts-expect-error unsupported import specifier
41		tsToJsdoc ??= await import("npm:ts-to-jsdoc@2.4.0") as TsToJsdoc;
	40	tsToJsdoc ??= await import("npm:" + libName + "@2.4.0") as TsToJsdoc;
42	41	}
43	42	const { transpileFile } = tsToJsdoc;
44	43

lib/string.ts+8

...	...	@@ -155,4 +155,12 @@ function segmentWidth(segment: string) {
155	155	);
156	156	}
157	157
	158	export function escapeShellArgument(s: string): string {
	159	if (s.length === 0) return "''";
	160	if (/^[a-zA-Z0-9_\-.\/@,+=%:]+$/.test(s) && s[0] !== ":") return s;
	161	if (!s.includes("'")) return `'${s}'`;
	162	if (!/["$`\\]/.test(s)) return `"${s}"`;
	163	return `'${s.replace(/'/g, "'\\''")}'`;
	164	}
	165
158	166	import { UNWRAP } from "lib/assert.ts";

readme.md+10-6

...	...	@@ -1,7 +1,9 @@
1		# clover sitegen framework
	1	# clover sitegen framework and typescript library
2	2
3	3	this repository contains clover's "sitegen" framework, which is a set of tools
4		that assist building websites. these tools power <https://paperclover.net>.
	4	that assist building websites. additionally, this is the home of her typescript
	5	library `lib`, containing many high quality standalone sub-projects. all these
	6	tools power <https://paperclover.net>.
5	7
6	8	- HTML ("Server Side Rendering") engine written from scratch. (~500 lines)
7	9	- A more practical JSX runtime (`class` instead of `className`, built-in
...	...	@@ -18,12 +20,14 @@ that assist building websites. these tools power <https://paperclover.net>.
18	20	- (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
19	21	checks when the files change. For example, changing a component re-tests
20	22	only pages that use that component and re-lints only the changed file.
21		- Integrated libraries for building complex, content heavy web sites.
22	23	- Static asset serving with automatic ETag and build-time compression.
23	24	- Dynamic pages. Static pages can be regenerated (and cached) at runtime.
24		- Databases with a minimal SQLite wrapper.
25		- Meta Tags and Open Graph generation.
26	25	- Font subsetting to reduce page bandwidth.
	26	- General purpose typescript libraries. (See [`./lib`](./lib/readme.md))
	27	- [Logging](./lib/log.ts) and [progress](./lib/log/progress.ts) indicators.
	28	- [Priority Queue](./lib/queue.ts) with automatic reschedulability.
	29	- [Meta Tags](./framework/lib/meta.ts) and Open Graph generation.
	30	- Databases with a minimal SQLite wrapper.
27	31	- Built on the stable and battle-tested Node.js runtime.
28	32
29	33	none of these tools are complex or revolutionary. rather, this project is the
...	...	@@ -32,7 +36,7 @@ example on how other over-complicate other frameworks.
32	36
33	37	[Marko]: https://next.markojs.com
34	38
35		Included is `src`, which contains `paperclover.net`. Website highlights:
	39	Included is `src`, which contains `paperclover.net`'s source code. Highlights:
36	40
37	41	- TODO: flashy homepage.
38	42	- [Question/Answer board, custom markdown parser and components][q+a].

run.js-2

...	...	@@ -53,8 +53,6 @@ process.on("uncaughtException", (error) => {
53	53	process.exit(1);
54	54	});
55	55
56		hot.load("node:fs").readFileSync(undefined);
57
58	56	// Load with hooks
59	57	if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
60	58	if (process.argv.length == 2) {

src/file-viewer/backend.tsx+1-1

...	...	@@ -126,7 +126,7 @@ app.get("/file/*", async (c, next) => {
126	126	rangeHeader = null;
127	127	}
128	128
129		const found = await cache.read(filePath
	129	const found = await cache.read(filePath);
130	130	// foundFile = await cache.read(filePath, encoding, {
131	131	// start: rangeHeader ? rangeHeader[0] : 0,
132	132	// end: rangeHeader ? rangeHeader[1] : Infinity,

src/file-viewer/bin/scan3.ts+6-1

...	...	@@ -22,6 +22,7 @@ export async function main() {
22	22	const walkQueue = new queue.PriorityQueue(10);
23	23
24	24	const dirsNode = progress.start("Walk Tree", { estimate: 1 });
	25	dirsNode.passive = true;
25	26	const fileNode = progress.start("Process File", { estimate: 0 });
26	27	fileNode.sortChildren = (a, b) => {
27	28	const ac = a.children.length > 0 ? 1 : 0;
...	...	@@ -260,7 +261,11 @@ export async function main() {
260	261	}
261	262
262	263	// Add the root & recursively iterate!
263		scanDirectory(Path.resolve(root));
	264	const rootPath = Path.resolve(root);
	265	if (!rootPath.ifExistsSync()) {
	266	throw new Error(`file store ${rootPath} is not mounted`);
	267	}
	268	scanDirectory(rootPath);
264	269	// await qList.done();
265	270	// await qMeta.done();
266	271	// await qProcess.done();

src/file-viewer/ffmpeg.ts+3-2

...	...	@@ -61,7 +61,7 @@ export async function spawn(options: SpawnOptions) {
61	61	if (code !== 0) {
62	62	const fmt = code ? `code ${code}` : `signal ${signal}`;
63	63	const e: any = new Error(`ffmpeg failed with ${fmt}`);
64		e.args = [ffmpeg, ...args].join(" ");
	64	e.args = [ffmpeg, ...args].map(string.escapeShellArgument).join(" ");
65	65	e.code = code;
66	66	e.signal = signal;
67	67	throw e;
...	...	@@ -155,7 +155,7 @@ export class Parse {
155	155	}
156	156	}
157	157
158		const hlsMatch = line.match(/^\[hls @ .*Opening '(.+)' for writing/);
	158	const hlsMatch = line.match(/.*Opening '(.+)' for writing/);
159	159	if (hlsMatch) {
160	160	if (!hlsMatch[1].endsWith(".tmp")) {
161	161	this.hlsFile = path.basename(hlsMatch[1]);
...	...	@@ -179,3 +179,4 @@ import events from "node:events";
179	179	import * as path from "node:path";
180	180	import * as progress from "../../lib/log/progress.ts";
181	181	import type { Path } from "#sitegen/path";
	182	import * as string from "lib/string.ts";

src/file-viewer/transcode-rules.ts+11-6

...	...	@@ -220,12 +220,17 @@ export function getMpegDashArgs(
220	220	cmd.push("-use_template", "1");
221	221
222	222	// build adaptation sets string
223		const videoIndices = videos.map((_, i) => i).join(",");
224		const audioIndices = audios.map((_, i) => videos.length + i).join(",");
225		cmd.push(
226		"-adaptation_sets",
227		`id=0,streams=${videoIndices} id=1,streams=${audioIndices}`,
228		);
	223	const videoAdaptationSets = videos.map((_, i) => `id=${i},streams=${i}`)
	224	.join(" ");
	225	let adaptationSets = videoAdaptationSets;
	226
	227	if (audios.length > 0) {
	228	const audioId = videos.length;
	229	const audioIndices = audios.map((_, i) => videos.length + i).join(",");
	230	adaptationSets += ` id=${audioId},streams=${audioIndices}`;
	231	}
	232
	233	cmd.push("-adaptation_sets", adaptationSets);
229	234
230	235	cmd.push("-y");
231	236	cmd.push(outDir.join("dash.mpd").toString());