Commit 2fa9b4eb6ee7d21505a0def86bd0c36801869e9a | sitegen

author	clover caruso <git@paperclover.net> 2025-06-09 00:12:41-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log	2fa9b4eb6ee7d21505a0def86bd0c36801869e9a
tree	aa4f1b43e028347a8a1242ef73f78136feb1ce63
parent	ad50869c177bec4aa2eb956e55acbee3ba04ad27
signature	Commit is signed but in an unrecognized format.

incremental generator

12 files changed, 435 insertions(+), 337 deletions(-)

framework
readme.md
src
- q+a
  - components
    QuestionForm.marko
  - pages
    q+a.marko
- static

framework/bundle.ts+10-4

...	...	@@ -38,6 +38,7 @@ export async function bundleClientJavaScript(
38	38	plugins,
39	39	splitting: true,
40	40	write: false,
	41	metafile: true,
41	42	});
42	43	if (bundle.errors.length \|\| bundle.warnings.length) {
43	44	throw new AggregateError(
...	...	@@ -45,29 +46,34 @@ export async function bundleClientJavaScript(
45	46	"JS bundle failed",
46	47	);
47	48	}
48		incr.invalidate("bundle-script");
49	49	const publicScriptRoutes = extraPublicScripts.map((file) =>
50	50	path.basename(file).replace(/\.client\.[tj]sx?/, "")
51	51	);
	52	const { metafile } = bundle;
	53	console.log(metafile);
52	54	const promises: Promise<void>[] = [];
53	55	// TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
54	56	for (const file of bundle.outputFiles) {
	57	const { text } = file;
55	58	let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
56		const text = file.text;
	59	const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
	60	const sources = Object.keys(inputs);
	61
57	62	// Register non-chunks as script entries.
58	63	const chunk = route.startsWith("/js/c.");
59	64	if (!chunk) {
60	65	route = route.replace(".client.js", ".js");
61	66	incr.put({
62		srcId: "bundle-script",
	67	sources,
63	68	type: "script",
64	69	key: route.slice("/js/".length, -".js".length),
65	70	value: text,
66	71	});
67	72	}
	73	// Register chunks and public scripts as assets.
68	74	if (chunk \|\| publicScriptRoutes.includes(route)) {
69	75	promises.push(incr.putAsset({
70		srcId: "bundle-script",
	76	sources,
71	77	key: route,
72	78	body: text,
73	79	}));

framework/css.ts+23-7

...	...	@@ -34,11 +34,19 @@ export function preprocess(css: string, theme: Theme): string {
34	34	);
35	35	}
36	36
	37	export interface Output {
	38	text: string;
	39	sources: string[];
	40	}
	41
37	42	export async function bundleCssFiles(
38	43	cssImports: string[],
39	44	theme: Theme,
40	45	dev: boolean = false,
41		): Promise<string> {
	46	): Promise<Output> {
	47	cssImports = cssImports.map((file) =>
	48	path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
	49	);
42	50	const plugin = {
43	51	name: "clover",
44	52	setup(b) {
...	...	@@ -68,13 +76,14 @@ export async function bundleCssFiles(
68	76	const build = await esbuild.build({
69	77	bundle: true,
70	78	entryPoints: ["$input$"],
71		write: false,
72		external: ["*.woff2"],
73		target: ["ie11"],
74		plugins: [plugin],
	79	external: [".woff2", ".ttf", ".png", ".jpeg"],
	80	metafile: true,
75	81	minify: !dev,
	82	plugins: [plugin],
	83	target: ["ie11"],
	84	write: false,
76	85	});
77		const { errors, warnings, outputFiles } = build;
	86	const { errors, warnings, outputFiles, metafile } = build;
78	87	if (errors.length > 0) {
79	88	throw new AggregateError(errors, "CSS Build Failed");
80	89	}
...	...	@@ -82,8 +91,15 @@ export async function bundleCssFiles(
82	91	throw new AggregateError(warnings, "CSS Build Failed");
83	92	}
84	93	if (outputFiles.length > 1) throw new Error("Too many output files");
85		return outputFiles[0].text;
	94	return {
	95	text: outputFiles[0].text,
	96	sources: Object.keys(metafile.outputs["$input$.css"].inputs)
	97	.filter((x) => x !== "input:."),
	98	};
86	99	}
87	100
88	101	import * as esbuild from "esbuild";
89	102	import * as fs from "#sitegen/fs";
	103	import * as hot from "./hot.ts";
	104	import * as path from "node:path";
	105	import { Incremental } from "./incremental.ts";

framework/generate.tsx+41-24

...	...	@@ -21,7 +21,8 @@ async function sitegen(status: Spinner) {
21	21
22	22	let root = path.resolve(import.meta.dirname, "../src");
23	23	const join = (...sub: string[]) => path.join(root, ...sub);
24		const incr = new Incremental();
	24	const incr = Incremental.fromDisk();
	25	await incr.statAllFiles();
25	26
26	27	// Sitegen reviews every defined section for resources to process
27	28	const sections: sg.Section[] =
...	...	@@ -68,20 +69,24 @@ async function sitegen(status: Spinner) {
68	69	];
69	70	for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
70	71	const items = fs.readDirRecOptionalSync(dir);
71		item: for (const item of items) {
72		if (item.isDirectory()) continue;
	72	item: for (const subPath of items) {
	73	const file = path.join(dir, subPath);
	74	const stat = fs.statSync(file);
	75	if (stat.isDirectory()) continue;
73	76	for (const e of exclude) {
74		if (item.name.endsWith(e)) continue item;
	77	if (subPath.endsWith(e)) continue item;
75	78	}
76		const file = path.relative(dir, item.parentPath + "/" + item.name);
77	79	const trim = ext
78		? file
79		: file.slice(0, -path.extname(file).length).replaceAll(".", "/");
	80	? subPath
	81	: subPath.slice(0, -path.extname(subPath).length).replaceAll(
	82	".",
	83	"/",
	84	);
80	85	let id = prefix + trim.replaceAll("\\", "/");
81	86	if (prefix === "/" && id.endsWith("/index")) {
82	87	id = id.slice(0, -"/index".length) \|\| "/";
83	88	}
84		list.push({ id, file: path.join(item.parentPath, item.name) });
	89	list.push({ id, file: file });
85	90	}
86	91	}
87	92	}
...	...	@@ -92,8 +97,8 @@ async function sitegen(status: Spinner) {
92	97
93	98	// -- server side render --
94	99	status.text = "Building";
95		const cssOnce = new OnceMap<string>();
96		const cssQueue = new Queue<[string, string[], css.Theme], string>({
	100	const cssOnce = new OnceMap<css.Output>();
	101	const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
97	102	name: "Bundle",
98	103	fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
99	104	passive: true,
...	...	@@ -103,7 +108,7 @@ async function sitegen(status: Spinner) {
103	108	interface RenderResult {
104	109	body: string;
105	110	head: string;
106		inlineCss: string;
	111	css: css.Output;
107	112	scriptFiles: string[];
108	113	item: FileItem;
109	114	}
...	...	@@ -139,11 +144,16 @@ async function sitegen(status: Spinner) {
139	144	() => cssQueue.add([item.id, cssImports, theme]),
140	145	);
141	146	// -- html --
142		const bodyPromise = await ssr.ssrAsync(<Page />, {
	147	let page = <Page />;
	148	if (layout?.default) {
	149	const Layout = layout.default;
	150	page = <Layout>{page}</Layout>;
	151	}
	152	const bodyPromise = await ssr.ssrAsync(page, {
143	153	sitegen: sg.initRender(),
144	154	});
145	155
146		const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([
	156	const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
147	157	bodyPromise,
148	158	cssPromise,
149	159	renderedMetaPromise,
...	...	@@ -160,13 +170,14 @@ async function sitegen(status: Spinner) {
160	170	renderResults.push({
161	171	body: text,
162	172	head: renderedMeta,
163		inlineCss,
	173	css: cssBundle,
164	174	scriptFiles: Array.from(addon.sitegen.scripts),
165	175	item: item,
166	176	});
167	177	}
168	178	// This is done in two passes so that a page that throws during evaluation
169	179	// will report "Load Render Module" instead of "Render Static Page".
	180	const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
170	181	const spinnerFormat = status.format;
171	182	status.format = () => "";
172	183	const moduleLoadQueue = new Queue({
...	...	@@ -175,7 +186,7 @@ async function sitegen(status: Spinner) {
175	186	getItemText,
176	187	maxJobs: 1,
177	188	});
178		moduleLoadQueue.addMany(pages);
	189	moduleLoadQueue.addMany(neededPages);
179	190	await moduleLoadQueue.done({ method: "stop" });
180	191	const pageQueue = new Queue({
181	192	name: "Render Static Page",
...	...	@@ -183,7 +194,7 @@ async function sitegen(status: Spinner) {
183	194	getItemText,
184	195	maxJobs: 2,
185	196	});
186		pageQueue.addMany(pages);
	197	pageQueue.addMany(neededPages);
187	198	await pageQueue.done({ method: "stop" });
188	199	status.format = spinnerFormat;
189	200
...	...	@@ -207,7 +218,7 @@ async function sitegen(status: Spinner) {
207	218	async function doStaticFile(item: FileItem) {
208	219	const body = await fs.readFile(item.file);
209	220	await incr.putAsset({
210		srcId: "static:" + item.file,
	221	sources: [item.file],
211	222	key: item.id,
212	223	body,
213	224	});
...	...	@@ -219,7 +230,9 @@ async function sitegen(status: Spinner) {
219	230	maxJobs: 16,
220	231	});
221	232	status.format = () => "";
222		staticQueue.addMany(staticFiles);
	233	staticQueue.addMany(
	234	staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
	235	);
223	236	await staticQueue.done({ method: "stop" });
224	237	status.format = spinnerFormat;
225	238
...	...	@@ -227,11 +240,13 @@ async function sitegen(status: Spinner) {
227	240	status.text = `Concat ${renderResults.length} Pages`;
228	241	await Promise.all(
229	242	renderResults.map(
230		async ({ item: page, body, head, inlineCss, scriptFiles }) => {
	243	async (
	244	{ item: page, body, head, css, scriptFiles },
	245	) => {
231	246	const doc = wrapDocument({
232	247	body,
233	248	head,
234		inlineCss,
	249	inlineCss: css.text,
235	250	scripts: scriptFiles.map(
236	251	(id) =>
237	252	UNWRAP(
...	...	@@ -242,7 +257,7 @@ async function sitegen(status: Spinner) {
242	257	).map((x) => `{${x}}`).join("\n"),
243	258	});
244	259	await incr.putAsset({
245		srcId: "page:" + page.file,
	260	sources: [page.file, ...css.sources],
246	261	key: page.id,
247	262	body: doc,
248	263	headers: {
...	...	@@ -262,7 +277,7 @@ async function sitegen(status: Spinner) {
262	277	status.format = spinnerFormat;
263	278	status.text = `Incremental Flush`;
264	279	incr.flush();
265		incr.serializeToDisk(); // Allows picking up this state again
	280	incr.toDisk(); // Allows picking up this state again
266	281	return { elapsed: (performance.now() - startTime) / 1000 };
267	282	}
268	283
...	...	@@ -281,9 +296,11 @@ function wrapDocument({
281	296	inlineCss: string;
282	297	scripts: string;
283	298	}) {
284		return `<!doctype html><head>${head}${
	299	return `<!doctype html><html lang=en><head>${head}${
285	300	inlineCss ? `<style>${inlineCss}</style>` : ""
286		}</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
	301	}</head><body>${body}${
	302	scripts ? `<script>${scripts}</script>` : ""
	303	}</body></html>`;
287	304	}
288	305
289	306	import { OnceMap, Queue } from "./queue.ts";

framework/hot.ts-11

...	...	@@ -34,8 +34,6 @@ export interface FileStat {
34	34	cssImportsRecursive: string[] \| null;
35	35	lastModified: number;
36	36	imports: string[];
37		/* Used by 'incremental.ts' */
38		srcIds: string[];
39	37	}
40	38	let fsGraph = new Map<string, FileStat>();
41	39	export function setFsGraph(g: Map<string, FileStat>) {
...	...	@@ -60,14 +58,6 @@ Module.prototype._compile = function (
60	58	filename: string,
61	59	format: "module" \| "commonjs",
62	60	) {
63		fs.writeMkdirSync(
64		".clover/debug-transpilation/" +
65		path.relative(projectRoot, filename).replaceAll("\\", "/").replaceAll(
66		"../",
67		"_/",
68		).replaceAll("/", "."),
69		content,
70		);
71	61	const result = ModulePrototypeUnderscoreCompile.call(
72	62	this,
73	63	content,
...	...	@@ -96,7 +86,6 @@ Module.prototype._compile = function (
96	86	: null,
97	87	imports,
98	88	lastModified: stat.mtimeMs,
99		srcIds: [],
100	89	});
101	90	}
102	91	return result;

framework/incremental.ts+223-73

...	...	@@ -1,54 +1,50 @@
1	1	// `Incremental` contains multiple maps for the different parts of a site
2	2	// build, and tracks reused items across builds. It also handles emitting and
3	3	// updating the built site. This structure is self contained and serializable.
4		//
5		// Tracking is simple: Files map to one or more 'source IDs', which map to one
6		// or more 'artifact'. This two layer approach allows many files (say a page +
7		// all its imports) to map to the build of a page, which produces an HTML file
8		// plus a list of scripts.
	4
	5	type SourceId = string; // relative to project root, e.g. 'src/global.css'
	6	type ArtifactId = string; // `${ArtifactType}#${string}`
	7	type Sha1Id = string; // Sha1 hex string
9	8
10	9	interface ArtifactMap {
11	10	asset: Asset;
12	11	script: string;
13	12	}
14		type AllArtifactMaps = {
15		[K in keyof ArtifactMap]: Map<string, ArtifactMap[K]>;
16		};
17	13	type ArtifactType = keyof ArtifactMap;
18
19	14	interface Asset {
20	15	buffer: Buffer;
21	16	headers: Record<string, string \| undefined>;
22	17	hash: string;
23	18	}
24
25	19	export interface PutBase {
26		srcTag?: string; // deprecated
27		srcId: string;
	20	sources: SourceId[];
28	21	key: string;
29	22	}
30
31	23	export interface Put<T extends ArtifactType> extends PutBase {
32	24	type: T;
33	25	value: ArtifactMap[T];
34	26	}
35
36		export interface Output {
37		type: ArtifactType;
38		key: string;
	27	export interface Invalidations {
	28	lastModified: number;
	29	outputs: Set<ArtifactId>;
	30	files: Set<SourceId>;
39	31	}
40	32
41		const gzip = util.promisify(zlib.gzip);
42		const zstd = util.promisify(zlib.zstdCompress);
43
44	33	export class Incremental {
45	34	/** The generated artifacts */
46		out: AllArtifactMaps = {
	35	out: {
	36	[K in keyof ArtifactMap]: Map<string, ArtifactMap[K]>;
	37	} = {
47	38	asset: new Map(),
48	39	script: new Map(),
49	40	};
	41	/** Tracking filesystem entries to `srcId` */
	42	invals = new Map<SourceId, Invalidations>();
	43	/** Tracking output keys to files */
	44	sources = new Map<ArtifactId, SourceId[]>();
	45
50	46	/** Compressed resources */
51		compress = new Map<string, Compressed>();
	47	compress = new Map<Sha1Id, Compressed>();
52	48	compressQueue = new Queue<CompressJob, void>({
53	49	name: "Compress",
54	50	maxJobs: 5,
...	...	@@ -56,59 +52,114 @@ export class Incremental {
56	52	passive: true,
57	53	getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
58	54	});
59		/** Tracking filesystem entries to `srcId` */
60		files = new Map<string, hot.FileStat>();
61		srcIds = new Map<string, Output[]>();
62	55
63		static fromSerialized() {
	56	/** Invalidation deletes build artifacts so the check is trivial. */
	57	needsBuild(type: ArtifactType, key: string) {
	58	return !this.out[type].has(key);
64	59	}
65		serialize() {
66		const writer = new BufferWriter();
67	60
68		const asset = Array.from(
69		this.out.asset,
70		([key, { buffer, hash, headers }]) => {
71		const raw = writer.write(buffer, hash);
72		const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
73		const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null;
74		const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null;
75		return [key, {
76		raw,
77		gzip,
78		zstd,
79		hash,
80		headers,
81		}];
82		},
83		);
84		const script = Array.from(this.out.script);
	61	/*
	62	* Put built artifacts into the incremental cache. The entry points
	63	* used to build this must be provided. 'Incremental' will trace JS
	64	* imports and file modification times tracked by 'hot.ts'.
	65	*/
	66	put<T extends ArtifactType>({
	67	sources,
	68	type,
	69	key,
	70	value,
	71	}: Put<T>) {
	72	this.out[type].set(key, value);
85	73
86		const meta = Buffer.from(
87		JSON.stringify({
88		asset,
89		script,
90		}),
91		"utf-8",
	74	// Update sources information
	75	ASSERT(sources.length > 0, "Missing sources for " + type + " " + key);
	76	sources = sources.map((src) => path.normalize(src));
	77	const fullKey = `${type}#${key}`;
	78	const prevSources = this.sources.get(fullKey);
	79	const newSources = new Set(
	80	sources.map((file) =>
	81	path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
	82	),
92	83	);
	84	this.sources.set(fullKey, [...newSources]);
	85	for (const source of prevSources ?? []) {
	86	if (sources.includes(source)) {
	87	newSources.delete(source);
	88	continue;
	89	}
	90	const invals = UNWRAP(this.invals.get(source));
	91	ASSERT(invals.outputs.has(fullKey));
	92	invals.outputs.delete(fullKey);
	93	}
	94	// Use reflection from the plugin system to get imports.
	95	for (const source of newSources) {
	96	const invals = this.#getOrInitInvals(source);
	97	invals.outputs.add(fullKey);
	98	this.#followImports(source);
	99	}
	100	}
93	101
94		const lengthBuffer = Buffer.alloc(4);
95		lengthBuffer.writeUInt32LE(meta.byteLength, 0);
	102	// TODO: this doesnt remove stuff when it disappeary
	103	#getOrInitInvals(source: string) {
	104	let invals = this.invals.get(source);
	105	if (!invals) {
	106	const g = hot.getFsGraph().get(source);
	107	const lastModified = g?.lastModified ??
	108	fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;
	109	this.invals.set(
	110	source,
	111	invals = {
	112	lastModified,
	113	files: new Set(),
	114	outputs: new Set(),
	115	},
	116	);
	117	}
	118	return invals;
	119	}
96	120
97		return Buffer.concat([meta, lengthBuffer, ...writer.buffers]);
	121	#followImports(file: string) {
	122	const graph = hot.getFsGraph();
	123	const stat = graph.get(file);
	124	if (!stat) return;
	125	for (const i of stat.imports) {
	126	const invals = this.#getOrInitInvals(i);
	127	invals.files.add(file);
	128	this.#followImports(i);
	129	}
98	130	}
99	131
100		serializeToDisk(file = ".clover/incr.state") {
101		const buffer = this.serialize();
102		fs.writeFileSync(file, buffer);
	132	async statAllFiles() {
	133	for (const file of this.invals.keys()) {
	134	const mtime = fs.statSync(file).mtimeMs;
	135	this.updateStat(file, mtime);
	136	}
103	137	}
104	138
105		put<T extends ArtifactType>({
106		srcId,
107		type,
108		key,
109		value,
110		}: Put<T>) {
111		this.out[type].set(key, value);
	139	updateStat(fileKey: string, newLastModified: number) {
	140	const stat = this.invals.get(fileKey);
	141	ASSERT(stat, "Updated stat on untracked file " + fileKey);
	142	if (stat.lastModified < newLastModified) {
	143	// Invalidate
	144	console.log(fileKey + " updated");
	145	const invalidQueue = [fileKey];
	146	let currentInvalid;
	147	while (currentInvalid = invalidQueue.pop()) {
	148	const invalidations = this.invals.get(currentInvalid);
	149	ASSERT(
	150	invalidations,
	151	"No reason to track file '" + currentInvalid +
	152	"' if it has no invalidations",
	153	);
	154	const { files, outputs } = invalidations;
	155	for (const out of outputs) {
	156	const [type, artifactKey] = out.split("#", 2);
	157	this.out[type as ArtifactType].delete(artifactKey);
	158	}
	159	invalidQueue.push(...files);
	160	}
	161	}
	162	stat.lastModified = newLastModified;
112	163	}
113	164
114	165	async putAsset(info: PutAsset) {
...	...	@@ -161,7 +212,99 @@ export class Incremental {
161	212	entry![algo] = out;
162	213	}
163	214
164		invalidate(srcId: string) {
	215	serialize() {
	216	const writer = new BufferWriter();
	217
	218	const asset = Array.from(
	219	this.out.asset,
	220	([key, { buffer, hash, headers }]) => {
	221	const raw = writer.write(buffer, hash);
	222	const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
	223	const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
	224	const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
	225	return [key, {
	226	raw,
	227	gzip,
	228	zstd,
	229	hash,
	230	headers: headers as Record<string, string>,
	231	}] satisfies SerializedMeta["asset"][0];
	232	},
	233	);
	234	const script = Array.from(this.out.script);
	235	const invals = Array.from(this.invals, ([key, value]) => {
	236	const { lastModified, files, outputs } = value;
	237	return [key, {
	238	m: lastModified,
	239	f: [...files],
	240	o: [...outputs],
	241	}] satisfies SerializedMeta["invals"][0];
	242	});
	243	const sources = Array.from(this.sources, ([key, value]) => {
	244	return [key, ...value] as [string, ...string[]];
	245	});
	246	const json = {
	247	asset,
	248	script,
	249	invals,
	250	sources,
	251	} satisfies SerializedMeta;
	252	const meta = Buffer.from(JSON.stringify(json), "utf-8");
	253
	254	const lengthBuffer = Buffer.alloc(4);
	255	lengthBuffer.writeUInt32LE(meta.byteLength, 0);
	256
	257	return Buffer.concat([lengthBuffer, meta, ...writer.buffers]);
	258	}
	259
	260	static fromSerialized(buffer: Buffer): Incremental {
	261	const metaLength = buffer.readUint32LE(0);
	262	const meta: SerializedMeta = JSON.parse(
	263	buffer.subarray(4, 4 + metaLength).toString("utf8"),
	264	);
	265	const view = ([start, end]: View) =>
	266	buffer.subarray(4 + metaLength + start, 4 + metaLength + end);
	267
	268	const incr = new Incremental();
	269	incr.out.asset = new Map<string, Asset>(meta.asset.map(([key, value]) => {
	270	const { hash, raw, gzip, zstd, headers } = value;
	271	if ((gzip \|\| zstd) && !incr.compress.has(hash)) {
	272	incr.compress.set(hash, {
	273	gzip: gzip ? view(gzip) : undefined,
	274	zstd: zstd ? view(zstd) : undefined,
	275	});
	276	}
	277	return [key, {
	278	buffer: view(raw),
	279	headers: headers,
	280	hash: hash,
	281	}];
	282	}));
	283	incr.out.script = new Map(meta.script);
	284	incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {
	285	return [key, {
	286	lastModified: m,
	287	files: new Set(f),
	288	outputs: new Set(o),
	289	}];
	290	}));
	291	incr.sources = new Map(meta.sources.map(([key, ...value]) => [key, value]));
	292	return incr;
	293	}
	294
	295	toDisk(file = ".clover/incr.state") {
	296	const buffer = this.serialize();
	297	fs.writeFileSync(file, buffer);
	298	}
	299
	300	static fromDisk(file = ".clover/incr.state"): Incremental {
	301	try {
	302	const buffer = fs.readFileSync(file);
	303	return Incremental.fromSerialized(buffer);
	304	} catch (err: any) {
	305	if (err.code === "ENOENT") return new Incremental();
	306	throw err;
	307	}
165	308	}
166	309
167	310	async wait() {
...	...	@@ -171,18 +314,14 @@ export class Incremental {
171	314	async flush() {
172	315	ASSERT(!this.compressQueue.active);
173	316	const writer = new BufferWriter();
	317	// TODO: ensure all assets are actually compressed and not fake lying.
174	318	const asset = Object.fromEntries(
175	319	Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
176	320	const raw = writer.write(buffer, hash);
177	321	const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
178	322	const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
179	323	const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
180		return [key, {
181		raw,
182		gzip,
183		zstd,
184		headers,
185		}];
	324	return [key, { raw, gzip, zstd, headers }];
186	325	}),
187	326	);
188	327	await Promise.all([
...	...	@@ -230,10 +369,10 @@ class BufferWriter {
230	369
231	370	export type View = [start: number, end: number];
232	371
233		// Alongside this type is a byte buffer, containing all the assets.
234	372	export interface BuiltAssetMap {
235	373	[route: string]: BuiltAsset;
236	374	}
	375
237	376	export interface BuiltAsset {
238	377	raw: View;
239	378	gzip: View;
...	...	@@ -249,13 +388,24 @@ export interface SerializedMeta {
249	388	hash: string;
250	389	headers: Record<string, string>;
251	390	}]>;
252		script: [key: string, value: string][];
	391	script: Array<[key: string, value: string]>;
	392	invals: Array<[key: string, {
	393	/** Modified */
	394	m: number;
	395	f: SourceId[];
	396	o: ArtifactId[];
	397	}]>;
	398	sources: Array<[string, ...string[]]>;
253	399	}
254	400
	401	const gzip = util.promisify(zlib.gzip);
	402	const zstd = util.promisify(zlib.zstdCompress);
	403
255	404	import * as fs from "#sitegen/fs";
256	405	import * as zlib from "node:zlib";
257	406	import * as util from "node:util";
258	407	import { Queue } from "./queue.ts";
259	408	import * as hot from "./hot.ts";
260	409	import * as mime from "#sitegen/mime";
	410	import * as path from "node:path";
261	411	import { Buffer } from "node:buffer";

framework/lib/fs.ts+1-1

...	...	@@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer \| string) {
34	34
35	35	export function readDirRecOptionalSync(dir: string) {
36	36	try {
37		return readdirSync(dir, { recursive: true, withFileTypes: true });
	37	return readdirSync(dir, { recursive: true, encoding: "utf8" });
38	38	} catch (err: any) {
39	39	if (err.code === "ENOENT") return [];
40	40	throw err;

readme.md+5

...	...	@@ -33,6 +33,11 @@ Included is `src`, which contains `paperclover.net`. Website highlights:
33	33
34	34	## Development
35	35
	36	minimum system requirements:
	37	- a cpu with at least 1 core.
	38	- random access memory.
	39	- windows 7 or later, macos, or linux operating system.
	40
36	41	```
37	42	npm install
38	43

src/q+a/components/QuestionForm.marko created+13

...	...	@@ -0,0 +1,13 @@
	1	<form action="/q+a" method="POST">
	2	<textarea
	3	name="text"
	4	placeholder="ask clover a question..."
	5	required
	6	minlength="1"
	7	maxlength="10000"
	8	/>
	9	<div aria-hidden class="title">ask a question</div>
	10	<button type="submit">send</button>
	11	<div class="disabled-button">send</div>
	12	</form>
	13

src/q+a/pages/q+a.marko+5-14

...	...	@@ -3,7 +3,6 @@ export * as layout from "../layout.tsx";
3	3	export interface Input {
4	4	admin?: boolean;
5	5	}
6
7	6	export const meta: Metadata = {
8	7	title: "paper clover q+a",
9	8	description: "ask clover a question",
...	...	@@ -12,24 +11,16 @@ export const meta: Metadata = {
12	11	<const/{ admin = false } = input />
13	12	<const/questions = [...Question.getAll()] />
14	13
15		<form action="/q+a" method="POST">
16		<textarea
17		name="text"
18		placeholder="ask clover a question..."
19		required
20		minlength="1"
21		maxlength="10000"
22		/>
23		<div aria-hidden class="title">ask a question</div>
24		<button type="submit">send</button>
25		<div class="disabled-button">send</div>
26		</form>
	14	<if=!admin>
	15	<QuestionForm />
	16	</>
27	17	<for\|q\| of=questions>
28		<QuestionRender question=q admin=admin />;
	18	<QuestionRender question=q admin=admin />
29	19	</>
30	20	<footer>
31	21	fun fact: clover has answered ${questions.length} questions
32	22	</footer>
33	23
34	24	import { Question } from "@/q+a/models/Question.ts";
	25	import { QuestionForm } from "@/q+a/components/QuestionForm.marko";
35	26	import QuestionRender from '@/q+a/components/Question.marko';

src/static/robots.txt+5-58

...	...	@@ -1,59 +1,6 @@
1		// __esModule
2		true
	1	User-agent: *
	2	Allow: /
3	3
4		// getEnd
5		function getEnd(file, node) {
6		// Restore if merged: https://github.com/babel/babel/pull/16849
7		// if (node.end != null) {
8		// return node.end;
9		// }
10
11		if (node.loc) {
12		return locToIndex(file, node.loc.end);
13		}
14
15		return null;
16		}
17
18		// getLoc
19		function getLoc(file, index) {
20		return findLoc(getLineIndexes(file), 0, index);
21		}
22
23		// getLocRange
24		function getLocRange(file, start, end) {
25		const lineIndexes = getLineIndexes(file);
26		const startLoc = findLoc(lineIndexes, 0, start);
27
28		if (startLoc) {
29		const endLoc =
30		start === end ? startLoc : findLoc(lineIndexes, startLoc.line - 1, end);
31
32		return {
33		start: startLoc,
34		end: endLoc
35		};
36		}
37		}
38
39		// getStart
40		function getStart(file, node) {
41		// Restore if merged: https://github.com/babel/babel/pull/16849
42		// if (node.start != null) {
43		// return node.start;
44		// }
45
46		if (node.loc) {
47		return locToIndex(file, node.loc.start);
48		}
49
50		return null;
51		}
52
53		// withLoc
54		function withLoc(file, node, start, end) {
55		node.loc = getLocRange(file, start, end);
56		node.start = start;
57		node.end = end;
58		return node;
59		}
	4	# Restrict access to media files to ensure bots do not unintentionally
	5	# break their license agreement (https://paperclover.net/license)
	6	Disallow: /file

src/static/rss.xml+43-115

...	...	@@ -1,115 +1,43 @@
1		// __esModule
2		true
3
4		// parseArgs
5		function parseArgs(file, str, sourceStart, sourceEnd) {
6		const parsed = parseExpression(file, `_(${str})`, sourceStart, sourceEnd, 2);
7
8		if (parsed.type === "CallExpression") {
9		return parsed.arguments;
10		}
11
12		return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
13		}
14
15		// parseExpression
16		function parseExpression(
17		file,
18		str,
19		sourceStart,
20		sourceEnd,
21		sourceOffset)
22		{
23		return tryParse(file, true, str, sourceStart, sourceEnd, sourceOffset);
24		}
25
26		// parseParams
27		function parseParams(file, str, sourceStart, sourceEnd) {
28		const parsed = parseExpression(
29		file,
30		`(${str})=>{}`,
31		sourceStart,
32		sourceEnd,
33		1
34		);
35
36		if (parsed.type === "ArrowFunctionExpression") {
37		return parsed.params;
38		}
39
40		return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
41		}
42
43		// parseStatements
44		function parseStatements(
45		file,
46		str,
47		sourceStart,
48		sourceEnd,
49		sourceOffset)
50		{
51		return tryParse(file, false, str, sourceStart, sourceEnd, sourceOffset);
52		}
53
54		// parseTemplateLiteral
55		function parseTemplateLiteral(file, str, sourceStart, sourceEnd) {
56		const parsed = parseExpression(
57		file,
58		"`" + str + "`",
59		sourceStart,
60		sourceEnd,
61		1
62		);
63
64		if (parsed.type === "TemplateLiteral") {
65		return _compiler.types.templateLiteral(parsed.quasis, parsed.expressions);
66		}
67
68		return ensureParseError(file, parsed, sourceStart, sourceEnd);
69		}
70
71		// parseTypeArgs
72		function parseTypeArgs(file, str, sourceStart, sourceEnd) {
73		const parsed = parseExpression(file, `_<${str}>`, sourceStart, sourceEnd, 2);
74
75		if (parsed.type === "TSInstantiationExpression") {
76		// typeArguments is Flow only (not TS), we need to use typeParameters
77		return parsed.typeParameters;
78		}
79
80		return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
81		}
82
83		// parseTypeParams
84		function parseTypeParams(file, str, sourceStart, sourceEnd) {
85		const parsed = parseExpression(
86		file,
87		`<${str}>()=>{}`,
88		sourceStart,
89		sourceEnd,
90		1
91		);
92
93		if (parsed.type === "ArrowFunctionExpression") {
94		return parsed.typeParameters;
95		}
96
97		return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
98		}
99
100		// parseVar
101		function parseVar(file, str, sourceStart, sourceEnd) {
102		const parsed = parseExpression(
103		file,
104		`(${str})=>{}`,
105		sourceStart,
106		sourceEnd,
107		1
108		);
109
110		if (parsed.type === "ArrowFunctionExpression" && parsed.params.length === 1) {
111		return parsed.params[0];
112		}
113
114		return ensureParseError(file, parsed, sourceStart, sourceEnd);
115		}
	1	<?xml version="1.0" encoding="UTF-8" ?>
	2	<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
	3
	4	<channel>
	5	<title>paper clover</title>
	6	<description>art</description>
	7	<link>https://paperclover.net</link>
	8	<item>
	9	<title>clo's files & cotyledon</title>
	10	<link>https://paperclover.net/file</link>
	11	<description>
	12	i rewrote the file viewer to be nicer to use. as well as uploading about
	13	10gb of old content. there's over a dozen hours worth of shit to explore.
	14	</description>
	15	<pubDate>Wed, 30 Apr 2025 00:00:00 -0000</pubDate>
	16	<guid>https://paperclover.net/file</guid>
	17	</item>
	18	<item>
	19	<title>waterfalls and gender identity update</title>
	20	<link>https://paperclover.net/waterfalls</link>
	21	<description>
	22	this is a song about identity, jealousness, and the seasons. more
	23	specifically, it's a personal account of how it felt realizing i
	24	was a transgender woman. enjoy.
	25	</description>
	26	<pubDate>Wed, 01 Jan 2025 00:00:00 -0000</pubDate>
	27	<guid>https://paperclover.net/waterfalls</guid>
	28
	29	<enclosure url="https://paperclover.net/file/2025/waterfalls/waterfalls.mp4"
	30	type="video/mp4"/>
	31	<media:content url="https://paperclover.net/file/2025/waterfalls/waterfalls.mp4"
	32	type="video/mp4"
	33	width="1920"
	34	height="1080">
	35	</media:content>
	36	<media:thumbnail url="https://paperclover.net/file/2025/waterfalls/thumbnail.jpeg"
	37	width="1920"
	38	height="1080"
	39	/>
	40	</item>
	41	</channel>
	42
	43	</rss>

src/static/wurtzhome.html+66-30

...	...	@@ -1,30 +1,66 @@
1		// __esModule
2		true
3
4		// getTagDefForTagName
5		function getTagDefForTagName(file, tagName) {
6		const tagDef = getTaglibLookup(file).getTag(tagName);
7
8		if (tagDef) {
9		let seen = file.metadata.marko[SEEN_TAGS_KEY];
10		if (!seen) {
11		seen = file.metadata.marko[SEEN_TAGS_KEY] = new Set();
12		}
13
14		if (!seen.has(tagDef)) {
15		seen.add(tagName);
16		const { filePath } = tagDef;
17		const len = filePath.length;
18
19		if (filePath[len - 14] === "m" && filePath.endsWith("marko-tag.json")) {
20		file.metadata.marko.watchFiles.push(filePath);
21		}
22		}
23		}
24		return tagDef;
25		}
26
27		// getTaglibLookup
28		function getTaglibLookup(file) {
29		return file.___taglibLookup;
30		}
	1	<html lang="en">
	2	<head>
	3	<title>paper clover</title>
	4	</head>
	5	<body bgcolor="black" style="word-wrap: initial">
	6	<main style="display: flex; flex-direction: column; justify-content: center; align-items: center; height: 100vh">
	7	<div>
	8	<p style="margin: 0.5rem 0">
	9	<a
	10	href="/waterfalls"
	11	style="text-decoration: none; font-family: times; font-size: 48px"
	12	>
	13	<font color="#FFB5E8">water</font><font color="#FFE3F6">falls</font>
	14	</a>
	15	</p>
	16	<p style="margin: 0.5rem 0">
	17	<a
	18	href="/q+a"
	19	style="text-decoration: none; font-family: times; font-size: 48px"
	20	>
	21	<font color="#97E5B9">questions</font>
	22	<font color="#D4F5E2">and answers</font>
	23	</a>
	24	</p>
	25	<p style="margin: 0.5rem 0">
	26	<a
	27	href="/file"
	28	style="text-decoration: none; font-family: times; font-size: 48px"
	29	>
	30	<font color="#B28DFF">file</font>
	31	<font color="#DDC6FF">browser</font>
	32	</a>
	33	</p>
	34	<p style="margin: 0.5rem 0">
	35	<a
	36	href="/discord"
	37	style="text-decoration: none; font-family: times; font-size: 48px"
	38	>
	39	<font color="#5865F2">dis</font><font color="#7289DA">cord</font>
	40	</a>
	41	</p>
	42	<p style="margin: 0.5rem 0">
	43	<a
	44	href="/github"
	45	style="text-decoration: none; font-family: times; font-size: 48px"
	46	>
	47	<font color="#ffffff">git</font><font color="#959DA5">hub</font>
	48	</a>
	49	</p>
	50	<p style="margin: 0.5rem 0">
	51	<a
	52	href="/rss.xml"
	53	style="text-decoration: none; font-family: times; font-size: 48px"
	54	>
	55	<font color="#F26522">rss</font>
	56	<font color="#FF8147">feed</font>
	57	</a>
	58	</p>
	59	<h1 style="margin: -1.5rem 0 3rem 0; font-size: 7rem; font-weight: 400; font-family: times">
	60	<font color="#B8E1FF">paper</font>
	61	<font color="#E8F4FF">clover</font>
	62	</h1>
	63	</div>
	64	</main>
	65	</body>
	66	</html>