Commit 630def77dd3df902c021bd8cc05db17578d21eda | sitegen

author	clover caruso <git@paperclover.net> 2025-08-02 17:31:58-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
log	630def77dd3df902c021bd8cc05db17578d21eda
tree	509334b5abfc6cd8074d920107423e0baa921fa1
parent	f723b6b33082666d0db8475cc3171333b5326275
signature	Commit is signed but in an unrecognized format.

rewrite `incremental.ts` (#21)

the problems with the original implementation was mostly around error
handling. sources had to be tracked manually and provided to each
incremental output. the `hasArtifact` check was frequently forgotten.
this has been re-abstracted through `incr.work()`, which is given an
`io` object. all fs reads and module loads go through this interface,
which allows the sources to be properly tracked, even if it throws.

closes #12

19 files changed, 1596 insertions(+), 1503 deletions(-)

framework
package-lock.json
package.json
src
- file-viewer
  - bin
    scan3.ts
- global.css
- pages
  - index.marko

framework/bundle.ts+189-219

...	...	@@ -1,17 +1,25 @@
	1	async function trackEsbuild(io: Io, metafile: esbuild.Metafile) {
	2	await Promise.all(Object.keys(metafile.inputs)
	3	.filter(file => !isIgnoredSource(file))
	4	.map(file => io.trackFile(file)));
	5	}
	6
1	7	// This file implements client-side bundling, mostly wrapping esbuild.
2	8	export async function bundleClientJavaScript(
3		referencedScripts: string[],
4		extraPublicScripts: string[],
5		incr: Incremental,
6		dev: boolean = false,
	9	io: Io,
	10	{ clientRefs, extraPublicScripts, dev = false }: {
	11	clientRefs: string[];
	12	extraPublicScripts: string[];
	13	dev: boolean;
	14	},
7	15	) {
8	16	const entryPoints = [
9	17	...new Set([
10		...referencedScripts.map((file) => path.resolve(hot.projectSrc, file)),
	18	...clientRefs.map((x) => `src/${x}`),
11	19	...extraPublicScripts,
12		]),
	20	].map(toAbs)),
13	21	];
14		if (entryPoints.length === 0) return;
	22	if (entryPoints.length === 0) return {};
15	23	const invalidFiles = entryPoints
16	24	.filter((file) => !file.match(/\.client\.[tj]sx?/));
17	25	if (invalidFiles.length > 0) {
...	...	@@ -24,7 +32,7 @@ export async function bundleClientJavaScript(
24	32
25	33	const clientPlugins: esbuild.Plugin[] = [
26	34	projectRelativeResolution(),
27		markoViaBuildCache(incr),
	35	markoViaBuildCache(),
28	36	];
29	37
30	38	const bundle = await esbuild.build({
...	...	@@ -65,256 +73,218 @@ export async function bundleClientJavaScript(
65	73	)
66	74	);
67	75	const { metafile, outputFiles } = bundle;
68		const promises: Promise<void>[] = [];
	76	const p = []
	77	p.push(trackEsbuild(io, metafile));
	78	const scripts: Record<string, string> = {};
69	79	for (const file of outputFiles) {
70	80	const { text } = file;
71	81	let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
72	82	const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
73		const sources = Object.keys(inputs)
74		.filter((x) => !x.startsWith("<define:"));
	83	const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x));
75	84
76	85	// Register non-chunks as script entries.
77	86	const chunk = route.startsWith("/js/c.");
78	87	if (!chunk) {
79	88	const key = hot.getScriptId(path.resolve(sources[sources.length - 1]));
80	89	route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
81		incr.put({
82		sources,
83		kind: "script",
84		key,
85		value: text,
86		});
	90	scripts[key] = text;
87	91	}
88	92	// Register chunks and public scripts as assets.
89	93	if (chunk \|\| publicScriptRoutes.includes(route)) {
90		promises.push(incr.putAsset({
91		sources,
92		key: route,
93		body: text,
94		}));
	94	p.push(io.writeAsset(route, text));
95	95	}
96	96	}
97		await Promise.all(promises);
	97	await Promise.all(p);
	98	return scripts;
98	99	}
99	100
100	101	export type ServerPlatform = "node" \| "passthru";
	102	export interface ServerSideOptions {
	103	entries: string[],
	104	viewItems: sg.FileItem[]
	105	viewRefs: incr.Ref<PreparedView>[],
	106	styleMap: Map<string, incr.Ref<string>>;
	107	scriptMap: incr.Ref<Record<string, string>>;
	108	platform: ServerPlatform,
	109	}
101	110	export async function bundleServerJavaScript(
102		incr: Incremental,
103		platform: ServerPlatform = "node",
	111	{ viewItems, viewRefs, styleMap, scriptMap: wScriptMap, entries, platform }: ServerSideOptions
104	112	) {
105		if (incr.hasArtifact("backendBundle", platform)) return;
106
107		// Comment
108		const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
	113	const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => {
	114	const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
	115	return {
	116	magicWord,
	117	file: [
	118	...viewItems.map((view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`),
	119	`const styles = ${magicWord}[-2]`,
	120	`export const scripts = ${magicWord}[-1]`,
	121	"export const views = {",
	122	...viewItems.map((view, i) => [
	123	` ${JSON.stringify(view.id)}: {`,
	124	` component: view${i}.default,`,
	125	` meta: view${i}.meta,`,
	126	` layout: view${i}.layout?.default ?? null,`,
	127	` inlineCss: styles[${magicWord}[${i}]]`,
	128	` },`,
	129	].join("\n")),
	130	"}",
	131	].join("\n")
	132	};
	133	}, viewItems)
109	134
110		const viewSource = [
111		...Array.from(
112		incr.out.viewMetadata,
113		([, view], i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
114		),
115		`const styles = ${magicWord}[-2]`,
116		`export const scripts = ${magicWord}[-1]`,
117		"export const views = {",
118		...Array.from(incr.out.viewMetadata, ([key, view], i) =>
119		[
120		` ${JSON.stringify(key)}: {`,
121		` component: view${i}.default,`,
122		// ` meta: ${
123		// view.staticMeta ? JSON.stringify(view.staticMeta) : `view${i}.meta`
124		// },`,
125		` meta: view${i}.meta,`,
126		` layout: ${view.hasLayout ? `view${i}.layout?.default` : "null"},`,
127		` inlineCss: styles[${magicWord}[${i}]]`,
128		` },`,
129		].join("\n")),
130		"}",
131		].join("\n");
	135	const wBundles = entries.map(entry => [entry, incr.work(async (io, entry) => {
	136	const pkg = await io.readJson<{ dependencies: Record<string, string>; }>("package.json");
132	137
133		// -- plugins --
134		const serverPlugins: esbuild.Plugin[] = [
135		virtualFiles({
136		"$views": viewSource,
137		}),
138		projectRelativeResolution(),
139		markoViaBuildCache(incr),
140		{
141		name: "replace client references",
142		setup(b) {
143		b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({
144		contents:
145		hot.resolveClientRefs(await fs.readFile(file, "utf-8"), file).code,
146		loader: path.extname(file).slice(1) as esbuild.Loader,
147		}));
	138	let magicWord = null as string \| null;
	139	// -- plugins --
	140	const serverPlugins: esbuild.Plugin[] = [
	141	virtualFiles({
	142	// only add dependency when imported.
	143	"$views": async () => {
	144	const view = await io.readWork(wViewSource);
	145	({ magicWord } = view);
	146	return view.file;
	147	},
	148	}),
	149	projectRelativeResolution(),
	150	markoViaBuildCache(),
	151	{
	152	name: "replace client references",
	153	setup(b) {
	154	b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({
	155	contents:
	156	hot.resolveClientRefs(await fs.readFile(file, "utf-8"), file).code,
	157	loader: path.extname(file).slice(1) as esbuild.Loader,
	158	}));
	159	},
148	160	},
149		},
150		{
151		name: "mark css external",
152		setup(b) {
153		b.onResolve(
154		{ filter: /\.css$/ },
155		() => ({ path: ".", namespace: "dropped" }),
156		);
157		b.onLoad(
158		{ filter: /./, namespace: "dropped" },
159		() => ({ contents: "" }),
160		);
	161	{
	162	name: "mark css external",
	163	setup(b) {
	164	b.onResolve(
	165	{ filter: /\.css$/ },
	166	() => ({ path: ".", namespace: "dropped" }),
	167	);
	168	b.onLoad(
	169	{ filter: /./, namespace: "dropped" },
	170	() => ({ contents: "" }),
	171	);
	172	},
161	173	},
162		},
163		];
164		const pkg = await fs.readJson("package.json") as {
165		dependencies: Record<string, string>;
166		};
167		const { metafile, outputFiles } = await esbuild.build({
168		bundle: true,
169		chunkNames: "c.[hash]",
170		entryNames: "server",
171		entryPoints: [
172		path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
173		],
174		platform: "node",
175		format: "esm",
176		minify: false,
177		outdir: "out!",
178		plugins: serverPlugins,
179		splitting: true,
180		logLevel: "silent",
181		write: false,
182		metafile: true,
183		jsx: "automatic",
184		jsxImportSource: "#ssr",
185		jsxDev: false,
186		define: {
187		MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
188		},
189		external: Object.keys(pkg.dependencies)
190		.filter((x) => !x.startsWith("@paperclover")),
191		});
	174	];
192	175
193		const files: Record<string, Buffer> = {};
194		let fileWithMagicWord: string \| null = null;
195		for (const output of outputFiles) {
196		const basename = output.path.replace(/^.*?!/, "");
197		const key = "out!" + basename.replaceAll("\\", "/");
198		// If this contains the generated "$views" file, then
199		// mark this file as the one for replacement. Because
200		// `splitting` is `true`, esbuild will not emit this
201		// file in more than one chunk.
202		if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
203		fileWithMagicWord = basename;
204		}
205		files[basename] = Buffer.from(output.contents);
206		}
207		incr.put({
208		kind: "backendBundle",
209		key: platform,
210		value: {
211		magicWord,
212		files,
213		fileWithMagicWord,
214		},
215		sources: Object.keys(metafile.inputs).filter((x) =>
216		!x.includes("<define:") &&
217		!x.startsWith("vfs:") &&
218		!x.startsWith("dropped:") &&
219		!x.includes("node_modules")
220		),
221		});
222		}
223
224		export async function finalizeServerJavaScript(
225		incr: Incremental,
226		platform: ServerPlatform,
227		) {
228		if (incr.hasArtifact("backendReplace", platform)) return;
229		const {
230		files,
231		fileWithMagicWord,
232		magicWord,
233		} = UNWRAP(incr.getArtifact("backendBundle", platform));
234
235		if (!fileWithMagicWord) return;
	176	const { metafile, outputFiles, errors, warnings } = await esbuild.build({
	177	bundle: true,
	178	chunkNames: "c.[hash]",
	179	entryNames: path.basename(entry, path.extname(entry)),
	180	entryPoints: [
	181	path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
	182	],
	183	platform: "node",
	184	format: "esm",
	185	minify: false,
	186	outdir: "out!",
	187	plugins: serverPlugins,
	188	splitting: true,
	189	logLevel: "silent",
	190	write: false,
	191	metafile: true,
	192	jsx: "automatic",
	193	jsxImportSource: "#ssr",
	194	jsxDev: false,
	195	define: {
	196	MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
	197	'globalThis.CLOVER_SERVER_ENTRY': JSON.stringify(entry),
	198	},
	199	external: Object.keys(pkg.dependencies)
	200	.filter((x) => !x.startsWith("@paperclover")),
	201	});
	202	await trackEsbuild(io, metafile)
236	203
237		// Only the reachable resources need to be inserted into the bundle.
238		const viewScriptsList = new Set(
239		Array.from(incr.out.viewMetadata.values())
240		.flatMap((view) => view.clientRefs),
241		);
242		const viewStyleKeys = Array.from(incr.out.viewMetadata.values())
243		.map((view) => css.styleKey(view.cssImports, view.theme));
244		const viewCssBundles = viewStyleKeys
245		.map((key) => UNWRAP(incr.out.style.get(key), "Style key: " + key));
	204	let fileWithMagicWord: {
	205	bytes: Buffer;
	206	basename: string;
	207	magicWord: string;
	208	} \| null = null;
	209	for (const output of outputFiles) {
	210	const basename = output.path.replace(/^.*?!(?:\/\|\\)/, "");
	211	const key = "out!/" + basename.replaceAll("\\", "/");
	212	// If this contains the generated "$views" file, then
	213	// mark this file as the one for replacement. Because
	214	// `splitting` is `true`, esbuild will not emit this
	215	// file in more than one chunk.
	216	if (magicWord && metafile.outputs[key].inputs["framework/lib/view.ts"]) {
	217	ASSERT(!fileWithMagicWord);
	218	fileWithMagicWord = {
	219	basename,
	220	bytes: Buffer.from(output.contents),
	221	magicWord,
	222	};
	223	} else {
	224	io.writeFile(basename, Buffer.from(output.contents))
	225	}
	226	}
	227	return fileWithMagicWord;
	228	}, entry)] as const);
246	229
247		// Deduplicate styles
248		const styleList = Array.from(new Set(viewCssBundles));
	230	const wProcessed = wBundles.map(async([entry, wBundle]) => {
	231	if (!await wBundle) return;
	232	await incr.work(async (io) => {
	233	// Only the reachable resources need to be read and inserted into the bundle.
	234	// This is what Map<string, incr.Ref> is for
	235	const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));
	236	const views = await Promise.all(viewRefs.map(ref => io.readWork(ref)));
249	237
250		// Replace the magic word
251		let text = files[fileWithMagicWord].toString("utf-8");
252		text = text.replace(
253		new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"),
254		(_, i) => {
255		i = Number(i);
256		// Inline the styling data
257		if (i === -2) {
258		return JSON.stringify(styleList.map((cssText) => cssText));
259		}
260		// Inline the script data
261		if (i === -1) {
262		return JSON.stringify(Object.fromEntries(incr.out.script));
263		}
264		// Reference an index into `styleList`
265		return `${styleList.indexOf(viewCssBundles[i])}`;
266		},
267		);
	238	// Client JS
	239	const scriptList = Object.entries(await io.readWork(wScriptMap));
	240	const viewScriptsList = new Set(views.flatMap(view => view.clientRefs));
	241	const neededScripts = scriptList.filter(([k]) => viewScriptsList.has(k));
268	242
269		incr.put({
270		kind: "backendReplace",
271		key: platform,
272		sources: [
273		// Backend input code (includes view code)
274		...incr.sourcesFor("backendBundle", platform),
275		// Script
276		...Array.from(viewScriptsList)
277		.flatMap((key) => incr.sourcesFor("script", hot.getScriptId(key))),
278		// Style
279		...viewStyleKeys.flatMap((key) => incr.sourcesFor("style", key)),
280		],
281		value: Buffer.from(text),
282		});
283		}
	243	// CSS
	244	const viewStyleKeys = views.map((view) => view.styleKey);
	245	const viewCssBundles = await Promise.all(
	246	viewStyleKeys.map((key) => io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key))));
	247	const styleList = Array.from(new Set(viewCssBundles));
284	248
285		function markoViaBuildCache(incr: Incremental): esbuild.Plugin {
286		return {
287		name: "marko via build cache",
288		setup(b) {
289		b.onLoad(
290		{ filter: /\.marko$/ },
291		async ({ path: file }) => {
292		const key = path.relative(hot.projectRoot, file)
293		.replaceAll("\\", "/");
294		const cacheEntry = incr.out.serverMarko.get(key);
295		if (!cacheEntry) {
296		if (!fs.existsSync(file)) {
297		console.log(`File does not exist: ${file}`);
298		}
299		throw new Error("Marko file not in cache: " + file);
	249	// Replace the magic word
	250	const text = bytes.toString("utf-8").replace(
	251	new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"),
	252	(_, i) => {
	253	i = Number(i);
	254	// Inline the styling data
	255	if (i === -2) {
	256	return JSON.stringify(styleList.map((cssText) => cssText));
	257	}
	258	// Inline the script data
	259	if (i === -1) {
	260	return JSON.stringify(Object.fromEntries(neededScripts));
300	261	}
301		return ({
302		loader: "ts",
303		contents: cacheEntry.src,
304		resolveDir: path.dirname(file),
305		});
	262	// Reference an index into `styleList`
	263	return `${styleList.indexOf(viewCssBundles[i])}`;
306	264	},
307	265	);
308		},
309		};
	266
	267	io.writeFile(basename, text);
	268	});
	269	})
	270
	271	await Promise.all(wProcessed);
310	272	}
311	273
	274
312	275	import * as esbuild from "esbuild";
313	276	import * as path from "node:path";
314	277	import process from "node:process";
315	278	import * as hot from "./hot.ts";
316		import { projectRelativeResolution, virtualFiles } from "./esbuild-support.ts";
317		import { Incremental } from "./incremental.ts";
	279	import {
	280	isIgnoredSource,
	281	markoViaBuildCache,
	282	projectRelativeResolution,
	283	virtualFiles,
	284	} from "./esbuild-support.ts";
	285	import { Io, toAbs, toRel } from "./incremental.ts";
318	286	import * as css from "./css.ts";
319	287	import * as fs from "#sitegen/fs";
320	288	import * as mime from "#sitegen/mime";
	289	import * as incr from "./incremental.ts";
	290	import * as sg from "#sitegen";import type { PreparedView } from "./generate2.ts";import { meta } from "@/file-viewer/pages/file.cotyledon_speedbump.tsx";

framework/css.ts+10-16

...	...	@@ -40,11 +40,6 @@ export function preprocess(css: string, theme: Theme): string {
40	40	);
41	41	}
42	42
43		export interface Output {
44		text: string;
45		sources: string[];
46		}
47
48	43	export function styleKey(
49	44	cssImports: string[],
50	45	theme: Theme,
...	...	@@ -60,11 +55,14 @@ export function styleKey(
60	55	}
61	56
62	57	export async function bundleCssFiles(
63		cssImports: string[],
64		theme: Theme,
65		dev: boolean = false,
66		): Promise<Output> {
67		cssImports = cssImports.map((file) => path.resolve(hot.projectSrc, file));
	58	io: Io,
	59	{ cssImports, theme, dev }: {
	60	cssImports: string[],
	61	theme: Theme,
	62	dev: boolean,
	63	}
	64	) {
	65	cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file)));
68	66	const plugin = {
69	67	name: "clover css",
70	68	setup(b) {
...	...	@@ -106,15 +104,11 @@ export async function bundleCssFiles(
106	104	throw new AggregateError(warnings, "CSS Build Failed");
107	105	}
108	106	if (outputFiles.length > 1) throw new Error("Too many output files");
109		return {
110		text: outputFiles[0].text,
111		sources: Object.keys(metafile.outputs["$input$.css"].inputs)
112		.filter((x) => !x.startsWith("vfs:")),
113		};
	107	return outputFiles[0].text;
114	108	}
115	109
116	110	import * as esbuild from "esbuild";
117	111	import * as fs from "#sitegen/fs";
118	112	import * as hot from "./hot.ts";
119	113	import * as path from "node:path";
120		import { virtualFiles } from "./esbuild-support.ts";
	114	import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts";

framework/debug.safe.ts+2-7

...	...	@@ -6,12 +6,7 @@ globalThis.UNWRAP = (t, ...args) => {
6	6	}
7	7	return t;
8	8	};
9		globalThis.ASSERT = (t, ...args) => {
10		if (!t) {
11		throw new Error(
12		args.length > 0 ? util.format(...args) : "Assertion Failed",
13		);
14		}
15		};
	9	globalThis.ASSERT = assert.ok;
16	10
17	11	import * as util from "node:util";
	12	import * as assert from 'node:assert'

framework/definitions.d.ts+1-1

...	...	@@ -1,4 +1,4 @@
1	1	declare function UNWRAP<T>(value: T \| null \| undefined, ...log: unknown[]): T;
2		declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
	2	declare function ASSERT(value: unknown, message?: string): asserts value;
3	3
4	4	type Timer = ReturnType<typeof setTimeout>;

framework/engine/ssr.ts+3-1

...	...	@@ -13,6 +13,7 @@ export function ssrSync<A extends Addons>(node: Node, addon: A = {} as A) {
13	13	const resolved = resolveNode(r, node);
14	14	return { text: renderNode(resolved), addon };
15	15	}
	16	export { ssrSync as sync };
16	17
17	18	export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
18	19	const r = initRender(true, addon);
...	...	@@ -20,7 +21,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
20	21	if (r.async === 0) {
21	22	return Promise.resolve({ text: renderNode(resolved), addon });
22	23	}
23		const { resolve, reject, promise } = Promise.withResolvers<Result>();
	24	const { resolve, reject, promise } = Promise.withResolvers<Result<A>>();
24	25	r.asyncDone = () => {
25	26	const rejections = r.rejections;
26	27	if (!rejections) return resolve({ text: renderNode(resolved), addon });
...	...	@@ -29,6 +30,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
29	30	};
30	31	return promise;
31	32	}
	33	export { ssrAsync as async };
32	34
33	35	/** Inline HTML into a render without escaping it */
34	36	export function html(rawText: ResolvedNode): DirectHtml {

framework/esbuild-support.ts+42-4

...	...	@@ -1,5 +1,7 @@
	1	type Awaitable<T> = T \| Promise<T>;
	2
1	3	export function virtualFiles(
2		map: Record<string, string \| esbuild.OnLoadResult>,
	4	map: Record<string, string \| esbuild.OnLoadResult \| (() => Awaitable<string \| esbuild.OnLoadResult>)>,
3	5	) {
4	6	return {
5	7	name: "clover vfs",
...	...	@@ -18,8 +20,9 @@ export function virtualFiles(
18	20	);
19	21	b.onLoad(
20	22	{ filter: /./, namespace: "vfs" },
21		({ path }) => {
22		const entry = map[path];
	23	async ({ path }) => {
	24	let entry = map[path];
	25	if (typeof entry === 'function') entry = await entry();
23	26	return ({
24	27	resolveDir: ".",
25	28	loader: "ts",
...	...	@@ -73,7 +76,42 @@ export function projectRelativeResolution(root = process.cwd() + "/src") {
73	76	} satisfies esbuild.Plugin;
74	77	}
75	78
	79	export function markoViaBuildCache(): esbuild.Plugin {
	80	return {
	81	name: "marko via build cache",
	82	setup(b) {
	83	b.onLoad(
	84	{ filter: /\.marko$/ },
	85	async ({ path: file }) => {
	86	const cacheEntry = markoCache.get(file);
	87	if (!cacheEntry) {
	88	if (!fs.existsSync(file)) {
	89	console.warn(`File does not exist: ${file}`);
	90	}
	91	console.log(markoCache.keys());
	92	throw new Error("Marko file not in cache: " + file);
	93	}
	94	return ({
	95	loader: "ts",
	96	contents: cacheEntry.src,
	97	resolveDir: path.dirname(file),
	98	});
	99	},
	100	);
	101	},
	102	};
	103	}
	104
	105	export function isIgnoredSource(source: string) {
	106	return source.includes("<define:") \|\|
	107	source.startsWith("vfs:") \|\|
	108	source.startsWith("dropped:") \|\|
	109	source.includes("node_modules")
	110	}
	111
76	112	import * as esbuild from "esbuild";
77	113	import * as string from "#sitegen/string";
78	114	import * as path from "node:path";
79		import * as hot from "./hot.ts";
	115	import * as fs from "#sitegen/fs";
	116	import * as incr from "./incremental.ts";
	117	import * as hot from "./hot.ts";import { markoCache } from "./marko.ts";

framework/generate.ts+280-399

...	...	@@ -1,51 +1,99 @@
1	1	// This file contains the main site generator build process.
2		// By using `Incremental`'s ability to automatically purge stale
3		// assets, the `sitegen` function performs partial rebuilds.
	2	// By using `incr.work`'s ability to cache work between runs,
	3	// the site generator is very fast to re-run.
	4	//
	5	// See `watch.ts` for a live development environment.
	6	const { toRel, toAbs } = incr;
	7	const globalCssPath = toAbs("src/global.css");
4	8
5		export function main() {
6		return withSpinner<Record<string, unknown>, any>({
7		text: "Recovering State",
8		successText,
9		failureText: () => "sitegen FAIL",
10		}, async (spinner) => {
11		// const incr = Incremental.fromDisk();
12		// await incr.statAllFiles();
13		const incr = new Incremental();
14		const result = await sitegen(spinner, incr);
15		incr.toDisk(); // Allows picking up this state again
16		return result;
17		}) as ReturnType<typeof sitegen>;
	9	export async function main() {
	10	await incr.restore();
	11	await incr.compile(generate);
18	12	}
19	13
20		export function successText({
21		elapsed,
22		inserted,
23		referenced,
24		unreferenced,
25		}: Awaited<ReturnType<typeof sitegen>>) {
26		const s = (array: unknown[]) => array.length === 1 ? "" : "s";
27		const kind = inserted.length === referenced.length ? "build" : "update";
28		const status = inserted.length > 0
29		? `${kind} ${inserted.length} key${s(inserted)}`
30		: unreferenced.length > 0
31		? `pruned ${unreferenced.length} key${s(unreferenced)}`
32		: `checked ${referenced.length} key${s(referenced)}`;
33		return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
34		}
	14	export async function generate() {
	15	// -- read config and discover files --
	16	const siteConfig = await incr.work(readManifest);
	17	const {
	18	staticFiles,
	19	scripts,
	20	views,
	21	pages,
	22	} = await discoverAllFiles(siteConfig);
35	23
36		export async function sitegen(
37		status: Spinner,
38		incr: Incremental,
39		) {
40		const startTime = performance.now();
	24	// TODO: make sure that `static` and `pages` does not overlap
	25
	26	// TODO: loadMarkoCache
	27
	28	// -- perform build-time rendering --
	29	const builtPages = pages.map((item) => incr.work(preparePage, item));
	30	const builtViews = views.map((item) => incr.work(prepareView, item));
	31	const builtStaticFiles = Promise.all((staticFiles.map((item) =>
	32	incr.work(
	33	async (io, { id, file }) => void await io.writeAsset(id, await io.readFile(file)),
	34	item,
	35	)
	36	)));
	37	const routes = await Promise.all([...builtViews, ...builtPages]);
	38
	39	// -- page resources --
	40	const scriptMap = incr.work(bundle.bundleClientJavaScript, {
	41	clientRefs: routes.flatMap((x) => x.clientRefs),
	42	extraPublicScripts: scripts.map((entry) => entry.file),
	43	dev: false,
	44	});
	45	const styleMap = prepareInlineCss(routes);
	46
	47	// -- backend --
	48	const builtBackend = bundle.bundleServerJavaScript({
	49	entries: siteConfig.backends,
	50	platform: 'node',
	51	styleMap,
	52	scriptMap,
	53	viewItems: views,
	54	viewRefs: builtViews,
	55	})
	56
	57	// -- assemble page assets --
	58	const pAssemblePages = builtPages.map((page) =>
	59	assembleAndWritePage(page, styleMap, scriptMap)
	60	);
	61
	62	await Promise.all([
	63	builtBackend,
	64	builtStaticFiles,
	65	...pAssemblePages,
	66	]);
	67	}
41	68
42		let root = path.resolve(import.meta.dirname, "../src");
43		const join = (...sub: string[]) => path.join(root, ...sub);
	69	export async function readManifest(io: Io) {
	70	const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
	71	return {
	72	siteSections: cfg.siteSections.map((section) => ({
	73	root: toRel(section.root),
	74	})),
	75	backends: cfg.backends.map(toRel),
	76	};
	77	}
44	78
45		// Sitegen reviews every defined section for resources to process
46		const sections: sg.Section[] =
47		require(path.join(root, "site.ts")).siteSections;
	79	export async function discoverAllFiles(
	80	siteConfig: Awaited<ReturnType<typeof readManifest>>,
	81	) {
	82	return (
	83	await Promise.all(
	84	siteConfig.siteSections.map(({ root: sectionRoot }) =>
	85	incr.work(scanSiteSection, toAbs(sectionRoot))
	86	),
	87	)
	88	).reduce((acc, next) => ({
	89	staticFiles: acc.staticFiles.concat(next.staticFiles),
	90	pages: acc.pages.concat(next.pages),
	91	views: acc.views.concat(next.views),
	92	scripts: acc.scripts.concat(next.scripts),
	93	}));
	94	}
48	95
	96	export async function scanSiteSection(io: Io, sectionRoot: string) {
49	97	// Static files are compressed and served as-is.
50	98	// - "{section}/static/*.png"
51	99	let staticFiles: FileItem[] = [];
...	...	@@ -61,394 +109,227 @@ export async function sitegen(
61	109	// - "{section}/scripts/*.client.ts"
62	110	let scripts: FileItem[] = [];
63	111
64		// -- Scan for files --
65		status.text = "Scanning Project";
66		for (const section of sections) {
67		const { root: sectionRoot } = section;
68		const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
69		const rootPrefix = root === sectionRoot
70		? ""
71		: path.relative(root, sectionRoot) + "/";
72		const kinds = [
73		{
74		dir: sectionPath("pages"),
75		list: pages,
76		prefix: "/",
77		include: [".tsx", ".mdx", ".marko"],
78		exclude: [".client.ts", ".client.tsx"],
79		},
80		{
81		dir: sectionPath("static"),
82		list: staticFiles,
83		prefix: "/",
84		ext: true,
85		},
86		{
87		dir: sectionPath("scripts"),
88		list: scripts,
89		prefix: rootPrefix,
90		include: [".client.ts", ".client.tsx"],
91		},
92		{
93		dir: sectionPath("views"),
94		list: views,
95		prefix: rootPrefix,
96		include: [".tsx", ".mdx", ".marko"],
97		exclude: [".client.ts", ".client.tsx"],
98		},
99		];
100		for (
101		const { dir, list, prefix, include = [""], exclude = [], ext = false }
102		of kinds
103		) {
104		const items = fs.readDirRecOptionalSync(dir);
105		for (const subPath of items) {
106		const file = path.join(dir, subPath);
107		const stat = fs.statSync(file);
108		if (stat.isDirectory()) continue;
109		if (!include.some((e) => subPath.endsWith(e))) continue;
110		if (exclude.some((e) => subPath.endsWith(e))) continue;
111		const trim = ext
112		? subPath
113		: subPath.slice(0, -path.extname(subPath).length).replaceAll(
114		".",
115		"/",
116		);
117		let id = prefix + trim.replaceAll("\\", "/");
118		if (prefix === "/" && id.endsWith("/index")) {
119		id = id.slice(0, -"/index".length) \|\| "/";
120		}
121		list.push({ id, file: file });
122		}
123		}
124		}
125		const globalCssPath = join("global.css");
126
127		// TODO: make sure that `static` and `pages` does not overlap
128
129		// -- inline style sheets, used and shared by pages and views --
130		status.text = "Building";
131		const cssOnce = new OnceMap();
132		const cssQueue = new Queue({
133		name: "Bundle",
134		async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
135		const { text, sources } = await css.bundleCssFiles(files, theme);
136		incr.put({
137		kind: "style",
138		key,
139		sources,
140		value: text,
141		});
	112	const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
	113	const rootPrefix = hot.projectSrc === sectionRoot
	114	? ""
	115	: path.relative(hot.projectSrc, sectionRoot) + "/";
	116	const kinds = [
	117	{
	118	dir: sectionPath("pages"),
	119	list: pages,
	120	prefix: "/",
	121	include: [".tsx", ".mdx", ".marko"],
	122	exclude: [".client.ts", ".client.tsx"],
142	123	},
143		passive: true,
144		getItemText: ([id]) => id,
145		maxJobs: 2,
146		});
147		function ensureCssGetsBuilt(
148		cssImports: string[],
149		theme: css.Theme,
150		referrer: string,
151		) {
152		const key = css.styleKey(cssImports, theme);
153		cssOnce.get(
154		key,
155		async () => {
156		incr.getArtifact("style", key) ??
157		await cssQueue.add([referrer, key, cssImports, theme]);
158		},
159		);
160		}
	124	{
	125	dir: sectionPath("static"),
	126	list: staticFiles,
	127	prefix: "/",
	128	ext: true,
	129	},
	130	{
	131	dir: sectionPath("scripts"),
	132	list: scripts,
	133	prefix: rootPrefix,
	134	include: [".client.ts", ".client.tsx"],
	135	},
	136	{
	137	dir: sectionPath("views"),
	138	list: views,
	139	prefix: rootPrefix,
	140	include: [".tsx", ".mdx", ".marko"],
	141	exclude: [".client.ts", ".client.tsx"],
	142	},
	143	];
	144	for (const kind of kinds) {
	145	const {
	146	dir,
	147	list,
	148	prefix,
	149	include = [""],
	150	exclude = [],
	151	ext = false,
	152	} = kind;
161	153
162		// -- server side render pages --
163		async function loadPageModule({ file }: FileItem) {
164		require(file);
165		}
166		async function renderPage(item: FileItem) {
167		// -- load and validate module --
168		let {
169		default: Page,
170		meta: metadata,
171		theme: pageTheme,
172		layout,
173		} = require(item.file);
174		if (!Page) {
175		throw new Error("Page is missing a 'default' export.");
	154	let items;
	155	try {
	156	items = await io.readDirRecursive(dir);
	157	} catch (err: any) {
	158	if (err.code === "ENOENT") continue;
	159	throw err;
176	160	}
177		if (!metadata) {
178		throw new Error("Page is missing 'meta' export with a title.");
	161	for (const subPath of items) {
	162	const file = path.join(dir, subPath);
	163	const stat = fs.statSync(file);
	164	if (stat.isDirectory()) continue;
	165	if (!include.some((e) => subPath.endsWith(e))) continue;
	166	if (exclude.some((e) => subPath.endsWith(e))) continue;
	167	const trim = ext
	168	? subPath
	169	: subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
	170	let id = prefix + trim.replaceAll("\\", "/");
	171	if (prefix === "/" && id.endsWith("/index")) {
	172	id = id.slice(0, -"/index".length) \|\| "/";
	173	}
	174	list.push({ id, file: path.relative(hot.projectRoot, file) });
179	175	}
	176	}
180	177
181		// -- css --
182		if (layout?.theme) pageTheme = layout.theme;
183		const theme: css.Theme = {
184		...css.defaultTheme,
185		...pageTheme,
186		};
187		const cssImports = Array.from(
188		new Set([globalCssPath, ...hot.getCssImports(item.file)]),
189		(file) => path.relative(hot.projectSrc, file),
190		);
191		ensureCssGetsBuilt(cssImports, theme, item.id);
192
193		// -- metadata --
194		const renderedMetaPromise = Promise.resolve(
195		typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
196		).then((m) => meta.renderMeta(m));
197
198		// -- html --
199		let page = [engine.kElement, Page, {}];
200		if (layout?.default) {
201		page = [engine.kElement, layout.default, { children: page }];
202		}
203		const bodyPromise = engine.ssrAsync(page, {
204		sitegen: sg.initRender(),
205		});
	178	return { staticFiles, pages, views, scripts };
	179	}
206	180
207		const [{ text, addon }, renderedMeta] = await Promise.all([
208		bodyPromise,
209		renderedMetaPromise,
210		]);
211		if (!renderedMeta.includes("<title>")) {
212		throw new Error(
213		"Page is missing 'meta.title'. " +
214		"All pages need a title tag.",
215		);
216		}
217		incr.put({
218		kind: "pageMetadata",
219		key: item.id,
220		// Incremental integrates with `hot.ts` + `require`
221		// to trace all the needed source files here.
222		sources: [item.file],
223		value: {
224		html: text,
225		meta: renderedMeta,
226		cssImports,
227		theme: theme ?? null,
228		clientRefs: Array.from(addon.sitegen.scripts),
229		},
230		});
	181	export async function preparePage(io: Io, item: sg.FileItem) {
	182	// -- load and validate module --
	183	let {
	184	default: Page,
	185	meta: metadata,
	186	theme: pageTheme,
	187	layout,
	188	} = await io.import<any>(item.file);
	189	if (!Page) {
	190	throw new Error("Page is missing a 'default' export.");
231	191	}
232		async function prepareView(item: FileItem) {
233		const module = require(item.file);
234		if (!module.meta) {
235		throw new Error(`${item.file} is missing 'export const meta'`);
236		}
237		if (!module.default) {
238		throw new Error(`${item.file} is missing a default export.`);
239		}
240		const pageTheme = module.layout?.theme ?? module.theme;
241		const theme: css.Theme = {
242		...css.defaultTheme,
243		...pageTheme,
244		};
245		const cssImports = Array.from(
246		new Set([globalCssPath, ...hot.getCssImports(item.file)]),
247		(file) => path.relative(hot.projectSrc, file),
248		);
249		ensureCssGetsBuilt(cssImports, theme, item.id);
250		incr.put({
251		kind: "viewMetadata",
252		key: item.id,
253		sources: [item.file],
254		value: {
255		file: path.relative(hot.projectRoot, item.file),
256		cssImports,
257		theme,
258		clientRefs: hot.getClientScriptRefs(item.file),
259		hasLayout: !!module.layout?.default,
260		},
261		});
	192	if (!metadata) {
	193	throw new Error("Page is missing 'meta' export with a title.");
262	194	}
263	195
264		// Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
265		// required so that it's (1) re-built if needed, (2) not pruned from build.
266		const neededPages = pages.filter((page) => {
267		const existing = incr.getArtifact("pageMetadata", page.id);
268		if (existing) {
269		const { cssImports, theme } = existing;
270		ensureCssGetsBuilt(cssImports, theme, page.id);
271		}
272		return !existing;
273		});
274		const neededViews = views.filter((view) => {
275		const existing = incr.getArtifact("viewMetadata", view.id);
276		if (existing) {
277		const { cssImports, theme } = existing;
278		ensureCssGetsBuilt(cssImports, theme, view.id);
279		}
280		return !existing;
281		});
	196	// -- css --
	197	if (layout?.theme) pageTheme = layout.theme;
	198	const theme: css.Theme = {
	199	...css.defaultTheme,
	200	...pageTheme,
	201	};
	202	const cssImports = Array.from(
	203	new Set([globalCssPath, ...hot.getCssImports(item.file)]),
	204	(file) => path.relative(hot.projectSrc, file),
	205	);
282	206
283		// Load the marko cache before render modules are loaded
284		incr.loadMarkoCache();
	207	// -- metadata --
	208	const renderedMetaPromise = Promise.resolve(
	209	typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
	210	).then((m) => meta.renderMeta(m));
285	211
286		// This is done in two passes so that a page that throws during evaluation
287		// will report "Load Render Module" instead of "Render Static Page".
288		const spinnerFormat = status.format;
289		status.format = () => "";
290		const moduleLoadQueue = new Queue({
291		name: "Load Render Module",
292		fn: loadPageModule,
293		getItemText,
294		maxJobs: 1,
295		});
296		moduleLoadQueue.addMany(neededPages);
297		moduleLoadQueue.addMany(neededViews);
298		await moduleLoadQueue.done({ method: "stop" });
299		const pageQueue = new Queue({
300		name: "Render Static Page",
301		fn: renderPage,
302		getItemText,
303		maxJobs: 2,
304		});
305		pageQueue.addMany(neededPages);
306		const viewQueue = new Queue({
307		name: "Build Dynamic View",
308		fn: prepareView,
309		getItemText,
310		maxJobs: 2,
	212	// -- html --
	213	let page = [engine.kElement, Page, {}];
	214	if (layout?.default) {
	215	page = [engine.kElement, layout.default, { children: page }];
	216	}
	217	const bodyPromise = engine.ssrAsync(page, {
	218	sitegen: sg.initRender(),
311	219	});
312		viewQueue.addMany(neededViews);
313		const pageAndViews = [
314		pageQueue.done({ method: "stop" }),
315		viewQueue.done({ method: "stop" }),
316		];
317		await Promise.allSettled(pageAndViews);
318		await Promise.all(pageAndViews);
319		status.format = spinnerFormat;
320
321		// -- bundle server javascript (backend and views) --
322		status.text = "Bundle JavaScript";
323		incr.snapshotMarkoCache();
324		const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
325	220
326		// -- bundle client javascript --
327		const referencedScripts = Array.from(
328		new Set(
329		[
330		...pages.map((item) =>
331		UNWRAP(
332		incr.getArtifact("pageMetadata", item.id),
333		`Missing pageMetadata ${item.id}`,
334		)
335		),
336		...views.map((item) =>
337		UNWRAP(
338		incr.getArtifact("viewMetadata", item.id),
339		`Missing viewMetadata ${item.id}`,
340		)
341		),
342		].flatMap((item) => item.clientRefs),
343		),
344		(script) => path.resolve(hot.projectSrc, script),
345		).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
346		const extraPublicScripts = scripts.map((entry) => entry.file);
347		const clientJavaScriptPromise = bundle.bundleClientJavaScript(
348		referencedScripts,
349		extraPublicScripts,
350		incr,
351		);
352		await Promise.all([
353		serverJavaScriptPromise,
354		clientJavaScriptPromise,
355		cssQueue.done({ method: "stop" }),
	221	const [{ text, addon }, renderedMeta] = await Promise.all([
	222	bodyPromise,
	223	renderedMetaPromise,
356	224	]);
357		await bundle.finalizeServerJavaScript(incr, "node");
	225	if (!renderedMeta.includes("<title>")) {
	226	throw new Error(
	227	"Page is missing 'meta.title'. " + "All pages need a title tag.",
	228	);
	229	}
	230	const styleKey = css.styleKey(cssImports, theme);
	231	return {
	232	id: item.id,
	233	html: text,
	234	meta: renderedMeta,
	235	cssImports,
	236	theme: theme ?? null,
	237	styleKey,
	238	clientRefs: Array.from(addon.sitegen.scripts),
	239	};
	240	}
358	241
359		// -- copy/compress static files --
360		async function doStaticFile(item: FileItem) {
361		const body = await fs.readFile(item.file);
362		await incr.putAsset({
363		sources: [item.file],
364		key: item.id,
365		body,
366		});
	242	export async function prepareView(io: Io, item: sg.FileItem) {
	243	const module = await io.import<any>(item.file);
	244	if (!module.meta) {
	245	throw new Error(`${item.file} is missing 'export const meta'`);
367	246	}
368		const staticQueue = new Queue({
369		name: "Load Static",
370		fn: doStaticFile,
371		getItemText,
372		maxJobs: 16,
373		});
374		status.format = () => "";
375		staticQueue.addMany(
376		staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
	247	if (!module.default) {
	248	throw new Error(`${item.file} is missing a default export.`);
	249	}
	250	const pageTheme = module.layout?.theme ?? module.theme;
	251	const theme: css.Theme = {
	252	...css.defaultTheme,
	253	...pageTheme,
	254	};
	255	const cssImports = Array.from(
	256	new Set([globalCssPath, ...hot.getCssImports(item.file)]),
	257	(file) => path.relative(hot.projectSrc, file),
377	258	);
378		await staticQueue.done({ method: "stop" });
379		status.format = spinnerFormat;
	259	const styleKey = css.styleKey(cssImports, theme);
	260	return {
	261	file: path.relative(hot.projectRoot, item.file),
	262	cssImports,
	263	theme,
	264	clientRefs: hot.getClientScriptRefs(item.file),
	265	hasLayout: !!module.layout?.default,
	266	styleKey,
	267	};
	268	}
	269	export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
380	270
381		// -- concatenate static rendered pages --
382		status.text = `Concat Pages`;
383		await Promise.all(pages.map(async (page) => {
384		if (incr.hasArtifact("asset", page.id)) return;
385		const {
386		html,
387		meta,
388		cssImports,
389		theme,
390		clientRefs,
391		} = UNWRAP(incr.out.pageMetadata.get(page.id));
392		const scriptIds = clientRefs.map(hot.getScriptId);
393		const styleKey = css.styleKey(cssImports, theme);
394		const style = UNWRAP(
395		incr.out.style.get(styleKey),
396		`Missing style ${styleKey}`,
	271	export function prepareInlineCss(
	272	items: Array<{
	273	styleKey: string;
	274	cssImports: string[];
	275	theme: css.Theme;
	276	}>,
	277	) {
	278	const map = new Map<string, incr.Ref<string>>();
	279	for (const { styleKey, cssImports, theme } of items) {
	280	if (map.has(styleKey)) continue;
	281	map.set(
	282	styleKey,
	283	incr.work(css.bundleCssFiles, {
	284	cssImports,
	285	theme,
	286	dev: false,
	287	}),
397	288	);
398		const doc = wrapDocument({
399		body: html,
400		head: meta,
401		inlineCss: style,
402		scripts: scriptIds.map(
403		(ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
404		).map((x) => `{${x}}`).join("\n"),
405		});
406		await incr.putAsset({
407		sources: [
408		page.file,
409		...incr.sourcesFor("style", styleKey),
410		...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
411		],
412		key: page.id,
413		body: doc,
414		headers: {
415		"Content-Type": "text/html",
416		},
417		});
418		}));
419		status.format = () => "";
420		status.text = ``;
421		// This will wait for all compression jobs to finish, which up
422		// to this point have been left as dangling promises.
423		await incr.wait();
	289	}
	290	return map;
	291	}
424	292
425		const { inserted, referenced, unreferenced } = incr.shake();
	293	export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
	294	export async function assembleAndWritePage(
	295	pageWork: incr.Ref<PreparedPage>,
	296	styleMap: Map<string, incr.Ref<string>>,
	297	scriptWork: incr.Ref<Record<string, string>>,
	298	) {
	299	const page = await pageWork;
	300	return incr.work(
	301	async (io, { id, html, meta, styleKey, clientRefs }) => {
	302	const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
426	303
427		// Flush the site to disk.
428		status.format = spinnerFormat;
429		status.text = `Incremental Flush`;
430		incr.flush("node"); // Write outputs
431		return {
432		incr,
433		inserted,
434		referenced,
435		unreferenced,
436		elapsed: (performance.now() - startTime) / 1000,
437		};
438		}
	304	const scriptIds = clientRefs.map(hot.getScriptId);
	305	const scriptMap = await io.readWork(scriptWork);
	306	const scripts = scriptIds.map((ref) =>
	307	UNWRAP(scriptMap[ref], `Missing script ${ref}`)
	308	)
	309	.map((x) => `{${x}}`).join("\n");
439	310
440		function getItemText({ file }: FileItem) {
441		return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
	311	const doc = wrapDocument({
	312	body: html,
	313	head: meta,
	314	inlineCss,
	315	scripts,
	316	});
	317	await io.writeAsset(id, doc, {
	318	"Content-Type": "text/html",
	319	});
	320	},
	321	page,
	322	);
442	323	}
443	324
444		import { OnceMap, Queue } from "#sitegen/async";
445		import { Incremental } from "./incremental.ts";
	325	import * as sg from "#sitegen";
	326	import * as incr from "./incremental.ts";
	327	import { Io } from "./incremental.ts";
446	328	import * as bundle from "./bundle.ts";
447	329	import * as css from "./css.ts";
448	330	import * as engine from "./engine/ssr.ts";
449	331	import * as hot from "./hot.ts";
450	332	import * as fs from "#sitegen/fs";
451		import * as sg from "#sitegen";
452	333	import type { FileItem } from "#sitegen";
453	334	import * as path from "node:path";
454	335	import * as meta from "#sitegen/meta";

framework/hot.ts+24-50

...	...	@@ -19,6 +19,8 @@ export const load = createRequire(
19	19	};
20	20	export const { cache } = load;
21	21
	22	load<any>("source-map-support").install({hookRequire: true});
	23
22	24	// Register extensions by overwriting `require.extensions`
23	25	const require = load;
24	26	const exts = require.extensions;
...	...	@@ -42,8 +44,7 @@ export function getFileStat(filepath: string) {
42	44	}
43	45
44	46	function shouldTrackPath(filename: string) {
45		return !filename.includes("node_modules") &&
46		!filename.includes(import.meta.dirname);
	47	return !filename.includes("node_modules");
47	48	}
48	49
49	50	const Module = load<typeof import("node:module")>("node:module");
...	...	@@ -59,11 +60,12 @@ Module.prototype._compile = function (
59	60	filename,
60	61	format,
61	62	);
62		const stat = fs.statSync(filename);
63	63	if (shouldTrackPath(filename)) {
	64	const stat = fs.statSync(filename);
64	65	const cssImportsMaybe: string[] = [];
65	66	const imports: string[] = [];
66		for (const { filename: file, cloverClientRefs } of this.children) {
	67	for (const childModule of this.children) {
	68	const { filename: file, cloverClientRefs } = childModule;
67	69	if (file.endsWith(".css")) cssImportsMaybe.push(file);
68	70	else {
69	71	const child = fileStats.get(file);
...	...	@@ -71,6 +73,7 @@ Module.prototype._compile = function (
71	73	const { cssImportsRecursive } = child;
72	74	if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
73	75	imports.push(file);
	76	(childModule.cloverImporters ??= []).push(this);
74	77	if (cloverClientRefs && cloverClientRefs.length > 0) {
75	78	(this.cloverClientRefs ??= [])
76	79	.push(...cloverClientRefs);
...	...	@@ -82,7 +85,7 @@ Module.prototype._compile = function (
82	85	? Array.from(new Set(cssImportsMaybe))
83	86	: null,
84	87	imports,
85		lastModified: stat.mtimeMs,
	88	lastModified: Math.floor(stat.mtimeMs),
86	89	});
87	90	}
88	91	return result;
...	...	@@ -113,7 +116,7 @@ function loadEsbuild(module: NodeJS.Module, filepath: string) {
113	116	interface LoadOptions {
114	117	scannedClientRefs?: string[];
115	118	}
116		function loadEsbuildCode(
	119	export function loadEsbuildCode(
117	120	module: NodeJS.Module,
118	121	filepath: string,
119	122	src: string,
...	...	@@ -139,7 +142,7 @@ function loadEsbuildCode(
139	142	import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
140	143	import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
141	144	import.meta.filename = ${JSON.stringify(filepath)};
142		`.trim().replace(/\n/g, "") + src;
	145	`.trim().replace(/[\n\s]/g, "") + src;
143	146	}
144	147	src = esbuild.transformSync(src, {
145	148	loader,
...	...	@@ -149,11 +152,12 @@ function loadEsbuildCode(
149	152	jsxImportSource: "#ssr",
150	153	jsxDev: true,
151	154	sourcefile: filepath,
	155	sourcemap: 'inline',
152	156	}).code;
153	157	return module._compile(src, filepath, "commonjs");
154	158	}
155	159
156		function resolveClientRef(sourcePath: string, ref: string) {
	160	export function resolveClientRef(sourcePath: string, ref: string) {
157	161	const filePath = resolveFrom(sourcePath, ref);
158	162	if (
159	163	!filePath.endsWith(".client.ts") &&
...	...	@@ -164,44 +168,10 @@ function resolveClientRef(sourcePath: string, ref: string) {
164	168	return path.relative(projectSrc, filePath);
165	169	}
166	170
167		// TODO: extract the marko compilation tools out, lazy load them
168		export interface MarkoCacheEntry {
169		src: string;
170		scannedClientRefs: string[];
171		}
172		export const markoCache = new Map<string, MarkoCacheEntry>();
	171	let lazyMarko: typeof import('./marko.ts') \| null = null;
173	172	function loadMarko(module: NodeJS.Module, filepath: string) {
174		let cache = markoCache.get(filepath);
175		if (!cache) {
176		let src = fs.readFileSync(filepath, "utf8");
177		// A non-standard thing here is Clover Sitegen implements
178		// its own client side scripting stuff, so it overrides
179		// bare client import statements to it's own usage.
180		const scannedClientRefs = new Set<string>();
181		if (src.match(/^\s*client\s+import\s+["']/m)) {
182		src = src.replace(
183		/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
184		(_, src) => {
185		const ref = JSON.parse(`"${src.slice(1, -1)}"`);
186		const resolved = resolveClientRef(filepath, ref);
187		scannedClientRefs.add(resolved);
188		return `<CloverScriptInclude=${
189		JSON.stringify(getScriptId(resolved))
190		} />`;
191		},
192		) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
193		}
194
195		src = marko.compileSync(src, filepath).code;
196		src = src.replace("marko/debug/html", "#ssr/marko");
197		cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
198		markoCache.set(filepath, cache);
199		}
200
201		const { src, scannedClientRefs } = cache;
202		return loadEsbuildCode(module, filepath, src, {
203		scannedClientRefs,
204		});
	173	lazyMarko ??= require<typeof import('./marko.ts')>("./framework/marko.ts");
	174	lazyMarko.loadMarko(module, filepath);
205	175	}
206	176
207	177	function loadMdx(module: NodeJS.Module, filepath: string) {
...	...	@@ -224,10 +194,14 @@ export function reloadRecursive(filepath: string) {
224	194	}
225	195
226	196	export function unload(filepath: string) {
	197	lazyMarko?.markoCache.delete(filepath)
227	198	filepath = path.resolve(filepath);
228		const existing = cache[filepath];
229		if (existing) delete cache[filepath];
230		fileStats.delete(filepath);
	199	const module = cache[filepath];
	200	if (!module) return;
	201	delete cache[filepath];
	202	for (const importer of module.cloverImporters ?? []) {
	203	unload(importer.filename);
	204	}
231	205	}
232	206
233	207	function deleteRecursiveInner(id: string, module: any) {
...	...	@@ -326,6 +300,7 @@ declare global {
326	300	interface Module {
327	301	cloverClientRefs?: string[];
328	302	cloverSourceCode?: string;
	303	cloverImporters?: Module[],
329	304
330	305	_compile(
331	306	this: NodeJS.Module,
...	...	@@ -343,11 +318,10 @@ declare module "node:module" {
343	318	): unknown;
344	319	}
345	320
346		import * as fs from "./lib/fs.ts";
	321	import * as fs from "#sitegen/fs";
347	322	import * as path from "node:path";
348	323	import { pathToFileURL } from "node:url";
349	324	import * as esbuild from "esbuild";
350		import * as marko from "@marko/compiler";
351	325	import { createRequire } from "node:module";
352	326	import * as mdx from "@mdx-js/mdx";
353	327	import * as self from "./hot.ts";

framework/incremental.test.ts created+56

...	...	@@ -0,0 +1,56 @@
	1	test("trivial case", async () => {
	2	incr.reset();
	3
	4	const file1 = tmpFile("example.txt");
	5	file1.write("one");
	6
	7	async function compilation() {
	8	const first = incr.work({
	9	label: "first compute",
	10	async run (io) {
	11	await setTimeout(1000);
	12	const contents = await io.readFile(file1.path);
	13	return [contents, Math.random()] as const;
	14	}
	15	});
	16	const second = incr.work({
	17	label: "second compute",
	18	wait: first,
	19	async run (io) {
	20	await setTimeout(1000);
	21	return io.readWork(first)[0].toUpperCase();
	22	}
	23	});
	24	const third = incr.work({
	25	label: "third compute",
	26	wait: first,
	27	async run (io) {
	28	await setTimeout(1000);
	29	return io.readWork(first)[1] * 1000;
	30	}
	31	});
	32	return incr.work({
	33	label: "last compute",
	34	wait: [second, third],
	35	async run (io) {
	36	await setTimeout(1000);
	37	return {
	38	second: io.readWork(second),
	39	third: io.readWork(third),
	40	}
	41	}
	42	});
	43	}
	44	const { value: first } = await incr.compile(compilation);
	45	const { value: second } = await incr.compile(compilation);
	46	ASSERT(first === second);
	47	incr.forceInvalidate(file1.path);
	48	const { value: third } = await incr.compile(compilation);
	49	ASSERT(first !== third);
	50	ASSERT(first[0] === third[0]);
	51	});
	52
	53	import * as incr from "./incremental2.ts";
	54	import { beforeEach, test } from "node:test";
	55	import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises";
	56

framework/incremental.ts+603-577

...	...	@@ -1,592 +1,575 @@
1		// Incremental contains multiple maps for the different kinds
2		// of Artifact, which contain a list of source files which
3		// were used to produce it. When files change, Incremental sees
4		// that the `mtime` is newer, and purges the referenced artifacts.
5
6		type SourceId = string; // relative to project root, e.g. 'src/global.css'
7		type ArtifactId = string; // `${ArtifactType}\0${string}`
8		type Sha1Id = string; // Sha1 hex string
9
10		// -- artifact types --
11		interface ArtifactMap {
12		/* An asset (serve with "#sitegen/asset" */
13		asset: Asset;
14		/* The bundled text of a '.client.ts' script */
15		// TODO: track imports this has into `asset`
16		script: string;
17		/* The bundled style tag contents. Keyed by 'css.styleKey' */
18		style: string;
19		/* Metadata about a static page */
20		pageMetadata: PageMetadata;
21		/* Metadata about a dynamic view */
22		viewMetadata: ViewMetadata;
23		/* Cached '.marko' server compilation */
24		serverMarko: hot.MarkoCacheEntry;
25		/* Backend source code, pre-replacement. Keyed by platform type. */
26		backendBundle: BackendBundle;
27		/* One file in the backend receives post-processing. */
28		backendReplace: Buffer;
29		}
30		type ArtifactKind = keyof ArtifactMap;
31		/* Automatic path tracing is performed to make it so that
32		* specifying 'sources: [file]' refers to it and everything it imports.
33		* These kinds do not have that behavior
34		*/
35		const exactDependencyKinds = ["serverMarko"];
36		export interface Asset {
37		buffer: Buffer;
38		headers: Record<string, string \| undefined>;
39		hash: string;
40		}
41		/**
42		* This interface intentionally omits the contents
43		* of its scripts and styles for fine-grained rebuilds.
44		*/
45		export interface PageMetadata {
46		html: string;
47		meta: string;
48		cssImports: string[];
49		theme: css.Theme;
50		clientRefs: string[];
	1	// Incremental build system using singleton state.
	2	// See `work()`, `compile()`, and `invalidate()` for details.
	3	//
	4	// All state is serializable to allow recovering state across sessions.
	5	// This framework special-cases the asset map, but is otherwise
	6	// agnostic of what it is a compiler for.
	7	let running = false;
	8	let jobs = 0;
	9	let newKeys = 0;
	10	let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
	11	let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
	12	let works = new Map<string, Work>();
	13	let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
	14	let writes = new Map<string, FileWrite>();
	15	let assets = new Map<string, Asset>(); // keyed by hash
	16
	17	export interface Ref<T> {
	18	/** This method is compatible with `await` syntax */
	19	then(
	20	onFulfilled: (value: T) => void,
	21	onRejected: (error: unknown) => void,
	22	): void;
	23	key: string;
51	24	}
	25	type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
	26
52	27	/**
53		* Like a page, this intentionally omits resources,
54		* but additionally omits the bundled server code.
	28	* Declare and a unit of work. Return value is memoized and
	29	* only rebuilt when inputs (declared via `Io`) change. Outputs
	30	* are written at the end of a compilation (see `compile`).
	31	*
	32	* If the returned `Ref` is not awaited or read
	33	* via io.readWork, the job is never started.
55	34	*/
56		export interface ViewMetadata {
57		file: string;
58		// staticMeta: string \| null; TODO
59		cssImports: string[];
60		theme: css.Theme;
61		clientRefs: string[];
62		hasLayout: boolean;
63		}
64		export interface BackendBundle {
65		magicWord: string;
66		fileWithMagicWord: string \| null;
67		files: Record<string, Buffer>;
68		}
	35	export function work<O>(job: Job<void, O>): Ref<O>;
	36	export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;
	37	export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
	38	const source = JSON.stringify(util.getCallSites(2)[1]);
	39	const keySource = [source, util.inspect(input)].join(":");
	40	const key = crypto.createHash("sha1").update(keySource).digest("base64url");
	41	ASSERT(running);
	42	ASSERT(
	43	!seenWorks.has(key),
	44	`Key '${key}' must be unique during the build. ` +
	45	`To fix this, provide a manual 'key' argument.`,
	46	);
	47	seenWorks.add(key);
	48
	49	const prev = works.get(key) as Work<O> \| null;
	50	if (prev) {
	51	return { key, then: (done) => done(prev.value) };
	52	}
69	53
70		// -- incremental support types --
71		export interface PutBase {
72		sources: SourceId[];
73		key: string;
74		}
75		export interface Put<T extends ArtifactKind> extends PutBase {
76		kind: T;
77		value: ArtifactMap[T];
78		}
79		export interface Invalidations {
80		lastModified: number;
81		outputs: Set<ArtifactId>;
82		files: Set<SourceId>;
83		}
	54	async function perform() {
	55	const io = new Io(key);
	56	jobs += 1;
	57	newKeys += 1;
	58	try {
	59	const value = await job(io, input);
	60	validateSerializable(value, "");
	61	const { reads, writes } = io;
	62	works.set(key, {
	63	value,
	64	affects: [],
	65	reads,
	66	writes,
	67	debug: source,
	68	});
	69	for (const add of reads.files) {
	70	const { affects } = UNWRAP(files.get(add));
	71	ASSERT(!affects.includes(key));
	72	affects.push(key);
	73	}
	74	for (const add of reads.works) {
	75	const { affects } = UNWRAP(works.get(add));
	76	ASSERT(!affects.includes(key));
	77	affects.push(key);
	78	}
	79	return value;
	80	} finally {
	81	jobs -= 1;
	82	}
	83	}
84	84
85		export class Incremental {
86		/** The generated artifacts */
87		out: {
88		[K in keyof ArtifactMap]: Map<string, ArtifactMap[K]>;
89		} = {
90		asset: new Map(),
91		script: new Map(),
92		style: new Map(),
93		pageMetadata: new Map(),
94		viewMetadata: new Map(),
95		serverMarko: new Map(),
96		backendBundle: new Map(),
97		backendReplace: new Map(),
98		};
99		/** Tracking filesystem entries to `srcId` */
100		invals = new Map<SourceId, Invalidations>();
101		/** Tracking output keys to files */
102		sources = new Map<ArtifactId, SourceId[]>();
103
104		/** Compressed resources */
105		compress = new Map<Sha1Id, Compressed>();
106		compressQueue = new Queue<CompressJob, void>({
107		name: "Compress",
108		maxJobs: 5,
109		fn: this.compressImpl.bind(this),
110		passive: true,
111		getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
112		});
113
114		/** Reset at the end of each update */
115		round = {
116		inserted: new Set<ArtifactId>(),
117		referenced: new Set<ArtifactId>(),
	85	let cached: Promise<O>;
	86	return {
	87	key,
	88	then: (fufill, reject) => void (cached ??= perform()).then(fufill, reject),
118	89	};
	90	}
119	91
120		getArtifact<T extends ArtifactKind>(kind: T, key: string) {
121		this.round.referenced.add(`${kind}\0${key}`);
122		return this.out[kind].get(key);
	92	export async function compile<T>(compiler: () => Promise<T>) {
	93	ASSERT(!running, `Cannot run twice at the same time.`);
	94	try {
	95	running = true;
	96	ASSERT(jobs === 0);
	97	const start = performance.now();
	98	const timerSpinner = new Spinner({
	99	text: () =>
	100	`sitegen! [${
	101	((performance.now() - start) / 1000).toFixed(
	102	1,
	103	)
	104	}s]`,
	105	fps: 10,
	106	});
	107	using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
	108
	109	const value = await compiler();
	110	ASSERT(jobs === 0);
	111	timerSpinner.text = "incremental flush";
	112	await flush(start);
	113	timerSpinner.stop();
	114	return {
	115	value,
	116	watchFiles: new Set(files.keys()),
	117	newOutputs: Array.from(seenWrites).filter(x => x.startsWith('f:')).map(x => x.slice(2)),
	118	newAssets: !Array.from(seenWrites).some(x => x.startsWith('a:')),
	119	};
	120	} finally {
	121	running = false;
	122	newKeys = 0;
	123	seenWrites.clear();
	124	seenWorks.clear();
123	125	}
	126	}
124	127
125		hasArtifact(kind: ArtifactKind, key: string) {
126		return this.getArtifact(kind, key) != null;
	128	export async function flush(start: number) {
	129	// Trim
	130	const detachedFiles = new Set<string>;
	131	const referencedAssets = new Set<string>;
	132	for (const [k, { writes: { assets } }] of works) {
	133	if (seenWorks.has(k)) {
	134	for (const asset of assets.values()) referencedAssets.add(asset.hash);
	135	continue;
	136	}
	137	deleteWork(k);
127	138	}
128
129		sourcesFor(kind: ArtifactKind, key: string) {
130		return UNWRAP(
131		this.sources.get(kind + "\0" + key),
132		`No artifact '${kind}' '${key}'`,
133		);
	139	for (const [k, file] of files) {
	140	if (file.affects.length > 0) continue;
	141	files.delete(k);
	142	detachedFiles.add(k);
	143	}
	144	for (const k of assets.keys()) {
	145	if (!referencedAssets.has(k))
	146	assets.delete(k);
134	147	}
135	148
136		shake() {
137		const toPublic = (str: string) => {
138		const [kind, key] = str.split("\0");
139		return { kind: kind as ArtifactKind, key };
140		};
141		const inserted = Array.from(this.round.inserted, toPublic);
142		const referenced = Array.from(this.round.referenced, toPublic);
143		const unreferenced: { kind: ArtifactKind; key: string }[] = [];
144
145		for (const kind in this.out) {
146		const map = this.out[kind as keyof typeof this.out];
147		if (!map) continue;
148		for (const key of map.keys()) {
149		if (!this.round.referenced.has(`${kind}\0${key}`)) {
150		unreferenced.push({ kind: kind as ArtifactKind, key });
151		// this.out[kind as ArtifactKind].delete(key);
152		}
153		}
154		}
	149	const p = [];
	150	// File writes
	151	let dist = 0;
	152	for (const [key, { buffer, size }] of writes) {
	153	if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
	154	dist += size;
	155	}
	156	// Asset map
	157	{
	158	const { json, blob } = getAssetManifest();
	159	const jsonString = Buffer.from(JSON.stringify(json));
	160	p.push(fs.writeMkdir(".clover/o/static.json", jsonString));
	161	p.push(fs.writeMkdir(".clover/o/static.blob", blob));
	162	dist += blob.byteLength + jsonString.byteLength;
	163	}
	164	await Promise.all(p);
	165
	166	// Incremental state
	167	const serialized = msgpackr.pack(serialize());
	168	await fs.writeMkdir(".clover/incr.state", serialized);
	169	const time = (performance.now() - start).toFixed(0);
	170	console.success(`sitegen! in ${time} ms`);
	171	console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);
	172	console.writeLine(` - ${assets.size} static assets`);
	173	console.writeLine(
	174	` - dist: ${formatSize(dist)}, incremental: ${
	175	formatSize(serialized.byteLength)
	176	}`,
	177	);
	178	}
155	179
156		this.round.inserted.clear();
157		this.round.referenced.clear();
	180	export async function restore() {
	181	let buffer;
	182	try {
	183	buffer = await fs.readFile(".clover/incr.state");
	184	} catch (err: any) {
	185	if (err.code !== "ENOENT") throw err;
	186	}
	187	if (!buffer) return;
	188	await deserialize(buffer);
	189	}
158	190
159		return { inserted, referenced, unreferenced };
	191	function forceInvalidate(entry: { affects: string[] }) {
	192	const queue = [...entry.affects];
	193	let key;
	194	while ((key = queue.shift())) {
	195	const affects = deleteWork(key);
	196	queue.push(...affects);
160	197	}
	198	}
161	199
162		/*
163		* Put built artifacts into the incremental cache. The entry points
164		* used to build this must be provided. 'Incremental' will trace JS
165		* imports and file modification times tracked by 'hot.ts'.
166		*/
167		put<T extends ArtifactKind>({
168		sources,
169		kind,
170		key,
171		value,
172		}: Put<T>) {
173		// These three invariants affect incremental accuracy.
174		if (this.round.inserted.has(`${kind}\0${key}`)) {
175		console.error(
176		`Artifact ${kind}:${key} was inserted multiple times in the same round!`,
177		);
178		} else if (!this.round.referenced.has(`${kind}\0${key}`)) {
179		console.error(
180		`Artifact ${kind}:${key} was inserted without checking if (!hasArtifact())`,
181		);
182		} else if (this.out[kind].has(key)) {
183		console.error(
184		`Artifact ${kind}:${key} is not stale, but overwritten.`,
185		);
186		}
	200	function deleteWork(key: string) {
	201	const work = works.get(key);
	202	if (!work) return [];
	203	const { reads, affects, writes: w } = work;
	204	for (const remove of reads.files) {
	205	const { affects } = UNWRAP(files.get(remove));
	206	ASSERT(affects.includes(key));
	207	affects.splice(affects.indexOf(key), 1);
	208	}
	209	for (const remove of reads.works) {
	210	const { affects } = UNWRAP(works.get(remove), remove);
	211	ASSERT(affects.includes(key));
	212	affects.splice(affects.indexOf(key), 1);
	213	}
	214	for (const remove of affects) {
	215	const { reads: { works: list } } = UNWRAP(works.get(remove), remove);
	216	ASSERT(list.has(key));
	217	list.delete(key);
	218	}
	219	for (const file of w.files) {
	220	if (UNWRAP(writes.get(file)).work === key)
	221	writes.delete(file);
	222	}
	223	// Assets are temporarily kept, trimmed via manual GC after compilation.
187	224
188		this.out[kind].set(key, value);
	225	works.delete(key);
	226	return affects;
	227	}
189	228
190		this.round.inserted.add(`${kind}\0${key}`);
	229	export function reset() {
	230	ASSERT(!running);
	231	works.clear();
	232	files.clear();
	233	assets.clear();
	234	}
191	235
192		// Update sources information
193		ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
194		sources = sources.map((src) => path.normalize(src));
195		const fullKey = `${kind}\0${key}`;
196		const prevSources = this.sources.get(fullKey);
197		const newSources = new Set(
198		sources.map((file) =>
199		path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
200		),
201		);
202		this.sources.set(fullKey, [...newSources]);
203		for (const source of prevSources ?? []) {
204		if (sources.includes(source)) {
205		newSources.delete(source);
206		continue;
207		}
208		const invals = UNWRAP(this.invals.get(source));
209		ASSERT(invals.outputs.has(fullKey));
210		invals.outputs.delete(fullKey);
211		}
212		// Use reflection from the plugin system to get imports.
213		for (const source of newSources) {
214		const invals = this.#getOrInitInvals(source);
215		invals.outputs.add(fullKey);
216		this.#followImports(source);
	236	export function serialize() {
	237	const fileEntries = Array.from(files, ([k, v]) =>
	238	[
	239	k,
	240	v.type,
	241	v.type === 'f' ? v.lastModified : v.type === 'd' ? v.contentHash : null,
	242	...v.affects,
	243	] as const);
	244	const workEntries = Array.from(works, ([k, v]) =>
	245	[
	246	k,
	247	v.value,
	248	Array.from(v.reads.files),
	249	Array.from(v.reads.works),
	250	Array.from(v.writes.files),
	251	Array.from(v.writes.assets, ([k, { headers }]) => [k, headers] as const),
	252	v.affects,
	253	] as const);
	254	const expectedFilesOnDisk = Array.from(
	255	writes,
	256	([k, { size, work }]) => [k, size, work] as const,
	257	);
	258	const assetEntries = Array.from(
	259	assets,
	260	([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
	261	);
	262	return [
	263	1,
	264	fileEntries,
	265	workEntries,
	266	expectedFilesOnDisk,
	267	assetEntries,
	268	] as const;
	269	}
	270	type SerializedState = ReturnType<typeof serialize>;
	271
	272	/* No-op on failure */
	273	async function deserialize(buffer: Buffer) {
	274	const decoded = msgpackr.decode(buffer) as SerializedState;
	275	if (!Array.isArray(decoded)) return false;
	276	if (decoded[0] !== 1) return false;
	277	const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] =
	278	decoded;
	279	for (const [k, type, content, ...affects] of fileEntries) {
	280	if (type === "f") {
	281	ASSERT(typeof content === "number");
	282	files.set(k, { type, affects, lastModified: content });
	283	} else if (type === 'd') {
	284	ASSERT(typeof content === "string");
	285	files.set(k, { type, affects, contentHash: content, contents: [] });
	286	} else {
	287	files.set(k, { type, affects });
217	288	}
218	289	}
219
220		// TODO: this doesnt remove stuff when it disappeary
221		#getOrInitInvals(source: string) {
222		let invals = this.invals.get(source);
223		if (!invals) {
224		const lastModified = hot.getFileStat(source)?.lastModified ??
225		fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;
226		this.invals.set(
227		source,
228		invals = {
229		lastModified,
230		files: new Set(),
231		outputs: new Set(),
232		},
233		);
234		}
235		return invals;
	290	for (const entry of workEntries) {
	291	const [
	292	k,
	293	value,
	294	readFiles,
	295	readWorks,
	296	writeFiles,
	297	writeAssets,
	298	affects,
	299	] = entry;
	300	works.set(k, {
	301	value,
	302	reads: {
	303	files: new Set(readFiles),
	304	works: new Set(readWorks),
	305	},
	306	writes: {
	307	files: new Set(writeFiles),
	308	assets: new Map(Array.from(writeAssets, ([k, headers]) => [k, {
	309	hash: JSON.parse(UNWRAP(headers.etag)),
	310	headers,
	311	}])),
	312	},
	313	affects,
	314	});
236	315	}
237
238		#followImports(file: string) {
239		const stat = hot.getFileStat(file);
240		if (!stat) return;
241		for (const i of stat.imports) {
242		const invals = this.#getOrInitInvals(i);
243		invals.files.add(file);
244		this.#followImports(i);
	316	const statFiles = await Promise.all(expectedFilesOnDisk
	317	.map(([k, size, work]) =>
	318	fs.stat(path.join(".clover/o", k))
	319	.catch((err) => {
	320	if (err.code === "ENOENT") return null;
	321	throw err;
	322	})
	323	.then((stat) => ({ k, size, work, stat }))
	324	));
	325	for (const { k, stat, work, size } of statFiles) {
	326	if (stat?.size === size) {
	327	writes.set(k, {
	328	size: size,
	329	buffer: null,
	330	work,
	331	});
	332	} else {
	333	forceInvalidate({ affects: [work] });
245	334	}
246	335	}
247
248		async statAllFiles() {
249		for (const file of this.invals.keys()) {
250		try {
251		const mtime = fs.statSync(file).mtimeMs;
252		this.updateStat(file, mtime);
253		} catch (err) {
254		}
255		}
	336	for (const [hash, raw, gzip, zstd] of assetEntries) {
	337	assets.set(hash, { raw, gzip, zstd });
256	338	}
257	339
258		updateStat(file: string, newLastModified: number \| null) {
259		file = path.relative(hot.projectRoot, file);
260		const stat = this.invals.get(file);
261		ASSERT(stat, "Updated stat on untracked file " + file);
262		const hasUpdate = !newLastModified \|\| stat.lastModified < newLastModified;
263		if (hasUpdate) {
264		// Invalidate
265		console.info(file + " " + (newLastModified ? "updated" : "deleted"));
266		hot.unload(file);
267		const invalidQueue = [file];
268		let currentInvalid;
269		while (currentInvalid = invalidQueue.pop()) {
270		const invalidations = this.invals.get(currentInvalid);
271		ASSERT(
272		invalidations,
273		"No reason to track file '" + currentInvalid +
274		"' if it has no invalidations",
275		);
276		const { files, outputs } = invalidations;
277		for (const out of outputs) {
278		const [kind, artifactKey] = out.split("\0");
279		this.out[kind as ArtifactKind].delete(artifactKey);
280		}
281		invalidQueue.push(...files);
	340	await Promise.all(Array.from(files, ([key, file]) => invalidateEntry(key, file)));
	341	}
	342
	343	export async function invalidate(filePath: string): Promise<boolean> {
	344	const key = toRel(toAbs(filePath));
	345	const file = UNWRAP(files.get(key), `Untracked file '${key}'`)
	346	return invalidateEntry(key, file)
	347	}
	348
	349	export async function invalidateEntry(key: string, file: TrackedFile): Promise<boolean> {
	350	try {
	351	if (file.type === "d") {
	352	const contents = file.contents = await fs.readdir(key);
	353	contents.sort();
	354	const contentHash = crypto
	355	.createHash("sha1")
	356	.update(contents.join("\0"))
	357	.digest("base64url");
	358	if (file.contentHash !== contentHash) {
	359	file.contentHash = contentHash;
	360	throw new Error();
	361	}
	362	} else if (file.type === 'f') {
	363	const lastModified = await fs.stat(key)
	364	.then(x => Math.floor(x.mtimeMs), () => 0);
	365	if (file.lastModified !== lastModified) {
	366	file.lastModified = lastModified;
	367	throw new Error();
282	368	}
283		}
284		if (newLastModified) {
285		stat.lastModified = newLastModified;
286	369	} else {
287		this.invals.delete(file);
	370	file.type satisfies 'null';
	371	const stat = await fs.stat(key).catch(() => null);
	372	if (stat) throw new Error();
288	373	}
289		return hasUpdate;
	374	return false;
	375	} catch (e) {
	376	forceInvalidate(file);
	377	hot.unload(toAbs(key));
	378	if (file.type === 'null') files.delete(key);
	379	return true;
290	380	}
	381	}
291	382
292		async putAsset(info: PutAsset) {
293		const { body, headers, key } = info;
294		const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body);
295		const hash = Buffer.from(await crypto.subtle.digest("sha-1", buffer))
296		.toString("hex");
297		const value: Asset = {
298		buffer,
299		headers: {
300		"Content-Type": headers?.["Content-Type"] ?? mime.contentTypeFor(key),
301		"ETag": JSON.stringify(hash),
302		...headers,
303		},
304		hash,
305		};
306		const a = this.put({ ...info, kind: "asset", value });
307		if (!this.compress.has(hash)) {
308		const label = info.key;
309		this.compress.set(hash, {
310		zstd: undefined,
311		gzip: undefined,
312		});
313		this.compressQueue.add({ label, buffer, algo: "zstd", hash });
314		this.compressQueue.add({ label, buffer, algo: "gzip", hash });
315		}
316		return a;
317		}
318
319		async compressImpl({ algo, buffer, hash }: CompressJob) {
320		let out;
321		if (algo === "zstd") out = await zstd(buffer);
322		else if (algo === "gzip") out = await gzip(buffer, { level: 9 });
323		else algo satisfies never;
324
325		let entry = this.compress.get(hash);
326		if (!entry) {
327		this.compress.set(
328		hash,
329		entry = {
330		zstd: undefined,
331		gzip: undefined,
332		},
333		);
334		}
335		entry![algo] = out;
336		}
337
338		serialize() {
339		const writer = new BufferWriter();
340
341		// -- artifact --
342		const asset = Array.from(
343		this.out.asset,
344		([key, { buffer, hash, headers }]) => {
345		const raw = writer.write(buffer, hash);
346		const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
347		const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
348		const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
349		return [key, {
350		raw,
351		gzip,
352		zstd,
353		hash,
354		headers: headers as Record<string, string>,
355		}] satisfies SerializedMeta["asset"][0];
356		},
357		);
358		const script = Array.from(this.out.script);
359		const style = Array.from(this.out.style);
360		const pageMetadata = Array.from(this.out.pageMetadata);
361		const viewMetadata = Array.from(this.out.viewMetadata);
362		const serverMarko = Array.from(this.out.serverMarko);
363		const backendBundle = Array.from(this.out.backendBundle, ([k, v]) => {
364		return [k, {
365		magicWord: v.magicWord,
366		fileWithMagicWord: v.fileWithMagicWord,
367		files: Object.entries(v.files).map(
368		([file, contents]) => [
369		file,
370		writer.write(contents, "backendBundle" + k + ":" + file),
371		],
372		),
373		}] satisfies SerializedMeta["backendBundle"][0];
374		});
375		const backendReplace = Array.from(
376		this.out.backendReplace,
377		([k, v]) =>
378		[
379		k,
380		writer.write(v, "backendReplace" + k),
381		] satisfies SerializedMeta["backendReplace"][0],
382		);
383		// -- incremental metadata --
384		const invals = Array.from(this.invals, ([key, value]) => {
385		const { lastModified, files, outputs } = value;
386		return [key, {
387		m: lastModified,
388		f: [...files],
389		o: [...outputs],
390		}] satisfies SerializedMeta["invals"][0];
391		});
392		const sources = Array.from(this.sources, ([key, value]) => {
393		return [key, ...value] as [string, ...string[]];
394		});
395		const json = {
396		asset,
397		script,
398		invals,
399		sources,
400		style,
401		pageMetadata,
402		viewMetadata,
403		serverMarko,
404		backendBundle,
405		backendReplace,
406		} satisfies SerializedMeta;
407		const meta = Buffer.from(JSON.stringify(json), "utf-8");
408
409		const lengthBuffer = Buffer.alloc(4);
410		lengthBuffer.writeUInt32LE(meta.byteLength, 0);
411
412		return Buffer.concat([lengthBuffer, meta, ...writer.buffers]);
413		}
414
415		static fromSerialized(buffer: Buffer): Incremental {
416		const metaLength = buffer.readUint32LE(0);
417		const meta: SerializedMeta = JSON.parse(
418		buffer.subarray(4, 4 + metaLength).toString("utf8"),
419		);
420		const view = ([start, end]: View) =>
421		buffer.subarray(4 + metaLength + start, 4 + metaLength + end);
422
423		const incr = new Incremental();
424		incr.out = {
425		asset: new Map(meta.asset.map(([key, value]) => {
426		const { hash, raw, gzip, zstd, headers } = value;
427		if ((gzip \|\| zstd) && !incr.compress.has(hash)) {
428		incr.compress.set(hash, {
429		gzip: gzip ? view(gzip) : undefined,
430		zstd: zstd ? view(zstd) : undefined,
431		});
432		}
433		return [key, {
434		buffer: view(raw),
435		headers: headers,
436		hash: hash,
437		}];
438		})),
439		script: new Map(meta.script),
440		style: new Map(meta.style),
441		pageMetadata: new Map(meta.pageMetadata),
442		viewMetadata: new Map(meta.viewMetadata),
443		serverMarko: new Map(meta.serverMarko),
444		backendBundle: new Map(meta.backendBundle.map(([key, value]) => {
445		return [key, {
446		magicWord: value.magicWord,
447		fileWithMagicWord: value.fileWithMagicWord,
448		files: Object.fromEntries(
449		value.files.map(([file, contents]) => [file, view(contents)]),
450		),
451		}];
452		})),
453		backendReplace: new Map(
454		meta.backendReplace.map(([key, contents]) => [key, view(contents)]),
	383	export function getAssetManifest() {
	384	const writer = new BufferWriter();
	385	const asset = Object.fromEntries(
	386	Array.from(works, (work) => work[1].writes.assets)
	387	.filter((map) => map.size > 0)
	388	.flatMap((map) =>
	389	Array.from(map, ([key, { hash, headers }]) => {
	390	const { raw, gzip, zstd } = UNWRAP(
	391	assets.get(hash),
	392	`Asset ${key} (${hash})`,
	393	);
	394	return [key, {
	395	raw: writer.write(raw, "raw:" + hash),
	396	gzip: writer.write(gzip, "gzip:" + hash),
	397	zstd: writer.write(zstd, "zstd:" + hash),
	398	headers,
	399	}] as const;
	400	})
455	401	),
456		};
457		incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {
458		return [key, {
459		lastModified: m,
460		files: new Set(f),
461		outputs: new Set(o),
462		}];
463		}));
464		incr.sources = new Map(meta.sources.map(([key, ...value]) => [key, value]));
465		return incr;
466		}
467
468		/*
469		* Move the cached (server) marko transpilations from this incremental
470		* into the running process.
471		*/
472		loadMarkoCache() {
473		hot.markoCache.clear();
474		for (const [key, value] of this.out.serverMarko) {
475		hot.markoCache.set(path.resolve(hot.projectRoot, key), value);
476		}
477		}
	402	) satisfies BuiltAssetMap;
	403	return { json: asset, blob: writer.get() };
	404	}
478	405
479		/*
480		* Move the cached (server) marko transpilations from this incremental
481		* into the running process.
482		*/
483		snapshotMarkoCache() {
484		for (const [file, value] of hot.markoCache) {
485		const key = path.relative(hot.projectRoot, file).replaceAll("\\", "/");
486		// Only insert if it doesn't exist. Calling 'put' when it
487		// already exists would inform the user of extra calls to put.
488		if (!this.hasArtifact("serverMarko", key)) {
489		this.put({
490		kind: "serverMarko",
491		sources: [file],
492		key,
493		value,
494		});
	406	/* Input/Output with automatic tracking.
	407	* - Inputs read with Io are tracked to know when to rebuild
	408	* - Outputs written with Io are deleted when abandoned.
	409	*/
	410	export class Io {
	411	constructor(public key: string) {}
	412	reads: Reads = { files: new Set(), works: new Set() };
	413	writes: Writes = { files: new Set(), assets: new Map() };
	414
	415	#trackFs(file: string) {
	416	const resolved = toAbs(file);
	417	const key = toRel(resolved);
	418	this.reads.files.add(key);
	419	return { resolved, key };
	420	}
	421	async readWork<T>(ref: Ref<T>): Promise<T> {
	422	this.reads.works.add(ref.key);
	423	return await ref;
	424	}
	425	/** Track a file in the compilation without reading it. */
	426	async trackFile(file: string) {
	427	const { key, resolved } = this.#trackFs(file);
	428	if (!files.get(key)) {
	429	let lastModified: number = 0;
	430	try {
	431	lastModified = Math.floor((await fs.stat(file)).mtimeMs);
	432	files.set(key, { type: "f", lastModified, affects: [] });
	433	} catch {
	434	files.set(key, { type: "null", affects: [] });
495	435	}
496	436	}
	437	return resolved;
497	438	}
498
499		toDisk(file = ".clover/incr.state") {
500		const buffer = this.serialize();
501		fs.writeFileSync(file, buffer);
	439	async readFile(file: string) {
	440	return fs.readFile(await this.trackFile(file), "utf-8");
502	441	}
503
504		static fromDisk(file = ".clover/incr.state"): Incremental {
	442	async readJson<T>(file: string) {
	443	return JSON.parse(await this.readFile(file)) as T;
	444	}
	445	async readDir(dir: string) {
	446	const { key, resolved } = this.#trackFs(dir);
	447	const existing = files.get(key);
505	448	try {
506		const buffer = fs.readFileSync(file);
507		return Incremental.fromSerialized(buffer);
508		} catch (err: any) {
509		if (err.code === "ENOENT") return new Incremental();
	449	if (existing?.type === 'd') return existing.contents;
	450	const contents = await fs.readdir(resolved);
	451	contents.sort();
	452	const contentHash = crypto
	453	.createHash("sha1")
	454	.update(contents.join("\0"))
	455	.digest("base64url");
	456	files.set(key, {
	457	type: "d",
	458	affects: [],
	459	contentHash,
	460	contents,
	461	});
	462	return contents;
	463	} catch (err) {
	464	if (!existing) files.set(key, { type: "null", affects: [] });
510	465	throw err;
511	466	}
512	467	}
513
514		async wait() {
515		await this.compressQueue.done({ method: "success" });
	468	async readDirRecursive(dir: string): Promise<string[]> {
	469	const dirs = await this.readDir(dir);
	470	return (
	471	await Promise.all(
	472	dirs.map(async (child) => {
	473	const abs = path.join(dir, child);
	474	const stat = await fs.stat(abs);
	475	if (stat.isDirectory()) {
	476	return (await this.readDirRecursive(abs)).map((grand) =>
	477	path.join(child, grand)
	478	);
	479	} else {
	480	return child;
	481	}
	482	}),
	483	)
	484	).flat();
516	485	}
517
518		async flush(
519		platform: bundle.ServerPlatform,
520		dir = path.resolve(".clover/out"),
	486	/* Track all dependencies of a module. */
	487	async import<T>(file: string): Promise<T> {
	488	const { resolved } = this.#trackFs(file);
	489	try {
	490	return require(resolved) as T;
	491	} finally {
	492	const queue = [resolved];
	493	const seen = new Set<string>();
	494	let current;
	495	while ((current = queue.shift())) {
	496	const stat = hot.getFileStat(current);
	497	if (!stat) continue;
	498	const { key } = this.#trackFs(current);
	499	if (!files.get(key)) {
	500	files.set(key, {
	501	type: "f",
	502	affects: [],
	503	lastModified: stat?.lastModified ?? 0,
	504	});
	505	}
	506	for (const imp of stat.imports) {
	507	if (!seen.has(imp)) {
	508	seen.add(imp);
	509	queue.push(imp);
	510	}
	511	}
	512	}
	513	}
	514	}
	515	async writeAsset(
	516	pathname: string,
	517	blob: string \| Buffer,
	518	headersOption?: HeadersInit,
521	519	) {
522		ASSERT(!this.compressQueue.active);
523		const join = (...args: string[]) => path.join(dir, ...args);
524		const writer = new BufferWriter();
525
526		// TODO: ensure all compressed got compressed
527
528		const asset = Object.fromEntries(
529		Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
530		const raw = writer.write(buffer, hash);
531		const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
532		const gzip = writer.write(UNWRAP(gzipBuf), hash + ".gz");
533		const zstd = writer.write(UNWRAP(zstdBuf), hash + ".zstd");
534		return [key, { raw, gzip, zstd, headers }];
535		}),
536		);
537		const backendBundle = UNWRAP(this.out.backendBundle.get(platform));
538
539		// Arrange output files
540		const outFiles: Array<[file: string, contents: string \| Buffer]> = [
541		// Asset manifest
542		["static.json", JSON.stringify(asset)],
543		["static.blob", writer.get()],
544
545		// Backend
546		...Object.entries(backendBundle.files).map(([subPath, contents]) =>
547		[
548		subPath,
549		subPath === backendBundle.fileWithMagicWord
550		? UNWRAP(this.out.backendReplace.get(platform))
551		: contents,
552		] as [string, Buffer]
553		),
554		];
	520	ASSERT(pathname.startsWith("/"));
	521	ASSERT(!seenWrites.has("a:" + pathname));
555	522
556		// TODO: check duplicates
	523	const buffer = typeof blob === "string" ? Buffer.from(blob) : blob;
557	524
558		// Perform all i/o
559		await Promise.all(
560		outFiles.map(([subPath, contents]) =>
561		fs.writeMkdir(join(subPath), contents, { flush: true })
562		),
	525	const headers = new Headers(headersOption ?? {});
	526	const hash = crypto.createHash("sha1").update(buffer).digest("hex");
	527	if (!headers.has("Content-Type")) {
	528	headers.set("Content-Type", mime.contentTypeFor(pathname));
	529	}
	530	headers.set("ETag", JSON.stringify(hash));
	531	this.writes.assets.set(pathname, {
	532	hash,
	533	// @ts-expect-error TODO
	534	headers: Object.fromEntries(headers),
	535	});
	536	if (!assets.has(hash)) {
	537	jobs += 1;
	538	assets.set(hash, undefined!);
	539	const [gzipBuffer, zstdBuffer] = await Promise.all([
	540	gzip(buffer),
	541	zstdCompress(buffer),
	542	]);
	543	assets.set(hash, {
	544	raw: buffer,
	545	gzip: gzipBuffer,
	546	zstd: zstdBuffer,
	547	});
	548	jobs -= 1;
	549	}
	550	}
	551	writeFile(subPath: string, blob: string \| Buffer) {
	552	ASSERT(!subPath.startsWith("/"));
	553	ASSERT(
	554	!seenWrites.has("f:" + subPath),
	555	`File overwritten: ${JSON.stringify(subPath)}`,
563	556	);
	557	seenWrites.add("f:" + subPath);
	558	const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
	559	writes.set(subPath, {
	560	buffer,
	561	size: buffer.byteLength,
	562	work: this.key,
	563	});
564	564	}
565	565	}
566	566
567		export interface PutAsset extends PutBase {
568		body: string \| Buffer;
569		headers?: Record<string, string \| undefined>;
570		}
571
572		export interface Compressed {
573		gzip?: Buffer;
574		zstd?: Buffer;
575		}
576
577		export interface CompressJob {
578		algo: "zstd" \| "gzip";
579		buffer: Buffer;
580		label: string;
581		hash: string;
582		}
583
584	567	class BufferWriter {
585	568	size = 0;
586		seen = new Map<string, View>();
	569	seen = new Map<string, BufferView>();
587	570	buffers: Buffer[] = [];
588	571
589		write(buffer: Buffer, hash: string): View {
	572	write(buffer: Buffer, hash: string): BufferView {
590	573	let view = this.seen.get(hash);
591	574	if (view) return view;
592	575	view = [this.size, this.size += buffer.byteLength];
...	...	@@ -600,58 +583,101 @@ class BufferWriter {
600	583	}
601	584	}
602	585
603		export type View = [start: number, end: number];
	586	export function validateSerializable(value: unknown, key: string) {
	587	if (typeof value === "string") {
	588	if (value.includes(hot.projectRoot)) {
	589	throw new Error(
	590	`Return value must not contain the CWD for portability, found at ${key}`,
	591	);
	592	}
	593	} else if (value && typeof value === "object") {
	594	if (Array.isArray(value)) {
	595	value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
	596	} else if (Object.getPrototypeOf(value) === Object.prototype \|\| Buffer.isBuffer(value)) {
	597	Object.entries(value).forEach(([k, v]) =>
	598	validateSerializable(v, `${key}.${k}`)
	599	);
	600	} else {
	601	throw new Error(
	602	`Return value must be a plain JS object, found ${
	603	Object.getPrototypeOf(value).constructor.name
	604	} at ${key}`,
	605	);
	606	}
	607	} else if (["bigint", "function", "symbol"].includes(typeof value)) {
	608	throw new Error(
	609	`Return value must be a plain JS object, found ${typeof value} at ${key}`,
	610	);
	611	}
	612	}
604	613
605		export interface BuiltAssetMap {
606		[route: string]: BuiltAsset;
	614	export function toAbs(absPath: string) {
	615	return path.resolve(hot.projectRoot, absPath);
607	616	}
608	617
609		export interface BuiltAsset {
610		raw: View;
611		gzip: View;
612		zstd: View;
613		headers: Record<string, string>;
	618	export function toRel(absPath: string) {
	619	return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
614	620	}
615	621
616		export interface SerializedMeta {
617		asset: Array<[route: string, data: {
618		raw: View;
619		gzip: View \| null;
620		zstd: View \| null;
	622	export type BufferView = [start: number, end: number];
	623	interface Reads {
	624	files: Set<string>;
	625	works: Set<string>;
	626	}
	627	interface FileWrite {
	628	buffer: Buffer \| null;
	629	size: number;
	630	work: string;
	631	}
	632	interface Writes {
	633	files: Set<string>;
	634	assets: Map<string, {
621	635	hash: string;
622	636	headers: Record<string, string>;
623		}]>;
624		script: Array<[key: string, value: string]>;
625		style: Array<[key: string, value: string]>;
626		pageMetadata: Array<[key: string, PageMetadata]>;
627		viewMetadata: Array<[key: string, ViewMetadata]>;
628		serverMarko: Array<[key: string, hot.MarkoCacheEntry]>;
629		backendBundle: Array<[platform: string, {
630		magicWord: string;
631		fileWithMagicWord: string \| null;
632		files: Array<[string, View]>;
633		}]>;
634		backendReplace: Array<[key: string, View]>;
635
636		invals: Array<[key: string, {
637		/** Modified */
638		m: number;
639		f: SourceId[];
640		o: ArtifactId[];
641		}]>;
642		sources: Array<[string, ...string[]]>;
	637	}>;
	638	}
	639	interface Asset {
	640	raw: Buffer;
	641	gzip: Buffer;
	642	zstd: Buffer;
	643	}
	644	interface Work<T = unknown> {
	645	debug?: string;
	646	value: T;
	647	reads: Reads;
	648	writes: Writes;
	649	affects: string[];
	650	}
	651	type TrackedFile =
	652	& {
	653	affects: string[];
	654	}
	655	& (
	656	\| { type: "f"; lastModified: number }
	657	\| { type: "d"; contentHash: string; contents: string[] }
	658	\| { type: "null"; }
	659	);
	660	export interface BuiltAssetMap {
	661	[route: string]: BuiltAsset;
	662	}
	663	export interface BuiltAsset {
	664	raw: BufferView;
	665	gzip: BufferView;
	666	zstd: BufferView;
	667	headers: Record<string, string>;
643	668	}
644	669
645	670	const gzip = util.promisify(zlib.gzip);
646		const zstd = util.promisify(zlib.zstdCompress);
	671	const zstdCompress = util.promisify(zlib.zstdCompress);
647	672
648	673	import * as fs from "#sitegen/fs";
649		import * as zlib from "node:zlib";
650		import * as util from "node:util";
651		import { Queue } from "#sitegen/async";
	674	import * as path from "node:path";
652	675	import * as hot from "./hot.ts";
	676	import * as util from "node:util";
	677	import * as crypto from "node:crypto";
653	678	import * as mime from "#sitegen/mime";
654		import * as path from "node:path";
655		import { Buffer } from "node:buffer";
656		import * as css from "./css.ts";
657		import type * as bundle from "./bundle.ts";
	679	import * as zlib from "node:zlib";
	680	import * as console from "@paperclover/console";
	681	import { Spinner } from "@paperclover/console/Spinner";
	682	import { formatSize } from "@/file-viewer/format.ts";
	683	import * as msgpackr from "msgpackr";

framework/lib/assets.ts+2-2

...	...	@@ -69,7 +69,7 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
69	69	return ifNoneMatch === etag \|\| ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
70	70	}
71	71
72		function subarrayAsset([start, end]: View) {
	72	function subarrayAsset([start, end]: BufferView) {
73	73	return assets!.buf.subarray(start, end);
74	74	}
75	75
...	...	@@ -115,6 +115,6 @@ process.on("message", (msg: any) => {
115	115	import * as fs from "#sitegen/fs";
116	116	import type { Context, Next } from "hono";
117	117	import type { StatusCode } from "hono/utils/http-status";
118		import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
	118	import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts";
119	119	import { Buffer } from "node:buffer";
120	120	import * as path from "node:path";

framework/lib/testing.ts created+11

...	...	@@ -0,0 +1,11 @@
	1	export function tmpFile(basename: string) {
	2	const file = path.join(import.meta.dirname, '../../.clover/testing', basename);
	3	return {
	4	path: file,
	5	read: fs.readFile.bind(fs, file),
	6	write: fs.writeMkdir.bind(fs, file),
	7	};
	8	}
	9
	10	import * as path from 'node:path';
	11	import * as fs from './fs.ts';

framework/marko.ts created+46

...	...	@@ -0,0 +1,46 @@
	1	console.log("MARKO");
	2	export interface MarkoCacheEntry {
	3	src: string;
	4	scannedClientRefs: string[];
	5	}
	6
	7	export const markoCache = new Map<string, MarkoCacheEntry>();
	8
	9	export function loadMarko(module: NodeJS.Module, filepath: string) {
	10	let cache = markoCache.get(filepath);
	11	console.log({ filepath, has: !!cache })
	12	if (!cache) {
	13	let src = fs.readFileSync(filepath, "utf8");
	14	// A non-standard thing here is Clover Sitegen implements
	15	// its own client side scripting stuff, so it overrides
	16	// bare client import statements to it's own usage.
	17	const scannedClientRefs = new Set<string>();
	18	if (src.match(/^\s*client\s+import\s+["']/m)) {
	19	src = src.replace(
	20	/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
	21	(_, src) => {
	22	const ref = JSON.parse(`"${src.slice(1, -1)}"`);
	23	const resolved = hot.resolveClientRef(filepath, ref);
	24	scannedClientRefs.add(resolved);
	25	return `<CloverScriptInclude=${
	26	JSON.stringify(hot.getScriptId(resolved))
	27	} />`;
	28	},
	29	) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
	30	}
	31
	32	src = marko.compileSync(src, filepath).code;
	33	src = src.replace("marko/debug/html", "#ssr/marko");
	34	cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
	35	markoCache.set(filepath, cache);
	36	}
	37
	38	const { src, scannedClientRefs } = cache;
	39	return hot.loadEsbuildCode(module, filepath, src, {
	40	scannedClientRefs,
	41	});
	42	}
	43
	44	import * as marko from "@marko/compiler";
	45	import * as hot from "./hot.ts";
	46	import * as fs from "#sitegen/fs";

framework/watch.ts+67-91

...	...	@@ -2,102 +2,67 @@
2	2
3	3	const debounceMilliseconds = 25;
4	4
5		export async function main() {
6		let subprocess: child_process.ChildProcess \| null = null;
	5	let subprocess: child_process.ChildProcess \| null = null;
	6	process.on("beforeExit", () => {
	7	subprocess?.removeListener("close", onSubprocessClose);
	8	});
	9
	10	let watch: Watch;
7	11
	12	export async function main() {
8	13	// Catch up state by running a main build.
9		const { incr } = await generate.main();
10		// ...and watch the files that cause invals.
11		const watch = new Watch(rebuild);
12		watch.add(...incr.invals.keys());
13		statusLine();
14		// ... and then serve it!
15		serve();
16
17		function serve() {
18		if (subprocess) {
19		subprocess.removeListener("close", onSubprocessClose);
20		subprocess.kill();
21		}
22		subprocess = child_process.fork(".clover/out/server.js", [
23		"--development",
24		], {
25		stdio: "inherit",
26		});
27		subprocess.on("close", onSubprocessClose);
28		}
	14	await incr.restore();
	15	watch = new Watch(rebuild);
	16	rebuild([]);
	17	}
29	18
30		function onSubprocessClose(code: number \| null, signal: string \| null) {
31		subprocess = null;
32		const status = code != null ? `code ${code}` : `signal ${signal}`;
33		console.error(`Backend process exited with ${status}`);
	19	function serve() {
	20	if (subprocess) {
	21	subprocess.removeListener("close", onSubprocessClose);
	22	subprocess.kill();
34	23	}
35
36		process.on("beforeExit", () => {
37		subprocess?.removeListener("close", onSubprocessClose);
	24	subprocess = child_process.fork(".clover/o/backend.js", [
	25	"--development",
	26	], {
	27	stdio: "inherit",
38	28	});
	29	subprocess.on("close", onSubprocessClose);
	30	}
39	31
40		function rebuild(files: string[]) {
41		files = files.map((file) => path.relative(hot.projectRoot, file));
42		const changed: string[] = [];
43		for (const file of files) {
44		let mtimeMs: number \| null = null;
45		try {
46		mtimeMs = fs.statSync(file).mtimeMs;
47		} catch (err: any) {
48		if (err?.code !== "ENOENT") throw err;
49		}
50		if (incr.updateStat(file, mtimeMs)) changed.push(file);
51		}
52		if (changed.length === 0) {
53		console.warn("Files were modified but the 'modify' time did not change.");
54		return;
	32	function onSubprocessClose(code: number \| null, signal: string \| null) {
	33	subprocess = null;
	34	const status = code != null ? `code ${code}` : `signal ${signal}`;
	35	console.error(`Backend process exited with ${status}`);
	36	}
	37
	38	async function rebuild(files: string[]) {
	39	const hasInvalidated = files.length === 0
	40	\|\| (await Promise.all(files.map(incr.invalidate))).some(Boolean);
	41	if (!hasInvalidated) return;
	42	incr.compile(generate.generate).then(({
	43	watchFiles,
	44	newOutputs,
	45	newAssets
	46	}) => {
	47	const removeWatch = [...watch.files].filter(x => !watchFiles.has(x))
	48	for (const file of removeWatch) watch.remove(file);
	49	watch.add(...watchFiles);
	50	// Restart the server if it was changed or not running.
	51	if (!subprocess \|\| newOutputs.includes("backend.js")) {
	52	serve();
	53	} else if (subprocess && newAssets) {
	54	subprocess.send({ type: "clover.assets.reload" });
55	55	}
56		withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
57		text: "Rebuilding",
58		successText: generate.successText,
59		failureText: () => "sitegen FAIL",
60		}, async (spinner) => {
61		console.info("---");
62		console.info(
63		"Updated" +
64		(changed.length === 1
65		? " " + changed[0]
66		: changed.map((file) => "\n- " + file)),
67		);
68		const result = await generate.sitegen(spinner, incr);
69		incr.toDisk(); // Allows picking up this state again
70		for (const file of watch.files) {
71		const relative = path.relative(hot.projectRoot, file);
72		if (!incr.invals.has(relative)) watch.remove(file);
73		}
74		return result;
75		}).then((result) => {
76		// Restart the server if it was changed or not running.
77		if (
78		!subprocess \|\|
79		result.inserted.some(({ kind }) => kind === "backendReplace")
80		) {
81		serve();
82		} else if (
83		subprocess &&
84		result.inserted.some(({ kind }) => kind === "asset")
85		) {
86		subprocess.send({ type: "clover.assets.reload" });
87		}
88		return result;
89		}).catch((err) => {
90		console.error(util.inspect(err));
91		}).finally(statusLine);
92		}
	56	}).catch((err) => {
	57	console.error(util.inspect(err));
	58	}).finally(statusLine);
	59	}
93	60
94		function statusLine() {
95		console.info(
96		`Watching ${incr.invals.size} files \x1b[36m[last change: ${
97		new Date().toLocaleTimeString()
98		}]\x1b[39m`,
99		);
100		}
	61	function statusLine() {
	62	console.info(
	63	`Watching ${watch.files.size} files `
	64	+ `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
	65	);
101	66	}
102	67
103	68	class Watch {
...	...	@@ -174,11 +139,21 @@ class Watch {
174	139	for (const w of this.watchers) w.close();
175	140	}
176	141
	142	#getFiles(absPath: string, event: fs.WatchEventType) {
	143	const files = [];
	144	if (this.files.has(absPath)) files.push(absPath);
	145	if (event === 'rename') {
	146	const dir = path.dirname(absPath);
	147	if (this.files.has(dir)) files.push(dir);
	148	}
	149	return files;
	150	}
	151
177	152	#handleEvent(root: string, event: fs.WatchEventType, subPath: string \| null) {
178	153	if (!subPath) return;
179		const file = path.join(root, subPath);
180		if (!this.files.has(file)) return;
181		this.stale.add(file);
	154	const files = this.#getFiles(path.join(root, subPath), event);
	155	if (files.length === 0) return;
	156	for(const file of files) this.stale.add(file);
182	157	const { debounce } = this;
183	158	if (debounce !== null) clearTimeout(debounce);
184	159	this.debounce = setTimeout(() => {
...	...	@@ -192,6 +167,7 @@ class Watch {
192	167	import * as fs from "node:fs";
193	168	import { withSpinner } from "@paperclover/console/Spinner";
194	169	import * as generate from "./generate.ts";
	170	import * as incr from "./incremental.ts";
195	171	import * as path from "node:path";
196	172	import * as util from "node:util";
197	173	import * as hot from "./hot.ts";

package-lock.json+126

...	...	@@ -15,8 +15,10 @@
15	15	"hls.js": "^1.6.5",
16	16	"hono": "^4.7.11",
17	17	"marko": "^6.0.20",
	18	"msgpackr": "^1.11.5",
18	19	"puppeteer": "^24.10.1",
19	20	"sharp": "^0.34.2",
	21	"source-map-support": "^0.5.21",
20	22	"unique-names-generator": "^4.7.1",
21	23	"vscode-oniguruma": "^2.0.1",
22	24	"vscode-textmate": "^9.2.0"
...	...	@@ -1478,6 +1480,84 @@
1478	1480	"url": "https://opencollective.com/unified"
1479	1481	}
1480	1482	},
	1483	"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
	1484	"version": "3.0.3",
	1485	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
	1486	"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
	1487	"cpu": [
	1488	"arm64"
	1489	],
	1490	"license": "MIT",
	1491	"optional": true,
	1492	"os": [
	1493	"darwin"
	1494	]
	1495	},
	1496	"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
	1497	"version": "3.0.3",
	1498	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
	1499	"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
	1500	"cpu": [
	1501	"x64"
	1502	],
	1503	"license": "MIT",
	1504	"optional": true,
	1505	"os": [
	1506	"darwin"
	1507	]
	1508	},
	1509	"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
	1510	"version": "3.0.3",
	1511	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
	1512	"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
	1513	"cpu": [
	1514	"arm"
	1515	],
	1516	"license": "MIT",
	1517	"optional": true,
	1518	"os": [
	1519	"linux"
	1520	]
	1521	},
	1522	"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
	1523	"version": "3.0.3",
	1524	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
	1525	"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
	1526	"cpu": [
	1527	"arm64"
	1528	],
	1529	"license": "MIT",
	1530	"optional": true,
	1531	"os": [
	1532	"linux"
	1533	]
	1534	},
	1535	"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
	1536	"version": "3.0.3",
	1537	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
	1538	"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
	1539	"cpu": [
	1540	"x64"
	1541	],
	1542	"license": "MIT",
	1543	"optional": true,
	1544	"os": [
	1545	"linux"
	1546	]
	1547	},
	1548	"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
	1549	"version": "3.0.3",
	1550	"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
	1551	"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
	1552	"cpu": [
	1553	"x64"
	1554	],
	1555	"license": "MIT",
	1556	"optional": true,
	1557	"os": [
	1558	"win32"
	1559	]
	1560	},
1481	1561	"node_modules/@paperclover/console": {
1482	1562	"resolved": "git+https://git.paperclover.net/clo/console.git#1a6ac2b79fdd8a21a1c57d25723975872bc07e3e",
1483	1563	"dependencies": {
...	...	@@ -3765,6 +3845,37 @@
3765	3845	"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
3766	3846	"license": "MIT"
3767	3847	},
	3848	"node_modules/msgpackr": {
	3849	"version": "1.11.5",
	3850	"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
	3851	"integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
	3852	"license": "MIT",
	3853	"optionalDependencies": {
	3854	"msgpackr-extract": "^3.0.2"
	3855	}
	3856	},
	3857	"node_modules/msgpackr-extract": {
	3858	"version": "3.0.3",
	3859	"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
	3860	"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
	3861	"hasInstallScript": true,
	3862	"license": "MIT",
	3863	"optional": true,
	3864	"dependencies": {
	3865	"node-gyp-build-optional-packages": "5.2.2"
	3866	},
	3867	"bin": {
	3868	"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
	3869	},
	3870	"optionalDependencies": {
	3871	"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
	3872	"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
	3873	"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
	3874	"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
	3875	"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
	3876	"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
	3877	}
	3878	},
3768	3879	"node_modules/netmask": {
3769	3880	"version": "2.0.2",
3770	3881	"resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz",
...	...	@@ -3774,6 +3885,21 @@
3774	3885	"node": ">= 0.4.0"
3775	3886	}
3776	3887	},
	3888	"node_modules/node-gyp-build-optional-packages": {
	3889	"version": "5.2.2",
	3890	"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
	3891	"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
	3892	"license": "MIT",
	3893	"optional": true,
	3894	"dependencies": {
	3895	"detect-libc": "^2.0.1"
	3896	},
	3897	"bin": {
	3898	"node-gyp-build-optional-packages": "bin.js",
	3899	"node-gyp-build-optional-packages-optional": "optional.js",
	3900	"node-gyp-build-optional-packages-test": "build-test.js"
	3901	}
	3902	},
3777	3903	"node_modules/node-releases": {
3778	3904	"version": "2.0.19",
3779	3905	"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",

package.json+2

...	...	@@ -11,8 +11,10 @@
11	11	"hls.js": "^1.6.5",
12	12	"hono": "^4.7.11",
13	13	"marko": "^6.0.20",
	14	"msgpackr": "^1.11.5",
14	15	"puppeteer": "^24.10.1",
15	16	"sharp": "^0.34.2",
	17	"source-map-support": "^0.5.21",
16	18	"unique-names-generator": "^4.7.1",
17	19	"vscode-oniguruma": "^2.0.1",
18	20	"vscode-textmate": "^9.2.0"

src/file-viewer/bin/scan3.ts+120-120

...	...	@@ -17,9 +17,9 @@ export async function main() {
17	17	const start = performance.now();
18	18	const timerSpinner = new Spinner({
19	19	text: () =>
20		`paper clover's scan3 [${
21		((performance.now() - start) / 1000).toFixed(1)
22		}s]`,
	20	`paper clover's scan3 [${((performance.now() - start) / 1000).toFixed(
	21	1,
	22	)}s]`,
23	23	fps: 10,
24	24	});
25	25	using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
...	...	@@ -38,20 +38,23 @@ export async function main() {
38	38	qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
39	39
40	40	if (mediaFile) {
41		const deleted = mediaFile.getChildren()
	41	const deleted = mediaFile
	42	.getChildren()
42	43	.filter((child) => !items.includes(child.basename))
43	44	.flatMap((child) =>
44	45	child.kind === MediaFileKind.directory
45	46	? child.getRecursiveFileChildren()
46		: child
	47	: child,
47	48	);
48	49
49		qMeta.addMany(deleted.map((mediaFile) => ({
50		absPath: path.join(root, mediaFile.path),
51		publicPath: mediaFile.path,
52		stat: null,
53		mediaFile,
54		})));
	50	qMeta.addMany(
	51	deleted.map((mediaFile) => ({
	52	absPath: path.join(root, mediaFile.path),
	53	publicPath: mediaFile.path,
	54	stat: null,
	55	mediaFile,
	56	})),
	57	);
55	58	}
56	59
57	60	return;
...	...	@@ -96,13 +99,13 @@ export async function main() {
96	99	if (
97	100	mediaFile &&
98	101	mediaFile.date.getTime() < stat.mtime.getTime() &&
99		(Date.now() - stat.mtime.getTime()) < monthMilliseconds
	102	Date.now() - stat.mtime.getTime() < monthMilliseconds
100	103	) {
101	104	date = mediaFile.date;
102	105	console.warn(
103		`M-time on ${publicPath} was likely corrupted. ${
104		formatDate(mediaFile.date)
105		} -> ${formatDate(stat.mtime)}`,
	106	`M-time on ${publicPath} was likely corrupted. ${formatDate(
	107	mediaFile.date,
	108	)} -> ${formatDate(stat.mtime)}`,
106	109	);
107	110	}
108	111	mediaFile = MediaFile.createFile({
...	...	@@ -129,7 +132,10 @@ export async function main() {
129	132	await processor.run({ absPath, stat, mediaFile, spin });
130	133	mediaFile.setProcessed(mediaFile.processed \| (1 << (16 + index)));
131	134	for (const dependantJob of after) {
132		ASSERT(dependantJob.needs > 0, `dependantJob.needs > 0, ${dependantJob.needs}`);
	135	ASSERT(
	136	dependantJob.needs > 0,
	137	`dependantJob.needs > 0, ${dependantJob.needs}`,
	138	);
133	139	dependantJob.needs -= 1;
134	140	if (dependantJob.needs == 0) qProcess.add(dependantJob);
135	141	}
...	...	@@ -149,25 +155,27 @@ export async function main() {
149	155	}));
150	156	}
151	157
152		async function queueProcessors(
153		{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
154		) {
	158	async function queueProcessors({
	159	absPath,
	160	stat,
	161	mediaFile,
	162	}: Omit<ProcessFileArgs, "spin">) {
155	163	const ext = mediaFile.extensionNonEmpty.toLowerCase();
156	164	let possible = processors.filter((p) =>
157		p.include ? p.include.has(ext) : !p.exclude?.has(ext)
	165	p.include ? p.include.has(ext) : !p.exclude?.has(ext),
158	166	);
159	167	if (possible.length === 0) return;
160	168
161	169	const hash = possible.reduce((a, b) => a ^ b.hash, 0) \| 1;
162		ASSERT(hash <= 0xFFFF, `${hash.toString(16)} has no bits above 16 set`);
	170	ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
163	171	let processed = mediaFile.processed;
164	172
165	173	// If the hash has changed, migrate the bitfield over.
166	174	// This also runs when the processor hash is in it's initial 0 state.
167	175	const order = decodeProcessors(mediaFile.processors);
168		if ((processed & 0xFFFF) !== hash) {
169		const previous = order.filter((_, i) =>
170		(processed & (1 << (16 + i))) !== 0
	176	if ((processed & 0xffff) !== hash) {
	177	const previous = order.filter(
	178	(_, i) => (processed & (1 << (16 + i))) !== 0,
171	179	);
172	180	processed = hash;
173	181	for (const { id, hash } of previous) {
...	...	@@ -182,13 +190,13 @@ export async function main() {
182	190	}
183	191	mediaFile.setProcessors(
184	192	processed,
185		possible.map((p) =>
186		p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF)
187		).join(";"),
	193	possible
	194	.map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
	195	.join(";"),
188	196	);
189	197	} else {
190	198	possible = order.map(({ id }) =>
191		UNWRAP(possible.find((p) => p.id === id))
	199	UNWRAP(possible.find((p) => p.id === id)),
192	200	);
193	201	}
194	202
...	...	@@ -225,8 +233,9 @@ export async function main() {
225	233
226	234	async function runUndoProcessors(mediaFile: MediaFile) {
227	235	const { processed } = mediaFile;
228		const previous = decodeProcessors(mediaFile.processors)
229		.filter((_, i) => (processed & (1 << (16 + i))) !== 0);
	236	const previous = decodeProcessors(mediaFile.processors).filter(
	237	(_, i) => (processed & (1 << (16 + i))) !== 0,
	238	);
230	239	for (const { id } of previous) {
231	240	const p = processors.find((p) => p.id === id);
232	241	if (!p) continue;
...	...	@@ -244,22 +253,23 @@ export async function main() {
244	253	await qProcess.done();
245	254
246	255	// Update directory metadata
247		const dirs = MediaFile.getDirectoriesToReindex()
248		.sort((a, b) => b.path.length - a.path.length);
	256	const dirs = MediaFile.getDirectoriesToReindex().sort(
	257	(a, b) => b.path.length - a.path.length,
	258	);
249	259	for (const dir of dirs) {
250	260	const children = dir.getChildren();
251	261
252	262	// readme.txt
253		const readmeContent = children.find((x) =>
254		x.basename === "readme.txt"
255		)?.contents ?? "";
	263	const readmeContent =
	264	children.find((x) => x.basename === "readme.txt")?.contents ?? "";
256	265
257	266	// dirsort
258	267	let dirsort: string[] \| null = null;
259	268	const dirSortRaw =
260	269	children.find((x) => x.basename === ".dirsort")?.contents ?? "";
261	270	if (dirSortRaw) {
262		dirsort = dirSortRaw.split("\n")
	271	dirsort = dirSortRaw
	272	.split("\n")
263	273	.map((x) => x.trim())
264	274	.filter(Boolean);
265	275	}
...	...	@@ -284,7 +294,8 @@ export async function main() {
284	294	}
285	295	}
286	296
287		const dirHash = crypto.createHash("sha1")
	297	const dirHash = crypto
	298	.createHash("sha1")
288	299	.update(dir.path + allHashes)
289	300	.digest("hex");
290	301
...	...	@@ -323,19 +334,21 @@ export async function main() {
323	334
324	335	console.info(
325	336	"Updated file viewer index in \x1b[1m" +
326		((performance.now() - start) / 1000).toFixed(1) + "s\x1b[0m",
	337	((performance.now() - start) / 1000).toFixed(1) +
	338	"s\x1b[0m",
327	339	);
328	340
329	341	MediaFile.db.prepare("VACUUM").run();
330		const { duration, count } = MediaFile.db.prepare<
331		[],
332		{ count: number; duration: number }
333		>(`
	342	const { duration, count } = MediaFile.db
	343	.prepare<[], { count: number; duration: number }>(
	344	`
334	345	select
335	346	count(*) as count,
336	347	sum(duration) as duration
337	348	from media_files
338		`).getNonNull();
	349	`,
	350	)
	351	.getNonNull();
339	352
340	353	console.info();
341	354	console.info(
...	...	@@ -365,7 +378,7 @@ const execFile: typeof execFileRaw = ((
365	378	) =>
366	379	execFileRaw(...args).catch((e: any) => {
367	380	if (e?.message?.startsWith?.("Command failed")) {
368		if (e.code > (2 ** 31)) e.code \|= 0;
	381	if (e.code > 2 ** 31) e.code \|= 0;
369	382	const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
370	383	e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
371	384	}
...	...	@@ -374,11 +387,7 @@ const execFile: typeof execFileRaw = ((
374	387	const ffprobeBin = testProgram("ffprobe", "--help");
375	388	const ffmpegBin = testProgram("ffmpeg", "--help");
376	389
377		const ffmpegOptions = [
378		"-hide_banner",
379		"-loglevel",
380		"warning",
381		];
	390	const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
382	391
383	392	const procDuration: Process = {
384	393	name: "calculate duration",
...	...	@@ -496,27 +505,23 @@ const procImageSubsets: Process = {
496	505	for (const size of targetSizes) {
497	506	const { w, h } = resizeDimensions(width, height, size);
498	507	for (const { ext, args } of transcodeRules.imagePresets) {
499		spin.text = baseStatus +
500		` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
	508	spin.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
501	509
502	510	stack.use(
503		await produceAsset(
504		`${mediaFile.hash}/${size}${ext}`,
505		async (out) => {
506		await fs.mkdir(path.dirname(out));
507		await fs.rm(out, { force: true });
508		await execFile(ffmpegBin!, [
509		...ffmpegOptions,
510		"-i",
511		absPath,
512		"-vf",
513		`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
514		...args,
515		out,
516		]);
517		return [out];
518		},
519		),
	511	await produceAsset(`${mediaFile.hash}/${size}${ext}`, async (out) => {
	512	await fs.mkdir(path.dirname(out));
	513	await fs.rm(out, { force: true });
	514	await execFile(ffmpegBin!, [
	515	...ffmpegOptions,
	516	"-i",
	517	absPath,
	518	"-vf",
	519	`scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
	520	...args,
	521	out,
	522	]);
	523	return [out];
	524	}),
520	525	);
521	526	}
522	527	}
...	...	@@ -561,21 +566,17 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
561	566	if (config.encoder && typeof config.encoder.videoSrc === "string") {
562	567	const { videoSrc, audioSrc, rate } = config.encoder;
563	568	inputArgs = [
564		...rate ? ["-r", String(rate)] : [],
	569	...(rate ? ["-r", String(rate)] : []),
565	570	"-i",
566	571	videoSrc,
567		...audioSrc ? ["-i", audioSrc] : [],
	572	...(audioSrc ? ["-i", audioSrc] : []),
568	573	];
569	574	}
570	575	} catch (err: any) {
571	576	if (err?.code !== "ENOENT") throw err;
572	577	}
573	578
574		const args = transcodeRules.getVideoArgs(
575		preset,
576		base,
577		inputArgs,
578		);
	579	const args = transcodeRules.getVideoArgs(preset, base, inputArgs);
579	580	try {
580	581	const fakeProgress = new Progress({ text: spin.text, spinner: null });
581	582	fakeProgress.stop();
...	...	@@ -612,22 +613,25 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
612	613	const procCompression = [
613	614	{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
614	615	{ name: "zstd", fn: () => zlib.createZstdCompress() },
615		].map(({ name, fn }) => ({
616		name: `compress ${name}`,
617		exclude: rules.extsPreCompressed,
618		async run({ absPath, mediaFile }) {
619		if ((mediaFile.size ?? 0) < 10) return;
620		await produceAsset(`${mediaFile.hash}/${name}`, async (base) => {
621		fs.mkdirSync(path.dirname(base));
622		await stream.promises.pipeline(
623		fs.createReadStream(absPath),
624		fn(),
625		fs.createWriteStream(base),
626		);
627		return [base];
628		});
629		},
630		} satisfies Process as Process));
	616	].map(
	617	({ name, fn }) =>
	618	({
	619	name: `compress ${name}`,
	620	exclude: rules.extsPreCompressed,
	621	async run({ absPath, mediaFile }) {
	622	if ((mediaFile.size ?? 0) < 10) return;
	623	await produceAsset(`${mediaFile.hash}/${name}`, async (base) => {
	624	fs.mkdirSync(path.dirname(base));
	625	await stream.promises.pipeline(
	626	fs.createReadStream(absPath),
	627	fn(),
	628	fs.createWriteStream(base),
	629	);
	630	return [base];
	631	});
	632	},
	633	}) satisfies Process as Process,
	634	);
631	635
632	636	const processors = [
633	637	procDimensions,
...	...	@@ -637,30 +641,29 @@ const processors = [
637	641	procImageSubsets,
638	642	...procVideos,
639	643	...procCompression,
640		]
641		.map((process, id, all) => {
642		const strIndex = (id: number) =>
643		String.fromCharCode("a".charCodeAt(0) + id);
644		return {
645		...process as Process,
646		id: strIndex(id),
647		// Create a unique key.
648		hash: new Uint16Array(
649		crypto.createHash("sha1")
650		.update(
651		process.run.toString() +
652		(process.version ? String(process.version) : ""),
653		)
654		.digest().buffer,
655		).reduce((a, b) => a ^ b),
656		depends: (process.depends ?? []).map((depend) => {
657		const index = all.findIndex((p) => p.name === depend);
658		if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
659		if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
660		return strIndex(index);
661		}),
662		};
663		});
	644	].map((process, id, all) => {
	645	const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
	646	return {
	647	...(process as Process),
	648	id: strIndex(id),
	649	// Create a unique key.
	650	hash: new Uint16Array(
	651	crypto
	652	.createHash("sha1")
	653	.update(
	654	process.run.toString() +
	655	(process.version ? String(process.version) : ""),
	656	)
	657	.digest().buffer,
	658	).reduce((a, b) => a ^ b),
	659	depends: (process.depends ?? []).map((depend) => {
	660	const index = all.findIndex((p) => p.name === depend);
	661	if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
	662	if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
	663	return strIndex(index);
	664	}),
	665	};
	666	});
664	667
665	668	function resizeDimensions(w: number, h: number, desiredWidth: number) {
666	669	ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
...	...	@@ -676,10 +679,7 @@ async function produceAsset(
676	679	if (asset.refs === 1) {
677	680	const paths = await builder(path.join(workDir, key));
678	681	asset.addFiles(
679		paths.map((file) =>
680		path.relative(workDir, file)
681		.replaceAll("\\", "/")
682		),
	682	paths.map((file) => path.relative(workDir, file).replaceAll("\\", "/")),
683	683	);
684	684	}
685	685	return {
...	...	@@ -719,7 +719,7 @@ interface ProcessJob {
719	719	absPath: string;
720	720	stat: fs.Stats;
721	721	mediaFile: MediaFile;
722		processor: typeof processors[0];
	722	processor: (typeof processors)[0];
723	723	index: number;
724	724	after: ProcessJob[];
725	725	needs: number;

src/global.css+11-15

...	...	@@ -4,7 +4,9 @@
4	4	font-weight: 400 750;
5	5	font-style: normal;
6	6	font-display: swap;
7		font-variation-settings: "CASL" 0.25, "MONO" 0;
	7	font-variation-settings:
	8	"CASL" 0.25,
	9	"MONO" 0;
8	10	font-style: oblique -15deg 0deg;
9	11	unicode-range: U+0020-007E;
10	12	}
...	...	@@ -14,7 +16,9 @@
14	16	font-weight: 400 800;
15	17	font-style: normal;
16	18	font-display: swap;
17		font-variation-settings: "CASL" 0.25, "MONO" 1;
	19	font-variation-settings:
	20	"CASL" 0.25,
	21	"MONO" 1;
18	22	font-style: oblique -15deg 0deg;
19	23	unicode-range: U+0020-007E;
20	24	}
...	...	@@ -24,21 +28,13 @@
24	28	font-weight: 400 800;
25	29	font-style: normal;
26	30	font-display: swap;
27		font-variation-settings: "CASL" 0.25, "MONO" 1;
	31	font-variation-settings:
	32	"CASL" 0.25,
	33	"MONO" 1;
28	34	font-style: oblique -15deg 0deg;
29	35	unicode-range:
30		U+00C0-00FF,
31		U+00A9,
32		U+2190-2193,
33		U+2018,
34		U+2019,
35		U+201C,
36		U+201D,
37		U+2022,
38		U+00A0-00A8,
39		U+00AA-00BF,
40		U+2194-2199,
41		U+0100-017F;
	36	U+00C0-00FF, U+00A9, U+2190-2193, U+2018, U+2019, U+201C, U+201D, U+2022,
	37	U+00A0-00A8, U+00AA-00BF, U+2194-2199, U+0100-017F;
42	38	}
43	39
44	40	*,

src/pages/index.marko+1-1

...	...	@@ -29,7 +29,7 @@ export const meta: Meta = {
29	29	<main>
30	30	<div>
31	31	<h2>posts</h2>
32		<p>song: <span>in the summer</span> (coming soon, 2025-07-12)</p>
	32	<p>song: <a href="/in-the-summer">in the summer</a> (2025-01-01)</p>
33	33	<p>song: <a href="/waterfalls">waterfalls</a> (2025-01-01)</p>
34	34	<h2>things</h2>
35	35	<p><a href="/q+a">questions and answers</a></p>