Commit e6cca067561965523a27ce63fabb551d3a6ef32f | sitegen

author	clover caruso <git@paperclover.net> 2025-06-13 00:13:22-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log	e6cca067561965523a27ce63fabb551d3a6ef32f
tree	0ff132f436e571b5d34c10e645dbf60934bb29dd
parent	896eb7a6b0005ea9214fc7f59ee0a5ac5e5d5a6d
signature	Commit is signed but in an unrecognized format.

incremental sitegen dev server!

8 files changed, 535 insertions(+), 233 deletions(-)

framework

framework/bundle.ts+126-105

...	...	@@ -1,8 +1,4 @@
1	1	// This file implements client-side bundling, mostly wrapping esbuild.
2		const clientPlugins: esbuild.Plugin[] = [
3		// There are currently no plugins needed by 'paperclover.net'
4		];
5
6	2	export async function bundleClientJavaScript(
7	3	referencedScripts: string[],
8	4	extraPublicScripts: string[],
...	...	@@ -11,7 +7,7 @@ export async function bundleClientJavaScript(
11	7	) {
12	8	const entryPoints = [
13	9	...new Set([
14		...referencedScripts,
	10	...referencedScripts.map((file) => path.resolve(hot.projectSrc, file)),
15	11	...extraPublicScripts,
16	12	]),
17	13	];
...	...	@@ -26,6 +22,10 @@ export async function bundleClientJavaScript(
26	22	);
27	23	}
28	24
	25	const clientPlugins: esbuild.Plugin[] = [
	26	// There are currently no plugins needed by 'paperclover.net'
	27	];
	28
29	29	const bundle = await esbuild.build({
30	30	bundle: true,
31	31	chunkNames: "/js/c.[hash]",
...	...	@@ -51,7 +51,6 @@ export async function bundleClientJavaScript(
51	51	);
52	52	const { metafile } = bundle;
53	53	const promises: Promise<void>[] = [];
54		// TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
55	54	for (const file of bundle.outputFiles) {
56	55	const { text } = file;
57	56	let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
...	...	@@ -61,7 +60,7 @@ export async function bundleClientJavaScript(
61	60	// Register non-chunks as script entries.
62	61	const chunk = route.startsWith("/js/c.");
63	62	if (!chunk) {
64		const key = hot.getScriptId(sources[0]);
	63	const key = hot.getScriptId(path.resolve(sources[0]));
65	64	route = "/js/" + key + ".js";
66	65	incr.put({
67	66	sources,
...	...	@@ -82,41 +81,40 @@ export async function bundleClientJavaScript(
82	81	await Promise.all(promises);
83	82	}
84	83
85		type ServerPlatform = "node" \| "passthru";
	84	export type ServerPlatform = "node" \| "passthru";
86	85	export async function bundleServerJavaScript(
87		/** Has 'export default app;' */
88		_: string,
89		/** Views for dynamic loading */
90		viewEntryPoints: FileItem[],
	86	incr: Incremental,
91	87	platform: ServerPlatform = "node",
92	88	) {
93		const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
	89	if (incr.hasArtifact("backendBundle", platform)) return;
	90
	91	// Comment
	92	const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
	93
94	94	const viewSource = [
95		...viewEntryPoints.map((view, i) =>
96		`import * as view${i} from ${JSON.stringify(view.file)}`
	95	...Array.from(
	96	incr.out.viewMetadata,
	97	([, view], i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
97	98	),
98		`const styles = ${scriptMagic}[-2]`,
99		`export const scripts = ${scriptMagic}[-1]`,
	99	`const styles = ${magicWord}[-2]`,
	100	`export const scripts = ${magicWord}[-1]`,
100	101	"export const views = {",
101		...viewModules.flatMap(({ view, module }, i) => [
102		` ${JSON.stringify(view.id)}: {`,
103		` component: view${i}.default,`,
104		` meta: view${i}.meta,`,
105		` layout: ${
106		module.layout?.default ? `view${i}.layout?.default` : "undefined"
107		},`,
108		` theme: ${
109		module.layout?.theme
110		? `view${i}.layout?.theme`
111		: module.theme
112		? `view${i}.theme`
113		: "undefined"
114		},`,
115		` inlineCss: styles[${scriptMagic}[${i}]]`,
116		` },`,
117		]),
	102	...Array.from(incr.out.viewMetadata, ([key, view], i) =>
	103	[
	104	` ${JSON.stringify(key)}: {`,
	105	` component: view${i}.default,`,
	106	// ` meta: ${
	107	// view.staticMeta ? JSON.stringify(view.staticMeta) : `view${i}.meta`
	108	// },`,
	109	` meta: view${i}.meta,`,
	110	` layout: ${view.hasLayout ? `view${i}.layout?.default` : "null"},`,
	111	` inlineCss: styles[${magicWord}[${i}]]`,
	112	` },`,
	113	].join("\n")),
118	114	"}",
119	115	].join("\n");
	116
	117	// -- plugins --
120	118	const serverPlugins: esbuild.Plugin[] = [
121	119	virtualFiles({
122	120	"$views": viewSource,
...	...	@@ -125,10 +123,21 @@ export async function bundleServerJavaScript(
125	123	{
126	124	name: "marko via build cache",
127	125	setup(b) {
128		b.onLoad({ filter: /\.marko$/ }, async ({ path: file }) => ({
129		loader: "ts",
130		contents: hot.getSourceCode(file),
131		}));
	126	b.onLoad(
	127	{ filter: /\.marko$/ },
	128	async ({ path: file }) => {
	129	const key = path.relative(hot.projectRoot, file)
	130	.replaceAll("\\", "/");
	131	const cacheEntry = incr.out.serverMarko.get(key);
	132	if (!cacheEntry) {
	133	throw new Error("Marko file not in cache: " + file);
	134	}
	135	return ({
	136	loader: "ts",
	137	contents: cacheEntry.src,
	138	});
	139	},
	140	);
132	141	},
133	142	},
134	143	{
...	...	@@ -145,10 +154,10 @@ export async function bundleServerJavaScript(
145	154	},
146	155	},
147	156	];
148		const bundle = await esbuild.build({
	157	const { metafile, outputFiles, warnings } = await esbuild.build({
149	158	bundle: true,
150	159	chunkNames: "c.[hash]",
151		entryNames: "[name]",
	160	entryNames: "server",
152	161	entryPoints: [
153	162	path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
154	163	],
...	...	@@ -161,81 +170,96 @@ export async function bundleServerJavaScript(
161	170	write: false,
162	171	metafile: true,
163	172	});
164		const viewModules = viewEntryPoints.map((view) => {
165		const module = require(view.file);
166		if (!module.meta) {
167		throw new Error(`${view.file} is missing 'export const meta'`);
168		}
169		if (!module.default) {
170		throw new Error(`${view.file} is missing a default export.`);
	173
	174	const files: Record<string, Buffer> = {};
	175	let fileWithMagicWord: string \| null = null;
	176	for (const output of outputFiles) {
	177	const basename = output.path.replace(/^.*?!/, "");
	178	const key = "out!" + basename.replaceAll("\\", "/");
	179	// If this contains the generated "$views" file, then
	180	// mark this file as the one for replacement. Because
	181	// `splitting` is `true`, esbuild will not emit this
	182	// file in more than one chunk.
	183	if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
	184	fileWithMagicWord = basename;
171	185	}
172		return { module, view };
173		});
174		const viewData = viewModules.map(({ module, view }) => {
175		return {
176		id: view.id,
177		theme: module.theme,
178		cssImports: hot.getCssImports(view.file)
179		.concat("src/global.css")
180		.map((file) => path.resolve(file)),
181		clientRefs: hot.getClientScriptRefs(view.file),
182		};
	186	files[basename] = Buffer.from(output.contents);
	187	}
	188	incr.put({
	189	kind: "backendBundle",
	190	key: platform,
	191	value: {
	192	magicWord,
	193	files,
	194	fileWithMagicWord,
	195	},
	196	sources: Object.keys(metafile.inputs).filter((x) =>
	197	!x.startsWith("vfs:") &&
	198	!x.startsWith("dropped:") &&
	199	!x.includes("node_modules")
	200	),
183	201	});
184		return {
185		views: viewData,
186		bundle,
187		scriptMagic,
188		};
189	202	}
190	203
191		type Await<T> = T extends Promise<infer R> ? R : T;
192
193		export function finalizeServerJavaScript(
194		backend: Await<ReturnType<typeof bundleServerJavaScript>>,
195		viewCssBundles: css.Output[],
	204	export async function finalizeServerJavaScript(
196	205	incr: Incremental,
	206	platform: ServerPlatform,
197	207	) {
198		const { metafile, outputFiles } = backend.bundle;
	208	if (incr.hasArtifact("backendReplace", platform)) return;
	209	const {
	210	files,
	211	fileWithMagicWord,
	212	magicWord,
	213	} = UNWRAP(incr.getArtifact("backendBundle", platform));
199	214
200		// Only the reachable script files need to be inserted into the bundle.
201		const reachableScriptKeys = new Set(
202		backend.views.flatMap((view) => view.clientRefs),
203		);
204		const reachableScripts = Object.fromEntries(
205		Array.from(incr.out.script)
206		.filter(([k]) => reachableScriptKeys.has(k)),
	215	if (!fileWithMagicWord) return;
	216
	217	// Only the reachable resources need to be inserted into the bundle.
	218	const viewScriptsList = new Set(
	219	Array.from(incr.out.viewMetadata.values())
	220	.flatMap((view) => view.clientRefs),
207	221	);
	222	const viewStyleKeys = Array.from(incr.out.viewMetadata.values())
	223	.map((view) => css.styleKey(view.cssImports, view.theme));
	224	const viewCssBundles = viewStyleKeys
	225	.map((key) => UNWRAP(incr.out.style.get(key), "Style key: " + key));
208	226
209	227	// Deduplicate styles
210	228	const styleList = Array.from(new Set(viewCssBundles));
211	229
212		for (const output of outputFiles) {
213		const basename = output.path.replace(/^.*?!/, "");
214		const key = "out!" + basename.replaceAll("\\", "/");
	230	// Replace the magic word
	231	let text = files[fileWithMagicWord].toString("utf-8");
	232	text = text.replace(
	233	new RegExp(magicWord + "\\[(-?\\d)\\]", "gs"),
	234	(_, i) => {
	235	i = Number(i);
	236	// Inline the styling data
	237	if (i === -2) {
	238	return JSON.stringify(styleList.map((cssText) => cssText));
	239	}
	240	// Inline the script data
	241	if (i === -1) {
	242	return JSON.stringify(Object.fromEntries(incr.out.script));
	243	}
	244	// Reference an index into `styleList`
	245	return `${styleList.indexOf(viewCssBundles[i])}`;
	246	},
	247	);
215	248
216		// If this contains the generated "$views" file, then
217		// replace the IDs with the bundled results.
218		let text = output.text;
219		if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
220		text = text.replace(
221		/CLOVER_CLIENT_SCRIPTS_DEFINITION\[(-?\d)\]/gs,
222		(_, i) => {
223		i = Number(i);
224		// Inline the styling data
225		if (i === -2) {
226		return JSON.stringify(styleList.map((item) => item.text));
227		}
228		// Inline the script data
229		if (i === -1) {
230		return JSON.stringify(Object.fromEntries(incr.out.script));
231		}
232		// Reference an index into `styleList`
233		return `${styleList.indexOf(viewCssBundles[i])}`;
234		},
235		);
236		}
237		fs.writeMkdirSync(path.join(".clover/backend/" + basename), text);
238		}
	249	incr.put({
	250	kind: "backendReplace",
	251	key: platform,
	252	sources: [
	253	// Backend input code (includes view code)
	254	...incr.sourcesFor("backendBundle", platform),
	255	// Script
	256	...Array.from(viewScriptsList)
	257	.flatMap((key) => incr.sourcesFor("script", hot.getScriptId(key))),
	258	// Style
	259	...viewStyleKeys.flatMap((key) => incr.sourcesFor("style", key)),
	260	],
	261	value: Buffer.from(text),
	262	});
239	263	}
240	264
241	265	import * as esbuild from "esbuild";
...	...	@@ -248,7 +272,4 @@ import {
248	272	virtualFiles,
249	273	} from "./esbuild-support.ts";
250	274	import { Incremental } from "./incremental.ts";
251		import type { FileItem } from "#sitegen";
252		import * as marko from "@marko/compiler";
253	275	import * as css from "./css.ts";
254		import * as fs from "./lib/fs.ts";

framework/css.ts+5-5

...	...	@@ -51,10 +51,12 @@ export function styleKey(
51	51	) {
52	52	cssImports = cssImports
53	53	.map((file) =>
54		path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
	54	(path.isAbsolute(file) ? path.relative(hot.projectSrc, file) : file)
	55	.replaceAll("\\", "/")
55	56	)
56	57	.sort();
57		return cssImports.join(":") + JSON.stringify(theme);
	58	return cssImports.join(":") + ":" +
	59	Object.entries(theme).map(([k, v]) => `${k}=${v}`);
58	60	}
59	61
60	62	export async function bundleCssFiles(
...	...	@@ -62,9 +64,7 @@ export async function bundleCssFiles(
62	64	theme: Theme,
63	65	dev: boolean = false,
64	66	): Promise<Output> {
65		cssImports = cssImports.map((file) =>
66		path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
67		);
	67	cssImports = cssImports.map((file) => path.resolve(hot.projectSrc, file));
68	68	const plugin = {
69	69	name: "clover css",
70	70	setup(b) {

framework/generate.ts+48-36

...	...	@@ -2,23 +2,35 @@
2	2	// By using `Incremental`'s ability to automatically purge stale
3	3	// assets, the `sitegen` function performs partial rebuilds.
4	4
5		export function main(incremental?: Incremental) {
	5	export function main() {
6	6	return withSpinner<Record<string, unknown>, any>({
7	7	text: "Recovering State",
8	8	successText,
9	9	failureText: () => "sitegen FAIL",
10	10	}, async (spinner) => {
11		// const incr = Incremental.fromDisk();
12		// await incr.statAllFiles();
13		const incr = new Incremental();
	11	const incr = Incremental.fromDisk();
	12	await incr.statAllFiles();
	13	// const incr = new Incremental();
14	14	const result = await sitegen(spinner, incr);
15	15	incr.toDisk(); // Allows picking up this state again
16	16	return result;
17	17	}) as ReturnType<typeof sitegen>;
18	18	}
19	19
20		export function successText({ elapsed }: { elapsed: number }) {
21		return "sitegen! update in " + elapsed.toFixed(1) + "s";
	20	export function successText({
	21	elapsed,
	22	inserted,
	23	referenced,
	24	unreferenced,
	25	}: Awaited<ReturnType<typeof sitegen>>) {
	26	const s = (array: unknown[]) => array.length === 1 ? "" : "s";
	27	const kind = inserted.length === referenced.length ? "build" : "update";
	28	const status = inserted.length > 0
	29	? `${kind} ${inserted.length} key${s(inserted)}`
	30	: unreferenced.length > 0
	31	? `pruned ${unreferenced.length} key${s(unreferenced)}`
	32	: `checked ${referenced.length} key${s(referenced)}`;
	33	return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
22	34	}
23	35
24	36	export async function sitegen(
...	...	@@ -159,7 +171,10 @@ export async function sitegen(
159	171	...css.defaultTheme,
160	172	...pageTheme,
161	173	};
162		const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
	174	const cssImports = Array.from(
	175	new Set([globalCssPath, ...hot.getCssImports(item.file)]),
	176	(file) => path.relative(hot.projectSrc, file),
	177	);
163	178	ensureCssGetsBuilt(cssImports, theme, item.id);
164	179
165	180	// -- metadata --
...	...	@@ -216,7 +231,7 @@ export async function sitegen(
216	231	};
217	232	const cssImports = hot.getCssImports(view.file)
218	233	.concat("src/global.css")
219		.map((file) => path.relative(hot.projectRoot, path.resolve(file)));
	234	.map((file) => path.relative(hot.projectSrc, path.resolve(file)));
220	235	incr.put({
221	236	kind: "viewMetadata",
222	237	key: view.id,
...	...	@@ -250,6 +265,9 @@ export async function sitegen(
250	265	return !existing;
251	266	});
252	267
	268	// Load the marko cache before render modules are loaded
	269	incr.loadMarkoCache();
	270
253	271	// This is done in two passes so that a page that throws during evaluation
254	272	// will report "Load Render Module" instead of "Render Static Page".
255	273	const spinnerFormat = status.format;
...	...	@@ -281,22 +299,12 @@ export async function sitegen(
281	299	await viewQueue.done({ method: "stop" });
282	300	status.format = spinnerFormat;
283	301
284		// -- bundle backend and views --
285		// status.text = "Bundle backend code";
286		// const backend = await bundle.bundleServerJavaScript(
287		// join("backend.ts"),
288		// views,
289		// );
290		// const viewCssPromise = await Promise.all(
291		// backend.views.map((view) =>
292		// cssOnce.get(
293		// view.cssImports.join(":") + JSON.stringify(view.theme),
294		// () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
295		// )
296		// ),
297		// );
	302	// -- bundle server javascript (backend and views) --
	303	status.text = "Bundle JavaScript";
	304	incr.snapshotMarkoCache();
	305	const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
298	306
299		// -- bundle scripts --
	307	// -- bundle client javascript --
300	308	const referencedScripts = Array.from(
301	309	new Set(
302	310	[
...	...	@@ -317,19 +325,17 @@ export async function sitegen(
317	325	(script) => path.resolve(hot.projectSrc, script),
318	326	).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
319	327	const extraPublicScripts = scripts.map((entry) => entry.file);
320		const uniqueCount = new Set([
321		...referencedScripts,
322		...extraPublicScripts,
323		]).size;
324		status.text = `Bundle ${uniqueCount} Scripts`;
325		await bundle.bundleClientJavaScript(
	328	const clientJavaScriptPromise = bundle.bundleClientJavaScript(
326	329	referencedScripts,
327	330	extraPublicScripts,
328	331	incr,
329	332	);
330
331		// -- finalize backend bundle --
332		// await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
	333	await Promise.all([
	334	serverJavaScriptPromise,
	335	clientJavaScriptPromise,
	336	cssQueue.done({ method: "stop" }),
	337	]);
	338	await bundle.finalizeServerJavaScript(incr, "node");
333	339
334	340	// -- copy/compress static files --
335	341	async function doStaticFile(item: FileItem) {
...	...	@@ -353,8 +359,6 @@ export async function sitegen(
353	359	await staticQueue.done({ method: "stop" });
354	360	status.format = spinnerFormat;
355	361
356		await cssQueue.done({ method: "stop" });
357
358	362	// -- concatenate static rendered pages --
359	363	status.text = `Concat Pages`;
360	364	await Promise.all(pages.map(async (page) => {
...	...	@@ -399,11 +403,19 @@ export async function sitegen(
399	403	// to this point have been left as dangling promises.
400	404	await incr.wait();
401	405
	406	const { inserted, referenced, unreferenced } = incr.shake();
	407
402	408	// Flush the site to disk.
403	409	status.format = spinnerFormat;
404	410	status.text = `Incremental Flush`;
405		incr.flush(); // Write outputs
406		return { incr, elapsed: (performance.now() - startTime) / 1000 };
	411	incr.flush("node"); // Write outputs
	412	return {
	413	incr,
	414	inserted,
	415	referenced,
	416	unreferenced,
	417	elapsed: (performance.now() - startTime) / 1000,
	418	};
407	419	}
408	420
409	421	function getItemText({ file }: FileItem) {

framework/hot.ts+54-40

...	...	@@ -35,9 +35,10 @@ export interface FileStat {
35	35	lastModified: number;
36	36	imports: string[];
37	37	}
38		let fsGraph = new Map<string, FileStat>();
39		export function getFsGraph() {
40		return fsGraph;
	38	const fileStats = new Map<string, FileStat>();
	39
	40	export function getFileStat(filepath: string) {
	41	return fileStats.get(path.resolve(filepath));
41	42	}
42	43
43	44	function shouldTrackPath(filename: string) {
...	...	@@ -63,18 +64,16 @@ Module.prototype._compile = function (
63	64	const cssImportsMaybe: string[] = [];
64	65	const imports: string[] = [];
65	66	for (const { filename: file } of this.children) {
66		const relative = path.relative(projectRoot, file);
67		if (file.endsWith(".css")) cssImportsMaybe.push(relative);
	67	if (file.endsWith(".css")) cssImportsMaybe.push(file);
68	68	else {
69		const child = fsGraph.get(relative);
	69	const child = fileStats.get(file);
70	70	if (!child) continue;
71	71	const { cssImportsRecursive } = child;
72	72	if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
73		imports.push(relative);
	73	imports.push(file);
74	74	}
75	75	}
76		const relative = path.relative(projectRoot, filename);
77		fsGraph.set(relative, {
	76	fileStats.set(filename, {
78	77	cssImportsRecursive: cssImportsMaybe.length > 0
79	78	? Array.from(new Set(cssImportsMaybe))
80	79	: null,
...	...	@@ -155,36 +154,46 @@ function resolveClientRef(sourcePath: string, ref: string) {
155	154	) {
156	155	throw new Error("addScript must be a .client.ts or .client.tsx");
157	156	}
158		return filePath;
	157	return path.relative(projectSrc, filePath);
159	158	}
160	159
	160	// TODO: extract the marko compilation tools out, lazy load them
	161	export interface MarkoCacheEntry {
	162	src: string;
	163	scannedClientRefs: string[];
	164	}
	165	export const markoCache = new Map<string, MarkoCacheEntry>();
161	166	function loadMarko(module: NodeJS.Module, filepath: string) {
162		let src = fs.readFileSync(filepath, "utf8");
163		// A non-standard thing here is Clover Sitegen implements
164		// its own client side scripting stuff, so it overrides
165		// bare client import statements to it's own usage.
166		const scannedClientRefs = new Set<string>();
167		if (src.match(/^\s*client\s+import\s+["']/m)) {
168		src = src.replace(
169		/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
170		(_, src) => {
171		const ref = JSON.parse(`"${src.slice(1, -1)}"`);
172		const resolved = resolveClientRef(filepath, ref);
173		scannedClientRefs.add(resolved);
174		return `<CloverScriptInclude=${
175		JSON.stringify(getScriptId(resolved))
176		} />`;
177		},
178		) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
179		}
180
181		src = marko.compileSync(src, filepath).code;
182		src = src.replace("marko/debug/html", "#ssr/marko");
	167	let cache = markoCache.get(filepath);
	168	if (!cache) {
	169	let src = fs.readFileSync(filepath, "utf8");
	170	// A non-standard thing here is Clover Sitegen implements
	171	// its own client side scripting stuff, so it overrides
	172	// bare client import statements to it's own usage.
	173	const scannedClientRefs = new Set<string>();
	174	if (src.match(/^\s*client\s+import\s+["']/m)) {
	175	src = src.replace(
	176	/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
	177	(_, src) => {
	178	const ref = JSON.parse(`"${src.slice(1, -1)}"`);
	179	const resolved = resolveClientRef(filepath, ref);
	180	scannedClientRefs.add(resolved);
	181	return `<CloverScriptInclude=${
	182	JSON.stringify(getScriptId(resolved))
	183	} />`;
	184	},
	185	) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
	186	}
183	187
184		module.cloverSourceCode = src;
	188	src = marko.compileSync(src, filepath).code;
	189	src = src.replace("marko/debug/html", "#ssr/marko");
	190	cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
	191	markoCache.set(filepath, cache);
	192	}
185	193
	194	const { src, scannedClientRefs } = cache;
186	195	return loadEsbuildCode(module, filepath, src, {
187		scannedClientRefs: Array.from(scannedClientRefs),
	196	scannedClientRefs,
188	197	});
189	198	}
190	199
...	...	@@ -202,12 +211,19 @@ function loadCss(module: NodeJS.Module, _filepath: string) {
202	211	export function reloadRecursive(filepath: string) {
203	212	filepath = path.resolve(filepath);
204	213	const existing = cache[filepath];
205		if (existing) deleteRecursive(filepath, existing);
206		fsGraph.clear();
	214	if (existing) deleteRecursiveInner(filepath, existing);
	215	fileStats.clear();
207	216	return require(filepath);
208	217	}
209	218
210		function deleteRecursive(id: string, module: any) {
	219	export function unload(filepath: string) {
	220	filepath = path.resolve(filepath);
	221	const existing = cache[filepath];
	222	if (existing) delete cache[filepath];
	223	fileStats.delete(filepath);
	224	}
	225
	226	function deleteRecursiveInner(id: string, module: any) {
211	227	if (id.includes(path.sep + "node_modules" + path.sep)) {
212	228	return;
213	229	}
...	...	@@ -215,15 +231,14 @@ function deleteRecursive(id: string, module: any) {
215	231	for (const child of module.children) {
216	232	if (child.filename.includes("/engine/")) return;
217	233	const existing = cache[child.filename];
218		if (existing === child) deleteRecursive(child.filename, existing);
	234	if (existing === child) deleteRecursiveInner(child.filename, existing);
219	235	}
220	236	}
221	237
222	238	export function getCssImports(filepath: string) {
223	239	filepath = path.resolve(filepath);
224	240	if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
225		return fsGraph.get(path.relative(projectRoot, filepath))
226		?.cssImportsRecursive ?? [];
	241	return fileStats.get(filepath)?.cssImportsRecursive ?? [];
227	242	}
228	243
229	244	export function getClientScriptRefs(filepath: string) {
...	...	@@ -296,7 +311,6 @@ export function resolveClientRefs(
296	311
297	312	export function getScriptId(file: string) {
298	313	return (path.isAbsolute(file) ? path.relative(projectSrc, file) : file)
299		.replace(/^\/?src\//, "")
300	314	.replaceAll("\\", "/");
301	315	}
302	316

framework/incremental.ts+236-38

...	...	@@ -20,8 +20,19 @@ interface ArtifactMap {
20	20	pageMetadata: PageMetadata;
21	21	/* Metadata about a dynamic view */
22	22	viewMetadata: ViewMetadata;
	23	/* Cached '.marko' server compilation */
	24	serverMarko: hot.MarkoCacheEntry;
	25	/* Backend source code, pre-replacement. Keyed by platform type. */
	26	backendBundle: BackendBundle;
	27	/* One file in the backend receives post-processing. */
	28	backendReplace: Buffer;
23	29	}
24	30	type ArtifactKind = keyof ArtifactMap;
	31	/* Automatic path tracing is performed to make it so that
	32	* specifying 'sources: [file]' refers to it and everything it imports.
	33	* These kinds do not have that behavior
	34	*/
	35	const exactDependencyKinds = ["serverMarko"];
25	36	export interface Asset {
26	37	buffer: Buffer;
27	38	headers: Record<string, string \| undefined>;
...	...	@@ -50,6 +61,11 @@ export interface ViewMetadata {
50	61	clientRefs: string[];
51	62	hasLayout: boolean;
52	63	}
	64	export interface BackendBundle {
	65	magicWord: string;
	66	fileWithMagicWord: string \| null;
	67	files: Record<string, Buffer>;
	68	}
53	69
54	70	// -- incremental support types --
55	71	export interface PutBase {
...	...	@@ -76,6 +92,9 @@ export class Incremental {
76	92	style: new Map(),
77	93	pageMetadata: new Map(),
78	94	viewMetadata: new Map(),
	95	serverMarko: new Map(),
	96	backendBundle: new Map(),
	97	backendReplace: new Map(),
79	98	};
80	99	/** Tracking filesystem entries to `srcId` */
81	100	invals = new Map<SourceId, Invalidations>();
...	...	@@ -92,16 +111,52 @@ export class Incremental {
92	111	getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
93	112	});
94	113
	114	/** Reset at the end of each update */
	115	round = {
	116	inserted: new Set<ArtifactId>(),
	117	referenced: new Set<ArtifactId>(),
	118	};
	119
95	120	getArtifact<T extends ArtifactKind>(kind: T, key: string) {
	121	this.round.referenced.add(`${kind}\0${key}`);
96	122	return this.out[kind].get(key);
97	123	}
98	124
99		hasArtifact<T extends ArtifactKind>(kind: T, key: string) {
100		return this.out[kind].has(key);
	125	hasArtifact(kind: ArtifactKind, key: string) {
	126	return this.getArtifact(kind, key) != null;
101	127	}
102	128
103	129	sourcesFor(kind: ArtifactKind, key: string) {
104		return UNWRAP(this.sources.get(kind + "\0" + key));
	130	return UNWRAP(
	131	this.sources.get(kind + "\0" + key),
	132	`No artifact '${kind}' '${key}'`,
	133	);
	134	}
	135
	136	shake() {
	137	const toPublic = (str: string) => {
	138	const [kind, key] = str.split("\0");
	139	return { kind: kind as ArtifactKind, key };
	140	};
	141	const inserted = Array.from(this.round.inserted, toPublic);
	142	const referenced = Array.from(this.round.referenced, toPublic);
	143	const unreferenced: { kind: ArtifactKind; key: string }[] = [];
	144
	145	for (const kind in this.out) {
	146	const map = this.out[kind as keyof typeof this.out];
	147	if (!map) continue;
	148	for (const key of map.keys()) {
	149	if (!this.round.referenced.has(`${kind}\0${key}`)) {
	150	unreferenced.push({ kind: kind as ArtifactKind, key });
	151	console.warn("unreferened " + kind + " : " + key);
	152	}
	153	}
	154	}
	155
	156	this.round.inserted.clear();
	157	this.round.referenced.clear();
	158
	159	return { inserted, referenced, unreferenced };
105	160	}
106	161
107	162	/*
...	...	@@ -115,9 +170,25 @@ export class Incremental {
115	170	key,
116	171	value,
117	172	}: Put<T>) {
118		console.log("put " + kind + ": " + key);
	173	// These three invariants affect incremental accuracy.
	174	if (this.round.inserted.has(`${kind}\0${key}`)) {
	175	console.error(
	176	`Artifact ${kind}:${key} was inserted multiple times in the same round!`,
	177	);
	178	} else if (!this.round.referenced.has(`${kind}\0${key}`)) {
	179	console.error(
	180	`Artifact ${kind}:${key} was inserted without checking if (!hasArtifact())`,
	181	);
	182	} else if (this.out[kind].has(key)) {
	183	console.error(
	184	`Artifact ${kind}:${key} is not stale, but overwritten.`,
	185	);
	186	}
	187
119	188	this.out[kind].set(key, value);
120	189
	190	this.round.inserted.add(`${kind}\0${key}`);
	191
121	192	// Update sources information
122	193	ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
123	194	sources = sources.map((src) => path.normalize(src));
...	...	@@ -150,8 +221,7 @@ export class Incremental {
150	221	#getOrInitInvals(source: string) {
151	222	let invals = this.invals.get(source);
152	223	if (!invals) {
153		const g = hot.getFsGraph().get(source);
154		const lastModified = g?.lastModified ??
	224	const lastModified = hot.getFileStat(source)?.lastModified ??
155	225	fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;
156	226	this.invals.set(
157	227	source,
...	...	@@ -166,8 +236,7 @@ export class Incremental {
166	236	}
167	237
168	238	#followImports(file: string) {
169		const graph = hot.getFsGraph();
170		const stat = graph.get(file);
	239	const stat = hot.getFileStat(file);
171	240	if (!stat) return;
172	241	for (const i of stat.imports) {
173	242	const invals = this.#getOrInitInvals(i);
...	...	@@ -178,19 +247,23 @@ export class Incremental {
178	247
179	248	async statAllFiles() {
180	249	for (const file of this.invals.keys()) {
181		const mtime = fs.statSync(file).mtimeMs;
182		this.updateStat(file, mtime);
	250	try {
	251	const mtime = fs.statSync(file).mtimeMs;
	252	this.updateStat(file, mtime);
	253	} catch (err) {
	254	}
183	255	}
184	256	}
185	257
186		updateStat(file: string, newLastModified: number) {
	258	updateStat(file: string, newLastModified: number \| null) {
187	259	file = path.relative(hot.projectRoot, file);
188	260	const stat = this.invals.get(file);
189	261	ASSERT(stat, "Updated stat on untracked file " + file);
190		const hasUpdate = stat.lastModified < newLastModified;
	262	const hasUpdate = !newLastModified \|\| stat.lastModified < newLastModified;
191	263	if (hasUpdate) {
192	264	// Invalidate
193		console.info(file + " updated");
	265	console.info(file + " " + (newLastModified ? "updated" : "deleted"));
	266	hot.unload(file);
194	267	const invalidQueue = [file];
195	268	let currentInvalid;
196	269	while (currentInvalid = invalidQueue.pop()) {
...	...	@@ -204,12 +277,15 @@ export class Incremental {
204	277	for (const out of outputs) {
205	278	const [kind, artifactKey] = out.split("\0");
206	279	this.out[kind as ArtifactKind].delete(artifactKey);
207		console.log("stale " + kind + ": " + artifactKey);
208	280	}
209	281	invalidQueue.push(...files);
210	282	}
211	283	}
212		stat.lastModified = newLastModified;
	284	if (newLastModified) {
	285	stat.lastModified = newLastModified;
	286	} else {
	287	this.invals.delete(file);
	288	}
213	289	return hasUpdate;
214	290	}
215	291
...	...	@@ -266,6 +342,7 @@ export class Incremental {
266	342	serialize() {
267	343	const writer = new BufferWriter();
268	344
	345	// -- artifact --
269	346	const asset = Array.from(
270	347	this.out.asset,
271	348	([key, { buffer, hash, headers }]) => {
...	...	@@ -283,6 +360,31 @@ export class Incremental {
283	360	},
284	361	);
285	362	const script = Array.from(this.out.script);
	363	const style = Array.from(this.out.style);
	364	const pageMetadata = Array.from(this.out.pageMetadata);
	365	const viewMetadata = Array.from(this.out.viewMetadata);
	366	const serverMarko = Array.from(this.out.serverMarko);
	367	const backendBundle = Array.from(this.out.backendBundle, ([k, v]) => {
	368	return [k, {
	369	magicWord: v.magicWord,
	370	fileWithMagicWord: v.fileWithMagicWord,
	371	files: Object.entries(v.files).map(
	372	([file, contents]) => [
	373	file,
	374	writer.write(contents, "backendBundle" + k + ":" + file),
	375	],
	376	),
	377	}] satisfies SerializedMeta["backendBundle"][0];
	378	});
	379	const backendReplace = Array.from(
	380	this.out.backendReplace,
	381	([k, v]) =>
	382	[
	383	k,
	384	writer.write(v, "backendReplace" + k),
	385	] satisfies SerializedMeta["backendReplace"][0],
	386	);
	387	// -- incremental metadata --
286	388	const invals = Array.from(this.invals, ([key, value]) => {
287	389	const { lastModified, files, outputs } = value;
288	390	return [key, {
...	...	@@ -299,6 +401,12 @@ export class Incremental {
299	401	script,
300	402	invals,
301	403	sources,
	404	style,
	405	pageMetadata,
	406	viewMetadata,
	407	serverMarko,
	408	backendBundle,
	409	backendReplace,
302	410	} satisfies SerializedMeta;
303	411	const meta = Buffer.from(JSON.stringify(json), "utf-8");
304	412
...	...	@@ -317,21 +425,39 @@ export class Incremental {
317	425	buffer.subarray(4 + metaLength + start, 4 + metaLength + end);
318	426
319	427	const incr = new Incremental();
320		incr.out.asset = new Map<string, Asset>(meta.asset.map(([key, value]) => {
321		const { hash, raw, gzip, zstd, headers } = value;
322		if ((gzip \|\| zstd) && !incr.compress.has(hash)) {
323		incr.compress.set(hash, {
324		gzip: gzip ? view(gzip) : undefined,
325		zstd: zstd ? view(zstd) : undefined,
326		});
327		}
328		return [key, {
329		buffer: view(raw),
330		headers: headers,
331		hash: hash,
332		}];
333		}));
334		incr.out.script = new Map(meta.script);
	428	incr.out = {
	429	asset: new Map(meta.asset.map(([key, value]) => {
	430	const { hash, raw, gzip, zstd, headers } = value;
	431	if ((gzip \|\| zstd) && !incr.compress.has(hash)) {
	432	incr.compress.set(hash, {
	433	gzip: gzip ? view(gzip) : undefined,
	434	zstd: zstd ? view(zstd) : undefined,
	435	});
	436	}
	437	return [key, {
	438	buffer: view(raw),
	439	headers: headers,
	440	hash: hash,
	441	}];
	442	})),
	443	script: new Map(meta.script),
	444	style: new Map(meta.style),
	445	pageMetadata: new Map(meta.pageMetadata),
	446	viewMetadata: new Map(meta.viewMetadata),
	447	serverMarko: new Map(meta.serverMarko),
	448	backendBundle: new Map(meta.backendBundle.map(([key, value]) => {
	449	return [key, {
	450	magicWord: value.magicWord,
	451	fileWithMagicWord: value.fileWithMagicWord,
	452	files: Object.fromEntries(
	453	value.files.map(([file, contents]) => [file, view(contents)]),
	454	),
	455	}];
	456	})),
	457	backendReplace: new Map(
	458	meta.backendReplace.map(([key, contents]) => [key, view(contents)]),
	459	),
	460	};
335	461	incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {
336	462	return [key, {
337	463	lastModified: m,
...	...	@@ -343,6 +469,37 @@ export class Incremental {
343	469	return incr;
344	470	}
345	471
	472	/*
	473	* Move the cached (server) marko transpilations from this incremental
	474	* into the running process.
	475	*/
	476	loadMarkoCache() {
	477	hot.markoCache.clear();
	478	for (const [key, value] of this.out.serverMarko) {
	479	hot.markoCache.set(path.resolve(hot.projectRoot, key), value);
	480	}
	481	}
	482
	483	/*
	484	* Move the cached (server) marko transpilations from this incremental
	485	* into the running process.
	486	*/
	487	snapshotMarkoCache() {
	488	for (const [file, value] of hot.markoCache) {
	489	const key = path.relative(hot.projectRoot, file).replaceAll("\\", "/");
	490	// Only insert if it doesn't exist. Calling 'put' when it
	491	// already exists would inform the user of extra calls to put.
	492	if (!this.hasArtifact("serverMarko", key)) {
	493	this.put({
	494	kind: "serverMarko",
	495	sources: [file],
	496	key,
	497	value,
	498	});
	499	}
	500	}
	501	}
	502
346	503	toDisk(file = ".clover/incr.state") {
347	504	const buffer = this.serialize();
348	505	fs.writeFileSync(file, buffer);
...	...	@@ -362,23 +519,52 @@ export class Incremental {
362	519	await this.compressQueue.done({ method: "success" });
363	520	}
364	521
365		async flush() {
	522	async flush(
	523	platform: bundle.ServerPlatform,
	524	dir = path.resolve(".clover/out"),
	525	) {
366	526	ASSERT(!this.compressQueue.active);
	527	const join = (...args: string[]) => path.join(dir, ...args);
367	528	const writer = new BufferWriter();
368		// TODO: ensure all assets are actually compressed and not fake lying.
	529
	530	// TODO: ensure all compressed got compressed
	531
369	532	const asset = Object.fromEntries(
370	533	Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
371	534	const raw = writer.write(buffer, hash);
372	535	const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
373		const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
374		const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
	536	const gzip = writer.write(UNWRAP(gzipBuf), hash + ".gz");
	537	const zstd = writer.write(UNWRAP(zstdBuf), hash + ".zstd");
375	538	return [key, { raw, gzip, zstd, headers }];
376	539	}),
377	540	);
378		await Promise.all([
379		fs.writeFile(".clover/static.json", JSON.stringify(asset)),
380		fs.writeFile(".clover/static.blob", writer.get()),
381		]);
	541	const backendBundle = UNWRAP(this.out.backendBundle.get(platform));
	542
	543	// Arrange output files
	544	const outFiles: Array<[file: string, contents: string \| Buffer]> = [
	545	// Asset manifest
	546	["static.json", JSON.stringify(asset)],
	547	["static.blob", writer.get()],
	548
	549	// Backend
	550	...Object.entries(backendBundle.files).map(([subPath, contents]) =>
	551	[
	552	subPath,
	553	subPath === backendBundle.fileWithMagicWord
	554	? UNWRAP(this.out.backendReplace.get(platform))
	555	: contents,
	556	] as [string, Buffer]
	557	),
	558	];
	559
	560	// TODO: check duplicates
	561
	562	// Perform all i/o
	563	await Promise.all(
	564	outFiles.map(([subPath, contents]) =>
	565	fs.writeMkdir(join(subPath), contents, { flush: true })
	566	),
	567	);
382	568	}
383	569	}
384	570
...	...	@@ -440,6 +626,17 @@ export interface SerializedMeta {
440	626	headers: Record<string, string>;
441	627	}]>;
442	628	script: Array<[key: string, value: string]>;
	629	style: Array<[key: string, value: string]>;
	630	pageMetadata: Array<[key: string, PageMetadata]>;
	631	viewMetadata: Array<[key: string, ViewMetadata]>;
	632	serverMarko: Array<[key: string, hot.MarkoCacheEntry]>;
	633	backendBundle: Array<[platform: string, {
	634	magicWord: string;
	635	fileWithMagicWord: string \| null;
	636	files: Array<[string, View]>;
	637	}]>;
	638	backendReplace: Array<[key: string, View]>;
	639
443	640	invals: Array<[key: string, {
444	641	/** Modified */
445	642	m: number;
...	...	@@ -461,3 +658,4 @@ import * as mime from "#sitegen/mime";
461	658	import * as path from "node:path";
462	659	import { Buffer } from "node:buffer";
463	660	import * as css from "./css.ts";
	661	import type * as bundle from "./bundle.ts";

framework/lib/assets.ts+14-4

...	...	@@ -8,9 +8,10 @@ export type StaticPageId = string;
8	8
9	9	export async function reload() {
10	10	const [map, buf] = await Promise.all([
11		fs.readFile(".clover/static.json", "utf8"),
12		fs.readFile(".clover/static.blob"),
	11	fs.readFile(path.join(import.meta.dirname, "static.json"), "utf8"),
	12	fs.readFile(path.join(import.meta.dirname, "static.blob")),
13	13	]);
	14	console.log("new buffer loaded");
14	15	assets = {
15	16	map: JSON.parse(map),
16	17	buf,
...	...	@@ -18,8 +19,11 @@ export async function reload() {
18	19	}
19	20
20	21	export async function reloadSync() {
21		const map = fs.readFileSync(".clover/static.json", "utf8");
22		const buf = fs.readFileSync(".clover/static.blob");
	22	const map = fs.readFileSync(
	23	path.join(import.meta.dirname, "static.json"),
	24	"utf8",
	25	);
	26	const buf = fs.readFileSync(path.join(import.meta.dirname, "static.blob"));
23	27	assets = {
24	28	map: JSON.parse(map),
25	29	buf,
...	...	@@ -105,8 +109,14 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
105	109	return c.res = new Response(body, { headers, status });
106	110	}
107	111
	112	process.on("message", (msg: any) => {
	113	console.log({ msg });
	114	if (msg?.type === "clover.assets.reload") reload();
	115	});
	116
108	117	import * as fs from "#sitegen/fs";
109	118	import type { Context, Next } from "hono";
110	119	import type { StatusCode } from "hono/utils/http-status";
111	120	import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
112	121	import { Buffer } from "node:buffer";
	122	import * as path from "node:path";

framework/lib/fs.ts+8-2

...	...	@@ -22,9 +22,15 @@ export function mkdirSync(dir: string) {
22	22	return nodeMkdirSync(dir, { recursive: true });
23	23	}
24	24
25		export async function writeMkdir(file: string, contents: Buffer \| string) {
	25	export type WriteFileAsyncOptions = Parameters<typeof writeFile>[2];
	26
	27	export async function writeMkdir(
	28	file: string,
	29	contents: Buffer \| string,
	30	options?: WriteFileAsyncOptions,
	31	) {
26	32	await mkdir(path.dirname(file));
27		return writeFile(file, contents);
	33	return writeFile(file, contents, options);
28	34	}
29	35
30	36	export function writeMkdirSync(file: string, contents: Buffer \| string) {

framework/watch.ts+44-3

...	...	@@ -3,12 +3,39 @@
3	3	const debounceMilliseconds = 25;
4	4
5	5	export async function main() {
	6	let subprocess: child_process.ChildProcess \| null = null;
	7
6	8	// Catch up state by running a main build.
7	9	const { incr } = await generate.main();
8	10	// ...and watch the files that cause invals.
9	11	const watch = new Watch(rebuild);
10	12	watch.add(...incr.invals.keys());
11	13	statusLine();
	14	// ... an
	15	serve();
	16
	17	function serve() {
	18	if (subprocess) {
	19	subprocess.removeListener("close", onSubprocessClose);
	20	subprocess.kill();
	21	}
	22	subprocess = child_process.fork(".clover/out/server.js", [
	23	"--development",
	24	], {
	25	stdio: "inherit",
	26	});
	27	subprocess.on("close", onSubprocessClose);
	28	}
	29
	30	function onSubprocessClose(code: number \| null, signal: string \| null) {
	31	subprocess = null;
	32	const status = code != null ? `code ${code}` : `signal ${signal}`;
	33	console.error(`Backend process exited with ${status}`);
	34	}
	35
	36	process.on("beforeExit", () => {
	37	subprocess?.removeListener("close", onSubprocessClose);
	38	});
12	39
13	40	function rebuild(files: string[]) {
14	41	files = files.map((file) => path.relative(hot.projectRoot, file));
...	...	@@ -20,7 +47,7 @@ export async function main() {
20	47	console.warn("Files were modified but the 'modify' time did not change.");
21	48	return;
22	49	}
23		withSpinner<Record<string, unknown>, any>({
	50	withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
24	51	text: "Rebuilding",
25	52	successText: generate.successText,
26	53	failureText: () => "sitegen FAIL",
...	...	@@ -39,6 +66,20 @@ export async function main() {
39	66	if (!incr.invals.has(relative)) watch.remove(file);
40	67	}
41	68	return result;
	69	}).then((result) => {
	70	// Restart the server if it was changed or not running.
	71	if (
	72	!subprocess \|\|
	73	result.inserted.some(({ kind }) => kind === "backendReplace")
	74	) {
	75	serve();
	76	} else if (
	77	subprocess &&
	78	result.inserted.some(({ kind }) => kind === "asset")
	79	) {
	80	subprocess.send({ type: "clover.assets.reload" });
	81	}
	82	return result;
42	83	}).catch((err) => {
43	84	console.error(util.inspect(err));
44	85	}).finally(statusLine);
...	...	@@ -142,10 +183,10 @@ class Watch {
142	183	}
143	184	}
144	185
145		import { Incremental } from "./incremental.ts";
146	186	import * as fs from "node:fs";
147		import { Spinner, withSpinner } from "@paperclover/console/Spinner";
	187	import { withSpinner } from "@paperclover/console/Spinner";
148	188	import * as generate from "./generate.ts";
149	189	import * as path from "node:path";
150	190	import * as util from "node:util";
151	191	import * as hot from "./hot.ts";
	192	import * as child_process from "node:child_process";