Commit 98be3e2bd40a40c6595de522d087092df63cdbb7 | sitegen

flake.nix+23-23

...	...	@@ -3,27 +3,27 @@
3	3	nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
4	4	utils.url = "github:numtide/flake-utils";
5	5	};
6		outputs =
7		{ nixpkgs, utils, ... }:
8		utils.lib.eachDefaultSystem (
9		system:
10		let
11		pkgs = nixpkgs.legacyPackages.${system};
12		in
13		{
14		devShells.default = pkgs.mkShell {
15		buildInputs = [
16		pkgs.nodejs_24 # runtime
17		pkgs.deno # formatter
18		(pkgs.ffmpeg.override {
19		withOpus = true;
20		withSvtav1 = true;
21		withJxl = true;
22		withWebp = true;
23		})
24		pkgs.rsync
25		];
26		};
27		}
28		);
	6	outputs = inputs: inputs.utils.lib.eachDefaultSystem (system:
	7	with inputs.nixpkgs.legacyPackages.${system}; {
	8	devShells.default = pkgs.mkShell {
	9	buildInputs = [
	10	pkgs.nodejs_24 # runtime
	11	pkgs.deno # formatter
	12	(pkgs.ffmpeg.override {
	13	withOpus = true;
	14	withSvtav1 = true;
	15	withJxl = true;
	16	withWebp = true;
	17	})
	18	pkgs.rsync
	19	];
	20	};
	21	devShells.min = pkgs.mkShell {
	22	buildInputs = [
	23	pkgs.nodejs_24 # runtime
	24	pkgs.deno # formatter
	25	pkgs.rsync
	26	];
	27	};
	28	});
29	29	}

framework/bundle.ts+189-139

...	...	@@ -88,19 +88,19 @@ export async function bundleClientJavaScript(
88	88	for (const file of outputFiles) {
89	89	const { text } = file;
90	90	let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
91		const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
92		const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x));
	91	const { entryPoint } = UNWRAP(metafile.outputs["out!" + route]);
93	92
94	93	// Register non-chunks as script entries.
95	94	const chunk = route.startsWith("/js/c.");
96	95	if (!chunk) {
97		const key = hot.getScriptId(path.resolve(sources[sources.length - 1]));
	96	const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
	97	console.log(route, key);
98	98	route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
99	99	scripts[key] = text;
100	100	}
101	101	// Register chunks and public scripts as assets.
102	102	if (chunk \|\| publicScriptRoutes.includes(route)) {
103		p.push(io.writeAsset(route, text));
	103	p.push(io.writeAsset({ pathname: route, buffer: text }));
104	104	}
105	105	}
106	106	await Promise.all(p);
...	...	@@ -111,7 +111,7 @@ export type ServerPlatform = "node" \| "passthru";
111	111	export interface ServerSideOptions {
112	112	entries: string[];
113	113	viewItems: sg.FileItem[];
114		viewRefs: incr.Ref<PreparedView>[];
	114	viewRefs: incr.Ref<PageOrView>[];
115	115	styleMap: Map<string, incr.Ref<string>>;
116	116	scriptMap: incr.Ref<Record<string, string>>;
117	117	platform: ServerPlatform;
...	...	@@ -124,144 +124,192 @@ export async function bundleServerJavaScript({
124	124	entries,
125	125	platform,
126	126	}: ServerSideOptions) {
127		const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => {
128		const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
129		return {
130		magicWord,
131		file: [
132		...viewItems.map(
133		(view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
134		),
135		`const styles = ${magicWord}[-2]`,
136		`export const scripts = ${magicWord}[-1]`,
137		"export const views = {",
138		...viewItems.map((view, i) =>
139		[
140		` ${JSON.stringify(view.id)}: {`,
141		` component: view${i}.default,`,
142		` meta: view${i}.meta,`,
143		` layout: view${i}.layout?.default ?? null,`,
144		` inlineCss: styles[${magicWord}[${i}]]`,
145		` },`,
146		].join("\n"),
147		),
148		"}",
149		].join("\n"),
150		};
151		}, viewItems);
152
153		const wBundles = entries.map(
154		(entry) =>
155		[
156		entry,
157		incr.work(async (io, entry) => {
158		const pkg = await io.readJson<{
159		dependencies: Record<string, string>;
160		}>("package.json");
	127	const regenKeys: Record<string, string[]> = {};
	128	const regenTtls: view.Ttl[] = [];
	129	for (const ref of viewRefs) {
	130	const value = UNWRAP(ref.value);
	131	if (value.type === "page" && (value.regenerate?.tags?.length ?? 0) > 0) {
	132	for (const tag of value.regenerate!.tags!) {
	133	(regenKeys[tag] ??= []).push(`page:${value.id}`);
	134	}
	135	}
	136	if (value.type === "page" && (value.regenerate?.seconds ?? 0) > 0) {
	137	regenTtls.push({
	138	key: `page:${value.id}` as view.Key,
	139	seconds: value.regenerate!.seconds!,
	140	});
	141	}
	142	}
161	143
162		let magicWord = null as string \| null;
163		// -- plugins --
164		const serverPlugins: esbuild.Plugin[] = [
165		virtualFiles({
166		// only add dependency when imported.
167		$views: async () => {
168		const view = await io.readWork(wViewSource);
169		({ magicWord } = view);
170		return view.file;
171		},
172		}),
173		projectRelativeResolution(),
174		markoViaBuildCache(),
175		{
176		name: "replace client references",
177		setup(b) {
178		b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({
179		contents: hot.resolveClientRefs(
180		await fs.readFile(file, "utf-8"),
181		file,
182		).code,
183		loader: path.extname(file).slice(1) as esbuild.Loader,
184		}));
185		},
186		},
187		{
188		name: "mark css external",
189		setup(b) {
190		b.onResolve({ filter: /\.css$/ }, () => ({
191		path: ".",
192		namespace: "dropped",
193		}));
194		b.onLoad({ filter: /./, namespace: "dropped" }, () => ({
195		contents: "",
196		}));
197		},
198		},
199		];
	144	const wViewSource = incr.work(
	145	async (
	146	_,
	147	{ viewItems, regenKeys, regenTtls }: {
	148	viewItems: sg.FileItem[];
	149	regenKeys: Record<string, string[]>;
	150	regenTtls: view.Ttl[];
	151	},
	152	) => {
	153	const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
	154	return {
	155	magicWord,
	156	file: [
	157	...viewItems.map(
	158	(view, i) =>
	159	`import * as view${i} from ${JSON.stringify(view.file)}`,
	160	),
	161	`const styles = ${magicWord}[-2]`,
	162	`export const scripts = ${magicWord}[-1]`,
	163	"export const views = {",
	164	...viewItems.map((view, i) =>
	165	[
	166	` ${JSON.stringify(view.id)}: {`,
	167	` component: view${i}.default,`,
	168	` meta: view${i}.meta,`,
	169	` layout: view${i}.layout?.default ?? null,`,
	170	` inlineCss: styles[${magicWord}[${i}]]`,
	171	` },`,
	172	].join("\n")
	173	),
	174	"}",
	175	`export const regenTags = ${JSON.stringify(regenKeys)};`,
	176	`export const regenTtls = ${JSON.stringify(regenTtls)};`,
	177	].join("\n"),
	178	};
	179	},
	180	{ viewItems, regenKeys, regenTtls },
	181	);
200	182
201		const { metafile, outputFiles } = await esbuild.build({
202		bundle: true,
203		chunkNames: "c.[hash]",
204		entryNames: path.basename(entry, path.extname(entry)),
205		entryPoints: [
206		path.join(
207		import.meta.dirname,
208		"backend/entry-" + platform + ".ts",
209		),
210		],
211		platform: "node",
212		format: "esm",
213		minify: false,
214		outdir: "out!",
215		plugins: serverPlugins,
216		splitting: true,
217		logLevel: "silent",
218		write: false,
219		metafile: true,
220		jsx: "automatic",
221		jsxImportSource: "#engine",
222		jsxDev: false,
223		define: {
224		MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
225		"globalThis.CLOVER_SERVER_ENTRY": JSON.stringify(entry),
226		},
227		external: Object.keys(pkg.dependencies).filter(
228		(x) => !x.startsWith("@paperclover"),
	183	await incr.work(
	184	async (io, { regenKeys, viewItems }) => {
	185	io.writeFile(
	186	"../ts/view.d.ts",
	187	[
	188	"export interface RegisteredViews {",
	189	...viewItems
	190	.filter((view) => !view.id.startsWith("page:"))
	191	.map(
	192	(view) =>
	193	` ${JSON.stringify(view.id)}: ` +
	194	`typeof import(${
	195	JSON.stringify(path.relative(".clover/ts", toAbs(view.file)))
	196	}),`,
229	197	),
230		});
231		await trackEsbuild(io, metafile);
	198	"}",
	199	"export type RegenKey = " +
	200	(regenKeys.map((key) => JSON.stringify(key)).join(" \| ") \|\|
	201	"never"),
	202	].join("\n"),
	203	);
	204	},
	205	{ regenKeys: Object.keys(regenKeys), viewItems },
	206	);
232	207
233		let fileWithMagicWord: {
234		bytes: Buffer;
235		basename: string;
236		magicWord: string;
237		} \| null = null;
238		for (const output of outputFiles) {
239		const basename = output.path.replace(/^.*?!(?:\/\|\\)/, "");
240		const key = "out!/" + basename.replaceAll("\\", "/");
241		// If this contains the generated "$views" file, then
242		// mark this file as the one for replacement. Because
243		// `splitting` is `true`, esbuild will not emit this
244		// file in more than one chunk.
245		if (
246		magicWord &&
247		metafile.outputs[key].inputs["framework/lib/view.ts"]
248		) {
249		ASSERT(!fileWithMagicWord);
250		fileWithMagicWord = {
251		basename,
252		bytes: Buffer.from(output.contents),
253		magicWord,
254		};
255		} else {
256		io.writeFile(basename, Buffer.from(output.contents));
257		}
258		}
259		return fileWithMagicWord;
260		}, entry),
261		] as const,
	208	const wBundles = entries.map((entry) =>
	209	incr.work(async (io, entry) => {
	210	const pkg = await io.readJson<{
	211	dependencies: Record<string, string>;
	212	}>("package.json");
	213
	214	let magicWord = null as string \| null;
	215	// -- plugins --
	216	const serverPlugins: esbuild.Plugin[] = [
	217	virtualFiles({
	218	// only add dependency when imported.
	219	$views: async () => {
	220	const view = await io.readWork(wViewSource);
	221	({ magicWord } = view);
	222	return view.file;
	223	},
	224	}),
	225	projectRelativeResolution(),
	226	markoViaBuildCache(),
	227	{
	228	name: "replace client references",
	229	setup(b) {
	230	b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({
	231	contents: hot.resolveClientRefs(
	232	await fs.readFile(file, "utf-8"),
	233	file,
	234	).code,
	235	loader: path.extname(file).slice(1) as esbuild.Loader,
	236	}));
	237	},
	238	},
	239	{
	240	name: "mark css external",
	241	setup(b) {
	242	b.onResolve({ filter: /\.css$/ }, () => ({
	243	path: ".",
	244	namespace: "dropped",
	245	}));
	246	b.onLoad({ filter: /./, namespace: "dropped" }, () => ({
	247	contents: "",
	248	}));
	249	},
	250	},
	251	];
	252
	253	const { metafile, outputFiles } = await esbuild.build({
	254	bundle: true,
	255	chunkNames: "c.[hash]",
	256	entryNames: path.basename(entry, path.extname(entry)),
	257	entryPoints: [
	258	path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
	259	],
	260	platform: "node",
	261	format: "esm",
	262	minify: false,
	263	outdir: "out!",
	264	plugins: serverPlugins,
	265	splitting: true,
	266	logLevel: "silent",
	267	write: false,
	268	metafile: true,
	269	jsx: "automatic",
	270	jsxImportSource: "#engine",
	271	jsxDev: false,
	272	define: {
	273	MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
	274	"globalThis.CLOVER_SERVER_ENTRY": JSON.stringify(entry),
	275	},
	276	external: Object.keys(pkg.dependencies).filter(
	277	(x) => !x.startsWith("@paperclover"),
	278	),
	279	});
	280	await trackEsbuild(io, metafile);
	281
	282	let fileWithMagicWord: {
	283	bytes: Buffer;
	284	basename: string;
	285	magicWord: string;
	286	} \| null = null;
	287	for (const output of outputFiles) {
	288	const basename = output.path.replace(/^.*?!(?:\/\|\\)/, "");
	289	const key = "out!/" + basename.replaceAll("\\", "/");
	290	// If this contains the generated "$views" file, then
	291	// mark this file as the one for replacement. Because
	292	// `splitting` is `true`, esbuild will not emit this
	293	// file in more than one chunk.
	294	if (
	295	magicWord &&
	296	UNWRAP(metafile.outputs[key]).inputs["framework/lib/view.ts"]
	297	) {
	298	ASSERT(!fileWithMagicWord);
	299	fileWithMagicWord = {
	300	basename,
	301	bytes: Buffer.from(output.contents),
	302	magicWord,
	303	};
	304	} else {
	305	io.writeFile(basename, Buffer.from(output.contents));
	306	}
	307	}
	308	return fileWithMagicWord;
	309	}, entry)
262	310	);
263	311
264		const wProcessed = wBundles.map(async ([entry, wBundle]) => {
	312	const wProcessed = wBundles.map(async (wBundle) => {
265	313	if (!(await wBundle)) return;
266	314	await incr.work(async (io) => {
267	315	// Only the reachable resources need to be read and inserted into the bundle.
...	...	@@ -278,7 +326,7 @@ export async function bundleServerJavaScript({
278	326	const viewStyleKeys = views.map((view) => view.styleKey);
279	327	const viewCssBundles = await Promise.all(
280	328	viewStyleKeys.map((key) =>
281		io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key)),
	329	io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key))
282	330	),
283	331	);
284	332	const styleList = Array.from(new Set(viewCssBundles));
...	...	@@ -297,7 +345,7 @@ export async function bundleServerJavaScript({
297	345	return JSON.stringify(Object.fromEntries(neededScripts));
298	346	}
299	347	// Reference an index into `styleList`
300		return `${styleList.indexOf(viewCssBundles[i])}`;
	348	return `${styleList.indexOf(UNWRAP(viewCssBundles[i]))}`;
301	349	});
302	350
303	351	io.writeFile(basename, text);
...	...	@@ -322,3 +370,5 @@ import * as fs from "#sitegen/fs";
322	370	import * as mime from "#sitegen/mime";
323	371	import * as incr from "./incremental.ts";
324	372	import * as sg from "#sitegen";
	373	import type { PageOrView } from "./generate.ts";
	374	import type * as view from "#sitegen/view";

framework/css.ts+9-6

...	...	@@ -57,12 +57,14 @@ export function styleKey(
57	57	export async function bundleCssFiles(
58	58	io: Io,
59	59	{ cssImports, theme, dev }: {
60		cssImports: string[],
61		theme: Theme,
62		dev: boolean,
63		}
	60	cssImports: string[];
	61	theme: Theme;
	62	dev: boolean;
	63	},
64	64	) {
65		cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file)));
	65	cssImports = await Promise.all(
	66	cssImports.map((file) => io.trackFile("src/" + file)),
	67	);
66	68	const plugin = {
67	69	name: "clover css",
68	70	setup(b) {
...	...	@@ -111,4 +113,5 @@ import * as esbuild from "esbuild";
111	113	import * as fs from "#sitegen/fs";
112	114	import * as hot from "./hot.ts";
113	115	import * as path from "node:path";
114		import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts";
	116	import { virtualFiles } from "./esbuild-support.ts";
	117	import type { Io } from "./incremental.ts";

framework/debug.safe.ts+1-1

...	...	@@ -9,4 +9,4 @@ globalThis.UNWRAP = (t, ...args) => {
9	9	globalThis.ASSERT = assert.ok;
10	10
11	11	import * as util from "node:util";
12		import * as assert from 'node:assert'
	12	import * as assert from "node:assert";

framework/engine/jsx-runtime.ts+1-1

...	...	@@ -51,4 +51,4 @@ declare global {
51	51	}
52	52	}
53	53
54		import * as render from "./render.ts";
	54	import * as render from "#engine/render";

framework/engine/marko-runtime.ts+1

...	...	@@ -18,6 +18,7 @@ export const createTemplate = (
18	18	const r = render.current;
19	19	// Support using Marko outside of Clover SSR
20	20	if (!r) return renderer(props, n);
	21	render.setCurrent(null);
21	22	const markoResult = renderFn.call(renderer, {
22	23	...props,
23	24	$global: { clover: r, cloverAsyncMarker },

framework/engine/render.test.tsx+1-1

...	...	@@ -1,5 +1,5 @@
1	1	import { test } from "node:test";
2		import * as render from "./render.ts";
	2	import * as render from "#engine/render";
3	3
4	4	test("sanity", (t) => t.assert.equal(render.sync("gm <3").text, "gm <3"));
5	5	test("simple tree", (t) =>

framework/engine/render.ts+30-8

...	...	@@ -118,8 +118,11 @@ export function resolveNode(r: State, node: unknown): ResolvedNode {
118	118	if (!node && node !== 0) return ""; // falsy, non numeric
119	119	if (typeof node !== "object") {
120	120	if (node === true) return ""; // booleans are ignored
121		if (typeof node === "string") return escapeHtml(node);
	121	if (typeof node === "string") return escapeHtmlContent(node);
122	122	if (typeof node === "number") return String(node); // no escaping ever
	123	if (typeof node === "symbol" && node.toString() === kElement.toString()) {
	124	throw new Error(`There are two instances of Clover SSR loaded!`);
	125	}
123	126	throw new Error(`Cannot render ${inspect(node)} to HTML`);
124	127	}
125	128	if (node instanceof Promise) {
...	...	@@ -217,12 +220,14 @@ function stringifyElement(element: ResolvedElement) {
217	220	let attr;
218	221	switch (prop) {
219	222	default:
220		attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`;
	223	attr = `${prop}=${quoteIfNeeded(escapeAttribute(String(value)))}`;
221	224	break;
222	225	case "className":
223	226	// Legacy React Compat
224	227	case "class":
225		attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`;
	228	attr = `class=${
	229	quoteIfNeeded(escapeAttribute(clsx(value as ClsxInput)))
	230	}`;
226	231	break;
227	232	case "htmlFor":
228	233	throw new Error("Do not use the `htmlFor` attribute. Use `for`");
...	...	@@ -233,7 +238,7 @@ function stringifyElement(element: ResolvedElement) {
233	238	case "key":
234	239	continue;
235	240	}
236		if (needSpace) (out += " "), (needSpace = !attr.endsWith('"'));
	241	if (needSpace) ((out += " "), (needSpace = !attr.endsWith('"')));
237	242	out += attr;
238	243	}
239	244	out += ">";
...	...	@@ -254,14 +259,16 @@ export function stringifyStyleAttribute(style: Record<string, string>) {
254	259	let out = ``;
255	260	for (const styleName in style) {
256	261	if (out) out += ";";
257		out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${escapeHtml(
258		String(style[styleName]),
259		)}`;
	262	out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${
	263	escapeAttribute(
	264	String(style[styleName]),
	265	)
	266	}`;
260	267	}
261	268	return "style=" + quoteIfNeeded(out);
262	269	}
263	270	export function quoteIfNeeded(text: string) {
264		if (text.includes(" ")) return '"' + text + '"';
	271	if (text.match(/["/>]/)) return '"' + text + '"';
265	272	return text;
266	273	}
267	274
...	...	@@ -303,6 +310,21 @@ export function clsx(mix: ClsxInput) {
303	310	return str;
304	311	}
305	312
	313	export const escapeHtmlContent = (unsafeText: string) =>
	314	String(unsafeText)
	315	.replace(/&/g, "&")
	316	.replace(/</g, "<")
	317	.replace(/>/g, ">")
	318	.replace(/"/g, """);
	319	// TODO: combine into one function which decides if an attribute needs quotes
	320	// and escapes it correctly depending on the context.
	321	const escapeAttribute = (unsafeText: string) =>
	322	String(unsafeText)
	323	.replace(/&/g, "&")
	324	.replace(/</g, "<")
	325	.replace(/>/g, ">")
	326	.replace(/"/g, """);
	327	/** @deprecated */
306	328	export const escapeHtml = (unsafeText: string) =>
307	329	String(unsafeText)
308	330	.replace(/&/g, "&")

framework/engine/suspense.ts+2-2

...	...	@@ -42,7 +42,7 @@ export function Suspense({ children, fallback }: SuspenseProps): render.Node {
42	42	r.asyncDone = () => {
43	43	const rejections = r.rejections;
44	44	if (rejections && rejections.length > 0) throw new Error("TODO");
45		state.pushChunk?.(name, (ip[0] = resolved));
	45	state.pushChunk?.(name, ip[0] = resolved);
46	46	};
47	47	return render.raw(ip);
48	48	}
...	...	@@ -99,4 +99,4 @@ export async function* renderStreaming<
99	99	return addonOutput as unknown as T;
100	100	}
101	101
102		import * as render from "./render.ts";
	102	import * as render from "#engine/render";

framework/esbuild-support.ts+10-5

...	...	@@ -1,7 +1,12 @@
1	1	type Awaitable<T> = T \| Promise<T>;
2	2
3	3	export function virtualFiles(
4		map: Record<string, string \| esbuild.OnLoadResult \| (() => Awaitable<string \| esbuild.OnLoadResult>)>,
	4	map: Record<
	5	string,
	6	\| string
	7	\| esbuild.OnLoadResult
	8	\| (() => Awaitable<string \| esbuild.OnLoadResult>)
	9	>,
5	10	) {
6	11	return {
7	12	name: "clover vfs",
...	...	@@ -22,7 +27,7 @@ export function virtualFiles(
22	27	{ filter: /./, namespace: "vfs" },
23	28	async ({ path }) => {
24	29	let entry = map[path];
25		if (typeof entry === 'function') entry = await entry();
	30	if (typeof entry === "function") entry = await entry();
26	31	return ({
27	32	resolveDir: ".",
28	33	loader: "ts",
...	...	@@ -88,7 +93,6 @@ export function markoViaBuildCache(): esbuild.Plugin {
88	93	if (!fs.existsSync(file)) {
89	94	console.warn(`File does not exist: ${file}`);
90	95	}
91		console.log(markoCache.keys());
92	96	throw new Error("Marko file not in cache: " + file);
93	97	}
94	98	return ({
...	...	@@ -106,7 +110,7 @@ export function isIgnoredSource(source: string) {
106	110	return source.includes("<define:") \|\|
107	111	source.startsWith("vfs:") \|\|
108	112	source.startsWith("dropped:") \|\|
109		source.includes("node_modules")
	113	source.includes("node_modules");
110	114	}
111	115
112	116	import * as esbuild from "esbuild";
...	...	@@ -114,4 +118,5 @@ import * as string from "#sitegen/string";
114	118	import * as path from "node:path";
115	119	import * as fs from "#sitegen/fs";
116	120	import * as incr from "./incremental.ts";
117		import * as hot from "./hot.ts";import { markoCache } from "./marko.ts";
	121	import * as hot from "./hot.ts";
	122	import { markoCache } from "./marko.ts";

framework/generate.ts+63-36

...	...	@@ -7,15 +7,16 @@ const { toRel, toAbs } = incr;
7	7	const globalCssPath = toAbs("src/global.css");
8	8
9	9	export async function main() {
10		await incr.restore();
	10	if (!process.argv.includes("-f")) await incr.restore();
11	11	await incr.compile(generate);
12	12	}
13	13
14	14	export async function generate() {
15	15	// -- read config and discover files --
16	16	const siteConfig = await incr.work(readManifest);
17		const { staticFiles, scripts, views, pages } =
18		await discoverAllFiles(siteConfig);
	17	const { staticFiles, scripts, views, pages } = await discoverAllFiles(
	18	siteConfig,
	19	);
19	20
20	21	// TODO: make sure that `static` and `pages` does not overlap
21	22
...	...	@@ -28,13 +29,21 @@ export async function generate() {
28	29	staticFiles.map((item) =>
29	30	incr.work(
30	31	async (io, { id, file }) =>
31		void (await io.writeAsset(id, await io.readFile(file))),
	32	void (await io.writeAsset({
	33	pathname: id,
	34	buffer: await io.readFile(file),
	35	})),
32	36	item,
33		),
	37	)
34	38	),
35	39	);
36	40	const routes = await Promise.all([...builtViews, ...builtPages]);
37	41
	42	const viewsAndDynPages: incr.Ref<PageOrView>[] = [
	43	...builtViews,
	44	...builtPages.filter((page) => UNWRAP(page.value).regenerate),
	45	];
	46
38	47	// -- page resources --
39	48	const scriptMap = incr.work(bundle.bundleClientJavaScript, {
40	49	clientRefs: routes.flatMap((x) => x.clientRefs),
...	...	@@ -49,13 +58,16 @@ export async function generate() {
49	58	platform: "node",
50	59	styleMap,
51	60	scriptMap,
52		viewItems: views,
53		viewRefs: builtViews,
	61	viewItems: viewsAndDynPages.map((ref) => {
	62	const { id, file, type } = UNWRAP(ref.value);
	63	return { id: type === "page" ? `page:${id}` : id, file };
	64	}),
	65	viewRefs: viewsAndDynPages,
54	66	});
55	67
56	68	// -- assemble page assets --
57	69	const pAssemblePages = builtPages.map((page) =>
58		assembleAndWritePage(page, styleMap, scriptMap),
	70	assembleAndWritePage(page, styleMap, scriptMap)
59	71	);
60	72
61	73	await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
...	...	@@ -77,7 +89,7 @@ export async function discoverAllFiles(
77	89	return (
78	90	await Promise.all(
79	91	siteConfig.siteSections.map(({ root: sectionRoot }) =>
80		incr.work(scanSiteSection, toAbs(sectionRoot)),
	92	incr.work(scanSiteSection, toAbs(sectionRoot))
81	93	),
82	94	)
83	95	).reduce((acc, next) => ({
...	...	@@ -105,10 +117,9 @@ export async function scanSiteSection(io: Io, sectionRoot: string) {
105	117	let scripts: FileItem[] = [];
106	118
107	119	const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
108		const rootPrefix =
109		hot.projectSrc === sectionRoot
110		? ""
111		: path.relative(hot.projectSrc, sectionRoot) + "/";
	120	const rootPrefix = hot.projectSrc === sectionRoot
	121	? ""
	122	: path.relative(hot.projectSrc, sectionRoot) + "/";
112	123	const kinds = [
113	124	{
114	125	dir: sectionPath("pages"),
...	...	@@ -181,7 +192,8 @@ export async function preparePage(io: Io, item: sg.FileItem) {
181	192	meta: metadata,
182	193	theme: pageTheme,
183	194	layout,
184		} = await io.import<any>(item.file);
	195	regenerate,
	196	} = await io.import<sg.PageExports>(item.file);
185	197	if (!Page) throw new Error("Page is missing a 'default' export.");
186	198	if (!metadata) throw new Error("Page is missing 'meta' export with a title.");
187	199
...	...	@@ -219,16 +231,20 @@ export async function preparePage(io: Io, item: sg.FileItem) {
219	231	"Page is missing 'meta.title'. " + "All pages need a title tag.",
220	232	);
221	233	}
	234
222	235	const styleKey = css.styleKey(cssImports, theme);
223	236	return {
	237	type: "page",
224	238	id: item.id,
	239	file: item.file,
	240	regenerate,
225	241	html: text,
226	242	meta: renderedMeta,
227	243	cssImports,
228	244	theme: theme ?? null,
229	245	styleKey,
230	246	clientRefs: Array.from(addon[sg.userData.key].scripts),
231		};
	247	} as const;
232	248	}
233	249
234	250	export async function prepareView(io: Io, item: sg.FileItem) {
...	...	@@ -243,13 +259,15 @@ export async function prepareView(io: Io, item: sg.FileItem) {
243	259	);
244	260	const styleKey = css.styleKey(cssImports, theme);
245	261	return {
246		file: path.relative(hot.projectRoot, item.file),
	262	type: "view",
	263	id: item.id,
	264	file: item.file,
247	265	cssImports,
248	266	theme,
249	267	clientRefs: hot.getClientScriptRefs(item.file),
250	268	hasLayout: !!module.layout?.default,
251	269	styleKey,
252		};
	270	} as const;
253	271	}
254	272	export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
255	273
...	...	@@ -274,28 +292,38 @@ export async function assembleAndWritePage(
274	292	scriptWork: incr.Ref<Record<string, string>>,
275	293	) {
276	294	const page = await pageWork;
277		return incr.work(async (io, { id, html, meta, styleKey, clientRefs }) => {
278		const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
	295	return incr.work(
	296	async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
	297	const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
279	298
280		const scriptIds = clientRefs.map(hot.getScriptId);
281		const scriptMap = await io.readWork(scriptWork);
282		const scripts = scriptIds
283		.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
284		.map((x) => `{${x}}`)
285		.join("\n");
	299	const scriptIds = clientRefs.map(hot.getScriptId);
	300	const scriptMap = await io.readWork(scriptWork);
	301	const scripts = scriptIds
	302	.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
	303	.map((x) => `{${x}}`)
	304	.join("\n");
286	305
287		const doc = wrapDocument({
288		body: html,
289		head: meta,
290		inlineCss,
291		scripts,
292		});
293		await io.writeAsset(id, doc, {
294		"Content-Type": "text/html",
295		});
296		}, page);
	306	const buffer = sg.wrapDocument({
	307	body: html,
	308	head: meta,
	309	inlineCss,
	310	scripts,
	311	});
	312	await io.writeAsset({
	313	pathname: id,
	314	buffer,
	315	headers: {
	316	"Content-Type": "text/html",
	317	},
	318	regenerative: !!regenerate,
	319	});
	320	},
	321	page,
	322	);
297	323	}
298	324
	325	export type PageOrView = PreparedPage \| PreparedView;
	326
299	327	import * as sg from "#sitegen";
300	328	import * as incr from "./incremental.ts";
301	329	import { Io } from "./incremental.ts";
...	...	@@ -307,4 +335,3 @@ import * as fs from "#sitegen/fs";
307	335	import type { FileItem } from "#sitegen";
308	336	import * as path from "node:path";
309	337	import * as meta from "#sitegen/meta";
310		import { wrapDocument } from "./lib/view.ts";

framework/hot.ts+6-8

...	...	@@ -78,10 +78,9 @@ Module.prototype._compile = function (
78	78	}
79	79	}
80	80	fileStats.set(filename, {
81		cssImportsRecursive:
82		cssImportsMaybe.length > 0
83		? Array.from(new Set(cssImportsMaybe))
84		: null,
	81	cssImportsRecursive: cssImportsMaybe.length > 0
	82	? Array.from(new Set(cssImportsMaybe))
	83	: null,
85	84	imports,
86	85	lastModified: Math.floor(stat.mtimeMs),
87	86	});
...	...	@@ -137,14 +136,13 @@ export function loadEsbuildCode(
137	136	src = code;
138	137	}
139	138	if (src.includes("import.meta")) {
140		src =
141		`
	139	src = `
142	140	import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
143	141	import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
144	142	import.meta.filename = ${JSON.stringify(filepath)};
145	143	`
146		.trim()
147		.replace(/[\n\s]/g, "") + src;
	144	.trim()
	145	.replace(/[\n\s]/g, "") + src;
148	146	}
149	147	src = esbuild.transformSync(src, {
150	148	loader,

framework/incremental.test.ts+11-11

...	...	@@ -7,38 +7,38 @@ test("trivial case", async () => {
7	7	async function compilation() {
8	8	const first = incr.work({
9	9	label: "first compute",
10		async run (io) {
	10	async run(io) {
11	11	await setTimeout(1000);
12	12	const contents = await io.readFile(file1.path);
13	13	return [contents, Math.random()] as const;
14		}
	14	},
15	15	});
16	16	const second = incr.work({
17	17	label: "second compute",
18	18	wait: first,
19		async run (io) {
	19	async run(io) {
20	20	await setTimeout(1000);
21	21	return io.readWork(first)[0].toUpperCase();
22		}
	22	},
23	23	});
24	24	const third = incr.work({
25	25	label: "third compute",
26	26	wait: first,
27		async run (io) {
	27	async run(io) {
28	28	await setTimeout(1000);
29	29	return io.readWork(first)[1] * 1000;
30		}
	30	},
31	31	});
32	32	return incr.work({
33	33	label: "last compute",
34	34	wait: [second, third],
35		async run (io) {
	35	async run(io) {
36	36	await setTimeout(1000);
37	37	return {
38	38	second: io.readWork(second),
39	39	third: io.readWork(third),
40		}
41		}
	40	};
	41	},
42	42	});
43	43	}
44	44	const { value: first } = await incr.compile(compilation);
...	...	@@ -52,5 +52,5 @@ test("trivial case", async () => {
52	52
53	53	import * as incr from "./incremental2.ts";
54	54	import { beforeEach, test } from "node:test";
55		import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises";
56
	55	import { tmpFile } from "#sitegen/testing";
	56	import { setTimeout } from "node:timers/promises";

framework/incremental.ts+186-142

...	...	@@ -2,8 +2,7 @@
2	2	// See `work()`, `compile()`, and `invalidate()` for details.
3	3	//
4	4	// All state is serializable to allow recovering state across sessions.
5		// This framework special-cases the asset map, but is otherwise
6		// agnostic of what it is a compiler for.
	5	// This library special-cases the asset map, but is otherwise agnostic.
7	6	let running = false;
8	7	let jobs = 0;
9	8	let newKeys = 0;
...	...	@@ -15,27 +14,25 @@ let writes = new Map<string, FileWrite>();
15	14	let assets = new Map<string, Asset>(); // keyed by hash
16	15
17	16	export interface Ref<T> {
18		/** This method is compatible with `await` syntax */
19		then(
20		onFulfilled: (value: T) => void,
21		onRejected: (error: unknown) => void,
22		): void;
23	17	key: string;
	18	/** This method is compatible with `await` syntax */
	19	then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
	20	get value(): T \| null;
24	21	}
25	22	type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
26	23
27	24	/**
28		* Declare and a unit of work. Return value is memoized and
29		* only rebuilt when inputs (declared via `Io`) change. Outputs
30		* are written at the end of a compilation (see `compile`).
	25	* Declare and a unit of work. Return value is memoized and only rebuilt when
	26	* inputs change. Inputs are tracked via the `io` interface, as well as a hash
	27	* of the `input` param and caller source code. Outputs are written at the end
	28	* of a compilation (see `compile`).
31	29	*
32		* If the returned `Ref` is not awaited or read
33		* via io.readWork, the job is never started.
	30	* Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
34	31	*/
35	32	export function work<O>(job: Job<void, O>): Ref<O>;
36	33	export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;
37	34	export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
38		const source = JSON.stringify(util.getCallSites(2)[1]);
	35	const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1]));
39	36	const keySource = [source, util.inspect(input)].join(":");
40	37	const key = crypto.createHash("sha1").update(keySource).digest("base64url");
41	38	ASSERT(running);
...	...	@@ -48,7 +45,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
48	45
49	46	const prev = works.get(key) as Work<O> \| null;
50	47	if (prev) {
51		return { key, then: (done) => done(prev.value) };
	48	return { key, then: (done) => done(prev.value), value: prev.value };
52	49	}
53	50
54	51	async function perform() {
...	...	@@ -59,13 +56,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
59	56	const value = await job(io, input);
60	57	validateSerializable(value, "");
61	58	const { reads, writes } = io;
62		works.set(key, {
63		value,
64		affects: [],
65		reads,
66		writes,
67		debug: source,
68		});
	59	works.set(key, { value, affects: [], reads, writes });
69	60	for (const add of reads.files) {
70	61	const { affects } = UNWRAP(files.get(add));
71	62	ASSERT(!affects.includes(key));
...	...	@@ -85,7 +76,12 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
85	76	let cached: Promise<O>;
86	77	return {
87	78	key,
88		then: (fufill, reject) => void (cached ??= perform()).then(fufill, reject),
	79	then(fufill, reject) {
	80	(cached ??= perform()).then(fufill, reject);
	81	},
	82	get value() {
	83	return (works.get(this.key)?.value as O) ?? null;
	84	},
89	85	};
90	86	}
91	87
...	...	@@ -97,11 +93,7 @@ export async function compile<T>(compiler: () => Promise<T>) {
97	93	const start = performance.now();
98	94	const timerSpinner = new Spinner({
99	95	text: () =>
100		`sitegen! [${
101		((performance.now() - start) / 1000).toFixed(
102		1,
103		)
104		}s]`,
	96	`sitegen! [${((performance.now() - start) / 1000).toFixed(1)}s]`,
105	97	fps: 10,
106	98	});
107	99	using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
...	...	@@ -114,8 +106,10 @@ export async function compile<T>(compiler: () => Promise<T>) {
114	106	return {
115	107	value,
116	108	watchFiles: new Set(files.keys()),
117		newOutputs: Array.from(seenWrites).filter(x => x.startsWith('f:')).map(x => x.slice(2)),
118		newAssets: !Array.from(seenWrites).some(x => x.startsWith('a:')),
	109	newOutputs: Array.from(seenWrites)
	110	.filter((x) => x.startsWith("f:"))
	111	.map((x) => x.slice(2)),
	112	newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
119	113	};
120	114	} finally {
121	115	running = false;
...	...	@@ -127,9 +121,10 @@ export async function compile<T>(compiler: () => Promise<T>) {
127	121
128	122	export async function flush(start: number) {
129	123	// Trim
130		const detachedFiles = new Set<string>;
131		const referencedAssets = new Set<string>;
132		for (const [k, { writes: { assets } }] of works) {
	124	const detachedFiles = new Set<string>();
	125	const referencedAssets = new Set<string>();
	126	for (const [k, v] of works) {
	127	const assets = v.writes.assets;
133	128	if (seenWorks.has(k)) {
134	129	for (const asset of assets.values()) referencedAssets.add(asset.hash);
135	130	continue;
...	...	@@ -141,12 +136,9 @@ export async function flush(start: number) {
141	136	files.delete(k);
142	137	detachedFiles.add(k);
143	138	}
144		for (const k of assets.keys()) {
145		if (!referencedAssets.has(k))
146		assets.delete(k);
147		}
	139	for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
148	140
149		const p = [];
	141	const p: Promise<void>[] = [];
150	142	// File writes
151	143	let dist = 0;
152	144	for (const [key, { buffer, size }] of writes) {
...	...	@@ -155,10 +147,14 @@ export async function flush(start: number) {
155	147	}
156	148	// Asset map
157	149	{
158		const { json, blob } = getAssetManifest();
	150	const { json, blob, dynamic, dts } = getAssetManifest();
159	151	const jsonString = Buffer.from(JSON.stringify(json));
160		p.push(fs.writeMkdir(".clover/o/static.json", jsonString));
161		p.push(fs.writeMkdir(".clover/o/static.blob", blob));
	152	p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
	153	p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
	154	p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
	155	for (const [k, v] of dynamic) {
	156	p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
	157	}
162	158	dist += blob.byteLength + jsonString.byteLength;
163	159	}
164	160	await Promise.all(p);
...	...	@@ -171,9 +167,8 @@ export async function flush(start: number) {
171	167	console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);
172	168	console.writeLine(` - ${assets.size} static assets`);
173	169	console.writeLine(
174		` - dist: ${formatSize(dist)}, incremental: ${
175		formatSize(serialized.byteLength)
176		}`,
	170	` - dist: ${formatSize(dist)},` +
	171	` incremental: ${formatSize(serialized.byteLength)}`,
177	172	);
178	173	}
179	174
...	...	@@ -212,13 +207,14 @@ function deleteWork(key: string) {
212	207	affects.splice(affects.indexOf(key), 1);
213	208	}
214	209	for (const remove of affects) {
215		const { reads: { works: list } } = UNWRAP(works.get(remove), remove);
	210	const {
	211	reads: { works: list },
	212	} = UNWRAP(works.get(remove), remove);
216	213	ASSERT(list.has(key));
217	214	list.delete(key);
218	215	}
219	216	for (const file of w.files) {
220		if (UNWRAP(writes.get(file)).work === key)
221		writes.delete(file);
	217	if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
222	218	}
223	219	// Assets are temporarily kept, trimmed via manual GC after compilation.
224	220
...	...	@@ -234,23 +230,32 @@ export function reset() {
234	230	}
235	231
236	232	export function serialize() {
237		const fileEntries = Array.from(files, ([k, v]) =>
238		[
239		k,
240		v.type,
241		v.type === 'f' ? v.lastModified : v.type === 'd' ? v.contentHash : null,
242		...v.affects,
243		] as const);
244		const workEntries = Array.from(works, ([k, v]) =>
245		[
246		k,
247		v.value,
248		Array.from(v.reads.files),
249		Array.from(v.reads.works),
250		Array.from(v.writes.files),
251		Array.from(v.writes.assets, ([k, { headers }]) => [k, headers] as const),
252		v.affects,
253		] as const);
	233	const fileEntries = Array.from(
	234	files,
	235	([k, v]) =>
	236	[
	237	k,
	238	v.type,
	239	v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
	240	...v.affects,
	241	] as const,
	242	);
	243	const workEntries = Array.from(
	244	works,
	245	([k, v]) =>
	246	[
	247	k,
	248	v.value,
	249	Array.from(v.reads.files),
	250	Array.from(v.reads.works),
	251	Array.from(v.writes.files),
	252	Array.from(
	253	v.writes.assets,
	254	([k, { headers }]) => [k, headers] as const,
	255	),
	256	v.affects,
	257	] as const,
	258	);
254	259	const expectedFilesOnDisk = Array.from(
255	260	writes,
256	261	([k, { size, work }]) => [k, size, work] as const,
...	...	@@ -280,7 +285,7 @@ async function deserialize(buffer: Buffer) {
280	285	if (type === "f") {
281	286	ASSERT(typeof content === "number");
282	287	files.set(k, { type, affects, lastModified: content });
283		} else if (type === 'd') {
	288	} else if (type === "d") {
284	289	ASSERT(typeof content === "string");
285	290	files.set(k, { type, affects, contentHash: content, contents: [] });
286	291	} else {
...	...	@@ -288,15 +293,8 @@ async function deserialize(buffer: Buffer) {
288	293	}
289	294	}
290	295	for (const entry of workEntries) {
291		const [
292		k,
293		value,
294		readFiles,
295		readWorks,
296		writeFiles,
297		writeAssets,
298		affects,
299		] = entry;
	296	const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] =
	297	entry;
300	298	works.set(k, {
301	299	value,
302	300	reads: {
...	...	@@ -305,23 +303,30 @@ async function deserialize(buffer: Buffer) {
305	303	},
306	304	writes: {
307	305	files: new Set(writeFiles),
308		assets: new Map(Array.from(writeAssets, ([k, headers]) => [k, {
309		hash: JSON.parse(UNWRAP(headers.etag)),
310		headers,
311		}])),
	306	assets: new Map(
	307	Array.from(writeAssets, ([k, headers]) => [
	308	k,
	309	{
	310	hash: JSON.parse(UNWRAP(headers.etag)),
	311	headers,
	312	},
	313	]),
	314	),
312	315	},
313	316	affects,
314	317	});
315	318	}
316		const statFiles = await Promise.all(expectedFilesOnDisk
317		.map(([k, size, work]) =>
318		fs.stat(path.join(".clover/o", k))
	319	const statFiles = await Promise.all(
	320	expectedFilesOnDisk.map(([k, size, work]) =>
	321	fs
	322	.stat(path.join(".clover/o", k))
319	323	.catch((err) => {
320	324	if (err.code === "ENOENT") return null;
321	325	throw err;
322	326	})
323	327	.then((stat) => ({ k, size, work, stat }))
324		));
	328	),
	329	);
325	330	for (const { k, stat, work, size } of statFiles) {
326	331	if (stat?.size === size) {
327	332	writes.set(k, {
...	...	@@ -337,19 +342,28 @@ async function deserialize(buffer: Buffer) {
337	342	assets.set(hash, { raw, gzip, zstd });
338	343	}
339	344
340		await Promise.all(Array.from(files, ([key, file]) => invalidateEntry(key, file)));
	345	await Promise.all(
	346	Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
	347	);
341	348	}
342	349
343		export async function invalidate(filePath: string): Promise<boolean> {
	350	export async function invalidate(
	351	filePath: string,
	352	unloadModule: boolean = true,
	353	): Promise<boolean> {
344	354	const key = toRel(toAbs(filePath));
345		const file = UNWRAP(files.get(key), `Untracked file '${key}'`)
346		return invalidateEntry(key, file)
	355	const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
	356	return invalidateEntry(key, file, unloadModule);
347	357	}
348	358
349		export async function invalidateEntry(key: string, file: TrackedFile): Promise<boolean> {
	359	export async function invalidateEntry(
	360	key: string,
	361	file: TrackedFile,
	362	unloadModule: boolean,
	363	): Promise<boolean> {
350	364	try {
351	365	if (file.type === "d") {
352		const contents = file.contents = await fs.readdir(key);
	366	const contents = (file.contents = await fs.readdir(key));
353	367	contents.sort();
354	368	const contentHash = crypto
355	369	.createHash("sha1")
...	...	@@ -359,48 +373,79 @@ export async function invalidateEntry(key: string, file: TrackedFile): Promise<b
359	373	file.contentHash = contentHash;
360	374	throw new Error();
361	375	}
362		} else if (file.type === 'f') {
363		const lastModified = await fs.stat(key)
364		.then(x => Math.floor(x.mtimeMs), () => 0);
	376	} else if (file.type === "f") {
	377	const lastModified = await fs.stat(key).then(
	378	(x) => Math.floor(x.mtimeMs),
	379	() => 0,
	380	);
365	381	if (file.lastModified !== lastModified) {
366	382	file.lastModified = lastModified;
367	383	throw new Error();
368	384	}
369	385	} else {
370		file.type satisfies 'null';
	386	file.type satisfies "null";
371	387	const stat = await fs.stat(key).catch(() => null);
372		if (stat) throw new Error();
	388	if (stat) throw new Error();
373	389	}
374	390	return false;
375	391	} catch (e) {
376	392	forceInvalidate(file);
377		hot.unload(toAbs(key));
378		if (file.type === 'null') files.delete(key);
	393	if (unloadModule) {
	394	// TODO: handle when this triggers unloading of `generate.ts`
	395	hot.unload(toAbs(key));
	396	}
	397	if (file.type === "null") files.delete(key);
379	398	return true;
380	399	}
381	400	}
382	401
383	402	export function getAssetManifest() {
	403	const dynamic = new Map<string, Buffer>();
384	404	const writer = new BufferWriter();
385		const asset = Object.fromEntries(
	405	const assetMap = Object.fromEntries(
386	406	Array.from(works, (work) => work[1].writes.assets)
387	407	.filter((map) => map.size > 0)
388	408	.flatMap((map) =>
389		Array.from(map, ([key, { hash, headers }]) => {
	409	Array.from(map, ([key, { hash, headers, regenerative }]) => {
390	410	const { raw, gzip, zstd } = UNWRAP(
391	411	assets.get(hash),
392	412	`Asset ${key} (${hash})`,
393	413	);
394		return [key, {
395		raw: writer.write(raw, "raw:" + hash),
396		gzip: writer.write(gzip, "gzip:" + hash),
397		zstd: writer.write(zstd, "zstd:" + hash),
398		headers,
399		}] as const;
	414	if (regenerative) {
	415	const id = crypto
	416	.createHash("sha1")
	417	.update(key)
	418	.digest("hex")
	419	.slice(0, 16); /* TODO */
	420	dynamic.set(
	421	id,
	422	manifest.packDynamicBuffer(raw, gzip, zstd, headers),
	423	);
	424	return [key, { type: 1, id }] as const;
	425	}
	426	return [
	427	key,
	428	{
	429	type: 0,
	430	raw: writer.write(raw, "raw:" + hash),
	431	gzip: writer.write(gzip, "gzip:" + hash),
	432	zstd: writer.write(zstd, "zstd:" + hash),
	433	headers,
	434	},
	435	] as const;
400	436	})
401	437	),
402		) satisfies BuiltAssetMap;
403		return { json: asset, blob: writer.get() };
	438	) satisfies manifest.Manifest;
	439	return {
	440	json: assetMap,
	441	blob: writer.get(),
	442	dynamic,
	443	dts: "export type AssetKey = " +
	444	Object.keys(assetMap)
	445	.map((key) => JSON.stringify(key))
	446	.join(" \| ") +
	447	"\n",
	448	};
404	449	}
405	450
406	451	/* Input/Output with automatic tracking.
...	...	@@ -444,9 +489,9 @@ export class Io {
444	489	}
445	490	async readDir(dir: string) {
446	491	const { key, resolved } = this.#trackFs(dir);
447		const existing = files.get(key);
	492	const existing = files.get(key);
448	493	try {
449		if (existing?.type === 'd') return existing.contents;
	494	if (existing?.type === "d") return existing.contents;
450	495	const contents = await fs.readdir(resolved);
451	496	contents.sort();
452	497	const contentHash = crypto
...	...	@@ -512,26 +557,27 @@ export class Io {
512	557	}
513	558	}
514	559	}
515		async writeAsset(
516		pathname: string,
517		blob: string \| Buffer,
518		headersOption?: HeadersInit,
519		) {
520		ASSERT(pathname.startsWith("/"));
521		ASSERT(!seenWrites.has("a:" + pathname));
522
523		const buffer = typeof blob === "string" ? Buffer.from(blob) : blob;
524
525		const headers = new Headers(headersOption ?? {});
	560	async writeAsset(asset: {
	561	pathname: string;
	562	buffer: string \| Buffer;
	563	regenerative?: boolean;
	564	headers?: HeadersInit;
	565	}) {
	566	ASSERT(asset.pathname.startsWith("/"));
	567	ASSERT(!seenWrites.has("a:" + asset.pathname));
	568	const buffer = typeof asset.buffer === "string"
	569	? Buffer.from(asset.buffer)
	570	: asset.buffer;
	571	const headers = new Headers(asset.headers ?? {});
526	572	const hash = crypto.createHash("sha1").update(buffer).digest("hex");
527	573	if (!headers.has("Content-Type")) {
528		headers.set("Content-Type", mime.contentTypeFor(pathname));
	574	headers.set("Content-Type", mime.contentTypeFor(asset.pathname));
529	575	}
530		headers.set("ETag", JSON.stringify(hash));
531		this.writes.assets.set(pathname, {
	576	headers.set("etag", JSON.stringify(hash));
	577	this.writes.assets.set(asset.pathname, {
532	578	hash,
533		// @ts-expect-error TODO
534	579	headers: Object.fromEntries(headers),
	580	regenerative: !!asset.regenerative,
535	581	});
536	582	if (!assets.has(hash)) {
537	583	jobs += 1;
...	...	@@ -593,7 +639,10 @@ export function validateSerializable(value: unknown, key: string) {
593	639	} else if (value && typeof value === "object") {
594	640	if (Array.isArray(value)) {
595	641	value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
596		} else if (Object.getPrototypeOf(value) === Object.prototype \|\| Buffer.isBuffer(value)) {
	642	} else if (
	643	Object.getPrototypeOf(value) === Object.prototype \|\|
	644	Buffer.isBuffer(value)
	645	) {
597	646	Object.entries(value).forEach(([k, v]) =>
598	647	validateSerializable(v, `${key}.${k}`)
599	648	);
...	...	@@ -631,53 +680,48 @@ interface FileWrite {
631	680	}
632	681	interface Writes {
633	682	files: Set<string>;
634		assets: Map<string, {
635		hash: string;
636		headers: Record<string, string>;
637		}>;
	683	assets: Map<string, AssetWrite>;
638	684	}
639	685	interface Asset {
640	686	raw: Buffer;
641	687	gzip: Buffer;
642	688	zstd: Buffer;
643	689	}
	690	interface AssetWrite {
	691	hash: string;
	692	headers: Record<string, string>;
	693	regenerative: boolean;
	694	}
644	695	interface Work<T = unknown> {
645		debug?: string;
646	696	value: T;
647	697	reads: Reads;
648	698	writes: Writes;
649	699	affects: string[];
650	700	}
651	701	type TrackedFile =
652		& {
653		affects: string[];
654		}
	702	& { affects: string[] }
655	703	& (
656	704	\| { type: "f"; lastModified: number }
657	705	\| { type: "d"; contentHash: string; contents: string[] }
658		\| { type: "null"; }
	706	\| { type: "null" }
659	707	);
660		export interface BuiltAssetMap {
661		[route: string]: BuiltAsset;
662		}
663		export interface BuiltAsset {
664		raw: BufferView;
665		gzip: BufferView;
666		zstd: BufferView;
667		headers: Record<string, string>;
668		}
669	708
670	709	const gzip = util.promisify(zlib.gzip);
671	710	const zstdCompress = util.promisify(zlib.zstdCompress);
672	711
	712	import * as hot from "./hot.ts";
	713
673	714	import * as fs from "#sitegen/fs";
	715	import * as mime from "#sitegen/mime";
	716	import * as manifest from "#sitegen/assets";
	717
674	718	import * as path from "node:path";
675		import * as hot from "./hot.ts";
676	719	import * as util from "node:util";
677	720	import * as crypto from "node:crypto";
678		import * as mime from "#sitegen/mime";
679	721	import * as zlib from "node:zlib";
	722
680	723	import * as console from "@paperclover/console";
681	724	import { Spinner } from "@paperclover/console/Spinner";
682	725	import { formatSize } from "@/file-viewer/format.ts";
	726
683	727	import * as msgpackr from "msgpackr";

framework/lib/assets.ts+167-43

...	...	@@ -1,73 +1,114 @@
	1	// Static and dynamic assets are built alongside the server binary.
	2	// This module implements decoding and serving of the asset blobs,
	3	// but also implements patching of dynamic assets. The `Manifest`
	4	// is generated by `incremental.ts`
	5	const root = import.meta.dirname;
	6	let current: Loaded \| null = null;
	7
	8	// TODO: rename all these types
	9	type DynamicId = string;
	10	export type { Key };
	11	export type Manifest =
	12	& {
	13	[K in Key]: StaticAsset \| DynamicAsset;
	14	}
	15	& {
	16	[string: string]: StaticAsset \| DynamicAsset;
	17	};
	18	export interface StaticAsset extends AssetBase {
	19	type: 0;
	20	}
	21	interface AssetBase {
	22	headers: Record<string, string> & { etag: string };
	23	raw: BufferView;
	24	gzip: BufferView;
	25	zstd: BufferView;
	26	}
	27	export interface DynamicAsset {
	28	type: 1;
	29	id: DynamicId;
	30	}
1	31	interface Loaded {
2		map: BuiltAssetMap;
3		buf: Buffer;
	32	map: Manifest;
	33	static: Buffer;
	34	dynamic: Map<DynamicId, DynamicEntry>;
4	35	}
5		let assets: Loaded \| null = null;
6
7		export type StaticPageId = string;
8
9		export async function reload() {
10		const [map, buf] = await Promise.all([
11		fs.readFile(path.join(import.meta.dirname, "static.json"), "utf8"),
12		fs.readFile(path.join(import.meta.dirname, "static.blob")),
13		]);
14		return (assets = { map: JSON.parse(map), buf });
	36	export interface DynamicEntry extends AssetBase {
	37	buffer: Buffer;
15	38	}
16	39
17		export function reloadSync() {
18		const map = fs.readFileSync(
19		path.join(import.meta.dirname, "static.json"),
20		"utf8",
	40	export async function reload() {
	41	const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
	42	const statics = await fs.readFile(path.join(root, "asset.blob"));
	43	const dynamic = new Map(
	44	await Promise.all(
	45	Object.entries(map)
	46	.filter((entry): entry is [string, DynamicAsset] => entry[1].type === 1)
	47	.map(async ([k, v]) =>
	48	[
	49	v.id,
	50	await fs.readFile(path.join(root, "dynamic", v.id))
	51	.then(loadRegenerative),
	52	] as const
	53	),
	54	),
21	55	);
22		const buf = fs.readFileSync(path.join(import.meta.dirname, "static.blob"));
23		return (assets = { map: JSON.parse(map), buf });
	56	return (current = { map, static: statics, dynamic });
24	57	}
25	58
26	59	export async function middleware(c: Context, next: Next) {
27		if (!assets) await reload();
28		const asset = assets!.map[c.req.path];
	60	if (!current) current = await reload();
	61	const asset = current.map[c.req.path];
29	62	if (asset) return assetInner(c, asset, 200);
30	63	return next();
31	64	}
32	65
33	66	export async function notFound(c: Context) {
34		if (!assets) await reload();
	67	if (!current) current = await reload();
35	68	let pathname = c.req.path;
36	69	do {
37		const asset = assets!.map[pathname + "/404"];
	70	const asset = current.map[pathname + "/404"];
38	71	if (asset) return assetInner(c, asset, 404);
39	72	pathname = pathname.slice(0, pathname.lastIndexOf("/"));
40	73	} while (pathname);
41		const asset = assets!.map["/404"];
	74	const asset = current.map["/404"];
42	75	if (asset) return assetInner(c, asset, 404);
43	76	return c.text("the 'Not Found' page was not found", 404);
44	77	}
45	78
46		export async function serveAsset(
47		c: Context,
48		id: StaticPageId,
49		status: StatusCode,
50		) {
51		return assetInner(c, (assets ?? (await reload())).map[id], status);
	79	export async function serveAsset(c: Context, id: Key, status: StatusCode) {
	80	return assetInner(c, (current ?? (await reload())).map[id], status);
52	81	}
53	82
	83	/** @deprecated */
54	84	export function hasAsset(id: string) {
55		return (assets ?? reloadSync()).map[id] !== undefined;
	85	return UNWRAP(current).map[id] !== undefined;
56	86	}
57	87
58	88	export function etagMatches(etag: string, ifNoneMatch: string) {
59	89	return ifNoneMatch === etag \|\| ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
60	90	}
61	91
62		function subarrayAsset([start, end]: BufferView) {
63		return assets!.buf.subarray(start, end);
	92	function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {
	93	ASSERT(current);
	94	if (asset.type === 0) {
	95	return respondWithBufferAndViews(c, current.static, asset, status);
	96	} else {
	97	const entry = UNWRAP(current.dynamic.get(asset.id));
	98	return respondWithBufferAndViews(c, entry.buffer, entry, status);
	99	}
64	100	}
65	101
66		function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
67		const ifnonematch = c.req.header("If-None-Match");
68		if (ifnonematch) {
69		const etag = asset.headers.ETag;
70		if (etagMatches(etag, ifnonematch)) {
	102	function respondWithBufferAndViews(
	103	c: Context,
	104	buffer: Buffer,
	105	asset: AssetBase,
	106	status: StatusCode,
	107	) {
	108	const ifNoneMatch = c.req.header("If-None-Match");
	109	if (ifNoneMatch) {
	110	const etag = asset.headers.etag;
	111	if (etagMatches(etag, ifNoneMatch)) {
71	112	return (c.res = new Response(null, {
72	113	status: 304,
73	114	statusText: "Not Modified",
...	...	@@ -80,24 +121,103 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
80	121	const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
81	122	let body;
82	123	let headers = asset.headers;
83		if (acceptEncoding.includes("zstd") && asset.zstd) {
84		body = subarrayAsset(asset.zstd);
	124	if (acceptEncoding.includes("zstd")) {
	125	body = buffer.subarray(...asset.zstd);
85	126	headers = {
86	127	...asset.headers,
87	128	"Content-Encoding": "zstd",
88	129	};
89		} else if (acceptEncoding.includes("gzip") && asset.gzip) {
90		body = subarrayAsset(asset.gzip);
	130	} else if (acceptEncoding.includes("gzip")) {
	131	body = buffer.subarray(...asset.gzip);
91	132	headers = {
92	133	...asset.headers,
93	134	"Content-Encoding": "gzip",
94	135	};
95	136	} else {
96		body = subarrayAsset(asset.raw);
	137	body = buffer.subarray(...asset.raw);
97	138	}
98	139	return (c.res = new Response(body, { headers, status }));
99	140	}
100	141
	142	export function packDynamicBuffer(
	143	raw: Buffer,
	144	gzip: Buffer,
	145	zstd: Buffer,
	146	headers: Record<string, string>,
	147	) {
	148	const headersBuffer = Buffer.from(
	149	Object.entries(headers)
	150	.map((entry) => entry.join(":"))
	151	.join("\n"),
	152	"utf-8",
	153	);
	154	const header = new Uint32Array(3);
	155	header[0] = headersBuffer.byteLength + header.byteLength;
	156	header[1] = header[0] + raw.byteLength;
	157	header[2] = header[1] + gzip.byteLength;
	158	return Buffer.concat([
	159	Buffer.from(header.buffer),
	160	headersBuffer,
	161	raw,
	162	gzip,
	163	zstd,
	164	]);
	165	}
	166
	167	function loadRegenerative(buffer: Buffer): DynamicEntry {
	168	const headersEnd = buffer.readUInt32LE(0);
	169	const headers = Object.fromEntries(
	170	buffer
	171	.subarray(3 * 4, headersEnd)
	172	.toString("utf-8")
	173	.split("\n")
	174	.map((line) => {
	175	const i = line.indexOf(":");
	176	return [line.slice(0, i), line.slice(i + 1)];
	177	}),
	178	);
	179	const raw = buffer.readUInt32LE(4);
	180	const gzip = buffer.readUInt32LE(8);
	181	const hasEtag = (v: object): v is typeof v & { etag: string } =>
	182	"etag" in v && typeof v.etag === "string";
	183	ASSERT(hasEtag(headers));
	184	return {
	185	headers,
	186	buffer,
	187	raw: [headersEnd, raw],
	188	gzip: [raw, gzip],
	189	zstd: [gzip, buffer.byteLength],
	190	};
	191	}
	192
	193	const gzip = util.promisify(zlib.gzip);
	194	const zstdCompress = util.promisify(zlib.zstdCompress);
	195
	196	export async function overwriteDynamic(
	197	key: Key,
	198	value: string \| Buffer,
	199	headers: Record<string, string>,
	200	) {
	201	if (!current) current = await reload();
	202	const asset = UNWRAP(current.map[key]);
	203	ASSERT(asset.type === 1);
	204	UNWRAP(current.dynamic.has(asset.id));
	205	const buffer = Buffer.from(value);
	206	const etag = JSON.stringify(
	207	crypto.createHash("sha1").update(buffer).digest("hex"),
	208	);
	209	const [gzipBuffer, zstdBuffer] = await Promise.all([
	210	gzip(buffer),
	211	zstdCompress(buffer),
	212	]);
	213	const packed = packDynamicBuffer(buffer, gzipBuffer, zstdBuffer, {
	214	...headers,
	215	etag,
	216	});
	217	current.dynamic.set(asset.id, loadRegenerative(packed));
	218	await fs.writeFile(path.join(root, "dynamic", asset.id), packed);
	219	}
	220
101	221	process.on("message", (msg: any) => {
102	222	if (msg?.type === "clover.assets.reload") reload();
103	223	});
...	...	@@ -105,6 +225,10 @@ process.on("message", (msg: any) => {
105	225	import * as fs from "#sitegen/fs";
106	226	import type { Context, Next } from "hono";
107	227	import type { StatusCode } from "hono/utils/http-status";
108		import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts";
	228	import type { BufferView } from "../incremental.ts";
109	229	import { Buffer } from "node:buffer";
110	230	import * as path from "node:path";
	231	import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
	232	import * as crypto from "node:crypto";
	233	import * as zlib from "node:zlib";
	234	import * as util from "node:util";

framework/lib/async.ts+2-2

...	...	@@ -51,8 +51,8 @@ export class Queue<T, R> {
51	51	let n = 0;
52	52	for (const item of active) {
53	53	let itemText = "- " + item.format(now);
54		text +=
55		`\n` + itemText.slice(0, Math.max(0, process.stdout.columns - 1));
	54	text += `\n` +
	55	itemText.slice(0, Math.max(0, process.stdout.columns - 1));
56	56	if (n > 10) {
57	57	text += `\n ... + ${active.length - n} more`;
58	58	break;

framework/lib/fs.ts+2-2

...	...	@@ -4,6 +4,7 @@ export {
4	4	createReadStream,
5	5	createWriteStream,
6	6	existsSync,
	7	type FileHandle,
7	8	open,
8	9	readdir,
9	10	readdirSync,
...	...	@@ -15,7 +16,6 @@ export {
15	16	statSync,
16	17	writeFile,
17	18	writeFileSync,
18		type FileHandle,
19	19	};
20	20
21	21	export function mkdir(dir: string) {
...	...	@@ -98,6 +98,7 @@ import {
98	98	writeFileSync,
99	99	} from "node:fs";
100	100	import {
	101	type FileHandle,
101	102	mkdir as nodeMkdir,
102	103	open,
103	104	readdir,
...	...	@@ -106,6 +107,5 @@ import {
106	107	rmdir,
107	108	stat,
108	109	writeFile,
109		type FileHandle,
110	110	} from "node:fs/promises";
111	111	export { Stats } from "node:fs";

framework/lib/markdown.tsx+12-8

...	...	@@ -1,9 +1,11 @@
1		/* Impementation of CommonMark specification for markdown with support
	1	/* Implementation of [CommonMark] specification for markdown with support
2	2	* for custom syntax extensions via the parser options. Instead of
3	3	* returning an AST that has a second conversion pass to JSX, the
4	4	* returned value of 'parse' is 'engine.Node' which can be stringified
5		* via clover's SSR engine. This way, generation optimizations, async
	5	* via Clover's SSR engine. This way, generation optimizations, async
6	6	* components, and other features are gained for free here.
	7	*
	8	* [CommonMark]: https://spec.commonmark.org/0.31.2/
7	9	*/
8	10	function parse(src: string, options: Partial<ParseOpts> = {}) {}
9	11
...	...	@@ -16,6 +18,9 @@ export function Markdown({
16	18	return parse(src, options);
17	19	}
18	20
	21	// TODO: This implementation is flawed because it is impossible to sanely handle
	22	// emphasis and strong emphasis, and all their edge cases. Instead of making these
	23	// using extensions interface, they should be special cased.
19	24	function parseInline(src: string, options: Partial<InlineOpts> = {}) {
20	25	const { rules = inlineRules, links = new Map() } = options;
21	26	const opts: InlineOpts = { rules, links };
...	...	@@ -110,12 +115,11 @@ export const inlineRules: Record<string, InlineRule> = {
110	115	} else if (afterText[0] === "[") {
111	116	const splitTarget = splitFirst(afterText.slice(1), /]/);
112	117	if (!splitTarget) return null;
113		const name =
114		splitTarget.first.trim().length === 0
115		? // Collapsed reference link
116		textSrc.trim()
117		: // Full Reference Link
118		splitTarget.first.trim();
	118	const name = splitTarget.first.trim().length === 0
	119	// Collapsed reference link
	120	? textSrc.trim()
	121	// Full Reference Link
	122	: splitTarget.first.trim();
119	123	const target = opts.links.get(name);
120	124	if (!target) return null;
121	125	({ href, title } = target);

framework/lib/sitegen.ts+40-1

...	...	@@ -1,6 +1,25 @@
1		// Import this file with 'import * as sg from "#sitegen";'
2	1	export type ScriptId = string;
3	2
	3	export interface PageExports extends ViewExports {
	4	regenerate?: PageRegenerateOptions;
	5	}
	6	export interface ViewExports {
	7	default: render.Component;
	8	meta: meta.Meta \| ((props: { ssr: true }) => Promise<meta.Meta> \| meta.Meta);
	9	theme?: css.Theme;
	10	layout?: Layout;
	11	}
	12	export interface Layout {
	13	default: render.Component;
	14	theme?: css.Theme;
	15	// TODO: nested layout
	16	}
	17	export interface PageRegenerateOptions {
	18	tags?: string[];
	19	seconds?: number;
	20	debounce?: number;
	21	}
	22
4	23	/**
5	24	* A filesystem object associated with some ID,
6	25	* such as a page's route to it's source file.
...	...	@@ -30,4 +49,24 @@ export function addScript(id: ScriptId \| { value: ScriptId }) {
30	49	userData.get().scripts.add(typeof id === "string" ? id : id.value);
31	50	}
32	51
	52	export function wrapDocument({
	53	body,
	54	head,
	55	inlineCss,
	56	scripts,
	57	}: {
	58	head: string;
	59	body: string;
	60	inlineCss: string;
	61	scripts: string;
	62	}) {
	63	return `<!doctype html><html lang=en><head>${head}${
	64	inlineCss ? `<style>${inlineCss}</style>` : ""
	65	}</head><body>${body}${
	66	scripts ? `<script>${scripts}</script>` : ""
	67	}</body></html>`;
	68	}
	69
33	70	import * as render from "#engine/render";
	71	import type * as meta from "./meta.ts";
	72	import type * as css from "../css.ts";

framework/lib/sqlite.ts+3-1

...	...	@@ -10,7 +10,9 @@ export function getDb(file: string) {
10	10	if (db) return db;
11	11	const fileWithExt = file.includes(".") ? file : file + ".sqlite";
12	12	db = new WrappedDatabase(
13		new DatabaseSync(path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt)),
	13	new DatabaseSync(
	14	path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
	15	),
14	16	);
15	17	map.set(file, db);
16	18	return db;

framework/lib/testing.ts+7-3

...	...	@@ -1,5 +1,9 @@
1	1	export function tmpFile(basename: string) {
2		const file = path.join(import.meta.dirname, '../../.clover/testing', basename);
	2	const file = path.join(
	3	import.meta.dirname,
	4	"../../.clover/testing",
	5	basename,
	6	);
3	7	return {
4	8	path: file,
5	9	read: fs.readFile.bind(fs, file),
...	...	@@ -7,5 +11,5 @@ export function tmpFile(basename: string) {
7	11	};
8	12	}
9	13
10		import * as path from 'node:path';
11		import * as fs from './fs.ts';
	14	import * as path from "node:path";
	15	import * as fs from "./fs.ts";

framework/lib/view.ts+70-44

...	...	@@ -1,38 +1,64 @@
1		export interface View {
	1	// The "view" system allows rendering dynamic pages within backends.
	2	// This is done by scanning all `views` dirs, bundling their client
	3	// resources, and then providing `serve` which renders a page.
	4	//
	5	// This system also implements page regeneration.
	6	let codegen: Codegen;
	7	try {
	8	codegen = require("$views");
	9	} catch {
	10	throw new Error("Can only import '#sitegen/view' in backends.");
	11	}
	12
	13	// Generated in `bundle.ts`
	14	export interface Codegen {
	15	views: { [K in Key]: View<PropsFromModule<ViewMap[K]>> };
	16	scripts: Record<string, string>;
	17	regenTtls: Ttl[];
	18	regenTags: Record<RegenKey, Key[]>;
	19	}
	20	// The view contains pre-bundled CSS and scripts, but keeps the scripts
	21	// separate for run-time dynamic scripts. For example, the file viewer
	22	// includes the canvas for the current page, but only the current page.
	23	export interface View<Props extends Record<string, unknown>> {
2	24	component: render.Component;
3		meta:
4		\| meta.Meta
5		\| ((props: { context?: hono.Context }) => Promise<meta.Meta> \| meta.Meta);
	25	meta: meta.Meta \| ((props: Props) => Promise<meta.Meta> \| meta.Meta);
6	26	layout?: render.Component;
7	27	inlineCss: string;
8	28	scripts: Record<string, string>;
9	29	}
	30	export interface Ttl {
	31	seconds: number;
	32	key: Key;
	33	}
	34	export type Key = keyof ViewMap;
10	35
11		let views: Record<string, View> = null!;
12		let scripts: Record<string, string> = null!;
13
14		export async function renderView(
	36	export async function serve<K extends Key>(
15	37	context: hono.Context,
16		id: string,
17		props: Record<string, unknown>,
	38	id: K,
	39	props: PropsFromModule<ViewMap[K]>,
18	40	) {
19		return context.html(await renderViewToString(id, { context, ...props }));
	41	return context.html(await renderToString(id, { context, ...props }));
20	42	}
21	43
22		export async function renderViewToString(
23		id: string,
24		props: Record<string, unknown>,
	44	type PropsFromModule<M extends any> = M extends {
	45	default: (props: infer T) => render.Node;
	46	} ? T
	47	: never;
	48
	49	export async function renderToString<K extends Key>(
	50	id: K,
	51	props: PropsFromModule<ViewMap[K]>,
25	52	) {
26		views ?? ({ views, scripts } = require("$views"));
27		// The view contains pre-bundled CSS and scripts, but keeps the scripts
28		// separate for run-time dynamic scripts. For example, the file viewer
29		// includes the canvas for the current page, but only the current page.
30	53	const {
31	54	component,
32	55	inlineCss,
33	56	layout,
34	57	meta: metadata,
35		}: View = UNWRAP(views[id], `Missing view ${id}`);
	58	}: View<PropsFromModule<ViewMap[K]>> = UNWRAP(
	59	codegen.views[id],
	60	`Missing view ${id}`,
	61	);
36	62
37	63	// -- metadata --
38	64	const renderedMetaPromise = Promise.resolve(
...	...	@@ -48,48 +74,48 @@ export async function renderViewToString(
48	74	} = await render.async(page, { [sg.userData.key]: sg.initRender() });
49	75
50	76	// -- join document and send --
51		return wrapDocument({
	77	return sg.wrapDocument({
52	78	body,
53	79	head: await renderedMetaPromise,
54	80	inlineCss,
55	81	scripts: joinScripts(
56		Array.from(sitegen.scripts, (id) =>
57		UNWRAP(scripts[id], `Missing script ${id}`),
	82	Array.from(
	83	sitegen!.scripts,
	84	(id) => UNWRAP(codegen.scripts[id], `Missing script ${id}`),
58	85	),
59	86	),
60	87	});
61	88	}
62	89
63		export function provideViewData(v: typeof views, s: typeof scripts) {
64		(views = v), (scripts = s);
	90	export function regenerate(tag: RegenKey) {
	91	for (const view of codegen.regenTags[tag]) {
	92	const key = view.slice("page:".length);
	93	renderToString(view, {})
	94	.then((result) => {
	95	console.info(`regenerate ${key}`);
	96	asset.overwriteDynamic(key as asset.Key, result, {
	97	"content-type": "text/html",
	98	});
	99	})
	100	.catch((e) => {
	101	console.error(`Failed regenerating ${view} from tag ${tag}`, e);
	102	});
	103	}
65	104	}
66	105
67		export function joinScripts(scriptSources: string[]) {
	106	function joinScripts(scriptSources: string[]) {
68	107	const { length } = scriptSources;
69	108	if (length === 0) return "";
70		if (length === 1) return scriptSources[0];
	109	if (0 in scriptSources) return scriptSources[0];
71	110	return scriptSources.map((source) => `{${source}}`).join(";");
72	111	}
73	112
74		export function wrapDocument({
75		body,
76		head,
77		inlineCss,
78		scripts,
79		}: {
80		head: string;
81		body: string;
82		inlineCss: string;
83		scripts: string;
84		}) {
85		return `<!doctype html><html lang=en><head>${head}${
86		inlineCss ? `<style>${inlineCss}</style>` : ""
87		}</head><body>${body}${
88		scripts ? `<script>${scripts}</script>` : ""
89		}</body></html>`;
90		}
91
92	113	import * as meta from "./meta.ts";
93	114	import type * as hono from "#hono";
94	115	import * as render from "#engine/render";
95	116	import * as sg from "./sitegen.ts";
	117	import * as asset from "./assets.ts";
	118	import type {
	119	RegenKey,
	120	RegisteredViews as ViewMap,
	121	} from "../../.clover/ts/view.d.ts";

framework/marko.ts+12-11

...	...	@@ -14,18 +14,19 @@ export function loadMarko(module: NodeJS.Module, filepath: string) {
14	14	// bare client import statements to it's own usage.
15	15	const scannedClientRefs = new Set<string>();
16	16	if (src.match(/^\s*client\s+import\s+["']/m)) {
17		src =
18		src.replace(
19		/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
20		(_, src) => {
21		const ref = JSON.parse(`"${src.slice(1, -1)}"`);
22		const resolved = hot.resolveClientRef(filepath, ref);
23		scannedClientRefs.add(resolved);
24		return `<CloverScriptInclude=${JSON.stringify(
	17	src = src.replace(
	18	/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
	19	(_, src) => {
	20	const ref = JSON.parse(`"${src.slice(1, -1)}"`);
	21	const resolved = hot.resolveClientRef(filepath, ref);
	22	scannedClientRefs.add(resolved);
	23	return `<CloverScriptInclude=${
	24	JSON.stringify(
25	25	hot.getScriptId(resolved),
26		)} />`;
27		},
28		) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
	26	)
	27	} />`;
	28	},
	29	) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
29	30	}
30	31
31	32	src = marko.compileSync(src, filepath).code;

framework/watch.ts+10-9

...	...	@@ -11,7 +11,7 @@ let watch: Watch;
11	11
12	12	export async function main() {
13	13	// Catch up state by running a main build.
14		await incr.restore();
	14	if (!process.argv.includes("-f")) await incr.restore();
15	15	watch = new Watch(rebuild);
16	16	rebuild([]);
17	17	}
...	...	@@ -36,15 +36,16 @@ function onSubprocessClose(code: number \| null, signal: string \| null) {
36	36	}
37	37
38	38	async function rebuild(files: string[]) {
39		const hasInvalidated = files.length === 0
40		\|\| (await Promise.all(files.map(incr.invalidate))).some(Boolean);
	39	const hasInvalidated = files.length === 0 \|\|
	40	(await Promise.all(files.map((file) => incr.invalidate(file))))
	41	.some(Boolean);
41	42	if (!hasInvalidated) return;
42	43	incr.compile(generate.generate).then(({
43	44	watchFiles,
44	45	newOutputs,
45		newAssets
	46	newAssets,
46	47	}) => {
47		const removeWatch = [...watch.files].filter(x => !watchFiles.has(x))
	48	const removeWatch = [...watch.files].filter((x) => !watchFiles.has(x));
48	49	for (const file of removeWatch) watch.remove(file);
49	50	watch.add(...watchFiles);
50	51	// Restart the server if it was changed or not running.
...	...	@@ -60,8 +61,8 @@ async function rebuild(files: string[]) {
60	61
61	62	function statusLine() {
62	63	console.info(
63		`Watching ${watch.files.size} files `
64		+ `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
	64	`Watching ${watch.files.size} files ` +
	65	`\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
65	66	);
66	67	}
67	68
...	...	@@ -142,7 +143,7 @@ class Watch {
142	143	#getFiles(absPath: string, event: fs.WatchEventType) {
143	144	const files = [];
144	145	if (this.files.has(absPath)) files.push(absPath);
145		if (event === 'rename') {
	146	if (event === "rename") {
146	147	const dir = path.dirname(absPath);
147	148	if (this.files.has(dir)) files.push(dir);
148	149	}
...	...	@@ -153,7 +154,7 @@ class Watch {
153	154	if (!subPath) return;
154	155	const files = this.#getFiles(path.join(root, subPath), event);
155	156	if (files.length === 0) return;
156		for(const file of files) this.stale.add(file);
	157	for (const file of files) this.stale.add(file);
157	158	const { debounce } = this;
158	159	if (debounce !== null) clearTimeout(debounce);
159	160	this.debounce = setTimeout(() => {

src/backend.ts+3-3

...	...	@@ -1,6 +1,6 @@
1		// This is the main file for the backend
	1	// This is the main file for paperclover.net's server.
2	2	const app = new Hono();
3		const logHttp = scoped("http", { color: "magenta" });
	3	const logHttp = console.scoped("http", { color: "magenta" });
4	4
5	5	// Middleware
6	6	app.use(trimTrailingSlash());
...	...	@@ -38,4 +38,4 @@ import { logger } from "hono/logger";
38	38	import { trimTrailingSlash } from "hono/trailing-slash";
39	39	import * as assets from "#sitegen/assets";
40	40	import * as admin from "./admin.ts";
41		import { scoped } from "@paperclover/console";
	41	import * as console from "@paperclover/console";

src/blog/pages/25/file-viewer-internals.mdx deleted-10

...	...	@@ -1,10 +0,0 @@
1		export const blog: BlogMeta = {
2		title: "Marko is the coziest HTML templating language",
3		desc: "...todo...",
4		date: "2025-07-08",
5		draft: true,
6		};
7		export const meta = formatBlogMeta(blob);
8		export * as layout from "@/blog/layout.tsx";
9
10

src/blog/pages/25/marko-intro.markodown deleted-290

...	...	@@ -1,290 +0,0 @@
1		export const blog: BlogMeta = {
2		title: "Marko is the coziest HTML templating language",
3		desc: "...todo...",
4		date: "2025-06-13",
5		draft: true,
6		};
7		export const meta = formatBlogMeta(blob);
8		export * as layout from "@/blog/layout.tsx";
9
10		I've been recently playing around [Marko][1], and after adding limited support
11		for it in my website generator, [sitegen][2], I instantly fell in love with how
12		minimalistic it is in comparison to JSX, Astro components, and Svelte.
13
14		## Introduction
15
16		If JSX was taking HTML and shoving its syntax into JavaScript, Marko is shoving
17		JavaScript into HTML. Attributes are JavaScript expressions.
18
19		```marko
20		<div>
21		// `input` is like props, but given in the top-level scope
22		<time datetime=input.date.toISOString()>
23		// Interpolation with JS template string syntax
24		${formatTimeNicely(input.date)}
25		</time>
26		<div>
27		<a href=`/users/${input.user.id}`>${input.user.name}</a>
28		</div>
29
30		// Capital letter variables for imported components
31		<MarkdownContent message=input.message />
32
33		// Components also can be auto-imported by lowercase.
34		// This will look upwards for a `tags/` folder containing
35		// "custom-footer.marko", similar to how Node.js finds
36		// package names in all upwards `node_modules` folders.
37		<custom-footer />
38		</div>
39
40		// ESM `import` / `export` just work as expected.
41		// I prefer my imports at the end, to highlight the markup.
42		import MarkdownContent from "./MarkdownContent.marko";
43		import { formatTimeNicely } from "../date-helpers.ts";
44		```
45
46		Tags with the `value` attribute have a shorthand, which is used by the built-in
47		`<if>` for conditional rendering.
48
49		```marko
50		// Sugar for <input value="string" />
51		<input="string" />
52
53		// and it composes amazingly to the 'if' built-in
54		<if=input.user>
55		<UserProfile=input.user />
56		</if>
57		```
58
59		Tags can also return values into the scope for use in the template using `/`, such as `<id>` for unique ID generation. This is available to components that `<return=output/>`.
60
61		```
62		<id/uniqueId />
63
64		<input id=uniqueId type="checkbox" name="allow_trans_rights" />
65		<label for=uniqueId>click me!</>
66		// ^ oh, you can also omit the
67		// closing tag name if you want.
68		```
69
70		It's important that I started with the two forms of "Tag I/O": `=` for input
71		and `/` for output. With those building blocks, we introduce local variables
72		with `const`
73
74		```
75		<const/rendered = markdownToHtml(input.value) />
76
77		// This is how you insert raw HTML to the document
78		<inline-html=rendered />
79
80		// It supports all of the cozy destructuring syntax JS has
81		<const/{ id, name } = user />
82		```
83
84		Unlike JSX, when you pass content within a tag (`input.content` instead of
85		JSX's `children`), instead of it being a JSX element, it is actually a
86		function. This means that the `for` tag can render the content multiple times.
87
88		```
89		<ul>
90		<for from=1 to=10>
91		// Renders a new random number for each iteration.
92		<li>${Math.random()}</li>
93		</>
94		</ul>
95		```
96
97		Since `content` is a function, it can take arguments. This is done with `\|`
98
99		```
100		<h1>my friends</h1>
101		<ul>
102		// I tend to omit the closing tag names for the built-in control
103		// flow tags, but I keep them for HTML tags. It's kinda like how
104		// in JavaScript you just write `}` to close your `if`s and loops.
105		//
106		// Anyways <for> also has 'of'
107		<for\|item\| of=user.friends>
108		<li class="friend">${item.name}</li>
109		</>
110
111		// They support the same syntax JavaScript function params allows,
112		// so you can have destructuring here too, and multiple params.
113		<for\|{ name }, index\| of=user.friends>
114		// By the way you can also use emmet-style class and ID shorthands.
115		<li.friend>My #${index + 1} friend is ${name}</li>
116		</>
117		</ul>
118		```
119
120		Instead of named slots, Marko has attribute tags. These are more powerful than
121		slots since they are functions, and can also act as sugar for more complicated
122		attributes.
123
124		```
125		<Layout title="Welcome">
126		<@header variant="big">
127		<h1>the next big thing</h1>
128		</@header>
129
130		<p>body text...</p>
131		</Layout>
132
133		// The `input` variable inside of <Layout /> is:
134		//
135		// {
136		// title: "Welcome",
137		// header: {
138		// content: /* function rendering "<h1>the next big thing</h1>" */,
139		// variant: "big",
140		// },
141		// content: /* function rendering "<p>body text</p>" */
142		// }
143		```
144
145		This layout could be implemented as such:
146
147		```marko
148		<main>
149		<if=input.header />
150		<const/{ ...headerProps, content }=input.header />
151		<header ...headerProps>
152		// Instead of assigning to a variable with a capital letter,
153		// template interpolation works on tag names. This can also
154		// be a string to render the native HTML tag of that kind.
155		<${content} />
156		</header>
157		<hr />
158		</>
159
160		<${input.content} />
161		</main>
162		```
163
164		The last syntax feature missing is calling a tag with parameters. That is done
165		just like a regular function call, with '('.
166
167		```
168		<Something(item, index) />
169		```
170
171		In fact, attributes can just be sugar over this syntax; _this technically isn't
172		true but it's close enough for the example_
173
174		```
175		<SpecialButton type="submit" class="red" />
176
177		// is equal to
178
179		<SpecialButton({ type: "submit", class: "red" }) />
180		```
181
182		All of the above is about how Marko's syntax works, and how it performs HTML
183		generation with components. Marko also allows interactive components, but an
184		explaination of that is beyond the scope of this page, mostly since I have not
185		used it. A brief example of it, modified from their documentation.
186
187		```marko
188		// Reactive variables with <let/> just work...
189		<let/basicCounter=0 />
190		<button onClick() { basicCounter += 1 }>${basicCounter}</button>
191		// ...but a counter is boring.
192
193		<let/todos=[
194		{ id: 0, text: "Learn Marko" },
195		{ id: 1, text: "Make a Website" },
196		]/>
197
198		// 'by' is like React JSX's "key" property, but it's optional.
199		<ul><for\|todo, i\| of=todos by=(todo => todo.id)>
200		<li.todo>
201		// this variable remains stable even if the list
202		// re-orders, because 'by' was specified.
203		<let/done=false/>
204		<label>
205		<span>${todo.text}</span>
206		// ':=' creates a two-way reactive binding,
207		// (it passes a callback for `checkedChanged`)
208		<input type="checkbox" checked:=done />
209		</label>
210		<button
211		title="delete"
212		disabled=!done
213		onClick() {
214		todos = todos.toSpliced(i, 1);
215		}
216		> × </button>
217		</li>
218		</></ul>
219
220		// Form example
221		<let/nextId=2/>
222		<form onSubmit(e) {
223		e.preventDefault();
224		todos = todos.concat({
225		id: nextId++,
226		// HTMLFormElement exposes all its named input
227		// elements as extra properties on the object.
228		text: e.target.text.value,
229		});
230		// And you can clear it with 'reset()'
231		e.target.reset();
232		}>
233		// We don't 'onChange' like a React loser. The form
234		// value can be read in the submit event like normal.
235		<input name="text" placeholder="Another Item">
236		<button type="submit">Add</button>
237		</form>
238		```
239
240		## Usage on `paperclover.net`
241
242		TODO: document a lot of feedback, how i embedded Marko
243
244		My website uses statically generated HTML. That is why I have not needed to use
245		reactive variables. My generator doesn't even try compiling components
246		client-side.
247
248		Here is the actual component used to render [questions on the clover q+a][/q+a].
249
250		```marko
251		// Renders a `Question` entry including its markdown body.
252		export interface Input {
253		question: Question;
254		admin?: boolean;
255		}
256
257		// 2024-12-31 05:00:00 EST
258		export const transitionDate = 1735639200000;
259
260		<const/{ question, admin } = input />
261		<const/{ id, date, text } = question/>
262
263		<${"e-"}
264		f=(date > transitionDate ? true : undefined)
265		id=admin ? `q${id}` : undefined
266		>
267		<if=admin>
268		<a
269		style="margin-right: 0.5rem"
270		href=`/admin/q+a/${id}`
271		>[EDIT]</a>
272		</>
273		<a>
274		<time
275		datetime=formatQuestionISOTimestamp(date)
276		>${formatQuestionTimestamp(date)}</time>
277		</a>
278
279		<CloverMarkdown ...{ text } />
280		</>
281
282		// this singleton script will make all the '<time>' tags clickable.
283		client import "./clickable-links.client.ts";
284
285		import type { Question } from "@/q+a/models/Question.ts";
286		import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";
287		import { CloverMarkdown } from "@/q+a/clover-markdown.tsx";
288		```
289
290		import { type BlogMeta, formatBlogMeta } from '@/blog/helpers.ts';

src/blog/pages/webdev/deranged-typescript.md created+87

...	...	@@ -0,0 +1,87 @@
	1	- imports at the bottom
	2	- order your file by importance.
	3	- 'G' to jump to imports, etc
	4	- prefer namespace imports
	5	- easier to type and refactor. easier to read.
	6	- large files are okay
	7	- all files are their own library
	8	- split files up by making components modular, not by "oh it's too big"
	9	- engine/render.ts is a standalone library, in order to split JSX, Suspense,
	10	and Marko out, the main file was made modular.
	11	- lowercase
	12	- name objects ultra-concisely
	13	- filenames are often one word describing what they contain
	14	- avoid useless descriptors like "utils", "helpers", and "data"
	15	- examples
	16	- async.ts contains all the async library functions.
	17	- watch.ts contains the file watcher and watch-reload mode.
	18	- render.*, Io
	19	- be ultra-concise in comments
	20	- no "discarded" variables, embrace `void x`
	21	- makes code more readable
	22	- note how i want to write a lint for this
	23	- note the one proposal i want about void
	24	- push the ts inference engine (as const, ReturnType, etc)
	25	- reduces how much you repeat yourself making it easier to refactor things
	26	- use the code as the source of truth
	27	- push the ts inference engine (generics)
	28	- do not implement crazy things with the TS engine, instead use generic input
	29	types, and then use regular control to narrow and transform the return type.
	30	source of truth is your code.
	31	- UNWRAP, ASSERT utility globals are amazing
	32	- ban postfix '!'
	33	- stripped for production frontend builds
	34	- destructure often
	35	- use the one example from work lol
	36	- package.json "imports" are amazing
	37	- remapping
	38	- implementation switching
	39	- testing
	40	- embrace the web and node.js APIs
	41	- sitegen relies on so many node features that bun and deno fail to run it.
	42	- overlay modules are great
	43	- avoid dependencies
	44	- once you build your own mini standard library you win
	45	- talk about regrets with mdx
	46
	47	## imports at the bottom
	48
	49	Here is an abridged version of my website's `backend.ts`. When reading it from
	50	top to bottom it is immediately obvious that it is a Hono web server.
	51
	52	```ts
	53	// This is the main file for paperclover.net's server.
	54	const app = new Hono();
	55	const logHttp = console.scoped("http", { color: "magenta" });
	56
	57	// Middleware
	58	app.use(...);
	59	...
	60
	61	// Backends
	62	app.route("", require("./q+a/backend.ts").app);
	63	...
	64
	65	export default app;
	66
	67	...
	68
	69	import { type Context, Hono, type Next } from "#hono";
	70	import { logger } from "hono/logger";
	71	import { trimTrailingSlash } from "hono/trailing-slash";
	72	import * as assets from "#sitegen/assets";
	73	import * as admin from "./admin.ts";
	74	import * as console from "@paperclover/console";
	75	```
	76
	77	Since `import`/`export` statements are hoisted like `var` and `function`, the
	78	position of these statements within the file does not matter. The imported
	79	modules have to be loaded first before this file can start. With this, I've
	80	found it nicer to sort the file by _importance_ rather than by arbitrary rules
	81	dictated by how C-style `#include`s worked.
	82
	83	Start with a documentation comment, then the most important
	84	functions/variables/types, sort the file by importance. Imports are not really
	85	important since you very quickly get to know where common namespaces come from.
	86	And since they're at the bottom, you can just press `G` in Vim or `CMD+Down` on
	87	the Mac to scroll to the end of the file.

src/blog/pages/webdev/marko-intro.markodown created+339

...	...	@@ -0,0 +1,339 @@
	1	export const blog: BlogMeta = {
	2	title: "Marko is the coziest HTML templating language",
	3	desc: "...todo...",
	4	created: "2025-06-13",
	5	draft: true,
	6	};
	7	export const meta = formatBlogMeta(blob);
	8	export * as layout from "@/blog/layout.tsx";
	9
	10	I've been recently playing around [Marko], and after adding limited support
	11	for it in my website generator, [sitegen], I instantly fell in love with how
	12	minimalistic it is in comparison to JSX, Astro components, and Svelte.
	13
	14	[Marko]: https://next.markojs.com
	15	[sitegen]: https://paperclover.dev/clo/sitegen
	16
	17	## Introduction to Marko
	18
	19	If JSX was taking HTML and shoving its syntax into JavaScript, Marko is shoving
	20	JavaScript into HTML. Attributes are JavaScript expressions.
	21
	22	```marko
	23	<div>
	24	// `input` is like props, but given in the top-level scope
	25	<time datetime=input.date.toISOString()>
	26	// Interpolation with JS template string syntax
	27	${formatTimeNicely(input.date)}
	28	</time>
	29	<div>
	30	<a href=`/users/${input.user.id}`>${input.user.name}</a>
	31	</div>
	32
	33	// Capital letter variables for imported components
	34	<MarkdownContent message=input.message />
	35
	36	// Components also can be auto-imported by lowercase.
	37	// This will look upwards for a `tags/` folder containing
	38	// "custom-footer.marko", similar to how Node.js finds
	39	// package names in all upwards `node_modules` folders.
	40	<custom-footer />
	41	</div>
	42
	43	// ESM `import` / `export` just work as expected.
	44	// I prefer my imports at the end, to highlight the markup.
	45	import MarkdownContent from "./MarkdownContent.marko";
	46	import { formatTimeNicely } from "../date-helpers.ts";
	47	```
	48
	49	Tags with the `value` attribute have a shorthand, which is used by the built-in
	50	`<if>` for conditional rendering.
	51
	52	```marko
	53	// Sugar for <input value="string" />
	54	<input="string" />
	55
	56	// and it composes amazingly to the 'if' built-in
	57	<if=input.user>
	58	<UserProfile=input.user />
	59	</if>
	60	```
	61
	62	Tags can also return values into the scope for use in the template using `/`, such as `<id>` for unique ID generation. This is available to components that `<return=output/>`.
	63
	64	```
	65	<id/uniqueId />
	66
	67	<input id=uniqueId type="checkbox" name="allow_trans_rights" />
	68	<label for=uniqueId>click me!</>
	69	// ^ oh, you can also omit the
	70	// closing tag name if you want.
	71	```
	72
	73	It's important that I started with the two forms of "Tag I/O": `=` for input
	74	and `/` for output. With those building blocks, we introduce local variables
	75	with `const`
	76
	77	```
	78	<const/rendered = markdownToHtml(input.value) />
	79
	80	// This is how you insert raw HTML to the document
	81	<inline-html=rendered />
	82
	83	// It supports all of the cozy destructuring syntax JS has
	84	<const/{ id, name } = user />
	85	```
	86
	87	Unlike JSX, when you pass content within a tag (`input.content` instead of
	88	JSX's `children`), instead of it being a JSX element, it is actually a
	89	function. This means that the `for` tag can render the content multiple times.
	90
	91	```
	92	<ul>
	93	<for from=1 to=10>
	94	// Renders a new random number for each iteration.
	95	<li>${Math.random()}</li>
	96	</>
	97	</ul>
	98	```
	99
	100	Since `content` is a function, it can take arguments. This is done with `\|`
	101
	102	```
	103	<h1>my friends</h1>
	104	<ul>
	105	// I tend to omit the closing tag names for the built-in control
	106	// flow tags, but I keep them for HTML tags. It's kinda like how
	107	// in JavaScript you just write `}` to close your `if`s and loops.
	108	//
	109	// Anyways <for> also has 'of'
	110	<for\|item\| of=user.friends>
	111	<li class="friend">${item.name}</li>
	112	</>
	113
	114	// They support the same syntax JavaScript function params allows,
	115	// so you can have destructuring here too, and multiple params.
	116	<for\|{ name }, index\| of=user.friends>
	117	// By the way you can also use emmet-style class and ID shorthands.
	118	<li.friend>My #${index + 1} friend is ${name}</li>
	119	</>
	120	</ul>
	121	```
	122
	123	Instead of named slots, Marko has attribute tags. These are more powerful than
	124	slots since they are functions, and can also act as sugar for more complicated
	125	attributes.
	126
	127	```
	128	<Layout title="Welcome">
	129	<@header variant="big">
	130	<h1>the next big thing</h1>
	131	</@header>
	132
	133	<p>body text...</p>
	134	</Layout>
	135
	136	// The `input` variable inside of <Layout /> is:
	137	//
	138	// {
	139	// title: "Welcome",
	140	// header: {
	141	// content: /* function rendering "<h1>the next big thing</h1>" */,
	142	// variant: "big",
	143	// },
	144	// content: /* function rendering "<p>body text</p>" */
	145	// }
	146	```
	147
	148	This layout could be implemented as such:
	149
	150	```marko
	151	<main>
	152	<if=input.header />
	153	<const/{ ...headerProps, content }=input.header />
	154	<header ...headerProps>
	155	// Instead of assigning to a variable with a capital letter,
	156	// template interpolation works on tag names. This can also
	157	// be a string to render the native HTML tag of that kind.
	158	<${content} />
	159	</header>
	160	<hr />
	161	</>
	162
	163	<${input.content} />
	164	</main>
	165	```
	166
	167	The last syntax feature missing is calling a tag with parameters. That is done
	168	just like a regular function call, with '('.
	169
	170	```
	171	<Something(item, index) />
	172	```
	173
	174	In fact, attributes can just be sugar over this syntax. (this technically isn't
	175	true but it's close enough for the example)
	176
	177	```
	178	<SpecialButton type="submit" class="red" />
	179
	180	// is equal to
	181
	182	<SpecialButton({ type: "submit", class: "red" }) />
	183	```
	184
	185	All of the above is about how Marko's syntax works, and how it performs HTML
	186	generation with components. Marko also allows interactive components, but an
	187	explaination of that is beyond the scope of this page, mostly since I have not
	188	used it. A brief example of it, modified from their documentation.
	189
	190	```marko
	191	// Reactive variables with <let/> just work...
	192	<let/basicCounter=0 />
	193	<button onClick() { basicCounter += 1 }>${basicCounter}</button>
	194	// ...but a counter is boring.
	195
	196	<let/todos=[
	197	{ id: 0, text: "Learn Marko" },
	198	{ id: 1, text: "Make a Website" },
	199	]/>
	200
	201	// 'by' is like React JSX's "key" property, but it's optional.
	202	<ul><for\|todo, i\| of=todos by=(todo => todo.id)>
	203	<li.todo>
	204	// this variable remains stable even if the list
	205	// re-orders, because 'by' was specified.
	206	<let/done=false/>
	207	<label>
	208	<span>${todo.text}</span>
	209	// ':=' creates a two-way reactive binding,
	210	// (it passes a callback for `checkedChanged`)
	211	<input type="checkbox" checked:=done />
	212	</label>
	213	<button
	214	title="delete"
	215	disabled=!done
	216	onClick() {
	217	todos = todos.toSpliced(i, 1);
	218	}
	219	> × </button>
	220	</li>
	221	</></ul>
	222
	223	// Form example
	224	<let/nextId=2/>
	225	<form onSubmit(e) {
	226	e.preventDefault();
	227	todos = todos.concat({
	228	id: nextId++,
	229	// HTMLFormElement exposes all its named input
	230	// elements as extra properties on the object.
	231	text: e.target.text.value,
	232	});
	233	// And you can clear it with 'reset()'
	234	e.target.reset();
	235	}>
	236	// We don't 'onChange' like a React loser. The form
	237	// value can be read in the submit event like normal.
	238	<input name="text" placeholder="Another Item">
	239	<button type="submit">Add</button>
	240	</form>
	241	```
	242
	243	<SectionHeader updated="2025-08-11">Usage on `paperclover.net`</Section>
	244
	245	Using Marko for HTML generation is quite easy. `.marko` files can be compiled
	246	into `.js` using the `@marko/compiler` library.
	247
	248	```ts
	249	const src = fs.readFileSync("page.marko", "utf8");
	250	const compile = marko.compileSync(src, filepath);
	251	fs.writeFileSync("page.js", compile.code);
	252
	253	const page = require("./page.js");
	254	console.info(page);
	255
	256	import * as fs from "node:fs";
	257	import * as marko from "@marko/compiler";
	258	```
	259
	260	To get client side JavaScript, an option can be passed to the Marko compiler to
	261	generate the client side code. While it is a big selling point of Marko, I do
	262	not use any of their client side features, instead deferring to manually-written
	263	frontend scripts. This is because that is how my website has been for years,
	264	statically generated. And for websites like mine that are content focused, this
	265	is the correct way to do things.
	266
	267	Since I have a custom HTML generation library (built on JSX and some React-like
	268	patterns), I have written a simple integration for it to utilize Marko
	269	components, which is loaded by replacing the generated import to `marko/html`,
	270	which lets me overwrite functions like `createTemplate` (to change the signature
	271	of a component), `dynamicTag` (to allow Marko to render non-Marko components),
	272	and `fork` (to enable async integration with the rendering framework). An
	273	additional feature of this is I have a Node.js loader hook to allow importing
	274	these files directly.
	275
	276	```tsx
	277	function Page() {
	278	const q = Question.getByDate(new Date("2025-06-07 12:12 EST"));
	279	return <div>
	280	<h1>example question</h1>
	281	<QuestionRender question={q} />
	282	</div>;
	283	}
	284
	285	// The synchronous render can be used because `Page` and `question.marko`
	286	// do not await any promises (SQLite runs synchronously)
	287	console.info(render.sync(<Page />).text);
	288
	289	import * as render from "#engine/render";
	290	import QuestionRender from "@/q+a/tags/question.marko";
	291	import { Question } from "@/q+a/models/Question.ts";
	292	```
	293
	294	Here is the `question.marko` tag used to render [questions on the clover q+a](/q+a).
	295
	296	```marko
	297	// Renders a `Question` entry including its markdown body.
	298	export interface Input {
	299	question: Question;
	300	admin?: boolean;
	301	}
	302
	303	// 2024-12-31 05:00:00 EST
	304	export const transitionDate = 1735639200000;
	305
	306	<const/{ question, admin } = input />
	307	<const/{ id, date, text } = question/>
	308
	309	<${"e-"}
	310	f=(date > transitionDate ? true : undefined)
	311	id=admin ? `q${id}` : undefined
	312	>
	313	<if=admin>
	314	<a
	315	style="margin-right: 0.5rem"
	316	href=`/admin/q+a/${id}`
	317	>[EDIT]</a>
	318	</>
	319	<a>
	320	<time
	321	datetime=formatQuestionISOTimestamp(date)
	322	>${formatQuestionTimestamp(date)}</time>
	323	</a>
	324
	325	<CloverMarkdown ...{ text } />
	326	</>
	327
	328	// this singleton script will make all the '<time>' tags clickable.
	329	client import "./clickable-links.client.ts";
	330
	331	import type { Question } from "@/q+a/models/Question.ts";
	332	import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";
	333	import { CloverMarkdown } from "@/q+a/clover-markdown.tsx";
	334	```
	335
	336	The integration is great, `client import` is quite a magical concept, and I've
	337	tuned it to do the expected thing in my framework.
	338
	339	import { type BlogMeta, formatBlogMeta } from '@/blog/helpers.ts';

src/file-viewer/backend.tsx+11-13

...	...	@@ -93,7 +93,7 @@ app.get("/file/*", async (c, next) => {
93	93	} satisfies APIDirectoryList;
94	94	return c.json(json);
95	95	}
96		c.res = await renderView(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {
	96	c.res = await view.serve(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {
97	97	file,
98	98	hasCotyledonCookie,
99	99	});
...	...	@@ -111,7 +111,7 @@ app.get("/file/*", async (c, next) => {
111	111	!lofi
112	112	) {
113	113	prefetchFile(file.path);
114		c.res = await renderView(c, "file-viewer/clofi", {
	114	c.res = await view.serve(c, "file-viewer/clofi", {
115	115	file,
116	116	hasCotyledonCookie,
117	117	});
...	...	@@ -125,11 +125,10 @@ app.get("/file/*", async (c, next) => {
125	125
126	126	let encoding = decideEncoding(c.req.header("Accept-Encoding"));
127	127
128		let sizeHeader =
129		encoding === "raw"
130		? expectedSize
131		: // Size cannot be known because of compression modes
132		undefined;
	128	let sizeHeader = encoding === "raw"
	129	? expectedSize
	130	// Size cannot be known because of compression modes
	131	: undefined;
133	132
134	133	// Etag
135	134	{
...	...	@@ -217,7 +216,7 @@ app.get("/canvas/:script", async (c, next) => {
217	216	if (!hasAsset(`/js/canvas/${script}.js`)) {
218	217	return next();
219	218	}
220		return renderView(c, "file-viewer/canvas", {
	219	return view.serve(c, "file-viewer/canvas", {
221	220	script,
222	221	});
223	222	});
...	...	@@ -295,10 +294,9 @@ function handleRanges(
295	294	): Response {
296	295	// TODO: multiple ranges
297	296	const rangeSize = ranges.reduce((a, b) => a + (b[1] - b[0] + 1), 0);
298		const rangeBody =
299		streamOrBuffer instanceof ReadableStream
300		? applySingleRangeToStream(streamOrBuffer, ranges)
301		: applyRangesToBuffer(streamOrBuffer, ranges, rangeSize);
	297	const rangeBody = streamOrBuffer instanceof ReadableStream
	298	? applySingleRangeToStream(streamOrBuffer, ranges)
	299	: applyRangesToBuffer(streamOrBuffer, ranges, rangeSize);
302	300	return new Response(rangeBody, {
303	301	status: 206,
304	302	headers: {
...	...	@@ -420,7 +418,7 @@ import { type Context, Hono } from "hono";
420	418
421	419	import * as render from "#engine/render";
422	420	import { etagMatches, hasAsset, serveAsset } from "#sitegen/assets";
423		import { renderView } from "#sitegen/view";
	421	import * as view from "#sitegen/view";
424	422	import { contentTypeFor } from "#sitegen/mime";
425	423
426	424	import { requireFriendAuth } from "@/friend-auth.ts";

src/file-viewer/bin/scan3.ts+20-14

...	...	@@ -17,9 +17,11 @@ export async function main() {
17	17	const start = performance.now();
18	18	const timerSpinner = new Spinner({
19	19	text: () =>
20		`paper clover's scan3 [${((performance.now() - start) / 1000).toFixed(
21		1,
22		)}s]`,
	20	`paper clover's scan3 [${
	21	((performance.now() - start) / 1000).toFixed(
	22	1,
	23	)
	24	}s]`,
23	25	fps: 10,
24	26	});
25	27	using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
...	...	@@ -44,7 +46,7 @@ export async function main() {
44	46	.flatMap((child) =>
45	47	child.kind === MediaFileKind.directory
46	48	? child.getRecursiveFileChildren()
47		: child,
	49	: child
48	50	);
49	51
50	52	qMeta.addMany(
...	...	@@ -103,9 +105,11 @@ export async function main() {
103	105	) {
104	106	date = mediaFile.date;
105	107	console.warn(
106		`M-time on ${publicPath} was likely corrupted. ${formatDate(
107		mediaFile.date,
108		)} -> ${formatDate(stat.mtime)}`,
	108	`M-time on ${publicPath} was likely corrupted. ${
	109	formatDate(
	110	mediaFile.date,
	111	)
	112	} -> ${formatDate(stat.mtime)}`,
109	113	);
110	114	}
111	115	mediaFile = MediaFile.createFile({
...	...	@@ -162,7 +166,7 @@ export async function main() {
162	166	}: Omit<ProcessFileArgs, "spin">) {
163	167	const ext = mediaFile.extensionNonEmpty.toLowerCase();
164	168	let possible = processors.filter((p) =>
165		p.include ? p.include.has(ext) : !p.exclude?.has(ext),
	169	p.include ? p.include.has(ext) : !p.exclude?.has(ext)
166	170	);
167	171	if (possible.length === 0) return;
168	172
...	...	@@ -196,7 +200,7 @@ export async function main() {
196	200	);
197	201	} else {
198	202	possible = order.map(({ id }) =>
199		UNWRAP(possible.find((p) => p.id === id)),
	203	UNWRAP(possible.find((p) => p.id === id))
200	204	);
201	205	}
202	206
...	...	@@ -260,8 +264,9 @@ export async function main() {
260	264	const children = dir.getChildren();
261	265
262	266	// readme.txt
263		const readmeContent =
264		children.find((x) => x.basename === "readme.txt")?.contents ?? "";
	267	const readmeContent = children.find((x) =>
	268	x.basename === "readme.txt"
	269	)?.contents ?? "";
265	270
266	271	// dirsort
267	272	let dirsort: string[] \| null = null;
...	...	@@ -354,7 +359,9 @@ export async function main() {
354	359	console.info(
355	360	"Global Stats:\n" +
356	361	`- File Count: \x1b[1m${count}\x1b[0m\n` +
357		`- Canonical Size: \x1b[1m${formatSize(MediaFile.getByPath("/")!.size)}\x1b[0m\n` +
	362	`- Canonical Size: \x1b[1m${
	363	formatSize(MediaFile.getByPath("/")!.size)
	364	}\x1b[0m\n` +
358	365	`- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n`,
359	366	);
360	367	}
...	...	@@ -459,7 +466,6 @@ const procLoadTextContents: Process = {
459	466	async run({ absPath, mediaFile, stat }) {
460	467	if (stat.size > 1_000_000) return;
461	468	const text = await fs.readFile(absPath, "utf-8");
462		console.log({ text });
463	469	mediaFile.setContents(text);
464	470	},
465	471	};
...	...	@@ -696,7 +702,7 @@ async function unproduceAsset(key: string) {
696	702	const ref = AssetRef.get(key);
697	703	if (ref) {
698	704	ref.unref();
699		console.log(`unref ${key}`);
	705	console.warn(`TODO: unref ${key}`);
700	706	// TODO: remove associated files from target
701	707	}
702	708	}

src/file-viewer/format.ts+34-20

...	...	@@ -29,7 +29,9 @@ export function formatDurationLong(seconds: number) {
29	29	const hours = Math.floor(seconds / 3600);
30	30	const minutes = Math.floor((seconds % 3600) / 60);
31	31	const remainingSeconds = seconds % 60;
32		return `${hours}:${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`;
	32	return `${hours}:${minutes.toString().padStart(2, "0")}:${
	33	remainingSeconds.toString().padStart(2, "0")
	34	}`;
33	35	}
34	36
35	37	export function escapeUri(uri: string) {
...	...	@@ -100,21 +102,27 @@ export function highlightLinksInTextView(
100	102	// Case 1: https:// or http:// URLs
101	103	if (match.startsWith("http")) {
102	104	if (match.includes(findDomain)) {
103		return `<a href="${match
104		.replace(/https?:\/\/paperclover\.net\/+/, "/")
105		.replace(/\/\/+/g, "/")}">${match}</a>`;
	105	return `<a href="${
	106	match
	107	.replace(/https?:\/\/paperclover\.net\/+/, "/")
	108	.replace(/\/\/+/g, "/")
	109	}">${match}</a>`;
106	110	}
107		return `<a href="${match.replace(
108		/\/\/+/g,
109		"/",
110		)}" target="_blank" rel="noopener noreferrer">${match}</a>`;
	111	return `<a href="${
	112	match.replace(
	113	/\/\/+/g,
	114	"/",
	115	)
	116	}" target="_blank" rel="noopener noreferrer">${match}</a>`;
111	117	}
112	118
113	119	// Case 2: domain URLs without protocol
114	120	if (match.startsWith(findDomain)) {
115		return `<a href="${match
116		.replace(findDomain + "/", "/")
117		.replace(/\/\/+/g, "/")}">${match}</a>`;
	121	return `<a href="${
	122	match
	123	.replace(findDomain + "/", "/")
	124	.replace(/\/\/+/g, "/")
	125	}">${match}</a>`;
118	126	}
119	127
120	128	// Case 3: /file/ URLs
...	...	@@ -145,7 +153,7 @@ export function highlightLinksInTextView(
145	153	// Match sibling file names (only if they're not already part of a link)
146	154	if (siblingFiles.length > 0) {
147	155	const escapedBasenames = siblingFiles.map((f) =>
148		f.basename.replace(/[.*+?^${}()\|[\]\\]/g, "\\$&"),
	156	f.basename.replace(/[.*+?^${}()\|[\]\\]/g, "\\$&")
149	157	);
150	158	const pattern = new RegExp(`\\b(${escapedBasenames.join("\|")})\\b`, "g");
151	159	const parts = processedText.split(/(<[^>]*>)/);
...	...	@@ -155,9 +163,11 @@ export function highlightLinksInTextView(
155	163	parts[i] = parts[i].replace(pattern, (match: string) => {
156	164	const file = siblingLookup[match];
157	165	if (file) {
158		return `<a href="/file/${file.path
159		.replace(/^\//, "")
160		.replace(/\/\/+/g, "/")}">${match}</a>`;
	166	return `<a href="/file/${
	167	file.path
	168	.replace(/^\//, "")
	169	.replace(/\/\/+/g, "/")
	170	}">${match}</a>`;
161	171	}
162	172	return match;
163	173	});
...	...	@@ -240,9 +250,11 @@ export function highlightConvo(text: string) {
240	250
241	251	return paras
242	252	.map(({ speaker, lines }) => {
243		return `<div class="s-${speaker}">${lines
244		.map((line) => `<div class="line">${line}</div>`)
245		.join("\n")}</div>`;
	253	return `<div class="s-${speaker}">${
	254	lines
	255	.map((line) => `<div class="line">${line}</div>`)
	256	.join("\n")
	257	}</div>`;
246	258	})
247	259	.join("\n");
248	260	}
...	...	@@ -267,10 +279,12 @@ export function formatDate(dateTime: Date) {
267	279	? dateTime < unknownDate
268	280	? "??.??.??"
269	281	: `xx.xx.${21 + Math.floor(dateTime.getTime() / 86400000)}`
270		: `${(dateTime.getMonth() + 1).toString().padStart(2, "0")}.${dateTime
	282	: `${(dateTime.getMonth() + 1).toString().padStart(2, "0")}.${
	283	dateTime
271	284	.getDate()
272	285	.toString()
273		.padStart(2, "0")}.${dateTime.getFullYear().toString().slice(2)}`;
	286	.padStart(2, "0")
	287	}.${dateTime.getFullYear().toString().slice(2)}`;
274	288	}
275	289
276	290	import type { MediaFile } from "@/file-viewer/models/MediaFile.ts";

src/file-viewer/rsync.ts+6-2

...	...	@@ -131,8 +131,12 @@ export class Parse {
131	131	percentage: Number(percentageStr),
132	132	timeElapsed,
133	133	transferNumber: this.currentTransfer,
134		filesToCheck: toCheckStr ? this.toCheck = Number(toCheckStr) : this.toCheck,
135		totalFiles: totalStr ? this.totalFiles = Number(totalStr) : this.totalFiles,
	134	filesToCheck: toCheckStr
	135	? this.toCheck = Number(toCheckStr)
	136	: this.toCheck,
	137	totalFiles: totalStr
	138	? this.totalFiles = Number(totalStr)
	139	: this.totalFiles,
136	140	speed: speed \|\| null,
137	141	};
138	142	}

src/global.css+15-11

...	...	@@ -4,9 +4,7 @@
4	4	font-weight: 400 750;
5	5	font-style: normal;
6	6	font-display: swap;
7		font-variation-settings:
8		"CASL" 0.25,
9		"MONO" 0;
	7	font-variation-settings: "CASL" 0.25, "MONO" 0;
10	8	font-style: oblique -15deg 0deg;
11	9	unicode-range: U+0020-007E;
12	10	}
...	...	@@ -16,9 +14,7 @@
16	14	font-weight: 400 800;
17	15	font-style: normal;
18	16	font-display: swap;
19		font-variation-settings:
20		"CASL" 0.25,
21		"MONO" 1;
	17	font-variation-settings: "CASL" 0.25, "MONO" 1;
22	18	font-style: oblique -15deg 0deg;
23	19	unicode-range: U+0020-007E;
24	20	}
...	...	@@ -28,13 +24,21 @@
28	24	font-weight: 400 800;
29	25	font-style: normal;
30	26	font-display: swap;
31		font-variation-settings:
32		"CASL" 0.25,
33		"MONO" 1;
	27	font-variation-settings: "CASL" 0.25, "MONO" 1;
34	28	font-style: oblique -15deg 0deg;
35	29	unicode-range:
36		U+00C0-00FF, U+00A9, U+2190-2193, U+2018, U+2019, U+201C, U+201D, U+2022,
37		U+00A0-00A8, U+00AA-00BF, U+2194-2199, U+0100-017F;
	30	U+00C0-00FF,
	31	U+00A9,
	32	U+2190-2193,
	33	U+2018,
	34	U+2019,
	35	U+201C,
	36	U+201D,
	37	U+2022,
	38	U+00A0-00A8,
	39	U+00AA-00BF,
	40	U+2194-2199,
	41	U+0100-017F;
38	42	}
39	43
40	44	*,

src/q+a/backend.ts+20-22

...	...	@@ -45,7 +45,7 @@ app.post("/q+a", async (c) => {
45	45	return sendSuccess(c, new Date());
46	46	}
47	47
48		const ipAddr = c.req.header("cf-connecting-ip");
	48	const ipAddr = c.req.header("x-forwarded-for");
49	49	if (ipAddr) {
50	50	input.sourceName = uniqueNamesGenerator({
51	51	dictionaries: [adjectives, colors, animals],
...	...	@@ -54,11 +54,6 @@ app.post("/q+a", async (c) => {
54	54	});
55	55	}
56	56
57		const cfIPCountry = c.req.header("cf-ipcountry");
58		if (cfIPCountry) {
59		input.sourceLocation = cfIPCountry;
60		}
61
62	57	if (ipAddr && PROXYCHECK_API_KEY) {
63	58	const proxyCheck = await fetch(
64	59	`https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,
...	...	@@ -77,17 +72,19 @@ app.post("/q+a", async (c) => {
77	72	proxyCheck[ipAddr].organisation ??
78	73	proxyCheck[ipAddr].provider ?? "unknown";
79	74	}
80		if (Number(proxyCheck[ipAddr].risk) > 72) {
	75	if (Number(proxyCheck[ipAddr].risk) > 78) {
81	76	return questionFailure(
82	77	c,
83	78	403,
84		"This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.",
	79	"This IP address has been flagged as a high risk IP address. If " +
	80	"you are using a VPN/Proxy, please disable it and try again.",
85	81	text,
86	82	);
87	83	}
88	84	}
89	85	}
90	86
	87	view.regenerate("q+a inbox");
91	88	const date = Question.create(
92	89	QuestionType.pending,
93	90	JSON.stringify(input),
...	...	@@ -104,7 +101,7 @@ async function sendSuccess(c: Context, date: Date) {
104	101	id: formatQuestionId(date),
105	102	}, { status: 200 });
106	103	}
107		c.res = await renderView(c, "q+a/success", {
	104	c.res = await view.serve(c, "q+a/success", {
108	105	permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,
109	106	});
110	107	}
...	...	@@ -130,7 +127,7 @@ app.get("/q+a/:id", async (c, next) => {
130	127	if (image) {
131	128	return getQuestionImage(question, c.req.method === "HEAD");
132	129	}
133		return renderView(c, "q+a/permalink", { question });
	130	return view.serve(c, "q+a/permalink", { question });
134	131	});
135	132
136	133	// Admin
...	...	@@ -138,7 +135,7 @@ app.get("/admin/q+a", async (c) => {
138	135	return serveAsset(c, "/admin/q+a", 200);
139	136	});
140	137	app.get("/admin/q+a/inbox", async (c) => {
141		return renderView(c, "q+a/backend-inbox", {});
	138	return view.serve(c, "q+a/backend-inbox", {});
142	139	});
143	140	app.delete("/admin/q+a/:id", async (c, next) => {
144	141	const id = c.req.param("id");
...	...	@@ -152,6 +149,7 @@ app.delete("/admin/q+a/:id", async (c, next) => {
152	149	} else {
153	150	Question.rejectByQmid(question.qmid);
154	151	}
	152	view.regenerate("q+a");
155	153	return c.json({ success: true, message: "ok" });
156	154	});
157	155	app.patch("/admin/q+a/:id", async (c, next) => {
...	...	@@ -165,6 +163,7 @@ app.patch("/admin/q+a/:id", async (c, next) => {
165	163	return questionFailure(c, 400, "Bad Request");
166	164	}
167	165	Question.updateByQmid(question.qmid, form.text, form.type);
	166	view.regenerate("q+a");
168	167	return c.json({ success: true, message: "ok" });
169	168	});
170	169	app.get("/admin/q+a/:id", async (c, next) => {
...	...	@@ -177,20 +176,22 @@ app.get("/admin/q+a/:id", async (c, next) => {
177	176	let pendingInfo: null \| PendingQuestionData = null;
178	177	if (question.type === QuestionType.pending) {
179	178	pendingInfo = JSON.parse(question.text) as PendingQuestionData;
180		question.text = pendingInfo.prompt.trim().split("\n").map((line) =>
181		line.trim().length === 0 ? "" : `q: ${line.trim()}`
182		).join("\n") + "\n\n";
	179	question.text = pendingInfo.prompt
	180	.trim()
	181	.split("\n")
	182	.map((line) => (line.trim().length === 0 ? "" : `q: ${line.trim()}`))
	183	.join("\n") + "\n\n";
183	184	question.type = QuestionType.normal;
184	185	}
185	186
186		return renderView(c, "q+a/editor", {
	187	return view.serve(c, "q+a/editor", {
187	188	pendingInfo,
188	189	question,
189	190	});
190	191	});
191	192
192	193	app.get("/q+a/things/random", async (c) => {
193		c.res = await renderView(c, "q+a/things-random", {});
	194	c.res = await view.serve(c, "q+a/things-random", {});
194	195	});
195	196
196	197	async function questionFailure(
...	...	@@ -202,7 +203,7 @@ async function questionFailure(
202	203	if (c.req.header("Accept")?.includes("application/json")) {
203	204	return c.json({ success: false, message, id: null }, { status });
204	205	}
205		return await renderView(c, "q+a/fail", {
	206	return await view.serve(c, "q+a/fail", {
206	207	error: message,
207	208	content,
208	209	});
...	...	@@ -218,11 +219,8 @@ import {
218	219	} from "unique-names-generator";
219	220	import { hasAdminToken } from "../admin.ts";
220	221	import { serveAsset } from "#sitegen/assets";
221		import {
222		PendingQuestion,
223		PendingQuestionData,
224		} from "./models/PendingQuestion.ts";
	222	import type { PendingQuestionData } from "./models/PendingQuestion.ts";
225	223	import { Question, QuestionType } from "./models/Question.ts";
226		import { renderView } from "#sitegen/view";
	224	import * as view from "#sitegen/view";
227	225	import { getQuestionImage } from "./image.tsx";
228	226	import { formatQuestionId, questionIdToTimestamp } from "./format.ts";

src/q+a/clover-markdown.tsx+1-3

...	...	@@ -144,9 +144,7 @@ function ListRenderer(node: ASTNode, children: any[]) {
144	144	const T = node.ordered ? "ol" : "ul";
145	145	return (
146	146	<T>
147		{children.map((child) => (
148		<li>{child}</li>
149		))}
	147	{children.map((child) => <li>{child}</li>)}
150	148	</T>
151	149	);
152	150	}

src/q+a/image.tsx+5-4

...	...	@@ -11,7 +11,7 @@ const getBrowser = RefCountedExpirable(
11	11	);
12	12
13	13	export async function renderQuestionImage(question: Question) {
14		const html = await renderViewToString("q+a/image-embed", { question });
	14	const html = await view.renderToString("q+a/image-embed", { question });
15	15
16	16	// this browser session will be reused if multiple images are generated
17	17	// either at the same time or within a 5-minute time span. the dispose
...	...	@@ -44,14 +44,15 @@ export async function getQuestionImage(
44	44	question: Question,
45	45	headOnly: boolean,
46	46	): Promise<Response> {
47		const hash = crypto.createHash("sha1")
	47	const hash = crypto
	48	.createHash("sha1")
48	49	.update(question.qmid + question.type + question.text)
49	50	.digest("hex");
50	51
51	52	const headers = {
52	53	"Content-Type": "image/png",
53	54	"Cache-Control": "public, max-age=31536000",
54		"ETag": `"${hash}"`,
	55	ETag: `"${hash}"`,
55	56	"Last-Modified": question.date.toUTCString(),
56	57	};
57	58
...	...	@@ -78,4 +79,4 @@ import * as path from "node:path";
78	79	import * as puppeteer from "puppeteer";
79	80	import { Question } from "@/q+a/models/Question.ts";
80	81	import { RefCountedExpirable } from "#sitegen/async";
81		import { renderViewToString } from "#sitegen/view";
	82	import * as view from "#sitegen/view";

src/q+a/pages/admin.q+a.marko+1-1

...	...	@@ -7,7 +7,7 @@ export const meta: Metadata = {
7	7	description: "ask clover a question",
8	8	};
9	9	export const regenerate = {
10		manual: true,
	10	tags: ["q+a", "q+a inbox"]
11	11	};
12	12
13	13	<const/inboxSize = PendingQuestion.getAll().length />

src/q+a/pages/q+a.marko+4-6

...	...	@@ -1,11 +1,9 @@
1		export * as layout from "../layout.tsx";
2		export const regenerate = {
3		manual: true,
4		};
5
6	1	export interface Input {
7	2	admin?: boolean;
8	3	}
	4
	5	export * as layout from "../layout.tsx";
	6	export const regenerate = { tags: ["q+a"] };
9	7	export const meta: Metadata = {
10	8	title: "paper clover q+a",
11	9	description: "ask clover a question",
...	...	@@ -14,7 +12,7 @@ export const meta: Metadata = {
14	12	<const/{ admin = false } = input />
15	13	<const/questions = [...Question.getAll()] />
16	14
17		<if=true>
	15	<if=!admin>
18	16	<question-form />
19	17	</>
20	18	<for\|question\| of=questions>

src/q+a/views/backend-inbox.marko+2-2

...	...	@@ -19,8 +19,8 @@ export const meta = { title: 'question answer inbox' };
19	19	</time>
20	20	<div style="color: dodgerblue; margin-bottom: 0.25rem">
21	21	${q.sourceName}
22		${q.sourceLocation !== "unknown" ? `from ${q.sourceLocation}` : null}
23		${q.sourceVPN ? `(VPN: ${q.sourceVPN})` : null}
	22	${q.sourceLocation !== "unknown" ? `from ${q.sourceLocation}` : ''}
	23	${q.sourceVPN ? `(VPN: ${q.sourceVPN})` : ''}
24	24	</div>
25	25	<p style="white-space: pre-wrap">${q.prompt}</p>
26	26	<p>

src/source-of-truth.ts+6-2

...	...	@@ -33,8 +33,12 @@ const rawFileRoot = process.env.CLOVER_FILE_RAW ??
33	33	const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
34	34	path.join(nasRoot, "Documents/Config/paperclover/derived");
35	35
36		if (!fs.existsSync(rawFileRoot)) throw new Error(`${rawFileRoot} does not exist`);
37		if (!fs.existsSync(derivedFileRoot)) throw new Error(`${derivedFileRoot} does not exist`);
	36	if (!fs.existsSync(rawFileRoot)) {
	37	throw new Error(`${rawFileRoot} does not exist`);
	38	}
	39	if (!fs.existsSync(derivedFileRoot)) {
	40	throw new Error(`${derivedFileRoot} does not exist`);
	41	}
38	42
39	43	type Awaitable<T> = T \| Promise<T>;
40	44

tsconfig.json+3-2

...	...	@@ -5,7 +5,7 @@
5	5	"incremental": true,
6	6	"jsx": "react-jsxdev",
7	7	"jsxImportSource": "#engine",
8		"lib": ["dom", "esnext", "esnext.iterator"],
	8	"lib": ["esnext", "dom", "dom.iterable"],
9	9	"module": "nodenext",
10	10	"noEmit": true,
11	11	"outDir": ".clover/ts",
...	...	@@ -14,7 +14,8 @@
14	14	"skipLibCheck": true,
15	15	"strict": true,
16	16	"verbatimModuleSyntax": true,
17		"target": "es2022"
	17	"target": "es2022",
	18	"noUncheckedIndexedAccess": true
18	19	},
19	20	"include": ["framework/*/", "src/*/"]
20	21	}

author	clover caruso <git@paperclover.net> 2025-08-11 22:43:27-07:00
committer	clover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
log	98be3e2bd40a40c6595de522d087092df63cdbb7
tree	ac5bf4b195a15f8e87222dad3ccceeff0592e8b4
parent	8d0985c06cb96067dc084cb17f5c3bfe8c43f56d
signature	Commit is signed but in an unrecognized format.

feat: dynamic page regeneration (#24)