Commit 6a4e985a960dc0c1fcf7776082292b717d25a6d8 | sitegen

author	clover caruso <git@paperclover.net> 2026-03-22 23:36:20-07:00
committer	clover caruso <git@paperclover.net> 2026-05-28 04:15:54-07:00
log	6a4e985a960dc0c1fcf7776082292b717d25a6d8
tree	5fa61e3cfe8b9a38a6ec90144cf44c13c7ab4031
parent	907e9083b681531d3be2f8ba0cafc63dede6e61d
signature	Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

WIP: feat(lib): Incremental Module Loader API

been playing with `node:vm`'s experimental modules. it is neat because
it gives us a hook into the module system that no other runtime has:

- able to reflect what files are pointed to by each file
- can adjust how files are processed before loading
- can unload/reload modules, with old ones being garbage collected

this commit rebuilds `hot.ts` and `incremental.ts` as a new system that
only depends on these node builtins, which (TODO) provides:

- simple module loading `new Incremental().import("src/main.ts");`
- plugin system to load extensions like `.tsx`, `.marko`, `.svelte`
- tracing the files used in resolution and initialization
  - allowing arbitrary dependencies to be added
- allowing arbitrary jobs that cache when re-run
- creating a filesystem watcher to detect changes
  - performing trivial module reloading (maybe full HMR)
- serializing the graph state to disk, along with v8 cache data

by implementing this, clover sitegen can:

- stop patching node.js internals
- drop the server-side esbuild code (frontend still needs it)
- use real es modules
- provide a base for more features (issue 4 and 5), which could possibly
  be solved as their own top-level projects.

this is also a great time to solve issue 27 (incremntal tracking)

24 files changed, 4335 insertions(+), 1543 deletions(-)

framework/hot.ts deleted-307

...	@@ -1,307 +0,0 @@
1	// This implements the ability to use TS, TSX, and more plugins
2	// in Node.js. It cannot be built on the ES module loader,
3	// because there is no exposed way to replace modules when
4	// needed (see nodejs#49442).
5	//
6	// It also allows using a simple compile cache, which is used by
7	// the site generator to determine when code changes.
8	export const projectRoot = path.resolve(import.meta.dirname, "../");
9	export const projectSrc = path.resolve(projectRoot, "src");
10
11	using _ = null;
12
13	// Create a project-relative require. For convenience, it is generic-typed.
14	export const load = createRequire(
15	pathToFileURL(path.join(projectRoot, "run.js")).toString(),
16	) as {
17	<T = unknown>(id: string): T;
18	extensions: NodeJS.Dict<(mod: NodeJS.Module, file: string) => unknown>;
19	cache: NodeJS.Dict<NodeJS.Module>;
20	resolve: (id: string, o?: { paths: string[] }) => string;
21	};
22	export const { cache } = load;
23
24	// Register extensions by overwriting `require.extensions`
25	const require = load;
26	const exts = require.extensions;
27	exts[".ts"] = loadEsbuild;
28	exts[".tsx"] = loadEsbuild;
29	exts[".jsx"] = loadEsbuild;
30	exts[".marko"] = loadMarko;
31	exts[".mdo"] = loadMarkodown;
32	exts[".css"] = loadCss;
33
34	// Intercept all module load calls to track CSS imports + file times.
35	export interface FileStat {
36	cssImportsRecursive: string[] \| null;
37	lastModified: number;
38	imports: string[];
39	}
40	const fileStats = new Map<string, FileStat>();
41
42	export function getFileStat(filepath: string) {
43	return fileStats.get(path.resolve(filepath));
44	}
45
46	function shouldTrackPath(filename: string) {
47	return !filename.includes("node_modules");
48	}
49
50	const Module = load<typeof import("node:module")>("node:module");
51	const ModulePrototypeUnderscoreCompile = Module.prototype._compile;
52	Module.prototype._compile = function(
53	content: string,
54	filename: string,
55	format: "module" \| "commonjs",
56	) {
57	const result = ModulePrototypeUnderscoreCompile.call(
58	this,
59	content,
60	filename,
61	format,
62	);
63	if (shouldTrackPath(filename)) {
64	const stat = fs.statSync(filename);
65	const cssImportsMaybe: string[] = [];
66	const imports: string[] = [];
67	for (const childModule of this.children) {
68	const { filename: file, cloverClientRefs } = childModule;
69	if (file.endsWith(".css")) cssImportsMaybe.push(file);
70	else {
71	const child = fileStats.get(file);
72	if (!child) continue;
73	const { cssImportsRecursive } = child;
74	if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
75	imports.push(file);
76	(childModule.cloverImporters ??= []).push(this);
77	if (cloverClientRefs && cloverClientRefs.length > 0) {
78	(this.cloverClientRefs ??= []).push(...cloverClientRefs);
79	}
80	}
81	}
82	fileStats.set(filename, {
83	cssImportsRecursive: cssImportsMaybe.length > 0
84	? Array.from(new Set(cssImportsMaybe))
85	: null,
86	imports,
87	lastModified: Math.floor(stat.mtimeMs),
88	});
89	}
90	return result;
91	};
92
93	function loadEsbuild(module: NodeJS.Module, filepath: string) {
94	return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
95	}
96
97	interface LoadOptions {
98	scannedClientRefs?: string[];
99	}
100	export function loadEsbuildCode(
101	module: NodeJS.Module,
102	filepath: string,
103	src: string,
104	opt: LoadOptions = {},
105	) {
106	if (filepath === import.meta.filename) {
107	module.exports = self;
108	return;
109	}
110	let loader: any = "tsx";
111	if (filepath.endsWith(".ts")) loader = "ts";
112	else if (filepath.endsWith(".jsx")) loader = "jsx";
113	else if (filepath.endsWith(".js")) loader = "js";
114	if (opt.scannedClientRefs) {
115	module.cloverClientRefs = opt.scannedClientRefs;
116	} else {
117	let { code, refs } = resolveClientRefs(src, filepath);
118	module.cloverClientRefs = refs;
119	src = code;
120	}
121	if (src.includes("import.meta")) {
122	src = `
123	import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
124	import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
125	import.meta.filename = ${JSON.stringify(filepath)};
126	import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r));
127	`
128	.trim()
129	.replace(/[\n\s]/g, "") + src;
130	}
131	src = esbuild.transformSync(src, {
132	loader,
133	format: "cjs",
134	target: "esnext",
135	jsx: "automatic",
136	jsxImportSource: "#jsx",
137	jsxDev: true,
138	sourcemap: "inline",
139	sourcefile: path.basename(filepath),
140	sourceRoot: path.dirname(filepath),
141	}).code;
142	return module._compile(src, filepath, "commonjs");
143	}
144
145	export function resolveClientRef(sourcePath: string, ref: string) {
146	const filePath = resolveFrom(sourcePath, ref);
147	if (!filePath.endsWith(".client.ts")) {
148	throw new Error("addScript must take a .client.ts");
149	}
150	return path.relative(projectSrc, filePath);
151	}
152
153	let lazyMarko: typeof import("./marko.ts") \| null = null;
154	function loadMarko(module: NodeJS.Module, filepath: string) {
155	lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
156	lazyMarko.loadMarko(module, filepath);
157	}
158	function loadMarkodown(module: NodeJS.Module, filepath: string) {
159	lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
160	lazyMarko.loadMarkodown(module, filepath);
161	}
162
163	function loadCss(module: NodeJS.Module, _filepath: string) {
164	module.exports = {};
165	}
166
167	export function reloadRecursive(filepath: string) {
168	filepath = path.resolve(filepath);
169	const existing = cache[filepath];
170	if (existing) deleteRecursiveInner(filepath, existing);
171	fileStats.clear();
172	return require(filepath);
173	}
174
175	export function unload(filepath: string) {
176	lazyMarko?.markoCache.delete(filepath);
177	filepath = path.resolve(filepath);
178	const module = cache[filepath];
179	if (!module) return;
180	delete cache[filepath];
181	for (const importer of module.cloverImporters ?? []) {
182	unload(importer.filename);
183	}
184	}
185
186	function deleteRecursiveInner(id: string, module: any) {
187	if (id.includes(path.sep + "node_modules" + path.sep)) {
188	return;
189	}
190	delete cache[id];
191	for (const child of module.children) {
192	if (child.filename.includes("/engine/")) return;
193	const existing = cache[child.filename];
194	if (existing === child) deleteRecursiveInner(child.filename, existing);
195	}
196	}
197
198	export function getCssImports(filepath: string) {
199	filepath = path.resolve(filepath);
200	if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
201	return fileStats.get(filepath)?.cssImportsRecursive ?? [];
202	}
203
204	export function getClientScriptRefs(filepath: string) {
205	filepath = path.resolve(filepath);
206	const module = require.cache[filepath];
207	if (!module) throw new Error(filepath + " was never loaded");
208	return module.cloverClientRefs ?? [];
209	}
210
211	export function getSourceCode(filepath: string) {
212	filepath = path.resolve(filepath);
213	const module = require.cache[filepath];
214	if (!module) throw new Error(filepath + " was never loaded");
215	if (!module.cloverSourceCode) {
216	throw new Error(filepath + " did not record source code");
217	}
218	return module.cloverSourceCode;
219	}
220
221	export function resolveFrom(src: string, dest: string) {
222	try {
223	return createRequire(src).resolve(dest);
224	} catch (err: any) {
225	if (
226	err.code === "MODULE_NOT_FOUND" && err.requireStack
227	&& err.requireStack.length <= 1
228	) {
229	err.message = err.message.split("\n")[0] + " from '" + src + "'";
230	}
231	throw err;
232	}
233	}
234
235	const importRegExp = /import\s+(\*\sas\s([a-zA-Z0-9$_]+)\|{[^}]+})\s+from\s+(?:"#sitegen"\|'#sitegen')/s;
236	const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))?/;
237	interface ResolvedClientRefs {
238	code: string;
239	refs: string[];
240	}
241	export function resolveClientRefs(
242	code: string,
243	filepath: string,
244	): ResolvedClientRefs {
245	// This match finds a call to 'import ... from "#sitegen"'
246	const importMatch = code.match(importRegExp);
247	if (!importMatch) return { code, refs: [] };
248	const items = UNWRAP(importMatch[1]);
249	let identifier = "";
250	if (items.startsWith("{")) {
251	const clauseMatch = items.match(getSitegenAddScriptRegExp);
252	if (!clauseMatch) return { code, refs: [] }; // did not import
253	identifier = clauseMatch[1] \|\| "addScript";
254	} else if (items.startsWith("*")) {
255	identifier = importMatch[2] + "\\s\\.\\saddScript";
256	} else {
257	throw new Error("Impossible");
258	}
259	identifier = identifier.replaceAll("$", "\\$"); // only needed escape
260	const findCallsRegExp = new RegExp(
261	`\\b(${identifier})\\s*\$("[^"]+"\|'[^']+')\$`,
262	"gs",
263	);
264	const scannedClientRefs = new Set<string>();
265	code = code.replace(findCallsRegExp, (_, call, arg) => {
266	const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
267	const resolved = resolveClientRef(filepath, ref);
268	scannedClientRefs.add(resolved);
269	return `${call}(${JSON.stringify(getScriptId(resolved))})`;
270	});
271	return { code, refs: Array.from(scannedClientRefs) };
272	}
273
274	export function getScriptId(file: string) {
275	return (
276	path.isAbsolute(file) ? path.relative(projectSrc, file) : file
277	).replaceAll("\\", "/");
278	}
279
280	declare global {
281	namespace NodeJS {
282	interface Module {
283	cloverClientRefs?: string[];
284	cloverSourceCode?: string;
285	cloverImporters?: Module[];
286
287	_compile(
288	this: NodeJS.Module,
289	content: string,
290	filepath: string,
291	format: "module" \| "commonjs",
292	): unknown;
293	}
294	}
295	}
296	declare module "node:module" {
297	export function _resolveFilename(id: string, parent: NodeJS.Module): unknown;
298	}
299
300	import * as fs from "#sitegen/fs";
301	import { UNWRAP } from "@clo/lib/assert";
302	import * as esbuild from "esbuild";
303	import { Buffer } from "node:buffer";
304	import { createRequire } from "node:module";
305	import * as path from "node:path";
306	import { pathToFileURL } from "node:url";
307	import * as self from "./hot.ts";

framework/incremental.ts deleted-738

...	@@ -1,738 +0,0 @@
1	// Incremental build system using singleton state.
2	// See `work()`, `compile()`, and `invalidate()` for details.
3	//
4	// All state is serializable to allow recovering state across sessions.
5	// This library special-cases the asset map, but is otherwise agnostic.
6	let running = false;
7	let jobs = 0;
8	let newKeys = 0;
9	let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
10	let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
11	let works = new Map<string, Work>();
12	let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
13	let writes = new Map<string, FileWrite>();
14	let assets = new Map<string, Asset>(); // keyed by hash
15
16	export interface Ref<T> {
17	key: string;
18	/** This method is compatible with `await` syntax */
19	then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20	get value(): T \| null;
21	/** The progress node passed into the input */
22	progress: progress.Node \| progress.Ref;
23	}
24	type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
25
26	/**
27	* Declare and a unit of work. Return value is memoized and only rebuilt when
28	* inputs change. Inputs are tracked via the `io` interface, as well as a hash
29	* of the `input` param and caller source code. Outputs are written at the end
30	* of a compilation (see `compile`).
31	*
32	* Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33	*/
34	export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
35	export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
36	export function work<I, O>(
37	p: progress.Ref,
38	job: Job<I, O>,
39	input?: I,
40	): Ref<O> {
41	const source = JSON.stringify(UNWRAP(stack.capture()[0]));
42	const keySource = [source, util.inspect(input)].join(":");
43	const key = crypto.createHash("sha1").update(keySource).digest("base64url");
44	ASSERT(running);
45	ASSERT(
46	!seenWorks.has(key),
47	`Key '${key}' must be unique during the build. `
48	+ `To fix this, provide a manual 'key' argument.`,
49	);
50	seenWorks.add(key);
51
52	const prev = works.get(key) as Work<O> \| null;
53	if (prev) {
54	return {
55	key,
56	then: (done) => done(prev.value),
57	value: prev.value,
58	progress: p,
59	};
60	}
61
62	async function perform() {
63	const io = new Io(key, p);
64	jobs += 1;
65	newKeys += 1;
66	try {
67	const value = await job(io, input as I);
68	validateSerializable(value, "");
69	const { reads, writes } = io;
70	works.set(key, { value, affects: [], reads, writes });
71	for (const add of reads.files) {
72	const { affects } = UNWRAP(files.get(add));
73	ASSERT(!affects.includes(key));
74	affects.push(key);
75	}
76	for (const add of reads.works) {
77	const { affects } = UNWRAP(works.get(add));
78	ASSERT(!affects.includes(key));
79	affects.push(key);
80	}
81	return value;
82	} finally {
83	jobs -= 1;
84	}
85	}
86
87	let cached: Promise<O>;
88	return {
89	key,
90	then(fufill, reject) {
91	(cached ??= perform()).then(fufill, reject);
92	},
93	get value() {
94	return (works.get(this.key)?.value as O) ?? null;
95	},
96	progress: p,
97	};
98	}
99
100	export async function compile<T>(
101	compiler: (node: progress.Ref) => Promise<T>,
102	p: progress.Ref = progress,
103	) {
104	ASSERT(!running, `Cannot run twice at the same time.`);
105	const startDate = new Date();
106	const start = performance.now();
107	const node = p.start("sitegen!");
108	try {
109	running = true;
110	ASSERT(jobs === 0);
111
112	const value = await compiler(node);
113	ASSERT(jobs === 0);
114	await flush(node, start);
115	return {
116	value,
117	watchFiles: new Set(files.keys()),
118	newOutputs: Array.from(seenWrites)
119	.filter((x) => x.startsWith("f:"))
120	.map((x) => x.slice(2)),
121	newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
122	};
123	} finally {
124	node.text = `sitegen at ${startDate.toLocaleDateString()} ${startDate.toLocaleTimeString()}`;
125	node.end();
126	running = false;
127	newKeys = 0;
128	seenWrites.clear();
129	seenWorks.clear();
130	}
131	}
132
133	export async function flush(progress: progress.Node, start: number) {
134	using _ = progress.start("incremental flush");
135	// Trim
136	const detachedFiles = new Set<string>();
137	const referencedAssets = new Set<string>();
138	for (const [k, v] of works) {
139	const assets = v.writes.assets;
140	if (seenWorks.has(k)) {
141	for (const asset of assets.values()) referencedAssets.add(asset.hash);
142	continue;
143	}
144	deleteWork(k);
145	}
146	for (const [k, file] of files) {
147	if (file.affects.length > 0) continue;
148	files.delete(k);
149	detachedFiles.add(k);
150	}
151	for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
152
153	const p: Promise<void>[] = [];
154	// File writes
155	let dist = 0;
156	for (const [key, { buffer, size }] of writes) {
157	if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
158	dist += size;
159	}
160	// Asset map
161	{
162	const { json, blob, dynamic, dts } = getAssetManifest();
163	const jsonString = Buffer.from(JSON.stringify(json));
164	p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
165	p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
166	p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
167	for (const [k, v] of dynamic) {
168	p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
169	}
170	dist += blob.byteLength + jsonString.byteLength;
171	}
172	await Promise.all(p);
173
174	// Incremental state
175	const serialized = msgpackr.pack(serialize());
176	await fs.writeMkdir(".clover/incr.state", serialized);
177	const time = (performance.now() - start).toFixed(0);
178	progress.log.info(
179	`sitegen! in ${time} ms\n`
180	+ ` - ${works.size} keys (${works.size - newKeys} cached)\n`
181	+ ` - ${assets.size} static assets\n`
182	+ ` - dist: ${formatSize(dist)},`
183	+ ` incremental: ${formatSize(serialized.byteLength)}`,
184	);
185	}
186
187	export async function restore() {
188	let buffer;
189	try {
190	buffer = await fs.readFile(".clover/incr.state");
191	} catch (err: any) {
192	if (err.code !== "ENOENT") throw err;
193	}
194	if (!buffer) return;
195	await deserialize(buffer);
196	}
197
198	function forceInvalidate(entry: { affects: string[] }) {
199	const queue = [...entry.affects];
200	let key;
201	while ((key = queue.shift())) {
202	const affects = deleteWork(key);
203	queue.push(...affects);
204	}
205	}
206
207	function deleteWork(key: string) {
208	const work = works.get(key);
209	if (!work) return [];
210	const { reads, affects, writes: w } = work;
211	for (const remove of reads.files) {
212	const { affects } = UNWRAP(files.get(remove));
213	ASSERT(affects.includes(key));
214	affects.splice(affects.indexOf(key), 1);
215	}
216	for (const remove of reads.works) {
217	const { affects } = UNWRAP(works.get(remove), remove);
218	ASSERT(affects.includes(key));
219	affects.splice(affects.indexOf(key), 1);
220	}
221	for (const remove of affects) {
222	const {
223	reads: { works: list },
224	} = UNWRAP(works.get(remove), remove);
225	ASSERT(list.has(key));
226	list.delete(key);
227	}
228	for (const file of w.files) {
229	if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
230	}
231	// Assets are temporarily kept, trimmed via manual GC after compilation.
232
233	works.delete(key);
234	return affects;
235	}
236
237	export function reset() {
238	ASSERT(!running);
239	works.clear();
240	files.clear();
241	assets.clear();
242	}
243
244	export function serialize() {
245	const fileEntries = Array.from(
246	files,
247	([k, v]) =>
248	[
249	k,
250	v.type,
251	v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
252	...v.affects,
253	] as const,
254	);
255	const workEntries = Array.from(
256	works,
257	([k, v]) =>
258	[
259	k,
260	v.value,
261	Array.from(v.reads.files),
262	Array.from(v.reads.works),
263	Array.from(v.writes.files),
264	Array.from(
265	v.writes.assets,
266	([k, { headers }]) => [k, headers] as const,
267	),
268	v.affects,
269	] as const,
270	);
271	const expectedFilesOnDisk = Array.from(
272	writes,
273	([k, { size, work }]) => [k, size, work] as const,
274	);
275	const assetEntries = Array.from(
276	assets,
277	([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
278	);
279	return [
280	1,
281	fileEntries,
282	workEntries,
283	expectedFilesOnDisk,
284	assetEntries,
285	] as const;
286	}
287	type SerializedState = ReturnType<typeof serialize>;
288
289	/* No-op on failure */
290	async function deserialize(buffer: Buffer) {
291	const decoded = msgpackr.decode(buffer) as SerializedState;
292	if (!Array.isArray(decoded)) return;
293	if (decoded[0] !== 1) return;
294	const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] = decoded;
295	for (const [k, type, content, ...affects] of fileEntries) {
296	if (type === "f") {
297	ASSERT(typeof content === "number");
298	files.set(k, { type, affects, lastModified: content });
299	} else if (type === "d") {
300	ASSERT(typeof content === "string");
301	files.set(k, { type, affects, contentHash: content, contents: [] });
302	} else {
303	files.set(k, { type, affects });
304	}
305	}
306	for (const entry of workEntries) {
307	const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] = entry;
308	works.set(k, {
309	value,
310	reads: {
311	files: new Set(readFiles),
312	works: new Set(readWorks),
313	},
314	writes: {
315	files: new Set(writeFiles),
316	assets: new Map(
317	Array.from(writeAssets, ([k, headers]) => [
318	k,
319	{
320	hash: JSON.parse(UNWRAP(headers.etag)),
321	headers,
322	},
323	]),
324	),
325	},
326	affects,
327	});
328	}
329	const statFiles = await Promise.all(
330	expectedFilesOnDisk.map(([k, size, work]) =>
331	fs
332	.stat(path.join(".clover/o", k))
333	.catch((err) => {
334	if (err.code === "ENOENT") return null;
335	throw err;
336	})
337	.then((stat) => ({ k, size, work, stat }))
338	),
339	);
340	for (const { k, stat, work, size } of statFiles) {
341	if (stat?.size === size) {
342	writes.set(k, {
343	size: size,
344	buffer: null,
345	work,
346	});
347	} else {
348	forceInvalidate({ affects: [work] });
349	}
350	}
351	for (const [hash, raw, gzip, zstd] of assetEntries) {
352	assets.set(hash, { raw, gzip, zstd });
353	}
354
355	await Promise.all(
356	Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
357	);
358	}
359
360	export async function invalidate(
361	filePath: string,
362	unloadModule: boolean = true,
363	): Promise<boolean> {
364	const key = toRel(toAbs(filePath));
365	const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
366	return invalidateEntry(key, file, unloadModule);
367	}
368
369	export async function invalidateEntry(
370	key: string,
371	file: TrackedFile,
372	unloadModule: boolean,
373	): Promise<boolean> {
374	try {
375	if (file.type === "d") {
376	const contents = (file.contents = await fs.readdir(key));
377	contents.sort();
378	const contentHash = crypto
379	.createHash("sha1")
380	.update(contents.join("\0"))
381	.digest("base64url");
382	if (file.contentHash !== contentHash) {
383	file.contentHash = contentHash;
384	throw new Error();
385	}
386	} else if (file.type === "f") {
387	const lastModified = await fs.stat(key).then(
388	(x) => Math.floor(x.mtimeMs),
389	() => 0,
390	);
391	if (file.lastModified !== lastModified) {
392	file.lastModified = lastModified;
393	throw new Error();
394	}
395	} else {
396	file.type satisfies "null";
397	const stat = await fs.stat(key).catch(() => null);
398	if (stat) throw new Error();
399	}
400	return false;
401	} catch (e) {
402	forceInvalidate(file);
403	if (unloadModule) {
404	// TODO: handle when this triggers unloading of `generate.ts`
405	hot.unload(toAbs(key));
406	}
407	if (file.type === "null") files.delete(key);
408	return true;
409	}
410	}
411
412	export function getAssetManifest() {
413	const dynamic = new Map<string, Buffer>();
414	const writer = new BufferWriter();
415	const assetMap = Object.fromEntries(
416	Array.from(works, (work) => work[1].writes.assets)
417	.filter((map) => map.size > 0)
418	.flatMap((map) =>
419	Array.from(map, ([key, { hash, headers, regenerative }]) => {
420	const { raw, gzip, zstd } = UNWRAP(
421	assets.get(hash),
422	`Asset ${key} (${hash})`,
423	);
424	if (regenerative) {
425	const id = crypto
426	.createHash("sha1")
427	.update(key)
428	.digest("hex")
429	.slice(0, 16); /* TODO */
430	dynamic.set(
431	id,
432	manifest.packDynamicBuffer(raw, gzip, zstd, headers),
433	);
434	return [key, { type: 1, id }] as const;
435	}
436	return [
437	key,
438	{
439	type: 0,
440	raw: writer.write(raw, "raw:" + hash),
441	gzip: writer.write(gzip, "gzip:" + hash),
442	zstd: writer.write(zstd, "zstd:" + hash),
443	headers,
444	},
445	] as const;
446	})
447	),
448	) satisfies manifest.Manifest;
449	return {
450	json: assetMap,
451	blob: writer.get(),
452	dynamic,
453	dts: "export type AssetKey = "
454	+ Object.keys(assetMap)
455	.map((key) => JSON.stringify(key))
456	.join(" \| ")
457	+ "\n",
458	};
459	}
460
461	/* Input/Output with automatic tracking.
462	* - Inputs read with Io are tracked to know when to rebuild
463	* - Outputs written with Io are deleted when abandoned.
464	*/
465	export class Io {
466	constructor(public key: string, private node: progress.Ref) {}
467	reads: Reads = { files: new Set(), works: new Set() };
468	writes: Writes = { files: new Set(), assets: new Map() };
469
470	start(text: string, options?: progress.StartOptions) {
471	return this.node.start(text, options);
472	}
473
474	#trackFs(file: string) {
475	const resolved = toAbs(file);
476	const key = toRel(resolved);
477	this.reads.files.add(key);
478	return { resolved, key };
479	}
480	async readWork<T>(ref: Ref<T>): Promise<T> {
481	this.reads.works.add(ref.key);
482	return await ref;
483	}
484	/** Track a file in the compilation without reading it. */
485	async trackFile(file: string) {
486	const { key, resolved } = this.#trackFs(file);
487	if (!files.get(key)) {
488	let lastModified: number = 0;
489	try {
490	lastModified = Math.floor((await fs.stat(file)).mtimeMs);
491	files.set(key, { type: "f", lastModified, affects: [] });
492	} catch {
493	files.set(key, { type: "null", affects: [] });
494	}
495	}
496	return resolved;
497	}
498	async readFile(file: string): Promise<Buffer>;
499	async readFile(file: string, encoding: "utf-8"): Promise<string>;
500	async readFile(file: string, encoding?: "utf-8") {
501	return fs.readFile(await this.trackFile(file), encoding);
502	}
503	async readJson<T>(file: string) {
504	return JSON.parse(await this.readFile(file, "utf-8")) as T;
505	}
506	async readDir(dir: string) {
507	const { key, resolved } = this.#trackFs(dir);
508	const existing = files.get(key);
509	try {
510	if (existing?.type === "d") return existing.contents;
511	const contents = await fs.readdir(resolved);
512	contents.sort();
513	const contentHash = crypto
514	.createHash("sha1")
515	.update(contents.join("\0"))
516	.digest("base64url");
517	files.set(key, {
518	type: "d",
519	affects: [],
520	contentHash,
521	contents,
522	});
523	return contents;
524	} catch (err) {
525	if (!existing) files.set(key, { type: "null", affects: [] });
526	throw err;
527	}
528	}
529	async readDirRecursive(dir: string): Promise<string[]> {
530	const dirs = await this.readDir(dir);
531	return (
532	await Promise.all(
533	dirs.map(async (child) => {
534	const abs = path.join(dir, child);
535	const stat = await fs.stat(abs);
536	if (stat.isDirectory()) {
537	return (await this.readDirRecursive(abs)).map((grand) => path.join(child, grand));
538	} else {
539	return child;
540	}
541	}),
542	)
543	).flat();
544	}
545	/* Track all dependencies of a module. */
546	async import<T>(file: string): Promise<T> {
547	const { resolved } = this.#trackFs(file);
548	try {
549	return require(resolved) as T;
550	} finally {
551	const queue = [resolved];
552	const seen = new Set<string>();
553	let current;
554	while ((current = queue.shift())) {
555	const stat = hot.getFileStat(current);
556	if (!stat) continue;
557	const { key } = this.#trackFs(current);
558	if (!files.get(key)) {
559	files.set(key, {
560	type: "f",
561	affects: [],
562	lastModified: stat?.lastModified ?? 0,
563	});
564	}
565	for (const imp of stat.imports) {
566	if (!seen.has(imp)) {
567	seen.add(imp);
568	queue.push(imp);
569	}
570	}
571	}
572	}
573	}
574	async writeAsset(asset: {
575	pathname: string;
576	buffer: string \| Uint8Array;
577	regenerative?: boolean;
578	headers?: HeadersInit;
579	}) {
580	ASSERT(asset.pathname.startsWith("/"));
581	ASSERT(!seenWrites.has("a:" + asset.pathname));
582	const buffer = typeof asset.buffer === "string"
583	? Buffer.from(asset.buffer)
584	: asset.buffer;
585	const headers = new Headers(asset.headers ?? {});
586	const hash = crypto.createHash("sha1").update(buffer).digest("hex");
587	if (!headers.has("Content-Type")) {
588	headers.set("Content-Type", mime.for(asset.pathname));
589	}
590	headers.set("etag", JSON.stringify(hash));
591	this.writes.assets.set(asset.pathname, {
592	hash,
593	headers: Object.fromEntries(headers),
594	regenerative: !!asset.regenerative,
595	});
596	if (!assets.has(hash)) {
597	jobs += 1;
598	assets.set(hash, undefined!);
599	const [gzipBuffer, zstdBuffer] = await Promise.all([
600	gzip(buffer),
601	zstdCompress(buffer),
602	]);
603	assets.set(hash, {
604	raw: buffer,
605	gzip: gzipBuffer,
606	zstd: zstdBuffer,
607	});
608	jobs -= 1;
609	}
610	}
611	writeFile(subPath: string, blob: string \| Buffer) {
612	ASSERT(!subPath.startsWith("/"));
613	ASSERT(
614	!seenWrites.has("f:" + subPath),
615	`File overwritten: ${JSON.stringify(subPath)}`,
616	);
617	seenWrites.add("f:" + subPath);
618	const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
619	writes.set(subPath, {
620	buffer,
621	size: buffer.byteLength,
622	work: this.key,
623	});
624	}
625	}
626
627	class BufferWriter {
628	size = 0;
629	seen = new Map<string, BufferView>();
630	buffers: Buffer[] = [];
631
632	write(buffer: Buffer, hash: string): BufferView {
633	let view = this.seen.get(hash);
634	if (view) return view;
635	view = [this.size, this.size += buffer.byteLength];
636	this.seen.set(hash, view);
637	this.buffers.push(buffer);
638	return view;
639	}
640
641	get() {
642	return Buffer.concat(this.buffers);
643	}
644	}
645
646	export function validateSerializable(value: unknown, key: string) {
647	if (typeof value === "string") {
648	if (value.includes(hot.projectRoot)) {
649	throw new Error(
650	`Return value must not contain the CWD for portability, found at ${key}`,
651	);
652	}
653	} else if (value && typeof value === "object") {
654	if (Array.isArray(value)) {
655	value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
656	} else if (
657	Object.getPrototypeOf(value) === Object.prototype
658	\|\| Buffer.isBuffer(value)
659	) {
660	Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`));
661	} else {
662	throw new Error(
663	`Return value must be a plain JS object, found ${Object.getPrototypeOf(value).constructor.name} at ${key}`,
664	);
665	}
666	} else if (["bigint", "function", "symbol"].includes(typeof value)) {
667	throw new Error(
668	`Return value must be a plain JS object, found ${typeof value} at ${key}`,
669	);
670	}
671	}
672
673	export function toAbs(absPath: string) {
674	return path.resolve(hot.projectRoot, absPath);
675	}
676
677	export function toRel(absPath: string) {
678	return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
679	}
680
681	export type BufferView = [start: number, end: number];
682	interface Reads {
683	files: Set<string>;
684	works: Set<string>;
685	}
686	interface FileWrite {
687	buffer: Buffer \| null;
688	size: number;
689	work: string;
690	}
691	interface Writes {
692	files: Set<string>;
693	assets: Map<string, AssetWrite>;
694	}
695	interface Asset {
696	raw: Buffer;
697	gzip: Buffer;
698	zstd: Buffer;
699	}
700	interface AssetWrite {
701	hash: string;
702	headers: Record<string, string>;
703	regenerative: boolean;
704	}
705	interface Work<T = unknown> {
706	value: T;
707	reads: Reads;
708	writes: Writes;
709	affects: string[];
710	}
711	type TrackedFile =
712	& { affects: string[] }
713	& (
714	\| { type: "f"; lastModified: number }
715	\| { type: "d"; contentHash: string; contents: string[] }
716	\| { type: "null" }
717	);
718
719	const gzip = util.promisify(zlib.gzip);
720	const zstdCompress = util.promisify(zlib.zstdCompress);
721
722	import * as hot from "./hot.ts";
723
724	import * as manifest from "#sitegen/assets";
725	import * as fs from "#sitegen/fs";
726	import * as mime from "@clo/lib/mime";
727
728	import * as crypto from "node:crypto";
729	import * as path from "node:path";
730	import * as util from "node:util";
731	import * as zlib from "node:zlib";
732
733	import { formatSize } from "#src/file-viewer/format.ts";
734	import * as stack from "@clo/lib/log/stack";
735	import * as progress from "@clo/lib/progress";
736
737	import { ASSERT, UNWRAP } from "@clo/lib/assert";
738	import * as msgpackr from "msgpackr";

framework/lib/path.ts+9-15

 @@ -11,10 +11,9 @@ export class Path {
       }
       return base;
     }
-    return new Path(path.resolve(
+    return new Path(
-      typeof base === "string" ? base : base.raw,
+      path.resolve(typeof base === "string" ? base : base.raw, ...more),
-      ...more,
+    );
-    ));
   }
   static createDirSync(...parts: string[]): Path;
 @@ -150,12 +149,8 @@ export class Path {
   async readIfExists(encoding?: "buffer"): Promise<Buffer | null>;
   async readIfExists<T = unknown>(encoding: "json"): Promise<T | null>;
   async readIfExists(encoding: "utf-8"): Promise<string | null>;
-  async readIfExists(
+  async readIfExists(encoding?: "utf-8" | "buffer" | "json"): Promise<unknown>;
-    encoding?: "utf-8" | "buffer" | "json",
+  async readIfExists(encoding?: "utf-8" | "buffer" | "json"): Promise<unknown> {
-  ): Promise<unknown>;
-  async readIfExists(
-    encoding?: "utf-8" | "buffer" | "json",
-  ): Promise<unknown> {
     try {
       return await this.read(encoding);
     } catch (err) {
 @@ -182,11 +177,10 @@ export class Path {
     return fs.readdirSync(this.raw).map((sub) => this.join(sub));
   }
-  async readDir(
+  async readDir({ recursive }: { recursive?: boolean } = {}): Promise<
-    { recursive }: { recursive?: boolean } = {},
+    Array<Path>
-  ): Promise<Array<Path>> {
+  > {
-    return (await fs.readdir(this.raw, { recursive }))
+    return (await fs.readdir(this.raw, { recursive })).map((sub) => this.join(sub));
-      .map((sub) => this.join(sub));
   }
   async writeJsonFile(json: unknown) {

framework/marko.ts+52-86

...	@@ -1,98 +1,64 @@	...	@@ -1,98 +1,64 @@
1	export interface MarkoCacheEntry {	1	export default function loadMarko({ source, file }: incremental.LoaderInput) {
2	src: string;	2	// transform markodown
3	scannedClientRefs: string[];	3	if (file.endsWith('.mdo')) {
4	}	4	let transform;
5		5	try {
6	export const markoCache = new Map<string, MarkoCacheEntry>();	6	transform = markodown.transform({
7		7	source,
8	export function loadMarko(module: NodeJS.Module, filepath: string) {	8	selfImport: `./${path.basename(file)}`,
9	return loadMarkoExtra(module, filepath, "html");	9	});
10	}	10	} catch (err) {
11	export function loadMarkodown(module: NodeJS.Module, filepath: string) {	11	throw new Error(`Failed ${file}: Markodown Compiler Panic`, {
12	return loadMarkoExtra(module, filepath, "markdown");	12	cause: err,
13	}	13	});
14		14	}
15	function loadMarkoExtra(	15	if (!transform.success) {
16	module: NodeJS.Module,	16	// TODO: better error handling
17	filepath: string,	17	throw new Error(
18	mode: "html" \| "markdown",	18	"Failed " + file + ": " + JSON.stringify(transform.errors),
19	) {
20	let cache = markoCache.get(filepath);
21	if (!cache) {
22	let src = fs.readFileSync(filepath, "utf8");
23	if (mode === "markdown") {
24	let transform;
25	try {
26	transform = markodown.transform({
27	source: src,
28	selfImport: `./${path.basename(filepath)}`,
29	});
30	} catch (err) {
31	throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
32	cause: err,
33	});
34	}
35	if (!transform.success) {
36	// TODO: better error handling
37	throw new Error(
38	"Failed " + filepath + ": " + JSON.stringify(transform.errors),
39	);
40	}
41	src = transform.text;
42
43	fs.writeMkdirSync(
44	".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
45	+ ".marko",
46	src,
47	);	19	);
48	}	20	}
		21	source = transform.text;
		22	}
49		23
50	// A non-standard thing here is Clover Sitegen implements	24	// a non-standard thing here is Clover Sitegen implements
51	// its own client side scripting stuff, so it overrides	25	// its own client side scripting stuff, so it overrides
52	// bare client import statements to it's own usage.	26	// bare client import statements to it's own usage.
53	const scannedClientRefs = new Set<string>();	27	const scannedClientRefs = new Set<string>();
54	if (src.match(/^\s*client\s+import\s+["']/m)) {	28	if (source.match(/^\s*client\s+import\s+["']/m)) {
55	src = src.replace(	29	source = source.replace(
56	/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,	30	/^\s*client\s+import\s+("[^"]+"\|'[^']+')[^\n]+/m,
57	(_, src) => {	31	(_, src) => {
58	const ref = JSON.parse(`"${src.slice(1, -1)}"`);	32	const ref = JSON.parse(`"${src.slice(1, -1)}"`);
59	const resolved = hot.resolveClientRef(filepath, ref);
60	scannedClientRefs.add(resolved);
61	return `<CloverScriptInclude=${
62	JSON.stringify(
63	hot.getScriptId(resolved),
64	)
65	} />`;
66	},
67	) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
68	}
69		33
70	let compiled = marko.compileSync(src, filepath, {	34	// TODO: WTF
71	sourceMaps: true,	35	const resolved = hot.resolveClientRef(file, ref);
72	});	36	scannedClientRefs.add(resolved);
73	// workaround an esbuild bug?	37
74	(compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";	38	return `<CloverScriptInclude=${JSON.stringify(resolved)} />`;
75	compiled.map.sources[0] = filepath;	39	},
76	src = compiled.code	40	) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
77	+ "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
78	+ Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");
79	cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
80	markoCache.set(filepath, cache);
81	fs.writeMkdirSync(
82	".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
83	+ ".js",
84	src,
85	);
86	}	41	}
87		42
88	const { src, scannedClientRefs } = cache;	43	let compiled = marko.compileSync(source, file, {
89	return hot.loadEsbuildCode(module, filepath, src, {	44	sourceMaps: true,
90	scannedClientRefs,
91	});	45	});
		46	// workaround an esbuild bug?
		47	(compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
		48	compiled.map.sources[0] = file;
		49
		50	return {
		51	format: "js",
		52	content: compiled.code
		53	+ "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
		54	+ Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64"),
		55	data: {
		56	scannedClientRefs: Array.from(scannedClientRefs),
		57	},
		58	};
92	}	59	}
93		60
94	import * as fs from "#sitegen/fs";
95	import * as markodown from "@clo/markodown";	61	import * as markodown from "@clo/markodown";
96	import * as marko from "@marko/compiler";	62	import * as marko from "@marko/compiler";
97	import * as path from "node:path";	63	import * as path from "node:path";
98	import * as hot from "./hot.ts";	64	import * as incremental from "@clo/lib/incremental";

lib/async.ts+1-1

...	@@ -152,7 +152,7 @@ export class DedupeConcurrent<Args extends unknown[], Ret> {	...	@@ -152,7 +152,7 @@ export class DedupeConcurrent<Args extends unknown[], Ret> {
152	if (promise) return promise;	152	if (promise) return promise;
153	promise = this.fn(...args);	153	promise = this.fn(...args);
154	this.pending.set(k, promise);	154	this.pending.set(k, promise);
155	void promise.finally(() => void this.pending.delete(k));	155	void promise.catch(() => {}).finally(() => void this.pending.delete(k));
156	return promise;	156	return promise;
157	};	157	};
158	}	158	}

lib/error.ts+2

...	@@ -2,6 +2,8 @@	...	@@ -2,6 +2,8 @@
2	* helper functions for dealing with the `unknown` type, mainly in `catch`	2	* helper functions for dealing with the `unknown` type, mainly in `catch`
3	* blocks or promise rejection callbacks.	3	* blocks or promise rejection callbacks.
4	*	4	*
		5	* TODO: rename from `error` to `exception`, less shadowing
		6	*
5	* @module	7	* @module
6	*/	8	*/
7		9

lib/examples/vm.ts created+17

...	@@ -0,0 +1,17 @@
1	import * as incremental from '../incremental.ts';
2
3	export async function main() {
4	const g = new incremental.Graph({
5	loaders: {
6	'.marko': ({ source, file }) => {
7	const { code } = marko.compileSync(source, file);
8	return { content: code, format: 'js' }
9	}
10	}
11	});
12	const log = await g.import('framework/generate.ts');
13	console.log(log);
14	console.log(g);
15	}
16
17	import * as marko from "@marko/compiler";

lib/incremental.esm.test.ts created+298

...	@@ -0,0 +1,298 @@
1	import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
2	import * as os from "node:os";
3	import * as path from "node:path";
4
5	import { expect, test } from "vitest";
6	import * as inc from "./incremental.ts";
7
8	test("Graph.import loads a typescript module graph", async () => {
9	const fs = new VirtualFs({
10	"/app/package.json": JSON.stringify({ type: "module" }),
11	"/app/main.ts": [
12	"import { value } from './dep.ts';",
13	"export const total: number = value + 1;",
14	"export default total;",
15	].join("\n"),
16	"/app/dep.ts": [
17	"export interface Thing { value: number }",
18	"export const value: number = 41;",
19	].join("\n"),
20	});
21
22	const graph = new inc.Graph({ fs });
23	const mod = await graph.import<{
24	total: number;
25	default: number;
26	}>("/app/main.ts");
27
28	expect(mod.total).toBe(42);
29	expect(mod.default).toBe(42);
30	});
31
32	test("Graph.import uses package imports during linking", async () => {
33	const fs = new VirtualFs({
34	"/app/package.json": JSON.stringify({
35	type: "module",
36	imports: {
37	"#dep": "./dep.ts",
38	},
39	}),
40	"/app/main.ts": [
41	"import { value } from '#dep';",
42	"export const loaded = value;",
43	].join("\n"),
44	"/app/dep.ts": "export const value: number = 7;",
45	});
46
47	const graph = new inc.Graph({ fs });
48	const mod = await graph.import<typeof import("/app/main.ts")>("/app/main.ts");
49
50	expect(mod.loaded).toBe(7);
51	});
52
53	test("Graph.import loads a .cjs entrypoint as a namespace", async () => {
54	await withTempDir(
55	{
56	"main.cjs": "module.exports = { value: 42, extra: true };",
57	},
58	async (dir) => {
59	const graph = new inc.Graph();
60	const mod = await graph.import<{
61	default: { value: number };
62	value: number;
63	}>(path.join(dir, "main.cjs"));
64
65	expect(mod.default).toEqual({ value: 42, extra: true });
66	expect(mod.value).toBe(42);
67	},
68	);
69	});
70
71	test("Graph.import rejects .cts entrypoints", async () => {
72	await withTempDir(
73	{
74	"main.cts": [
75	"const value: number = 7;",
76	"module.exports = { value };",
77	].join("\n"),
78	},
79	async (dir) => {
80	const graph = new inc.Graph();
81	await expect(graph.import(path.join(dir, "main.cts"))).rejects.toThrow(
82	"CommonJS TypeScript (.cts) is not supported",
83	);
84	},
85	);
86	});
87
88	test("Graph.import lets ESM import CommonJS default and named exports", async () => {
89	await withTempDir(
90	{
91	"package.json": JSON.stringify({ type: "module" }),
92	"main.ts": [
93	"import dep, { named } from './dep.cjs';",
94	"export const total = dep.named + named;",
95	"export const direct = named;",
96	].join("\n"),
97	"dep.cjs": "module.exports = { named: 5, other: 1 };",
98	},
99	async (dir) => {
100	const graph = new inc.Graph();
101	const mod = await graph.import<typeof import("/app/main.ts")>(
102	path.join(dir, "main.ts"),
103	);
104
105	expect(mod.total).toBe(10);
106	expect(mod.direct).toBe(5);
107	},
108	);
109	});
110
111	test("Graph.import executes native CommonJS require for json, builtins, and sloppy package resolution", async () => {
112	await withTempDir(
113	{
114	"package.json": JSON.stringify({ type: "commonjs" }),
115	"main.js": [
116	"const dep = require('./dep');",
117	"const dir = require('./dir');",
118	"const data = require('./data.json');",
119	"const pkg = require('pkg-sloppy');",
120	"const { sep } = require('node:path');",
121	"module.exports = { total: dep.value + dir.value + data.value + pkg.value, sep };",
122	].join("\n"),
123	"dep.js": "module.exports = { value: 1 };",
124	"dir/index.js": "module.exports = { value: 2 };",
125	"data.json": JSON.stringify({ value: 3 }),
126	"node_modules/pkg-sloppy/package.json": JSON.stringify({
127	name: "pkg-sloppy",
128	main: "./dist/index",
129	}),
130	"node_modules/pkg-sloppy/dist/index.js": "module.exports = { value: 4 };",
131	},
132	async (dir) => {
133	const graph = new inc.Graph();
134	const mod = await graph.import<{
135	default: { total: number; sep: string };
136	}>(path.join(dir, "main.js"));
137
138	expect(mod.default.total).toBe(10);
139	expect(mod.default.sep).toBeTypeOf("string");
140	},
141	);
142	});
143
144	test("Graph.import treats .ts as CommonJS under package type commonjs", async () => {
145	await withTempDir(
146	{
147	"package.json": JSON.stringify({ type: "commonjs" }),
148	"main.ts": "module.exports = { value: 9 };",
149	},
150	async (dir) => {
151	const graph = new inc.Graph();
152	const mod = await graph.import<{
153	default: { value: number };
154	value: number;
155	}>(path.join(dir, "main.ts"));
156
157	expect(mod.default).toEqual({ value: 9 });
158	expect(mod.value).toBe(9);
159	},
160	);
161	});
162
163	test("Graph.import reuses a shared package.json cache populated by resolve", async () => {
164	await withTempDir(
165	{
166	"package.json": JSON.stringify({
167	type: "commonjs",
168	imports: { "#dep": "./dep.js" },
169	}),
170	"dep.js": "module.exports = 1;",
171	"main.js": "module.exports = { value: require('./dep.js') };",
172	},
173	async (dir) => {
174	const packageJsonPath = path.join(dir, "package.json");
175	const packageJsonCache = new Map<string, inc.PackageMeta>();
176
177	await expect(
178	inc.resolve({
179	specifier: "#dep",
180	resolveDir: dir,
181	packageJsonCache,
182	}),
183	).resolves.toMatchObject({ type: "success" });
184	expect(packageJsonCache.get(packageJsonPath)?.type).toBe("commonjs");
185
186	const graph = new inc.Graph({ packageJsonCache });
187	const mod = await graph.import<{
188	default: { value: number };
189	value: number;
190	}>(path.join(dir, "main.js"));
191
192	expect(mod.default).toEqual({ value: 1 });
193	expect(packageJsonCache.get(packageJsonPath)?.type).toBe("commonjs");
194	},
195	);
196	});
197
198	test("Graph.import supports CommonJS cycles through the native module cache", async () => {
199	await withTempDir(
200	{
201	"a.cjs": [
202	"module.exports = { name: 'a' };",
203	"const b = require('./b.cjs');",
204	"module.exports.fromB = b.name;",
205	].join("\n"),
206	"b.cjs": [
207	"module.exports = { name: 'b' };",
208	"const a = require('./a.cjs');",
209	"module.exports.fromA = a.name;",
210	].join("\n"),
211	},
212	async (dir) => {
213	const graph = new inc.Graph();
214	const a = await graph.import<{
215	default: { name: string; fromB: string };
216	}>(path.join(dir, "a.cjs"));
217	const b = await graph.import<{
218	default: { name: string; fromA: string };
219	}>(path.join(dir, "b.cjs"));
220
221	expect(a.default).toEqual({ name: "a", fromB: "b" });
222	expect(b.default).toEqual({ name: "b", fromA: "a" });
223	},
224	);
225	});
226
227	async function withTempDir<T>(
228	files: Record<string, string>,
229	run: (dir: string) => Promise<T>,
230	): Promise<T> {
231	const dir = await mkdtemp(path.join(os.tmpdir(), "incremental-test-"));
232	try {
233	for (const [relative, content] of Object.entries(files)) {
234	const abs = path.join(dir, relative);
235	await mkdir(path.dirname(abs), { recursive: true });
236	await writeFile(abs, content);
237	}
238	return await run(dir);
239	} finally {
240	await rm(dir, { recursive: true, force: true });
241	}
242	}
243
244	class VirtualFs implements inc.Fs {
245	#files: Map<string, string>;
246	#directories: Set<string>;
247
248	constructor(files: Record<string, string>) {
249	this.#files = new Map(Object.entries(files));
250	this.#directories = new Set(["/"]);
251
252	for (const file of this.#files.keys()) {
253	const parts = file.split("/").filter(Boolean);
254	let current = "";
255	for (let i = 0; i < parts.length - 1; i++) {
256	current += `/${parts[i]}`;
257	this.#directories.add(current);
258	}
259	}
260	}
261
262	async readTextFile(path: string): Promise<string> {
263	const contents = this.#files.get(path);
264	if (contents !== undefined) return contents;
265	const error = new Error(`ENOENT: no such file, open '${path}'`);
266	// @ts-expect-error node-style error code
267	error.code = "ENOENT";
268	throw error;
269	}
270
271	async readDir(path: string): Promise<string[]> {
272	if (this.#directories.has(path)) return [];
273	const error = new Error(`ENOENT: no such directory, scandir '${path}'`);
274	// @ts-expect-error node-style error code
275	error.code = "ENOENT";
276	throw error;
277	}
278
279	async realPath(path: string): Promise<string> {
280	return path;
281	}
282
283	async stat(path: string): Promise<inc.Stat> {
284	const contents = this.#files.get(path);
285	if (contents !== undefined) {
286	return { kind: "file", size: contents.length, time: 0 };
287	}
288	if (this.#directories.has(path)) {
289	return { kind: "directory", size: 0, time: 0 };
290	}
291	const error = new Error(
292	`ENOENT: no such file or directory, stat '${path}'`,
293	);
294	// @ts-expect-error node-style error code
295	error.code = "ENOENT";
296	throw error;
297	}
298	}

lib/incremental.resolution.test.ts created+435

...	@@ -0,0 +1,435 @@
1	import { expect, test } from "vitest";
2	import * as inc from "./incremental.ts";
3
4	test("finds node.js builtin module", async () => {
5	const fs = new VirtualFs({});
6
7	await expect(inc.resolve({ specifier: "node:fs", resolveDir: "/", fs })).resolves.toEqual({
8	type: "success",
9	namespace: "node",
10	subPath: "fs",
11	files: [],
12	});
13	await expect(inc.resolve({ specifier: "fs", resolveDir: "/", fs })).resolves.toEqual({
14	type: "success",
15	namespace: "node",
16	subPath: "fs",
17	files: [],
18	});
19	await expect(inc.resolve({ specifier: "util/types", resolveDir: "/", fs })).resolves.toEqual({
20	type: "success",
21	namespace: "node",
22	subPath: "util/types",
23	files: [],
24	});
25	await expect(inc.resolve({ specifier: "node:test", resolveDir: "/", fs })).resolves.toEqual({
26	type: "success",
27	namespace: "node",
28	subPath: "test",
29	files: [],
30	});
31	await expect(inc.resolve({ specifier: "test", resolveDir: "/", fs })).resolves.toEqual({
32	type: "error",
33	error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
34	files: [
35	"/node_modules/test/package.json",
36	],
37	});
38	});
39
40	test("relative file path imports", async () => {
41	const fs = new VirtualFs({
42	"/src/a.js": "",
43	"/src/b.ts": "",
44	"/src/shared/util.js": "",
45	"/src/feature/main.js": "",
46	"/src/node_modules/a/index.js": "",
47	});
48
49	await expect(inc.resolve({ specifier: "./a.js", resolveDir: "/src", fs })).resolves.toEqual({
50	type: "success",
51	namespace: "file",
52	subPath: "/src/a.js",
53	files: ["/src/a.js"],
54	});
55
56	await expect(inc.resolve({ specifier: "./c.ts", resolveDir: "/src", fs })).resolves.toEqual({
57	type: "error",
58	error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
59	files: ["/src/c.ts"],
60	});
61
62	await expect(
63	inc.resolve({ specifier: "../shared/util.js", resolveDir: "/src/feature", fs }),
64	).resolves.toEqual({
65	type: "success",
66	namespace: "file",
67	subPath: "/src/shared/util.js",
68	files: ["/src/shared/util.js"],
69	});
70
71	await expect(inc.resolve({ specifier: "./a", resolveDir: "/src", fs })).resolves.toEqual({
72	type: "error",
73	error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
74	files: ["/src/a"],
75	});
76	await expect(inc.resolve({ specifier: "./node_modules/a", resolveDir: "/src", fs })).resolves.toEqual({
77	type: "error",
78	error: expect.objectContaining({ code: "ERR_UNSUPPORTED_DIR_IMPORT" }),
79	files: ["/src/node_modules/a"],
80	});
81	await expect(inc.resolve({ specifier: "./node_modules/a/index", resolveDir: "/src", fs })).resolves.toEqual({
82	type: "error",
83	error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
84	files: ["/src/node_modules/a/index"],
85	});
86	await expect(inc.resolve({ specifier: "./node_modules/a/index.js", resolveDir: "/src", fs })).resolves.toEqual({
87	type: "success",
88	namespace: "file",
89	subPath: "/src/node_modules/a/index.js",
90	files: ["/src/node_modules/a/index.js"],
91	});
92	});
93
94	test("sloppy resolution", async () => {
95	const fs = new VirtualFs({
96	"/src/a.js": "",
97	"/src/dir/index.js": "",
98	});
99
100	await expect(inc.resolve({
101	specifier: "./a",
102	resolveDir: "/src",
103	fs,
104	sloppy: true,
105	})).resolves.toEqual({
106	type: "success",
107	namespace: "file",
108	subPath: "/src/a.js",
109	files: ["/src/a", "/src/a.js"],
110	});
111
112	await expect(inc.resolve({
113	specifier: "./dir",
114	resolveDir: "/src",
115	fs,
116	sloppy: true,
117	})).resolves.toEqual({
118	type: "success",
119	namespace: "file",
120	subPath: "/src/dir/index.js",
121	files: [
122	"/src/dir",
123	"/src/dir.js",
124	"/src/dir.json",
125	"/src/dir.node",
126	"/src/dir/package.json",
127	"/src/dir/index.js",
128	],
129	});
130
131	await expect(
132	inc.resolve({ specifier: "./dir", resolveDir: "/src", fs, sloppy: false }),
133	).resolves.toEqual({
134	type: "error",
135	error: expect.objectContaining({ code: "ERR_UNSUPPORTED_DIR_IMPORT" }),
136	files: ["/src/dir"],
137	});
138	});
139
140	test("package resolution handles exports and imports maps", async () => {
141	const fs = new VirtualFs({
142	"/app/src/main.js": "",
143	"/app/package.json": JSON.stringify({
144	imports: {
145	"#src/": "./src/.js",
146	},
147	}),
148	"/app/src/feature.js": "export const feature = true;",
149	"/app/node_modules/pkg/package.json": JSON.stringify({
150	name: "pkg",
151	exports: "./dist/index.js",
152	}),
153	"/app/node_modules/pkg/dist/index.js": "export const value = 1;",
154	"/app/node_modules/pkg/dist/feature.js": "export const feature = true;",
155	"/app/node_modules/pkg-subpath/package.json": JSON.stringify({
156	name: "pkg-subpath",
157	exports: {
158	"./feature": "./dist/feature.js",
159	},
160	}),
161	"/app/node_modules/pkg-subpath/dist/feature.js": "export const feature = true;",
162	});
163
164	await expect(inc.resolve({ specifier: "pkg", resolveDir: "/app/src", fs })).resolves.toEqual({
165	type: "success",
166	namespace: "file",
167	subPath: "/app/node_modules/pkg/dist/index.js",
168	files: [
169	"/app/src/node_modules/pkg/package.json",
170	"/app/node_modules/pkg/package.json",
171	"/app/node_modules/pkg/dist/index.js",
172	],
173	});
174
175	await expect(inc.resolve({ specifier: "pkg-subpath/feature", resolveDir: "/app/src", fs })).resolves
176	.toEqual({
177	type: "success",
178	namespace: "file",
179	subPath: "/app/node_modules/pkg-subpath/dist/feature.js",
180	files: [
181	"/app/src/node_modules/pkg-subpath/package.json",
182	"/app/node_modules/pkg-subpath/package.json",
183	"/app/node_modules/pkg-subpath/dist/feature.js",
184	],
185	});
186
187	await expect(inc.resolve({
188	specifier: "#src/feature",
189	resolveDir: "/app/src",
190	fs,
191	})).resolves.toEqual({
192	type: "success",
193	namespace: "file",
194	subPath: "/app/src/feature.js",
195	files: [
196	"/app/src/package.json",
197	"/app/package.json",
198	"/app/src/feature.js",
199	],
200	});
201	});
202
203	test("package resolution reports export and import errors and supports sloppy main fallback", async () => {
204	const fs = new VirtualFs({
205	"/app/package.json": JSON.stringify({
206	imports: {
207	"#src/": "./src/.js",
208	},
209	}),
210	"/app/node_modules/pkg/package.json": JSON.stringify({
211	name: "pkg",
212	exports: {
213	"./feature": "./dist/feature.js",
214	},
215	}),
216	"/app/node_modules/pkg-sloppy/package.json": JSON.stringify({
217	name: "pkg-sloppy",
218	main: "./dist/index",
219	}),
220	"/app/node_modules/pkg-sloppy/dist/index.js": "export const value = 1;",
221	"/app/node_modules/pkg-legacy-main/package.json": JSON.stringify({
222	name: "pkg-legacy-main",
223	main: "lib/main.js",
224	}),
225	"/app/node_modules/pkg-legacy-main/lib/main.js": "module.exports = {};",
226	"/app/node_modules/pkg-legacy-module/package.json": JSON.stringify({
227	name: "pkg-legacy-module",
228	module: "lib/module.js",
229	}),
230	"/app/node_modules/pkg-legacy-module/lib/module.js": "module.exports = {};",
231	});
232
233	await expect(inc.resolve({ specifier: "pkg/missing", resolveDir: "/app/src", fs })).resolves
234	.toEqual({
235	type: "error",
236	error: expect.objectContaining({ code: "ERR_PACKAGE_PATH_NOT_EXPORTED" }),
237	files: ["/app/src/node_modules/pkg/package.json", "/app/node_modules/pkg/package.json"],
238	});
239
240	await expect(inc.resolve({ specifier: "#missing", resolveDir: "/app/src", fs })).resolves
241	.toEqual({
242	type: "error",
243	error: expect.objectContaining({ code: "ERR_PACKAGE_IMPORT_NOT_DEFINED" }),
244	files: ["/app/src/package.json", "/app/package.json"],
245	});
246
247	await expect(inc.resolve({
248	specifier: "pkg-sloppy",
249	resolveDir: "/app/src",
250	fs,
251	sloppy: true,
252	})).resolves.toEqual({
253	type: "success",
254	namespace: "file",
255	subPath: "/app/node_modules/pkg-sloppy/dist/index.js",
256	files: [
257	"/app/src/node_modules/pkg-sloppy/package.json",
258	"/app/node_modules/pkg-sloppy/package.json",
259	"/app/node_modules/pkg-sloppy/dist/index",
260	"/app/node_modules/pkg-sloppy/dist/index.js",
261	],
262	});
263
264	await expect(inc.resolve({
265	specifier: "pkg-legacy-main",
266	resolveDir: "/app/src",
267	fs,
268	sloppy: true,
269	})).resolves.toEqual({
270	type: "success",
271	namespace: "file",
272	subPath: "/app/node_modules/pkg-legacy-main/lib/main.js",
273	files: [
274	"/app/src/node_modules/pkg-legacy-main/package.json",
275	"/app/node_modules/pkg-legacy-main/package.json",
276	"/app/node_modules/pkg-legacy-main/lib/main.js",
277	],
278	});
279
280	await expect(inc.resolve({
281	specifier: "pkg-legacy-module",
282	resolveDir: "/app/src",
283	fs,
284	sloppy: true,
285	})).resolves.toEqual({
286	type: "success",
287	namespace: "file",
288	subPath: "/app/node_modules/pkg-legacy-module/lib/module.js",
289	files: [
290	"/app/src/node_modules/pkg-legacy-module/package.json",
291	"/app/node_modules/pkg-legacy-module/package.json",
292	"/app/node_modules/pkg-legacy-module/lib/module.js",
293	],
294	});
295	});
296
297	test("resolve reuses package.json cache for repeated imports lookups", async () => {
298	const fs = new VirtualFs({
299	"/app/package.json": JSON.stringify({
300	imports: {
301	"#dep": "./src/dep.js",
302	},
303	}),
304	"/app/src/dep.js": "export const dep = true;",
305	});
306	const packageJsonCache = new Map<string, inc.PackageMeta>();
307
308	await expect(inc.resolve({
309	specifier: "#dep",
310	resolveDir: "/app",
311	fs,
312	packageJsonCache,
313	})).resolves.toMatchObject({
314	type: "success",
315	subPath: "/app/src/dep.js",
316	});
317	expect(fs.readCount("/app/package.json")).toBe(1);
318
319	await expect(inc.resolve({
320	specifier: "#dep",
321	resolveDir: "/app",
322	fs,
323	packageJsonCache,
324	})).resolves.toMatchObject({
325	type: "success",
326	subPath: "/app/src/dep.js",
327	files: ["/app/package.json", "/app/src/dep.js"],
328	});
329	expect(fs.readCount("/app/package.json")).toBe(1);
330	});
331
332	test("resolve reuses package.json cache for repeated exports lookups", async () => {
333	const fs = new VirtualFs({
334	"/app/src/main.js": "",
335	"/app/node_modules/pkg/package.json": JSON.stringify({
336	name: "pkg",
337	exports: {
338	".": "./dist/index.js",
339	},
340	}),
341	"/app/node_modules/pkg/dist/index.js": "export const value = true;",
342	});
343	const packageJsonCache = new Map<string, inc.PackageMeta>();
344
345	await expect(inc.resolve({
346	specifier: "pkg",
347	resolveDir: "/app/src",
348	fs,
349	packageJsonCache,
350	})).resolves.toMatchObject({
351	type: "success",
352	subPath: "/app/node_modules/pkg/dist/index.js",
353	});
354	expect(fs.readCount("/app/node_modules/pkg/package.json")).toBe(1);
355
356	await expect(inc.resolve({
357	specifier: "pkg",
358	resolveDir: "/app/src",
359	fs,
360	packageJsonCache,
361	})).resolves.toMatchObject({
362	type: "success",
363	subPath: "/app/node_modules/pkg/dist/index.js",
364	files: [
365	"/app/src/node_modules/pkg/package.json",
366	"/app/node_modules/pkg/package.json",
367	"/app/node_modules/pkg/dist/index.js",
368	],
369	});
370	expect(fs.readCount("/app/node_modules/pkg/package.json")).toBe(1);
371	});
372
373	class VirtualFs implements inc.Fs {
374	#files: Map<string, string>;
375	#directories: Set<string>;
376	#readCounts: Map<string, number>;
377
378	constructor(files: Record<string, string>) {
379	this.#files = new Map(Object.entries(files));
380	this.#directories = new Set(["/"]);
381	this.#readCounts = new Map();
382
383	for (const file of this.#files.keys()) {
384	const parts = file.split("/").filter(Boolean);
385	let current = "";
386	for (let i = 0; i < parts.length - 1; i++) {
387	current += `/${parts[i]}`;
388	this.#directories.add(current);
389	}
390	}
391	}
392
393	async readTextFile(path: string): Promise<string> {
394	this.#readCounts.set(path, (this.#readCounts.get(path) ?? 0) + 1);
395	const contents = this.#files.get(path);
396	if (contents !== undefined) return contents;
397	const error = new Error(`ENOENT: no such file, open '${path}'`);
398	// @ts-expect-error node-style error code
399	error.code = "ENOENT";
400	throw error;
401	}
402
403	async readDir(path: string): Promise<string[]> {
404	if (!this.#directories.has(path)) {
405	const error = new Error(`ENOENT: no such directory, scandir '${path}'`);
406	// @ts-expect-error node-style error code
407	error.code = "ENOENT";
408	throw error;
409	}
410
411	return [];
412	}
413
414	async stat(path: string): Promise<inc.Stat> {
415	const contents = this.#files.get(path);
416	if (contents !== undefined) {
417	return { kind: "file", size: contents.length, time: 0 };
418	}
419	if (this.#directories.has(path)) {
420	return { kind: "directory", size: 0, time: 0 };
421	}
422	const error = new Error(`ENOENT: no such file or directory, stat '${path}'`);
423	// @ts-expect-error node-style error code
424	error.code = "ENOENT";
425	throw error;
426	}
427
428	async realPath(path: string): Promise<string> {
429	return path;
430	}
431
432	readCount(path: string): number {
433	return this.#readCounts.get(path) ?? 0;
434	}
435	}

lib/incremental.serialization.test.ts created+49

...	@@ -0,0 +1,49 @@
1	import { expect, test } from "vitest";
2	import * as inc from "./incremental.ts";
3
4	test("readBlob returns the cached entry when available", () => {
5	const db = new inc.Database(null);
6	const value = {
7	message: "hello",
8	bytes: new Uint8Array([1, 2, 3]),
9	when: new Date("2024-01-02T03:04:05.000Z"),
10	} satisfies inc.Serializable;
11
12	const ref = db.writeBlob(value);
13	const roundTrip = db.readBlob(ref.blob);
14
15	expect(roundTrip).toBe(value);
16	});
17
18	test("readBlob falls back to the database and revives serialized values", () => {
19	const db = new inc.Database(null);
20	const value = {
21	text: "hello",
22	number: Infinity,
23	missing: undefined,
24	when: new Date("2024-01-02T03:04:05.000Z"),
25	set: new Set<inc.Serializable>([1, "two"]),
26	map: new Map<inc.Serializable, inc.Serializable>([
27	["three", 3],
28	[4, "four"],
29	]),
30	bytes: new Uint8Array([4, 5, 6]),
31	nested: {
32	$clover: "escaped",
33	blob: new Uint8Array([7, 8, 9]),
34	},
35	} satisfies inc.Serializable;
36
37	const ref = db.writeBlob(value, false);
38	db.clearInMemoryCache();
39
40	const roundTrip = db.readBlob(ref.blob) as typeof value;
41
42	expect(roundTrip).toEqual(value);
43	expect(roundTrip).not.toBe(value);
44	expect(roundTrip.when).toBeInstanceOf(Date);
45	expect(roundTrip.set).toBeInstanceOf(Set);
46	expect(roundTrip.map).toBeInstanceOf(Map);
47	expect(roundTrip.bytes).toBeInstanceOf(Uint8Array);
48	expect(roundTrip.nested.blob).toBeInstanceOf(Uint8Array);
49	});

lib/incremental.sources.test.ts created+125

...	@@ -0,0 +1,125 @@
1	import { expect, test } from "vitest";
2	import * as inc from "./incremental.ts";
3
4	test("Database stores cache entries with complete URL sources", () => {
5	const db = new inc.Database(null);
6	const file = {
7	url: "file:///site/src/page.ts",
8	kind: "file",
9	time: 100,
10	size: 12,
11	hash: "sha1-page-v1",
12	} satisfies inc.Input;
13	const query = {
14	url: "sql://content/posts/by-slug?slug=hello",
15	kind: "query",
16	time: 7,
17	size: 1,
18	hash: "revision-7",
19	} satisfies inc.Input;
20
21	db.writeCache("page:hello", { html: "<h1>Hello</h1>" }, [file, query]);
22	const cache = db.readCache("page:hello")!;
23
24	expect(db.readBlob(cache.blob)).toEqual({ html: "<h1>Hello</h1>" });
25	expect(collect(cache.inputs)).toEqual([
26	{ id: expect.anything(), ...file },
27	{ id: expect.anything(), ...query },
28	]);
29	});
30
31	test("Database invalidates caches that depend on changed sources", () => {
32	const db = new inc.Database(null);
33	const page = {
34	url: "file:///site/src/page.ts",
35	kind: "file",
36	time: 100,
37	size: 12,
38	hash: "sha1-page-v1",
39	} satisfies inc.Input;
40	const layout = {
41	url: "file:///site/src/layout.ts",
42	kind: "file",
43	time: 200,
44	size: 34,
45	hash: "sha1-layout-v1",
46	} satisfies inc.Input;
47
48	db.writeCache("page", "page result", [page]);
49	db.writeCache("layout", "layout result", [layout]);
50
51	const storedPage = collect(db.sources()).find(
52	(source) => source.url === page.url,
53	)!;
54	db.invalidate(storedPage.id);
55	db.writeSource({ ...page, time: 101, size: 13, hash: "sha1-page-v2" });
56
57	expect(db.readCache("page")).toBeNull();
58	expect(db.readBlob(db.readCache("layout")!.blob)).toBe("layout result");
59	});
60
61	test("Database treats source kind changes as invalidation", () => {
62	const db = new inc.Database(null);
63	const missing = {
64	url: "file:///site/src/optional.ts",
65	kind: "unknown",
66	time: 0,
67	size: 0,
68	hash: "",
69	} satisfies inc.Input;
70
71	db.writeCache("resolution:optional", "missing", [missing]);
72
73	const [source] = collect(db.sources());
74	db.invalidate(source!.id);
75	db.writeSource({
76	...missing,
77	kind: "file",
78	time: 300,
79	size: 44,
80	hash: "sha1-optional",
81	});
82
83	expect(db.readCache("resolution:optional")).toBeNull();
84	});
85
86	test("Database lets callers skip invalidation when source state is unchanged", () => {
87	const db = new inc.Database(null);
88	const source = {
89	url: "https://example.com/feed.xml",
90	kind: "response",
91	time: 500,
92	size: 2048,
93	hash: "etag-feed-v1",
94	} satisfies inc.Input;
95
96	db.writeCache("feed", new Uint8Array([1, 2, 3]), [source]);
97
98	const [stored] = collect(db.sources());
99	expect(stored).toMatchObject(source);
100	expect(db.readBlob(db.readCache("feed")!.blob)).toEqual(
101	new Uint8Array([1, 2, 3]),
102	);
103	});
104
105	test("Database requires source identities to be complete URLs", () => {
106	const db = new inc.Database(null);
107
108	expect(() =>
109	db.writeSource({
110	url: "file:./relative.ts",
111	kind: "file",
112	time: 1,
113	size: 1,
114	hash: "sha1-relative",
115	}),
116	).toThrow("source url must be canonical");
117	});
118
119	function collect<T>(iterator: Iterator<T>): T[] {
120	const out: T[] = [];
121	for (let next = iterator.next(); !next.done; next = iterator.next()) {
122	out.push(next.value);
123	}
124	return out;
125	}

lib/incremental.ts created+1890

...	@@ -0,0 +1,1890 @@
1	/**
2	* Clover's `incremental` module provides a way to load and reload modules in
3	* Node.js, as well as a foundation to build incremental developer tools.
4	* Incremental build systems stem from the {@linkcode Ref#step\|step} function,
5	* which runs a closure given a IO object that allows tracing
6	* {@linkcode Input\|inputs} ({@link File\|file reads}, network fetches, database
7	* calls, and more). {@linkcode Step}s cache their output data and re-run when
8	* changed. By archetecting the build system steps, you can balance 'size of
9	* the cache' vs 'how fine grained is tracing'.
10	*
11	* The entire system is built around the idea of being able to flush incremental
12	* state to the filesystem, and revive cached work on a second run, so many
13	* places require {@link Serializable\|serializable} data. upon revival, the
14	* filesystem (and other input sources) are re-queried for changes. Because of
15	* this design, reading inputs is not allowed outside of steps.
16	*
17	* A central part of building incremental systems is being able to trace source
18	* code itself, so Clover Incremental includes a full set of tools for
19	*
20	* `--experimental-vm-modules` support is required as it is the only way to
21	* interact with an ES Module Loader with the ability to garbage collect modules.
22	*
23	* TODO, explain these visions more in details:
24	* - simple module loading `new incremental.Graph().import("src/main.ts");`
25	* - plugin system to load extensions like `.tsx`, `.marko`, `.svelte`
26	* - tracing the files used in resolution and initialization
27	* - allowing arbitrary dependencies to be added
28	* - allowing arbitrary jobs that cache when re-run
29	* - easy base to implement web frameworks, testing tools, linters, etc
30	* - creating a filesystem watcher to detect changes
31	* - performing trivial module reloading (perhaps... full HMR)
32	* - serializing the graph state to disk, along with jobs and v8 caches
33	*
34	* `@clo/lib/incremental` is implemented as a single file depending only on the
35	* node.js builtins and other `@clo/lib` libraries. i intend for this codebase
36	* to be a good introduction to how an incremental system can function.
37	*
38	* @module
39	*/
40
41	// TODO (this list is NOT in order)
42	// - [X] ES and CJS module loading
43	// - [ ] perfectly imports into the database
44	// - [ ] save and restore v8 cache
45	// - [ ] represent all loaders as virtual "steps"
46	// - [ ] partial hot graph replacement
47	// - [ ] steps
48	// - [ ] re-validating file inputs
49	// - [ ] re-validating network inputs
50	// - [ ] custom source types
51	// - [ ] depend on the results of other steps `readStep`
52	// - [ ] shaking old steps away
53	// - [ ] build output
54	// - [ ] write file (without storing the file's contents entirely in-database)
55	// - [ ] copy file
56	// - [ ] module tracing API (example: "if i imported X, list all the .css files")
57	// - [X] serialization API
58	// - [ ] filesystem watcher
59	// - [ ] ability to watch for other source types
60	// - [ ] trace every file ever + other insane details like runtime info
61
62	/**
63	* serialization special cases top-level `Uint8Array`s and `string`s to use more
64	* effective data storage strategies. other variants use JSON + a reviver.
65	*/
66	export type Serializable =
67	\| string
68	\| number
69	\| boolean
70	\| null
71	\| undefined
72	\| Uint8Array
73	\| Date
74	\| Set<Serializable>
75	\| Map<Serializable, Serializable>
76	\| Serializable[]
77	\| { [k: string]: Serializable };
78
79	export class Ref implements progress.Ref {
80	/** owning graph */
81	#graph: Graph;
82	/** parent progress ref */
83	#ref: progress.Ref;
84	/** promises for the build */
85	#build: PendingBuild;
86
87	constructor(options: {
88	graph: Graph;
89	progress: progress.Ref;
90	/** build status */
91	build: PendingBuild;
92	}) {
93	this.#graph = options.graph;
94	this.#build = options.build;
95	this.#ref = options.progress;
96	}
97
98	/**
99	* creates a new trackable unit of work as a child of this one.
100	* when given an estimate, a progress bar is rendered.
101	*/
102	start = (text: string, opts?: progress.StartOptions): ProgressNode => {
103	const node = this.#ref.start(text, opts);
104	return ts.mixin(
105	node,
106	new Ref({
107	graph: this.#graph,
108	progress: node,
109	build: this.#build,
110	}),
111	);
112	};
113
114	/**
115	* run a deterministic "step function" and cache the result. the cache entry
116	* is busted if the given key or any referenced {@linkcode Input}s are
117	* changed. inside the step, the {@linkcode Run} provides access to the
118	* filesystem, network, and a base for other input sources to use.
119	*
120	* on a cache hit, all {@linkcode Input}s are re-queried for changes before
121	* the step re-runs. in the case some inputs changed, the ones that stayed the
122	* same can be re-used from this querying pass.
123	*
124	* implicitly, the caller file is added as an `Input` so the implementation of
125	* the builder is tracked incrementally. the caller of `new incremental.Graph`
126	* is also given this behavior.
127	*
128	* NOTE: the provided callback MAY capture arbitrary values that are not a part
129	* of `key`. this is by design to bypass the cache key when needed. the step's
130	* logic MUST be deterministic, and future debugging tools reserve the right to
131	* arbitrarily re-call the step function multiple times per build, or even
132	* after a build has completed.
133	*/
134	step<Key extends Serializable, Result extends Serializable>(
135	key: Key,
136	fn: StepFn<Key, Result>,
137	): Step<Result> {
138	ASSERT(!this.#build.closed, "cannot start a step after the build has closed");
139	const callerFile = UNWRAP(stack.capture()[0], "runtime bug").file;
140	const keyHash = hashSerializable(key, callerFile);
141	ASSERT(
142	!this.#build.steps.has(keyHash),
143	() => `duplicate step key. caller may not be passing enough information to it's "key"`,
144	);
145	this.#build.steps.add(keyHash);
146
147	// TODO: validate cache inputs before accepting an existing entry.
148	const existing = this.#graph.db.readCache(keyHash);
149	if (existing) {
150	const value = this.#graph.db.readBlob(existing.blob) as Result;
151	return {
152	id: existing.id,
153	value,
154	then: (onfufilled) => Promise.resolve(value).then(onfufilled),
155	hit: true,
156	};
157	}
158
159	const run = new Run({
160	key,
161	build: this.#build,
162	progress: this.#ref,
163	});
164	const defer = Promise.withResolvers<Result>();
165	defer.promise.catch(() => {}); // handled by `step.then` or `incrementalBuild`
166
167	let rejectionObserved = false;
168	const step: Step<Result> = {
169	id: null,
170	value: null,
171	then: (onfufilled, onrejected) => {
172	if (typeof onrejected === "function") rejectionObserved = true;
173	return defer.promise.then(onfufilled, onrejected);
174	},
175	hit: false,
176	};
177
178	void (async () => {
179	this.#build.activeSteps.add(defer.promise);
180	try {
181	const result = await fn(run);
182	const inputs = await run.finish();
183	step.id = this.#graph.db.writeCache(keyHash, result, inputs);
184	step.value = result;
185	defer.resolve(result);
186	} catch (error) {
187	run.close();
188	this.#build.failures.push(() => rejectionObserved ? { observed: true } : { observed: false, error });
189	defer.reject(error);
190	} finally {
191	this.#build.activeSteps.delete(defer.promise);
192	}
193	})();
194
195	return step;
196	}
197
198	// TODO: function to write a file
199	// TODO: function to copy a file
200	}
201
202	export type StepFn<Key extends Serializable, Result extends Serializable> = (
203	t: Run<Key>,
204	) => Result \| Promise<Result>;
205
206	export type ProgressNode = progress.Node & Ref;
207
208	export interface Step<T> extends PromiseLike<T> {
209	/** @internal */
210	// unset until resolved.
211	id: RowId \| null;
212	/** returns `null` if the value is not yet done being computed */
213	value: T \| null;
214	/** true if this was a cache hit */
215	hit: boolean;
216	}
217
218	/** state shared between different cache boundaries throughout a single `incrementalBuild` call */
219	interface PendingBuild {
220	/** key is a URL. values may be waiting on an expensive freshness token. */
221	inputs: Map<string, Promise<Input>>;
222	steps: Set<string>;
223	activeSteps: Set<Promise<unknown>>;
224	/** failures from steps that were started but not awaited by user code. */
225	failures: Array<() => StepFailureStatus>;
226	// TODO: invert this
227	closed: boolean;
228	// TODO: output? want to store a list of written + copied files. tho some more details to this tbh.
229	}
230
231	async function waitForBuildSteps(build: PendingBuild): Promise<void> {
232	while (build.activeSteps.size > 0) {
233	await Promise.all(build.activeSteps);
234	}
235	}
236
237	// TODO: this is cooked
238	type StepFailureStatus =
239	\| { observed: true }
240	\| { observed: false; error: unknown };
241
242	/**
243	* blob-like object. the wrapper allows `@clo/lib/incremental` to hash the file
244	* in combination to the caller's usage, ensuring there is at most one full read.
245	*/
246	export class File {
247	path: string;
248	#handle: unknown;
249
250	/** file modification time */
251	time: number;
252	/** size in bytes */
253	size: number;
254	#fs: Fs;
255	#hash: string \| null = null;
256
257	constructor(fs: Fs, path: string, stat: Stat) {
258	ASSERT(stat.kind === "file");
259	this.path = path;
260	this.time = stat.time;
261	this.size = stat.size;
262	this.#fs = fs;
263	}
264
265	async stream(options?: { chunkSize: number }): Promise<ReadableStream> {
266	const chunkSize = options?.chunkSize ?? 65536;
267	// return a pullable readable stream, hashing every chunk as it is read
268	}
269	async arrayBuffer(): Promise<ArrayBuffer> {
270	// allocate the backing buffer, then incrementally read into it. hash the
271	// entire buffer at the end.
272	}
273	async text(): Promise<string> {
274	const decoder = new TextDecoder("utf-8");
275	const buffer = new Uint8Array(Math.min(this.size, 65536));
276	// take turns filling the buffer, hashing and decoding it as you go
277	let str = "";
278	// str += decoder.decode(buffer, { stream: true })
279	}
280
281	async hash(): Promise<string> {
282	// retrieve the hash once its done
283	}
284
285	async bytes(): Promise<Uint8Array> {
286	return new Uint8Array(await this.arrayBuffer());
287	}
288	}
289
290	/**
291	* state of a running {@linkcode StepFn}, contains a {@linkcode progress.Ref}
292	* and methods to track {@linkcode Input}s.
293	*/
294	// TODO: still not happy with the naming of this structure
295	export class Run<Key extends Serializable = Serializable> implements progress.Ref {
296	#build: PendingBuild;
297	#inputs: Promise<Input>[] = [];
298	#closed = false;
299
300	constructor(options: {
301	key: Key;
302	progress: progress.Ref;
303	build: PendingBuild;
304	}) {
305	this.key = options.key;
306	this.start = options.progress.start;
307	this.#build = options.build;
308	}
309
310	start: progress.Ref["start"];
311	/** as-is from the key argument in `.cachedWork()` */
312	key: Key;
313	/** appends an {@linkcode Input} to the work's inputs list */
314	track(input: Input \| Promise<Input>) {
315	ASSERT(!this.#closed, "cannot track an input after the step has finished");
316	ASSERT(!this.#build.closed, "cannot track an input after the build has closed");
317	this.#inputs.push(Promise.resolve(input));
318	}
319
320	/** @internal stop tracking and return the inputs recorded by this step. */
321	async finish(): Promise<Input[]> {
322	this.close();
323	return await Promise.all(this.#inputs);
324	}
325
326	/** @internal stop accepting new inputs. */
327	close(): void {
328	this.#closed = true;
329	}
330
331	/** appends all of the inputs from `step`, unwrapping it's promise */
332	async readStep<T>(step: Step<T>): Promise<T> {
333	void step;
334	throw new Error("Run.readStep is not implemented yet");
335	}
336	/** opens a file for reading */
337	async readFile(path: string): Promise<File> {
338	void path;
339	throw new Error("Run.readFile is not implemented yet");
340	}
341	// /** lists all of the children of a directory */
342	// async readDir(path: string): Promise<string[]> {}
343	// /** throws on HTTP or network I/O failure */
344	// fetchFile(input: string \| URL \| Request, init?: RequestInit): Promise<File>;
345	// /** throws only on network I/O failure */
346	// fetchResponse(
347	// input: string \| URL \| Request,
348	// init?: RequestInit,
349	// options?: { cache?: "all" \| "body" },
350	// ): Promise<Response>;
351	// /** load a module, with a "trust me bro" type parameter */
352	// import<T = unknown>(absPath: string): Promise<T> { }
353	// /** follow all the imports of a module without actually executing it. throws on syntax and resolution errors. */
354	// trackModule(absPath: string): Promise<void>;
355	}
356
357	export interface Fs {
358	/** read a whole UTF-8 text file */
359	readTextFile(path: string): Promise<string>;
360	readDir(path: string): Promise<string[]>;
361	realPath(abs: string): Promise<string>;
362	stat(path: string): Promise<Stat>;
363
364	open(path: string): Promise<FileHandle>;
365	statHandle(handle: FileHandle): Promise<Stat>;
366	readHandle(
367	handle: FileHandle,
368	offset: number,
369	length: number,
370	destination: Uint8Array,
371	): Promise<number>;
372	closeHandle(handle: FileHandle): Promise<void>;
373	}
374
375	export type FileHandle = ts.Brand<"@clo/lib/incremental/FileHandle"> & number;
376
377	export interface Stat {
378	kind: "file" \| "directory" \| "other";
379	size: number;
380	time: number;
381	}
382
383	function defaultFs(): Fs {
384	return {
385	readTextFile: (path: string) => fs.readFile(path, "utf-8"),
386	readDir: fs.readdir,
387	realPath: fs.realpath,
388	async stat(path) {
389	const st = await fs.stat(path);
390	return {
391	kind: st.isFile() ? "file" : st.isDirectory() ? "directory" : "other",
392	size: st.size,
393	time: st.mtimeMs,
394	};
395	},
396	async open(path) {
397	return await new Promise<FileHandle>((resolve, reject) => {
398	fscb.open(path, "r", (error, fd) => {
399	if (error) reject(error);
400	else resolve(fd as FileHandle);
401	});
402	});
403	},
404	async readHandle(handle, offset, length, destination) {
405	const result = await new Promise<number>((resolve, reject) => {
406	fscb.read(
407	handle,
408	destination,
409	0,
410	length,
411	offset,
412	(error, bytesRead) => {
413	if (error) reject(error);
414	else resolve(bytesRead);
415	},
416	);
417	});
418	return result;
419	},
420	async statHandle(handle) {
421	const result = await new Promise<Stat>((resolve, reject) => {
422	fscb.fstat(
423	handle,
424	(error, st) => {
425	if (error) reject(error);
426	else {resolve({
427	kind: st.isFile() ? "file" : st.isDirectory() ? "directory" : "other",
428	size: st.size,
429	time: st.mtimeMs,
430	});}
431	},
432	);
433	});
434	return result;
435	},
436	async closeHandle(handle) {
437	await new Promise<void>((resolve, reject) => {
438	fscb.close(handle, (error) => {
439	if (error) reject(error);
440	else resolve();
441	});
442	});
443	},
444	};
445	}
446
447	// TODO one day, this will just be a variant of the `Trace` object. for now just passing the source string
448	export interface LoaderInput {
449	source: string;
450	file: string;
451	getPackageMeta(): Promise<PackageMeta \| null>;
452	}
453
454	export type LoaderResult =
455	\| {
456	content: string;
457	format: "js" \| "ts" \| "cjs";
458	}
459	\| {
460	content: Serializable;
461	format: "serializable";
462	};
463
464	type LoaderFunction = (input: LoaderInput) => ts.Awaitable<LoaderResult>;
465
466	export interface GraphOptions {
467	/** file path to a database, or `null` for an in-memory database. */
468	db: Database \| string \| null;
469
470	/** register dynamic sources. */
471	sources: SourceOptions[];
472
473	/** override the filesystem implementation. disables CommonJS support. */
474	fs?: Fs;
475	/**
476	* key is an extension including the `.`, mapping to a function to load it's
477	* source code. appended or overwrites the default JavaScript+Typescript
478	* (`.[mc]?[tj]s`), `.json`, `.node` loaders.
479	*/
480	loaders?: Record<string, LoaderFunction>;
481	/** appended to the defaults. */
482	conditions?: string[];
483	/** caller-owned shared package.json metadata cache. */
484	packageJsonCache?: Map<string, PackageMeta>;
485	}
486
487	/**
488	* a reference to this object is needed to track sources
489	*/
490	interface SourceOptions {
491	protocol: string;
492	verify(
493	record: SourceRecord,
494	): Promise<{ value: Serializable; kind: string } \| "unchanged" \| "changed">;
495	query(url: URL): Promise<{ value: Serializable } & Input>;
496	}
497
498	let fileSource: SourceOptions = {
499	protocol: "file",
500	async verify(record) {
501	throw new Error("Function not implemented.");
502	},
503	async query(url) {
504	throw new Error("Function not implemented.");
505	},
506	};
507
508	/** in form `${namespace}:${subPath}` */
509	type Key = `${string}:${string}`;
510
511	function toKey(namespace: string, subPath: string): Key {
512	ASSERT(!namespace.includes(":"));
513	return `${namespace}:${subPath}` as Key;
514	}
515
516	function parseKey(key: Key): { namespace: string; subpath: string } {
517	const [_, namespace, subpath] = UNWRAP(
518	key.match(/^([^:]+):(.*)$/),
519	`invalid key format: ${key}`,
520	) as [string, string, string];
521	return { namespace, subpath };
522	}
523
524	/** holds module loading state */
525	export class Graph {
526	db: Database;
527	#fs: Fs;
528	#loaders: Record<string, LoaderFunction>;
529	#conditions: string[];
530	#packageJsonCache: Map<string, PackageMeta>;
531	#usesNativeFs: boolean;
532	registry: Map<Key, GraphEntry>;
533
534	constructor(options: GraphOptions) {
535	if (typeof vm.SourceTextModule !== "function") {
536	throw new Error(
537	"@clo/lib/incremental requires node to run with --experimental-vm-modules",
538	);
539	}
540	this.db = options.db instanceof Database ? options.db : new Database(options.db);
541	this.#fs = options.fs ?? defaultFs();
542	this.#loaders = { ...defaultLoaders, ...options.loaders };
543	this.#conditions = [...defaultConditions, ...(options.conditions ?? [])];
544	this.#packageJsonCache = options.packageJsonCache ?? new Map();
545	this.#usesNativeFs = !options.fs;
546	this.registry = new Map();
547	}
548
549	/**
550	* runs the provided build pipeline. the pipeline is an orchestration of
551	* {@linkcode StepFn\|steps}, async functions that read and process data. by
552	* keeping all actual reads in these step functions, this means an incremental
553	* re-run can quickly skip all of the actual logic. this can only be done
554	* because the caching primitives are perfect. at the end, the pipeline should
555	* write/copy its intended outputs with the output APIs in {@linkcode Ref}.
556	*/
557	async incrementalBuild<T>(options: {
558	buildAndWriteOutput: (topLevel: Ref) => Promise<T>;
559	progress: progress.Ref;
560	}): Promise<T> {
561	const build: PendingBuild = {
562	inputs: new Map(),
563	steps: new Set(),
564	activeSteps: new Set(),
565	failures: [],
566	closed: false,
567	};
568	let result: T;
569	let thrown: unknown;
570	let didThrow = false;
571	try {
572	result = await options.buildAndWriteOutput(
573	new Ref({
574	graph: this,
575	progress: options.progress,
576	build,
577	}),
578	);
579	} catch (error) {
580	thrown = error;
581	didThrow = true;
582	} finally {
583	await waitForBuildSteps(build);
584	build.closed = true;
585	}
586
587	if (didThrow) throw thrown;
588	const failures = build.failures.flatMap((failure) => {
589	const result = failure();
590	return result.observed ? [] : [result.error];
591	});
592	if (failures.length > 0) throw new AggregateError(failures);
593
594	// TODO: post-build stuff. for example, assert that all input files are still ok?
595	return result!;
596	}
597
598	/** load a module without tracking its owner */
599	// TODO: tracing
600	async import<T = unknown>(absPath: string): Promise<T> {
601	const root = toKey("file", await this.#fs.realPath(absPath));
602	const entry = await this.#loadModule.get(root);
603	const queue = [entry];
604	const visited = new Set<Key>([root]);
605
606	while (queue.length > 0) {
607	const current = queue.shift()!;
608	if (current.module instanceof vm.SyntheticModule) continue;
609
610	const dependencies = await Promise.all(
611	current.resolutions.map(
612	async (key) => (await this.#loadModule.get(key)).module,
613	),
614	);
615	current.module.linkRequests(dependencies);
616
617	for (let i = 0; i < current.resolutions.length; i++) {
618	const dependencyKey = current.resolutions[i]!;
619	if (visited.has(dependencyKey)) continue;
620	visited.add(dependencyKey);
621	queue.push(await this.#loadModule.get(dependencyKey));
622	}
623	}
624
625	if (entry.module instanceof vm.SourceTextModule) entry.module.instantiate();
626	await entry.module.evaluate();
627	return entry.module.namespace as T;
628	}
629
630	#loadModule = new async.DedupeConcurrent(
631	async (key: Key): Promise<GraphEntry> => {
632	const cached = this.registry.get(key);
633	if (cached) return cached;
634
635	const { namespace, subpath } = parseKey(key);
636	try {
637	let entry: GraphEntry;
638	if (namespace === "file") {
639	entry = await this.#createEntryFromFile(subpath);
640	} else if (namespace === "node") {
641	entry = await this.#createEntryFromNodeBuiltin(subpath);
642	} else {
643	throw new Error(`unsupported module namespace: ${namespace}`);
644	}
645
646	this.registry.set(key, entry);
647	return entry;
648	} catch (err) {
649	exception.rethrowWithMetadata(err, { namespace, subpath });
650	}
651	},
652	([key]) => key,
653	);
654
655	/** constructs a new graph entry */
656	async #createEntryFromFile(absPath: string): Promise<GraphEntry> {
657	const ext = path.extname(absPath);
658	const loader = this.#loaders[ext];
659	if (!loader) {
660	throw new Error(
661	`No loader registered for extension ${JSON.stringify(ext)}`,
662	);
663	}
664
665	const loaded = await loader({
666	file: absPath,
667	source: await this.#fs.readTextFile(absPath),
668	getPackageMeta: async () => {
669	return await this.lookupPackageMeta(path.dirname(absPath));
670	},
671	});
672
673	if (loaded.format === "serializable") {
674	const value = loaded.content;
675	return {
676	module: new vm.SyntheticModule(
677	["default"],
678	function() {
679	this.setExport("default", value);
680	},
681	{ identifier: absPath },
682	),
683	resolutions: [],
684	};
685	}
686
687	let content = loaded.content;
688	if (loaded.format === "ts") {
689	try {
690	content = module.stripTypeScriptTypes(content);
691	} catch {
692	content = module.stripTypeScriptTypes(content, { mode: "transform" });
693	}
694	}
695
696	if (loaded.format === "cjs") {
697	if (!this.#usesNativeFs) {
698	throw new Error(
699	`CommonJS support is not implemented for a virtual fs.`,
700	);
701	}
702	return await this.#createEntryFromCommonJs(absPath);
703	}
704
705	const mod = new vm.SourceTextModule(content, {
706	identifier: absPath,
707	initializeImportMeta(meta) {
708	meta.dirname = path.dirname(absPath);
709	meta.filename = absPath;
710	meta.url = url.pathToFileURL(absPath).href;
711	meta.main = false;
712	},
713	});
714
715	const resolutions = await Promise.all(
716	mod.moduleRequests.map(({ specifier }) => this.#resolveDependency(absPath, specifier, false)),
717	);
718
719	return { module: mod, resolutions };
720	}
721
722	/** constructs a new graph entry */
723	async #createEntryFromNodeBuiltin(name: string): Promise<GraphEntry> {
724	const builtin = node.process?.getBuiltinModule(name as never);
725	if (!builtin) {
726	throw new Error(`Could not load builtin module ${JSON.stringify(name)}`);
727	}
728	return {
729	module: new vm.SyntheticModule(
730	Object.keys(builtin).concat("default"),
731	function() {
732	for (const key of Object.keys(builtin)) {
733	this.setExport(key, builtin[key as keyof typeof builtin]);
734	}
735	this.setExport("default", builtin);
736	},
737	{ identifier: `node:${name}` },
738	),
739	resolutions: [],
740	};
741	}
742
743	async #resolveDependency(
744	fromFile: string,
745	specifier: string,
746	sloppy: boolean,
747	): Promise<Key> {
748	const result = await resolve({
749	specifier,
750	resolveDir: path.dirname(fromFile),
751	fs: this.#fs,
752	sloppy,
753	conditions: this.#conditions,
754	packageJsonCache: this.#packageJsonCache,
755	});
756	if (result.type === "error") {
757	exception.rethrowWithMetadata(result.error, {
758	source: fromFile,
759	specifier, // TODO: error message should always be clear as day
760	});
761	}
762	return toKey(result.namespace, result.subPath);
763	}
764
765	async lookupPackageMeta(startDir: string): Promise<PackageMeta \| null> {
766	for (const dir of walkUpDirectories(startDir)) {
767	const packageJsonPath = path.join(dir, "package.json");
768	try {
769	const stat = await this.#fs.stat(packageJsonPath);
770	if (stat.kind !== "file") continue;
771	const meta = await readPackageJsonMeta(
772	this.#fs,
773	packageJsonPath,
774	this.#packageJsonCache,
775	);
776	return meta;
777	} catch (error) {
778	if (exception.code(error) === "ENOENT") continue;
779	throw error;
780	}
781	}
782	return null;
783	}
784
785	async #createEntryFromCommonJs(absPath: string): Promise<GraphEntry> {
786	const exportsValue = module.createRequire(absPath)(absPath);
787	const exportNames = ["default", ...getCommonJsNamedExports(exportsValue)];
788	const synthetic = new vm.SyntheticModule(
789	exportNames,
790	function() {
791	this.setExport("default", exportsValue);
792	for (const name of exportNames) {
793	if (name === "default") continue;
794	this.setExport(name, (exportsValue as Record<string, unknown>)[name]);
795	}
796	},
797	{ identifier: absPath },
798	);
799	return {
800	module: synthetic,
801	resolutions: [],
802	};
803	}
804	}
805
806	interface GraphEntry {
807	module: vm.SourceTextModule \| vm.SyntheticModule;
808	resolutions: Key[];
809	}
810
811	export interface PackageMeta {
812	type: "module" \| "commonjs" \| null;
813	imports: unknown;
814	exports: unknown;
815	}
816
817	const defaultLoaders: Record<string, LoaderFunction> = {
818	".js": async ({ source, getPackageMeta }) => ({
819	content: source,
820	format: (
821	{
822	commonjs: "cjs",
823	module: "js",
824	} as const
825	)[(await getPackageMeta())?.type ?? "commonjs"],
826	}),
827	".ts": async ({ source }) => ({ content: source, format: "ts" }),
828	".mjs": async ({ source }) => ({ content: source, format: "js" }),
829	".cjs": async ({ source }) => ({ content: source, format: "cjs" }),
830	".mts": async ({ source }) => ({ content: source, format: "ts" }),
831
832	".json": async ({ source }) => ({
833	content: JSON.parse(source),
834	format: "serializable",
835	}),
836	};
837
838	// --- module resolution ---
839
840	export interface ResolveOptions {
841	specifier: string;
842	resolveDir: string;
843	/** override the filesystem implementation */
844	fs?: Fs;
845	/**
846	* enable legacy commonjs resolution features
847	* @default false
848	*/
849	sloppy?: boolean;
850	conditions?: string[];
851	/** caller-owned shared package.json metadata cache */
852	packageJsonCache?: Map<string, PackageMeta>;
853	}
854
855	export type Resolution =
856	& {
857	/** the files that could have affected this resolution */
858	files: string[];
859	}
860	& (
861	\| {
862	type: "success";
863	namespace: string;
864	subPath: string;
865	error?: undefined;
866	}
867	\| {
868	type: "error";
869	error: unknown;
870	namespace?: undefined;
871	subPath?: undefined;
872	}
873	);
874
875	const defaultConditions = ["node", "import", "default"];
876	const sloppyExtensions = [".js", ".json", ".node"];
877
878	/**
879	* perform filesystem-based module resolution. by default, this adheres to
880	* everything node.js supports in it's ESM import resolution. the difference
881	* between this and something like `require.resolve` is it returns the full list
882	* of paths that could affect the outcome of the resolution.
883	*/
884	export async function resolve({
885	specifier,
886	resolveDir,
887	fs = defaultFs(),
888	sloppy = false,
889	conditions = defaultConditions,
890	packageJsonCache = new Map<string, PackageMeta>(),
891	}: ResolveOptions): Promise<Resolution> {
892	const r = new Resolver(fs, conditions, packageJsonCache);
893	try {
894	const { namespace, subPath } = await r.resolve(
895	specifier,
896	resolveDir,
897	sloppy,
898	);
899	return { type: "success", namespace, subPath, files: Array.from(r.files) };
900	} catch (error) {
901	return { type: "error", error, files: Array.from(r.files) };
902	}
903	}
904
905	/** sanity class to track file read states during module resolution */
906	class Resolver {
907	fs: Fs;
908	files = new Set<string>();
909	conditions: string[];
910	packageJsonCache: Map<string, PackageMeta>;
911
912	constructor(
913	fs: Fs,
914	conditions: string[],
915	packageJsonCache: Map<string, PackageMeta>,
916	) {
917	this.fs = fs;
918	this.conditions = conditions;
919	this.packageJsonCache = packageJsonCache;
920	}
921
922	async resolve(
923	specifier: string,
924	resolveDir: string,
925	sloppy: boolean,
926	): Promise<{ namespace: string; subPath: string }> {
927	const isNodeBuiltin = specifier.startsWith("node:");
928	if (isNodeBuiltin \|\| module.builtinModules.includes(specifier)) {
929	return {
930	namespace: "node",
931	subPath: isNodeBuiltin ? specifier.slice(5) : specifier,
932	};
933	}
934
935	if (
936	specifier.startsWith("/")
937	\|\| specifier.startsWith("./")
938	\|\| specifier.startsWith("../")
939	) {
940	const abs = path.resolve(resolveDir, specifier);
941	const result = sloppy
942	? await this.resolvePathSloppy(abs)
943	: await this.resolvePathExact(abs);
944	return { namespace: "file", subPath: await this.fs.realPath(result) };
945	}
946
947	if (specifier.startsWith("#")) {
948	const packageJsonPath = await this.findClosestPackageJson(resolveDir);
949	if (!packageJsonPath) {
950	this.throwWithCode(
951	`Cannot find package.json for import ${JSON.stringify(specifier)} from ${resolveDir}`,
952	"ERR_PACKAGE_IMPORT_NOT_DEFINED",
953	);
954	}
955	const meta = await this.readPackageJsonMeta(packageJsonPath);
956	const target = this.resolvePackageImportCondition(
957	meta.imports,
958	specifier,
959	);
960	if (!target) {
961	this.throwWithCode(
962	`Package import specifier ${JSON.stringify(specifier)} is not defined in ${packageJsonPath}`,
963	"ERR_PACKAGE_IMPORT_NOT_DEFINED",
964	);
965	}
966	return {
967	namespace: "file",
968	subPath: await this.resolvePackageTarget(
969	path.dirname(packageJsonPath),
970	target,
971	sloppy,
972	),
973	};
974	}
975
976	if (!this.isBareSpecifier(specifier)) {
977	this.throwWithCode(
978	`Invalid module specifier ${JSON.stringify(specifier)}`,
979	"ERR_INVALID_MODULE_SPECIFIER",
980	);
981	}
982
983	const resolved = await this.resolveBareSpecifier(
984	specifier,
985	resolveDir,
986	sloppy,
987	);
988	return { namespace: "file", subPath: await this.fs.realPath(resolved) };
989	}
990
991	async stat(file: string): Promise<Stat \| null> {
992	this.files.add(file);
993	try {
994	return await this.fs.stat(file);
995	} catch (error) {
996	if (exception.code(error) === "ENOENT") return null;
997	throw error;
998	}
999	}
1000
1001	async readJsonObject(file: string): Promise<Record<string, unknown>> {
1002	this.files.add(file);
1003	return JSON.parse(await this.fs.readTextFile(file)) as Record<string, unknown>;
1004	}
1005
1006	async readPackageJsonMeta(file: string): Promise<PackageMeta> {
1007	this.files.add(file);
1008	const cached = this.packageJsonCache.get(file);
1009	if (cached) return cached;
1010
1011	const json = JSON.parse(await this.fs.readTextFile(file)) as Record<
1012	string,
1013	unknown
1014	>;
1015	const meta: PackageMeta = {
1016	type: json.type === "module" \|\| json.type === "commonjs" ? json.type : null,
1017	imports: json.imports,
1018	exports: json.exports,
1019	};
1020	this.packageJsonCache.set(file, meta);
1021	return meta;
1022	}
1023
1024	async resolvePathExact(abs: string): Promise<string> {
1025	const stat = await this.stat(abs);
1026	if (!stat) {
1027	this.throwWithCode(
1028	`Cannot find module ${JSON.stringify(abs)}`,
1029	"ERR_MODULE_NOT_FOUND",
1030	);
1031	}
1032	if (stat.kind === "directory") {
1033	this.throwWithCode(
1034	`Directory import ${JSON.stringify(abs)} is not supported`,
1035	"ERR_UNSUPPORTED_DIR_IMPORT",
1036	);
1037	}
1038	if (stat.kind !== "file") {
1039	this.throwWithCode(
1040	`Cannot load module ${JSON.stringify(abs)}`,
1041	"ERR_MODULE_NOT_FOUND",
1042	);
1043	}
1044	return abs;
1045	}
1046
1047	async resolvePathSloppy(abs: string): Promise<string> {
1048	const exact = await this.stat(abs);
1049	if (exact?.kind === "file") return abs;
1050
1051	for (const ext of sloppyExtensions) {
1052	const candidate = `${abs}${ext}`;
1053	const stat = await this.stat(candidate);
1054	if (stat?.kind === "file") return candidate;
1055	}
1056
1057	const packageJson = path.join(abs, "package.json");
1058	const packageStat = await this.stat(packageJson);
1059	if (packageStat?.kind === "file") {
1060	const meta = await this.readPackageJsonMeta(packageJson);
1061	let target = this.resolvePackageImportCondition(meta.exports, ".");
1062	if (!target) {
1063	const json = await this.readJsonObject(packageJson);
1064	const moduleField = typeof json.module === "string" ? json.module : null;
1065	const mainField = typeof json.main === "string" ? json.main : null;
1066	target = moduleField ?? mainField;
1067	}
1068	if (target) {
1069	return await this.resolveLegacyPackageTarget(abs, target, true);
1070	}
1071	}
1072
1073	for (const ext of sloppyExtensions) {
1074	const indexCandidate = path.join(abs, `index${ext}`);
1075	const stat = await this.stat(indexCandidate);
1076	if (stat?.kind === "file") return indexCandidate;
1077	}
1078
1079	if (exact?.kind === "directory" \|\| packageStat?.kind === "file") {
1080	this.throwWithCode(
1081	`Directory import ${JSON.stringify(abs)} is not supported`,
1082	"ERR_UNSUPPORTED_DIR_IMPORT",
1083	);
1084	}
1085
1086	this.throwWithCode(
1087	`Cannot find module ${JSON.stringify(abs)}`,
1088	"ERR_MODULE_NOT_FOUND",
1089	);
1090	}
1091
1092	async findClosestPackageJson(startDir: string): Promise<string \| null> {
1093	for (const dir of walkUpDirectories(startDir)) {
1094	const candidate = path.join(dir, "package.json");
1095	const stat = await this.stat(candidate);
1096	if (stat?.kind === "file") return candidate;
1097	}
1098	return null;
1099	}
1100
1101	async resolveBareSpecifier(
1102	specifier: string,
1103	resolveDir: string,
1104	sloppy: boolean,
1105	): Promise<string> {
1106	const { packageName, packageSubpath } = this.parseBareSpecifier(specifier);
1107
1108	for (const dir of walkUpDirectories(resolveDir)) {
1109	const packageDir = path.join(dir, "node_modules", packageName);
1110	const packageJsonPath = path.join(packageDir, "package.json");
1111	const packageJsonStat = await this.stat(packageJsonPath);
1112	if (!packageJsonStat) continue;
1113	if (packageJsonStat.kind !== "file") {
1114	this.throwWithCode(
1115	`Invalid package.json for ${packageName}`,
1116	"ERR_MODULE_NOT_FOUND",
1117	);
1118	}
1119
1120	const meta = await this.readPackageJsonMeta(packageJsonPath);
1121	const subpath = packageSubpath ? `./${packageSubpath}` : ".";
1122
1123	if (meta.exports !== undefined) {
1124	const target = this.resolvePackageImportCondition(
1125	meta.exports,
1126	subpath,
1127	);
1128	if (!target) {
1129	this.throwWithCode(
1130	`Package subpath ${JSON.stringify(subpath)} is not exported by ${packageName}`,
1131	"ERR_PACKAGE_PATH_NOT_EXPORTED",
1132	);
1133	}
1134	return await this.resolvePackageTarget(packageDir, target, sloppy);
1135	}
1136
1137	if (subpath !== ".") {
1138	return sloppy
1139	? await this.resolvePathSloppy(path.join(packageDir, packageSubpath))
1140	: await this.resolvePathExact(path.join(packageDir, packageSubpath));
1141	}
1142
1143	const json = await this.readJsonObject(packageJsonPath);
1144	const mainField = typeof json.module === "string"
1145	? json.module
1146	: typeof json.main === "string"
1147	? json.main
1148	: null;
1149	if (mainField) {
1150	return await this.resolveLegacyPackageTarget(
1151	packageDir,
1152	mainField,
1153	sloppy,
1154	);
1155	}
1156
1157	return sloppy
1158	? await this.resolvePathSloppy(path.join(packageDir, "index"))
1159	: await this.resolvePathExact(path.join(packageDir, "index.js"));
1160	}
1161
1162	this.throwWithCode(
1163	`Cannot find package ${JSON.stringify(packageName)}`,
1164	"ERR_MODULE_NOT_FOUND",
1165	);
1166	}
1167
1168	async resolvePackageTarget(
1169	packageDir: string,
1170	target: string,
1171	sloppy: boolean,
1172	): Promise<string> {
1173	if (target.startsWith("#")) {
1174	const packageJsonPath = await this.findClosestPackageJson(packageDir);
1175	if (!packageJsonPath) {
1176	this.throwWithCode(
1177	`Package import specifier ${JSON.stringify(target)} is not defined`,
1178	"ERR_PACKAGE_IMPORT_NOT_DEFINED",
1179	);
1180	}
1181	const meta = await this.readPackageJsonMeta(packageJsonPath);
1182	const nestedTarget = this.resolvePackageImportCondition(
1183	meta.imports,
1184	target,
1185	);
1186	if (!nestedTarget) {
1187	this.throwWithCode(
1188	`Package import specifier ${JSON.stringify(target)} is not defined in ${packageJsonPath}`,
1189	"ERR_PACKAGE_IMPORT_NOT_DEFINED",
1190	);
1191	}
1192	return await this.resolvePackageTarget(
1193	path.dirname(packageJsonPath),
1194	nestedTarget,
1195	sloppy,
1196	);
1197	}
1198	if (
1199	!target.startsWith("./")
1200	&& !target.startsWith("../")
1201	&& !target.startsWith("/")
1202	) {
1203	this.throwWithCode(
1204	`Invalid package target ${JSON.stringify(target)}`,
1205	"ERR_INVALID_PACKAGE_TARGET",
1206	);
1207	}
1208	const abs = path.resolve(packageDir, target);
1209	return sloppy
1210	? await this.resolvePathSloppy(abs)
1211	: await this.resolvePathExact(abs);
1212	}
1213
1214	async resolveLegacyPackageTarget(
1215	packageDir: string,
1216	target: string,
1217	sloppy: boolean,
1218	): Promise<string> {
1219	if (target.startsWith("#")) {
1220	return await this.resolvePackageTarget(packageDir, target, sloppy);
1221	}
1222	if (
1223	target.startsWith("./")
1224	\|\| target.startsWith("../")
1225	\|\| target.startsWith("/")
1226	) {
1227	return await this.resolvePackageTarget(packageDir, target, sloppy);
1228	}
1229	const abs = path.resolve(packageDir, target);
1230	return sloppy
1231	? await this.resolvePathSloppy(abs)
1232	: await this.resolvePathExact(abs);
1233	}
1234
1235	isBareSpecifier(specifier: string): boolean {
1236	return (
1237	!specifier.startsWith(".")
1238	&& !specifier.startsWith("/")
1239	&& !specifier.startsWith("#")
1240	);
1241	}
1242
1243	parseBareSpecifier(specifier: string): {
1244	packageName: string;
1245	packageSubpath: string;
1246	} {
1247	const parts = specifier.split("/");
1248	const packageName = specifier.startsWith("@")
1249	? parts.slice(0, 2).join("/")
1250	: (parts[0] ?? "");
1251	const packageSubpath = specifier
1252	.slice(packageName.length)
1253	.replace(/^\/+/, "");
1254	return { packageName, packageSubpath };
1255	}
1256
1257	resolvePackageImportCondition(
1258	map: unknown,
1259	specifier: string,
1260	): string \| null {
1261	if (map !== null && typeof map === "object" && !Array.isArray(map)) {
1262	const keys = Object.keys(map as object);
1263	const hasPathKeys = keys.some(
1264	(k) => k.startsWith(".") \|\| k.startsWith("#"),
1265	);
1266	if (hasPathKeys) {
1267	return this.resolveSubpathMap(
1268	map as Record<string, unknown>,
1269	specifier,
1270	);
1271	}
1272	}
1273	return this.resolveTarget(map, null);
1274	}
1275
1276	private resolveSubpathMap(
1277	matchObj: Record<string, unknown>,
1278	matchKey: string,
1279	): string \| null {
1280	if (matchKey in matchObj) {
1281	return this.resolveTarget(matchObj[matchKey], null);
1282	}
1283
1284	const patternKeys = Object.keys(matchObj)
1285	.filter((k) => k.includes("*"))
1286	.sort((a, b) => b.indexOf("") - a.indexOf(""));
1287
1288	for (const expansionKey of patternKeys) {
1289	const starIdx = expansionKey.indexOf("*");
1290	const patternBase = expansionKey.slice(0, starIdx);
1291	const patternTrailer = expansionKey.slice(starIdx + 1);
1292
1293	if (!matchKey.startsWith(patternBase)) continue;
1294	if (matchKey.length < expansionKey.length) continue;
1295	if (patternTrailer.length > 0 && !matchKey.endsWith(patternTrailer)) {
1296	continue;
1297	}
1298
1299	const patternMatch = matchKey.slice(
1300	patternBase.length,
1301	matchKey.length - patternTrailer.length,
1302	);
1303	return this.resolveTarget(matchObj[expansionKey], patternMatch);
1304	}
1305
1306	return null;
1307	}
1308
1309	private resolveTarget(
1310	target: unknown,
1311	patternMatch: string \| null,
1312	): string \| null {
1313	if (typeof target === "string") {
1314	if (!target.startsWith("./") && !target.startsWith("#")) {
1315	this.throwWithCode(
1316	`invalid target: "${target}"`,
1317	"ERR_INVALID_PACKAGE_TARGET",
1318	);
1319	}
1320	return patternMatch !== null
1321	? target.replaceAll("*", patternMatch)
1322	: target;
1323	}
1324
1325	if (Array.isArray(target)) {
1326	for (const item of target) {
1327	try {
1328	const result = this.resolveTarget(item, patternMatch);
1329	if (result !== null) return result;
1330	} catch {
1331	//
1332	}
1333	}
1334	return null;
1335	}
1336
1337	if (target !== null && typeof target === "object") {
1338	for (
1339	const [key, value] of Object.entries(
1340	target as Record<string, unknown>,
1341	)
1342	) {
1343	if (key === "default" \|\| this.conditions.includes(key)) {
1344	const result = this.resolveTarget(value, patternMatch);
1345	if (result !== null) return result;
1346	}
1347	}
1348	}
1349
1350	return null;
1351	}
1352
1353	private throwWithCode(message: string, code: node.ErrorCode): never {
1354	exception.rethrowWithMetadata(new Error(message), { code });
1355	}
1356	}
1357
1358	async function readPackageJsonMeta(
1359	fs: Fs,
1360	file: string,
1361	packageJsonCache: Map<string, PackageMeta>,
1362	): Promise<PackageMeta> {
1363	const cached = packageJsonCache.get(file);
1364	if (cached) return cached;
1365
1366	const json = JSON.parse(await fs.readTextFile(file)) as Record<string, unknown>;
1367	const meta: PackageMeta = {
1368	type: json.type === "module" \|\| json.type === "commonjs" ? json.type : null,
1369	imports: json.imports,
1370	exports: json.exports,
1371	};
1372	packageJsonCache.set(file, meta);
1373	return meta;
1374	}
1375
1376	function* walkUpDirectories(startDir: string): Generator<string> {
1377	let current = path.resolve(startDir);
1378	while (true) {
1379	yield current;
1380	const parent = path.dirname(current);
1381	if (parent === current) break;
1382	current = parent;
1383	}
1384	}
1385
1386	function getCommonJsNamedExports(value: unknown): string[] {
1387	if (!value \|\| typeof value !== "object" \|\| Array.isArray(value)) return [];
1388	const proto = Object.getPrototypeOf(value);
1389	if (proto !== Object.prototype && proto !== null) return [];
1390	return Array.from(
1391	new Set(Object.keys(value).filter((key) => key !== "default")),
1392	);
1393	}
1394
1395	// --- database & serialization ---
1396	export type RowId = number \| bigint;
1397
1398	/**
1399	* a file, network request, or other input referenced within a
1400	* {@linkcode StepFn} function. unlike the `key` to a cache boundary, `Input`
1401	* urls can stay constant but have underlying changes. file paths are the main
1402	* use case, but this also extends to something like a database query.
1403	*/
1404	export interface Input {
1405	/** global key for the changable object, with enough information to re-query. */
1406	url: string;
1407	/** inline data known about this object, such as a file kind. */
1408	kind: string;
1409
1410	// these values are dependant on the implementation of whatever revalidates if
1411	// an `Input` has changed, and are modelled after a filesystem stat.
1412
1413	/** modification time or revision number. */
1414	time: number;
1415	/** byte size or other cheap source-specific magnitude. */
1416	size: number;
1417	/** a freshness token, such as a content hash. */
1418	hash: string;
1419	}
1420
1421	export interface SourceRecord extends Input {
1422	id: RowId;
1423	}
1424
1425	export interface CacheRecord {
1426	id: RowId;
1427	inputs: IterableIterator<SourceRecord>;
1428	/** read via `readBlob` */
1429	blob: RowId;
1430	}
1431
1432	/** dangerous trust-me-bro types to quickly add some type safety to sqlite */
1433	type TypedSqliteStatement<
1434	Input extends object,
1435	Output extends object = never,
1436	> = Omit<sqlite.StatementSync, "run" \| "get" \| "all" \| "iterate"> & {
1437	run(input: Input): sqlite.StatementResultingChanges;
1438	get(input: Input): Output \| undefined;
1439	all(input: Input): Output[];
1440	iterate(input: Input): IterableIterator<Output>;
1441	};
1442
1443	function prepareQueries(sql: sqlite.DatabaseSync) {
1444	return {
1445	insertBlob: sql.prepare(
1446	"insert into blobs (type, content) values ($type, $content);",
1447	) as unknown as TypedSqliteStatement<{
1448	type: "j" \| "s" \| "b";
1449	content: string \| Uint8Array;
1450	}>,
1451	readBlob: sql.prepare(
1452	"select type, content from blobs where id = $id;",
1453	) as unknown as TypedSqliteStatement<
1454	{ id: RowId },
1455	{ type: "j" \| "s" \| "b"; content: string \| Uint8Array }
1456	>,
1457	upsertSource: sql.prepare(
1458	"insert into sources (url, kind, time, size, hash) "
1459	+ "values ($url, $kind, $time, $size, $hash) "
1460	+ "on conflict(url) do update set "
1461	+ "kind = excluded.kind, "
1462	+ "time = excluded.time, "
1463	+ "size = excluded.size, "
1464	+ "hash = excluded.hash;",
1465	) as unknown as TypedSqliteStatement<{
1466	url: string;
1467	kind: string;
1468	time: number;
1469	size: number;
1470	hash: string;
1471	}>,
1472	readSource: sql.prepare(
1473	"select id, url, kind, time, size, hash from sources where url = $url;",
1474	) as unknown as TypedSqliteStatement<{ url: string }, SourceRecord>,
1475	readSources: sql.prepare(
1476	"select id, url, kind, time, size, hash from sources order by url;",
1477	) as unknown as TypedSqliteStatement<Record<string, never>, SourceRecord>,
1478	selectSourceId: sql.prepare(
1479	"select id from sources where url = $url;",
1480	) as unknown as TypedSqliteStatement<{ url: string }, { id: RowId }>,
1481	insertCache: sql.prepare(
1482	"insert into caches (key, blob) values ($key, $blob) "
1483	+ "on conflict(key) do update set blob = excluded.blob;",
1484	) as unknown as TypedSqliteStatement<{ key: string; blob: RowId }>,
1485	selectCache: sql.prepare(
1486	"select id, blob from caches where key = $key;",
1487	) as unknown as TypedSqliteStatement<
1488	{ key: string },
1489	{ id: RowId; blob: RowId }
1490	>,
1491	deleteCacheInputs: sql.prepare(
1492	"delete from inputs where cache = $cache;",
1493	) as unknown as TypedSqliteStatement<{ cache: RowId }>,
1494	insertInput: sql.prepare(
1495	"insert or ignore into inputs (source, cache) values ($source, $cache);",
1496	) as unknown as TypedSqliteStatement<{ source: RowId; cache: RowId }>,
1497	readCacheInputs: sql.prepare(
1498	"select sources.id, sources.url, sources.kind, sources.time, "
1499	+ "sources.size, sources.hash "
1500	+ "from sources "
1501	+ "join inputs on inputs.source = sources.id "
1502	+ "where inputs.cache = $cache "
1503	+ "order by sources.url;",
1504	) as unknown as TypedSqliteStatement<{ cache: RowId }, SourceRecord>,
1505	};
1506	}
1507
1508	/** backing storage for incremental artifacts built on `node:sqlite` */
1509	export class Database {
1510	sql: sqlite.DatabaseSync;
1511	#prep: ReturnType<typeof prepareQueries>;
1512	#cachedBlobs = new Map<RowId, Serializable>(); // WeakRef?
1513
1514	constructor(path: string \| null) {
1515	ASSERT(!path?.startsWith(":"));
1516	this.sql = new sqlite.DatabaseSync(path ?? ":memory:");
1517	this.sql.exec(
1518	[
1519	"pragma foreign_keys = ON;",
1520
1521	"create table if not exists blobs (",
1522	/**/ "id integer primary key asc,",
1523	/**/ "type text CHECK(type IN ('j','s','b')) not null,",
1524	/**/ "content any not null",
1525	") strict;",
1526
1527	"create table if not exists sources (",
1528	/**/ "id integer primary key asc,",
1529	/**/ "url text unique not null,",
1530	/**/ "kind text not null,",
1531	/**/ "time integer not null,",
1532	/**/ "size integer not null,",
1533	/**/ "hash text not null",
1534	") strict;",
1535
1536	"create table if not exists caches (",
1537	/**/ "id integer primary key asc,",
1538	/**/ "key text unique not null,",
1539	/**/ "blob integer not null references blobs(id) on delete cascade",
1540	") strict;",
1541
1542	"create table if not exists inputs (",
1543	/**/ "source integer not null references sources(id) on delete cascade,",
1544	/**/ "cache integer not null references caches(id) on delete cascade,",
1545	/**/ "primary key(source, cache)",
1546	") strict;",
1547
1548	// TODO: name `a` and `b` to clearly draw the line
1549	"create table if not exists deps (",
1550	/**/ "a integer not null references sources(id) on delete cascade,",
1551	/**/ "b integer not null references sources(id) on delete cascade,",
1552	/**/ "primary key(a, b)",
1553	") strict;",
1554	].join(""),
1555	);
1556	this.#prep = prepareQueries(this.sql);
1557	}
1558
1559	/** forget serialized blobs that were held in memory */
1560	clearInMemoryCache() {
1561	this.#cachedBlobs.clear();
1562	}
1563
1564	writeSource(source: Input): RowId {
1565	this.#validateSource(source);
1566	this.#prep.upsertSource.run(source);
1567	return UNWRAP(this.#prep.selectSourceId.get({ url: source.url })).id;
1568	}
1569
1570	sources(): IterableIterator<SourceRecord> {
1571	return this.#prep.readSources.iterate({}) as IterableIterator<SourceRecord>;
1572	}
1573
1574	invalidate(source: RowId \| Iterable<RowId>): void {
1575	const sources = typeof source === "number" \|\| typeof source === "bigint"
1576	? [source]
1577	: Array.from(source);
1578	if (sources.length === 0) return;
1579	const params = Object.fromEntries(
1580	sources.map((source, i) => [`source${i}`, source]),
1581	);
1582	const placeholders = sources.map((_, i) => `$source${i}`).join(", ");
1583	this.sql
1584	.prepare(
1585	`delete from caches where id in (`
1586	+ `select cache from inputs where source in (${placeholders})`
1587	+ `);`,
1588	)
1589	.run(params);
1590	}
1591
1592	/**
1593	* read a cache entry. caller should first examine `.inputs` to determine if
1594	* the cache value is truly fresh, and only then deserialize the blob.
1595	*/
1596	readCache(key: string): CacheRecord \| null {
1597	const row = this.#prep.selectCache.get({ key });
1598	if (!row) return null;
1599	return {
1600	id: row.id,
1601	blob: row.blob,
1602	inputs: this.#prep.readCacheInputs.iterate({ cache: row.id }),
1603	};
1604	}
1605
1606	/** read the source records linked to a cache entry. */
1607	readCacheInputs(cache: RowId): IterableIterator<SourceRecord> {
1608	return this.#prep.readCacheInputs.iterate({ cache });
1609	}
1610
1611	/** link a key to a value, with a set of `inputs` that bust this cache entry */
1612	writeCache(key: string, value: Serializable, inputs: Input[]): RowId {
1613	const { blob } = this.writeBlob(value);
1614	this.#prep.insertCache.run({ key, blob });
1615	const cache = UNWRAP(this.#prep.selectCache.get({ key })).id;
1616	this.#prep.deleteCacheInputs.run({ cache });
1617	for (const input of inputs) {
1618	const source = this.writeSource(input);
1619	this.#prep.insertInput.run({ source, cache });
1620	}
1621	return cache;
1622	}
1623
1624	/** write a {@linkcode Serializable} to the database */
1625	writeBlob(data: Serializable, cache = true): { blob: number \| bigint } {
1626	let content: string \| Uint8Array;
1627	let type: "j" \| "s" \| "b";
1628	if (typeof data === "string") {
1629	type = "s";
1630	content = data;
1631	} else if (data instanceof Uint8Array) {
1632	type = "b";
1633	content = data;
1634	} else {
1635	type = "j";
1636	content = JSON.stringify(this.#serializeObject(data, cache));
1637	}
1638	const insert = this.#prep.insertBlob.run({ type, content });
1639	if (cache) this.#cachedBlobs.set(insert.lastInsertRowid, data);
1640	return { blob: insert.lastInsertRowid };
1641	}
1642
1643	/** write a {@linkcode Serializable} via it's ID */
1644	readBlob(blob: RowId): Serializable {
1645	const cached = this.#cachedBlobs.get(blob);
1646	if (cached !== undefined \|\| this.#cachedBlobs.has(blob)) return cached;
1647
1648	const row = this.#prep.readBlob.get({ id: blob });
1649	if (!row) throw new Error(`missing blob row ${String(blob)}`);
1650
1651	const value = row.type === "s"
1652	? (row.content as string)
1653	: row.type === "b"
1654	? (row.content as Uint8Array)
1655	: this.#deserializeObject(JSON.parse(row.content as string));
1656
1657	this.#cachedBlobs.set(blob, value);
1658	return value;
1659	}
1660
1661	#validateSource(source: Input): void {
1662	let parsed: URL;
1663	try {
1664	parsed = new URL(source.url);
1665	} catch {
1666	throw new Error(`source url must be a complete URL: ${source.url}`);
1667	}
1668	if (parsed.href !== source.url) {
1669	throw new Error(`source url must be canonical: ${source.url}`);
1670	}
1671	if (!Number.isFinite(source.time)) {
1672	throw new Error(`source time must be a finite number: ${source.url}`);
1673	}
1674	if (!Number.isFinite(source.size)) {
1675	throw new Error(`source size must be a finite number: ${source.url}`);
1676	}
1677	}
1678
1679	#serializeObject(data: Serializable, cache = true): unknown {
1680	if (
1681	data === null
1682	\|\| typeof data === "string"
1683	\|\| typeof data === "boolean"
1684	) {
1685	return data;
1686	}
1687	if (typeof data === "number") {
1688	return Number.isFinite(data)
1689	? data
1690	: { $clover: "number", value: String(data) };
1691	}
1692	if (typeof data === "undefined") {
1693	return { $clover: "undefined" };
1694	}
1695	if (data instanceof Uint8Array) {
1696	return { $clover: "buffer", blob: this.writeBlob(data, cache).blob };
1697	}
1698	if (data instanceof Date) {
1699	return { $clover: "date", time: data.getTime() };
1700	}
1701	if (data instanceof Set) {
1702	return {
1703	$clover: "set",
1704	entries: Array.from(data, (value) => this.#serializeObject(value, cache)),
1705	};
1706	}
1707	if (data instanceof Map) {
1708	return {
1709	$clover: "map",
1710	entries: Array.from(data, ([key, value]) => [
1711	this.#serializeObject(key, cache),
1712	this.#serializeObject(value, cache),
1713	]),
1714	};
1715	}
1716	if (Array.isArray(data)) {
1717	return data.map((value) => this.#serializeObject(value, cache));
1718	}
1719	const out: Record<string, unknown> = {};
1720	for (const [key, value] of Object.entries(data)) {
1721	out[key] = this.#serializeObject(value, cache);
1722	}
1723	if (Object.hasOwn(out, "$clover")) {
1724	return { $clover: "escaped", value: out };
1725	}
1726	return out;
1727	}
1728
1729	#deserializeObject(data: unknown): Serializable {
1730	if (
1731	data === null
1732	\|\| typeof data === "string"
1733	\|\| typeof data === "boolean"
1734	) {
1735	return data;
1736	}
1737	if (typeof data === "number") return data;
1738	if (Array.isArray(data)) {
1739	return data.map((value) => this.#deserializeObject(value));
1740	}
1741	if (!data \|\| typeof data !== "object") {
1742	throw new Error(`unsupported serialized value: ${String(data)}`);
1743	}
1744
1745	const object = data as Record<string, unknown>;
1746	switch (object.$clover) {
1747	case undefined: {
1748	const out: Record<string, Serializable> = {};
1749	for (const [key, value] of Object.entries(object)) {
1750	out[key] = this.#deserializeObject(value);
1751	}
1752	return out;
1753	}
1754	case "number":
1755	return Number(object.value);
1756	case "undefined":
1757	return undefined;
1758	case "buffer":
1759	return this.readBlob(object.blob as RowId) as Uint8Array;
1760	case "date":
1761	return new Date(object.time as number);
1762	case "set":
1763	return new Set(
1764	(object.entries as unknown[]).map((value) => this.#deserializeObject(value)),
1765	);
1766	case "map":
1767	return new Map(
1768	(object.entries as [unknown, unknown][]).map(([key, value]) => [
1769	this.#deserializeObject(key),
1770	this.#deserializeObject(value),
1771	]),
1772	);
1773	case "escaped": {
1774	const value = object.value as Record<string, unknown>;
1775	const out: Record<string, Serializable> = {};
1776	for (const [key, entry] of Object.entries(value)) {
1777	out[key] = this.#deserializeObject(entry);
1778	}
1779	return out;
1780	}
1781	default:
1782	throw new Error(`unknown serialized marker: ${String(object.$clover)}`);
1783	}
1784	}
1785	}
1786
1787	export function lazyLoader<T extends { default: LoaderFunction }>(
1788	load: () => Promise<T>,
1789	): LoaderFunction {
1790	const once = async.once(load);
1791	return async (input) => (await once()).default(input);
1792	}
1793
1794	function hashSerializable(...values: Serializable[]): string {
1795	const hash = crypto.createHash("sha256");
1796	writeSerializableToHasher(hash, values);
1797	return hash.digest("base64url");
1798	}
1799
1800	function writeSerializableToHasher(
1801	hash: crypto.Hash,
1802	value: Serializable,
1803	): void {
1804	if (value === null) {
1805	hash.update("n;");
1806	return;
1807	}
1808	if (typeof value === "undefined") {
1809	hash.update("u;");
1810	return;
1811	}
1812	if (typeof value === "boolean") {
1813	hash.update(value ? "t;" : "f;");
1814	return;
1815	}
1816	if (typeof value === "number") {
1817	writeStringToHasher(hash, "#", Object.is(value, -0) ? "-0" : String(value));
1818	return;
1819	}
1820	if (typeof value === "string") {
1821	writeStringToHasher(hash, "s", value);
1822	return;
1823	}
1824	if (value instanceof Uint8Array) {
1825	hash.update(`b${value.byteLength}:`);
1826	hash.update(value);
1827	hash.update(";");
1828	return;
1829	}
1830	if (value instanceof Date) {
1831	writeStringToHasher(hash, "d", String(value.getTime()));
1832	return;
1833	}
1834	if (value instanceof Set) {
1835	hash.update(`e${value.size}:[`);
1836	for (const entry of value) writeSerializableToHasher(hash, entry);
1837	hash.update("]");
1838	return;
1839	}
1840	if (value instanceof Map) {
1841	hash.update(`m${value.size}:{`);
1842	for (const [key, entry] of value) {
1843	writeSerializableToHasher(hash, key);
1844	writeSerializableToHasher(hash, entry);
1845	}
1846	hash.update("}");
1847	return;
1848	}
1849	if (Array.isArray(value)) {
1850	hash.update(`a${value.length}:[`);
1851	for (const entry of value) writeSerializableToHasher(hash, entry);
1852	hash.update("]");
1853	return;
1854	}
1855	const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
1856	hash.update(`o${entries.length}:{`);
1857	for (const [key, entry] of entries) {
1858	writeStringToHasher(hash, "k", key);
1859	writeSerializableToHasher(hash, entry);
1860	}
1861	hash.update("}");
1862	}
1863
1864	function writeStringToHasher(
1865	hash: crypto.Hash,
1866	tag: string,
1867	value: string,
1868	): void {
1869	const byteLength = Buffer.byteLength(value);
1870	hash.update(`${tag}${byteLength}:`);
1871	hash.update(value);
1872	hash.update(";");
1873	}
1874
1875	import { ASSERT, UNWRAP } from "./assert.ts";
1876	import * as async from "./async.ts";
1877	import * as exception from "./error.ts";
1878	import * as stack from "./log/stack.ts";
1879	import * as node from "./node.ts";
1880	import * as progress from "./progress.ts";
1881	import * as ts from "./ts.ts";
1882
1883	import * as crypto from "node:crypto";
1884	import * as fscb from "node:fs";
1885	import * as fs from "node:fs/promises";
1886	import * as module from "node:module";
1887	import * as path from "node:path";
1888	import * as sqlite from "node:sqlite";
1889	import * as url from "node:url";
1890	import * as vm from "node:vm";

lib/log/stack.ts+1-1

...	@@ -90,7 +90,7 @@ export function format(error: Error, colors = false): string {	...	@@ -90,7 +90,7 @@ export function format(error: Error, colors = false): string {
90	const { name, message, stack: _, ...payload } = error;	90	const { name, message, stack: _, ...payload } = error;
91		91
92	let out = (colors ? ansi.fgRed : "")	92	let out = (colors ? ansi.fgRed : "")
93	+ (name && name !== "Error" ? `[${name}] ` : "")	93	+ (name && name !== "Error" && name !== "TypeError" ? `[${name}] ` : "")
94	+ message + (colors ? ansi.fgReset : "") + "\n";	94	+ message + (colors ? ansi.fgReset : "") + "\n";
95		95
96	out += Object.entries(payload)	96	out += Object.entries(payload)

lib/mime.ts+2-3

...	@@ -127,7 +127,6 @@ const extensions = new Map(	...	@@ -127,7 +127,6 @@ const extensions = new Map(
127	+ ".xls" + "#" + "application/vnd.ms-excel" + "\n"	127	+ ".xls" + "#" + "application/vnd.ms-excel" + "\n"
128	+ ".xlsx" + "#" + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + "\n"	128	+ ".xlsx" + "#" + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + "\n"
129	+ ".xml" + "#" + "application/xml" + "\n"	129	+ ".xml" + "#" + "application/xml" + "\n"
130	+ ".xps" + "#" + "application/vnd.ms-xpsdocument" + "\n"	130	+ ".xps" + "#" + "application/vnd.ms-xpsdocument"
131	+ +""	131	).split("\n").map((line) => line.split("#") as [string, string]),
132	).slice(0, -1).split("\n").map((line) => line.split("#") as [string, string]),
133	);	132	);

lib/package.json-4

 @@ -1,9 +1,5 @@
 {
   "name": "@clo/lib",
   "type": "module",
-  "dependencies": {
-    "@types/node": "^24.0.0",
-    "@clo/lib": "file:."
-  },
   "exports": { "./*": "./*.ts" }
 }

lib/progress.ts+271-225

...	@@ -85,7 +85,7 @@ export interface Ref {	...	@@ -85,7 +85,7 @@ export interface Ref {
85	* creates a new trackable unit of work as a child of this one.	85	* creates a new trackable unit of work as a child of this one.
86	* when given an estimate, a progress bar is rendered.	86	* when given an estimate, a progress bar is rendered.
87	*/	87	*/
88	start(text: string, opts?: StartOptions): Node;	88	start: (text: string, opts?: StartOptions) => Node;
89	}	89	}
90		90
91	/**	91	/**
...	@@ -205,17 +205,19 @@ export function slashValueFormatter(	...	@@ -205,17 +205,19 @@ export function slashValueFormatter(
205	): ValueFormatter {	205	): ValueFormatter {
206	return (value: number, total: number \| null) =>	206	return (value: number, total: number \| null) =>
207	value > 0	207	value > 0
208	? fmtNumber(value)	208	? fmtNumber(value) +
209	+ (total != null && total > 0 ? "/" + fmtNumber(total) : "")	209	(total != null && total > 0 ? "/" + fmtNumber(total) : "")
210	: "";	210	: "";
211	}	211	}
212		212
213	/** the formatter used when passing `units: "count"` to {@linkcode start} */	213	/** the formatter used when passing `units: "count"` to {@linkcode start} */
214	export const defaultValueFormatter: ValueFormatter = /* @__PURE__ */	214	export const defaultValueFormatter: ValueFormatter =
		215	/* @__PURE__ */
215	slashValueFormatter(String);	216	slashValueFormatter(String);
216		217
217	/** the formatter used when passing `units: "bytes"` to {@linkcode start} */	218	/** the formatter used when passing `units: "bytes"` to {@linkcode start} */
218	export const bytesValueFormatter: ValueFormatter = /* @__PURE__ */	219	export const bytesValueFormatter: ValueFormatter =
		220	/* @__PURE__ */
219	slashValueFormatter(string.formatByteSize);	221	slashValueFormatter(string.formatByteSize);
220		222
221	/** the formatter used when passing `units: "percent"` to {@linkcode start} */	223	/** the formatter used when passing `units: "percent"` to {@linkcode start} */
...	@@ -342,18 +344,17 @@ export class Root<	...	@@ -342,18 +344,17 @@ export class Root<
342		344
343	#emitChangeSoon = () => {	345	#emitChangeSoon = () => {
344	if (this.#debounce) return;	346	if (this.#debounce) return;
345	(this.#debounce = this.delay(1))	347	(this.#debounce = this.delay(1)).then(() => {
346	.then(() => {	348	this.#debounce = null;
347	this.#debounce = null;	349	this.emit("change", this.#active);
348	this.emit("change", this.#active);	350	});
349	});
350	};	351	};
351	}	352	}
352		353
353	/** the built in events provided by {@linkcode Root} */	354	/** the built in events provided by {@linkcode Root} */
354	export type RootEventMap<Result = void> = {	355	export type RootEventMap<Result = void> = {
355	// this event is the debounced "ready to re-render" event. do not emit.	356	// this event is the debounced "ready to re-render" event. do not emit.
356	"change": [rootNodes: readonly ReadOnlyNode[]];	357	change: [rootNodes: readonly ReadOnlyNode[]];
357	// these events are emitted without a debounce, and are used for general	358	// these events are emitted without a debounce, and are used for general
358	// communication within `Root`'s implementation. do not emit.	359	// communication within `Root`'s implementation. do not emit.
359	"node-start": [newNode: ReadOnlyNode, parentNode: ReadOnlyNode \| null];	360	"node-start": [newNode: ReadOnlyNode, parentNode: ReadOnlyNode \| null];
...	@@ -363,8 +364,8 @@ export type RootEventMap<Result = void> = {	...	@@ -363,8 +364,8 @@ export type RootEventMap<Result = void> = {
363	// these events are used for result communication. they can be sent via	364	// these events are used for result communication. they can be sent via
364	// `emit` or the shorthand methods `end` and `error`. sending either will put	365	// `emit` or the shorthand methods `end` and `error`. sending either will put
365	// the `Root` into a "done" state.	366	// the `Root` into a "done" state.
366	"end": [result: Result];	367	end: [result: Result];
367	"error": [error: unknown];	368	error: [error: unknown];
368	};	369	};
369		370
370	const rootEvents: ReadonlySet<unknown> = new Set([	371	const rootEvents: ReadonlySet<unknown> = new Set([
...	@@ -378,13 +379,14 @@ const rootEvents: ReadonlySet<unknown> = new Set([	...	@@ -378,13 +379,14 @@ const rootEvents: ReadonlySet<unknown> = new Set([
378	]);	379	]);
379		380
380	type MergeRootEvents<Result, Map extends Events.Map> = {	381	type MergeRootEvents<Result, Map extends Events.Map> = {
381	[K in keyof Map \| keyof RootEventMap]: K extends keyof RootEventMap ? RootEventMap<Result>[K]	382	[K in keyof Map \| keyof RootEventMap]: K extends keyof RootEventMap
		383	? RootEventMap<Result>[K]
382	: Map[K];	384	: Map[K];
383	};	385	};
384		386
385	/** a read-only version of {@linkcode Node}, provided to renderers. */	387	/** a read-only version of {@linkcode Node}, provided to renderers. */
386	export type ReadOnlyNode = Readonly<	388	export type ReadOnlyNode = Readonly<
387	& Pick<	389	Pick<
388	Required<Node>,	390	Required<Node>,
389	\| "text"	391	\| "text"
390	\| "value"	392	\| "value"
...	@@ -395,8 +397,7 @@ export type ReadOnlyNode = Readonly<	...	@@ -395,8 +397,7 @@ export type ReadOnlyNode = Readonly<
395	\| "showTotal"	397	\| "showTotal"
396	\| "sortChildren"	398	\| "sortChildren"
397	\| "valueFormatter"	399	\| "valueFormatter"
398	>	400	> & {
399	& {
400	/** unique identifier. once {@linkcode Node} is disposed, the key is reused. */	401	/** unique identifier. once {@linkcode Node} is disposed, the key is reused. */
401	key: number;	402	key: number;
402	logs: readonly log.Message[];	403	logs: readonly log.Message[];
...	@@ -416,8 +417,8 @@ interface Internal extends ts.Writeable<ReadOnlyNode> {	...	@@ -416,8 +417,8 @@ interface Internal extends ts.Writeable<ReadOnlyNode> {
416	}	417	}
417		418
418	// TypeScript Test	419	// TypeScript Test
419	void function(unknown: unknown) {	420	void function (unknown: unknown) {
420	(unknown as Internal) satisfies ReadOnlyNode;	421	unknown as Internal satisfies ReadOnlyNode;
421	};	422	};
422		423
423	/**	424	/**
...	@@ -445,11 +446,13 @@ function newNode<Result, EventMap extends Events.Map>(	...	@@ -445,11 +446,13 @@ function newNode<Result, EventMap extends Events.Map>(
445	children: [],	446	children: [],
446	parent: null,	447	parent: null,
447	sortChildren: opts.sortChildren ?? null,	448	sortChildren: opts.sortChildren ?? null,
448	valueFormatter: opts.valueFormatter ?? {	449	valueFormatter:
449	count: defaultValueFormatter,	450	opts.valueFormatter ??
450	bytes: bytesValueFormatter,	451	{
451	percent: percentValueFormatter,	452	count: defaultValueFormatter,
452	}[opts.units ?? "count"],	453	bytes: bytesValueFormatter,
		454	percent: percentValueFormatter,
		455	}[opts.units ?? "count"],
453	detached: false,	456	detached: false,
454	};	457	};
455		458
...	@@ -464,8 +467,11 @@ function newNode<Result, EventMap extends Events.Map>(	...	@@ -464,8 +467,11 @@ function newNode<Result, EventMap extends Events.Map>(
464	function mutate(key: keyof ReadOnlyNode) {	467	function mutate(key: keyof ReadOnlyNode) {
465	if (state.detached) return;	468	if (state.detached) return;
466	if (	469	if (
467	key === "value" && estimator && state.total > 0 && state.value > 0	470	key === "value" &&
468	&& state.value < state.total	471	estimator &&
		472	state.total > 0 &&
		473	state.value > 0 &&
		474	state.value < state.total
469	) {	475	) {
470	// originally, auto-estimation would use the Root's timing primitives,	476	// originally, auto-estimation would use the Root's timing primitives,
471	// but this makes it so that `state.estimatedTime` is in terms of the	477	// but this makes it so that `state.estimatedTime` is in terms of the
...	@@ -587,18 +593,15 @@ export const nullNode: Node = {	...	@@ -587,18 +593,15 @@ export const nullNode: Node = {
587	get text() {	593	get text() {
588	return "[detached]";	594	return "[detached]";
589	},	595	},
590	set text(_) {	596	set text(_) {},
591	},
592	get value() {	597	get value() {
593	return 0;	598	return 0;
594	},	599	},
595	set value(_) {	600	set value(_) {},
596	},
597	get total() {	601	get total() {
598	return 0;	602	return 0;
599	},	603	},
600	set total(_) {	604	set total(_) {},
601	},
602	inc: noop,	605	inc: noop,
603	log: {	606	log: {
604	info: noop,	607	info: noop,
...	@@ -627,7 +630,7 @@ function endNode<R, M extends Events.Map>(owner: Root<R, M>, state: Internal) {	...	@@ -627,7 +630,7 @@ function endNode<R, M extends Events.Map>(owner: Root<R, M>, state: Internal) {
627	const i = state.parent.children.indexOf(state);	630	const i = state.parent.children.indexOf(state);
628	ASSERT(i !== -1);	631	ASSERT(i !== -1);
629	let p: Internal \| null = state;	632	let p: Internal \| null = state;
630	while (p = p.parent) title = p.text + " / " + title;	633	while ((p = p.parent)) title = p.text + " / " + title;
631	state.parent.children.splice(i, 1);	634	state.parent.children.splice(i, 1);
632	}	635	}
633	owner.emit("node-end", state);	636	owner.emit("node-end", state);
...	@@ -650,36 +653,45 @@ const spinnerFps = 12.5;	...	@@ -650,36 +653,45 @@ const spinnerFps = 12.5;
650	const box = { tee: "├─ ", line: "│ ", langle: "└─ " };	653	const box = { tee: "├─ ", line: "│ ", langle: "└─ " };
651	const barChars = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"];	654	const barChars = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"];
652	const fullBar = "█";	655	const fullBar = "█";
653	const spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]	656	const spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"].map(
654	.map((frame) => ansi.style(ansi.fgBlue, frame));	657	(frame) => ansi.style(ansi.fgBlue, frame),
		658	);
655		659
656	function renderAnsiMainLine(state: ReadOnlyNode, now: number, depth: number) {	660	function renderAnsiMainLine(state: ReadOnlyNode, now: number, depth: number) {
657	const { text, total, showTotal, value, estimatedTime } = state;	661	const { text, total, showTotal, value, estimatedTime } = state;
658	const dateNow = Date.now();	662	const dateNow = Date.now();
659	const showEstimate = estimatedTime && !findLongerEstimate(state, estimatedTime);	663	const showEstimate =
660	const estimate = showEstimate && (estimatedTime > dateNow + 1000)	664	estimatedTime && !findLongerEstimate(state, estimatedTime);
661	? ", "	665	const estimate =
662	+ string.formatDurationLetters(Math.round((estimatedTime - dateNow) / 1000))	666	showEstimate && estimatedTime > dateNow + 1000
663	: "";	667	? ", " +
		668	string.formatDurationLetters(
		669	Math.round((estimatedTime - dateNow) / 1000),
		670	)
		671	: "";
664	const valueFormatted = state.valueFormatter(value, total);	672	const valueFormatted = state.valueFormatter(value, total);
665	if (total && showTotal) {	673	if (total && showTotal) {
666	return ansi.style(	674	return (
667	ansi.bgBrightBlack + ansi.fgBlue,	675	ansi.style(
668	formatUnicodeBar(	676	ansi.bgBrightBlack + ansi.fgBlue,
669	value / total,	677	formatUnicodeBar(value / total, Math.max(4, depth > 0 ? 12 : 25)),
670	Math.max(4, depth > 0 ? 12 : 25),	678	) +
671	),	679	(valueFormatted \|\| estimate ? ` [${valueFormatted}${estimate}] ` : " ") +
672	)	680	text
673	+ (valueFormatted \|\| estimate ? ` [${valueFormatted}${estimate}] ` : " ")	681	);
674	+ text;
675	}	682	}
676	const frame = Math.floor(now / (1000 / spinnerFps)) % spinner.length;	683	const frame = Math.floor(now / (1000 / spinnerFps)) % spinner.length;
677	return ((depth === 0 ? spinner[frame] + " " : "")	684	return (
678	+ (valueFormatted ? `[${valueFormatted}] ` : "") + text);	685	(depth === 0 ? spinner[frame] + " " : "") +
		686	(valueFormatted ? `[${valueFormatted}] ` : "") +
		687	text
		688	);
679	}	689	}
680		690
681	function hasChildren(states: ReadOnlyNode): boolean {	691	function hasChildren(states: ReadOnlyNode): boolean {
682	return states.children.some((x) => !x.hidden && (!x.passive \|\| hasChildren(x)));	692	return states.children.some(
		693	(x) => !x.hidden && (!x.passive \|\| hasChildren(x)),
		694	);
683	}	695	}
684		696
685	function findLongerEstimate(	697	function findLongerEstimate(
...	@@ -701,12 +713,14 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {	...	@@ -701,12 +713,14 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {
701	let truncated = 0;	713	let truncated = 0;
702	let out = "";	714	let out = "";
703	let { children } = state;	715	let { children } = state;
704	children = children.filter((x) => !x.hidden && (!x.passive \|\| hasChildren(x)));	716	children = children.filter(
		717	(x) => !x.hidden && (!x.passive \|\| hasChildren(x)),
		718	);
705	if (children.length === 0) return "";	719	if (children.length === 0) return "";
706	for (let i = 0, { length } = children; i < length; i += 1) {	720	for (let i = 0, { length } = children; i < length; i += 1) {
707	const child = UNWRAP(children[i]);	721	const child = UNWRAP(children[i]);
708	let item = "";	722	let item = "";
709	const left = depth.map((x) => x ? box.line : " ").join("");	723	const left = depth.map((x) => (x ? box.line : " ")).join("");
710	item += left + (i === length - 1 && !truncated ? box.langle : box.tee);	724	item += left + (i === length - 1 && !truncated ? box.langle : box.tee);
711	item += renderAnsiMainLine(child, now, 1 + depth.length) + "\n";	725	item += renderAnsiMainLine(child, now, 1 + depth.length) + "\n";
712	item += renderChildren(child, now, depth.concat(i < length - 1));	726	item += renderChildren(child, now, depth.concat(i < length - 1));
...	@@ -721,17 +735,21 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {	...	@@ -721,17 +735,21 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {
721	.trim();	735	.trim();
722	if (logLines) {	736	if (logLines) {
723	for (const line of logLines.split("\n").slice(-Math.min(3, maxHeight))) {	737	for (const line of logLines.split("\n").slice(-Math.min(3, maxHeight))) {
724	item += left + (i === length - 1 && !truncated ? " " : box.line)	738	item +=
725	+ " "	739	left +
726	+ ansi.style(ansi.fgBrightBlack, ">")	740	(i === length - 1 && !truncated ? " " : box.line) +
727	+ " " + line + "\n";	741	" " +
		742	ansi.style(ansi.fgBrightBlack, ">") +
		743	" " +
		744	line +
		745	"\n";
728	}	746	}
729	}	747	}
730	maxHeight -= h + string.countNewlines(logLines);	748	maxHeight -= h + string.countNewlines(logLines);
731	out += item;	749	out += item;
732	}	750	}
733	if (truncated) {	751	if (truncated) {
734	out += depth.map((x) => x ? box.line : " ").join("") + box.langle;	752	out += depth.map((x) => (x ? box.line : " ")).join("") + box.langle;
735	out += ansi.style(ansi.fgBrightBlack, `[${truncated} more]`);	753	out += ansi.style(ansi.fgBrightBlack, `[${truncated} more]`);
736	}	754	}
737	return out;	755	return out;
...	@@ -779,43 +797,52 @@ export function formatUnicodeBar(progress: number, width: number): string {	...	@@ -779,43 +797,52 @@ export function formatUnicodeBar(progress: number, width: number): string {
779	*/	797	*/
780	export function attachToScreen(	798	export function attachToScreen(
781	root: Root,	799	root: Root,
782	{ writeOutput, startWidget }: Pick<	800	{
783	log.WidgetHost,	801	writeOutput,
784	"writeOutput" \| "startWidget"	802	startWidget,
785	>,	803	}: Pick<log.WidgetHost, "writeOutput" \| "startWidget">,
786	): ts.Dispose {	804	): ts.Dispose {
787	const stack = new DisposableStack();	805	const stack = new DisposableStack();
788	let widget: log.WidgetInstance \| null = null;	806	let widget: log.WidgetInstance \| null = null;
789		807
790	stack.use(root.on("change", (items) => {	808	stack.use(
791	if (items.length > 0) {	809	root.on("change", (items) => {
792	widget ??= startWidget({	810	if (items.length > 0) {
793	format: ({ now }) => formatAnsi(now, root.active),	811	widget ??=
794	}) ?? null;	812	startWidget({
795	if (!widget) return;	813	format: ({ now }) => formatAnsi(now, root.active),
796	widget.fps = items.some((x) => x.showTotal !== false && x.total > 0)	814	}) ?? null;
797	? spinnerFps	815	if (!widget) return;
798	: null;	816	widget.fps = items.some((x) => x.showTotal !== false && x.total > 0)
799	widget.redraw();	817	? spinnerFps
800	} else {	818	: null;
801	widget?.stop();	819	widget.redraw();
802	widget = null;	820	} else {
803	}	821	widget?.stop();
804	}));	822	widget = null;
805	stack.use(root.on("node-detached-log", (msg) => {	823	}
806	writeOutput(log.formatAnsiMessage(msg, true));	824	}),
807	}));	825	);
808	stack.use(root.on("node-end", (node) => {	826	stack.use(
809	let title = node.text;	827	root.on("node-detached-log", (msg) => {
810	let p: ReadOnlyNode \| null = node;	828	writeOutput(log.formatAnsiMessage(msg, true));
811	while (p = p.parent) title = p.text + " / " + title;	829	}),
812	const { logs } = node;	830	);
813	if (logs.length > 0) {	831	stack.use(
814	const header = `[logs from ${title}]`;	832	root.on("node-end", (node) => {
815	writeOutput(ansi.style(ansi.fgBrightBlack, header) + "\n");	833	let title = node.text;
816	writeOutput(logs.map((msg) => log.formatAnsiMessage(msg, true)).join(""));	834	let p: ReadOnlyNode \| null = node;
817	}	835	while ((p = p.parent)) title = p.text + " / " + title;
818	}));	836	const { logs } = node;
		837	if (logs.length > 0) {
		838	const header = `[logs from ${title}]`;
		839	writeOutput(ansi.style(ansi.fgBrightBlack, header) + "\n");
		840	writeOutput(
		841	logs.map((msg) => log.formatAnsiMessage(msg, true)).join(""),
		842	);
		843	}
		844	}),
		845	);
819		846
820	return ts.defer(() => stack[Symbol.dispose]);	847	return ts.defer(() => stack[Symbol.dispose]);
821	}	848	}
...	@@ -902,7 +929,7 @@ export function encodeEventStream<	...	@@ -902,7 +929,7 @@ export function encodeEventStream<
902	// fun fact, this type is impossible to write without generics	929	// fun fact, this type is impossible to write without generics
903	root: Root<Result, Map>,	930	root: Root<Result, Map>,
904	{	931	{
905	throttleMs = 1000 / 30, /* 30fps */	932	throttleMs = 1000 / 30 /* 30fps */,
906	serializeStackTraces = false,	933	serializeStackTraces = false,
907	}: EncodeStreamOptions = {},	934	}: EncodeStreamOptions = {},
908	): ReadableStream<StreamEvent> {	935	): ReadableStream<StreamEvent> {
...	@@ -934,47 +961,56 @@ export function encodeEventStream<	...	@@ -934,47 +961,56 @@ export function encodeEventStream<
934	});	961	});
935		962
936	let isFirst = true;	963	let isFirst = true;
937	return new ReadableStream({	964	return new ReadableStream(
938	start(controller) {	965	{
939	if (root.active.length > 0) {	966	start(controller) {
		967	if (root.active.length > 0) {
		968	controller.enqueue(s.getDelta(isFirst));
		969	isFirst = false;
		970	}
		971
		972	// TODO: why does ts require two args
		973	stack.use(
		974	root.on("node-start", (newNode, _) => {
		975	if (!newNode.parent) s.rootChildrenUpdated = true;
		976	s.changed.set(newNode, new Set());
		977	emitSoon();
		978	}),
		979	);
		980	stack.use(
		981	root.on("node-change", (node, key) => {
		982	let set = s.changed.get(node);
		983	if (!set) s.changed.set(node, (set = new Set()));
		984	set.add(key);
		985	emitSoon();
		986	}),
		987	);
		988	stack.use(
		989	root.on("node-end", (node) => {
		990	if (!node.parent) s.rootChildrenUpdated = true;
		991	s.changed.delete(node);
		992	s.deleted.add(node); // handle add and remove in same frame
		993	emitSoon();
		994	}),
		995	);
		996	},
		997	async pull(controller) {
		998	await ready.until((x) => x === true);
		999	ready.value = false;
		1000	if (timer) {
		1001	timer.cancel();
		1002	timer = null;
		1003	}
940	controller.enqueue(s.getDelta(isFirst));	1004	controller.enqueue(s.getDelta(isFirst));
941	isFirst = false;	1005	isFirst = false;
942	}	1006	},
943		1007	cancel() {
944	// TODO: why does ts require two args	1008	stack.dispose();
945	stack.use(root.on("node-start", (newNode, _) => {	1009	(0, performance.now)();
946	if (!newNode.parent) s.rootChildrenUpdated = true;	1010	},
947	s.changed.set(newNode, new Set());
948	emitSoon();
949	}));
950	stack.use(root.on("node-change", (node, key) => {
951	let set = s.changed.get(node);
952	if (!set) s.changed.set(node, set = new Set());
953	set.add(key);
954	emitSoon();
955	}));
956	stack.use(root.on("node-end", (node) => {
957	if (!node.parent) s.rootChildrenUpdated = true;
958	s.changed.delete(node);
959	s.deleted.add(node); // handle add and remove in same frame
960	emitSoon();
961	}));
962	},
963	async pull(controller) {
964	await ready.until((x) => x === true);
965	ready.value = false;
966	if (timer) {
967	timer.cancel();
968	timer = null;
969	}
970	controller.enqueue(s.getDelta(isFirst));
971	isFirst = false;
972	},
973	cancel() {
974	stack.dispose();
975	(0, performance.now)();
976	},	1011	},
977	}, { highWaterMark: 0 });	1012	{ highWaterMark: 0 },
		1013	);
978	}	1014	}
979		1015
980	/**	1016	/**
...	@@ -1064,13 +1100,10 @@ class Encoder<	...	@@ -1064,13 +1100,10 @@ class Encoder<
1064		1100
1065	const { updated = [], started = [] } = Object.groupBy(	1101	const { updated = [], started = [] } = Object.groupBy(
1066	this.changed,	1102	this.changed,
1067	([node]) => this.keyMap.has(node) ? "updated" : "started",	1103	([node]) => (this.keyMap.has(node) ? "updated" : "started"),
1068	);	1104	);
1069		1105
1070	const payload: StreamEvent = [	1106	const payload: StreamEvent = [isFirst ? -1 : deleted.length, ...deleted];
1071	isFirst ? -1 : deleted.length,
1072	...deleted,
1073	];
1074		1107
1075	if (started.length > 0) {	1108	if (started.length > 0) {
1076	this.beginNodesDepthFirst(	1109	this.beginNodesDepthFirst(
...	@@ -1082,8 +1115,8 @@ class Encoder<	...	@@ -1082,8 +1115,8 @@ class Encoder<
1082	for (const [node, props] of updated) {	1115	for (const [node, props] of updated) {
1083	let logs = undefined;	1116	let logs = undefined;
1084	if (	1117	if (
1085	props.has("logs")	1118	props.has("logs") &&
1086	&& node.logs.length > (this.logsLength.get(node) ?? 0)	1119	node.logs.length > (this.logsLength.get(node) ?? 0)
1087	) {	1120	) {
1088	logs = this.redactLogsIfNeeded(	1121	logs = this.redactLogsIfNeeded(
1089	node.logs.slice(this.logsLength.get(node) ?? 0),	1122	node.logs.slice(this.logsLength.get(node) ?? 0),
...	@@ -1103,11 +1136,13 @@ class Encoder<	...	@@ -1103,11 +1136,13 @@ class Encoder<
1103	? node.children.map((node) => UNWRAP(this.keyMap.get(node)))	1136	? node.children.map((node) => UNWRAP(this.keyMap.get(node)))
1104	: undefined,	1137	: undefined,
1105	E: props.has("estimatedTime") ? node.estimatedTime : undefined,	1138	E: props.has("estimatedTime") ? node.estimatedTime : undefined,
1106	V: node.valueFormatter !== defaultValueFormatter	1139	V:
1107	&& (props.has("value") \|\| props.has("total")	1140	node.valueFormatter !== defaultValueFormatter &&
1108	\|\| props.has("valueFormatter"))	1141	(props.has("value") \|\|
1109	? node.valueFormatter(node.value, node.total)	1142	props.has("total") \|\|
1110	: undefined,	1143	props.has("valueFormatter"))
		1144	? node.valueFormatter(node.value, node.total)
		1145	: undefined,
1111	[kNode]: node,	1146	[kNode]: node,
1112	});	1147	});
1113	}	1148	}
...	@@ -1125,16 +1160,13 @@ class Encoder<	...	@@ -1125,16 +1160,13 @@ class Encoder<
1125	return payload;	1160	return payload;
1126	}	1161	}
1127		1162
1128	beginNodesDepthFirst(	1163	beginNodesDepthFirst(nodes: ReadOnlyNode[], out: StreamEvent) {
1129	nodes: ReadOnlyNode[],
1130	out: StreamEvent,
1131	) {
1132	for (let i = 0; i < nodes.length; i += 1) {	1164	for (let i = 0; i < nodes.length; i += 1) {
1133	const node = UNWRAP(nodes[i]);	1165	const node = UNWRAP(nodes[i]);
1134	nodes.push(...node.children);	1166	nodes.push(...node.children);
1135	}	1167	}
1136	let next;	1168	let next;
1137	while (next = nodes.pop()) {	1169	while ((next = nodes.pop())) {
1138	if (!this.keyMap.has(next)) {	1170	if (!this.keyMap.has(next)) {
1139	out.push(this.beginNode(next));	1171	out.push(this.beginNode(next));
1140	}	1172	}
...	@@ -1150,19 +1182,25 @@ class Encoder<	...	@@ -1150,19 +1182,25 @@ class Encoder<
1150	v: node.value !== 0 ? node.value : undefined,	1182	v: node.value !== 0 ? node.value : undefined,
1151	e: node.total !== 0 ? node.total : undefined,	1183	e: node.total !== 0 ? node.total : undefined,
1152	t: node.text.length > 0 ? node.text : undefined,	1184	t: node.text.length > 0 ? node.text : undefined,
1153	l: node.logs.length > 0	1185	l:
1154	? (this.logsLength.set(node, node.logs.length), this.redactLogsIfNeeded(node.logs.slice()))	1186	node.logs.length > 0
1155	: undefined,	1187	? (this.logsLength.set(node, node.logs.length),
1156	c: node.children.length > 0	1188	this.redactLogsIfNeeded(node.logs.slice()))
1157	? node.children.map((node) => UNWRAP(this.keyMap.get(node), () => [node.children, this.keyMap]))	1189	: undefined,
1158	: undefined,	1190	c:
		1191	node.children.length > 0
		1192	? node.children.map((node) =>
		1193	UNWRAP(this.keyMap.get(node), () => [node.children, this.keyMap]),
		1194	)
		1195	: undefined,
1159	p: node.passive === true ? true : undefined,	1196	p: node.passive === true ? true : undefined,
1160	s: node.showTotal === false ? false : undefined,	1197	s: node.showTotal === false ? false : undefined,
1161	h: node.hidden === true ? true : undefined,	1198	h: node.hidden === true ? true : undefined,
1162	E: node.estimatedTime != null ? node.estimatedTime : undefined,	1199	E: node.estimatedTime != null ? node.estimatedTime : undefined,
1163	V: node.valueFormatter !== defaultValueFormatter	1200	V:
1164	? node.valueFormatter(node.value, node.total)	1201	node.valueFormatter !== defaultValueFormatter
1165	: undefined,	1202	? node.valueFormatter(node.value, node.total)
		1203	: undefined,
1166	[kNode]: node,	1204	[kNode]: node,
1167	};	1205	};
1168	}	1206	}
...	@@ -1173,10 +1211,7 @@ class Encoder<	...	@@ -1173,10 +1211,7 @@ class Encoder<
1173	}	1211	}
1174	}	1212	}
1175		1213
1176	class Decoder<	1214	class Decoder<Result = void, EventMap extends Events.Map = ts.EmptyObject> {
1177	Result = void,
1178	EventMap extends Events.Map = ts.EmptyObject,
1179	> {
1180	target: Ref;	1215	target: Ref;
1181	events: Events<EventMap>;	1216	events: Events<EventMap>;
1182	resolve: (result: Result) => void;	1217	resolve: (result: Result) => void;
...	@@ -1284,7 +1319,7 @@ class Decoder<	...	@@ -1284,7 +1319,7 @@ class Decoder<
1284	}	1319	}
1285		1320
1286	let entry;	1321	let entry;
1287	while (entry = this.pendingStart.entries().next().value) {	1322	while ((entry = this.pendingStart.entries().next().value)) {
1288	const [key, value] = entry;	1323	const [key, value] = entry;
1289	this.startRecursive(key, value);	1324	this.startRecursive(key, value);
1290	}	1325	}
...	@@ -1302,9 +1337,8 @@ class Decoder<	...	@@ -1302,9 +1337,8 @@ class Decoder<
1302	startRecursive(key: EncodedKey, opts: PendingStart): Ref {	1337	startRecursive(key: EncodedKey, opts: PendingStart): Ref {
1303	ASSERT(this.pendingStart.delete(key));	1338	ASSERT(this.pendingStart.delete(key));
1304	const parentId = UNWRAP(this.pendingParents.get(key));	1339	const parentId = UNWRAP(this.pendingParents.get(key));
1305	let ref: Ref \| null = parentId === 0	1340	let ref: Ref \| null =
1306	? this.target	1341	parentId === 0 ? this.target : (this.active.get(parentId) ?? null);
1307	: this.active.get(parentId) ?? null;
1308	if (!ref) {	1342	if (!ref) {
1309	ref = this.startRecursive(	1343	ref = this.startRecursive(
1310	parentId,	1344	parentId,
...	@@ -1379,10 +1413,7 @@ export function encodeByteStream<	...	@@ -1379,10 +1413,7 @@ export function encodeByteStream<
1379	}	1413	}
1380		1414
1381	/** convert {@linkcode StreamEvent} into a byte payload. Mutates the input event. */	1415	/** convert {@linkcode StreamEvent} into a byte payload. Mutates the input event. */
1382	function writeStreamEvent(	1416	function writeStreamEvent(event: StreamEvent, writer: stream.WriteTarget) {
1383	event: StreamEvent,
1384	writer: stream.WriteTarget,
1385	) {
1386	using w = new stream.BufferedWriter(writer);	1417	using w = new stream.BufferedWriter(writer);
1387		1418
1388	let deleteCount = event.shift();	1419	let deleteCount = event.shift();
...	@@ -1402,8 +1433,13 @@ function writeStreamEvent(	...	@@ -1402,8 +1433,13 @@ function writeStreamEvent(
1402	}	1433	}
1403	}	1434	}
1404		1435
1405	const rootChildren = event.find((x): x is StreamRootChildren => Array.isArray(x) && typeof x[0] === "number");	1436	const rootChildren = event.find(
1406	const changedNodes = event.filter((x): x is StreamNode => typeof x === "object" && !Array.isArray(x));	1437	(x): x is StreamRootChildren =>
		1438	Array.isArray(x) && typeof x[0] === "number",
		1439	);
		1440	const changedNodes = event.filter(
		1441	(x): x is StreamNode => typeof x === "object" && !Array.isArray(x),
		1442	);
1407	w.varUint(changedNodes.length + +!!rootChildren);	1443	w.varUint(changedNodes.length + +!!rootChildren);
1408	for (const obj of changedNodes) {	1444	for (const obj of changedNodes) {
1409	ASSERT(typeof obj === "object" && "k" in obj);	1445	ASSERT(typeof obj === "object" && "k" in obj);
...	@@ -1422,19 +1458,21 @@ function writeStreamEvent(	...	@@ -1422,19 +1458,21 @@ function writeStreamEvent(
1422	ASSERT(messages == null \|\| messages.length > 0);	1458	ASSERT(messages == null \|\| messages.length > 0);
1423		1459
1424	w.varUint(key);	1460	w.varUint(key);
1425	w.u16(encodeDeltaFlags({	1461	w.u16(
1426	hidden,	1462	encodeDeltaFlags({
1427	passive,	1463	hidden,
1428	showTotal,	1464	passive,
1429	changedValue: value !== undefined,	1465	showTotal,
1430	changedTotal: total !== undefined,	1466	changedValue: value !== undefined,
1431	changedLogs: messages !== undefined,	1467	changedTotal: total !== undefined,
1432	changedText: text !== undefined,	1468	changedLogs: messages !== undefined,
1433	changedChildren: childrenSet !== undefined,	1469	changedText: text !== undefined,
1434	changedEstimatedTime: estimatedTime !== undefined,	1470	changedChildren: childrenSet !== undefined,
1435	changedFormattedValue: formattedValue !== undefined,	1471	changedEstimatedTime: estimatedTime !== undefined,
1436	estimatedTimeNonNull: estimatedTime !== null,	1472	changedFormattedValue: formattedValue !== undefined,
1437	}));	1473	estimatedTimeNonNull: estimatedTime !== null,
		1474	}),
		1475	);
1438	if (value !== undefined) w.f64(value);	1476	if (value !== undefined) w.f64(value);
1439	if (total !== undefined) w.f64(total);	1477	if (total !== undefined) w.f64(total);
1440	if (text !== undefined) w.stringWithLength(text);	1478	if (text !== undefined) w.stringWithLength(text);
...	@@ -1444,10 +1482,10 @@ function writeStreamEvent(	...	@@ -1444,10 +1482,10 @@ function writeStreamEvent(
1444	let level = logLevelSerialize.indexOf(msg.level ?? "info");	1482	let level = logLevelSerialize.indexOf(msg.level ?? "info");
1445	if (level === -1) level = 0;	1483	if (level === -1) level = 0;
1446	w.u8(	1484	w.u8(
1447	level	1485	level +
1448	+ (msg.scope ? 1 << 5 : 0)	1486	(msg.scope ? 1 << 5 : 0) +
1449	+ (msg.stack && msg.stack.length > 0 ? 1 << 6 : 0)	1487	(msg.stack && msg.stack.length > 0 ? 1 << 6 : 0) +
1450	+ (msg.custom !== undefined ? 1 << 7 : 0),	1488	(msg.custom !== undefined ? 1 << 7 : 0),
1451	);	1489	);
1452	w.stringWithLength(msg.text);	1490	w.stringWithLength(msg.text);
1453	w.varUint(Math.floor(msg.time));	1491	w.varUint(Math.floor(msg.time));
...	@@ -1484,7 +1522,9 @@ function writeStreamEvent(	...	@@ -1484,7 +1522,9 @@ function writeStreamEvent(
1484	}	1522	}
1485	}	1523	}
1486		1524
1487	const customEvents = event.filter((x): x is StreamCustomEvent => Array.isArray(x) && typeof x[0] === "string");	1525	const customEvents = event.filter(
		1526	(x): x is StreamCustomEvent => Array.isArray(x) && typeof x[0] === "string",
		1527	);
1488	w.varUint(customEvents.length);	1528	w.varUint(customEvents.length);
1489	for (const [event, ...args] of customEvents) {	1529	for (const [event, ...args] of customEvents) {
1490	w.stringWithLength(event);	1530	w.stringWithLength(event);
...	@@ -1563,9 +1603,9 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {	...	@@ -1563,9 +1603,9 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
1563	while (len > 0) {	1603	while (len > 0) {
1564	const msgFlags = await r.u8();	1604	const msgFlags = await r.u8();
1565	const level = logLevelSerialize[msgFlags & 0b1111] ?? "info";	1605	const level = logLevelSerialize[msgFlags & 0b1111] ?? "info";
1566	const hasScope = msgFlags && (1 << 5) > 0;	1606	const hasScope = msgFlags && 1 << 5 > 0;
1567	const hasStack = msgFlags && (1 << 6) > 0;	1607	const hasStack = msgFlags && 1 << 6 > 0;
1568	const hasCustom = msgFlags && (1 << 7) > 0;	1608	const hasCustom = msgFlags && 1 << 7 > 0;
1569	const text = await r.stringWithLength();	1609	const text = await r.stringWithLength();
1570	const time = await r.varUint();	1610	const time = await r.varUint();
1571	const scope = hasScope ? await r.stringWithLength() : undefined;	1611	const scope = hasScope ? await r.stringWithLength() : undefined;
...	@@ -1596,10 +1636,12 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {	...	@@ -1596,10 +1636,12 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
1596	if (flags.changedChildren) {	1636	if (flags.changedChildren) {
1597	c = [];	1637	c = [];
1598	let len = await r.varUint();	1638	let len = await r.varUint();
1599	while (len-- > 0) c.push(await r.varUint() as EncodedKey);	1639	while (len-- > 0) c.push((await r.varUint()) as EncodedKey);
1600	}	1640	}
1601	const E = flags.changedEstimatedTime	1641	const E = flags.changedEstimatedTime
1602	? flags.estimatedTimeNonNull ? await r.varUint() : null	1642	? flags.estimatedTimeNonNull
		1643	? await r.varUint()
		1644	: null
1603	: undefined;	1645	: undefined;
1604	const V = flags.changedFormattedValue	1646	const V = flags.changedFormattedValue
1605	? await r.stringWithLength()	1647	? await r.stringWithLength()
...	@@ -1608,7 +1650,7 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {	...	@@ -1608,7 +1650,7 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
1608	// a "v2"'s packet will go here, contents ignored by this version.	1650	// a "v2"'s packet will go here, contents ignored by this version.
1609	if (flags.futureProofingPacket) {	1651	if (flags.futureProofingPacket) {
1610	const len = await r.varUint();	1652	const len = await r.varUint();
1611	void await r.readExactly(len);	1653	void (await r.readExactly(len));
1612	}	1654	}
1613		1655
1614	event.push({	1656	event.push({
...	@@ -1641,37 +1683,37 @@ const logLevelSerialize = ["info", "error", "warn", "debug"] as const;	...	@@ -1641,37 +1683,37 @@ const logLevelSerialize = ["info", "error", "warn", "debug"] as const;
1641		1683
1642	function encodeDeltaFlags(node: Partial<DeltaFlags>) {	1684	function encodeDeltaFlags(node: Partial<DeltaFlags>) {
1643	return (	1685	return (
1644	(node.hidden ? 1 << 0 : 0)	1686	(node.hidden ? 1 << 0 : 0) +
1645	+ (node.passive ? 1 << 1 : 0)	1687	(node.passive ? 1 << 1 : 0) +
1646	+ (node.showTotal ? 1 << 2 : 0)	1688	(node.showTotal ? 1 << 2 : 0) +
1647	+ (node.changedValue ? 1 << 3 : 0)	1689	(node.changedValue ? 1 << 3 : 0) +
1648	+ (node.changedTotal ? 1 << 4 : 0)	1690	(node.changedTotal ? 1 << 4 : 0) +
1649	+ (node.changedLogs ? 1 << 6 : 0)	1691	(node.changedLogs ? 1 << 6 : 0) +
1650	+ (node.changedText ? 1 << 7 : 0)	1692	(node.changedText ? 1 << 7 : 0) +
1651	+ (node.changedChildren ? 1 << 8 : 0)	1693	(node.changedChildren ? 1 << 8 : 0) +
1652	+ (node.changedEstimatedTime ? 1 << 9 : 0)	1694	(node.changedEstimatedTime ? 1 << 9 : 0) +
1653	+ (node.changedFormattedValue ? 1 << 10 : 0)	1695	(node.changedFormattedValue ? 1 << 10 : 0) +
1654	+ (node.estimatedTimeNonNull ? 1 << 11 : 0)	1696	(node.estimatedTimeNonNull ? 1 << 11 : 0) +
1655	+ (node.futureProofingPacket ? 1 << 14 : 0)	1697	(node.futureProofingPacket ? 1 << 14 : 0) +
1656	+ (node.deleted ? 1 << 15 : 0)	1698	(node.deleted ? 1 << 15 : 0)
1657	);	1699	);
1658	}	1700	}
1659		1701
1660	function decodeDeltaFlags(flags: number): DeltaFlags {	1702	function decodeDeltaFlags(flags: number): DeltaFlags {
1661	return {	1703	return {
1662	hidden: (flags & 1 << 0) !== 0,	1704	hidden: (flags & (1 << 0)) !== 0,
1663	passive: (flags & 1 << 1) !== 0,	1705	passive: (flags & (1 << 1)) !== 0,
1664	showTotal: (flags & 1 << 2) !== 0,	1706	showTotal: (flags & (1 << 2)) !== 0,
1665	changedValue: (flags & 1 << 3) !== 0,	1707	changedValue: (flags & (1 << 3)) !== 0,
1666	changedTotal: (flags & 1 << 4) !== 0,	1708	changedTotal: (flags & (1 << 4)) !== 0,
1667	changedLogs: (flags & 1 << 6) !== 0,	1709	changedLogs: (flags & (1 << 6)) !== 0,
1668	changedText: (flags & 1 << 7) !== 0,	1710	changedText: (flags & (1 << 7)) !== 0,
1669	changedChildren: (flags & 1 << 8) !== 0,	1711	changedChildren: (flags & (1 << 8)) !== 0,
1670	changedEstimatedTime: (flags & 1 << 9) !== 0,	1712	changedEstimatedTime: (flags & (1 << 9)) !== 0,
1671	changedFormattedValue: (flags & 1 << 10) !== 0,	1713	changedFormattedValue: (flags & (1 << 10)) !== 0,
1672	estimatedTimeNonNull: (flags & 1 << 11) !== 0,	1714	estimatedTimeNonNull: (flags & (1 << 11)) !== 0,
1673	futureProofingPacket: (flags & 1 << 14) !== 0,	1715	futureProofingPacket: (flags & (1 << 14)) !== 0,
1674	deleted: (flags & 1 << 15) !== 0,	1716	deleted: (flags & (1 << 15)) !== 0,
1675	};	1717	};
1676	}	1718	}
1677		1719
...	@@ -1731,9 +1773,10 @@ export class Ema implements EstimationAlgorithm {	...	@@ -1731,9 +1773,10 @@ export class Ema implements EstimationAlgorithm {
1731	if (elapsed === 0) return null; // too quick	1773	if (elapsed === 0) return null; // too quick
1732		1774
1733	const rate = current / elapsed;	1775	const rate = current / elapsed;
1734	this.estimate = this.estimate != null	1776	this.estimate =
1735	? (1 - this.alpha) * this.estimate + this.alpha * rate	1777	this.estimate != null
1736	: rate;	1778	? (1 - this.alpha) * this.estimate + this.alpha * rate
		1779	: rate;
1737	if (elapsed < 10_000 \|\| this.estimate <= 0 \|\| this.samples < 3) return null;	1780	if (elapsed < 10_000 \|\| this.estimate <= 0 \|\| this.samples < 3) return null;
1738	const remaining = total - current;	1781	const remaining = total - current;
1739	return time + remaining / this.estimate;	1782	return time + remaining / this.estimate;
...	@@ -1750,7 +1793,10 @@ type EncodedKey = number & { brand: typeof kNode };	...	@@ -1750,7 +1793,10 @@ type EncodedKey = number & { brand: typeof kNode };
1750		1793
1751	const globalKeyPool = new KeyPool<number>();	1794	const globalKeyPool = new KeyPool<number>();
1752		1795
1753	const global: Ref = /** @__PURE__ */ ((root = new Root()) => (attachToScreen(root, log), root))();	1796	const global: Ref = /** @__PURE__ */ ((root = new Root()) => (
		1797	attachToScreen(root, log),
		1798	root
		1799	))();
1754		1800
1755	/**	1801	/**
1756	* a {@linkcode Ref} to the global progress root. unlike referencing the	1802	* a {@linkcode Ref} to the global progress root. unlike referencing the

lib/readme.md+3-2

lib/ts.ts+27-17

 @@ -10,13 +10,19 @@ export type Timer = ReturnType<typeof setTimeout>;
 export type Interval = ReturnType<typeof setInterval>;
 /** opposite of the built-in `Readonly` type */
 export type Writeable<T> = { -readonly [P in keyof T]: T[P] };
+/** A type that can be `await`ed to return `T` */
+export type Awaitable<T> = T | PromiseLike<T>;
 /** opposite of the built-in `Readonly` type, but recursive */
 // TODO: tuples?
 export type DeepWriteable<T> = {
-  -readonly [P in keyof T]: T[P] extends ReadonlyArray<infer I> ? Array<I>
+  -readonly [P in keyof T]: T[P] extends ReadonlyArray<infer I>
+    ? Array<I>
     : DeepWriteable<T[P]>;
 };
+declare const brand: unique symbol;
+export type Brand<T> = { [brand]: T };
 /**
  * redeclared here because it is only provided by `lib: ["dom"]` and not
  * the JavaScript standard types.
 @@ -25,7 +31,7 @@ export interface VoidFunction {
   (): void;
 }
-/** Convenience type. Construct with `defer` */
+/** convenience type. Construct with `defer` */
 export interface Dispose extends Disposable, VoidFunction {}
 /**
 @@ -51,12 +57,10 @@ export function mixin<A extends object, B extends object>(
   other: B,
 ): Mixin<A, B> {
   const prototype = Object.getPrototypeOf(other);
-  for (
+  for (const key of [
-    const key of [
+    ...Object.getOwnPropertyNames(prototype),
-      ...Object.getOwnPropertyNames(prototype),
+    ...Object.getOwnPropertySymbols(prototype),
-      ...Object.getOwnPropertySymbols(prototype),
+  ]) {
-    ]
-  ) {
     if (key === "constructor") continue;
     if (key in base) continue;
     const fn = prototype[key];
 @@ -69,15 +73,21 @@ export function mixin<A extends object, B extends object>(
 export type EmptyObject = Record<never, unknown>;
 /** TODO: this type has many subtle bugs */
-export type ToJson<T> = T extends JsonValue ? T
+export type ToJson<T> = T extends JsonValue
-  : T extends { toJSON(): infer J } ? ToJson<J>
+  ? T
-  : T extends Set<unknown> | Map<unknown, unknown> | Record<string, never> ? EmptyObject
+  : T extends { toJSON(): infer J }
-  : T extends unknown[] ? {
+    ? ToJson<J>
-      [K in keyof Omit<T, keyof unknown[]>]: T[K] extends undefined | void ? null
+    : T extends Set<unknown> | Map<unknown, unknown> | Record<string, never>
-        : ToJson<T[K]>;
+      ? EmptyObject
-    }
+      : T extends unknown[]
-  : T extends object ? { [K in keyof Omit<T, symbol>]: ToJson<T[K]> }
+        ? {
-  : never;
+            [K in keyof Omit<T, keyof unknown[]>]: T[K] extends undefined | void
+              ? null
+              : ToJson<T[K]>;
+          }
+        : T extends object
+          ? { [K in keyof Omit<T, symbol>]: ToJson<T[K]> }
+          : never;
 export type Json = JsonValue | undefined | void;
 export type JsonValue =
   | string

old_stuff/hot.ts created+306

...	@@ -0,0 +1,306 @@
1	// This implements the ability to use TS, TSX, and more plugins
2	// in Node.js. It cannot be built on the ES module loader,
3	// because there is no exposed way to replace modules when
4	// needed (see nodejs#49442).
5	//
6	// It also allows using a simple compile cache, which is used by
7	// the site generator to determine when code changes.
8	export const projectRoot = path.resolve(import.meta.dirname, "../");
9	export const projectSrc = path.resolve(projectRoot, "src");
10
11	using _ = null;
12
13	// Create a project-relative require. For convenience, it is generic-typed.
14	export const load = createRequire(
15	pathToFileURL(path.join(projectRoot, "run.js")).toString(),
16	) as {
17	<T = unknown>(id: string): T;
18	extensions: NodeJS.Dict<(mod: NodeJS.Module, file: string) => unknown>;
19	cache: NodeJS.Dict<NodeJS.Module>;
20	resolve: (id: string, o?: { paths: string[] }) => string;
21	};
22	export const { cache } = load;
23
24	// Register extensions by overwriting `require.extensions`
25	const require = load;
26	const exts = require.extensions;
27	exts[".ts"] = loadEsbuild;
28	exts[".tsx"] = loadEsbuild;
29	exts[".jsx"] = loadEsbuild;
30	exts[".marko"] = loadMarko;
31	exts[".mdo"] = loadMarkodown;
32	exts[".css"] = loadCss;
33
34	// Intercept all module load calls to track CSS imports + file times.
35	export interface FileStat {
36	cssImportsRecursive: string[] \| null;
37	lastModified: number;
38	imports: string[];
39	}
40	const fileStats = new Map<string, FileStat>();
41
42	export function getFileStat(filepath: string) {
43	return fileStats.get(path.resolve(filepath));
44	}
45
46	function shouldTrackPath(filename: string) {
47	return !filename.includes("node_modules");
48	}
49
50	const Module = load<typeof import("node:module")>("node:module");
51	const ModulePrototypeUnderscoreCompile = Module.prototype._compile;
52	Module.prototype._compile = function(
53	content: string,
54	filename: string,
55	format: "module" \| "commonjs",
56	) {
57	const result = ModulePrototypeUnderscoreCompile.call(
58	this,
59	content,
60	filename,
61	format,
62	);
63	if (shouldTrackPath(filename)) {
64	const stat = fs.statSync(filename);
65	const cssImportsMaybe: string[] = [];
66	const imports: string[] = [];
67	for (const childModule of this.children) {
68	const { filename: file, cloverClientRefs } = childModule;
69	if (file.endsWith(".css")) cssImportsMaybe.push(file);
70	else {
71	const child = fileStats.get(file);
72	if (!child) continue;
73	const { cssImportsRecursive } = child;
74	if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
75	imports.push(file);
76	(childModule.cloverImporters ??= []).push(this);
77	if (cloverClientRefs && cloverClientRefs.length > 0) {
78	(this.cloverClientRefs ??= []).push(...cloverClientRefs);
79	}
80	}
81	}
82	fileStats.set(filename, {
83	cssImportsRecursive: cssImportsMaybe.length > 0
84	? Array.from(new Set(cssImportsMaybe))
85	: null,
86	imports,
87	lastModified: Math.floor(stat.mtimeMs),
88	});
89	}
90	return result;
91	};
92
93	function loadEsbuild(module: NodeJS.Module, filepath: string) {
94	return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
95	}
96
97	interface LoadOptions {
98	scannedClientRefs?: string[];
99	}
100	export function loadEsbuildCode(
101	module: NodeJS.Module,
102	filepath: string,
103	src: string,
104	opt: LoadOptions = {},
105	) {
106	if (filepath === import.meta.filename) {
107	module.exports = self;
108	return;
109	}
110	let loader: any = "tsx";
111	if (filepath.endsWith(".ts")) loader = "ts";
112	else if (filepath.endsWith(".jsx")) loader = "jsx";
113	else if (filepath.endsWith(".js")) loader = "js";
114	if (opt.scannedClientRefs) {
115	module.cloverClientRefs = opt.scannedClientRefs;
116	} else {
117	let { code, refs } = resolveClientRefs(src, filepath);
118	module.cloverClientRefs = refs;
119	src = code;
120	}
121	if (src.includes("import.meta")) {
122	src = `
123	import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
124	import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
125	import.meta.filename = ${JSON.stringify(filepath)};
126	import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r));
127	`
128	.trim()
129	.replace(/[\n\s]/g, "") + src;
130	}
131	src = esbuild.transformSync(src, {
132	loader,
133	format: "cjs",
134	target: "esnext",
135	jsx: "automatic",
136	jsxImportSource: "#jsx",
137	jsxDev: true,
138	sourcemap: "inline",
139	sourcefile: path.basename(filepath),
140	sourceRoot: path.dirname(filepath),
141	}).code;
142	return module._compile(src, filepath, "commonjs");
143	}
144
145	export function resolveClientRef(sourcePath: string, ref: string) {
146	const filePath = resolveFrom(sourcePath, ref);
147	if (!filePath.endsWith(".client.ts")) {
148	throw new Error("addScript must take a .client.ts");
149	}
150	return path.relative(projectSrc, filePath);
151	}
152
153	let lazyMarko: typeof import("./marko.ts") \| null = null;
154	function loadMarko(module: NodeJS.Module, filepath: string) {
155	lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
156	lazyMarko.loadMarko(module, filepath);
157	}
158	function loadMarkodown(module: NodeJS.Module, filepath: string) {
159	lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
160	lazyMarko.loadMarkodown(module, filepath);
161	}
162
163	function loadCss(module: NodeJS.Module, _filepath: string) {
164	module.exports = {};
165	}
166
167	export function reloadRecursive(filepath: string) {
168	filepath = path.resolve(filepath);
169	const existing = cache[filepath];
170	if (existing) deleteRecursiveInner(filepath, existing);
171	fileStats.clear();
172	return require(filepath);
173	}
174
175	export function unload(filepath: string) {
176	lazyMarko?.markoCache.delete(filepath);
177	filepath = path.resolve(filepath);
178	const module = cache[filepath];
179	if (!module) return;
180	delete cache[filepath];
181	for (const importer of module.cloverImporters ?? []) {
182	unload(importer.filename);
183	}
184	}
185
186	function deleteRecursiveInner(id: string, module: any) {
187	if (id.includes(path.sep + "node_modules" + path.sep)) {
188	return;
189	}
190	delete cache[id];
191	for (const child of module.children) {
192	if (child.filename.includes("/engine/")) return;
193	const existing = cache[child.filename];
194	if (existing === child) deleteRecursiveInner(child.filename, existing);
195	}
196	}
197
198	export function getCssImports(filepath: string) {
199	filepath = path.resolve(filepath);
200	if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
201	return fileStats.get(filepath)?.cssImportsRecursive ?? [];
202	}
203
204	export function getClientScriptRefs(filepath: string) {
205	filepath = path.resolve(filepath);
206	const module = require.cache[filepath];
207	if (!module) throw new Error(filepath + " was never loaded");
208	return module.cloverClientRefs ?? [];
209	}
210
211	export function getSourceCode(filepath: string) {
212	filepath = path.resolve(filepath);
213	const module = require.cache[filepath];
214	if (!module) throw new Error(filepath + " was never loaded");
215	if (!module.cloverSourceCode) {
216	throw new Error(filepath + " did not record source code");
217	}
218	return module.cloverSourceCode;
219	}
220
221	export function resolveFrom(src: string, dest: string) {
222	try {
223	return createRequire(src).resolve(dest);
224	} catch (err: any) {
225	if (
226	err.code === "MODULE_NOT_FOUND" && err.requireStack
227	&& err.requireStack.length <= 1
228	) {
229	err.message = err.message.split("\n")[0] + " from '" + src + "'";
230	}
231	throw err;
232	}
233	}
234
235	const importRegExp = /import\s+(\*\sas\s([a-zA-Z0-9$_]+)\|{[^}]+})\s+from\s+(?:"#sitegen"\|'#sitegen')/s;
236	const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))?/;
237	interface ResolvedClientRefs {
238	code: string;
239	refs: string[];
240	}
241	export function resolveClientRefs(
242	code: string,
243	filepath: string,
244	): ResolvedClientRefs {
245	// This match finds a call to 'import ... from "#sitegen"'
246	const importMatch = code.match(importRegExp);
247	if (!importMatch) return { code, refs: [] };
248	const items = UNWRAP(importMatch[1]);
249	let identifier = "";
250	if (items.startsWith("{")) {
251	const clauseMatch = items.match(getSitegenAddScriptRegExp);
252	if (!clauseMatch) return { code, refs: [] }; // did not import
253	identifier = clauseMatch[1] \|\| "addScript";
254	} else if (items.startsWith("*")) {
255	identifier = importMatch[2] + "\\s\\.\\saddScript";
256	} else {
257	throw new Error("Impossible");
258	}
259	identifier = identifier.replaceAll("$", "\\$"); // only needed escape
260	const findCallsRegExp = new RegExp(
261	`\\b(${identifier})\\s*\$("[^"]+"\|'[^']+')\$`,
262	"gs",
263	);
264	const scannedClientRefs = new Set<string>();
265	code = code.replace(findCallsRegExp, (_, call, arg) => {
266	const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
267	const resolved = resolveClientRef(filepath, ref);
268	scannedClientRefs.add(resolved);
269	return `${call}(${JSON.stringify(getScriptId(resolved))})`;
270	});
271	return { code, refs: Array.from(scannedClientRefs) };
272	}
273
274	export function getScriptId(file: string) {
275	return (
276	path.isAbsolute(file) ? path.relative(projectSrc, file) : file
277	).replaceAll("\\", "/");
278	}
279
280	declare global {
281	namespace NodeJS {
282	interface Module {
283	cloverClientRefs?: string[];
284	cloverSourceCode?: string;
285	cloverImporters?: Module[];
286
287	_compile(
288	this: NodeJS.Module,
289	content: string,
290	filepath: string,
291	format: "module" \| "commonjs",
292	): unknown;
293	}
294	}
295	}
296	declare module "node:module" {
297	export function _resolveFilename(id: string, parent: NodeJS.Module): unknown;
298	}
299
300	import * as fs from "#sitegen/fs";
301	import { UNWRAP } from "@clo/lib/assert";
302	import * as esbuild from "esbuild";
303	import { createRequire } from "node:module";
304	import * as path from "node:path";
305	import { pathToFileURL } from "node:url";
306	import * as self from "./hot.ts";

old_stuff/incremental.ts created+738

...	@@ -0,0 +1,738 @@
1	// Incremental build system using singleton state.
2	// See `work()`, `compile()`, and `invalidate()` for details.
3	//
4	// All state is serializable to allow recovering state across sessions.
5	// This library special-cases the asset map, but is otherwise agnostic.
6	let running = false;
7	let jobs = 0;
8	let newKeys = 0;
9	let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
10	let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
11	let works = new Map<string, Work>();
12	let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
13	let writes = new Map<string, FileWrite>();
14	let assets = new Map<string, Asset>(); // keyed by hash
15
16	export interface Ref<T> {
17	key: string;
18	/** This method is compatible with `await` syntax */
19	then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20	get value(): T \| null;
21	/** The progress node passed into the input */
22	progress: progress.Node \| progress.Ref;
23	}
24	type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
25
26	/**
27	* Declare and a unit of work. Return value is memoized and only rebuilt when
28	* inputs change. Inputs are tracked via the `io` interface, as well as a hash
29	* of the `input` param and caller source code. Outputs are written at the end
30	* of a compilation (see `compile`).
31	*
32	* Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33	*/
34	export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
35	export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
36	export function work<I, O>(
37	p: progress.Ref,
38	job: Job<I, O>,
39	input?: I,
40	): Ref<O> {
41	const source = JSON.stringify(UNWRAP(stack.capture()[0]));
42	const keySource = [source, util.inspect(input)].join(":");
43	const key = crypto.createHash("sha1").update(keySource).digest("base64url");
44	ASSERT(running);
45	ASSERT(
46	!seenWorks.has(key),
47	`Key '${key}' must be unique during the build. `
48	+ `To fix this, provide a manual 'key' argument.`,
49	);
50	seenWorks.add(key);
51
52	const prev = works.get(key) as Work<O> \| null;
53	if (prev) {
54	return {
55	key,
56	then: (done) => done(prev.value),
57	value: prev.value,
58	progress: p,
59	};
60	}
61
62	async function perform() {
63	const io = new Io(key, p);
64	jobs += 1;
65	newKeys += 1;
66	try {
67	const value = await job(io, input as I);
68	validateSerializable(value, "");
69	const { reads, writes } = io;
70	works.set(key, { value, affects: [], reads, writes });
71	for (const add of reads.files) {
72	const { affects } = UNWRAP(files.get(add));
73	ASSERT(!affects.includes(key));
74	affects.push(key);
75	}
76	for (const add of reads.works) {
77	const { affects } = UNWRAP(works.get(add));
78	ASSERT(!affects.includes(key));
79	affects.push(key);
80	}
81	return value;
82	} finally {
83	jobs -= 1;
84	}
85	}
86
87	let cached: Promise<O>;
88	return {
89	key,
90	then(fufill, reject) {
91	(cached ??= perform()).then(fufill, reject);
92	},
93	get value() {
94	return (works.get(this.key)?.value as O) ?? null;
95	},
96	progress: p,
97	};
98	}
99
100	export async function compile<T>(
101	compiler: (node: progress.Ref) => Promise<T>,
102	p: progress.Ref = progress,
103	) {
104	ASSERT(!running, `Cannot run twice at the same time.`);
105	const startDate = new Date();
106	const start = performance.now();
107	const node = p.start("sitegen!");
108	try {
109	running = true;
110	ASSERT(jobs === 0);
111
112	const value = await compiler(node);
113	ASSERT(jobs === 0);
114	await flush(node, start);
115	return {
116	value,
117	watchFiles: new Set(files.keys()),
118	newOutputs: Array.from(seenWrites)
119	.filter((x) => x.startsWith("f:"))
120	.map((x) => x.slice(2)),
121	newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
122	};
123	} finally {
124	node.text = `sitegen at ${startDate.toLocaleDateString()} ${startDate.toLocaleTimeString()}`;
125	node.end();
126	running = false;
127	newKeys = 0;
128	seenWrites.clear();
129	seenWorks.clear();
130	}
131	}
132
133	export async function flush(progress: progress.Node, start: number) {
134	using _ = progress.start("incremental flush");
135	// Trim
136	const detachedFiles = new Set<string>();
137	const referencedAssets = new Set<string>();
138	for (const [k, v] of works) {
139	const assets = v.writes.assets;
140	if (seenWorks.has(k)) {
141	for (const asset of assets.values()) referencedAssets.add(asset.hash);
142	continue;
143	}
144	deleteWork(k);
145	}
146	for (const [k, file] of files) {
147	if (file.affects.length > 0) continue;
148	files.delete(k);
149	detachedFiles.add(k);
150	}
151	for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
152
153	const p: Promise<void>[] = [];
154	// File writes
155	let dist = 0;
156	for (const [key, { buffer, size }] of writes) {
157	if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
158	dist += size;
159	}
160	// Asset map
161	{
162	const { json, blob, dynamic, dts } = getAssetManifest();
163	const jsonString = Buffer.from(JSON.stringify(json));
164	p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
165	p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
166	p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
167	for (const [k, v] of dynamic) {
168	p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
169	}
170	dist += blob.byteLength + jsonString.byteLength;
171	}
172	await Promise.all(p);
173
174	// Incremental state
175	const serialized = msgpackr.pack(serialize());
176	await fs.writeMkdir(".clover/incr.state", serialized);
177	const time = (performance.now() - start).toFixed(0);
178	progress.log.info(
179	`sitegen! in ${time} ms\n`
180	+ ` - ${works.size} keys (${works.size - newKeys} cached)\n`
181	+ ` - ${assets.size} static assets\n`
182	+ ` - dist: ${formatSize(dist)},`
183	+ ` incremental: ${formatSize(serialized.byteLength)}`,
184	);
185	}
186
187	export async function restore() {
188	let buffer;
189	try {
190	buffer = await fs.readFile(".clover/incr.state");
191	} catch (err: any) {
192	if (err.code !== "ENOENT") throw err;
193	}
194	if (!buffer) return;
195	await deserialize(buffer);
196	}
197
198	function forceInvalidate(entry: { affects: string[] }) {
199	const queue = [...entry.affects];
200	let key;
201	while ((key = queue.shift())) {
202	const affects = deleteWork(key);
203	queue.push(...affects);
204	}
205	}
206
207	function deleteWork(key: string) {
208	const work = works.get(key);
209	if (!work) return [];
210	const { reads, affects, writes: w } = work;
211	for (const remove of reads.files) {
212	const { affects } = UNWRAP(files.get(remove));
213	ASSERT(affects.includes(key));
214	affects.splice(affects.indexOf(key), 1);
215	}
216	for (const remove of reads.works) {
217	const { affects } = UNWRAP(works.get(remove), remove);
218	ASSERT(affects.includes(key));
219	affects.splice(affects.indexOf(key), 1);
220	}
221	for (const remove of affects) {
222	const {
223	reads: { works: list },
224	} = UNWRAP(works.get(remove), remove);
225	ASSERT(list.has(key));
226	list.delete(key);
227	}
228	for (const file of w.files) {
229	if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
230	}
231	// Assets are temporarily kept, trimmed via manual GC after compilation.
232
233	works.delete(key);
234	return affects;
235	}
236
237	export function reset() {
238	ASSERT(!running);
239	works.clear();
240	files.clear();
241	assets.clear();
242	}
243
244	export function serialize() {
245	const fileEntries = Array.from(
246	files,
247	([k, v]) =>
248	[
249	k,
250	v.type,
251	v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
252	...v.affects,
253	] as const,
254	);
255	const workEntries = Array.from(
256	works,
257	([k, v]) =>
258	[
259	k,
260	v.value,
261	Array.from(v.reads.files),
262	Array.from(v.reads.works),
263	Array.from(v.writes.files),
264	Array.from(
265	v.writes.assets,
266	([k, { headers }]) => [k, headers] as const,
267	),
268	v.affects,
269	] as const,
270	);
271	const expectedFilesOnDisk = Array.from(
272	writes,
273	([k, { size, work }]) => [k, size, work] as const,
274	);
275	const assetEntries = Array.from(
276	assets,
277	([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
278	);
279	return [
280	1,
281	fileEntries,
282	workEntries,
283	expectedFilesOnDisk,
284	assetEntries,
285	] as const;
286	}
287	type SerializedState = ReturnType<typeof serialize>;
288
289	/* No-op on failure */
290	async function deserialize(buffer: Buffer) {
291	const decoded = msgpackr.decode(buffer) as SerializedState;
292	if (!Array.isArray(decoded)) return;
293	if (decoded[0] !== 1) return;
294	const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] = decoded;
295	for (const [k, type, content, ...affects] of fileEntries) {
296	if (type === "f") {
297	ASSERT(typeof content === "number");
298	files.set(k, { type, affects, lastModified: content });
299	} else if (type === "d") {
300	ASSERT(typeof content === "string");
301	files.set(k, { type, affects, contentHash: content, contents: [] });
302	} else {
303	files.set(k, { type, affects });
304	}
305	}
306	for (const entry of workEntries) {
307	const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] = entry;
308	works.set(k, {
309	value,
310	reads: {
311	files: new Set(readFiles),
312	works: new Set(readWorks),
313	},
314	writes: {
315	files: new Set(writeFiles),
316	assets: new Map(
317	Array.from(writeAssets, ([k, headers]) => [
318	k,
319	{
320	hash: JSON.parse(UNWRAP(headers.etag)),
321	headers,
322	},
323	]),
324	),
325	},
326	affects,
327	});
328	}
329	const statFiles = await Promise.all(
330	expectedFilesOnDisk.map(([k, size, work]) =>
331	fs
332	.stat(path.join(".clover/o", k))
333	.catch((err) => {
334	if (err.code === "ENOENT") return null;
335	throw err;
336	})
337	.then((stat) => ({ k, size, work, stat }))
338	),
339	);
340	for (const { k, stat, work, size } of statFiles) {
341	if (stat?.size === size) {
342	writes.set(k, {
343	size: size,
344	buffer: null,
345	work,
346	});
347	} else {
348	forceInvalidate({ affects: [work] });
349	}
350	}
351	for (const [hash, raw, gzip, zstd] of assetEntries) {
352	assets.set(hash, { raw, gzip, zstd });
353	}
354
355	await Promise.all(
356	Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
357	);
358	}
359
360	export async function invalidate(
361	filePath: string,
362	unloadModule: boolean = true,
363	): Promise<boolean> {
364	const key = toRel(toAbs(filePath));
365	const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
366	return invalidateEntry(key, file, unloadModule);
367	}
368
369	export async function invalidateEntry(
370	key: string,
371	file: TrackedFile,
372	unloadModule: boolean,
373	): Promise<boolean> {
374	try {
375	if (file.type === "d") {
376	const contents = (file.contents = await fs.readdir(key));
377	contents.sort();
378	const contentHash = crypto
379	.createHash("sha1")
380	.update(contents.join("\0"))
381	.digest("base64url");
382	if (file.contentHash !== contentHash) {
383	file.contentHash = contentHash;
384	throw new Error();
385	}
386	} else if (file.type === "f") {
387	const lastModified = await fs.stat(key).then(
388	(x) => Math.floor(x.mtimeMs),
389	() => 0,
390	);
391	if (file.lastModified !== lastModified) {
392	file.lastModified = lastModified;
393	throw new Error();
394	}
395	} else {
396	file.type satisfies "null";
397	const stat = await fs.stat(key).catch(() => null);
398	if (stat) throw new Error();
399	}
400	return false;
401	} catch (e) {
402	forceInvalidate(file);
403	if (unloadModule) {
404	// TODO: handle when this triggers unloading of `generate.ts`
405	hot.unload(toAbs(key));
406	}
407	if (file.type === "null") files.delete(key);
408	return true;
409	}
410	}
411
412	export function getAssetManifest() {
413	const dynamic = new Map<string, Buffer>();
414	const writer = new BufferWriter();
415	const assetMap = Object.fromEntries(
416	Array.from(works, (work) => work[1].writes.assets)
417	.filter((map) => map.size > 0)
418	.flatMap((map) =>
419	Array.from(map, ([key, { hash, headers, regenerative }]) => {
420	const { raw, gzip, zstd } = UNWRAP(
421	assets.get(hash),
422	`Asset ${key} (${hash})`,
423	);
424	if (regenerative) {
425	const id = crypto
426	.createHash("sha1")
427	.update(key)
428	.digest("hex")
429	.slice(0, 16); /* TODO */
430	dynamic.set(
431	id,
432	manifest.packDynamicBuffer(raw, gzip, zstd, headers),
433	);
434	return [key, { type: 1, id }] as const;
435	}
436	return [
437	key,
438	{
439	type: 0,
440	raw: writer.write(raw, "raw:" + hash),
441	gzip: writer.write(gzip, "gzip:" + hash),
442	zstd: writer.write(zstd, "zstd:" + hash),
443	headers,
444	},
445	] as const;
446	})
447	),
448	) satisfies manifest.Manifest;
449	return {
450	json: assetMap,
451	blob: writer.get(),
452	dynamic,
453	dts: "export type AssetKey = "
454	+ Object.keys(assetMap)
455	.map((key) => JSON.stringify(key))
456	.join(" \| ")
457	+ "\n",
458	};
459	}
460
461	/* Input/Output with automatic tracking.
462	* - Inputs read with Io are tracked to know when to rebuild
463	* - Outputs written with Io are deleted when abandoned.
464	*/
465	export class Io {
466	constructor(public key: string, private node: progress.Ref) {}
467	reads: Reads = { files: new Set(), works: new Set() };
468	writes: Writes = { files: new Set(), assets: new Map() };
469
470	start(text: string, options?: progress.StartOptions) {
471	return this.node.start(text, options);
472	}
473
474	#trackFs(file: string) {
475	const resolved = toAbs(file);
476	const key = toRel(resolved);
477	this.reads.files.add(key);
478	return { resolved, key };
479	}
480	async readWork<T>(ref: Ref<T>): Promise<T> {
481	this.reads.works.add(ref.key);
482	return await ref;
483	}
484	/** Track a file in the compilation without reading it. */
485	async trackFile(file: string) {
486	const { key, resolved } = this.#trackFs(file);
487	if (!files.get(key)) {
488	let lastModified: number = 0;
489	try {
490	lastModified = Math.floor((await fs.stat(file)).mtimeMs);
491	files.set(key, { type: "f", lastModified, affects: [] });
492	} catch {
493	files.set(key, { type: "null", affects: [] });
494	}
495	}
496	return resolved;
497	}
498	async readFile(file: string): Promise<Buffer>;
499	async readFile(file: string, encoding: "utf-8"): Promise<string>;
500	async readFile(file: string, encoding?: "utf-8") {
501	return fs.readFile(await this.trackFile(file), encoding);
502	}
503	async readJson<T>(file: string) {
504	return JSON.parse(await this.readFile(file, "utf-8")) as T;
505	}
506	async readDir(dir: string) {
507	const { key, resolved } = this.#trackFs(dir);
508	const existing = files.get(key);
509	try {
510	if (existing?.type === "d") return existing.contents;
511	const contents = await fs.readdir(resolved);
512	contents.sort();
513	const contentHash = crypto
514	.createHash("sha1")
515	.update(contents.join("\0"))
516	.digest("base64url");
517	files.set(key, {
518	type: "d",
519	affects: [],
520	contentHash,
521	contents,
522	});
523	return contents;
524	} catch (err) {
525	if (!existing) files.set(key, { type: "null", affects: [] });
526	throw err;
527	}
528	}
529	async readDirRecursive(dir: string): Promise<string[]> {
530	const dirs = await this.readDir(dir);
531	return (
532	await Promise.all(
533	dirs.map(async (child) => {
534	const abs = path.join(dir, child);
535	const stat = await fs.stat(abs);
536	if (stat.isDirectory()) {
537	return (await this.readDirRecursive(abs)).map((grand) => path.join(child, grand));
538	} else {
539	return child;
540	}
541	}),
542	)
543	).flat();
544	}
545	/* Track all dependencies of a module. */
546	async import<T>(file: string): Promise<T> {
547	const { resolved } = this.#trackFs(file);
548	try {
549	return require(resolved) as T;
550	} finally {
551	const queue = [resolved];
552	const seen = new Set<string>();
553	let current;
554	while ((current = queue.shift())) {
555	const stat = hot.getFileStat(current);
556	if (!stat) continue;
557	const { key } = this.#trackFs(current);
558	if (!files.get(key)) {
559	files.set(key, {
560	type: "f",
561	affects: [],
562	lastModified: stat?.lastModified ?? 0,
563	});
564	}
565	for (const imp of stat.imports) {
566	if (!seen.has(imp)) {
567	seen.add(imp);
568	queue.push(imp);
569	}
570	}
571	}
572	}
573	}
574	async writeAsset(asset: {
575	pathname: string;
576	buffer: string \| Uint8Array;
577	regenerative?: boolean;
578	headers?: HeadersInit;
579	}) {
580	ASSERT(asset.pathname.startsWith("/"));
581	ASSERT(!seenWrites.has("a:" + asset.pathname));
582	const buffer = typeof asset.buffer === "string"
583	? Buffer.from(asset.buffer)
584	: asset.buffer;
585	const headers = new Headers(asset.headers ?? {});
586	const hash = crypto.createHash("sha1").update(buffer).digest("hex");
587	if (!headers.has("Content-Type")) {
588	headers.set("Content-Type", mime.for(asset.pathname));
589	}
590	headers.set("etag", JSON.stringify(hash));
591	this.writes.assets.set(asset.pathname, {
592	hash,
593	headers: Object.fromEntries(headers),
594	regenerative: !!asset.regenerative,
595	});
596	if (!assets.has(hash)) {
597	jobs += 1;
598	assets.set(hash, undefined!);
599	const [gzipBuffer, zstdBuffer] = await Promise.all([
600	gzip(buffer),
601	zstdCompress(buffer),
602	]);
603	assets.set(hash, {
604	raw: buffer,
605	gzip: gzipBuffer,
606	zstd: zstdBuffer,
607	});
608	jobs -= 1;
609	}
610	}
611	writeFile(subPath: string, blob: string \| Buffer) {
612	ASSERT(!subPath.startsWith("/"));
613	ASSERT(
614	!seenWrites.has("f:" + subPath),
615	`File overwritten: ${JSON.stringify(subPath)}`,
616	);
617	seenWrites.add("f:" + subPath);
618	const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
619	writes.set(subPath, {
620	buffer,
621	size: buffer.byteLength,
622	work: this.key,
623	});
624	}
625	}
626
627	class BufferWriter {
628	size = 0;
629	seen = new Map<string, BufferView>();
630	buffers: Buffer[] = [];
631
632	write(buffer: Buffer, hash: string): BufferView {
633	let view = this.seen.get(hash);
634	if (view) return view;
635	view = [this.size, this.size += buffer.byteLength];
636	this.seen.set(hash, view);
637	this.buffers.push(buffer);
638	return view;
639	}
640
641	get() {
642	return Buffer.concat(this.buffers);
643	}
644	}
645
646	export function validateSerializable(value: unknown, key: string) {
647	if (typeof value === "string") {
648	if (value.includes(hot.projectRoot)) {
649	throw new Error(
650	`Return value must not contain the CWD for portability, found at ${key}`,
651	);
652	}
653	} else if (value && typeof value === "object") {
654	if (Array.isArray(value)) {
655	value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
656	} else if (
657	Object.getPrototypeOf(value) === Object.prototype
658	\|\| Buffer.isBuffer(value)
659	) {
660	Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`));
661	} else {
662	throw new Error(
663	`Return value must be a plain JS object, found ${Object.getPrototypeOf(value).constructor.name} at ${key}`,
664	);
665	}
666	} else if (["bigint", "function", "symbol"].includes(typeof value)) {
667	throw new Error(
668	`Return value must be a plain JS object, found ${typeof value} at ${key}`,
669	);
670	}
671	}
672
673	export function toAbs(absPath: string) {
674	return path.resolve(hot.projectRoot, absPath);
675	}
676
677	export function toRel(absPath: string) {
678	return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
679	}
680
681	export type BufferView = [start: number, end: number];
682	interface Reads {
683	files: Set<string>;
684	works: Set<string>;
685	}
686	interface FileWrite {
687	buffer: Buffer \| null;
688	size: number;
689	work: string;
690	}
691	interface Writes {
692	files: Set<string>;
693	assets: Map<string, AssetWrite>;
694	}
695	interface Asset {
696	raw: Buffer;
697	gzip: Buffer;
698	zstd: Buffer;
699	}
700	interface AssetWrite {
701	hash: string;
702	headers: Record<string, string>;
703	regenerative: boolean;
704	}
705	interface Work<T = unknown> {
706	value: T;
707	reads: Reads;
708	writes: Writes;
709	affects: string[];
710	}
711	type TrackedFile =
712	& { affects: string[] }
713	& (
714	\| { type: "f"; lastModified: number }
715	\| { type: "d"; contentHash: string; contents: string[] }
716	\| { type: "null" }
717	);
718
719	const gzip = util.promisify(zlib.gzip);
720	const zstdCompress = util.promisify(zlib.zstdCompress);
721
722	import * as hot from "./hot.ts";
723
724	import * as manifest from "#sitegen/assets";
725	import * as fs from "#sitegen/fs";
726	import * as mime from "@clo/lib/mime";
727
728	import * as crypto from "node:crypto";
729	import * as path from "node:path";
730	import * as util from "node:util";
731	import * as zlib from "node:zlib";
732
733	import { formatSize } from "#src/file-viewer/format.ts";
734	import * as stack from "@clo/lib/log/stack";
735	import * as progress from "@clo/lib/progress";
736
737	import { ASSERT, UNWRAP } from "@clo/lib/assert";
738	import * as msgpackr from "msgpackr";

package-lock.json+1-5

 @@ -29,11 +29,7 @@
       }
     },
     "lib": {
-      "name": "@clo/lib",
+      "name": "@clo/lib"
-      "dependencies": {
-        "@clo/lib": "file:.",
-        "@types/node": "^24.0.0"
-      }
     },
     "node_modules/@clo/lib": {
       "resolved": "lib",

playground.ts created+19

...	@@ -0,0 +1,19 @@
1	import * as incremental from "./lib/incremental.ts";
2	import * as progress from "./lib/progress.ts";
3
4	const g = new incremental.Graph({ db: "" });
5	g.incrementalBuild({
6	async buildAndWriteOutput(ref) {
7	using node = ref.start("sitegen!");
8
9	const y = node.cache("meow", async (t) => {
10	const meow = JSON.parse(
11	await t.fs.readFile("package.json"),
12	) as typeof import("./package.json");
13	return meow.imports;
14	});
15
16	console.log({ y });
17	},
18	progress,
19	});

run.js+88-137

...	@@ -1,152 +1,103 @@	...	@@ -1,152 +1,103 @@
1	#!/usr/bin/env node	1	#!/usr/bin/env node
2	// This file allows using Node.js in combination with	2	// This file allows using Node.js in combination with
3	// all available plugins. Usage: "node run <script>"	3	// all available plugins. Usage: "node run <script>"
4	import * as child_process from "node:child_process";	4	import crypto from "node:crypto";
5	import * as crypto from "node:crypto";	5	import fs from "node:fs";
6	import * as fs from "node:fs";	6	import path from "node:path";
7	import * as os from "node:os";
8	import * as path from "node:path";
9	import process from "node:process";	7	import process from "node:process";
10	import * as url from "node:url";	8	import url from "node:url";
11	import * as zlib from "node:zlib";
12
13	// Disable experimental warnings (Type Stripping, etc)
14	const { emit: originalEmit } = process;
15	const warnings = ["ExperimentalWarning"];
16	process.emit = function(event, error) {
17	return event === "warning" && warnings.includes(error.name)
18	? false
19	: originalEmit.apply(process, arguments);
20	};
21
22	// Auto-install with npm if needed.
23	{
24	const rootDir = path.dirname(url.fileURLToPath(import.meta.url));
25	const cloverDir = path.join(rootDir, ".clover");
26	fs.mkdirSync(cloverDir, { recursive: true });
27	function computeHash() {
28	const hash = crypto.createHash("sha1");
29	hash.update("npm");
30	hash.update(fs.readFileSync(path.join(rootDir, "package.json")));
31	hash.update(fs.readFileSync(path.join(rootDir, "package-lock.json")));
32	hash.update(fs.readdirSync(path.join(rootDir, "node_modules")).join("\0"));
33	return hash.digest("hex");
34	}
35	try {
36	const expected = fs.readFileSync(path.join(cloverDir, "deps-hash.txt"), "utf-8");
37	const actual = computeHash();
38	if (expected !== actual) throw new Error();
39	} catch {
40	child_process.execSync("npm i --no-audit --no-fund", { stdio: "inherit" });
41	console.info();
42	fs.writeFileSync(path.join(cloverDir, "deps-hash.txt"), computeHash());
43	}
44	}
45		9
46	let hot;	10	/** @type {import('@clo/lib/incremental').Graph \| null} */
47	try {	11	export let graph = null;
48	// zstd is not supported in node 22
49	await new Promise((resolve) => zlib.zstdCompress(new Uint8Array(), resolve));
50
51	// seems to fail in bun, run it earlier to catch
52	(await import("source-map-support")).install({ hookRequire: true });
53
54	// init hooks. some runtimes have bugs here. and long term this system should be rebuilt
55	hot = await import("./framework/hot.ts");
56	hot.load("./lib/log.ts");
57	} catch (e) {
58	const brand = process.versions.bun
59	? `bun ${process.versions.bun}`
60	: process.versions.deno
61	? `deno ${process.versions.deno}`
62	: null;
63	const brandText = brand ? ` (${brand})` : "";
64	globalThis.console.error(
65	`sitegen depends on a node.js v24. your runtime is missing features.\n`
66	+ `this is node.js version ${process.version}${brandText}\n\n`
67	+ `get node.js --> https://nodejs.org/en/download/current\n`,
68	);
69	globalThis.console.error(e);
70	process.exit(1);
71	await new Promise(() => {});
72	}
73		12
74	const log = hot.load("./lib/log.ts");	13	// parse cli options
75	console.info = log.info;	14	if (import.meta.main) {
76	console.warn = log.warn;	15	// TODO: re-launch process with --experimental-vm-modules --no-warnings
77	console.error = log.error;
78	console.debug = log.debug;
79	console["log"] = log.log;
80
81	process.on("uncaughtException", (error) => {
82	console.error("Uncaught Exception");
83	console.error(error);
84	process.exit(1);
85	});
86
87	if (fs.existsSync(".env")) {
88	const env = fs.readFileSync(".env", "utf-8")
89	.split("\n").filter(Boolean)
90	.map((x) => x.trim().split("="));
91	for (const [k, v] of env) process.env[k] = v;
92	}
93		16
94	// Load with hooks	17	// disable experimental warnings (Type Stripping, etc)
95	if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {	18	const { emit: originalEmit } = process;
96	if (process.argv.length == 2) {	19	const warnings = ["ExperimentalWarning"];
97	log.error("usage: node run <script> [...args]");	20	process.emit = function(event, error) {
98	process.exit(1);	21	return event === "warning" && warnings.includes(error.name)
99	}	22	? false
100	let found;	23	: originalEmit.apply(process, arguments);
101	for (const dir of fs.globSync("{lib,{src,framework},{src,lib}/**/bin}")) {	24	};
		25
		26	// auto-install packages with npm if needed.
		27	{
		28	const rootDir = path.dirname(url.fileURLToPath(import.meta.url));
		29	const cloverDir = path.join(rootDir, ".clover");
		30	fs.mkdirSync(cloverDir, { recursive: true });
		31	function computeHash() {
		32	const hash = crypto.createHash("sha1");
		33	hash.update("npm");
		34	hash.update(fs.readFileSync(path.join(rootDir, "package.json")));
		35	hash.update(fs.readFileSync(path.join(rootDir, "package-lock.json")));
		36	hash.update(fs.readdirSync(path.join(rootDir, "node_modules")).join("\0"));
		37	return hash.digest("hex");
		38	}
102	try {	39	try {
103	found = hot.resolveFrom(	40	const expected = fs.readFileSync(path.join(cloverDir, "deps-hash.txt"), "utf-8");
104	import.meta.filename,	41	const actual = computeHash();
105	`./${dir}/${process.argv[2]}`,	42	if (expected !== actual) throw new Error();
106	);	43	} catch {
107	break;	44	child_process.execSync("npm i --no-audit --no-fund", { stdio: "inherit" });
108	} catch (e) {	45	console.info();
109	continue;	46	fs.writeFileSync(path.join(cloverDir, "deps-hash.txt"), computeHash());
110	}	47	}
111	}	48	}
112	if (!found) {	49
113	log.error("Cannot find script: " + process.argv[2]);	50
		51	if (process.argv.length == 2) {
		52	log.error("usage: node run <script> [...args]");
114	process.exit(1);	53	process.exit(1);
115	}	54	}
		55
		56	// load system
		57	const incremental = await import("@clo/lib/incremental");
		58	const log = await import("@clo/lib/log");
		59	const ansi = await import("@clo/lib/string/ansi");
		60	const progress = await import("@clo/lib/progress");
		61	const stack = await import("@clo/lib/log/stack");
		62
		63	process.on("uncaughtException", (error) => {
		64	log.writeOutput(ansi.style(ansi.fgBlack + ansi.bgRed, " UNCAUGHT ") + ' ' + stack.format(error, process.stderr.isTTY) + '\n');
		65	process.exit(1);
		66	});
		67
		68	console.info = log.info;
		69	console.warn = log.warn;
		70	console.error = log.error;
		71	console.debug = log.debug;
		72	console.log = log.log;
		73
		74	// resolve entrypoint
		75	const mainFileRaw = process.argv[1];
		76	const mainFile = "wow";
116	process.argv = [process.argv[0], ...process.argv.slice(2)];	77	process.argv = [process.argv[0], ...process.argv.slice(2)];
117	try {	78
118	const mod = await hot.load(found);	79	// revive and load
119	if (mod.main) mod.main();	80
120	else if (mod.default?.fetch) {	81	const markoLoader = incremental.lazyLoader(() => import('./framework/marko'));
121	const protocol = "http";	82
122	const { serve } = hot.load("@hono/node-server");	83	graph = await incremental.Graph.revive({
123	serve(	84	file: path.resolve(".clover/incremental.db"),
124	{	85	progress,
125	fetch: mod.default.fetch,	86
126	port: mod.port ?? 3000,	87	loaders: {
127	},	88	'.marko': markoLoader,
128	({ address, port }) => {	89	'.mdo': markoLoader,
129	if (address === "::" \|\| address === "127.0.0.1") {	90	},
130	address = os.hostname().replace(/\.local$/, "");	91	});
131	}	92
132	log.info(	93	const module = await graph.import(mainFile, { main: true });
133	url.format({	94
134	protocol,	95	if (typeof (module?.default?.fetch ?? module?.default?.respond) === 'function') {
135	hostname: address,	96	const http = await import('@clo/lib/http');
136	port,	97	const { url } = await http.serve({
137	}),	98	respond: module?.default?.fetch ?? module?.default?.respond,
138	);	99	port: 3000,
139	},	100	});
140	);	101	log.info(`-> ${url}`);
141	} else if (!found.match(/\.test\.[tj]sx?$/)) {
142	log.warn(
143	hot.load("path").relative(".", found),
144	"does not export a \"main\" function",
145	);
146	}
147	} catch (e) {
148	log.error(e);
149	}	102	}
150	}	103	}
151
152	export { hot };

tsconfig.json+1-2

...	@@ -1,7 +1,6 @@	...	@@ -1,7 +1,6 @@
1	{	1	{
2	"compilerOptions": {	2	"compilerOptions": {
3	"allowImportingTsExtensions": true,	3	"allowImportingTsExtensions": true,
4	"baseUrl": ".",
5	"incremental": true,	4	"incremental": true,
6	"jsx": "react-jsxdev",	5	"jsx": "react-jsxdev",
7	"jsxImportSource": "#jsx",	6	"jsxImportSource": "#jsx",
...	@@ -16,7 +15,7 @@	...	@@ -16,7 +15,7 @@
16	"target": "esnext",	15	"target": "esnext",
17	"noUncheckedIndexedAccess": true,	16	"noUncheckedIndexedAccess": true,
18	"resolveJsonModule": true,	17	"resolveJsonModule": true,
19	"types": ["marko", "node"]	18	"types": ["node", "marko"]
20	},	19	},
21	"include": ["./lib/*/", "src/*/", "framework/*/"]	20	"include": ["./lib/*/", "src/*/", "framework/*/"]
22	}	21	}

...	@@ -19,11 +19,12 @@ depend on each other. there are no external dependencies; code is easy to audit.	...	@@ -19,11 +19,12 @@ depend on each other. there are no external dependencies; code is easy to audit.
19	- `bytes` - `Uint8Array` helpers	19	- `bytes` - `Uint8Array` helpers
20	- `error` - interop with the `unknown` type in error handlers	20	- `error` - interop with the `unknown` type in error handlers
21	- `Events` - typed event emitter	21	- `Events` - typed event emitter
		22	- `incremental` - for build tools, tracable `node:vm`, job caching
22	- `log` - logging, terminal i/o, interactive widgets	23	- `log` - logging, terminal i/o, interactive widgets
23	- `log/stack` - stack trace parse and formatter	24	- `log/stack` - stack trace parse and formatter
24	- `Lru` - least recently used cache	25	- `Lru` - least recently used cache
25	- `mime` - trimmed database of ext -> mime types	26	- `mime` - trimmed database of ext -> mime types
26	- `node` - the Node.js javascript runtime	27	- `node` - tree-shakable loaders for Node.js builtins
27	- `progress` - cli progress bars, nesting status indication	28	- `progress` - cli progress bars, nesting status indication
28	- `queue` - priority queue with rescheduling	29	- `queue` - priority queue with rescheduling
29	- `stream` - ReadableStream helpers	30	- `stream` - ReadableStream helpers
...	@@ -35,7 +36,7 @@ depend on each other. there are no external dependencies; code is easy to audit.	...	@@ -35,7 +36,7 @@ depend on each other. there are no external dependencies; code is easy to audit.
35	- `ts` typescript types and general purpose code	36	- `ts` typescript types and general purpose code
36		37
37	there is currently no hosted documentation site. please refer to the source	38	there is currently no hosted documentation site. please refer to the source
38	files' comments.	39	files' comments and the JSR generated documentation.
39		40
40	## usage	41	## usage
41		42