authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-03-22 23:36:20-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-05-28 04:15:54-07:00
log6a4e985a960dc0c1fcf7776082292b717d25a6d8
tree5fa61e3cfe8b9a38a6ec90144cf44c13c7ab4031
parent907e9083b681531d3be2f8ba0cafc63dede6e61d
signature Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

WIP: feat(lib): Incremental Module Loader API

been playing with `node:vm`'s experimental modules. it is neat because it gives us a hook into the module system that no other runtime has: - able to reflect what files are pointed to by each file - can adjust how files are processed before loading - can unload/reload modules, with old ones being garbage collected this commit rebuilds `hot.ts` and `incremental.ts` as a new system that only depends on these node builtins, which (TODO) provides: - simple module loading `new Incremental().import("src/main.ts");` - plugin system to load extensions like `.tsx`, `.marko`, `.svelte` - tracing the files used in resolution and initialization - allowing arbitrary dependencies to be added - allowing arbitrary jobs that cache when re-run - creating a filesystem watcher to detect changes - performing trivial module reloading (maybe full HMR) - serializing the graph state to disk, along with v8 cache data by implementing this, clover sitegen can: - stop patching node.js internals - drop the server-side esbuild code (frontend still needs it) - use real es modules - provide a base for more features (issue 4 and 5), which could possibly be solved as their own top-level projects. this is also a great time to solve issue 27 (incremntal tracking)

24 files changed, 4335 insertions(+), 1543 deletions(-)

framework/hot.ts deleted-307
......@@ -1,307 +0,0 @@
1// This implements the ability to use TS, TSX, and more plugins
2// in Node.js. It cannot be built on the ES module loader,
3// because there is no exposed way to replace modules when
4// needed (see nodejs#49442).
5//
6// It also allows using a simple compile cache, which is used by
7// the site generator to determine when code changes.
8export const projectRoot = path.resolve(import.meta.dirname, "../");
9export const projectSrc = path.resolve(projectRoot, "src");
10
11using _ = null;
12
13// Create a project-relative require. For convenience, it is generic-typed.
14export const load = createRequire(
15 pathToFileURL(path.join(projectRoot, "run.js")).toString(),
16) as {
17 <T = unknown>(id: string): T;
18 extensions: NodeJS.Dict<(mod: NodeJS.Module, file: string) => unknown>;
19 cache: NodeJS.Dict<NodeJS.Module>;
20 resolve: (id: string, o?: { paths: string[] }) => string;
21};
22export const { cache } = load;
23
24// Register extensions by overwriting `require.extensions`
25const require = load;
26const exts = require.extensions;
27exts[".ts"] = loadEsbuild;
28exts[".tsx"] = loadEsbuild;
29exts[".jsx"] = loadEsbuild;
30exts[".marko"] = loadMarko;
31exts[".mdo"] = loadMarkodown;
32exts[".css"] = loadCss;
33
34// Intercept all module load calls to track CSS imports + file times.
35export interface FileStat {
36 cssImportsRecursive: string[] | null;
37 lastModified: number;
38 imports: string[];
39}
40const fileStats = new Map<string, FileStat>();
41
42export function getFileStat(filepath: string) {
43 return fileStats.get(path.resolve(filepath));
44}
45
46function shouldTrackPath(filename: string) {
47 return !filename.includes("node_modules");
48}
49
50const Module = load<typeof import("node:module")>("node:module");
51const ModulePrototypeUnderscoreCompile = Module.prototype._compile;
52Module.prototype._compile = function(
53 content: string,
54 filename: string,
55 format: "module" | "commonjs",
56) {
57 const result = ModulePrototypeUnderscoreCompile.call(
58 this,
59 content,
60 filename,
61 format,
62 );
63 if (shouldTrackPath(filename)) {
64 const stat = fs.statSync(filename);
65 const cssImportsMaybe: string[] = [];
66 const imports: string[] = [];
67 for (const childModule of this.children) {
68 const { filename: file, cloverClientRefs } = childModule;
69 if (file.endsWith(".css")) cssImportsMaybe.push(file);
70 else {
71 const child = fileStats.get(file);
72 if (!child) continue;
73 const { cssImportsRecursive } = child;
74 if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
75 imports.push(file);
76 (childModule.cloverImporters ??= []).push(this);
77 if (cloverClientRefs && cloverClientRefs.length > 0) {
78 (this.cloverClientRefs ??= []).push(...cloverClientRefs);
79 }
80 }
81 }
82 fileStats.set(filename, {
83 cssImportsRecursive: cssImportsMaybe.length > 0
84 ? Array.from(new Set(cssImportsMaybe))
85 : null,
86 imports,
87 lastModified: Math.floor(stat.mtimeMs),
88 });
89 }
90 return result;
91};
92
93function loadEsbuild(module: NodeJS.Module, filepath: string) {
94 return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
95}
96
97interface LoadOptions {
98 scannedClientRefs?: string[];
99}
100export function loadEsbuildCode(
101 module: NodeJS.Module,
102 filepath: string,
103 src: string,
104 opt: LoadOptions = {},
105) {
106 if (filepath === import.meta.filename) {
107 module.exports = self;
108 return;
109 }
110 let loader: any = "tsx";
111 if (filepath.endsWith(".ts")) loader = "ts";
112 else if (filepath.endsWith(".jsx")) loader = "jsx";
113 else if (filepath.endsWith(".js")) loader = "js";
114 if (opt.scannedClientRefs) {
115 module.cloverClientRefs = opt.scannedClientRefs;
116 } else {
117 let { code, refs } = resolveClientRefs(src, filepath);
118 module.cloverClientRefs = refs;
119 src = code;
120 }
121 if (src.includes("import.meta")) {
122 src = `
123 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
124 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
125 import.meta.filename = ${JSON.stringify(filepath)};
126 import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r));
127 `
128 .trim()
129 .replace(/[\n\s]/g, "") + src;
130 }
131 src = esbuild.transformSync(src, {
132 loader,
133 format: "cjs",
134 target: "esnext",
135 jsx: "automatic",
136 jsxImportSource: "#jsx",
137 jsxDev: true,
138 sourcemap: "inline",
139 sourcefile: path.basename(filepath),
140 sourceRoot: path.dirname(filepath),
141 }).code;
142 return module._compile(src, filepath, "commonjs");
143}
144
145export function resolveClientRef(sourcePath: string, ref: string) {
146 const filePath = resolveFrom(sourcePath, ref);
147 if (!filePath.endsWith(".client.ts")) {
148 throw new Error("addScript must take a .client.ts");
149 }
150 return path.relative(projectSrc, filePath);
151}
152
153let lazyMarko: typeof import("./marko.ts") | null = null;
154function loadMarko(module: NodeJS.Module, filepath: string) {
155 lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
156 lazyMarko.loadMarko(module, filepath);
157}
158function loadMarkodown(module: NodeJS.Module, filepath: string) {
159 lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
160 lazyMarko.loadMarkodown(module, filepath);
161}
162
163function loadCss(module: NodeJS.Module, _filepath: string) {
164 module.exports = {};
165}
166
167export function reloadRecursive(filepath: string) {
168 filepath = path.resolve(filepath);
169 const existing = cache[filepath];
170 if (existing) deleteRecursiveInner(filepath, existing);
171 fileStats.clear();
172 return require(filepath);
173}
174
175export function unload(filepath: string) {
176 lazyMarko?.markoCache.delete(filepath);
177 filepath = path.resolve(filepath);
178 const module = cache[filepath];
179 if (!module) return;
180 delete cache[filepath];
181 for (const importer of module.cloverImporters ?? []) {
182 unload(importer.filename);
183 }
184}
185
186function deleteRecursiveInner(id: string, module: any) {
187 if (id.includes(path.sep + "node_modules" + path.sep)) {
188 return;
189 }
190 delete cache[id];
191 for (const child of module.children) {
192 if (child.filename.includes("/engine/")) return;
193 const existing = cache[child.filename];
194 if (existing === child) deleteRecursiveInner(child.filename, existing);
195 }
196}
197
198export function getCssImports(filepath: string) {
199 filepath = path.resolve(filepath);
200 if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
201 return fileStats.get(filepath)?.cssImportsRecursive ?? [];
202}
203
204export function getClientScriptRefs(filepath: string) {
205 filepath = path.resolve(filepath);
206 const module = require.cache[filepath];
207 if (!module) throw new Error(filepath + " was never loaded");
208 return module.cloverClientRefs ?? [];
209}
210
211export function getSourceCode(filepath: string) {
212 filepath = path.resolve(filepath);
213 const module = require.cache[filepath];
214 if (!module) throw new Error(filepath + " was never loaded");
215 if (!module.cloverSourceCode) {
216 throw new Error(filepath + " did not record source code");
217 }
218 return module.cloverSourceCode;
219}
220
221export function resolveFrom(src: string, dest: string) {
222 try {
223 return createRequire(src).resolve(dest);
224 } catch (err: any) {
225 if (
226 err.code === "MODULE_NOT_FOUND" && err.requireStack
227 && err.requireStack.length <= 1
228 ) {
229 err.message = err.message.split("\n")[0] + " from '" + src + "'";
230 }
231 throw err;
232 }
233}
234
235const importRegExp = /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;
236const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))?/;
237interface ResolvedClientRefs {
238 code: string;
239 refs: string[];
240}
241export function resolveClientRefs(
242 code: string,
243 filepath: string,
244): ResolvedClientRefs {
245 // This match finds a call to 'import ... from "#sitegen"'
246 const importMatch = code.match(importRegExp);
247 if (!importMatch) return { code, refs: [] };
248 const items = UNWRAP(importMatch[1]);
249 let identifier = "";
250 if (items.startsWith("{")) {
251 const clauseMatch = items.match(getSitegenAddScriptRegExp);
252 if (!clauseMatch) return { code, refs: [] }; // did not import
253 identifier = clauseMatch[1] || "addScript";
254 } else if (items.startsWith("*")) {
255 identifier = importMatch[2] + "\\s*\\.\\s*addScript";
256 } else {
257 throw new Error("Impossible");
258 }
259 identifier = identifier.replaceAll("$", "\\$"); // only needed escape
260 const findCallsRegExp = new RegExp(
261 `\\b(${identifier})\\s*\\(("[^"]+"|'[^']+')\\)`,
262 "gs",
263 );
264 const scannedClientRefs = new Set<string>();
265 code = code.replace(findCallsRegExp, (_, call, arg) => {
266 const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
267 const resolved = resolveClientRef(filepath, ref);
268 scannedClientRefs.add(resolved);
269 return `${call}(${JSON.stringify(getScriptId(resolved))})`;
270 });
271 return { code, refs: Array.from(scannedClientRefs) };
272}
273
274export function getScriptId(file: string) {
275 return (
276 path.isAbsolute(file) ? path.relative(projectSrc, file) : file
277 ).replaceAll("\\", "/");
278}
279
280declare global {
281 namespace NodeJS {
282 interface Module {
283 cloverClientRefs?: string[];
284 cloverSourceCode?: string;
285 cloverImporters?: Module[];
286
287 _compile(
288 this: NodeJS.Module,
289 content: string,
290 filepath: string,
291 format: "module" | "commonjs",
292 ): unknown;
293 }
294 }
295}
296declare module "node:module" {
297 export function _resolveFilename(id: string, parent: NodeJS.Module): unknown;
298}
299
300import * as fs from "#sitegen/fs";
301import { UNWRAP } from "@clo/lib/assert";
302import * as esbuild from "esbuild";
303import { Buffer } from "node:buffer";
304import { createRequire } from "node:module";
305import * as path from "node:path";
306import { pathToFileURL } from "node:url";
307import * as self from "./hot.ts";
framework/incremental.ts deleted-738
......@@ -1,738 +0,0 @@
1// Incremental build system using singleton state.
2// See `work()`, `compile()`, and `invalidate()` for details.
3//
4// All state is serializable to allow recovering state across sessions.
5// This library special-cases the asset map, but is otherwise agnostic.
6let running = false;
7let jobs = 0;
8let newKeys = 0;
9let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
10let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
11let works = new Map<string, Work>();
12let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
13let writes = new Map<string, FileWrite>();
14let assets = new Map<string, Asset>(); // keyed by hash
15
16export interface Ref<T> {
17 key: string;
18 /** This method is compatible with `await` syntax */
19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20 get value(): T | null;
21 /** The progress node passed into the input */
22 progress: progress.Node | progress.Ref;
23}
24type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
25
26/**
27 * Declare and a unit of work. Return value is memoized and only rebuilt when
28 * inputs change. Inputs are tracked via the `io` interface, as well as a hash
29 * of the `input` param and caller source code. Outputs are written at the end
30 * of a compilation (see `compile`).
31 *
32 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33 */
34export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
35export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
36export function work<I, O>(
37 p: progress.Ref,
38 job: Job<I, O>,
39 input?: I,
40): Ref<O> {
41 const source = JSON.stringify(UNWRAP(stack.capture()[0]));
42 const keySource = [source, util.inspect(input)].join(":");
43 const key = crypto.createHash("sha1").update(keySource).digest("base64url");
44 ASSERT(running);
45 ASSERT(
46 !seenWorks.has(key),
47 `Key '${key}' must be unique during the build. `
48 + `To fix this, provide a manual 'key' argument.`,
49 );
50 seenWorks.add(key);
51
52 const prev = works.get(key) as Work<O> | null;
53 if (prev) {
54 return {
55 key,
56 then: (done) => done(prev.value),
57 value: prev.value,
58 progress: p,
59 };
60 }
61
62 async function perform() {
63 const io = new Io(key, p);
64 jobs += 1;
65 newKeys += 1;
66 try {
67 const value = await job(io, input as I);
68 validateSerializable(value, "");
69 const { reads, writes } = io;
70 works.set(key, { value, affects: [], reads, writes });
71 for (const add of reads.files) {
72 const { affects } = UNWRAP(files.get(add));
73 ASSERT(!affects.includes(key));
74 affects.push(key);
75 }
76 for (const add of reads.works) {
77 const { affects } = UNWRAP(works.get(add));
78 ASSERT(!affects.includes(key));
79 affects.push(key);
80 }
81 return value;
82 } finally {
83 jobs -= 1;
84 }
85 }
86
87 let cached: Promise<O>;
88 return {
89 key,
90 then(fufill, reject) {
91 (cached ??= perform()).then(fufill, reject);
92 },
93 get value() {
94 return (works.get(this.key)?.value as O) ?? null;
95 },
96 progress: p,
97 };
98}
99
100export async function compile<T>(
101 compiler: (node: progress.Ref) => Promise<T>,
102 p: progress.Ref = progress,
103) {
104 ASSERT(!running, `Cannot run twice at the same time.`);
105 const startDate = new Date();
106 const start = performance.now();
107 const node = p.start("sitegen!");
108 try {
109 running = true;
110 ASSERT(jobs === 0);
111
112 const value = await compiler(node);
113 ASSERT(jobs === 0);
114 await flush(node, start);
115 return {
116 value,
117 watchFiles: new Set(files.keys()),
118 newOutputs: Array.from(seenWrites)
119 .filter((x) => x.startsWith("f:"))
120 .map((x) => x.slice(2)),
121 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
122 };
123 } finally {
124 node.text = `sitegen at ${startDate.toLocaleDateString()} ${startDate.toLocaleTimeString()}`;
125 node.end();
126 running = false;
127 newKeys = 0;
128 seenWrites.clear();
129 seenWorks.clear();
130 }
131}
132
133export async function flush(progress: progress.Node, start: number) {
134 using _ = progress.start("incremental flush");
135 // Trim
136 const detachedFiles = new Set<string>();
137 const referencedAssets = new Set<string>();
138 for (const [k, v] of works) {
139 const assets = v.writes.assets;
140 if (seenWorks.has(k)) {
141 for (const asset of assets.values()) referencedAssets.add(asset.hash);
142 continue;
143 }
144 deleteWork(k);
145 }
146 for (const [k, file] of files) {
147 if (file.affects.length > 0) continue;
148 files.delete(k);
149 detachedFiles.add(k);
150 }
151 for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
152
153 const p: Promise<void>[] = [];
154 // File writes
155 let dist = 0;
156 for (const [key, { buffer, size }] of writes) {
157 if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
158 dist += size;
159 }
160 // Asset map
161 {
162 const { json, blob, dynamic, dts } = getAssetManifest();
163 const jsonString = Buffer.from(JSON.stringify(json));
164 p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
165 p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
166 p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
167 for (const [k, v] of dynamic) {
168 p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
169 }
170 dist += blob.byteLength + jsonString.byteLength;
171 }
172 await Promise.all(p);
173
174 // Incremental state
175 const serialized = msgpackr.pack(serialize());
176 await fs.writeMkdir(".clover/incr.state", serialized);
177 const time = (performance.now() - start).toFixed(0);
178 progress.log.info(
179 `sitegen! in ${time} ms\n`
180 + ` - ${works.size} keys (${works.size - newKeys} cached)\n`
181 + ` - ${assets.size} static assets\n`
182 + ` - dist: ${formatSize(dist)},`
183 + ` incremental: ${formatSize(serialized.byteLength)}`,
184 );
185}
186
187export async function restore() {
188 let buffer;
189 try {
190 buffer = await fs.readFile(".clover/incr.state");
191 } catch (err: any) {
192 if (err.code !== "ENOENT") throw err;
193 }
194 if (!buffer) return;
195 await deserialize(buffer);
196}
197
198function forceInvalidate(entry: { affects: string[] }) {
199 const queue = [...entry.affects];
200 let key;
201 while ((key = queue.shift())) {
202 const affects = deleteWork(key);
203 queue.push(...affects);
204 }
205}
206
207function deleteWork(key: string) {
208 const work = works.get(key);
209 if (!work) return [];
210 const { reads, affects, writes: w } = work;
211 for (const remove of reads.files) {
212 const { affects } = UNWRAP(files.get(remove));
213 ASSERT(affects.includes(key));
214 affects.splice(affects.indexOf(key), 1);
215 }
216 for (const remove of reads.works) {
217 const { affects } = UNWRAP(works.get(remove), remove);
218 ASSERT(affects.includes(key));
219 affects.splice(affects.indexOf(key), 1);
220 }
221 for (const remove of affects) {
222 const {
223 reads: { works: list },
224 } = UNWRAP(works.get(remove), remove);
225 ASSERT(list.has(key));
226 list.delete(key);
227 }
228 for (const file of w.files) {
229 if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
230 }
231 // Assets are temporarily kept, trimmed via manual GC after compilation.
232
233 works.delete(key);
234 return affects;
235}
236
237export function reset() {
238 ASSERT(!running);
239 works.clear();
240 files.clear();
241 assets.clear();
242}
243
244export function serialize() {
245 const fileEntries = Array.from(
246 files,
247 ([k, v]) =>
248 [
249 k,
250 v.type,
251 v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
252 ...v.affects,
253 ] as const,
254 );
255 const workEntries = Array.from(
256 works,
257 ([k, v]) =>
258 [
259 k,
260 v.value,
261 Array.from(v.reads.files),
262 Array.from(v.reads.works),
263 Array.from(v.writes.files),
264 Array.from(
265 v.writes.assets,
266 ([k, { headers }]) => [k, headers] as const,
267 ),
268 v.affects,
269 ] as const,
270 );
271 const expectedFilesOnDisk = Array.from(
272 writes,
273 ([k, { size, work }]) => [k, size, work] as const,
274 );
275 const assetEntries = Array.from(
276 assets,
277 ([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
278 );
279 return [
280 1,
281 fileEntries,
282 workEntries,
283 expectedFilesOnDisk,
284 assetEntries,
285 ] as const;
286}
287type SerializedState = ReturnType<typeof serialize>;
288
289/* No-op on failure */
290async function deserialize(buffer: Buffer) {
291 const decoded = msgpackr.decode(buffer) as SerializedState;
292 if (!Array.isArray(decoded)) return;
293 if (decoded[0] !== 1) return;
294 const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] = decoded;
295 for (const [k, type, content, ...affects] of fileEntries) {
296 if (type === "f") {
297 ASSERT(typeof content === "number");
298 files.set(k, { type, affects, lastModified: content });
299 } else if (type === "d") {
300 ASSERT(typeof content === "string");
301 files.set(k, { type, affects, contentHash: content, contents: [] });
302 } else {
303 files.set(k, { type, affects });
304 }
305 }
306 for (const entry of workEntries) {
307 const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] = entry;
308 works.set(k, {
309 value,
310 reads: {
311 files: new Set(readFiles),
312 works: new Set(readWorks),
313 },
314 writes: {
315 files: new Set(writeFiles),
316 assets: new Map(
317 Array.from(writeAssets, ([k, headers]) => [
318 k,
319 {
320 hash: JSON.parse(UNWRAP(headers.etag)),
321 headers,
322 },
323 ]),
324 ),
325 },
326 affects,
327 });
328 }
329 const statFiles = await Promise.all(
330 expectedFilesOnDisk.map(([k, size, work]) =>
331 fs
332 .stat(path.join(".clover/o", k))
333 .catch((err) => {
334 if (err.code === "ENOENT") return null;
335 throw err;
336 })
337 .then((stat) => ({ k, size, work, stat }))
338 ),
339 );
340 for (const { k, stat, work, size } of statFiles) {
341 if (stat?.size === size) {
342 writes.set(k, {
343 size: size,
344 buffer: null,
345 work,
346 });
347 } else {
348 forceInvalidate({ affects: [work] });
349 }
350 }
351 for (const [hash, raw, gzip, zstd] of assetEntries) {
352 assets.set(hash, { raw, gzip, zstd });
353 }
354
355 await Promise.all(
356 Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
357 );
358}
359
360export async function invalidate(
361 filePath: string,
362 unloadModule: boolean = true,
363): Promise<boolean> {
364 const key = toRel(toAbs(filePath));
365 const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
366 return invalidateEntry(key, file, unloadModule);
367}
368
369export async function invalidateEntry(
370 key: string,
371 file: TrackedFile,
372 unloadModule: boolean,
373): Promise<boolean> {
374 try {
375 if (file.type === "d") {
376 const contents = (file.contents = await fs.readdir(key));
377 contents.sort();
378 const contentHash = crypto
379 .createHash("sha1")
380 .update(contents.join("\0"))
381 .digest("base64url");
382 if (file.contentHash !== contentHash) {
383 file.contentHash = contentHash;
384 throw new Error();
385 }
386 } else if (file.type === "f") {
387 const lastModified = await fs.stat(key).then(
388 (x) => Math.floor(x.mtimeMs),
389 () => 0,
390 );
391 if (file.lastModified !== lastModified) {
392 file.lastModified = lastModified;
393 throw new Error();
394 }
395 } else {
396 file.type satisfies "null";
397 const stat = await fs.stat(key).catch(() => null);
398 if (stat) throw new Error();
399 }
400 return false;
401 } catch (e) {
402 forceInvalidate(file);
403 if (unloadModule) {
404 // TODO: handle when this triggers unloading of `generate.ts`
405 hot.unload(toAbs(key));
406 }
407 if (file.type === "null") files.delete(key);
408 return true;
409 }
410}
411
412export function getAssetManifest() {
413 const dynamic = new Map<string, Buffer>();
414 const writer = new BufferWriter();
415 const assetMap = Object.fromEntries(
416 Array.from(works, (work) => work[1].writes.assets)
417 .filter((map) => map.size > 0)
418 .flatMap((map) =>
419 Array.from(map, ([key, { hash, headers, regenerative }]) => {
420 const { raw, gzip, zstd } = UNWRAP(
421 assets.get(hash),
422 `Asset ${key} (${hash})`,
423 );
424 if (regenerative) {
425 const id = crypto
426 .createHash("sha1")
427 .update(key)
428 .digest("hex")
429 .slice(0, 16); /* TODO */
430 dynamic.set(
431 id,
432 manifest.packDynamicBuffer(raw, gzip, zstd, headers),
433 );
434 return [key, { type: 1, id }] as const;
435 }
436 return [
437 key,
438 {
439 type: 0,
440 raw: writer.write(raw, "raw:" + hash),
441 gzip: writer.write(gzip, "gzip:" + hash),
442 zstd: writer.write(zstd, "zstd:" + hash),
443 headers,
444 },
445 ] as const;
446 })
447 ),
448 ) satisfies manifest.Manifest;
449 return {
450 json: assetMap,
451 blob: writer.get(),
452 dynamic,
453 dts: "export type AssetKey = "
454 + Object.keys(assetMap)
455 .map((key) => JSON.stringify(key))
456 .join(" | ")
457 + "\n",
458 };
459}
460
461/* Input/Output with automatic tracking.
462 * - Inputs read with Io are tracked to know when to rebuild
463 * - Outputs written with Io are deleted when abandoned.
464 */
465export class Io {
466 constructor(public key: string, private node: progress.Ref) {}
467 reads: Reads = { files: new Set(), works: new Set() };
468 writes: Writes = { files: new Set(), assets: new Map() };
469
470 start(text: string, options?: progress.StartOptions) {
471 return this.node.start(text, options);
472 }
473
474 #trackFs(file: string) {
475 const resolved = toAbs(file);
476 const key = toRel(resolved);
477 this.reads.files.add(key);
478 return { resolved, key };
479 }
480 async readWork<T>(ref: Ref<T>): Promise<T> {
481 this.reads.works.add(ref.key);
482 return await ref;
483 }
484 /** Track a file in the compilation without reading it. */
485 async trackFile(file: string) {
486 const { key, resolved } = this.#trackFs(file);
487 if (!files.get(key)) {
488 let lastModified: number = 0;
489 try {
490 lastModified = Math.floor((await fs.stat(file)).mtimeMs);
491 files.set(key, { type: "f", lastModified, affects: [] });
492 } catch {
493 files.set(key, { type: "null", affects: [] });
494 }
495 }
496 return resolved;
497 }
498 async readFile(file: string): Promise<Buffer>;
499 async readFile(file: string, encoding: "utf-8"): Promise<string>;
500 async readFile(file: string, encoding?: "utf-8") {
501 return fs.readFile(await this.trackFile(file), encoding);
502 }
503 async readJson<T>(file: string) {
504 return JSON.parse(await this.readFile(file, "utf-8")) as T;
505 }
506 async readDir(dir: string) {
507 const { key, resolved } = this.#trackFs(dir);
508 const existing = files.get(key);
509 try {
510 if (existing?.type === "d") return existing.contents;
511 const contents = await fs.readdir(resolved);
512 contents.sort();
513 const contentHash = crypto
514 .createHash("sha1")
515 .update(contents.join("\0"))
516 .digest("base64url");
517 files.set(key, {
518 type: "d",
519 affects: [],
520 contentHash,
521 contents,
522 });
523 return contents;
524 } catch (err) {
525 if (!existing) files.set(key, { type: "null", affects: [] });
526 throw err;
527 }
528 }
529 async readDirRecursive(dir: string): Promise<string[]> {
530 const dirs = await this.readDir(dir);
531 return (
532 await Promise.all(
533 dirs.map(async (child) => {
534 const abs = path.join(dir, child);
535 const stat = await fs.stat(abs);
536 if (stat.isDirectory()) {
537 return (await this.readDirRecursive(abs)).map((grand) => path.join(child, grand));
538 } else {
539 return child;
540 }
541 }),
542 )
543 ).flat();
544 }
545 /* Track all dependencies of a module. */
546 async import<T>(file: string): Promise<T> {
547 const { resolved } = this.#trackFs(file);
548 try {
549 return require(resolved) as T;
550 } finally {
551 const queue = [resolved];
552 const seen = new Set<string>();
553 let current;
554 while ((current = queue.shift())) {
555 const stat = hot.getFileStat(current);
556 if (!stat) continue;
557 const { key } = this.#trackFs(current);
558 if (!files.get(key)) {
559 files.set(key, {
560 type: "f",
561 affects: [],
562 lastModified: stat?.lastModified ?? 0,
563 });
564 }
565 for (const imp of stat.imports) {
566 if (!seen.has(imp)) {
567 seen.add(imp);
568 queue.push(imp);
569 }
570 }
571 }
572 }
573 }
574 async writeAsset(asset: {
575 pathname: string;
576 buffer: string | Uint8Array;
577 regenerative?: boolean;
578 headers?: HeadersInit;
579 }) {
580 ASSERT(asset.pathname.startsWith("/"));
581 ASSERT(!seenWrites.has("a:" + asset.pathname));
582 const buffer = typeof asset.buffer === "string"
583 ? Buffer.from(asset.buffer)
584 : asset.buffer;
585 const headers = new Headers(asset.headers ?? {});
586 const hash = crypto.createHash("sha1").update(buffer).digest("hex");
587 if (!headers.has("Content-Type")) {
588 headers.set("Content-Type", mime.for(asset.pathname));
589 }
590 headers.set("etag", JSON.stringify(hash));
591 this.writes.assets.set(asset.pathname, {
592 hash,
593 headers: Object.fromEntries(headers),
594 regenerative: !!asset.regenerative,
595 });
596 if (!assets.has(hash)) {
597 jobs += 1;
598 assets.set(hash, undefined!);
599 const [gzipBuffer, zstdBuffer] = await Promise.all([
600 gzip(buffer),
601 zstdCompress(buffer),
602 ]);
603 assets.set(hash, {
604 raw: buffer,
605 gzip: gzipBuffer,
606 zstd: zstdBuffer,
607 });
608 jobs -= 1;
609 }
610 }
611 writeFile(subPath: string, blob: string | Buffer) {
612 ASSERT(!subPath.startsWith("/"));
613 ASSERT(
614 !seenWrites.has("f:" + subPath),
615 `File overwritten: ${JSON.stringify(subPath)}`,
616 );
617 seenWrites.add("f:" + subPath);
618 const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
619 writes.set(subPath, {
620 buffer,
621 size: buffer.byteLength,
622 work: this.key,
623 });
624 }
625}
626
627class BufferWriter {
628 size = 0;
629 seen = new Map<string, BufferView>();
630 buffers: Buffer[] = [];
631
632 write(buffer: Buffer, hash: string): BufferView {
633 let view = this.seen.get(hash);
634 if (view) return view;
635 view = [this.size, this.size += buffer.byteLength];
636 this.seen.set(hash, view);
637 this.buffers.push(buffer);
638 return view;
639 }
640
641 get() {
642 return Buffer.concat(this.buffers);
643 }
644}
645
646export function validateSerializable(value: unknown, key: string) {
647 if (typeof value === "string") {
648 if (value.includes(hot.projectRoot)) {
649 throw new Error(
650 `Return value must not contain the CWD for portability, found at ${key}`,
651 );
652 }
653 } else if (value && typeof value === "object") {
654 if (Array.isArray(value)) {
655 value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
656 } else if (
657 Object.getPrototypeOf(value) === Object.prototype
658 || Buffer.isBuffer(value)
659 ) {
660 Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`));
661 } else {
662 throw new Error(
663 `Return value must be a plain JS object, found ${Object.getPrototypeOf(value).constructor.name} at ${key}`,
664 );
665 }
666 } else if (["bigint", "function", "symbol"].includes(typeof value)) {
667 throw new Error(
668 `Return value must be a plain JS object, found ${typeof value} at ${key}`,
669 );
670 }
671}
672
673export function toAbs(absPath: string) {
674 return path.resolve(hot.projectRoot, absPath);
675}
676
677export function toRel(absPath: string) {
678 return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
679}
680
681export type BufferView = [start: number, end: number];
682interface Reads {
683 files: Set<string>;
684 works: Set<string>;
685}
686interface FileWrite {
687 buffer: Buffer | null;
688 size: number;
689 work: string;
690}
691interface Writes {
692 files: Set<string>;
693 assets: Map<string, AssetWrite>;
694}
695interface Asset {
696 raw: Buffer;
697 gzip: Buffer;
698 zstd: Buffer;
699}
700interface AssetWrite {
701 hash: string;
702 headers: Record<string, string>;
703 regenerative: boolean;
704}
705interface Work<T = unknown> {
706 value: T;
707 reads: Reads;
708 writes: Writes;
709 affects: string[];
710}
711type TrackedFile =
712 & { affects: string[] }
713 & (
714 | { type: "f"; lastModified: number }
715 | { type: "d"; contentHash: string; contents: string[] }
716 | { type: "null" }
717 );
718
719const gzip = util.promisify(zlib.gzip);
720const zstdCompress = util.promisify(zlib.zstdCompress);
721
722import * as hot from "./hot.ts";
723
724import * as manifest from "#sitegen/assets";
725import * as fs from "#sitegen/fs";
726import * as mime from "@clo/lib/mime";
727
728import * as crypto from "node:crypto";
729import * as path from "node:path";
730import * as util from "node:util";
731import * as zlib from "node:zlib";
732
733import { formatSize } from "#src/file-viewer/format.ts";
734import * as stack from "@clo/lib/log/stack";
735import * as progress from "@clo/lib/progress";
736
737import { ASSERT, UNWRAP } from "@clo/lib/assert";
738import * as msgpackr from "msgpackr";
framework/lib/path.ts+9-15
......@@ -11,10 +11,9 @@ export class Path {
1111 }
1212 return base;
1313 }
14 return new Path(path.resolve(
15 typeof base === "string" ? base : base.raw,
16 ...more,
17 ));
14 return new Path(
15 path.resolve(typeof base === "string" ? base : base.raw, ...more),
16 );
1817 }
1918
2019 static createDirSync(...parts: string[]): Path;
......@@ -150,12 +149,8 @@ export class Path {
150149 async readIfExists(encoding?: "buffer"): Promise<Buffer | null>;
151150 async readIfExists<T = unknown>(encoding: "json"): Promise<T | null>;
152151 async readIfExists(encoding: "utf-8"): Promise<string | null>;
153 async readIfExists(
154 encoding?: "utf-8" | "buffer" | "json",
155 ): Promise<unknown>;
156 async readIfExists(
157 encoding?: "utf-8" | "buffer" | "json",
158 ): Promise<unknown> {
152 async readIfExists(encoding?: "utf-8" | "buffer" | "json"): Promise<unknown>;
153 async readIfExists(encoding?: "utf-8" | "buffer" | "json"): Promise<unknown> {
159154 try {
160155 return await this.read(encoding);
161156 } catch (err) {
......@@ -182,11 +177,10 @@ export class Path {
182177 return fs.readdirSync(this.raw).map((sub) => this.join(sub));
183178 }
184179
185 async readDir(
186 { recursive }: { recursive?: boolean } = {},
187 ): Promise<Array<Path>> {
188 return (await fs.readdir(this.raw, { recursive }))
189 .map((sub) => this.join(sub));
180 async readDir({ recursive }: { recursive?: boolean } = {}): Promise<
181 Array<Path>
182 > {
183 return (await fs.readdir(this.raw, { recursive })).map((sub) => this.join(sub));
190184 }
191185
192186 async writeJsonFile(json: unknown) {
framework/marko.ts+52-86
......@@ -1,98 +1,64 @@
1export interface MarkoCacheEntry {
2 src: string;
3 scannedClientRefs: string[];
4}
5
6export const markoCache = new Map<string, MarkoCacheEntry>();
7
8export function loadMarko(module: NodeJS.Module, filepath: string) {
9 return loadMarkoExtra(module, filepath, "html");
10}
11export function loadMarkodown(module: NodeJS.Module, filepath: string) {
12 return loadMarkoExtra(module, filepath, "markdown");
13}
14
15function loadMarkoExtra(
16 module: NodeJS.Module,
17 filepath: string,
18 mode: "html" | "markdown",
19) {
20 let cache = markoCache.get(filepath);
21 if (!cache) {
22 let src = fs.readFileSync(filepath, "utf8");
23 if (mode === "markdown") {
24 let transform;
25 try {
26 transform = markodown.transform({
27 source: src,
28 selfImport: `./${path.basename(filepath)}`,
29 });
30 } catch (err) {
31 throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
32 cause: err,
33 });
34 }
35 if (!transform.success) {
36 // TODO: better error handling
37 throw new Error(
38 "Failed " + filepath + ": " + JSON.stringify(transform.errors),
39 );
40 }
41 src = transform.text;
42
43 fs.writeMkdirSync(
44 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
45 + ".marko",
46 src,
1export default function loadMarko({ source, file }: incremental.LoaderInput) {
2 // transform markodown
3 if (file.endsWith('.mdo')) {
4 let transform;
5 try {
6 transform = markodown.transform({
7 source,
8 selfImport: `./${path.basename(file)}`,
9 });
10 } catch (err) {
11 throw new Error(`Failed ${file}: Markodown Compiler Panic`, {
12 cause: err,
13 });
14 }
15 if (!transform.success) {
16 // TODO: better error handling
17 throw new Error(
18 "Failed " + file + ": " + JSON.stringify(transform.errors),
4719 );
4820 }
21 source = transform.text;
22 }
4923
50 // A non-standard thing here is Clover Sitegen implements
51 // its own client side scripting stuff, so it overrides
52 // bare client import statements to it's own usage.
53 const scannedClientRefs = new Set<string>();
54 if (src.match(/^\s*client\s+import\s+["']/m)) {
55 src = src.replace(
56 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
57 (_, src) => {
58 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
59 const resolved = hot.resolveClientRef(filepath, ref);
60 scannedClientRefs.add(resolved);
61 return `<CloverScriptInclude=${
62 JSON.stringify(
63 hot.getScriptId(resolved),
64 )
65 } />`;
66 },
67 ) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
68 }
24 // a non-standard thing here is Clover Sitegen implements
25 // its own client side scripting stuff, so it overrides
26 // bare client import statements to it's own usage.
27 const scannedClientRefs = new Set<string>();
28 if (source.match(/^\s*client\s+import\s+["']/m)) {
29 source = source.replace(
30 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
31 (_, src) => {
32 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
6933
70 let compiled = marko.compileSync(src, filepath, {
71 sourceMaps: true,
72 });
73 // workaround an esbuild bug?
74 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
75 compiled.map.sources[0] = filepath;
76 src = compiled.code
77 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
78 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");
79 cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
80 markoCache.set(filepath, cache);
81 fs.writeMkdirSync(
82 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
83 + ".js",
84 src,
85 );
34 // TODO: WTF
35 const resolved = hot.resolveClientRef(file, ref);
36 scannedClientRefs.add(resolved);
37
38 return `<CloverScriptInclude=${JSON.stringify(resolved)} />`;
39 },
40 ) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
8641 }
8742
88 const { src, scannedClientRefs } = cache;
89 return hot.loadEsbuildCode(module, filepath, src, {
90 scannedClientRefs,
43 let compiled = marko.compileSync(source, file, {
44 sourceMaps: true,
9145 });
46 // workaround an esbuild bug?
47 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
48 compiled.map.sources[0] = file;
49
50 return {
51 format: "js",
52 content: compiled.code
53 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
54 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64"),
55 data: {
56 scannedClientRefs: Array.from(scannedClientRefs),
57 },
58 };
9259}
9360
94import * as fs from "#sitegen/fs";
9561import * as markodown from "@clo/markodown";
9662import * as marko from "@marko/compiler";
9763import * as path from "node:path";
98import * as hot from "./hot.ts";
64import * as incremental from "@clo/lib/incremental";
lib/async.ts+1-1
......@@ -152,7 +152,7 @@ export class DedupeConcurrent<Args extends unknown[], Ret> {
152152 if (promise) return promise;
153153 promise = this.fn(...args);
154154 this.pending.set(k, promise);
155 void promise.finally(() => void this.pending.delete(k));
155 void promise.catch(() => {}).finally(() => void this.pending.delete(k));
156156 return promise;
157157 };
158158}
lib/error.ts+2
......@@ -2,6 +2,8 @@
22 * helper functions for dealing with the `unknown` type, mainly in `catch`
33 * blocks or promise rejection callbacks.
44 *
5 * TODO: rename from `error` to `exception`, less shadowing
6 *
57 * @module
68 */
79
lib/examples/vm.ts created+17
......@@ -0,0 +1,17 @@
1import * as incremental from '../incremental.ts';
2
3export async function main() {
4 const g = new incremental.Graph({
5 loaders: {
6 '.marko': ({ source, file }) => {
7 const { code } = marko.compileSync(source, file);
8 return { content: code, format: 'js' }
9 }
10 }
11 });
12 const log = await g.import('framework/generate.ts');
13 console.log(log);
14 console.log(g);
15}
16
17import * as marko from "@marko/compiler";
lib/incremental.esm.test.ts created+298
......@@ -0,0 +1,298 @@
1import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
2import * as os from "node:os";
3import * as path from "node:path";
4
5import { expect, test } from "vitest";
6import * as inc from "./incremental.ts";
7
8test("Graph.import loads a typescript module graph", async () => {
9 const fs = new VirtualFs({
10 "/app/package.json": JSON.stringify({ type: "module" }),
11 "/app/main.ts": [
12 "import { value } from './dep.ts';",
13 "export const total: number = value + 1;",
14 "export default total;",
15 ].join("\n"),
16 "/app/dep.ts": [
17 "export interface Thing { value: number }",
18 "export const value: number = 41;",
19 ].join("\n"),
20 });
21
22 const graph = new inc.Graph({ fs });
23 const mod = await graph.import<{
24 total: number;
25 default: number;
26 }>("/app/main.ts");
27
28 expect(mod.total).toBe(42);
29 expect(mod.default).toBe(42);
30});
31
32test("Graph.import uses package imports during linking", async () => {
33 const fs = new VirtualFs({
34 "/app/package.json": JSON.stringify({
35 type: "module",
36 imports: {
37 "#dep": "./dep.ts",
38 },
39 }),
40 "/app/main.ts": [
41 "import { value } from '#dep';",
42 "export const loaded = value;",
43 ].join("\n"),
44 "/app/dep.ts": "export const value: number = 7;",
45 });
46
47 const graph = new inc.Graph({ fs });
48 const mod = await graph.import<typeof import("/app/main.ts")>("/app/main.ts");
49
50 expect(mod.loaded).toBe(7);
51});
52
53test("Graph.import loads a .cjs entrypoint as a namespace", async () => {
54 await withTempDir(
55 {
56 "main.cjs": "module.exports = { value: 42, extra: true };",
57 },
58 async (dir) => {
59 const graph = new inc.Graph();
60 const mod = await graph.import<{
61 default: { value: number };
62 value: number;
63 }>(path.join(dir, "main.cjs"));
64
65 expect(mod.default).toEqual({ value: 42, extra: true });
66 expect(mod.value).toBe(42);
67 },
68 );
69});
70
71test("Graph.import rejects .cts entrypoints", async () => {
72 await withTempDir(
73 {
74 "main.cts": [
75 "const value: number = 7;",
76 "module.exports = { value };",
77 ].join("\n"),
78 },
79 async (dir) => {
80 const graph = new inc.Graph();
81 await expect(graph.import(path.join(dir, "main.cts"))).rejects.toThrow(
82 "CommonJS TypeScript (.cts) is not supported",
83 );
84 },
85 );
86});
87
88test("Graph.import lets ESM import CommonJS default and named exports", async () => {
89 await withTempDir(
90 {
91 "package.json": JSON.stringify({ type: "module" }),
92 "main.ts": [
93 "import dep, { named } from './dep.cjs';",
94 "export const total = dep.named + named;",
95 "export const direct = named;",
96 ].join("\n"),
97 "dep.cjs": "module.exports = { named: 5, other: 1 };",
98 },
99 async (dir) => {
100 const graph = new inc.Graph();
101 const mod = await graph.import<typeof import("/app/main.ts")>(
102 path.join(dir, "main.ts"),
103 );
104
105 expect(mod.total).toBe(10);
106 expect(mod.direct).toBe(5);
107 },
108 );
109});
110
111test("Graph.import executes native CommonJS require for json, builtins, and sloppy package resolution", async () => {
112 await withTempDir(
113 {
114 "package.json": JSON.stringify({ type: "commonjs" }),
115 "main.js": [
116 "const dep = require('./dep');",
117 "const dir = require('./dir');",
118 "const data = require('./data.json');",
119 "const pkg = require('pkg-sloppy');",
120 "const { sep } = require('node:path');",
121 "module.exports = { total: dep.value + dir.value + data.value + pkg.value, sep };",
122 ].join("\n"),
123 "dep.js": "module.exports = { value: 1 };",
124 "dir/index.js": "module.exports = { value: 2 };",
125 "data.json": JSON.stringify({ value: 3 }),
126 "node_modules/pkg-sloppy/package.json": JSON.stringify({
127 name: "pkg-sloppy",
128 main: "./dist/index",
129 }),
130 "node_modules/pkg-sloppy/dist/index.js": "module.exports = { value: 4 };",
131 },
132 async (dir) => {
133 const graph = new inc.Graph();
134 const mod = await graph.import<{
135 default: { total: number; sep: string };
136 }>(path.join(dir, "main.js"));
137
138 expect(mod.default.total).toBe(10);
139 expect(mod.default.sep).toBeTypeOf("string");
140 },
141 );
142});
143
144test("Graph.import treats .ts as CommonJS under package type commonjs", async () => {
145 await withTempDir(
146 {
147 "package.json": JSON.stringify({ type: "commonjs" }),
148 "main.ts": "module.exports = { value: 9 };",
149 },
150 async (dir) => {
151 const graph = new inc.Graph();
152 const mod = await graph.import<{
153 default: { value: number };
154 value: number;
155 }>(path.join(dir, "main.ts"));
156
157 expect(mod.default).toEqual({ value: 9 });
158 expect(mod.value).toBe(9);
159 },
160 );
161});
162
163test("Graph.import reuses a shared package.json cache populated by resolve", async () => {
164 await withTempDir(
165 {
166 "package.json": JSON.stringify({
167 type: "commonjs",
168 imports: { "#dep": "./dep.js" },
169 }),
170 "dep.js": "module.exports = 1;",
171 "main.js": "module.exports = { value: require('./dep.js') };",
172 },
173 async (dir) => {
174 const packageJsonPath = path.join(dir, "package.json");
175 const packageJsonCache = new Map<string, inc.PackageMeta>();
176
177 await expect(
178 inc.resolve({
179 specifier: "#dep",
180 resolveDir: dir,
181 packageJsonCache,
182 }),
183 ).resolves.toMatchObject({ type: "success" });
184 expect(packageJsonCache.get(packageJsonPath)?.type).toBe("commonjs");
185
186 const graph = new inc.Graph({ packageJsonCache });
187 const mod = await graph.import<{
188 default: { value: number };
189 value: number;
190 }>(path.join(dir, "main.js"));
191
192 expect(mod.default).toEqual({ value: 1 });
193 expect(packageJsonCache.get(packageJsonPath)?.type).toBe("commonjs");
194 },
195 );
196});
197
198test("Graph.import supports CommonJS cycles through the native module cache", async () => {
199 await withTempDir(
200 {
201 "a.cjs": [
202 "module.exports = { name: 'a' };",
203 "const b = require('./b.cjs');",
204 "module.exports.fromB = b.name;",
205 ].join("\n"),
206 "b.cjs": [
207 "module.exports = { name: 'b' };",
208 "const a = require('./a.cjs');",
209 "module.exports.fromA = a.name;",
210 ].join("\n"),
211 },
212 async (dir) => {
213 const graph = new inc.Graph();
214 const a = await graph.import<{
215 default: { name: string; fromB: string };
216 }>(path.join(dir, "a.cjs"));
217 const b = await graph.import<{
218 default: { name: string; fromA: string };
219 }>(path.join(dir, "b.cjs"));
220
221 expect(a.default).toEqual({ name: "a", fromB: "b" });
222 expect(b.default).toEqual({ name: "b", fromA: "a" });
223 },
224 );
225});
226
227async function withTempDir<T>(
228 files: Record<string, string>,
229 run: (dir: string) => Promise<T>,
230): Promise<T> {
231 const dir = await mkdtemp(path.join(os.tmpdir(), "incremental-test-"));
232 try {
233 for (const [relative, content] of Object.entries(files)) {
234 const abs = path.join(dir, relative);
235 await mkdir(path.dirname(abs), { recursive: true });
236 await writeFile(abs, content);
237 }
238 return await run(dir);
239 } finally {
240 await rm(dir, { recursive: true, force: true });
241 }
242}
243
244class VirtualFs implements inc.Fs {
245 #files: Map<string, string>;
246 #directories: Set<string>;
247
248 constructor(files: Record<string, string>) {
249 this.#files = new Map(Object.entries(files));
250 this.#directories = new Set(["/"]);
251
252 for (const file of this.#files.keys()) {
253 const parts = file.split("/").filter(Boolean);
254 let current = "";
255 for (let i = 0; i < parts.length - 1; i++) {
256 current += `/${parts[i]}`;
257 this.#directories.add(current);
258 }
259 }
260 }
261
262 async readTextFile(path: string): Promise<string> {
263 const contents = this.#files.get(path);
264 if (contents !== undefined) return contents;
265 const error = new Error(`ENOENT: no such file, open '${path}'`);
266 // @ts-expect-error node-style error code
267 error.code = "ENOENT";
268 throw error;
269 }
270
271 async readDir(path: string): Promise<string[]> {
272 if (this.#directories.has(path)) return [];
273 const error = new Error(`ENOENT: no such directory, scandir '${path}'`);
274 // @ts-expect-error node-style error code
275 error.code = "ENOENT";
276 throw error;
277 }
278
279 async realPath(path: string): Promise<string> {
280 return path;
281 }
282
283 async stat(path: string): Promise<inc.Stat> {
284 const contents = this.#files.get(path);
285 if (contents !== undefined) {
286 return { kind: "file", size: contents.length, time: 0 };
287 }
288 if (this.#directories.has(path)) {
289 return { kind: "directory", size: 0, time: 0 };
290 }
291 const error = new Error(
292 `ENOENT: no such file or directory, stat '${path}'`,
293 );
294 // @ts-expect-error node-style error code
295 error.code = "ENOENT";
296 throw error;
297 }
298}
lib/incremental.resolution.test.ts created+435
......@@ -0,0 +1,435 @@
1import { expect, test } from "vitest";
2import * as inc from "./incremental.ts";
3
4test("finds node.js builtin module", async () => {
5 const fs = new VirtualFs({});
6
7 await expect(inc.resolve({ specifier: "node:fs", resolveDir: "/", fs })).resolves.toEqual({
8 type: "success",
9 namespace: "node",
10 subPath: "fs",
11 files: [],
12 });
13 await expect(inc.resolve({ specifier: "fs", resolveDir: "/", fs })).resolves.toEqual({
14 type: "success",
15 namespace: "node",
16 subPath: "fs",
17 files: [],
18 });
19 await expect(inc.resolve({ specifier: "util/types", resolveDir: "/", fs })).resolves.toEqual({
20 type: "success",
21 namespace: "node",
22 subPath: "util/types",
23 files: [],
24 });
25 await expect(inc.resolve({ specifier: "node:test", resolveDir: "/", fs })).resolves.toEqual({
26 type: "success",
27 namespace: "node",
28 subPath: "test",
29 files: [],
30 });
31 await expect(inc.resolve({ specifier: "test", resolveDir: "/", fs })).resolves.toEqual({
32 type: "error",
33 error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
34 files: [
35 "/node_modules/test/package.json",
36 ],
37 });
38});
39
40test("relative file path imports", async () => {
41 const fs = new VirtualFs({
42 "/src/a.js": "",
43 "/src/b.ts": "",
44 "/src/shared/util.js": "",
45 "/src/feature/main.js": "",
46 "/src/node_modules/a/index.js": "",
47 });
48
49 await expect(inc.resolve({ specifier: "./a.js", resolveDir: "/src", fs })).resolves.toEqual({
50 type: "success",
51 namespace: "file",
52 subPath: "/src/a.js",
53 files: ["/src/a.js"],
54 });
55
56 await expect(inc.resolve({ specifier: "./c.ts", resolveDir: "/src", fs })).resolves.toEqual({
57 type: "error",
58 error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
59 files: ["/src/c.ts"],
60 });
61
62 await expect(
63 inc.resolve({ specifier: "../shared/util.js", resolveDir: "/src/feature", fs }),
64 ).resolves.toEqual({
65 type: "success",
66 namespace: "file",
67 subPath: "/src/shared/util.js",
68 files: ["/src/shared/util.js"],
69 });
70
71 await expect(inc.resolve({ specifier: "./a", resolveDir: "/src", fs })).resolves.toEqual({
72 type: "error",
73 error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
74 files: ["/src/a"],
75 });
76 await expect(inc.resolve({ specifier: "./node_modules/a", resolveDir: "/src", fs })).resolves.toEqual({
77 type: "error",
78 error: expect.objectContaining({ code: "ERR_UNSUPPORTED_DIR_IMPORT" }),
79 files: ["/src/node_modules/a"],
80 });
81 await expect(inc.resolve({ specifier: "./node_modules/a/index", resolveDir: "/src", fs })).resolves.toEqual({
82 type: "error",
83 error: expect.objectContaining({ code: "ERR_MODULE_NOT_FOUND" }),
84 files: ["/src/node_modules/a/index"],
85 });
86 await expect(inc.resolve({ specifier: "./node_modules/a/index.js", resolveDir: "/src", fs })).resolves.toEqual({
87 type: "success",
88 namespace: "file",
89 subPath: "/src/node_modules/a/index.js",
90 files: ["/src/node_modules/a/index.js"],
91 });
92});
93
94test("sloppy resolution", async () => {
95 const fs = new VirtualFs({
96 "/src/a.js": "",
97 "/src/dir/index.js": "",
98 });
99
100 await expect(inc.resolve({
101 specifier: "./a",
102 resolveDir: "/src",
103 fs,
104 sloppy: true,
105 })).resolves.toEqual({
106 type: "success",
107 namespace: "file",
108 subPath: "/src/a.js",
109 files: ["/src/a", "/src/a.js"],
110 });
111
112 await expect(inc.resolve({
113 specifier: "./dir",
114 resolveDir: "/src",
115 fs,
116 sloppy: true,
117 })).resolves.toEqual({
118 type: "success",
119 namespace: "file",
120 subPath: "/src/dir/index.js",
121 files: [
122 "/src/dir",
123 "/src/dir.js",
124 "/src/dir.json",
125 "/src/dir.node",
126 "/src/dir/package.json",
127 "/src/dir/index.js",
128 ],
129 });
130
131 await expect(
132 inc.resolve({ specifier: "./dir", resolveDir: "/src", fs, sloppy: false }),
133 ).resolves.toEqual({
134 type: "error",
135 error: expect.objectContaining({ code: "ERR_UNSUPPORTED_DIR_IMPORT" }),
136 files: ["/src/dir"],
137 });
138});
139
140test("package resolution handles exports and imports maps", async () => {
141 const fs = new VirtualFs({
142 "/app/src/main.js": "",
143 "/app/package.json": JSON.stringify({
144 imports: {
145 "#src/*": "./src/*.js",
146 },
147 }),
148 "/app/src/feature.js": "export const feature = true;",
149 "/app/node_modules/pkg/package.json": JSON.stringify({
150 name: "pkg",
151 exports: "./dist/index.js",
152 }),
153 "/app/node_modules/pkg/dist/index.js": "export const value = 1;",
154 "/app/node_modules/pkg/dist/feature.js": "export const feature = true;",
155 "/app/node_modules/pkg-subpath/package.json": JSON.stringify({
156 name: "pkg-subpath",
157 exports: {
158 "./feature": "./dist/feature.js",
159 },
160 }),
161 "/app/node_modules/pkg-subpath/dist/feature.js": "export const feature = true;",
162 });
163
164 await expect(inc.resolve({ specifier: "pkg", resolveDir: "/app/src", fs })).resolves.toEqual({
165 type: "success",
166 namespace: "file",
167 subPath: "/app/node_modules/pkg/dist/index.js",
168 files: [
169 "/app/src/node_modules/pkg/package.json",
170 "/app/node_modules/pkg/package.json",
171 "/app/node_modules/pkg/dist/index.js",
172 ],
173 });
174
175 await expect(inc.resolve({ specifier: "pkg-subpath/feature", resolveDir: "/app/src", fs })).resolves
176 .toEqual({
177 type: "success",
178 namespace: "file",
179 subPath: "/app/node_modules/pkg-subpath/dist/feature.js",
180 files: [
181 "/app/src/node_modules/pkg-subpath/package.json",
182 "/app/node_modules/pkg-subpath/package.json",
183 "/app/node_modules/pkg-subpath/dist/feature.js",
184 ],
185 });
186
187 await expect(inc.resolve({
188 specifier: "#src/feature",
189 resolveDir: "/app/src",
190 fs,
191 })).resolves.toEqual({
192 type: "success",
193 namespace: "file",
194 subPath: "/app/src/feature.js",
195 files: [
196 "/app/src/package.json",
197 "/app/package.json",
198 "/app/src/feature.js",
199 ],
200 });
201});
202
203test("package resolution reports export and import errors and supports sloppy main fallback", async () => {
204 const fs = new VirtualFs({
205 "/app/package.json": JSON.stringify({
206 imports: {
207 "#src/*": "./src/*.js",
208 },
209 }),
210 "/app/node_modules/pkg/package.json": JSON.stringify({
211 name: "pkg",
212 exports: {
213 "./feature": "./dist/feature.js",
214 },
215 }),
216 "/app/node_modules/pkg-sloppy/package.json": JSON.stringify({
217 name: "pkg-sloppy",
218 main: "./dist/index",
219 }),
220 "/app/node_modules/pkg-sloppy/dist/index.js": "export const value = 1;",
221 "/app/node_modules/pkg-legacy-main/package.json": JSON.stringify({
222 name: "pkg-legacy-main",
223 main: "lib/main.js",
224 }),
225 "/app/node_modules/pkg-legacy-main/lib/main.js": "module.exports = {};",
226 "/app/node_modules/pkg-legacy-module/package.json": JSON.stringify({
227 name: "pkg-legacy-module",
228 module: "lib/module.js",
229 }),
230 "/app/node_modules/pkg-legacy-module/lib/module.js": "module.exports = {};",
231 });
232
233 await expect(inc.resolve({ specifier: "pkg/missing", resolveDir: "/app/src", fs })).resolves
234 .toEqual({
235 type: "error",
236 error: expect.objectContaining({ code: "ERR_PACKAGE_PATH_NOT_EXPORTED" }),
237 files: ["/app/src/node_modules/pkg/package.json", "/app/node_modules/pkg/package.json"],
238 });
239
240 await expect(inc.resolve({ specifier: "#missing", resolveDir: "/app/src", fs })).resolves
241 .toEqual({
242 type: "error",
243 error: expect.objectContaining({ code: "ERR_PACKAGE_IMPORT_NOT_DEFINED" }),
244 files: ["/app/src/package.json", "/app/package.json"],
245 });
246
247 await expect(inc.resolve({
248 specifier: "pkg-sloppy",
249 resolveDir: "/app/src",
250 fs,
251 sloppy: true,
252 })).resolves.toEqual({
253 type: "success",
254 namespace: "file",
255 subPath: "/app/node_modules/pkg-sloppy/dist/index.js",
256 files: [
257 "/app/src/node_modules/pkg-sloppy/package.json",
258 "/app/node_modules/pkg-sloppy/package.json",
259 "/app/node_modules/pkg-sloppy/dist/index",
260 "/app/node_modules/pkg-sloppy/dist/index.js",
261 ],
262 });
263
264 await expect(inc.resolve({
265 specifier: "pkg-legacy-main",
266 resolveDir: "/app/src",
267 fs,
268 sloppy: true,
269 })).resolves.toEqual({
270 type: "success",
271 namespace: "file",
272 subPath: "/app/node_modules/pkg-legacy-main/lib/main.js",
273 files: [
274 "/app/src/node_modules/pkg-legacy-main/package.json",
275 "/app/node_modules/pkg-legacy-main/package.json",
276 "/app/node_modules/pkg-legacy-main/lib/main.js",
277 ],
278 });
279
280 await expect(inc.resolve({
281 specifier: "pkg-legacy-module",
282 resolveDir: "/app/src",
283 fs,
284 sloppy: true,
285 })).resolves.toEqual({
286 type: "success",
287 namespace: "file",
288 subPath: "/app/node_modules/pkg-legacy-module/lib/module.js",
289 files: [
290 "/app/src/node_modules/pkg-legacy-module/package.json",
291 "/app/node_modules/pkg-legacy-module/package.json",
292 "/app/node_modules/pkg-legacy-module/lib/module.js",
293 ],
294 });
295});
296
297test("resolve reuses package.json cache for repeated imports lookups", async () => {
298 const fs = new VirtualFs({
299 "/app/package.json": JSON.stringify({
300 imports: {
301 "#dep": "./src/dep.js",
302 },
303 }),
304 "/app/src/dep.js": "export const dep = true;",
305 });
306 const packageJsonCache = new Map<string, inc.PackageMeta>();
307
308 await expect(inc.resolve({
309 specifier: "#dep",
310 resolveDir: "/app",
311 fs,
312 packageJsonCache,
313 })).resolves.toMatchObject({
314 type: "success",
315 subPath: "/app/src/dep.js",
316 });
317 expect(fs.readCount("/app/package.json")).toBe(1);
318
319 await expect(inc.resolve({
320 specifier: "#dep",
321 resolveDir: "/app",
322 fs,
323 packageJsonCache,
324 })).resolves.toMatchObject({
325 type: "success",
326 subPath: "/app/src/dep.js",
327 files: ["/app/package.json", "/app/src/dep.js"],
328 });
329 expect(fs.readCount("/app/package.json")).toBe(1);
330});
331
332test("resolve reuses package.json cache for repeated exports lookups", async () => {
333 const fs = new VirtualFs({
334 "/app/src/main.js": "",
335 "/app/node_modules/pkg/package.json": JSON.stringify({
336 name: "pkg",
337 exports: {
338 ".": "./dist/index.js",
339 },
340 }),
341 "/app/node_modules/pkg/dist/index.js": "export const value = true;",
342 });
343 const packageJsonCache = new Map<string, inc.PackageMeta>();
344
345 await expect(inc.resolve({
346 specifier: "pkg",
347 resolveDir: "/app/src",
348 fs,
349 packageJsonCache,
350 })).resolves.toMatchObject({
351 type: "success",
352 subPath: "/app/node_modules/pkg/dist/index.js",
353 });
354 expect(fs.readCount("/app/node_modules/pkg/package.json")).toBe(1);
355
356 await expect(inc.resolve({
357 specifier: "pkg",
358 resolveDir: "/app/src",
359 fs,
360 packageJsonCache,
361 })).resolves.toMatchObject({
362 type: "success",
363 subPath: "/app/node_modules/pkg/dist/index.js",
364 files: [
365 "/app/src/node_modules/pkg/package.json",
366 "/app/node_modules/pkg/package.json",
367 "/app/node_modules/pkg/dist/index.js",
368 ],
369 });
370 expect(fs.readCount("/app/node_modules/pkg/package.json")).toBe(1);
371});
372
373class VirtualFs implements inc.Fs {
374 #files: Map<string, string>;
375 #directories: Set<string>;
376 #readCounts: Map<string, number>;
377
378 constructor(files: Record<string, string>) {
379 this.#files = new Map(Object.entries(files));
380 this.#directories = new Set(["/"]);
381 this.#readCounts = new Map();
382
383 for (const file of this.#files.keys()) {
384 const parts = file.split("/").filter(Boolean);
385 let current = "";
386 for (let i = 0; i < parts.length - 1; i++) {
387 current += `/${parts[i]}`;
388 this.#directories.add(current);
389 }
390 }
391 }
392
393 async readTextFile(path: string): Promise<string> {
394 this.#readCounts.set(path, (this.#readCounts.get(path) ?? 0) + 1);
395 const contents = this.#files.get(path);
396 if (contents !== undefined) return contents;
397 const error = new Error(`ENOENT: no such file, open '${path}'`);
398 // @ts-expect-error node-style error code
399 error.code = "ENOENT";
400 throw error;
401 }
402
403 async readDir(path: string): Promise<string[]> {
404 if (!this.#directories.has(path)) {
405 const error = new Error(`ENOENT: no such directory, scandir '${path}'`);
406 // @ts-expect-error node-style error code
407 error.code = "ENOENT";
408 throw error;
409 }
410
411 return [];
412 }
413
414 async stat(path: string): Promise<inc.Stat> {
415 const contents = this.#files.get(path);
416 if (contents !== undefined) {
417 return { kind: "file", size: contents.length, time: 0 };
418 }
419 if (this.#directories.has(path)) {
420 return { kind: "directory", size: 0, time: 0 };
421 }
422 const error = new Error(`ENOENT: no such file or directory, stat '${path}'`);
423 // @ts-expect-error node-style error code
424 error.code = "ENOENT";
425 throw error;
426 }
427
428 async realPath(path: string): Promise<string> {
429 return path;
430 }
431
432 readCount(path: string): number {
433 return this.#readCounts.get(path) ?? 0;
434 }
435}
lib/incremental.serialization.test.ts created+49
......@@ -0,0 +1,49 @@
1import { expect, test } from "vitest";
2import * as inc from "./incremental.ts";
3
4test("readBlob returns the cached entry when available", () => {
5 const db = new inc.Database(null);
6 const value = {
7 message: "hello",
8 bytes: new Uint8Array([1, 2, 3]),
9 when: new Date("2024-01-02T03:04:05.000Z"),
10 } satisfies inc.Serializable;
11
12 const ref = db.writeBlob(value);
13 const roundTrip = db.readBlob(ref.blob);
14
15 expect(roundTrip).toBe(value);
16});
17
18test("readBlob falls back to the database and revives serialized values", () => {
19 const db = new inc.Database(null);
20 const value = {
21 text: "hello",
22 number: Infinity,
23 missing: undefined,
24 when: new Date("2024-01-02T03:04:05.000Z"),
25 set: new Set<inc.Serializable>([1, "two"]),
26 map: new Map<inc.Serializable, inc.Serializable>([
27 ["three", 3],
28 [4, "four"],
29 ]),
30 bytes: new Uint8Array([4, 5, 6]),
31 nested: {
32 $clover: "escaped",
33 blob: new Uint8Array([7, 8, 9]),
34 },
35 } satisfies inc.Serializable;
36
37 const ref = db.writeBlob(value, false);
38 db.clearInMemoryCache();
39
40 const roundTrip = db.readBlob(ref.blob) as typeof value;
41
42 expect(roundTrip).toEqual(value);
43 expect(roundTrip).not.toBe(value);
44 expect(roundTrip.when).toBeInstanceOf(Date);
45 expect(roundTrip.set).toBeInstanceOf(Set);
46 expect(roundTrip.map).toBeInstanceOf(Map);
47 expect(roundTrip.bytes).toBeInstanceOf(Uint8Array);
48 expect(roundTrip.nested.blob).toBeInstanceOf(Uint8Array);
49});
lib/incremental.sources.test.ts created+125
......@@ -0,0 +1,125 @@
1import { expect, test } from "vitest";
2import * as inc from "./incremental.ts";
3
4test("Database stores cache entries with complete URL sources", () => {
5 const db = new inc.Database(null);
6 const file = {
7 url: "file:///site/src/page.ts",
8 kind: "file",
9 time: 100,
10 size: 12,
11 hash: "sha1-page-v1",
12 } satisfies inc.Input;
13 const query = {
14 url: "sql://content/posts/by-slug?slug=hello",
15 kind: "query",
16 time: 7,
17 size: 1,
18 hash: "revision-7",
19 } satisfies inc.Input;
20
21 db.writeCache("page:hello", { html: "<h1>Hello</h1>" }, [file, query]);
22 const cache = db.readCache("page:hello")!;
23
24 expect(db.readBlob(cache.blob)).toEqual({ html: "<h1>Hello</h1>" });
25 expect(collect(cache.inputs)).toEqual([
26 { id: expect.anything(), ...file },
27 { id: expect.anything(), ...query },
28 ]);
29});
30
31test("Database invalidates caches that depend on changed sources", () => {
32 const db = new inc.Database(null);
33 const page = {
34 url: "file:///site/src/page.ts",
35 kind: "file",
36 time: 100,
37 size: 12,
38 hash: "sha1-page-v1",
39 } satisfies inc.Input;
40 const layout = {
41 url: "file:///site/src/layout.ts",
42 kind: "file",
43 time: 200,
44 size: 34,
45 hash: "sha1-layout-v1",
46 } satisfies inc.Input;
47
48 db.writeCache("page", "page result", [page]);
49 db.writeCache("layout", "layout result", [layout]);
50
51 const storedPage = collect(db.sources()).find(
52 (source) => source.url === page.url,
53 )!;
54 db.invalidate(storedPage.id);
55 db.writeSource({ ...page, time: 101, size: 13, hash: "sha1-page-v2" });
56
57 expect(db.readCache("page")).toBeNull();
58 expect(db.readBlob(db.readCache("layout")!.blob)).toBe("layout result");
59});
60
61test("Database treats source kind changes as invalidation", () => {
62 const db = new inc.Database(null);
63 const missing = {
64 url: "file:///site/src/optional.ts",
65 kind: "unknown",
66 time: 0,
67 size: 0,
68 hash: "",
69 } satisfies inc.Input;
70
71 db.writeCache("resolution:optional", "missing", [missing]);
72
73 const [source] = collect(db.sources());
74 db.invalidate(source!.id);
75 db.writeSource({
76 ...missing,
77 kind: "file",
78 time: 300,
79 size: 44,
80 hash: "sha1-optional",
81 });
82
83 expect(db.readCache("resolution:optional")).toBeNull();
84});
85
86test("Database lets callers skip invalidation when source state is unchanged", () => {
87 const db = new inc.Database(null);
88 const source = {
89 url: "https://example.com/feed.xml",
90 kind: "response",
91 time: 500,
92 size: 2048,
93 hash: "etag-feed-v1",
94 } satisfies inc.Input;
95
96 db.writeCache("feed", new Uint8Array([1, 2, 3]), [source]);
97
98 const [stored] = collect(db.sources());
99 expect(stored).toMatchObject(source);
100 expect(db.readBlob(db.readCache("feed")!.blob)).toEqual(
101 new Uint8Array([1, 2, 3]),
102 );
103});
104
105test("Database requires source identities to be complete URLs", () => {
106 const db = new inc.Database(null);
107
108 expect(() =>
109 db.writeSource({
110 url: "file:./relative.ts",
111 kind: "file",
112 time: 1,
113 size: 1,
114 hash: "sha1-relative",
115 }),
116 ).toThrow("source url must be canonical");
117});
118
119function collect<T>(iterator: Iterator<T>): T[] {
120 const out: T[] = [];
121 for (let next = iterator.next(); !next.done; next = iterator.next()) {
122 out.push(next.value);
123 }
124 return out;
125}
lib/incremental.ts created+1890
......@@ -0,0 +1,1890 @@
1/**
2 * Clover's `incremental` module provides a way to load and reload modules in
3 * Node.js, as well as a foundation to build incremental developer tools.
4 * Incremental build systems stem from the {@linkcode Ref#step|step} function,
5 * which runs a closure given a IO object that allows tracing
6 * {@linkcode Input|inputs} ({@link File|file reads}, network fetches, database
7 * calls, and more). {@linkcode Step}s cache their output data and re-run when
8 * changed. By archetecting the build system steps, you can balance 'size of
9 * the cache' vs 'how fine grained is tracing'.
10 *
11 * The entire system is built around the idea of being able to flush incremental
12 * state to the filesystem, and revive cached work on a second run, so many
13 * places require {@link Serializable|serializable} data. upon revival, the
14 * filesystem (and other input sources) are re-queried for changes. Because of
15 * this design, reading inputs is not allowed outside of steps.
16 *
17 * A central part of building incremental systems is being able to trace source
18 * code itself, so Clover Incremental includes a full set of tools for
19 *
20 * `--experimental-vm-modules` support is required as it is the only way to
21 * interact with an ES Module Loader with the ability to garbage collect modules.
22 *
23 * TODO, explain these visions more in details:
24 * - simple module loading `new incremental.Graph().import("src/main.ts");`
25 * - plugin system to load extensions like `.tsx`, `.marko`, `.svelte`
26 * - tracing the files used in resolution and initialization
27 * - allowing arbitrary dependencies to be added
28 * - allowing arbitrary jobs that cache when re-run
29 * - easy base to implement web frameworks, testing tools, linters, etc
30 * - creating a filesystem watcher to detect changes
31 * - performing trivial module reloading (perhaps... full HMR)
32 * - serializing the graph state to disk, along with jobs and v8 caches
33 *
34 * `@clo/lib/incremental` is implemented as a single file depending only on the
35 * node.js builtins and other `@clo/lib` libraries. i intend for this codebase
36 * to be a good introduction to how an incremental system can function.
37 *
38 * @module
39 */
40
41// TODO (this list is NOT in order)
42// - [X] ES and CJS module loading
43// - [ ] perfectly imports into the database
44// - [ ] save and restore v8 cache
45// - [ ] represent all loaders as virtual "steps"
46// - [ ] partial hot graph replacement
47// - [ ] steps
48// - [ ] re-validating file inputs
49// - [ ] re-validating network inputs
50// - [ ] custom source types
51// - [ ] depend on the results of other steps `readStep`
52// - [ ] shaking old steps away
53// - [ ] build output
54// - [ ] write file (without storing the file's contents entirely in-database)
55// - [ ] copy file
56// - [ ] module tracing API (example: "if i imported X, list all the .css files")
57// - [X] serialization API
58// - [ ] filesystem watcher
59// - [ ] ability to watch for other source types
60// - [ ] trace every file ever + other insane details like runtime info
61
62/**
63 * serialization special cases top-level `Uint8Array`s and `string`s to use more
64 * effective data storage strategies. other variants use JSON + a reviver.
65 */
66export type Serializable =
67 | string
68 | number
69 | boolean
70 | null
71 | undefined
72 | Uint8Array
73 | Date
74 | Set<Serializable>
75 | Map<Serializable, Serializable>
76 | Serializable[]
77 | { [k: string]: Serializable };
78
79export class Ref implements progress.Ref {
80 /** owning graph */
81 #graph: Graph;
82 /** parent progress ref */
83 #ref: progress.Ref;
84 /** promises for the build */
85 #build: PendingBuild;
86
87 constructor(options: {
88 graph: Graph;
89 progress: progress.Ref;
90 /** build status */
91 build: PendingBuild;
92 }) {
93 this.#graph = options.graph;
94 this.#build = options.build;
95 this.#ref = options.progress;
96 }
97
98 /**
99 * creates a new trackable unit of work as a child of this one.
100 * when given an estimate, a progress bar is rendered.
101 */
102 start = (text: string, opts?: progress.StartOptions): ProgressNode => {
103 const node = this.#ref.start(text, opts);
104 return ts.mixin(
105 node,
106 new Ref({
107 graph: this.#graph,
108 progress: node,
109 build: this.#build,
110 }),
111 );
112 };
113
114 /**
115 * run a deterministic "step function" and cache the result. the cache entry
116 * is busted if the given key or any referenced {@linkcode Input}s are
117 * changed. inside the step, the {@linkcode Run} provides access to the
118 * filesystem, network, and a base for other input sources to use.
119 *
120 * on a cache hit, all {@linkcode Input}s are re-queried for changes before
121 * the step re-runs. in the case some inputs changed, the ones that stayed the
122 * same can be re-used from this querying pass.
123 *
124 * implicitly, the caller file is added as an `Input` so the implementation of
125 * the builder is tracked incrementally. the caller of `new incremental.Graph`
126 * is also given this behavior.
127 *
128 * NOTE: the provided callback MAY capture arbitrary values that are not a part
129 * of `key`. this is by design to bypass the cache key when needed. the step's
130 * logic MUST be deterministic, and future debugging tools reserve the right to
131 * arbitrarily re-call the step function multiple times per build, or even
132 * after a build has completed.
133 */
134 step<Key extends Serializable, Result extends Serializable>(
135 key: Key,
136 fn: StepFn<Key, Result>,
137 ): Step<Result> {
138 ASSERT(!this.#build.closed, "cannot start a step after the build has closed");
139 const callerFile = UNWRAP(stack.capture()[0], "runtime bug").file;
140 const keyHash = hashSerializable(key, callerFile);
141 ASSERT(
142 !this.#build.steps.has(keyHash),
143 () => `duplicate step key. caller may not be passing enough information to it's "key"`,
144 );
145 this.#build.steps.add(keyHash);
146
147 // TODO: validate cache inputs before accepting an existing entry.
148 const existing = this.#graph.db.readCache(keyHash);
149 if (existing) {
150 const value = this.#graph.db.readBlob(existing.blob) as Result;
151 return {
152 id: existing.id,
153 value,
154 then: (onfufilled) => Promise.resolve(value).then(onfufilled),
155 hit: true,
156 };
157 }
158
159 const run = new Run({
160 key,
161 build: this.#build,
162 progress: this.#ref,
163 });
164 const defer = Promise.withResolvers<Result>();
165 defer.promise.catch(() => {}); // handled by `step.then` or `incrementalBuild`
166
167 let rejectionObserved = false;
168 const step: Step<Result> = {
169 id: null,
170 value: null,
171 then: (onfufilled, onrejected) => {
172 if (typeof onrejected === "function") rejectionObserved = true;
173 return defer.promise.then(onfufilled, onrejected);
174 },
175 hit: false,
176 };
177
178 void (async () => {
179 this.#build.activeSteps.add(defer.promise);
180 try {
181 const result = await fn(run);
182 const inputs = await run.finish();
183 step.id = this.#graph.db.writeCache(keyHash, result, inputs);
184 step.value = result;
185 defer.resolve(result);
186 } catch (error) {
187 run.close();
188 this.#build.failures.push(() => rejectionObserved ? { observed: true } : { observed: false, error });
189 defer.reject(error);
190 } finally {
191 this.#build.activeSteps.delete(defer.promise);
192 }
193 })();
194
195 return step;
196 }
197
198 // TODO: function to write a file
199 // TODO: function to copy a file
200}
201
202export type StepFn<Key extends Serializable, Result extends Serializable> = (
203 t: Run<Key>,
204) => Result | Promise<Result>;
205
206export type ProgressNode = progress.Node & Ref;
207
208export interface Step<T> extends PromiseLike<T> {
209 /** @internal */
210 // unset until resolved.
211 id: RowId | null;
212 /** returns `null` if the value is not yet done being computed */
213 value: T | null;
214 /** true if this was a cache hit */
215 hit: boolean;
216}
217
218/** state shared between different cache boundaries throughout a single `incrementalBuild` call */
219interface PendingBuild {
220 /** key is a URL. values may be waiting on an expensive freshness token. */
221 inputs: Map<string, Promise<Input>>;
222 steps: Set<string>;
223 activeSteps: Set<Promise<unknown>>;
224 /** failures from steps that were started but not awaited by user code. */
225 failures: Array<() => StepFailureStatus>;
226 // TODO: invert this
227 closed: boolean;
228 // TODO: output? want to store a list of written + copied files. tho some more details to this tbh.
229}
230
231async function waitForBuildSteps(build: PendingBuild): Promise<void> {
232 while (build.activeSteps.size > 0) {
233 await Promise.all(build.activeSteps);
234 }
235}
236
237// TODO: this is cooked
238type StepFailureStatus =
239 | { observed: true }
240 | { observed: false; error: unknown };
241
242/**
243 * blob-like object. the wrapper allows `@clo/lib/incremental` to hash the file
244 * in combination to the caller's usage, ensuring there is at most one full read.
245 */
246export class File {
247 path: string;
248 #handle: unknown;
249
250 /** file modification time */
251 time: number;
252 /** size in bytes */
253 size: number;
254 #fs: Fs;
255 #hash: string | null = null;
256
257 constructor(fs: Fs, path: string, stat: Stat) {
258 ASSERT(stat.kind === "file");
259 this.path = path;
260 this.time = stat.time;
261 this.size = stat.size;
262 this.#fs = fs;
263 }
264
265 async stream(options?: { chunkSize: number }): Promise<ReadableStream> {
266 const chunkSize = options?.chunkSize ?? 65536;
267 // return a pullable readable stream, hashing every chunk as it is read
268 }
269 async arrayBuffer(): Promise<ArrayBuffer> {
270 // allocate the backing buffer, then incrementally read into it. hash the
271 // entire buffer at the end.
272 }
273 async text(): Promise<string> {
274 const decoder = new TextDecoder("utf-8");
275 const buffer = new Uint8Array(Math.min(this.size, 65536));
276 // take turns filling the buffer, hashing and decoding it as you go
277 let str = "";
278 // str += decoder.decode(buffer, { stream: true })
279 }
280
281 async hash(): Promise<string> {
282 // retrieve the hash once its done
283 }
284
285 async bytes(): Promise<Uint8Array> {
286 return new Uint8Array(await this.arrayBuffer());
287 }
288}
289
290/**
291 * state of a running {@linkcode StepFn}, contains a {@linkcode progress.Ref}
292 * and methods to track {@linkcode Input}s.
293 */
294// TODO: still not happy with the naming of this structure
295export class Run<Key extends Serializable = Serializable> implements progress.Ref {
296 #build: PendingBuild;
297 #inputs: Promise<Input>[] = [];
298 #closed = false;
299
300 constructor(options: {
301 key: Key;
302 progress: progress.Ref;
303 build: PendingBuild;
304 }) {
305 this.key = options.key;
306 this.start = options.progress.start;
307 this.#build = options.build;
308 }
309
310 start: progress.Ref["start"];
311 /** as-is from the key argument in `.cachedWork()` */
312 key: Key;
313 /** appends an {@linkcode Input} to the work's inputs list */
314 track(input: Input | Promise<Input>) {
315 ASSERT(!this.#closed, "cannot track an input after the step has finished");
316 ASSERT(!this.#build.closed, "cannot track an input after the build has closed");
317 this.#inputs.push(Promise.resolve(input));
318 }
319
320 /** @internal stop tracking and return the inputs recorded by this step. */
321 async finish(): Promise<Input[]> {
322 this.close();
323 return await Promise.all(this.#inputs);
324 }
325
326 /** @internal stop accepting new inputs. */
327 close(): void {
328 this.#closed = true;
329 }
330
331 /** appends all of the inputs from `step`, unwrapping it's promise */
332 async readStep<T>(step: Step<T>): Promise<T> {
333 void step;
334 throw new Error("Run.readStep is not implemented yet");
335 }
336 /** opens a file for reading */
337 async readFile(path: string): Promise<File> {
338 void path;
339 throw new Error("Run.readFile is not implemented yet");
340 }
341 // /** lists all of the children of a directory */
342 // async readDir(path: string): Promise<string[]> {}
343 // /** throws on HTTP or network I/O failure */
344 // fetchFile(input: string | URL | Request, init?: RequestInit): Promise<File>;
345 // /** throws only on network I/O failure */
346 // fetchResponse(
347 // input: string | URL | Request,
348 // init?: RequestInit,
349 // options?: { cache?: "all" | "body" },
350 // ): Promise<Response>;
351 // /** load a module, with a "trust me bro" type parameter */
352 // import<T = unknown>(absPath: string): Promise<T> { }
353 // /** follow all the imports of a module without actually executing it. throws on syntax and resolution errors. */
354 // trackModule(absPath: string): Promise<void>;
355}
356
357export interface Fs {
358 /** read a whole UTF-8 text file */
359 readTextFile(path: string): Promise<string>;
360 readDir(path: string): Promise<string[]>;
361 realPath(abs: string): Promise<string>;
362 stat(path: string): Promise<Stat>;
363
364 open(path: string): Promise<FileHandle>;
365 statHandle(handle: FileHandle): Promise<Stat>;
366 readHandle(
367 handle: FileHandle,
368 offset: number,
369 length: number,
370 destination: Uint8Array,
371 ): Promise<number>;
372 closeHandle(handle: FileHandle): Promise<void>;
373}
374
375export type FileHandle = ts.Brand<"@clo/lib/incremental/FileHandle"> & number;
376
377export interface Stat {
378 kind: "file" | "directory" | "other";
379 size: number;
380 time: number;
381}
382
383function defaultFs(): Fs {
384 return {
385 readTextFile: (path: string) => fs.readFile(path, "utf-8"),
386 readDir: fs.readdir,
387 realPath: fs.realpath,
388 async stat(path) {
389 const st = await fs.stat(path);
390 return {
391 kind: st.isFile() ? "file" : st.isDirectory() ? "directory" : "other",
392 size: st.size,
393 time: st.mtimeMs,
394 };
395 },
396 async open(path) {
397 return await new Promise<FileHandle>((resolve, reject) => {
398 fscb.open(path, "r", (error, fd) => {
399 if (error) reject(error);
400 else resolve(fd as FileHandle);
401 });
402 });
403 },
404 async readHandle(handle, offset, length, destination) {
405 const result = await new Promise<number>((resolve, reject) => {
406 fscb.read(
407 handle,
408 destination,
409 0,
410 length,
411 offset,
412 (error, bytesRead) => {
413 if (error) reject(error);
414 else resolve(bytesRead);
415 },
416 );
417 });
418 return result;
419 },
420 async statHandle(handle) {
421 const result = await new Promise<Stat>((resolve, reject) => {
422 fscb.fstat(
423 handle,
424 (error, st) => {
425 if (error) reject(error);
426 else {resolve({
427 kind: st.isFile() ? "file" : st.isDirectory() ? "directory" : "other",
428 size: st.size,
429 time: st.mtimeMs,
430 });}
431 },
432 );
433 });
434 return result;
435 },
436 async closeHandle(handle) {
437 await new Promise<void>((resolve, reject) => {
438 fscb.close(handle, (error) => {
439 if (error) reject(error);
440 else resolve();
441 });
442 });
443 },
444 };
445}
446
447// TODO one day, this will just be a variant of the `Trace` object. for now just passing the source string
448export interface LoaderInput {
449 source: string;
450 file: string;
451 getPackageMeta(): Promise<PackageMeta | null>;
452}
453
454export type LoaderResult =
455 | {
456 content: string;
457 format: "js" | "ts" | "cjs";
458 }
459 | {
460 content: Serializable;
461 format: "serializable";
462 };
463
464type LoaderFunction = (input: LoaderInput) => ts.Awaitable<LoaderResult>;
465
466export interface GraphOptions {
467 /** file path to a database, or `null` for an in-memory database. */
468 db: Database | string | null;
469
470 /** register dynamic sources. */
471 sources: SourceOptions[];
472
473 /** override the filesystem implementation. disables CommonJS support. */
474 fs?: Fs;
475 /**
476 * key is an extension including the `.`, mapping to a function to load it's
477 * source code. appended or overwrites the default JavaScript+Typescript
478 * (`.[mc]?[tj]s`), `.json`, `.node` loaders.
479 */
480 loaders?: Record<string, LoaderFunction>;
481 /** appended to the defaults. */
482 conditions?: string[];
483 /** caller-owned shared package.json metadata cache. */
484 packageJsonCache?: Map<string, PackageMeta>;
485}
486
487/**
488 * a reference to this object is needed to track sources
489 */
490interface SourceOptions {
491 protocol: string;
492 verify(
493 record: SourceRecord,
494 ): Promise<{ value: Serializable; kind: string } | "unchanged" | "changed">;
495 query(url: URL): Promise<{ value: Serializable } & Input>;
496}
497
498let fileSource: SourceOptions = {
499 protocol: "file",
500 async verify(record) {
501 throw new Error("Function not implemented.");
502 },
503 async query(url) {
504 throw new Error("Function not implemented.");
505 },
506};
507
508/** in form `${namespace}:${subPath}` */
509type Key = `${string}:${string}`;
510
511function toKey(namespace: string, subPath: string): Key {
512 ASSERT(!namespace.includes(":"));
513 return `${namespace}:${subPath}` as Key;
514}
515
516function parseKey(key: Key): { namespace: string; subpath: string } {
517 const [_, namespace, subpath] = UNWRAP(
518 key.match(/^([^:]+):(.*)$/),
519 `invalid key format: ${key}`,
520 ) as [string, string, string];
521 return { namespace, subpath };
522}
523
524/** holds module loading state */
525export class Graph {
526 db: Database;
527 #fs: Fs;
528 #loaders: Record<string, LoaderFunction>;
529 #conditions: string[];
530 #packageJsonCache: Map<string, PackageMeta>;
531 #usesNativeFs: boolean;
532 registry: Map<Key, GraphEntry>;
533
534 constructor(options: GraphOptions) {
535 if (typeof vm.SourceTextModule !== "function") {
536 throw new Error(
537 "@clo/lib/incremental requires node to run with --experimental-vm-modules",
538 );
539 }
540 this.db = options.db instanceof Database ? options.db : new Database(options.db);
541 this.#fs = options.fs ?? defaultFs();
542 this.#loaders = { ...defaultLoaders, ...options.loaders };
543 this.#conditions = [...defaultConditions, ...(options.conditions ?? [])];
544 this.#packageJsonCache = options.packageJsonCache ?? new Map();
545 this.#usesNativeFs = !options.fs;
546 this.registry = new Map();
547 }
548
549 /**
550 * runs the provided build pipeline. the pipeline is an orchestration of
551 * {@linkcode StepFn|steps}, async functions that read and process data. by
552 * keeping all actual reads in these step functions, this means an incremental
553 * re-run can quickly skip all of the actual logic. this can only be done
554 * because the caching primitives are perfect. at the end, the pipeline should
555 * write/copy its intended outputs with the output APIs in {@linkcode Ref}.
556 */
557 async incrementalBuild<T>(options: {
558 buildAndWriteOutput: (topLevel: Ref) => Promise<T>;
559 progress: progress.Ref;
560 }): Promise<T> {
561 const build: PendingBuild = {
562 inputs: new Map(),
563 steps: new Set(),
564 activeSteps: new Set(),
565 failures: [],
566 closed: false,
567 };
568 let result: T;
569 let thrown: unknown;
570 let didThrow = false;
571 try {
572 result = await options.buildAndWriteOutput(
573 new Ref({
574 graph: this,
575 progress: options.progress,
576 build,
577 }),
578 );
579 } catch (error) {
580 thrown = error;
581 didThrow = true;
582 } finally {
583 await waitForBuildSteps(build);
584 build.closed = true;
585 }
586
587 if (didThrow) throw thrown;
588 const failures = build.failures.flatMap((failure) => {
589 const result = failure();
590 return result.observed ? [] : [result.error];
591 });
592 if (failures.length > 0) throw new AggregateError(failures);
593
594 // TODO: post-build stuff. for example, assert that all input files are still ok?
595 return result!;
596 }
597
598 /** load a module without tracking its owner */
599 // TODO: tracing
600 async import<T = unknown>(absPath: string): Promise<T> {
601 const root = toKey("file", await this.#fs.realPath(absPath));
602 const entry = await this.#loadModule.get(root);
603 const queue = [entry];
604 const visited = new Set<Key>([root]);
605
606 while (queue.length > 0) {
607 const current = queue.shift()!;
608 if (current.module instanceof vm.SyntheticModule) continue;
609
610 const dependencies = await Promise.all(
611 current.resolutions.map(
612 async (key) => (await this.#loadModule.get(key)).module,
613 ),
614 );
615 current.module.linkRequests(dependencies);
616
617 for (let i = 0; i < current.resolutions.length; i++) {
618 const dependencyKey = current.resolutions[i]!;
619 if (visited.has(dependencyKey)) continue;
620 visited.add(dependencyKey);
621 queue.push(await this.#loadModule.get(dependencyKey));
622 }
623 }
624
625 if (entry.module instanceof vm.SourceTextModule) entry.module.instantiate();
626 await entry.module.evaluate();
627 return entry.module.namespace as T;
628 }
629
630 #loadModule = new async.DedupeConcurrent(
631 async (key: Key): Promise<GraphEntry> => {
632 const cached = this.registry.get(key);
633 if (cached) return cached;
634
635 const { namespace, subpath } = parseKey(key);
636 try {
637 let entry: GraphEntry;
638 if (namespace === "file") {
639 entry = await this.#createEntryFromFile(subpath);
640 } else if (namespace === "node") {
641 entry = await this.#createEntryFromNodeBuiltin(subpath);
642 } else {
643 throw new Error(`unsupported module namespace: ${namespace}`);
644 }
645
646 this.registry.set(key, entry);
647 return entry;
648 } catch (err) {
649 exception.rethrowWithMetadata(err, { namespace, subpath });
650 }
651 },
652 ([key]) => key,
653 );
654
655 /** constructs a new graph entry */
656 async #createEntryFromFile(absPath: string): Promise<GraphEntry> {
657 const ext = path.extname(absPath);
658 const loader = this.#loaders[ext];
659 if (!loader) {
660 throw new Error(
661 `No loader registered for extension ${JSON.stringify(ext)}`,
662 );
663 }
664
665 const loaded = await loader({
666 file: absPath,
667 source: await this.#fs.readTextFile(absPath),
668 getPackageMeta: async () => {
669 return await this.lookupPackageMeta(path.dirname(absPath));
670 },
671 });
672
673 if (loaded.format === "serializable") {
674 const value = loaded.content;
675 return {
676 module: new vm.SyntheticModule(
677 ["default"],
678 function() {
679 this.setExport("default", value);
680 },
681 { identifier: absPath },
682 ),
683 resolutions: [],
684 };
685 }
686
687 let content = loaded.content;
688 if (loaded.format === "ts") {
689 try {
690 content = module.stripTypeScriptTypes(content);
691 } catch {
692 content = module.stripTypeScriptTypes(content, { mode: "transform" });
693 }
694 }
695
696 if (loaded.format === "cjs") {
697 if (!this.#usesNativeFs) {
698 throw new Error(
699 `CommonJS support is not implemented for a virtual fs.`,
700 );
701 }
702 return await this.#createEntryFromCommonJs(absPath);
703 }
704
705 const mod = new vm.SourceTextModule(content, {
706 identifier: absPath,
707 initializeImportMeta(meta) {
708 meta.dirname = path.dirname(absPath);
709 meta.filename = absPath;
710 meta.url = url.pathToFileURL(absPath).href;
711 meta.main = false;
712 },
713 });
714
715 const resolutions = await Promise.all(
716 mod.moduleRequests.map(({ specifier }) => this.#resolveDependency(absPath, specifier, false)),
717 );
718
719 return { module: mod, resolutions };
720 }
721
722 /** constructs a new graph entry */
723 async #createEntryFromNodeBuiltin(name: string): Promise<GraphEntry> {
724 const builtin = node.process?.getBuiltinModule(name as never);
725 if (!builtin) {
726 throw new Error(`Could not load builtin module ${JSON.stringify(name)}`);
727 }
728 return {
729 module: new vm.SyntheticModule(
730 Object.keys(builtin).concat("default"),
731 function() {
732 for (const key of Object.keys(builtin)) {
733 this.setExport(key, builtin[key as keyof typeof builtin]);
734 }
735 this.setExport("default", builtin);
736 },
737 { identifier: `node:${name}` },
738 ),
739 resolutions: [],
740 };
741 }
742
743 async #resolveDependency(
744 fromFile: string,
745 specifier: string,
746 sloppy: boolean,
747 ): Promise<Key> {
748 const result = await resolve({
749 specifier,
750 resolveDir: path.dirname(fromFile),
751 fs: this.#fs,
752 sloppy,
753 conditions: this.#conditions,
754 packageJsonCache: this.#packageJsonCache,
755 });
756 if (result.type === "error") {
757 exception.rethrowWithMetadata(result.error, {
758 source: fromFile,
759 specifier, // TODO: error message should always be clear as day
760 });
761 }
762 return toKey(result.namespace, result.subPath);
763 }
764
765 async lookupPackageMeta(startDir: string): Promise<PackageMeta | null> {
766 for (const dir of walkUpDirectories(startDir)) {
767 const packageJsonPath = path.join(dir, "package.json");
768 try {
769 const stat = await this.#fs.stat(packageJsonPath);
770 if (stat.kind !== "file") continue;
771 const meta = await readPackageJsonMeta(
772 this.#fs,
773 packageJsonPath,
774 this.#packageJsonCache,
775 );
776 return meta;
777 } catch (error) {
778 if (exception.code(error) === "ENOENT") continue;
779 throw error;
780 }
781 }
782 return null;
783 }
784
785 async #createEntryFromCommonJs(absPath: string): Promise<GraphEntry> {
786 const exportsValue = module.createRequire(absPath)(absPath);
787 const exportNames = ["default", ...getCommonJsNamedExports(exportsValue)];
788 const synthetic = new vm.SyntheticModule(
789 exportNames,
790 function() {
791 this.setExport("default", exportsValue);
792 for (const name of exportNames) {
793 if (name === "default") continue;
794 this.setExport(name, (exportsValue as Record<string, unknown>)[name]);
795 }
796 },
797 { identifier: absPath },
798 );
799 return {
800 module: synthetic,
801 resolutions: [],
802 };
803 }
804}
805
806interface GraphEntry {
807 module: vm.SourceTextModule | vm.SyntheticModule;
808 resolutions: Key[];
809}
810
811export interface PackageMeta {
812 type: "module" | "commonjs" | null;
813 imports: unknown;
814 exports: unknown;
815}
816
817const defaultLoaders: Record<string, LoaderFunction> = {
818 ".js": async ({ source, getPackageMeta }) => ({
819 content: source,
820 format: (
821 {
822 commonjs: "cjs",
823 module: "js",
824 } as const
825 )[(await getPackageMeta())?.type ?? "commonjs"],
826 }),
827 ".ts": async ({ source }) => ({ content: source, format: "ts" }),
828 ".mjs": async ({ source }) => ({ content: source, format: "js" }),
829 ".cjs": async ({ source }) => ({ content: source, format: "cjs" }),
830 ".mts": async ({ source }) => ({ content: source, format: "ts" }),
831
832 ".json": async ({ source }) => ({
833 content: JSON.parse(source),
834 format: "serializable",
835 }),
836};
837
838// --- module resolution ---
839
840export interface ResolveOptions {
841 specifier: string;
842 resolveDir: string;
843 /** override the filesystem implementation */
844 fs?: Fs;
845 /**
846 * enable legacy commonjs resolution features
847 * @default false
848 */
849 sloppy?: boolean;
850 conditions?: string[];
851 /** caller-owned shared package.json metadata cache */
852 packageJsonCache?: Map<string, PackageMeta>;
853}
854
855export type Resolution =
856 & {
857 /** the files that could have affected this resolution */
858 files: string[];
859 }
860 & (
861 | {
862 type: "success";
863 namespace: string;
864 subPath: string;
865 error?: undefined;
866 }
867 | {
868 type: "error";
869 error: unknown;
870 namespace?: undefined;
871 subPath?: undefined;
872 }
873 );
874
875const defaultConditions = ["node", "import", "default"];
876const sloppyExtensions = [".js", ".json", ".node"];
877
878/**
879 * perform filesystem-based module resolution. by default, this adheres to
880 * everything node.js supports in it's ESM import resolution. the difference
881 * between this and something like `require.resolve` is it returns the full list
882 * of paths that could affect the outcome of the resolution.
883 */
884export async function resolve({
885 specifier,
886 resolveDir,
887 fs = defaultFs(),
888 sloppy = false,
889 conditions = defaultConditions,
890 packageJsonCache = new Map<string, PackageMeta>(),
891}: ResolveOptions): Promise<Resolution> {
892 const r = new Resolver(fs, conditions, packageJsonCache);
893 try {
894 const { namespace, subPath } = await r.resolve(
895 specifier,
896 resolveDir,
897 sloppy,
898 );
899 return { type: "success", namespace, subPath, files: Array.from(r.files) };
900 } catch (error) {
901 return { type: "error", error, files: Array.from(r.files) };
902 }
903}
904
905/** sanity class to track file read states during module resolution */
906class Resolver {
907 fs: Fs;
908 files = new Set<string>();
909 conditions: string[];
910 packageJsonCache: Map<string, PackageMeta>;
911
912 constructor(
913 fs: Fs,
914 conditions: string[],
915 packageJsonCache: Map<string, PackageMeta>,
916 ) {
917 this.fs = fs;
918 this.conditions = conditions;
919 this.packageJsonCache = packageJsonCache;
920 }
921
922 async resolve(
923 specifier: string,
924 resolveDir: string,
925 sloppy: boolean,
926 ): Promise<{ namespace: string; subPath: string }> {
927 const isNodeBuiltin = specifier.startsWith("node:");
928 if (isNodeBuiltin || module.builtinModules.includes(specifier)) {
929 return {
930 namespace: "node",
931 subPath: isNodeBuiltin ? specifier.slice(5) : specifier,
932 };
933 }
934
935 if (
936 specifier.startsWith("/")
937 || specifier.startsWith("./")
938 || specifier.startsWith("../")
939 ) {
940 const abs = path.resolve(resolveDir, specifier);
941 const result = sloppy
942 ? await this.resolvePathSloppy(abs)
943 : await this.resolvePathExact(abs);
944 return { namespace: "file", subPath: await this.fs.realPath(result) };
945 }
946
947 if (specifier.startsWith("#")) {
948 const packageJsonPath = await this.findClosestPackageJson(resolveDir);
949 if (!packageJsonPath) {
950 this.throwWithCode(
951 `Cannot find package.json for import ${JSON.stringify(specifier)} from ${resolveDir}`,
952 "ERR_PACKAGE_IMPORT_NOT_DEFINED",
953 );
954 }
955 const meta = await this.readPackageJsonMeta(packageJsonPath);
956 const target = this.resolvePackageImportCondition(
957 meta.imports,
958 specifier,
959 );
960 if (!target) {
961 this.throwWithCode(
962 `Package import specifier ${JSON.stringify(specifier)} is not defined in ${packageJsonPath}`,
963 "ERR_PACKAGE_IMPORT_NOT_DEFINED",
964 );
965 }
966 return {
967 namespace: "file",
968 subPath: await this.resolvePackageTarget(
969 path.dirname(packageJsonPath),
970 target,
971 sloppy,
972 ),
973 };
974 }
975
976 if (!this.isBareSpecifier(specifier)) {
977 this.throwWithCode(
978 `Invalid module specifier ${JSON.stringify(specifier)}`,
979 "ERR_INVALID_MODULE_SPECIFIER",
980 );
981 }
982
983 const resolved = await this.resolveBareSpecifier(
984 specifier,
985 resolveDir,
986 sloppy,
987 );
988 return { namespace: "file", subPath: await this.fs.realPath(resolved) };
989 }
990
991 async stat(file: string): Promise<Stat | null> {
992 this.files.add(file);
993 try {
994 return await this.fs.stat(file);
995 } catch (error) {
996 if (exception.code(error) === "ENOENT") return null;
997 throw error;
998 }
999 }
1000
1001 async readJsonObject(file: string): Promise<Record<string, unknown>> {
1002 this.files.add(file);
1003 return JSON.parse(await this.fs.readTextFile(file)) as Record<string, unknown>;
1004 }
1005
1006 async readPackageJsonMeta(file: string): Promise<PackageMeta> {
1007 this.files.add(file);
1008 const cached = this.packageJsonCache.get(file);
1009 if (cached) return cached;
1010
1011 const json = JSON.parse(await this.fs.readTextFile(file)) as Record<
1012 string,
1013 unknown
1014 >;
1015 const meta: PackageMeta = {
1016 type: json.type === "module" || json.type === "commonjs" ? json.type : null,
1017 imports: json.imports,
1018 exports: json.exports,
1019 };
1020 this.packageJsonCache.set(file, meta);
1021 return meta;
1022 }
1023
1024 async resolvePathExact(abs: string): Promise<string> {
1025 const stat = await this.stat(abs);
1026 if (!stat) {
1027 this.throwWithCode(
1028 `Cannot find module ${JSON.stringify(abs)}`,
1029 "ERR_MODULE_NOT_FOUND",
1030 );
1031 }
1032 if (stat.kind === "directory") {
1033 this.throwWithCode(
1034 `Directory import ${JSON.stringify(abs)} is not supported`,
1035 "ERR_UNSUPPORTED_DIR_IMPORT",
1036 );
1037 }
1038 if (stat.kind !== "file") {
1039 this.throwWithCode(
1040 `Cannot load module ${JSON.stringify(abs)}`,
1041 "ERR_MODULE_NOT_FOUND",
1042 );
1043 }
1044 return abs;
1045 }
1046
1047 async resolvePathSloppy(abs: string): Promise<string> {
1048 const exact = await this.stat(abs);
1049 if (exact?.kind === "file") return abs;
1050
1051 for (const ext of sloppyExtensions) {
1052 const candidate = `${abs}${ext}`;
1053 const stat = await this.stat(candidate);
1054 if (stat?.kind === "file") return candidate;
1055 }
1056
1057 const packageJson = path.join(abs, "package.json");
1058 const packageStat = await this.stat(packageJson);
1059 if (packageStat?.kind === "file") {
1060 const meta = await this.readPackageJsonMeta(packageJson);
1061 let target = this.resolvePackageImportCondition(meta.exports, ".");
1062 if (!target) {
1063 const json = await this.readJsonObject(packageJson);
1064 const moduleField = typeof json.module === "string" ? json.module : null;
1065 const mainField = typeof json.main === "string" ? json.main : null;
1066 target = moduleField ?? mainField;
1067 }
1068 if (target) {
1069 return await this.resolveLegacyPackageTarget(abs, target, true);
1070 }
1071 }
1072
1073 for (const ext of sloppyExtensions) {
1074 const indexCandidate = path.join(abs, `index${ext}`);
1075 const stat = await this.stat(indexCandidate);
1076 if (stat?.kind === "file") return indexCandidate;
1077 }
1078
1079 if (exact?.kind === "directory" || packageStat?.kind === "file") {
1080 this.throwWithCode(
1081 `Directory import ${JSON.stringify(abs)} is not supported`,
1082 "ERR_UNSUPPORTED_DIR_IMPORT",
1083 );
1084 }
1085
1086 this.throwWithCode(
1087 `Cannot find module ${JSON.stringify(abs)}`,
1088 "ERR_MODULE_NOT_FOUND",
1089 );
1090 }
1091
1092 async findClosestPackageJson(startDir: string): Promise<string | null> {
1093 for (const dir of walkUpDirectories(startDir)) {
1094 const candidate = path.join(dir, "package.json");
1095 const stat = await this.stat(candidate);
1096 if (stat?.kind === "file") return candidate;
1097 }
1098 return null;
1099 }
1100
1101 async resolveBareSpecifier(
1102 specifier: string,
1103 resolveDir: string,
1104 sloppy: boolean,
1105 ): Promise<string> {
1106 const { packageName, packageSubpath } = this.parseBareSpecifier(specifier);
1107
1108 for (const dir of walkUpDirectories(resolveDir)) {
1109 const packageDir = path.join(dir, "node_modules", packageName);
1110 const packageJsonPath = path.join(packageDir, "package.json");
1111 const packageJsonStat = await this.stat(packageJsonPath);
1112 if (!packageJsonStat) continue;
1113 if (packageJsonStat.kind !== "file") {
1114 this.throwWithCode(
1115 `Invalid package.json for ${packageName}`,
1116 "ERR_MODULE_NOT_FOUND",
1117 );
1118 }
1119
1120 const meta = await this.readPackageJsonMeta(packageJsonPath);
1121 const subpath = packageSubpath ? `./${packageSubpath}` : ".";
1122
1123 if (meta.exports !== undefined) {
1124 const target = this.resolvePackageImportCondition(
1125 meta.exports,
1126 subpath,
1127 );
1128 if (!target) {
1129 this.throwWithCode(
1130 `Package subpath ${JSON.stringify(subpath)} is not exported by ${packageName}`,
1131 "ERR_PACKAGE_PATH_NOT_EXPORTED",
1132 );
1133 }
1134 return await this.resolvePackageTarget(packageDir, target, sloppy);
1135 }
1136
1137 if (subpath !== ".") {
1138 return sloppy
1139 ? await this.resolvePathSloppy(path.join(packageDir, packageSubpath))
1140 : await this.resolvePathExact(path.join(packageDir, packageSubpath));
1141 }
1142
1143 const json = await this.readJsonObject(packageJsonPath);
1144 const mainField = typeof json.module === "string"
1145 ? json.module
1146 : typeof json.main === "string"
1147 ? json.main
1148 : null;
1149 if (mainField) {
1150 return await this.resolveLegacyPackageTarget(
1151 packageDir,
1152 mainField,
1153 sloppy,
1154 );
1155 }
1156
1157 return sloppy
1158 ? await this.resolvePathSloppy(path.join(packageDir, "index"))
1159 : await this.resolvePathExact(path.join(packageDir, "index.js"));
1160 }
1161
1162 this.throwWithCode(
1163 `Cannot find package ${JSON.stringify(packageName)}`,
1164 "ERR_MODULE_NOT_FOUND",
1165 );
1166 }
1167
1168 async resolvePackageTarget(
1169 packageDir: string,
1170 target: string,
1171 sloppy: boolean,
1172 ): Promise<string> {
1173 if (target.startsWith("#")) {
1174 const packageJsonPath = await this.findClosestPackageJson(packageDir);
1175 if (!packageJsonPath) {
1176 this.throwWithCode(
1177 `Package import specifier ${JSON.stringify(target)} is not defined`,
1178 "ERR_PACKAGE_IMPORT_NOT_DEFINED",
1179 );
1180 }
1181 const meta = await this.readPackageJsonMeta(packageJsonPath);
1182 const nestedTarget = this.resolvePackageImportCondition(
1183 meta.imports,
1184 target,
1185 );
1186 if (!nestedTarget) {
1187 this.throwWithCode(
1188 `Package import specifier ${JSON.stringify(target)} is not defined in ${packageJsonPath}`,
1189 "ERR_PACKAGE_IMPORT_NOT_DEFINED",
1190 );
1191 }
1192 return await this.resolvePackageTarget(
1193 path.dirname(packageJsonPath),
1194 nestedTarget,
1195 sloppy,
1196 );
1197 }
1198 if (
1199 !target.startsWith("./")
1200 && !target.startsWith("../")
1201 && !target.startsWith("/")
1202 ) {
1203 this.throwWithCode(
1204 `Invalid package target ${JSON.stringify(target)}`,
1205 "ERR_INVALID_PACKAGE_TARGET",
1206 );
1207 }
1208 const abs = path.resolve(packageDir, target);
1209 return sloppy
1210 ? await this.resolvePathSloppy(abs)
1211 : await this.resolvePathExact(abs);
1212 }
1213
1214 async resolveLegacyPackageTarget(
1215 packageDir: string,
1216 target: string,
1217 sloppy: boolean,
1218 ): Promise<string> {
1219 if (target.startsWith("#")) {
1220 return await this.resolvePackageTarget(packageDir, target, sloppy);
1221 }
1222 if (
1223 target.startsWith("./")
1224 || target.startsWith("../")
1225 || target.startsWith("/")
1226 ) {
1227 return await this.resolvePackageTarget(packageDir, target, sloppy);
1228 }
1229 const abs = path.resolve(packageDir, target);
1230 return sloppy
1231 ? await this.resolvePathSloppy(abs)
1232 : await this.resolvePathExact(abs);
1233 }
1234
1235 isBareSpecifier(specifier: string): boolean {
1236 return (
1237 !specifier.startsWith(".")
1238 && !specifier.startsWith("/")
1239 && !specifier.startsWith("#")
1240 );
1241 }
1242
1243 parseBareSpecifier(specifier: string): {
1244 packageName: string;
1245 packageSubpath: string;
1246 } {
1247 const parts = specifier.split("/");
1248 const packageName = specifier.startsWith("@")
1249 ? parts.slice(0, 2).join("/")
1250 : (parts[0] ?? "");
1251 const packageSubpath = specifier
1252 .slice(packageName.length)
1253 .replace(/^\/+/, "");
1254 return { packageName, packageSubpath };
1255 }
1256
1257 resolvePackageImportCondition(
1258 map: unknown,
1259 specifier: string,
1260 ): string | null {
1261 if (map !== null && typeof map === "object" && !Array.isArray(map)) {
1262 const keys = Object.keys(map as object);
1263 const hasPathKeys = keys.some(
1264 (k) => k.startsWith(".") || k.startsWith("#"),
1265 );
1266 if (hasPathKeys) {
1267 return this.resolveSubpathMap(
1268 map as Record<string, unknown>,
1269 specifier,
1270 );
1271 }
1272 }
1273 return this.resolveTarget(map, null);
1274 }
1275
1276 private resolveSubpathMap(
1277 matchObj: Record<string, unknown>,
1278 matchKey: string,
1279 ): string | null {
1280 if (matchKey in matchObj) {
1281 return this.resolveTarget(matchObj[matchKey], null);
1282 }
1283
1284 const patternKeys = Object.keys(matchObj)
1285 .filter((k) => k.includes("*"))
1286 .sort((a, b) => b.indexOf("*") - a.indexOf("*"));
1287
1288 for (const expansionKey of patternKeys) {
1289 const starIdx = expansionKey.indexOf("*");
1290 const patternBase = expansionKey.slice(0, starIdx);
1291 const patternTrailer = expansionKey.slice(starIdx + 1);
1292
1293 if (!matchKey.startsWith(patternBase)) continue;
1294 if (matchKey.length < expansionKey.length) continue;
1295 if (patternTrailer.length > 0 && !matchKey.endsWith(patternTrailer)) {
1296 continue;
1297 }
1298
1299 const patternMatch = matchKey.slice(
1300 patternBase.length,
1301 matchKey.length - patternTrailer.length,
1302 );
1303 return this.resolveTarget(matchObj[expansionKey], patternMatch);
1304 }
1305
1306 return null;
1307 }
1308
1309 private resolveTarget(
1310 target: unknown,
1311 patternMatch: string | null,
1312 ): string | null {
1313 if (typeof target === "string") {
1314 if (!target.startsWith("./") && !target.startsWith("#")) {
1315 this.throwWithCode(
1316 `invalid target: "${target}"`,
1317 "ERR_INVALID_PACKAGE_TARGET",
1318 );
1319 }
1320 return patternMatch !== null
1321 ? target.replaceAll("*", patternMatch)
1322 : target;
1323 }
1324
1325 if (Array.isArray(target)) {
1326 for (const item of target) {
1327 try {
1328 const result = this.resolveTarget(item, patternMatch);
1329 if (result !== null) return result;
1330 } catch {
1331 //
1332 }
1333 }
1334 return null;
1335 }
1336
1337 if (target !== null && typeof target === "object") {
1338 for (
1339 const [key, value] of Object.entries(
1340 target as Record<string, unknown>,
1341 )
1342 ) {
1343 if (key === "default" || this.conditions.includes(key)) {
1344 const result = this.resolveTarget(value, patternMatch);
1345 if (result !== null) return result;
1346 }
1347 }
1348 }
1349
1350 return null;
1351 }
1352
1353 private throwWithCode(message: string, code: node.ErrorCode): never {
1354 exception.rethrowWithMetadata(new Error(message), { code });
1355 }
1356}
1357
1358async function readPackageJsonMeta(
1359 fs: Fs,
1360 file: string,
1361 packageJsonCache: Map<string, PackageMeta>,
1362): Promise<PackageMeta> {
1363 const cached = packageJsonCache.get(file);
1364 if (cached) return cached;
1365
1366 const json = JSON.parse(await fs.readTextFile(file)) as Record<string, unknown>;
1367 const meta: PackageMeta = {
1368 type: json.type === "module" || json.type === "commonjs" ? json.type : null,
1369 imports: json.imports,
1370 exports: json.exports,
1371 };
1372 packageJsonCache.set(file, meta);
1373 return meta;
1374}
1375
1376function* walkUpDirectories(startDir: string): Generator<string> {
1377 let current = path.resolve(startDir);
1378 while (true) {
1379 yield current;
1380 const parent = path.dirname(current);
1381 if (parent === current) break;
1382 current = parent;
1383 }
1384}
1385
1386function getCommonJsNamedExports(value: unknown): string[] {
1387 if (!value || typeof value !== "object" || Array.isArray(value)) return [];
1388 const proto = Object.getPrototypeOf(value);
1389 if (proto !== Object.prototype && proto !== null) return [];
1390 return Array.from(
1391 new Set(Object.keys(value).filter((key) => key !== "default")),
1392 );
1393}
1394
1395// --- database & serialization ---
1396export type RowId = number | bigint;
1397
1398/**
1399 * a file, network request, or other input referenced within a
1400 * {@linkcode StepFn} function. unlike the `key` to a cache boundary, `Input`
1401 * urls can stay constant but have underlying changes. file paths are the main
1402 * use case, but this also extends to something like a database query.
1403 */
1404export interface Input {
1405 /** global key for the changable object, with enough information to re-query. */
1406 url: string;
1407 /** inline data known about this object, such as a file kind. */
1408 kind: string;
1409
1410 // these values are dependant on the implementation of whatever revalidates if
1411 // an `Input` has changed, and are modelled after a filesystem stat.
1412
1413 /** modification time or revision number. */
1414 time: number;
1415 /** byte size or other cheap source-specific magnitude. */
1416 size: number;
1417 /** a freshness token, such as a content hash. */
1418 hash: string;
1419}
1420
1421export interface SourceRecord extends Input {
1422 id: RowId;
1423}
1424
1425export interface CacheRecord {
1426 id: RowId;
1427 inputs: IterableIterator<SourceRecord>;
1428 /** read via `readBlob` */
1429 blob: RowId;
1430}
1431
1432/** dangerous trust-me-bro types to quickly add some type safety to sqlite */
1433type TypedSqliteStatement<
1434 Input extends object,
1435 Output extends object = never,
1436> = Omit<sqlite.StatementSync, "run" | "get" | "all" | "iterate"> & {
1437 run(input: Input): sqlite.StatementResultingChanges;
1438 get(input: Input): Output | undefined;
1439 all(input: Input): Output[];
1440 iterate(input: Input): IterableIterator<Output>;
1441};
1442
1443function prepareQueries(sql: sqlite.DatabaseSync) {
1444 return {
1445 insertBlob: sql.prepare(
1446 "insert into blobs (type, content) values ($type, $content);",
1447 ) as unknown as TypedSqliteStatement<{
1448 type: "j" | "s" | "b";
1449 content: string | Uint8Array;
1450 }>,
1451 readBlob: sql.prepare(
1452 "select type, content from blobs where id = $id;",
1453 ) as unknown as TypedSqliteStatement<
1454 { id: RowId },
1455 { type: "j" | "s" | "b"; content: string | Uint8Array }
1456 >,
1457 upsertSource: sql.prepare(
1458 "insert into sources (url, kind, time, size, hash) "
1459 + "values ($url, $kind, $time, $size, $hash) "
1460 + "on conflict(url) do update set "
1461 + "kind = excluded.kind, "
1462 + "time = excluded.time, "
1463 + "size = excluded.size, "
1464 + "hash = excluded.hash;",
1465 ) as unknown as TypedSqliteStatement<{
1466 url: string;
1467 kind: string;
1468 time: number;
1469 size: number;
1470 hash: string;
1471 }>,
1472 readSource: sql.prepare(
1473 "select id, url, kind, time, size, hash from sources where url = $url;",
1474 ) as unknown as TypedSqliteStatement<{ url: string }, SourceRecord>,
1475 readSources: sql.prepare(
1476 "select id, url, kind, time, size, hash from sources order by url;",
1477 ) as unknown as TypedSqliteStatement<Record<string, never>, SourceRecord>,
1478 selectSourceId: sql.prepare(
1479 "select id from sources where url = $url;",
1480 ) as unknown as TypedSqliteStatement<{ url: string }, { id: RowId }>,
1481 insertCache: sql.prepare(
1482 "insert into caches (key, blob) values ($key, $blob) "
1483 + "on conflict(key) do update set blob = excluded.blob;",
1484 ) as unknown as TypedSqliteStatement<{ key: string; blob: RowId }>,
1485 selectCache: sql.prepare(
1486 "select id, blob from caches where key = $key;",
1487 ) as unknown as TypedSqliteStatement<
1488 { key: string },
1489 { id: RowId; blob: RowId }
1490 >,
1491 deleteCacheInputs: sql.prepare(
1492 "delete from inputs where cache = $cache;",
1493 ) as unknown as TypedSqliteStatement<{ cache: RowId }>,
1494 insertInput: sql.prepare(
1495 "insert or ignore into inputs (source, cache) values ($source, $cache);",
1496 ) as unknown as TypedSqliteStatement<{ source: RowId; cache: RowId }>,
1497 readCacheInputs: sql.prepare(
1498 "select sources.id, sources.url, sources.kind, sources.time, "
1499 + "sources.size, sources.hash "
1500 + "from sources "
1501 + "join inputs on inputs.source = sources.id "
1502 + "where inputs.cache = $cache "
1503 + "order by sources.url;",
1504 ) as unknown as TypedSqliteStatement<{ cache: RowId }, SourceRecord>,
1505 };
1506}
1507
1508/** backing storage for incremental artifacts built on `node:sqlite` */
1509export class Database {
1510 sql: sqlite.DatabaseSync;
1511 #prep: ReturnType<typeof prepareQueries>;
1512 #cachedBlobs = new Map<RowId, Serializable>(); // WeakRef?
1513
1514 constructor(path: string | null) {
1515 ASSERT(!path?.startsWith(":"));
1516 this.sql = new sqlite.DatabaseSync(path ?? ":memory:");
1517 this.sql.exec(
1518 [
1519 "pragma foreign_keys = ON;",
1520
1521 "create table if not exists blobs (",
1522 /**/ "id integer primary key asc,",
1523 /**/ "type text CHECK(type IN ('j','s','b')) not null,",
1524 /**/ "content any not null",
1525 ") strict;",
1526
1527 "create table if not exists sources (",
1528 /**/ "id integer primary key asc,",
1529 /**/ "url text unique not null,",
1530 /**/ "kind text not null,",
1531 /**/ "time integer not null,",
1532 /**/ "size integer not null,",
1533 /**/ "hash text not null",
1534 ") strict;",
1535
1536 "create table if not exists caches (",
1537 /**/ "id integer primary key asc,",
1538 /**/ "key text unique not null,",
1539 /**/ "blob integer not null references blobs(id) on delete cascade",
1540 ") strict;",
1541
1542 "create table if not exists inputs (",
1543 /**/ "source integer not null references sources(id) on delete cascade,",
1544 /**/ "cache integer not null references caches(id) on delete cascade,",
1545 /**/ "primary key(source, cache)",
1546 ") strict;",
1547
1548 // TODO: name `a` and `b` to clearly draw the line
1549 "create table if not exists deps (",
1550 /**/ "a integer not null references sources(id) on delete cascade,",
1551 /**/ "b integer not null references sources(id) on delete cascade,",
1552 /**/ "primary key(a, b)",
1553 ") strict;",
1554 ].join(""),
1555 );
1556 this.#prep = prepareQueries(this.sql);
1557 }
1558
1559 /** forget serialized blobs that were held in memory */
1560 clearInMemoryCache() {
1561 this.#cachedBlobs.clear();
1562 }
1563
1564 writeSource(source: Input): RowId {
1565 this.#validateSource(source);
1566 this.#prep.upsertSource.run(source);
1567 return UNWRAP(this.#prep.selectSourceId.get({ url: source.url })).id;
1568 }
1569
1570 sources(): IterableIterator<SourceRecord> {
1571 return this.#prep.readSources.iterate({}) as IterableIterator<SourceRecord>;
1572 }
1573
1574 invalidate(source: RowId | Iterable<RowId>): void {
1575 const sources = typeof source === "number" || typeof source === "bigint"
1576 ? [source]
1577 : Array.from(source);
1578 if (sources.length === 0) return;
1579 const params = Object.fromEntries(
1580 sources.map((source, i) => [`source${i}`, source]),
1581 );
1582 const placeholders = sources.map((_, i) => `$source${i}`).join(", ");
1583 this.sql
1584 .prepare(
1585 `delete from caches where id in (`
1586 + `select cache from inputs where source in (${placeholders})`
1587 + `);`,
1588 )
1589 .run(params);
1590 }
1591
1592 /**
1593 * read a cache entry. caller should first examine `.inputs` to determine if
1594 * the cache value is truly fresh, and only then deserialize the blob.
1595 */
1596 readCache(key: string): CacheRecord | null {
1597 const row = this.#prep.selectCache.get({ key });
1598 if (!row) return null;
1599 return {
1600 id: row.id,
1601 blob: row.blob,
1602 inputs: this.#prep.readCacheInputs.iterate({ cache: row.id }),
1603 };
1604 }
1605
1606 /** read the source records linked to a cache entry. */
1607 readCacheInputs(cache: RowId): IterableIterator<SourceRecord> {
1608 return this.#prep.readCacheInputs.iterate({ cache });
1609 }
1610
1611 /** link a key to a value, with a set of `inputs` that bust this cache entry */
1612 writeCache(key: string, value: Serializable, inputs: Input[]): RowId {
1613 const { blob } = this.writeBlob(value);
1614 this.#prep.insertCache.run({ key, blob });
1615 const cache = UNWRAP(this.#prep.selectCache.get({ key })).id;
1616 this.#prep.deleteCacheInputs.run({ cache });
1617 for (const input of inputs) {
1618 const source = this.writeSource(input);
1619 this.#prep.insertInput.run({ source, cache });
1620 }
1621 return cache;
1622 }
1623
1624 /** write a {@linkcode Serializable} to the database */
1625 writeBlob(data: Serializable, cache = true): { blob: number | bigint } {
1626 let content: string | Uint8Array;
1627 let type: "j" | "s" | "b";
1628 if (typeof data === "string") {
1629 type = "s";
1630 content = data;
1631 } else if (data instanceof Uint8Array) {
1632 type = "b";
1633 content = data;
1634 } else {
1635 type = "j";
1636 content = JSON.stringify(this.#serializeObject(data, cache));
1637 }
1638 const insert = this.#prep.insertBlob.run({ type, content });
1639 if (cache) this.#cachedBlobs.set(insert.lastInsertRowid, data);
1640 return { blob: insert.lastInsertRowid };
1641 }
1642
1643 /** write a {@linkcode Serializable} via it's ID */
1644 readBlob(blob: RowId): Serializable {
1645 const cached = this.#cachedBlobs.get(blob);
1646 if (cached !== undefined || this.#cachedBlobs.has(blob)) return cached;
1647
1648 const row = this.#prep.readBlob.get({ id: blob });
1649 if (!row) throw new Error(`missing blob row ${String(blob)}`);
1650
1651 const value = row.type === "s"
1652 ? (row.content as string)
1653 : row.type === "b"
1654 ? (row.content as Uint8Array)
1655 : this.#deserializeObject(JSON.parse(row.content as string));
1656
1657 this.#cachedBlobs.set(blob, value);
1658 return value;
1659 }
1660
1661 #validateSource(source: Input): void {
1662 let parsed: URL;
1663 try {
1664 parsed = new URL(source.url);
1665 } catch {
1666 throw new Error(`source url must be a complete URL: ${source.url}`);
1667 }
1668 if (parsed.href !== source.url) {
1669 throw new Error(`source url must be canonical: ${source.url}`);
1670 }
1671 if (!Number.isFinite(source.time)) {
1672 throw new Error(`source time must be a finite number: ${source.url}`);
1673 }
1674 if (!Number.isFinite(source.size)) {
1675 throw new Error(`source size must be a finite number: ${source.url}`);
1676 }
1677 }
1678
1679 #serializeObject(data: Serializable, cache = true): unknown {
1680 if (
1681 data === null
1682 || typeof data === "string"
1683 || typeof data === "boolean"
1684 ) {
1685 return data;
1686 }
1687 if (typeof data === "number") {
1688 return Number.isFinite(data)
1689 ? data
1690 : { $clover: "number", value: String(data) };
1691 }
1692 if (typeof data === "undefined") {
1693 return { $clover: "undefined" };
1694 }
1695 if (data instanceof Uint8Array) {
1696 return { $clover: "buffer", blob: this.writeBlob(data, cache).blob };
1697 }
1698 if (data instanceof Date) {
1699 return { $clover: "date", time: data.getTime() };
1700 }
1701 if (data instanceof Set) {
1702 return {
1703 $clover: "set",
1704 entries: Array.from(data, (value) => this.#serializeObject(value, cache)),
1705 };
1706 }
1707 if (data instanceof Map) {
1708 return {
1709 $clover: "map",
1710 entries: Array.from(data, ([key, value]) => [
1711 this.#serializeObject(key, cache),
1712 this.#serializeObject(value, cache),
1713 ]),
1714 };
1715 }
1716 if (Array.isArray(data)) {
1717 return data.map((value) => this.#serializeObject(value, cache));
1718 }
1719 const out: Record<string, unknown> = {};
1720 for (const [key, value] of Object.entries(data)) {
1721 out[key] = this.#serializeObject(value, cache);
1722 }
1723 if (Object.hasOwn(out, "$clover")) {
1724 return { $clover: "escaped", value: out };
1725 }
1726 return out;
1727 }
1728
1729 #deserializeObject(data: unknown): Serializable {
1730 if (
1731 data === null
1732 || typeof data === "string"
1733 || typeof data === "boolean"
1734 ) {
1735 return data;
1736 }
1737 if (typeof data === "number") return data;
1738 if (Array.isArray(data)) {
1739 return data.map((value) => this.#deserializeObject(value));
1740 }
1741 if (!data || typeof data !== "object") {
1742 throw new Error(`unsupported serialized value: ${String(data)}`);
1743 }
1744
1745 const object = data as Record<string, unknown>;
1746 switch (object.$clover) {
1747 case undefined: {
1748 const out: Record<string, Serializable> = {};
1749 for (const [key, value] of Object.entries(object)) {
1750 out[key] = this.#deserializeObject(value);
1751 }
1752 return out;
1753 }
1754 case "number":
1755 return Number(object.value);
1756 case "undefined":
1757 return undefined;
1758 case "buffer":
1759 return this.readBlob(object.blob as RowId) as Uint8Array;
1760 case "date":
1761 return new Date(object.time as number);
1762 case "set":
1763 return new Set(
1764 (object.entries as unknown[]).map((value) => this.#deserializeObject(value)),
1765 );
1766 case "map":
1767 return new Map(
1768 (object.entries as [unknown, unknown][]).map(([key, value]) => [
1769 this.#deserializeObject(key),
1770 this.#deserializeObject(value),
1771 ]),
1772 );
1773 case "escaped": {
1774 const value = object.value as Record<string, unknown>;
1775 const out: Record<string, Serializable> = {};
1776 for (const [key, entry] of Object.entries(value)) {
1777 out[key] = this.#deserializeObject(entry);
1778 }
1779 return out;
1780 }
1781 default:
1782 throw new Error(`unknown serialized marker: ${String(object.$clover)}`);
1783 }
1784 }
1785}
1786
1787export function lazyLoader<T extends { default: LoaderFunction }>(
1788 load: () => Promise<T>,
1789): LoaderFunction {
1790 const once = async.once(load);
1791 return async (input) => (await once()).default(input);
1792}
1793
1794function hashSerializable(...values: Serializable[]): string {
1795 const hash = crypto.createHash("sha256");
1796 writeSerializableToHasher(hash, values);
1797 return hash.digest("base64url");
1798}
1799
1800function writeSerializableToHasher(
1801 hash: crypto.Hash,
1802 value: Serializable,
1803): void {
1804 if (value === null) {
1805 hash.update("n;");
1806 return;
1807 }
1808 if (typeof value === "undefined") {
1809 hash.update("u;");
1810 return;
1811 }
1812 if (typeof value === "boolean") {
1813 hash.update(value ? "t;" : "f;");
1814 return;
1815 }
1816 if (typeof value === "number") {
1817 writeStringToHasher(hash, "#", Object.is(value, -0) ? "-0" : String(value));
1818 return;
1819 }
1820 if (typeof value === "string") {
1821 writeStringToHasher(hash, "s", value);
1822 return;
1823 }
1824 if (value instanceof Uint8Array) {
1825 hash.update(`b${value.byteLength}:`);
1826 hash.update(value);
1827 hash.update(";");
1828 return;
1829 }
1830 if (value instanceof Date) {
1831 writeStringToHasher(hash, "d", String(value.getTime()));
1832 return;
1833 }
1834 if (value instanceof Set) {
1835 hash.update(`e${value.size}:[`);
1836 for (const entry of value) writeSerializableToHasher(hash, entry);
1837 hash.update("]");
1838 return;
1839 }
1840 if (value instanceof Map) {
1841 hash.update(`m${value.size}:{`);
1842 for (const [key, entry] of value) {
1843 writeSerializableToHasher(hash, key);
1844 writeSerializableToHasher(hash, entry);
1845 }
1846 hash.update("}");
1847 return;
1848 }
1849 if (Array.isArray(value)) {
1850 hash.update(`a${value.length}:[`);
1851 for (const entry of value) writeSerializableToHasher(hash, entry);
1852 hash.update("]");
1853 return;
1854 }
1855 const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
1856 hash.update(`o${entries.length}:{`);
1857 for (const [key, entry] of entries) {
1858 writeStringToHasher(hash, "k", key);
1859 writeSerializableToHasher(hash, entry);
1860 }
1861 hash.update("}");
1862}
1863
1864function writeStringToHasher(
1865 hash: crypto.Hash,
1866 tag: string,
1867 value: string,
1868): void {
1869 const byteLength = Buffer.byteLength(value);
1870 hash.update(`${tag}${byteLength}:`);
1871 hash.update(value);
1872 hash.update(";");
1873}
1874
1875import { ASSERT, UNWRAP } from "./assert.ts";
1876import * as async from "./async.ts";
1877import * as exception from "./error.ts";
1878import * as stack from "./log/stack.ts";
1879import * as node from "./node.ts";
1880import * as progress from "./progress.ts";
1881import * as ts from "./ts.ts";
1882
1883import * as crypto from "node:crypto";
1884import * as fscb from "node:fs";
1885import * as fs from "node:fs/promises";
1886import * as module from "node:module";
1887import * as path from "node:path";
1888import * as sqlite from "node:sqlite";
1889import * as url from "node:url";
1890import * as vm from "node:vm";
lib/log/stack.ts+1-1
......@@ -90,7 +90,7 @@ export function format(error: Error, colors = false): string {
9090 const { name, message, stack: _, ...payload } = error;
9191
9292 let out = (colors ? ansi.fgRed : "")
93 + (name && name !== "Error" ? `[${name}] ` : "")
93 + (name && name !== "Error" && name !== "TypeError" ? `[${name}] ` : "")
9494 + message + (colors ? ansi.fgReset : "") + "\n";
9595
9696 out += Object.entries(payload)
lib/mime.ts+2-3
......@@ -127,7 +127,6 @@ const extensions = new Map(
127127 + ".xls" + "#" + "application/vnd.ms-excel" + "\n"
128128 + ".xlsx" + "#" + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + "\n"
129129 + ".xml" + "#" + "application/xml" + "\n"
130 + ".xps" + "#" + "application/vnd.ms-xpsdocument" + "\n"
131 + +""
132 ).slice(0, -1).split("\n").map((line) => line.split("#") as [string, string]),
130 + ".xps" + "#" + "application/vnd.ms-xpsdocument"
131 ).split("\n").map((line) => line.split("#") as [string, string]),
133132);
lib/package.json-4
......@@ -1,9 +1,5 @@
11{
22 "name": "@clo/lib",
33 "type": "module",
4 "dependencies": {
5 "@types/node": "^24.0.0",
6 "@clo/lib": "file:."
7 },
84 "exports": { "./*": "./*.ts" }
95}
lib/progress.ts+271-225
......@@ -85,7 +85,7 @@ export interface Ref {
8585 * creates a new trackable unit of work as a child of this one.
8686 * when given an estimate, a progress bar is rendered.
8787 */
88 start(text: string, opts?: StartOptions): Node;
88 start: (text: string, opts?: StartOptions) => Node;
8989}
9090
9191/**
......@@ -205,17 +205,19 @@ export function slashValueFormatter(
205205): ValueFormatter {
206206 return (value: number, total: number | null) =>
207207 value > 0
208 ? fmtNumber(value)
209 + (total != null && total > 0 ? "/" + fmtNumber(total) : "")
208 ? fmtNumber(value) +
209 (total != null && total > 0 ? "/" + fmtNumber(total) : "")
210210 : "";
211211}
212212
213213/** the formatter used when passing `units: "count"` to {@linkcode start} */
214export const defaultValueFormatter: ValueFormatter = /* @__PURE__ */
214export const defaultValueFormatter: ValueFormatter =
215 /* @__PURE__ */
215216 slashValueFormatter(String);
216217
217218/** the formatter used when passing `units: "bytes"` to {@linkcode start} */
218export const bytesValueFormatter: ValueFormatter = /* @__PURE__ */
219export const bytesValueFormatter: ValueFormatter =
220 /* @__PURE__ */
219221 slashValueFormatter(string.formatByteSize);
220222
221223/** the formatter used when passing `units: "percent"` to {@linkcode start} */
......@@ -342,18 +344,17 @@ export class Root<
342344
343345 #emitChangeSoon = () => {
344346 if (this.#debounce) return;
345 (this.#debounce = this.delay(1))
346 .then(() => {
347 this.#debounce = null;
348 this.emit("change", this.#active);
349 });
347 (this.#debounce = this.delay(1)).then(() => {
348 this.#debounce = null;
349 this.emit("change", this.#active);
350 });
350351 };
351352}
352353
353354/** the built in events provided by {@linkcode Root} */
354355export type RootEventMap<Result = void> = {
355356 // this event is the debounced "ready to re-render" event. do not emit.
356 "change": [rootNodes: readonly ReadOnlyNode[]];
357 change: [rootNodes: readonly ReadOnlyNode[]];
357358 // these events are emitted without a debounce, and are used for general
358359 // communication within `Root`'s implementation. do not emit.
359360 "node-start": [newNode: ReadOnlyNode, parentNode: ReadOnlyNode | null];
......@@ -363,8 +364,8 @@ export type RootEventMap<Result = void> = {
363364 // these events are used for result communication. they can be sent via
364365 // `emit` or the shorthand methods `end` and `error`. sending either will put
365366 // the `Root` into a "done" state.
366 "end": [result: Result];
367 "error": [error: unknown];
367 end: [result: Result];
368 error: [error: unknown];
368369};
369370
370371const rootEvents: ReadonlySet<unknown> = new Set([
......@@ -378,13 +379,14 @@ const rootEvents: ReadonlySet<unknown> = new Set([
378379]);
379380
380381type MergeRootEvents<Result, Map extends Events.Map> = {
381 [K in keyof Map | keyof RootEventMap]: K extends keyof RootEventMap ? RootEventMap<Result>[K]
382 [K in keyof Map | keyof RootEventMap]: K extends keyof RootEventMap
383 ? RootEventMap<Result>[K]
382384 : Map[K];
383385};
384386
385387/** a read-only version of {@linkcode Node}, provided to renderers. */
386388export type ReadOnlyNode = Readonly<
387 & Pick<
389 Pick<
388390 Required<Node>,
389391 | "text"
390392 | "value"
......@@ -395,8 +397,7 @@ export type ReadOnlyNode = Readonly<
395397 | "showTotal"
396398 | "sortChildren"
397399 | "valueFormatter"
398 >
399 & {
400 > & {
400401 /** unique identifier. once {@linkcode Node} is disposed, the key is reused. */
401402 key: number;
402403 logs: readonly log.Message[];
......@@ -416,8 +417,8 @@ interface Internal extends ts.Writeable<ReadOnlyNode> {
416417}
417418
418419// TypeScript Test
419void function(unknown: unknown) {
420 (unknown as Internal) satisfies ReadOnlyNode;
420void function (unknown: unknown) {
421 unknown as Internal satisfies ReadOnlyNode;
421422};
422423
423424/**
......@@ -445,11 +446,13 @@ function newNode<Result, EventMap extends Events.Map>(
445446 children: [],
446447 parent: null,
447448 sortChildren: opts.sortChildren ?? null,
448 valueFormatter: opts.valueFormatter ?? {
449 count: defaultValueFormatter,
450 bytes: bytesValueFormatter,
451 percent: percentValueFormatter,
452 }[opts.units ?? "count"],
449 valueFormatter:
450 opts.valueFormatter ??
451 {
452 count: defaultValueFormatter,
453 bytes: bytesValueFormatter,
454 percent: percentValueFormatter,
455 }[opts.units ?? "count"],
453456 detached: false,
454457 };
455458
......@@ -464,8 +467,11 @@ function newNode<Result, EventMap extends Events.Map>(
464467 function mutate(key: keyof ReadOnlyNode) {
465468 if (state.detached) return;
466469 if (
467 key === "value" && estimator && state.total > 0 && state.value > 0
468 && state.value < state.total
470 key === "value" &&
471 estimator &&
472 state.total > 0 &&
473 state.value > 0 &&
474 state.value < state.total
469475 ) {
470476 // originally, auto-estimation would use the Root's timing primitives,
471477 // but this makes it so that `state.estimatedTime` is in terms of the
......@@ -587,18 +593,15 @@ export const nullNode: Node = {
587593 get text() {
588594 return "[detached]";
589595 },
590 set text(_) {
591 },
596 set text(_) {},
592597 get value() {
593598 return 0;
594599 },
595 set value(_) {
596 },
600 set value(_) {},
597601 get total() {
598602 return 0;
599603 },
600 set total(_) {
601 },
604 set total(_) {},
602605 inc: noop,
603606 log: {
604607 info: noop,
......@@ -627,7 +630,7 @@ function endNode<R, M extends Events.Map>(owner: Root<R, M>, state: Internal) {
627630 const i = state.parent.children.indexOf(state);
628631 ASSERT(i !== -1);
629632 let p: Internal | null = state;
630 while (p = p.parent) title = p.text + " / " + title;
633 while ((p = p.parent)) title = p.text + " / " + title;
631634 state.parent.children.splice(i, 1);
632635 }
633636 owner.emit("node-end", state);
......@@ -650,36 +653,45 @@ const spinnerFps = 12.5;
650653const box = { tee: "├─ ", line: "│ ", langle: "└─ " };
651654const barChars = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"];
652655const fullBar = "█";
653const spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
654 .map((frame) => ansi.style(ansi.fgBlue, frame));
656const spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"].map(
657 (frame) => ansi.style(ansi.fgBlue, frame),
658);
655659
656660function renderAnsiMainLine(state: ReadOnlyNode, now: number, depth: number) {
657661 const { text, total, showTotal, value, estimatedTime } = state;
658662 const dateNow = Date.now();
659 const showEstimate = estimatedTime && !findLongerEstimate(state, estimatedTime);
660 const estimate = showEstimate && (estimatedTime > dateNow + 1000)
661 ? ", "
662 + string.formatDurationLetters(Math.round((estimatedTime - dateNow) / 1000))
663 : "";
663 const showEstimate =
664 estimatedTime && !findLongerEstimate(state, estimatedTime);
665 const estimate =
666 showEstimate && estimatedTime > dateNow + 1000
667 ? ", " +
668 string.formatDurationLetters(
669 Math.round((estimatedTime - dateNow) / 1000),
670 )
671 : "";
664672 const valueFormatted = state.valueFormatter(value, total);
665673 if (total && showTotal) {
666 return ansi.style(
667 ansi.bgBrightBlack + ansi.fgBlue,
668 formatUnicodeBar(
669 value / total,
670 Math.max(4, depth > 0 ? 12 : 25),
671 ),
672 )
673 + (valueFormatted || estimate ? ` [${valueFormatted}${estimate}] ` : " ")
674 + text;
674 return (
675 ansi.style(
676 ansi.bgBrightBlack + ansi.fgBlue,
677 formatUnicodeBar(value / total, Math.max(4, depth > 0 ? 12 : 25)),
678 ) +
679 (valueFormatted || estimate ? ` [${valueFormatted}${estimate}] ` : " ") +
680 text
681 );
675682 }
676683 const frame = Math.floor(now / (1000 / spinnerFps)) % spinner.length;
677 return ((depth === 0 ? spinner[frame] + " " : "")
678 + (valueFormatted ? `[${valueFormatted}] ` : "") + text);
684 return (
685 (depth === 0 ? spinner[frame] + " " : "") +
686 (valueFormatted ? `[${valueFormatted}] ` : "") +
687 text
688 );
679689}
680690
681691function hasChildren(states: ReadOnlyNode): boolean {
682 return states.children.some((x) => !x.hidden && (!x.passive || hasChildren(x)));
692 return states.children.some(
693 (x) => !x.hidden && (!x.passive || hasChildren(x)),
694 );
683695}
684696
685697function findLongerEstimate(
......@@ -701,12 +713,14 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {
701713 let truncated = 0;
702714 let out = "";
703715 let { children } = state;
704 children = children.filter((x) => !x.hidden && (!x.passive || hasChildren(x)));
716 children = children.filter(
717 (x) => !x.hidden && (!x.passive || hasChildren(x)),
718 );
705719 if (children.length === 0) return "";
706720 for (let i = 0, { length } = children; i < length; i += 1) {
707721 const child = UNWRAP(children[i]);
708722 let item = "";
709 const left = depth.map((x) => x ? box.line : " ").join("");
723 const left = depth.map((x) => (x ? box.line : " ")).join("");
710724 item += left + (i === length - 1 && !truncated ? box.langle : box.tee);
711725 item += renderAnsiMainLine(child, now, 1 + depth.length) + "\n";
712726 item += renderChildren(child, now, depth.concat(i < length - 1));
......@@ -721,17 +735,21 @@ function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {
721735 .trim();
722736 if (logLines) {
723737 for (const line of logLines.split("\n").slice(-Math.min(3, maxHeight))) {
724 item += left + (i === length - 1 && !truncated ? " " : box.line)
725 + " "
726 + ansi.style(ansi.fgBrightBlack, ">")
727 + " " + line + "\n";
738 item +=
739 left +
740 (i === length - 1 && !truncated ? " " : box.line) +
741 " " +
742 ansi.style(ansi.fgBrightBlack, ">") +
743 " " +
744 line +
745 "\n";
728746 }
729747 }
730748 maxHeight -= h + string.countNewlines(logLines);
731749 out += item;
732750 }
733751 if (truncated) {
734 out += depth.map((x) => x ? box.line : " ").join("") + box.langle;
752 out += depth.map((x) => (x ? box.line : " ")).join("") + box.langle;
735753 out += ansi.style(ansi.fgBrightBlack, `[${truncated} more]`);
736754 }
737755 return out;
......@@ -779,43 +797,52 @@ export function formatUnicodeBar(progress: number, width: number): string {
779797 */
780798export function attachToScreen(
781799 root: Root,
782 { writeOutput, startWidget }: Pick<
783 log.WidgetHost,
784 "writeOutput" | "startWidget"
785 >,
800 {
801 writeOutput,
802 startWidget,
803 }: Pick<log.WidgetHost, "writeOutput" | "startWidget">,
786804): ts.Dispose {
787805 const stack = new DisposableStack();
788806 let widget: log.WidgetInstance | null = null;
789807
790 stack.use(root.on("change", (items) => {
791 if (items.length > 0) {
792 widget ??= startWidget({
793 format: ({ now }) => formatAnsi(now, root.active),
794 }) ?? null;
795 if (!widget) return;
796 widget.fps = items.some((x) => x.showTotal !== false && x.total > 0)
797 ? spinnerFps
798 : null;
799 widget.redraw();
800 } else {
801 widget?.stop();
802 widget = null;
803 }
804 }));
805 stack.use(root.on("node-detached-log", (msg) => {
806 writeOutput(log.formatAnsiMessage(msg, true));
807 }));
808 stack.use(root.on("node-end", (node) => {
809 let title = node.text;
810 let p: ReadOnlyNode | null = node;
811 while (p = p.parent) title = p.text + " / " + title;
812 const { logs } = node;
813 if (logs.length > 0) {
814 const header = `[logs from ${title}]`;
815 writeOutput(ansi.style(ansi.fgBrightBlack, header) + "\n");
816 writeOutput(logs.map((msg) => log.formatAnsiMessage(msg, true)).join(""));
817 }
818 }));
808 stack.use(
809 root.on("change", (items) => {
810 if (items.length > 0) {
811 widget ??=
812 startWidget({
813 format: ({ now }) => formatAnsi(now, root.active),
814 }) ?? null;
815 if (!widget) return;
816 widget.fps = items.some((x) => x.showTotal !== false && x.total > 0)
817 ? spinnerFps
818 : null;
819 widget.redraw();
820 } else {
821 widget?.stop();
822 widget = null;
823 }
824 }),
825 );
826 stack.use(
827 root.on("node-detached-log", (msg) => {
828 writeOutput(log.formatAnsiMessage(msg, true));
829 }),
830 );
831 stack.use(
832 root.on("node-end", (node) => {
833 let title = node.text;
834 let p: ReadOnlyNode | null = node;
835 while ((p = p.parent)) title = p.text + " / " + title;
836 const { logs } = node;
837 if (logs.length > 0) {
838 const header = `[logs from ${title}]`;
839 writeOutput(ansi.style(ansi.fgBrightBlack, header) + "\n");
840 writeOutput(
841 logs.map((msg) => log.formatAnsiMessage(msg, true)).join(""),
842 );
843 }
844 }),
845 );
819846
820847 return ts.defer(() => stack[Symbol.dispose]);
821848}
......@@ -902,7 +929,7 @@ export function encodeEventStream<
902929 // fun fact, this type is impossible to write without generics
903930 root: Root<Result, Map>,
904931 {
905 throttleMs = 1000 / 30, /* 30fps */
932 throttleMs = 1000 / 30 /* 30fps */,
906933 serializeStackTraces = false,
907934 }: EncodeStreamOptions = {},
908935): ReadableStream<StreamEvent> {
......@@ -934,47 +961,56 @@ export function encodeEventStream<
934961 });
935962
936963 let isFirst = true;
937 return new ReadableStream({
938 start(controller) {
939 if (root.active.length > 0) {
964 return new ReadableStream(
965 {
966 start(controller) {
967 if (root.active.length > 0) {
968 controller.enqueue(s.getDelta(isFirst));
969 isFirst = false;
970 }
971
972 // TODO: why does ts require two args
973 stack.use(
974 root.on("node-start", (newNode, _) => {
975 if (!newNode.parent) s.rootChildrenUpdated = true;
976 s.changed.set(newNode, new Set());
977 emitSoon();
978 }),
979 );
980 stack.use(
981 root.on("node-change", (node, key) => {
982 let set = s.changed.get(node);
983 if (!set) s.changed.set(node, (set = new Set()));
984 set.add(key);
985 emitSoon();
986 }),
987 );
988 stack.use(
989 root.on("node-end", (node) => {
990 if (!node.parent) s.rootChildrenUpdated = true;
991 s.changed.delete(node);
992 s.deleted.add(node); // handle add and remove in same frame
993 emitSoon();
994 }),
995 );
996 },
997 async pull(controller) {
998 await ready.until((x) => x === true);
999 ready.value = false;
1000 if (timer) {
1001 timer.cancel();
1002 timer = null;
1003 }
9401004 controller.enqueue(s.getDelta(isFirst));
9411005 isFirst = false;
942 }
943
944 // TODO: why does ts require two args
945 stack.use(root.on("node-start", (newNode, _) => {
946 if (!newNode.parent) s.rootChildrenUpdated = true;
947 s.changed.set(newNode, new Set());
948 emitSoon();
949 }));
950 stack.use(root.on("node-change", (node, key) => {
951 let set = s.changed.get(node);
952 if (!set) s.changed.set(node, set = new Set());
953 set.add(key);
954 emitSoon();
955 }));
956 stack.use(root.on("node-end", (node) => {
957 if (!node.parent) s.rootChildrenUpdated = true;
958 s.changed.delete(node);
959 s.deleted.add(node); // handle add and remove in same frame
960 emitSoon();
961 }));
962 },
963 async pull(controller) {
964 await ready.until((x) => x === true);
965 ready.value = false;
966 if (timer) {
967 timer.cancel();
968 timer = null;
969 }
970 controller.enqueue(s.getDelta(isFirst));
971 isFirst = false;
972 },
973 cancel() {
974 stack.dispose();
975 (0, performance.now)();
1006 },
1007 cancel() {
1008 stack.dispose();
1009 (0, performance.now)();
1010 },
9761011 },
977 }, { highWaterMark: 0 });
1012 { highWaterMark: 0 },
1013 );
9781014}
9791015
9801016/**
......@@ -1064,13 +1100,10 @@ class Encoder<
10641100
10651101 const { updated = [], started = [] } = Object.groupBy(
10661102 this.changed,
1067 ([node]) => this.keyMap.has(node) ? "updated" : "started",
1103 ([node]) => (this.keyMap.has(node) ? "updated" : "started"),
10681104 );
10691105
1070 const payload: StreamEvent = [
1071 isFirst ? -1 : deleted.length,
1072 ...deleted,
1073 ];
1106 const payload: StreamEvent = [isFirst ? -1 : deleted.length, ...deleted];
10741107
10751108 if (started.length > 0) {
10761109 this.beginNodesDepthFirst(
......@@ -1082,8 +1115,8 @@ class Encoder<
10821115 for (const [node, props] of updated) {
10831116 let logs = undefined;
10841117 if (
1085 props.has("logs")
1086 && node.logs.length > (this.logsLength.get(node) ?? 0)
1118 props.has("logs") &&
1119 node.logs.length > (this.logsLength.get(node) ?? 0)
10871120 ) {
10881121 logs = this.redactLogsIfNeeded(
10891122 node.logs.slice(this.logsLength.get(node) ?? 0),
......@@ -1103,11 +1136,13 @@ class Encoder<
11031136 ? node.children.map((node) => UNWRAP(this.keyMap.get(node)))
11041137 : undefined,
11051138 E: props.has("estimatedTime") ? node.estimatedTime : undefined,
1106 V: node.valueFormatter !== defaultValueFormatter
1107 && (props.has("value") || props.has("total")
1108 || props.has("valueFormatter"))
1109 ? node.valueFormatter(node.value, node.total)
1110 : undefined,
1139 V:
1140 node.valueFormatter !== defaultValueFormatter &&
1141 (props.has("value") ||
1142 props.has("total") ||
1143 props.has("valueFormatter"))
1144 ? node.valueFormatter(node.value, node.total)
1145 : undefined,
11111146 [kNode]: node,
11121147 });
11131148 }
......@@ -1125,16 +1160,13 @@ class Encoder<
11251160 return payload;
11261161 }
11271162
1128 beginNodesDepthFirst(
1129 nodes: ReadOnlyNode[],
1130 out: StreamEvent,
1131 ) {
1163 beginNodesDepthFirst(nodes: ReadOnlyNode[], out: StreamEvent) {
11321164 for (let i = 0; i < nodes.length; i += 1) {
11331165 const node = UNWRAP(nodes[i]);
11341166 nodes.push(...node.children);
11351167 }
11361168 let next;
1137 while (next = nodes.pop()) {
1169 while ((next = nodes.pop())) {
11381170 if (!this.keyMap.has(next)) {
11391171 out.push(this.beginNode(next));
11401172 }
......@@ -1150,19 +1182,25 @@ class Encoder<
11501182 v: node.value !== 0 ? node.value : undefined,
11511183 e: node.total !== 0 ? node.total : undefined,
11521184 t: node.text.length > 0 ? node.text : undefined,
1153 l: node.logs.length > 0
1154 ? (this.logsLength.set(node, node.logs.length), this.redactLogsIfNeeded(node.logs.slice()))
1155 : undefined,
1156 c: node.children.length > 0
1157 ? node.children.map((node) => UNWRAP(this.keyMap.get(node), () => [node.children, this.keyMap]))
1158 : undefined,
1185 l:
1186 node.logs.length > 0
1187 ? (this.logsLength.set(node, node.logs.length),
1188 this.redactLogsIfNeeded(node.logs.slice()))
1189 : undefined,
1190 c:
1191 node.children.length > 0
1192 ? node.children.map((node) =>
1193 UNWRAP(this.keyMap.get(node), () => [node.children, this.keyMap]),
1194 )
1195 : undefined,
11591196 p: node.passive === true ? true : undefined,
11601197 s: node.showTotal === false ? false : undefined,
11611198 h: node.hidden === true ? true : undefined,
11621199 E: node.estimatedTime != null ? node.estimatedTime : undefined,
1163 V: node.valueFormatter !== defaultValueFormatter
1164 ? node.valueFormatter(node.value, node.total)
1165 : undefined,
1200 V:
1201 node.valueFormatter !== defaultValueFormatter
1202 ? node.valueFormatter(node.value, node.total)
1203 : undefined,
11661204 [kNode]: node,
11671205 };
11681206 }
......@@ -1173,10 +1211,7 @@ class Encoder<
11731211 }
11741212}
11751213
1176class Decoder<
1177 Result = void,
1178 EventMap extends Events.Map = ts.EmptyObject,
1179> {
1214class Decoder<Result = void, EventMap extends Events.Map = ts.EmptyObject> {
11801215 target: Ref;
11811216 events: Events<EventMap>;
11821217 resolve: (result: Result) => void;
......@@ -1284,7 +1319,7 @@ class Decoder<
12841319 }
12851320
12861321 let entry;
1287 while (entry = this.pendingStart.entries().next().value) {
1322 while ((entry = this.pendingStart.entries().next().value)) {
12881323 const [key, value] = entry;
12891324 this.startRecursive(key, value);
12901325 }
......@@ -1302,9 +1337,8 @@ class Decoder<
13021337 startRecursive(key: EncodedKey, opts: PendingStart): Ref {
13031338 ASSERT(this.pendingStart.delete(key));
13041339 const parentId = UNWRAP(this.pendingParents.get(key));
1305 let ref: Ref | null = parentId === 0
1306 ? this.target
1307 : this.active.get(parentId) ?? null;
1340 let ref: Ref | null =
1341 parentId === 0 ? this.target : (this.active.get(parentId) ?? null);
13081342 if (!ref) {
13091343 ref = this.startRecursive(
13101344 parentId,
......@@ -1379,10 +1413,7 @@ export function encodeByteStream<
13791413}
13801414
13811415/** convert {@linkcode StreamEvent} into a byte payload. Mutates the input event. */
1382function writeStreamEvent(
1383 event: StreamEvent,
1384 writer: stream.WriteTarget,
1385) {
1416function writeStreamEvent(event: StreamEvent, writer: stream.WriteTarget) {
13861417 using w = new stream.BufferedWriter(writer);
13871418
13881419 let deleteCount = event.shift();
......@@ -1402,8 +1433,13 @@ function writeStreamEvent(
14021433 }
14031434 }
14041435
1405 const rootChildren = event.find((x): x is StreamRootChildren => Array.isArray(x) && typeof x[0] === "number");
1406 const changedNodes = event.filter((x): x is StreamNode => typeof x === "object" && !Array.isArray(x));
1436 const rootChildren = event.find(
1437 (x): x is StreamRootChildren =>
1438 Array.isArray(x) && typeof x[0] === "number",
1439 );
1440 const changedNodes = event.filter(
1441 (x): x is StreamNode => typeof x === "object" && !Array.isArray(x),
1442 );
14071443 w.varUint(changedNodes.length + +!!rootChildren);
14081444 for (const obj of changedNodes) {
14091445 ASSERT(typeof obj === "object" && "k" in obj);
......@@ -1422,19 +1458,21 @@ function writeStreamEvent(
14221458 ASSERT(messages == null || messages.length > 0);
14231459
14241460 w.varUint(key);
1425 w.u16(encodeDeltaFlags({
1426 hidden,
1427 passive,
1428 showTotal,
1429 changedValue: value !== undefined,
1430 changedTotal: total !== undefined,
1431 changedLogs: messages !== undefined,
1432 changedText: text !== undefined,
1433 changedChildren: childrenSet !== undefined,
1434 changedEstimatedTime: estimatedTime !== undefined,
1435 changedFormattedValue: formattedValue !== undefined,
1436 estimatedTimeNonNull: estimatedTime !== null,
1437 }));
1461 w.u16(
1462 encodeDeltaFlags({
1463 hidden,
1464 passive,
1465 showTotal,
1466 changedValue: value !== undefined,
1467 changedTotal: total !== undefined,
1468 changedLogs: messages !== undefined,
1469 changedText: text !== undefined,
1470 changedChildren: childrenSet !== undefined,
1471 changedEstimatedTime: estimatedTime !== undefined,
1472 changedFormattedValue: formattedValue !== undefined,
1473 estimatedTimeNonNull: estimatedTime !== null,
1474 }),
1475 );
14381476 if (value !== undefined) w.f64(value);
14391477 if (total !== undefined) w.f64(total);
14401478 if (text !== undefined) w.stringWithLength(text);
......@@ -1444,10 +1482,10 @@ function writeStreamEvent(
14441482 let level = logLevelSerialize.indexOf(msg.level ?? "info");
14451483 if (level === -1) level = 0;
14461484 w.u8(
1447 level
1448 + (msg.scope ? 1 << 5 : 0)
1449 + (msg.stack && msg.stack.length > 0 ? 1 << 6 : 0)
1450 + (msg.custom !== undefined ? 1 << 7 : 0),
1485 level +
1486 (msg.scope ? 1 << 5 : 0) +
1487 (msg.stack && msg.stack.length > 0 ? 1 << 6 : 0) +
1488 (msg.custom !== undefined ? 1 << 7 : 0),
14511489 );
14521490 w.stringWithLength(msg.text);
14531491 w.varUint(Math.floor(msg.time));
......@@ -1484,7 +1522,9 @@ function writeStreamEvent(
14841522 }
14851523 }
14861524
1487 const customEvents = event.filter((x): x is StreamCustomEvent => Array.isArray(x) && typeof x[0] === "string");
1525 const customEvents = event.filter(
1526 (x): x is StreamCustomEvent => Array.isArray(x) && typeof x[0] === "string",
1527 );
14881528 w.varUint(customEvents.length);
14891529 for (const [event, ...args] of customEvents) {
14901530 w.stringWithLength(event);
......@@ -1563,9 +1603,9 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
15631603 while (len > 0) {
15641604 const msgFlags = await r.u8();
15651605 const level = logLevelSerialize[msgFlags & 0b1111] ?? "info";
1566 const hasScope = msgFlags && (1 << 5) > 0;
1567 const hasStack = msgFlags && (1 << 6) > 0;
1568 const hasCustom = msgFlags && (1 << 7) > 0;
1606 const hasScope = msgFlags && 1 << 5 > 0;
1607 const hasStack = msgFlags && 1 << 6 > 0;
1608 const hasCustom = msgFlags && 1 << 7 > 0;
15691609 const text = await r.stringWithLength();
15701610 const time = await r.varUint();
15711611 const scope = hasScope ? await r.stringWithLength() : undefined;
......@@ -1596,10 +1636,12 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
15961636 if (flags.changedChildren) {
15971637 c = [];
15981638 let len = await r.varUint();
1599 while (len-- > 0) c.push(await r.varUint() as EncodedKey);
1639 while (len-- > 0) c.push((await r.varUint()) as EncodedKey);
16001640 }
16011641 const E = flags.changedEstimatedTime
1602 ? flags.estimatedTimeNonNull ? await r.varUint() : null
1642 ? flags.estimatedTimeNonNull
1643 ? await r.varUint()
1644 : null
16031645 : undefined;
16041646 const V = flags.changedFormattedValue
16051647 ? await r.stringWithLength()
......@@ -1608,7 +1650,7 @@ async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
16081650 // a "v2"'s packet will go here, contents ignored by this version.
16091651 if (flags.futureProofingPacket) {
16101652 const len = await r.varUint();
1611 void await r.readExactly(len);
1653 void (await r.readExactly(len));
16121654 }
16131655
16141656 event.push({
......@@ -1641,37 +1683,37 @@ const logLevelSerialize = ["info", "error", "warn", "debug"] as const;
16411683
16421684function encodeDeltaFlags(node: Partial<DeltaFlags>) {
16431685 return (
1644 (node.hidden ? 1 << 0 : 0)
1645 + (node.passive ? 1 << 1 : 0)
1646 + (node.showTotal ? 1 << 2 : 0)
1647 + (node.changedValue ? 1 << 3 : 0)
1648 + (node.changedTotal ? 1 << 4 : 0)
1649 + (node.changedLogs ? 1 << 6 : 0)
1650 + (node.changedText ? 1 << 7 : 0)
1651 + (node.changedChildren ? 1 << 8 : 0)
1652 + (node.changedEstimatedTime ? 1 << 9 : 0)
1653 + (node.changedFormattedValue ? 1 << 10 : 0)
1654 + (node.estimatedTimeNonNull ? 1 << 11 : 0)
1655 + (node.futureProofingPacket ? 1 << 14 : 0)
1656 + (node.deleted ? 1 << 15 : 0)
1686 (node.hidden ? 1 << 0 : 0) +
1687 (node.passive ? 1 << 1 : 0) +
1688 (node.showTotal ? 1 << 2 : 0) +
1689 (node.changedValue ? 1 << 3 : 0) +
1690 (node.changedTotal ? 1 << 4 : 0) +
1691 (node.changedLogs ? 1 << 6 : 0) +
1692 (node.changedText ? 1 << 7 : 0) +
1693 (node.changedChildren ? 1 << 8 : 0) +
1694 (node.changedEstimatedTime ? 1 << 9 : 0) +
1695 (node.changedFormattedValue ? 1 << 10 : 0) +
1696 (node.estimatedTimeNonNull ? 1 << 11 : 0) +
1697 (node.futureProofingPacket ? 1 << 14 : 0) +
1698 (node.deleted ? 1 << 15 : 0)
16571699 );
16581700}
16591701
16601702function decodeDeltaFlags(flags: number): DeltaFlags {
16611703 return {
1662 hidden: (flags & 1 << 0) !== 0,
1663 passive: (flags & 1 << 1) !== 0,
1664 showTotal: (flags & 1 << 2) !== 0,
1665 changedValue: (flags & 1 << 3) !== 0,
1666 changedTotal: (flags & 1 << 4) !== 0,
1667 changedLogs: (flags & 1 << 6) !== 0,
1668 changedText: (flags & 1 << 7) !== 0,
1669 changedChildren: (flags & 1 << 8) !== 0,
1670 changedEstimatedTime: (flags & 1 << 9) !== 0,
1671 changedFormattedValue: (flags & 1 << 10) !== 0,
1672 estimatedTimeNonNull: (flags & 1 << 11) !== 0,
1673 futureProofingPacket: (flags & 1 << 14) !== 0,
1674 deleted: (flags & 1 << 15) !== 0,
1704 hidden: (flags & (1 << 0)) !== 0,
1705 passive: (flags & (1 << 1)) !== 0,
1706 showTotal: (flags & (1 << 2)) !== 0,
1707 changedValue: (flags & (1 << 3)) !== 0,
1708 changedTotal: (flags & (1 << 4)) !== 0,
1709 changedLogs: (flags & (1 << 6)) !== 0,
1710 changedText: (flags & (1 << 7)) !== 0,
1711 changedChildren: (flags & (1 << 8)) !== 0,
1712 changedEstimatedTime: (flags & (1 << 9)) !== 0,
1713 changedFormattedValue: (flags & (1 << 10)) !== 0,
1714 estimatedTimeNonNull: (flags & (1 << 11)) !== 0,
1715 futureProofingPacket: (flags & (1 << 14)) !== 0,
1716 deleted: (flags & (1 << 15)) !== 0,
16751717 };
16761718}
16771719
......@@ -1731,9 +1773,10 @@ export class Ema implements EstimationAlgorithm {
17311773 if (elapsed === 0) return null; // too quick
17321774
17331775 const rate = current / elapsed;
1734 this.estimate = this.estimate != null
1735 ? (1 - this.alpha) * this.estimate + this.alpha * rate
1736 : rate;
1776 this.estimate =
1777 this.estimate != null
1778 ? (1 - this.alpha) * this.estimate + this.alpha * rate
1779 : rate;
17371780 if (elapsed < 10_000 || this.estimate <= 0 || this.samples < 3) return null;
17381781 const remaining = total - current;
17391782 return time + remaining / this.estimate;
......@@ -1750,7 +1793,10 @@ type EncodedKey = number & { brand: typeof kNode };
17501793
17511794const globalKeyPool = new KeyPool<number>();
17521795
1753const global: Ref = /** @__PURE__ */ ((root = new Root()) => (attachToScreen(root, log), root))();
1796const global: Ref = /** @__PURE__ */ ((root = new Root()) => (
1797 attachToScreen(root, log),
1798 root
1799))();
17541800
17551801/**
17561802 * a {@linkcode Ref} to the global progress root. unlike referencing the
lib/readme.md+3-2
......@@ -19,11 +19,12 @@ depend on each other. there are no external dependencies; code is easy to audit.
1919- `bytes` - `Uint8Array` helpers
2020- `error` - interop with the `unknown` type in error handlers
2121- `Events` - typed event emitter
22- `incremental` - for build tools, tracable `node:vm`, job caching
2223- `log` - logging, terminal i/o, interactive widgets
2324- `log/stack` - stack trace parse and formatter
2425- `Lru` - least recently used cache
2526- `mime` - trimmed database of ext -> mime types
26- `node` - the Node.js javascript runtime
27- `node` - tree-shakable loaders for Node.js builtins
2728- `progress` - cli progress bars, nesting status indication
2829- `queue` - priority queue with rescheduling
2930- `stream` - ReadableStream helpers
......@@ -35,7 +36,7 @@ depend on each other. there are no external dependencies; code is easy to audit.
3536- `ts` typescript types and general purpose code
3637
3738there is currently no hosted documentation site. please refer to the source
38files' comments.
39files' comments and the JSR generated documentation.
3940
4041## usage
4142
lib/ts.ts+27-17
......@@ -10,13 +10,19 @@ export type Timer = ReturnType<typeof setTimeout>;
1010export type Interval = ReturnType<typeof setInterval>;
1111/** opposite of the built-in `Readonly` type */
1212export type Writeable<T> = { -readonly [P in keyof T]: T[P] };
13/** A type that can be `await`ed to return `T` */
14export type Awaitable<T> = T | PromiseLike<T>;
1315/** opposite of the built-in `Readonly` type, but recursive */
1416// TODO: tuples?
1517export type DeepWriteable<T> = {
16 -readonly [P in keyof T]: T[P] extends ReadonlyArray<infer I> ? Array<I>
18 -readonly [P in keyof T]: T[P] extends ReadonlyArray<infer I>
19 ? Array<I>
1720 : DeepWriteable<T[P]>;
1821};
1922
23declare const brand: unique symbol;
24export type Brand<T> = { [brand]: T };
25
2026/**
2127 * redeclared here because it is only provided by `lib: ["dom"]` and not
2228 * the JavaScript standard types.
......@@ -25,7 +31,7 @@ export interface VoidFunction {
2531 (): void;
2632}
2733
28/** Convenience type. Construct with `defer` */
34/** convenience type. Construct with `defer` */
2935export interface Dispose extends Disposable, VoidFunction {}
3036
3137/**
......@@ -51,12 +57,10 @@ export function mixin<A extends object, B extends object>(
5157 other: B,
5258): Mixin<A, B> {
5359 const prototype = Object.getPrototypeOf(other);
54 for (
55 const key of [
56 ...Object.getOwnPropertyNames(prototype),
57 ...Object.getOwnPropertySymbols(prototype),
58 ]
59 ) {
60 for (const key of [
61 ...Object.getOwnPropertyNames(prototype),
62 ...Object.getOwnPropertySymbols(prototype),
63 ]) {
6064 if (key === "constructor") continue;
6165 if (key in base) continue;
6266 const fn = prototype[key];
......@@ -69,15 +73,21 @@ export function mixin<A extends object, B extends object>(
6973export type EmptyObject = Record<never, unknown>;
7074
7175/** TODO: this type has many subtle bugs */
72export type ToJson<T> = T extends JsonValue ? T
73 : T extends { toJSON(): infer J } ? ToJson<J>
74 : T extends Set<unknown> | Map<unknown, unknown> | Record<string, never> ? EmptyObject
75 : T extends unknown[] ? {
76 [K in keyof Omit<T, keyof unknown[]>]: T[K] extends undefined | void ? null
77 : ToJson<T[K]>;
78 }
79 : T extends object ? { [K in keyof Omit<T, symbol>]: ToJson<T[K]> }
80 : never;
76export type ToJson<T> = T extends JsonValue
77 ? T
78 : T extends { toJSON(): infer J }
79 ? ToJson<J>
80 : T extends Set<unknown> | Map<unknown, unknown> | Record<string, never>
81 ? EmptyObject
82 : T extends unknown[]
83 ? {
84 [K in keyof Omit<T, keyof unknown[]>]: T[K] extends undefined | void
85 ? null
86 : ToJson<T[K]>;
87 }
88 : T extends object
89 ? { [K in keyof Omit<T, symbol>]: ToJson<T[K]> }
90 : never;
8191export type Json = JsonValue | undefined | void;
8292export type JsonValue =
8393 | string
old_stuff/hot.ts created+306
......@@ -0,0 +1,306 @@
1// This implements the ability to use TS, TSX, and more plugins
2// in Node.js. It cannot be built on the ES module loader,
3// because there is no exposed way to replace modules when
4// needed (see nodejs#49442).
5//
6// It also allows using a simple compile cache, which is used by
7// the site generator to determine when code changes.
8export const projectRoot = path.resolve(import.meta.dirname, "../");
9export const projectSrc = path.resolve(projectRoot, "src");
10
11using _ = null;
12
13// Create a project-relative require. For convenience, it is generic-typed.
14export const load = createRequire(
15 pathToFileURL(path.join(projectRoot, "run.js")).toString(),
16) as {
17 <T = unknown>(id: string): T;
18 extensions: NodeJS.Dict<(mod: NodeJS.Module, file: string) => unknown>;
19 cache: NodeJS.Dict<NodeJS.Module>;
20 resolve: (id: string, o?: { paths: string[] }) => string;
21};
22export const { cache } = load;
23
24// Register extensions by overwriting `require.extensions`
25const require = load;
26const exts = require.extensions;
27exts[".ts"] = loadEsbuild;
28exts[".tsx"] = loadEsbuild;
29exts[".jsx"] = loadEsbuild;
30exts[".marko"] = loadMarko;
31exts[".mdo"] = loadMarkodown;
32exts[".css"] = loadCss;
33
34// Intercept all module load calls to track CSS imports + file times.
35export interface FileStat {
36 cssImportsRecursive: string[] | null;
37 lastModified: number;
38 imports: string[];
39}
40const fileStats = new Map<string, FileStat>();
41
42export function getFileStat(filepath: string) {
43 return fileStats.get(path.resolve(filepath));
44}
45
46function shouldTrackPath(filename: string) {
47 return !filename.includes("node_modules");
48}
49
50const Module = load<typeof import("node:module")>("node:module");
51const ModulePrototypeUnderscoreCompile = Module.prototype._compile;
52Module.prototype._compile = function(
53 content: string,
54 filename: string,
55 format: "module" | "commonjs",
56) {
57 const result = ModulePrototypeUnderscoreCompile.call(
58 this,
59 content,
60 filename,
61 format,
62 );
63 if (shouldTrackPath(filename)) {
64 const stat = fs.statSync(filename);
65 const cssImportsMaybe: string[] = [];
66 const imports: string[] = [];
67 for (const childModule of this.children) {
68 const { filename: file, cloverClientRefs } = childModule;
69 if (file.endsWith(".css")) cssImportsMaybe.push(file);
70 else {
71 const child = fileStats.get(file);
72 if (!child) continue;
73 const { cssImportsRecursive } = child;
74 if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
75 imports.push(file);
76 (childModule.cloverImporters ??= []).push(this);
77 if (cloverClientRefs && cloverClientRefs.length > 0) {
78 (this.cloverClientRefs ??= []).push(...cloverClientRefs);
79 }
80 }
81 }
82 fileStats.set(filename, {
83 cssImportsRecursive: cssImportsMaybe.length > 0
84 ? Array.from(new Set(cssImportsMaybe))
85 : null,
86 imports,
87 lastModified: Math.floor(stat.mtimeMs),
88 });
89 }
90 return result;
91};
92
93function loadEsbuild(module: NodeJS.Module, filepath: string) {
94 return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
95}
96
97interface LoadOptions {
98 scannedClientRefs?: string[];
99}
100export function loadEsbuildCode(
101 module: NodeJS.Module,
102 filepath: string,
103 src: string,
104 opt: LoadOptions = {},
105) {
106 if (filepath === import.meta.filename) {
107 module.exports = self;
108 return;
109 }
110 let loader: any = "tsx";
111 if (filepath.endsWith(".ts")) loader = "ts";
112 else if (filepath.endsWith(".jsx")) loader = "jsx";
113 else if (filepath.endsWith(".js")) loader = "js";
114 if (opt.scannedClientRefs) {
115 module.cloverClientRefs = opt.scannedClientRefs;
116 } else {
117 let { code, refs } = resolveClientRefs(src, filepath);
118 module.cloverClientRefs = refs;
119 src = code;
120 }
121 if (src.includes("import.meta")) {
122 src = `
123 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
124 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
125 import.meta.filename = ${JSON.stringify(filepath)};
126 import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r));
127 `
128 .trim()
129 .replace(/[\n\s]/g, "") + src;
130 }
131 src = esbuild.transformSync(src, {
132 loader,
133 format: "cjs",
134 target: "esnext",
135 jsx: "automatic",
136 jsxImportSource: "#jsx",
137 jsxDev: true,
138 sourcemap: "inline",
139 sourcefile: path.basename(filepath),
140 sourceRoot: path.dirname(filepath),
141 }).code;
142 return module._compile(src, filepath, "commonjs");
143}
144
145export function resolveClientRef(sourcePath: string, ref: string) {
146 const filePath = resolveFrom(sourcePath, ref);
147 if (!filePath.endsWith(".client.ts")) {
148 throw new Error("addScript must take a .client.ts");
149 }
150 return path.relative(projectSrc, filePath);
151}
152
153let lazyMarko: typeof import("./marko.ts") | null = null;
154function loadMarko(module: NodeJS.Module, filepath: string) {
155 lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
156 lazyMarko.loadMarko(module, filepath);
157}
158function loadMarkodown(module: NodeJS.Module, filepath: string) {
159 lazyMarko ??= require<typeof import("./marko.ts")>("./framework/marko.ts");
160 lazyMarko.loadMarkodown(module, filepath);
161}
162
163function loadCss(module: NodeJS.Module, _filepath: string) {
164 module.exports = {};
165}
166
167export function reloadRecursive(filepath: string) {
168 filepath = path.resolve(filepath);
169 const existing = cache[filepath];
170 if (existing) deleteRecursiveInner(filepath, existing);
171 fileStats.clear();
172 return require(filepath);
173}
174
175export function unload(filepath: string) {
176 lazyMarko?.markoCache.delete(filepath);
177 filepath = path.resolve(filepath);
178 const module = cache[filepath];
179 if (!module) return;
180 delete cache[filepath];
181 for (const importer of module.cloverImporters ?? []) {
182 unload(importer.filename);
183 }
184}
185
186function deleteRecursiveInner(id: string, module: any) {
187 if (id.includes(path.sep + "node_modules" + path.sep)) {
188 return;
189 }
190 delete cache[id];
191 for (const child of module.children) {
192 if (child.filename.includes("/engine/")) return;
193 const existing = cache[child.filename];
194 if (existing === child) deleteRecursiveInner(child.filename, existing);
195 }
196}
197
198export function getCssImports(filepath: string) {
199 filepath = path.resolve(filepath);
200 if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
201 return fileStats.get(filepath)?.cssImportsRecursive ?? [];
202}
203
204export function getClientScriptRefs(filepath: string) {
205 filepath = path.resolve(filepath);
206 const module = require.cache[filepath];
207 if (!module) throw new Error(filepath + " was never loaded");
208 return module.cloverClientRefs ?? [];
209}
210
211export function getSourceCode(filepath: string) {
212 filepath = path.resolve(filepath);
213 const module = require.cache[filepath];
214 if (!module) throw new Error(filepath + " was never loaded");
215 if (!module.cloverSourceCode) {
216 throw new Error(filepath + " did not record source code");
217 }
218 return module.cloverSourceCode;
219}
220
221export function resolveFrom(src: string, dest: string) {
222 try {
223 return createRequire(src).resolve(dest);
224 } catch (err: any) {
225 if (
226 err.code === "MODULE_NOT_FOUND" && err.requireStack
227 && err.requireStack.length <= 1
228 ) {
229 err.message = err.message.split("\n")[0] + " from '" + src + "'";
230 }
231 throw err;
232 }
233}
234
235const importRegExp = /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;
236const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))?/;
237interface ResolvedClientRefs {
238 code: string;
239 refs: string[];
240}
241export function resolveClientRefs(
242 code: string,
243 filepath: string,
244): ResolvedClientRefs {
245 // This match finds a call to 'import ... from "#sitegen"'
246 const importMatch = code.match(importRegExp);
247 if (!importMatch) return { code, refs: [] };
248 const items = UNWRAP(importMatch[1]);
249 let identifier = "";
250 if (items.startsWith("{")) {
251 const clauseMatch = items.match(getSitegenAddScriptRegExp);
252 if (!clauseMatch) return { code, refs: [] }; // did not import
253 identifier = clauseMatch[1] || "addScript";
254 } else if (items.startsWith("*")) {
255 identifier = importMatch[2] + "\\s*\\.\\s*addScript";
256 } else {
257 throw new Error("Impossible");
258 }
259 identifier = identifier.replaceAll("$", "\\$"); // only needed escape
260 const findCallsRegExp = new RegExp(
261 `\\b(${identifier})\\s*\\(("[^"]+"|'[^']+')\\)`,
262 "gs",
263 );
264 const scannedClientRefs = new Set<string>();
265 code = code.replace(findCallsRegExp, (_, call, arg) => {
266 const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
267 const resolved = resolveClientRef(filepath, ref);
268 scannedClientRefs.add(resolved);
269 return `${call}(${JSON.stringify(getScriptId(resolved))})`;
270 });
271 return { code, refs: Array.from(scannedClientRefs) };
272}
273
274export function getScriptId(file: string) {
275 return (
276 path.isAbsolute(file) ? path.relative(projectSrc, file) : file
277 ).replaceAll("\\", "/");
278}
279
280declare global {
281 namespace NodeJS {
282 interface Module {
283 cloverClientRefs?: string[];
284 cloverSourceCode?: string;
285 cloverImporters?: Module[];
286
287 _compile(
288 this: NodeJS.Module,
289 content: string,
290 filepath: string,
291 format: "module" | "commonjs",
292 ): unknown;
293 }
294 }
295}
296declare module "node:module" {
297 export function _resolveFilename(id: string, parent: NodeJS.Module): unknown;
298}
299
300import * as fs from "#sitegen/fs";
301import { UNWRAP } from "@clo/lib/assert";
302import * as esbuild from "esbuild";
303import { createRequire } from "node:module";
304import * as path from "node:path";
305import { pathToFileURL } from "node:url";
306import * as self from "./hot.ts";
old_stuff/incremental.ts created+738
......@@ -0,0 +1,738 @@
1// Incremental build system using singleton state.
2// See `work()`, `compile()`, and `invalidate()` for details.
3//
4// All state is serializable to allow recovering state across sessions.
5// This library special-cases the asset map, but is otherwise agnostic.
6let running = false;
7let jobs = 0;
8let newKeys = 0;
9let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
10let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
11let works = new Map<string, Work>();
12let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
13let writes = new Map<string, FileWrite>();
14let assets = new Map<string, Asset>(); // keyed by hash
15
16export interface Ref<T> {
17 key: string;
18 /** This method is compatible with `await` syntax */
19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20 get value(): T | null;
21 /** The progress node passed into the input */
22 progress: progress.Node | progress.Ref;
23}
24type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
25
26/**
27 * Declare and a unit of work. Return value is memoized and only rebuilt when
28 * inputs change. Inputs are tracked via the `io` interface, as well as a hash
29 * of the `input` param and caller source code. Outputs are written at the end
30 * of a compilation (see `compile`).
31 *
32 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33 */
34export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
35export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
36export function work<I, O>(
37 p: progress.Ref,
38 job: Job<I, O>,
39 input?: I,
40): Ref<O> {
41 const source = JSON.stringify(UNWRAP(stack.capture()[0]));
42 const keySource = [source, util.inspect(input)].join(":");
43 const key = crypto.createHash("sha1").update(keySource).digest("base64url");
44 ASSERT(running);
45 ASSERT(
46 !seenWorks.has(key),
47 `Key '${key}' must be unique during the build. `
48 + `To fix this, provide a manual 'key' argument.`,
49 );
50 seenWorks.add(key);
51
52 const prev = works.get(key) as Work<O> | null;
53 if (prev) {
54 return {
55 key,
56 then: (done) => done(prev.value),
57 value: prev.value,
58 progress: p,
59 };
60 }
61
62 async function perform() {
63 const io = new Io(key, p);
64 jobs += 1;
65 newKeys += 1;
66 try {
67 const value = await job(io, input as I);
68 validateSerializable(value, "");
69 const { reads, writes } = io;
70 works.set(key, { value, affects: [], reads, writes });
71 for (const add of reads.files) {
72 const { affects } = UNWRAP(files.get(add));
73 ASSERT(!affects.includes(key));
74 affects.push(key);
75 }
76 for (const add of reads.works) {
77 const { affects } = UNWRAP(works.get(add));
78 ASSERT(!affects.includes(key));
79 affects.push(key);
80 }
81 return value;
82 } finally {
83 jobs -= 1;
84 }
85 }
86
87 let cached: Promise<O>;
88 return {
89 key,
90 then(fufill, reject) {
91 (cached ??= perform()).then(fufill, reject);
92 },
93 get value() {
94 return (works.get(this.key)?.value as O) ?? null;
95 },
96 progress: p,
97 };
98}
99
100export async function compile<T>(
101 compiler: (node: progress.Ref) => Promise<T>,
102 p: progress.Ref = progress,
103) {
104 ASSERT(!running, `Cannot run twice at the same time.`);
105 const startDate = new Date();
106 const start = performance.now();
107 const node = p.start("sitegen!");
108 try {
109 running = true;
110 ASSERT(jobs === 0);
111
112 const value = await compiler(node);
113 ASSERT(jobs === 0);
114 await flush(node, start);
115 return {
116 value,
117 watchFiles: new Set(files.keys()),
118 newOutputs: Array.from(seenWrites)
119 .filter((x) => x.startsWith("f:"))
120 .map((x) => x.slice(2)),
121 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
122 };
123 } finally {
124 node.text = `sitegen at ${startDate.toLocaleDateString()} ${startDate.toLocaleTimeString()}`;
125 node.end();
126 running = false;
127 newKeys = 0;
128 seenWrites.clear();
129 seenWorks.clear();
130 }
131}
132
133export async function flush(progress: progress.Node, start: number) {
134 using _ = progress.start("incremental flush");
135 // Trim
136 const detachedFiles = new Set<string>();
137 const referencedAssets = new Set<string>();
138 for (const [k, v] of works) {
139 const assets = v.writes.assets;
140 if (seenWorks.has(k)) {
141 for (const asset of assets.values()) referencedAssets.add(asset.hash);
142 continue;
143 }
144 deleteWork(k);
145 }
146 for (const [k, file] of files) {
147 if (file.affects.length > 0) continue;
148 files.delete(k);
149 detachedFiles.add(k);
150 }
151 for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
152
153 const p: Promise<void>[] = [];
154 // File writes
155 let dist = 0;
156 for (const [key, { buffer, size }] of writes) {
157 if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
158 dist += size;
159 }
160 // Asset map
161 {
162 const { json, blob, dynamic, dts } = getAssetManifest();
163 const jsonString = Buffer.from(JSON.stringify(json));
164 p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
165 p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
166 p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
167 for (const [k, v] of dynamic) {
168 p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
169 }
170 dist += blob.byteLength + jsonString.byteLength;
171 }
172 await Promise.all(p);
173
174 // Incremental state
175 const serialized = msgpackr.pack(serialize());
176 await fs.writeMkdir(".clover/incr.state", serialized);
177 const time = (performance.now() - start).toFixed(0);
178 progress.log.info(
179 `sitegen! in ${time} ms\n`
180 + ` - ${works.size} keys (${works.size - newKeys} cached)\n`
181 + ` - ${assets.size} static assets\n`
182 + ` - dist: ${formatSize(dist)},`
183 + ` incremental: ${formatSize(serialized.byteLength)}`,
184 );
185}
186
187export async function restore() {
188 let buffer;
189 try {
190 buffer = await fs.readFile(".clover/incr.state");
191 } catch (err: any) {
192 if (err.code !== "ENOENT") throw err;
193 }
194 if (!buffer) return;
195 await deserialize(buffer);
196}
197
198function forceInvalidate(entry: { affects: string[] }) {
199 const queue = [...entry.affects];
200 let key;
201 while ((key = queue.shift())) {
202 const affects = deleteWork(key);
203 queue.push(...affects);
204 }
205}
206
207function deleteWork(key: string) {
208 const work = works.get(key);
209 if (!work) return [];
210 const { reads, affects, writes: w } = work;
211 for (const remove of reads.files) {
212 const { affects } = UNWRAP(files.get(remove));
213 ASSERT(affects.includes(key));
214 affects.splice(affects.indexOf(key), 1);
215 }
216 for (const remove of reads.works) {
217 const { affects } = UNWRAP(works.get(remove), remove);
218 ASSERT(affects.includes(key));
219 affects.splice(affects.indexOf(key), 1);
220 }
221 for (const remove of affects) {
222 const {
223 reads: { works: list },
224 } = UNWRAP(works.get(remove), remove);
225 ASSERT(list.has(key));
226 list.delete(key);
227 }
228 for (const file of w.files) {
229 if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
230 }
231 // Assets are temporarily kept, trimmed via manual GC after compilation.
232
233 works.delete(key);
234 return affects;
235}
236
237export function reset() {
238 ASSERT(!running);
239 works.clear();
240 files.clear();
241 assets.clear();
242}
243
244export function serialize() {
245 const fileEntries = Array.from(
246 files,
247 ([k, v]) =>
248 [
249 k,
250 v.type,
251 v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
252 ...v.affects,
253 ] as const,
254 );
255 const workEntries = Array.from(
256 works,
257 ([k, v]) =>
258 [
259 k,
260 v.value,
261 Array.from(v.reads.files),
262 Array.from(v.reads.works),
263 Array.from(v.writes.files),
264 Array.from(
265 v.writes.assets,
266 ([k, { headers }]) => [k, headers] as const,
267 ),
268 v.affects,
269 ] as const,
270 );
271 const expectedFilesOnDisk = Array.from(
272 writes,
273 ([k, { size, work }]) => [k, size, work] as const,
274 );
275 const assetEntries = Array.from(
276 assets,
277 ([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
278 );
279 return [
280 1,
281 fileEntries,
282 workEntries,
283 expectedFilesOnDisk,
284 assetEntries,
285 ] as const;
286}
287type SerializedState = ReturnType<typeof serialize>;
288
289/* No-op on failure */
290async function deserialize(buffer: Buffer) {
291 const decoded = msgpackr.decode(buffer) as SerializedState;
292 if (!Array.isArray(decoded)) return;
293 if (decoded[0] !== 1) return;
294 const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] = decoded;
295 for (const [k, type, content, ...affects] of fileEntries) {
296 if (type === "f") {
297 ASSERT(typeof content === "number");
298 files.set(k, { type, affects, lastModified: content });
299 } else if (type === "d") {
300 ASSERT(typeof content === "string");
301 files.set(k, { type, affects, contentHash: content, contents: [] });
302 } else {
303 files.set(k, { type, affects });
304 }
305 }
306 for (const entry of workEntries) {
307 const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] = entry;
308 works.set(k, {
309 value,
310 reads: {
311 files: new Set(readFiles),
312 works: new Set(readWorks),
313 },
314 writes: {
315 files: new Set(writeFiles),
316 assets: new Map(
317 Array.from(writeAssets, ([k, headers]) => [
318 k,
319 {
320 hash: JSON.parse(UNWRAP(headers.etag)),
321 headers,
322 },
323 ]),
324 ),
325 },
326 affects,
327 });
328 }
329 const statFiles = await Promise.all(
330 expectedFilesOnDisk.map(([k, size, work]) =>
331 fs
332 .stat(path.join(".clover/o", k))
333 .catch((err) => {
334 if (err.code === "ENOENT") return null;
335 throw err;
336 })
337 .then((stat) => ({ k, size, work, stat }))
338 ),
339 );
340 for (const { k, stat, work, size } of statFiles) {
341 if (stat?.size === size) {
342 writes.set(k, {
343 size: size,
344 buffer: null,
345 work,
346 });
347 } else {
348 forceInvalidate({ affects: [work] });
349 }
350 }
351 for (const [hash, raw, gzip, zstd] of assetEntries) {
352 assets.set(hash, { raw, gzip, zstd });
353 }
354
355 await Promise.all(
356 Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
357 );
358}
359
360export async function invalidate(
361 filePath: string,
362 unloadModule: boolean = true,
363): Promise<boolean> {
364 const key = toRel(toAbs(filePath));
365 const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
366 return invalidateEntry(key, file, unloadModule);
367}
368
369export async function invalidateEntry(
370 key: string,
371 file: TrackedFile,
372 unloadModule: boolean,
373): Promise<boolean> {
374 try {
375 if (file.type === "d") {
376 const contents = (file.contents = await fs.readdir(key));
377 contents.sort();
378 const contentHash = crypto
379 .createHash("sha1")
380 .update(contents.join("\0"))
381 .digest("base64url");
382 if (file.contentHash !== contentHash) {
383 file.contentHash = contentHash;
384 throw new Error();
385 }
386 } else if (file.type === "f") {
387 const lastModified = await fs.stat(key).then(
388 (x) => Math.floor(x.mtimeMs),
389 () => 0,
390 );
391 if (file.lastModified !== lastModified) {
392 file.lastModified = lastModified;
393 throw new Error();
394 }
395 } else {
396 file.type satisfies "null";
397 const stat = await fs.stat(key).catch(() => null);
398 if (stat) throw new Error();
399 }
400 return false;
401 } catch (e) {
402 forceInvalidate(file);
403 if (unloadModule) {
404 // TODO: handle when this triggers unloading of `generate.ts`
405 hot.unload(toAbs(key));
406 }
407 if (file.type === "null") files.delete(key);
408 return true;
409 }
410}
411
412export function getAssetManifest() {
413 const dynamic = new Map<string, Buffer>();
414 const writer = new BufferWriter();
415 const assetMap = Object.fromEntries(
416 Array.from(works, (work) => work[1].writes.assets)
417 .filter((map) => map.size > 0)
418 .flatMap((map) =>
419 Array.from(map, ([key, { hash, headers, regenerative }]) => {
420 const { raw, gzip, zstd } = UNWRAP(
421 assets.get(hash),
422 `Asset ${key} (${hash})`,
423 );
424 if (regenerative) {
425 const id = crypto
426 .createHash("sha1")
427 .update(key)
428 .digest("hex")
429 .slice(0, 16); /* TODO */
430 dynamic.set(
431 id,
432 manifest.packDynamicBuffer(raw, gzip, zstd, headers),
433 );
434 return [key, { type: 1, id }] as const;
435 }
436 return [
437 key,
438 {
439 type: 0,
440 raw: writer.write(raw, "raw:" + hash),
441 gzip: writer.write(gzip, "gzip:" + hash),
442 zstd: writer.write(zstd, "zstd:" + hash),
443 headers,
444 },
445 ] as const;
446 })
447 ),
448 ) satisfies manifest.Manifest;
449 return {
450 json: assetMap,
451 blob: writer.get(),
452 dynamic,
453 dts: "export type AssetKey = "
454 + Object.keys(assetMap)
455 .map((key) => JSON.stringify(key))
456 .join(" | ")
457 + "\n",
458 };
459}
460
461/* Input/Output with automatic tracking.
462 * - Inputs read with Io are tracked to know when to rebuild
463 * - Outputs written with Io are deleted when abandoned.
464 */
465export class Io {
466 constructor(public key: string, private node: progress.Ref) {}
467 reads: Reads = { files: new Set(), works: new Set() };
468 writes: Writes = { files: new Set(), assets: new Map() };
469
470 start(text: string, options?: progress.StartOptions) {
471 return this.node.start(text, options);
472 }
473
474 #trackFs(file: string) {
475 const resolved = toAbs(file);
476 const key = toRel(resolved);
477 this.reads.files.add(key);
478 return { resolved, key };
479 }
480 async readWork<T>(ref: Ref<T>): Promise<T> {
481 this.reads.works.add(ref.key);
482 return await ref;
483 }
484 /** Track a file in the compilation without reading it. */
485 async trackFile(file: string) {
486 const { key, resolved } = this.#trackFs(file);
487 if (!files.get(key)) {
488 let lastModified: number = 0;
489 try {
490 lastModified = Math.floor((await fs.stat(file)).mtimeMs);
491 files.set(key, { type: "f", lastModified, affects: [] });
492 } catch {
493 files.set(key, { type: "null", affects: [] });
494 }
495 }
496 return resolved;
497 }
498 async readFile(file: string): Promise<Buffer>;
499 async readFile(file: string, encoding: "utf-8"): Promise<string>;
500 async readFile(file: string, encoding?: "utf-8") {
501 return fs.readFile(await this.trackFile(file), encoding);
502 }
503 async readJson<T>(file: string) {
504 return JSON.parse(await this.readFile(file, "utf-8")) as T;
505 }
506 async readDir(dir: string) {
507 const { key, resolved } = this.#trackFs(dir);
508 const existing = files.get(key);
509 try {
510 if (existing?.type === "d") return existing.contents;
511 const contents = await fs.readdir(resolved);
512 contents.sort();
513 const contentHash = crypto
514 .createHash("sha1")
515 .update(contents.join("\0"))
516 .digest("base64url");
517 files.set(key, {
518 type: "d",
519 affects: [],
520 contentHash,
521 contents,
522 });
523 return contents;
524 } catch (err) {
525 if (!existing) files.set(key, { type: "null", affects: [] });
526 throw err;
527 }
528 }
529 async readDirRecursive(dir: string): Promise<string[]> {
530 const dirs = await this.readDir(dir);
531 return (
532 await Promise.all(
533 dirs.map(async (child) => {
534 const abs = path.join(dir, child);
535 const stat = await fs.stat(abs);
536 if (stat.isDirectory()) {
537 return (await this.readDirRecursive(abs)).map((grand) => path.join(child, grand));
538 } else {
539 return child;
540 }
541 }),
542 )
543 ).flat();
544 }
545 /* Track all dependencies of a module. */
546 async import<T>(file: string): Promise<T> {
547 const { resolved } = this.#trackFs(file);
548 try {
549 return require(resolved) as T;
550 } finally {
551 const queue = [resolved];
552 const seen = new Set<string>();
553 let current;
554 while ((current = queue.shift())) {
555 const stat = hot.getFileStat(current);
556 if (!stat) continue;
557 const { key } = this.#trackFs(current);
558 if (!files.get(key)) {
559 files.set(key, {
560 type: "f",
561 affects: [],
562 lastModified: stat?.lastModified ?? 0,
563 });
564 }
565 for (const imp of stat.imports) {
566 if (!seen.has(imp)) {
567 seen.add(imp);
568 queue.push(imp);
569 }
570 }
571 }
572 }
573 }
574 async writeAsset(asset: {
575 pathname: string;
576 buffer: string | Uint8Array;
577 regenerative?: boolean;
578 headers?: HeadersInit;
579 }) {
580 ASSERT(asset.pathname.startsWith("/"));
581 ASSERT(!seenWrites.has("a:" + asset.pathname));
582 const buffer = typeof asset.buffer === "string"
583 ? Buffer.from(asset.buffer)
584 : asset.buffer;
585 const headers = new Headers(asset.headers ?? {});
586 const hash = crypto.createHash("sha1").update(buffer).digest("hex");
587 if (!headers.has("Content-Type")) {
588 headers.set("Content-Type", mime.for(asset.pathname));
589 }
590 headers.set("etag", JSON.stringify(hash));
591 this.writes.assets.set(asset.pathname, {
592 hash,
593 headers: Object.fromEntries(headers),
594 regenerative: !!asset.regenerative,
595 });
596 if (!assets.has(hash)) {
597 jobs += 1;
598 assets.set(hash, undefined!);
599 const [gzipBuffer, zstdBuffer] = await Promise.all([
600 gzip(buffer),
601 zstdCompress(buffer),
602 ]);
603 assets.set(hash, {
604 raw: buffer,
605 gzip: gzipBuffer,
606 zstd: zstdBuffer,
607 });
608 jobs -= 1;
609 }
610 }
611 writeFile(subPath: string, blob: string | Buffer) {
612 ASSERT(!subPath.startsWith("/"));
613 ASSERT(
614 !seenWrites.has("f:" + subPath),
615 `File overwritten: ${JSON.stringify(subPath)}`,
616 );
617 seenWrites.add("f:" + subPath);
618 const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
619 writes.set(subPath, {
620 buffer,
621 size: buffer.byteLength,
622 work: this.key,
623 });
624 }
625}
626
627class BufferWriter {
628 size = 0;
629 seen = new Map<string, BufferView>();
630 buffers: Buffer[] = [];
631
632 write(buffer: Buffer, hash: string): BufferView {
633 let view = this.seen.get(hash);
634 if (view) return view;
635 view = [this.size, this.size += buffer.byteLength];
636 this.seen.set(hash, view);
637 this.buffers.push(buffer);
638 return view;
639 }
640
641 get() {
642 return Buffer.concat(this.buffers);
643 }
644}
645
646export function validateSerializable(value: unknown, key: string) {
647 if (typeof value === "string") {
648 if (value.includes(hot.projectRoot)) {
649 throw new Error(
650 `Return value must not contain the CWD for portability, found at ${key}`,
651 );
652 }
653 } else if (value && typeof value === "object") {
654 if (Array.isArray(value)) {
655 value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
656 } else if (
657 Object.getPrototypeOf(value) === Object.prototype
658 || Buffer.isBuffer(value)
659 ) {
660 Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`));
661 } else {
662 throw new Error(
663 `Return value must be a plain JS object, found ${Object.getPrototypeOf(value).constructor.name} at ${key}`,
664 );
665 }
666 } else if (["bigint", "function", "symbol"].includes(typeof value)) {
667 throw new Error(
668 `Return value must be a plain JS object, found ${typeof value} at ${key}`,
669 );
670 }
671}
672
673export function toAbs(absPath: string) {
674 return path.resolve(hot.projectRoot, absPath);
675}
676
677export function toRel(absPath: string) {
678 return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
679}
680
681export type BufferView = [start: number, end: number];
682interface Reads {
683 files: Set<string>;
684 works: Set<string>;
685}
686interface FileWrite {
687 buffer: Buffer | null;
688 size: number;
689 work: string;
690}
691interface Writes {
692 files: Set<string>;
693 assets: Map<string, AssetWrite>;
694}
695interface Asset {
696 raw: Buffer;
697 gzip: Buffer;
698 zstd: Buffer;
699}
700interface AssetWrite {
701 hash: string;
702 headers: Record<string, string>;
703 regenerative: boolean;
704}
705interface Work<T = unknown> {
706 value: T;
707 reads: Reads;
708 writes: Writes;
709 affects: string[];
710}
711type TrackedFile =
712 & { affects: string[] }
713 & (
714 | { type: "f"; lastModified: number }
715 | { type: "d"; contentHash: string; contents: string[] }
716 | { type: "null" }
717 );
718
719const gzip = util.promisify(zlib.gzip);
720const zstdCompress = util.promisify(zlib.zstdCompress);
721
722import * as hot from "./hot.ts";
723
724import * as manifest from "#sitegen/assets";
725import * as fs from "#sitegen/fs";
726import * as mime from "@clo/lib/mime";
727
728import * as crypto from "node:crypto";
729import * as path from "node:path";
730import * as util from "node:util";
731import * as zlib from "node:zlib";
732
733import { formatSize } from "#src/file-viewer/format.ts";
734import * as stack from "@clo/lib/log/stack";
735import * as progress from "@clo/lib/progress";
736
737import { ASSERT, UNWRAP } from "@clo/lib/assert";
738import * as msgpackr from "msgpackr";
package-lock.json+1-5
......@@ -29,11 +29,7 @@
2929 }
3030 },
3131 "lib": {
32 "name": "@clo/lib",
33 "dependencies": {
34 "@clo/lib": "file:.",
35 "@types/node": "^24.0.0"
36 }
32 "name": "@clo/lib"
3733 },
3834 "node_modules/@clo/lib": {
3935 "resolved": "lib",
playground.ts created+19
......@@ -0,0 +1,19 @@
1import * as incremental from "./lib/incremental.ts";
2import * as progress from "./lib/progress.ts";
3
4const g = new incremental.Graph({ db: "" });
5g.incrementalBuild({
6 async buildAndWriteOutput(ref) {
7 using node = ref.start("sitegen!");
8
9 const y = node.cache("meow", async (t) => {
10 const meow = JSON.parse(
11 await t.fs.readFile("package.json"),
12 ) as typeof import("./package.json");
13 return meow.imports;
14 });
15
16 console.log({ y });
17 },
18 progress,
19});
run.js+88-137
......@@ -1,152 +1,103 @@
11#!/usr/bin/env node
22// This file allows using Node.js in combination with
33// all available plugins. Usage: "node run <script>"
4import * as child_process from "node:child_process";
5import * as crypto from "node:crypto";
6import * as fs from "node:fs";
7import * as os from "node:os";
8import * as path from "node:path";
4import crypto from "node:crypto";
5import fs from "node:fs";
6import path from "node:path";
97import process from "node:process";
10import * as url from "node:url";
11import * as zlib from "node:zlib";
12
13// Disable experimental warnings (Type Stripping, etc)
14const { emit: originalEmit } = process;
15const warnings = ["ExperimentalWarning"];
16process.emit = function(event, error) {
17 return event === "warning" && warnings.includes(error.name)
18 ? false
19 : originalEmit.apply(process, arguments);
20};
21
22// Auto-install with npm if needed.
23{
24 const rootDir = path.dirname(url.fileURLToPath(import.meta.url));
25 const cloverDir = path.join(rootDir, ".clover");
26 fs.mkdirSync(cloverDir, { recursive: true });
27 function computeHash() {
28 const hash = crypto.createHash("sha1");
29 hash.update("npm");
30 hash.update(fs.readFileSync(path.join(rootDir, "package.json")));
31 hash.update(fs.readFileSync(path.join(rootDir, "package-lock.json")));
32 hash.update(fs.readdirSync(path.join(rootDir, "node_modules")).join("\0"));
33 return hash.digest("hex");
34 }
35 try {
36 const expected = fs.readFileSync(path.join(cloverDir, "deps-hash.txt"), "utf-8");
37 const actual = computeHash();
38 if (expected !== actual) throw new Error();
39 } catch {
40 child_process.execSync("npm i --no-audit --no-fund", { stdio: "inherit" });
41 console.info();
42 fs.writeFileSync(path.join(cloverDir, "deps-hash.txt"), computeHash());
43 }
44}
8import url from "node:url";
459
46let hot;
47try {
48 // zstd is not supported in node 22
49 await new Promise((resolve) => zlib.zstdCompress(new Uint8Array(), resolve));
50
51 // seems to fail in bun, run it earlier to catch
52 (await import("source-map-support")).install({ hookRequire: true });
53
54 // init hooks. some runtimes have bugs here. and long term this system should be rebuilt
55 hot = await import("./framework/hot.ts");
56 hot.load("./lib/log.ts");
57} catch (e) {
58 const brand = process.versions.bun
59 ? `bun ${process.versions.bun}`
60 : process.versions.deno
61 ? `deno ${process.versions.deno}`
62 : null;
63 const brandText = brand ? ` (${brand})` : "";
64 globalThis.console.error(
65 `sitegen depends on a node.js v24. your runtime is missing features.\n`
66 + `this is node.js version ${process.version}${brandText}\n\n`
67 + `get node.js --> https://nodejs.org/en/download/current\n`,
68 );
69 globalThis.console.error(e);
70 process.exit(1);
71 await new Promise(() => {});
72}
10/** @type {import('@clo/lib/incremental').Graph | null} */
11export let graph = null;
7312
74const log = hot.load("./lib/log.ts");
75console.info = log.info;
76console.warn = log.warn;
77console.error = log.error;
78console.debug = log.debug;
79console["log"] = log.log;
80
81process.on("uncaughtException", (error) => {
82 console.error("Uncaught Exception");
83 console.error(error);
84 process.exit(1);
85});
86
87if (fs.existsSync(".env")) {
88 const env = fs.readFileSync(".env", "utf-8")
89 .split("\n").filter(Boolean)
90 .map((x) => x.trim().split("="));
91 for (const [k, v] of env) process.env[k] = v;
92}
13// parse cli options
14if (import.meta.main) {
15 // TODO: re-launch process with --experimental-vm-modules --no-warnings
9316
94// Load with hooks
95if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
96 if (process.argv.length == 2) {
97 log.error("usage: node run <script> [...args]");
98 process.exit(1);
99 }
100 let found;
101 for (const dir of fs.globSync("{lib,{src,framework},{src,lib}/**/bin}")) {
17 // disable experimental warnings (Type Stripping, etc)
18 const { emit: originalEmit } = process;
19 const warnings = ["ExperimentalWarning"];
20 process.emit = function(event, error) {
21 return event === "warning" && warnings.includes(error.name)
22 ? false
23 : originalEmit.apply(process, arguments);
24 };
25
26 // auto-install packages with npm if needed.
27 {
28 const rootDir = path.dirname(url.fileURLToPath(import.meta.url));
29 const cloverDir = path.join(rootDir, ".clover");
30 fs.mkdirSync(cloverDir, { recursive: true });
31 function computeHash() {
32 const hash = crypto.createHash("sha1");
33 hash.update("npm");
34 hash.update(fs.readFileSync(path.join(rootDir, "package.json")));
35 hash.update(fs.readFileSync(path.join(rootDir, "package-lock.json")));
36 hash.update(fs.readdirSync(path.join(rootDir, "node_modules")).join("\0"));
37 return hash.digest("hex");
38 }
10239 try {
103 found = hot.resolveFrom(
104 import.meta.filename,
105 `./${dir}/${process.argv[2]}`,
106 );
107 break;
108 } catch (e) {
109 continue;
40 const expected = fs.readFileSync(path.join(cloverDir, "deps-hash.txt"), "utf-8");
41 const actual = computeHash();
42 if (expected !== actual) throw new Error();
43 } catch {
44 child_process.execSync("npm i --no-audit --no-fund", { stdio: "inherit" });
45 console.info();
46 fs.writeFileSync(path.join(cloverDir, "deps-hash.txt"), computeHash());
11047 }
11148 }
112 if (!found) {
113 log.error("Cannot find script: " + process.argv[2]);
49
50
51 if (process.argv.length == 2) {
52 log.error("usage: node run <script> [...args]");
11453 process.exit(1);
11554 }
55
56 // load system
57 const incremental = await import("@clo/lib/incremental");
58 const log = await import("@clo/lib/log");
59 const ansi = await import("@clo/lib/string/ansi");
60 const progress = await import("@clo/lib/progress");
61 const stack = await import("@clo/lib/log/stack");
62
63 process.on("uncaughtException", (error) => {
64 log.writeOutput(ansi.style(ansi.fgBlack + ansi.bgRed, " UNCAUGHT ") + ' ' + stack.format(error, process.stderr.isTTY) + '\n');
65 process.exit(1);
66 });
67
68 console.info = log.info;
69 console.warn = log.warn;
70 console.error = log.error;
71 console.debug = log.debug;
72 console.log = log.log;
73
74 // resolve entrypoint
75 const mainFileRaw = process.argv[1];
76 const mainFile = "wow";
11677 process.argv = [process.argv[0], ...process.argv.slice(2)];
117 try {
118 const mod = await hot.load(found);
119 if (mod.main) mod.main();
120 else if (mod.default?.fetch) {
121 const protocol = "http";
122 const { serve } = hot.load("@hono/node-server");
123 serve(
124 {
125 fetch: mod.default.fetch,
126 port: mod.port ?? 3000,
127 },
128 ({ address, port }) => {
129 if (address === "::" || address === "127.0.0.1") {
130 address = os.hostname().replace(/\.local$/, "");
131 }
132 log.info(
133 url.format({
134 protocol,
135 hostname: address,
136 port,
137 }),
138 );
139 },
140 );
141 } else if (!found.match(/\.test\.[tj]sx?$/)) {
142 log.warn(
143 hot.load("path").relative(".", found),
144 "does not export a \"main\" function",
145 );
146 }
147 } catch (e) {
148 log.error(e);
78
79 // revive and load
80
81 const markoLoader = incremental.lazyLoader(() => import('./framework/marko'));
82
83 graph = await incremental.Graph.revive({
84 file: path.resolve(".clover/incremental.db"),
85 progress,
86
87 loaders: {
88 '.marko': markoLoader,
89 '.mdo': markoLoader,
90 },
91 });
92
93 const module = await graph.import(mainFile, { main: true });
94
95 if (typeof (module?.default?.fetch ?? module?.default?.respond) === 'function') {
96 const http = await import('@clo/lib/http');
97 const { url } = await http.serve({
98 respond: module?.default?.fetch ?? module?.default?.respond,
99 port: 3000,
100 });
101 log.info(`-> ${url}`);
149102 }
150103}
151
152export { hot };
tsconfig.json+1-2
......@@ -1,7 +1,6 @@
11{
22 "compilerOptions": {
33 "allowImportingTsExtensions": true,
4 "baseUrl": ".",
54 "incremental": true,
65 "jsx": "react-jsxdev",
76 "jsxImportSource": "#jsx",
......@@ -16,7 +15,7 @@
1615 "target": "esnext",
1716 "noUncheckedIndexedAccess": true,
1817 "resolveJsonModule": true,
19 "types": ["marko", "node"]
18 "types": ["node", "marko"]
2019 },
2120 "include": ["./lib/**/*", "src/**/*", "framework/**/*"]
2221}