1// Incremental build system using singleton state.
2// See `work()`, `compile()`, and `invalidate()` for details.
3//
4// All state is serializable to allow recovering state across sessions.
5// This library special-cases the asset map, but is otherwise agnostic.
6let running = false;
7let jobs = 0;
8let newKeys = 0;
9let seenWorks = new Set<string>(); // for detecting conflict vs overwrite
10let seenWrites = new Set<string>(); // for detecting conflict vs overwrite
11let works = new Map<string, Work>();
12let files = new Map<string, TrackedFile>(); // keyed by `toRel` path
13let writes = new Map<string, FileWrite>();
14let assets = new Map<string, Asset>(); // keyed by hash
15
16export interface Ref<T> {
17 key: string;
18 /** This method is compatible with `await` syntax */
19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20 get value(): T | null;
21 /** The progress node passed into the input */
22 progress: progress.Node | progress.Ref;
23}
24type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
25
26/**
27 * Declare and a unit of work. Return value is memoized and only rebuilt when
28 * inputs change. Inputs are tracked via the `io` interface, as well as a hash
29 * of the `input` param and caller source code. Outputs are written at the end
30 * of a compilation (see `compile`).
31 *
32 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33 */
34export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
35export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
36export function work<I, O>(
37 p: progress.Ref,
38 job: Job<I, O>,
39 input?: I,
40): Ref<O> {
41 const source = JSON.stringify(UNWRAP(stack.capture()[0]));
42 const keySource = [source, util.inspect(input)].join(":");
43 const key = crypto.createHash("sha1").update(keySource).digest("base64url");
44 ASSERT(running);
45 ASSERT(
46 !seenWorks.has(key),
47 `Key '${key}' must be unique during the build. `
48 + `To fix this, provide a manual 'key' argument.`,
49 );
50 seenWorks.add(key);
51
52 const prev = works.get(key) as Work<O> | null;
53 if (prev) {
54 return {
55 key,
56 then: (done) => done(prev.value),
57 value: prev.value,
58 progress: p,
59 };
60 }
61
62 async function perform() {
63 const io = new Io(key, p);
64 jobs += 1;
65 newKeys += 1;
66 try {
67 const value = await job(io, input as I);
68 validateSerializable(value, "");
69 const { reads, writes } = io;
70 works.set(key, { value, affects: [], reads, writes });
71 for (const add of reads.files) {
72 const { affects } = UNWRAP(files.get(add));
73 ASSERT(!affects.includes(key));
74 affects.push(key);
75 }
76 for (const add of reads.works) {
77 const { affects } = UNWRAP(works.get(add));
78 ASSERT(!affects.includes(key));
79 affects.push(key);
80 }
81 return value;
82 } finally {
83 jobs -= 1;
84 }
85 }
86
87 let cached: Promise<O>;
88 return {
89 key,
90 then(fufill, reject) {
91 (cached ??= perform()).then(fufill, reject);
92 },
93 get value() {
94 return (works.get(this.key)?.value as O) ?? null;
95 },
96 progress: p,
97 };
98}
99
100export async function compile<T>(
101 compiler: (node: progress.Ref) => Promise<T>,
102 p: progress.Ref = progress,
103) {
104 ASSERT(!running, `Cannot run twice at the same time.`);
105 const startDate = new Date();
106 const start = performance.now();
107 const node = p.start("sitegen!");
108 try {
109 running = true;
110 ASSERT(jobs === 0);
111
112 const value = await compiler(node);
113 ASSERT(jobs === 0);
114 await flush(node, start);
115 return {
116 value,
117 watchFiles: new Set(files.keys()),
118 newOutputs: Array.from(seenWrites)
119 .filter((x) => x.startsWith("f:"))
120 .map((x) => x.slice(2)),
121 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
122 };
123 } finally {
124 node.text = `sitegen at ${startDate.toLocaleDateString()} ${startDate.toLocaleTimeString()}`;
125 node.end();
126 running = false;
127 newKeys = 0;
128 seenWrites.clear();
129 seenWorks.clear();
130 }
131}
132
133export async function flush(progress: progress.Node, start: number) {
134 using _ = progress.start("incremental flush");
135 // Trim
136 const detachedFiles = new Set<string>();
137 const referencedAssets = new Set<string>();
138 for (const [k, v] of works) {
139 const assets = v.writes.assets;
140 if (seenWorks.has(k)) {
141 for (const asset of assets.values()) referencedAssets.add(asset.hash);
142 continue;
143 }
144 deleteWork(k);
145 }
146 for (const [k, file] of files) {
147 if (file.affects.length > 0) continue;
148 files.delete(k);
149 detachedFiles.add(k);
150 }
151 for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
152
153 const p: Promise<void>[] = [];
154 // File writes
155 let dist = 0;
156 for (const [key, { buffer, size }] of writes) {
157 if (buffer) p.push(fs.writeMkdir(path.join(`.clover/o/${key}`), buffer));
158 dist += size;
159 }
160 // Asset map
161 {
162 const { json, blob, dynamic, dts } = getAssetManifest();
163 const jsonString = Buffer.from(JSON.stringify(json));
164 p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
165 p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
166 p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
167 for (const [k, v] of dynamic) {
168 p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
169 }
170 dist += blob.byteLength + jsonString.byteLength;
171 }
172 await Promise.all(p);
173
174 // Incremental state
175 const serialized = msgpackr.pack(serialize());
176 await fs.writeMkdir(".clover/incr.state", serialized);
177 const time = (performance.now() - start).toFixed(0);
178 progress.log.info(
179 `sitegen! in ${time} ms\n`
180 + ` - ${works.size} keys (${works.size - newKeys} cached)\n`
181 + ` - ${assets.size} static assets\n`
182 + ` - dist: ${string.formatByteSize(dist)},`
183 + ` incremental: ${string.formatByteSize(serialized.byteLength)}`,
184 );
185}
186
187export async function restore() {
188 let buffer;
189 try {
190 buffer = await fs.readFile(".clover/incr.state");
191 } catch (err: any) {
192 if (err.code !== "ENOENT") throw err;
193 }
194 if (!buffer) return;
195 await deserialize(buffer);
196}
197
198function forceInvalidate(entry: { affects: string[] }) {
199 const queue = [...entry.affects];
200 let key;
201 while ((key = queue.shift())) {
202 const affects = deleteWork(key);
203 queue.push(...affects);
204 }
205}
206
207function deleteWork(key: string) {
208 const work = works.get(key);
209 if (!work) return [];
210 const { reads, affects, writes: w } = work;
211 for (const remove of reads.files) {
212 const { affects } = UNWRAP(files.get(remove));
213 ASSERT(affects.includes(key));
214 affects.splice(affects.indexOf(key), 1);
215 }
216 for (const remove of reads.works) {
217 const { affects } = UNWRAP(works.get(remove), remove);
218 ASSERT(affects.includes(key));
219 affects.splice(affects.indexOf(key), 1);
220 }
221 for (const remove of affects) {
222 const {
223 reads: { works: list },
224 } = UNWRAP(works.get(remove), remove);
225 ASSERT(list.has(key));
226 list.delete(key);
227 }
228 for (const file of w.files) {
229 if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
230 }
231 // Assets are temporarily kept, trimmed via manual GC after compilation.
232
233 works.delete(key);
234 return affects;
235}
236
237export function reset() {
238 ASSERT(!running);
239 works.clear();
240 files.clear();
241 assets.clear();
242}
243
244export function serialize() {
245 const fileEntries = Array.from(
246 files,
247 ([k, v]) =>
248 [
249 k,
250 v.type,
251 v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
252 ...v.affects,
253 ] as const,
254 );
255 const workEntries = Array.from(
256 works,
257 ([k, v]) =>
258 [
259 k,
260 v.value,
261 Array.from(v.reads.files),
262 Array.from(v.reads.works),
263 Array.from(v.writes.files),
264 Array.from(
265 v.writes.assets,
266 ([k, { headers }]) => [k, headers] as const,
267 ),
268 v.affects,
269 ] as const,
270 );
271 const expectedFilesOnDisk = Array.from(
272 writes,
273 ([k, { size, work }]) => [k, size, work] as const,
274 );
275 const assetEntries = Array.from(
276 assets,
277 ([k, asset]) => [k, asset.raw, asset.gzip, asset.zstd] as const,
278 );
279 return [
280 1,
281 fileEntries,
282 workEntries,
283 expectedFilesOnDisk,
284 assetEntries,
285 ] as const;
286}
287type SerializedState = ReturnType<typeof serialize>;
288
289/* No-op on failure */
290async function deserialize(buffer: Buffer) {
291 const decoded = msgpackr.decode(buffer) as SerializedState;
292 if (!Array.isArray(decoded)) return;
293 if (decoded[0] !== 1) return;
294 const [, fileEntries, workEntries, expectedFilesOnDisk, assetEntries] = decoded;
295 for (const [k, type, content, ...affects] of fileEntries) {
296 if (type === "f") {
297 ASSERT(typeof content === "number");
298 files.set(k, { type, affects, lastModified: content });
299 } else if (type === "d") {
300 ASSERT(typeof content === "string");
301 files.set(k, { type, affects, contentHash: content, contents: [] });
302 } else {
303 files.set(k, { type, affects });
304 }
305 }
306 for (const entry of workEntries) {
307 const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] = entry;
308 works.set(k, {
309 value,
310 reads: {
311 files: new Set(readFiles),
312 works: new Set(readWorks),
313 },
314 writes: {
315 files: new Set(writeFiles),
316 assets: new Map(
317 Array.from(writeAssets, ([k, headers]) => [
318 k,
319 {
320 hash: JSON.parse(UNWRAP(headers.etag)),
321 headers,
322 },
323 ]),
324 ),
325 },
326 affects,
327 });
328 }
329 const statFiles = await Promise.all(
330 expectedFilesOnDisk.map(([k, size, work]) =>
331 fs
332 .stat(path.join(".clover/o", k))
333 .catch((err) => {
334 if (err.code === "ENOENT") return null;
335 throw err;
336 })
337 .then((stat) => ({ k, size, work, stat }))
338 ),
339 );
340 for (const { k, stat, work, size } of statFiles) {
341 if (stat?.size === size) {
342 writes.set(k, {
343 size: size,
344 buffer: null,
345 work,
346 });
347 } else {
348 forceInvalidate({ affects: [work] });
349 }
350 }
351 for (const [hash, raw, gzip, zstd] of assetEntries) {
352 assets.set(hash, { raw, gzip, zstd });
353 }
354
355 await Promise.all(
356 Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
357 );
358}
359
360export async function invalidate(
361 filePath: string,
362 unloadModule: boolean = true,
363): Promise<boolean> {
364 const key = toRel(toAbs(filePath));
365 const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
366 return invalidateEntry(key, file, unloadModule);
367}
368
369export async function invalidateEntry(
370 key: string,
371 file: TrackedFile,
372 unloadModule: boolean,
373): Promise<boolean> {
374 try {
375 if (file.type === "d") {
376 const contents = (file.contents = await fs.readdir(key));
377 contents.sort();
378 const contentHash = crypto
379 .createHash("sha1")
380 .update(contents.join("\0"))
381 .digest("base64url");
382 if (file.contentHash !== contentHash) {
383 file.contentHash = contentHash;
384 throw new Error();
385 }
386 } else if (file.type === "f") {
387 const lastModified = await fs.stat(key).then(
388 (x) => Math.floor(x.mtimeMs),
389 () => 0,
390 );
391 if (file.lastModified !== lastModified) {
392 file.lastModified = lastModified;
393 throw new Error();
394 }
395 } else {
396 file.type satisfies "null";
397 const stat = await fs.stat(key).catch(() => null);
398 if (stat) throw new Error();
399 }
400 return false;
401 } catch (e) {
402 forceInvalidate(file);
403 if (unloadModule) {
404 // TODO: handle when this triggers unloading of `generate.ts`
405 hot.unload(toAbs(key));
406 }
407 if (file.type === "null") files.delete(key);
408 return true;
409 }
410}
411
412export function getAssetManifest() {
413 const dynamic = new Map<string, Buffer>();
414 const writer = new BufferWriter();
415 const assetMap = Object.fromEntries(
416 Array.from(works, (work) => work[1].writes.assets)
417 .filter((map) => map.size > 0)
418 .flatMap((map) =>
419 Array.from(map, ([key, { hash, headers, regenerative }]) => {
420 const { raw, gzip, zstd } = UNWRAP(
421 assets.get(hash),
422 `Asset ${key} (${hash})`,
423 );
424 if (regenerative) {
425 const id = crypto
426 .createHash("sha1")
427 .update(key)
428 .digest("hex")
429 .slice(0, 16); /* TODO */
430 dynamic.set(
431 id,
432 manifest.packDynamicBuffer(raw, gzip, zstd, headers),
433 );
434 return [key, { type: 1, id }] as const;
435 }
436 return [
437 key,
438 {
439 type: 0,
440 raw: writer.write(raw, "raw:" + hash),
441 gzip: writer.write(gzip, "gzip:" + hash),
442 zstd: writer.write(zstd, "zstd:" + hash),
443 headers,
444 },
445 ] as const;
446 })
447 ),
448 ) satisfies manifest.Manifest;
449 return {
450 json: assetMap,
451 blob: writer.get(),
452 dynamic,
453 dts: "export type AssetKey = "
454 + Object.keys(assetMap)
455 .map((key) => JSON.stringify(key))
456 .join(" | ")
457 + "\n",
458 };
459}
460
461/* Input/Output with automatic tracking.
462 * - Inputs read with Io are tracked to know when to rebuild
463 * - Outputs written with Io are deleted when abandoned.
464 */
465export class Io {
466 constructor(public key: string, private node: progress.Ref) {}
467 reads: Reads = { files: new Set(), works: new Set() };
468 writes: Writes = { files: new Set(), assets: new Map() };
469
470 start(text: string, options?: progress.StartOptions) {
471 return this.node.start(text, options);
472 }
473
474 #trackFs(file: string) {
475 const resolved = toAbs(file);
476 const key = toRel(resolved);
477 this.reads.files.add(key);
478 return { resolved, key };
479 }
480 async readWork<T>(ref: Ref<T>): Promise<T> {
481 this.reads.works.add(ref.key);
482 return await ref;
483 }
484 /** Track a file in the compilation without reading it. */
485 async trackFile(file: string) {
486 const { key, resolved } = this.#trackFs(file);
487 if (!files.get(key)) {
488 let lastModified: number = 0;
489 try {
490 lastModified = Math.floor((await fs.stat(file)).mtimeMs);
491 files.set(key, { type: "f", lastModified, affects: [] });
492 } catch {
493 files.set(key, { type: "null", affects: [] });
494 }
495 }
496 return resolved;
497 }
498 async readFile(file: string): Promise<Buffer>;
499 async readFile(file: string, encoding: "utf-8"): Promise<string>;
500 async readFile(file: string, encoding?: "utf-8") {
501 return fs.readFile(await this.trackFile(file), encoding);
502 }
503 async readJson<T>(file: string) {
504 return JSON.parse(await this.readFile(file, "utf-8")) as T;
505 }
506 async readDir(dir: string) {
507 const { key, resolved } = this.#trackFs(dir);
508 const existing = files.get(key);
509 try {
510 if (existing?.type === "d") return existing.contents;
511 const contents = await fs.readdir(resolved);
512 contents.sort();
513 const contentHash = crypto
514 .createHash("sha1")
515 .update(contents.join("\0"))
516 .digest("base64url");
517 files.set(key, {
518 type: "d",
519 affects: [],
520 contentHash,
521 contents,
522 });
523 return contents;
524 } catch (err) {
525 if (!existing) files.set(key, { type: "null", affects: [] });
526 throw err;
527 }
528 }
529 async readDirRecursive(dir: string): Promise<string[]> {
530 const dirs = await this.readDir(dir);
531 return (
532 await Promise.all(
533 dirs.map(async (child) => {
534 const abs = path.join(dir, child);
535 const stat = await fs.stat(abs);
536 if (stat.isDirectory()) {
537 return (await this.readDirRecursive(abs)).map((grand) => path.join(child, grand));
538 } else {
539 return child;
540 }
541 }),
542 )
543 ).flat();
544 }
545 /* Track all dependencies of a module. */
546 async import<T>(file: string): Promise<T> {
547 const { resolved } = this.#trackFs(file);
548 try {
549 return require(resolved) as T;
550 } finally {
551 const queue = [resolved];
552 const seen = new Set<string>();
553 let current;
554 while ((current = queue.shift())) {
555 const stat = hot.getFileStat(current);
556 if (!stat) continue;
557 const { key } = this.#trackFs(current);
558 if (!files.get(key)) {
559 files.set(key, {
560 type: "f",
561 affects: [],
562 lastModified: stat?.lastModified ?? 0,
563 });
564 }
565 for (const imp of stat.imports) {
566 if (!seen.has(imp)) {
567 seen.add(imp);
568 queue.push(imp);
569 }
570 }
571 }
572 }
573 }
574 async writeAsset(asset: {
575 pathname: string;
576 buffer: string | Uint8Array;
577 regenerative?: boolean;
578 headers?: HeadersInit;
579 }) {
580 ASSERT(asset.pathname.startsWith("/"));
581 ASSERT(!seenWrites.has("a:" + asset.pathname));
582 const buffer = typeof asset.buffer === "string"
583 ? Buffer.from(asset.buffer)
584 : asset.buffer;
585 const headers = new Headers(asset.headers ?? {});
586 const hash = crypto.createHash("sha1").update(buffer).digest("hex");
587 if (!headers.has("Content-Type")) {
588 headers.set("Content-Type", mime.for(asset.pathname));
589 }
590 headers.set("etag", JSON.stringify(hash));
591 this.writes.assets.set(asset.pathname, {
592 hash,
593 headers: Object.fromEntries(headers),
594 regenerative: !!asset.regenerative,
595 });
596 if (!assets.has(hash)) {
597 jobs += 1;
598 assets.set(hash, undefined!);
599 const [gzipBuffer, zstdBuffer] = await Promise.all([
600 gzip(buffer),
601 zstdCompress(buffer),
602 ]);
603 assets.set(hash, {
604 raw: buffer,
605 gzip: gzipBuffer,
606 zstd: zstdBuffer,
607 });
608 jobs -= 1;
609 }
610 }
611 writeFile(subPath: string, blob: string | Buffer) {
612 ASSERT(!subPath.startsWith("/"));
613 ASSERT(
614 !seenWrites.has("f:" + subPath),
615 `File overwritten: ${JSON.stringify(subPath)}`,
616 );
617 seenWrites.add("f:" + subPath);
618 const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
619 writes.set(subPath, {
620 buffer,
621 size: buffer.byteLength,
622 work: this.key,
623 });
624 }
625}
626
627class BufferWriter {
628 size = 0;
629 seen = new Map<string, BufferView>();
630 buffers: Buffer[] = [];
631
632 write(buffer: Buffer, hash: string): BufferView {
633 let view = this.seen.get(hash);
634 if (view) return view;
635 view = [this.size, this.size += buffer.byteLength];
636 this.seen.set(hash, view);
637 this.buffers.push(buffer);
638 return view;
639 }
640
641 get() {
642 return Buffer.concat(this.buffers);
643 }
644}
645
646export function validateSerializable(value: unknown, key: string) {
647 if (typeof value === "string") {
648 if (value.includes(hot.projectRoot)) {
649 throw new Error(
650 `Return value must not contain the CWD for portability, found at ${key}`,
651 );
652 }
653 } else if (value && typeof value === "object") {
654 if (Array.isArray(value)) {
655 value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
656 } else if (
657 Object.getPrototypeOf(value) === Object.prototype
658 || Buffer.isBuffer(value)
659 ) {
660 Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`));
661 } else {
662 throw new Error(
663 `Return value must be a plain JS object, found ${Object.getPrototypeOf(value).constructor.name} at ${key}`,
664 );
665 }
666 } else if (["bigint", "function", "symbol"].includes(typeof value)) {
667 throw new Error(
668 `Return value must be a plain JS object, found ${typeof value} at ${key}`,
669 );
670 }
671}
672
673export function toAbs(absPath: string) {
674 return path.resolve(hot.projectRoot, absPath);
675}
676
677export function toRel(absPath: string) {
678 return path.relative(hot.projectRoot, absPath).replaceAll("\\", "/");
679}
680
681export type BufferView = [start: number, end: number];
682interface Reads {
683 files: Set<string>;
684 works: Set<string>;
685}
686interface FileWrite {
687 buffer: Buffer | null;
688 size: number;
689 work: string;
690}
691interface Writes {
692 files: Set<string>;
693 assets: Map<string, AssetWrite>;
694}
695interface Asset {
696 raw: Buffer;
697 gzip: Buffer;
698 zstd: Buffer;
699}
700interface AssetWrite {
701 hash: string;
702 headers: Record<string, string>;
703 regenerative: boolean;
704}
705interface Work<T = unknown> {
706 value: T;
707 reads: Reads;
708 writes: Writes;
709 affects: string[];
710}
711type TrackedFile =
712 & { affects: string[] }
713 & (
714 | { type: "f"; lastModified: number }
715 | { type: "d"; contentHash: string; contents: string[] }
716 | { type: "null" }
717 );
718
719const gzip = util.promisify(zlib.gzip);
720const zstdCompress = util.promisify(zlib.zstdCompress);
721
722import * as hot from "./hot.ts";
723
724import * as manifest from "#sitegen/assets";
725import * as fs from "#sitegen/fs";
726import * as mime from "@clo/lib/mime";
727
728import * as crypto from "node:crypto";
729import * as path from "node:path";
730import * as util from "node:util";
731import * as zlib from "node:zlib";
732
733import * as stack from "@clo/lib/log/stack";
734import * as progress from "@clo/lib/progress";
735import * as string from "@clo/lib/string";
736
737import { ASSERT, UNWRAP } from "@clo/lib/assert";
738import * as msgpackr from "msgpackr";