authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-11 00:17:58-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log896eb7a6b0005ea9214fc7f59ee0a5ac5e5d5a6d
tree07077ed1f53173b9ca9122f69ba9fbf83f1598e0
parent69a5d0ab42a5b739a835fea2fe2735e93eba30fd
signature Commit is signed but in an unrecognized format.

fine grained incremental rebuilding

7 files changed, 288 insertions(+), 117 deletions(-)

framework/bundle.ts+11-11
......@@ -65,7 +65,7 @@ export async function bundleClientJavaScript(
6565 route = "/js/" + key + ".js";
6666 incr.put({
6767 sources,
68 type: "script",
68 kind: "script",
6969 key,
7070 value: text,
7171 });
......@@ -91,16 +91,6 @@ export async function bundleServerJavaScript(
9191 platform: ServerPlatform = "node",
9292) {
9393 const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
94 const viewModules = viewEntryPoints.map((view) => {
95 const module = require(view.file);
96 if (!module.meta) {
97 throw new Error(`${view.file} is missing 'export const meta'`);
98 }
99 if (!module.default) {
100 throw new Error(`${view.file} is missing a default export.`);
101 }
102 return { module, view };
103 });
10494 const viewSource = [
10595 ...viewEntryPoints.map((view, i) =>
10696 `import * as view${i} from ${JSON.stringify(view.file)}`
......@@ -171,6 +161,16 @@ export async function bundleServerJavaScript(
171161 write: false,
172162 metafile: true,
173163 });
164 const viewModules = viewEntryPoints.map((view) => {
165 const module = require(view.file);
166 if (!module.meta) {
167 throw new Error(`${view.file} is missing 'export const meta'`);
168 }
169 if (!module.default) {
170 throw new Error(`${view.file} is missing a default export.`);
171 }
172 return { module, view };
173 });
174174 const viewData = viewModules.map(({ module, view }) => {
175175 return {
176176 id: view.id,
framework/css.ts+18
......@@ -5,6 +5,12 @@ export interface Theme {
55 h1?: string;
66}
77
8export const defaultTheme: Theme = {
9 bg: "#ffffff",
10 fg: "#050505",
11 primary: "#2e7dab",
12};
13
814export function stringifyTheme(theme: Theme) {
915 return [
1016 ":root {",
......@@ -39,6 +45,18 @@ export interface Output {
3945 sources: string[];
4046}
4147
48export function styleKey(
49 cssImports: string[],
50 theme: Theme,
51) {
52 cssImports = cssImports
53 .map((file) =>
54 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
55 )
56 .sort();
57 return cssImports.join(":") + JSON.stringify(theme);
58}
59
4260export async function bundleCssFiles(
4361 cssImports: string[],
4462 theme: Theme,
framework/definitions.d.ts+1-1
......@@ -1,4 +1,4 @@
1declare function UNWRAP<T>(value: T | null | undefined): T;
1declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
22declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
33
44type Timer = ReturnType<typeof setTimeout>;
framework/generate.ts+187-81
......@@ -1,11 +1,16 @@
1// This file contains the main site generator build process.
2// By using `Incremental`'s ability to automatically purge stale
3// assets, the `sitegen` function performs partial rebuilds.
4
15export function main(incremental?: Incremental) {
26 return withSpinner<Record<string, unknown>, any>({
37 text: "Recovering State",
48 successText,
59 failureText: () => "sitegen FAIL",
610 }, async (spinner) => {
7 const incr = Incremental.fromDisk();
8 await incr.statAllFiles();
11 // const incr = Incremental.fromDisk();
12 // await incr.statAllFiles();
13 const incr = new Incremental();
914 const result = await sitegen(spinner, incr);
1015 incr.toDisk(); // Allows picking up this state again
1116 return result;
......@@ -94,26 +99,42 @@ export async function sitegen(
9499 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
95100 const globalCssPath = join("global.css");
96101
97 // TODO: invalidate incremental resources
102 // TODO: make sure that `static` and `pages` does not overlap
98103
99 // -- server side render --
104 // -- inline style sheets, used and shared by pages and views --
100105 status.text = "Building";
101 const cssOnce = new OnceMap<css.Output>();
102 const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
106 const cssOnce = new OnceMap();
107 const cssQueue = new Queue({
103108 name: "Bundle",
104 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
109 async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
110 const { text, sources } = await css.bundleCssFiles(files, theme);
111 incr.put({
112 kind: "style",
113 key,
114 sources,
115 value: text,
116 });
117 },
105118 passive: true,
106119 getItemText: ([id]) => id,
107120 maxJobs: 2,
108121 });
109 interface RenderResult {
110 body: string;
111 head: string;
112 css: css.Output;
113 clientRefs: string[];
114 item: FileItem;
122 function ensureCssGetsBuilt(
123 cssImports: string[],
124 theme: css.Theme,
125 referrer: string,
126 ) {
127 const key = css.styleKey(cssImports, theme);
128 cssOnce.get(
129 key,
130 async () => {
131 incr.getArtifact("style", key) ??
132 await cssQueue.add([referrer, key, cssImports, theme]);
133 },
134 );
115135 }
116 const renderResults: RenderResult[] = [];
136
137 // -- server side render pages --
117138 async function loadPageModule({ file }: FileItem) {
118139 require(file);
119140 }
......@@ -125,28 +146,27 @@ export async function sitegen(
125146 theme: pageTheme,
126147 layout,
127148 } = require(item.file);
128 if (!Page) throw new Error("Page is missing a 'default' export.");
149 if (!Page) {
150 throw new Error("Page is missing a 'default' export.");
151 }
129152 if (!metadata) {
130153 throw new Error("Page is missing 'meta' export with a title.");
131154 }
155
156 // -- css --
132157 if (layout?.theme) pageTheme = layout.theme;
133 const theme = {
134 bg: "#fff",
135 fg: "#050505",
136 primary: "#2e7dab",
158 const theme: css.Theme = {
159 ...css.defaultTheme,
137160 ...pageTheme,
138161 };
162 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
163 ensureCssGetsBuilt(cssImports, theme, item.id);
139164
140165 // -- metadata --
141166 const renderedMetaPromise = Promise.resolve(
142167 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
143168 ).then((m) => meta.renderMeta(m));
144 // -- css --
145 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
146 const cssPromise = cssOnce.get(
147 cssImports.join(":") + JSON.stringify(theme),
148 () => cssQueue.add([item.id, cssImports, theme]),
149 );
169
150170 // -- html --
151171 let page = [engine.kElement, Page, {}];
152172 if (layout?.default) {
......@@ -156,9 +176,8 @@ export async function sitegen(
156176 sitegen: sg.initRender(),
157177 });
158178
159 const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
179 const [{ text, addon }, renderedMeta] = await Promise.all([
160180 bodyPromise,
161 cssPromise,
162181 renderedMetaPromise,
163182 ]);
164183 if (!renderedMeta.includes("<title>")) {
......@@ -167,20 +186,72 @@ export async function sitegen(
167186 "All pages need a title tag.",
168187 );
169188 }
170 // The script content is not ready, allow another page to Render. The page
171 // contents will be rebuilt at the end. This is more convenient anyways
172 // because it means client scripts don't re-render the page.
173 renderResults.push({
174 body: text,
175 head: renderedMeta,
176 css: cssBundle,
177 clientRefs: Array.from(addon.sitegen.scripts),
178 item: item,
189 incr.put({
190 kind: "pageMetadata",
191 key: item.id,
192 // Incremental integrates with `hot.ts` + `require`
193 // to trace all the needed source files here.
194 sources: [item.file],
195 value: {
196 html: text,
197 meta: renderedMeta,
198 cssImports,
199 theme: theme ?? null,
200 clientRefs: Array.from(addon.sitegen.scripts),
201 },
202 });
203 }
204 async function prepareView(view: FileItem) {
205 const module = require(view.file);
206 if (!module.meta) {
207 throw new Error(`${view.file} is missing 'export const meta'`);
208 }
209 if (!module.default) {
210 throw new Error(`${view.file} is missing a default export.`);
211 }
212 const pageTheme = module.layout?.theme ?? module.theme;
213 const theme: css.Theme = {
214 ...css.defaultTheme,
215 ...pageTheme,
216 };
217 const cssImports = hot.getCssImports(view.file)
218 .concat("src/global.css")
219 .map((file) => path.relative(hot.projectRoot, path.resolve(file)));
220 incr.put({
221 kind: "viewMetadata",
222 key: view.id,
223 sources: [view.file],
224 value: {
225 file: path.relative(hot.projectRoot, view.file),
226 cssImports,
227 theme,
228 clientRefs: hot.getClientScriptRefs(view.file),
229 hasLayout: !!module.layout?.default,
230 },
179231 });
180232 }
233
234 // Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
235 // required so that it's (1) re-built if needed, (2) not pruned from build.
236 const neededPages = pages.filter((page) => {
237 const existing = incr.getArtifact("pageMetadata", page.id);
238 if (existing) {
239 const { cssImports, theme } = existing;
240 ensureCssGetsBuilt(cssImports, theme, page.id);
241 }
242 return !existing;
243 });
244 const neededViews = views.filter((view) => {
245 const existing = incr.getArtifact("viewMetadata", view.id);
246 if (existing) {
247 const { cssImports, theme } = existing;
248 ensureCssGetsBuilt(cssImports, theme, view.id);
249 }
250 return !existing;
251 });
252
181253 // This is done in two passes so that a page that throws during evaluation
182254 // will report "Load Render Module" instead of "Render Static Page".
183 const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
184255 const spinnerFormat = status.format;
185256 status.format = () => "";
186257 const moduleLoadQueue = new Queue({
......@@ -190,6 +261,7 @@ export async function sitegen(
190261 maxJobs: 1,
191262 });
192263 moduleLoadQueue.addMany(neededPages);
264 moduleLoadQueue.addMany(neededViews);
193265 await moduleLoadQueue.done({ method: "stop" });
194266 const pageQueue = new Queue({
195267 name: "Render Static Page",
......@@ -198,32 +270,52 @@ export async function sitegen(
198270 maxJobs: 2,
199271 });
200272 pageQueue.addMany(neededPages);
273 const viewQueue = new Queue({
274 name: "Build Dynamic View",
275 fn: prepareView,
276 getItemText,
277 maxJobs: 2,
278 });
279 viewQueue.addMany(neededViews);
201280 await pageQueue.done({ method: "stop" });
281 await viewQueue.done({ method: "stop" });
202282 status.format = spinnerFormat;
203283
204284 // -- bundle backend and views --
205 status.text = "Bundle backend code";
206 const backend = await bundle.bundleServerJavaScript(
207 join("backend.ts"),
208 views,
209 );
210 const viewCssPromise = await Promise.all(
211 backend.views.map((view) =>
212 cssOnce.get(
213 view.cssImports.join(":") + JSON.stringify(view.theme),
214 () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
215 )
216 ),
217 );
285 // status.text = "Bundle backend code";
286 // const backend = await bundle.bundleServerJavaScript(
287 // join("backend.ts"),
288 // views,
289 // );
290 // const viewCssPromise = await Promise.all(
291 // backend.views.map((view) =>
292 // cssOnce.get(
293 // view.cssImports.join(":") + JSON.stringify(view.theme),
294 // () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
295 // )
296 // ),
297 // );
218298
219299 // -- bundle scripts --
220300 const referencedScripts = Array.from(
221 new Set([
222 ...renderResults.flatMap((r) => r.clientRefs),
223 ...backend.views.flatMap((r) => r.clientRefs),
224 ]),
301 new Set(
302 [
303 ...pages.map((item) =>
304 UNWRAP(
305 incr.getArtifact("pageMetadata", item.id),
306 `Missing pageMetadata ${item.id}`,
307 )
308 ),
309 ...views.map((item) =>
310 UNWRAP(
311 incr.getArtifact("viewMetadata", item.id),
312 `Missing viewMetadata ${item.id}`,
313 )
314 ),
315 ].flatMap((item) => item.clientRefs),
316 ),
225317 (script) => path.resolve(hot.projectSrc, script),
226 );
318 ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
227319 const extraPublicScripts = scripts.map((entry) => entry.file);
228320 const uniqueCount = new Set([
229321 ...referencedScripts,
......@@ -237,7 +329,7 @@ export async function sitegen(
237329 );
238330
239331 // -- finalize backend bundle --
240 await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
332 // await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
241333
242334 // -- copy/compress static files --
243335 async function doStaticFile(item: FileItem) {
......@@ -256,37 +348,51 @@ export async function sitegen(
256348 });
257349 status.format = () => "";
258350 staticQueue.addMany(
259 staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
351 staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
260352 );
261353 await staticQueue.done({ method: "stop" });
262354 status.format = spinnerFormat;
263355
356 await cssQueue.done({ method: "stop" });
357
264358 // -- concatenate static rendered pages --
265 status.text = `Concat ${renderResults.length} Pages`;
266 await Promise.all(
267 renderResults.map(
268 async (
269 { item: page, body, head, css, clientRefs: scriptFiles },
270 ) => {
271 const doc = wrapDocument({
272 body,
273 head,
274 inlineCss: css.text,
275 scripts: scriptFiles.map(
276 (file) => UNWRAP(incr.out.script.get(hot.getScriptId(file))),
277 ).map((x) => `{${x}}`).join("\n"),
278 });
279 await incr.putAsset({
280 sources: [page.file, ...css.sources],
281 key: page.id,
282 body: doc,
283 headers: {
284 "Content-Type": "text/html",
285 },
286 });
359 status.text = `Concat Pages`;
360 await Promise.all(pages.map(async (page) => {
361 if (incr.hasArtifact("asset", page.id)) return;
362 const {
363 html,
364 meta,
365 cssImports,
366 theme,
367 clientRefs,
368 } = UNWRAP(incr.out.pageMetadata.get(page.id));
369 const scriptIds = clientRefs.map(hot.getScriptId);
370 const styleKey = css.styleKey(cssImports, theme);
371 const style = UNWRAP(
372 incr.out.style.get(styleKey),
373 `Missing style ${styleKey}`,
374 );
375 const doc = wrapDocument({
376 body: html,
377 head: meta,
378 inlineCss: style,
379 scripts: scriptIds.map(
380 (ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
381 ).map((x) => `{${x}}`).join("\n"),
382 });
383 await incr.putAsset({
384 sources: [
385 page.file,
386 ...incr.sourcesFor("style", styleKey),
387 ...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
388 ],
389 key: page.id,
390 body: doc,
391 headers: {
392 "Content-Type": "text/html",
287393 },
288 ),
289 );
394 });
395 }));
290396 status.format = () => "";
291397 status.text = ``;
292398 // This will wait for all compression jobs to finish, which up
framework/incremental.ts+68-19
......@@ -1,27 +1,63 @@
1// `Incremental` contains multiple maps for the different parts of a site
2// build, and tracks reused items across builds. It also handles emitting and
3// updating the built site. This structure is self contained and serializable.
1// Incremental contains multiple maps for the different kinds
2// of Artifact, which contain a list of source files which
3// were used to produce it. When files change, Incremental sees
4// that the `mtime` is newer, and purges the referenced artifacts.
45
56type SourceId = string; // relative to project root, e.g. 'src/global.css'
6type ArtifactId = string; // `${ArtifactType}#${string}`
7type ArtifactId = string; // `${ArtifactType}\0${string}`
78type Sha1Id = string; // Sha1 hex string
89
10// -- artifact types --
911interface ArtifactMap {
12 /* An asset (serve with "#sitegen/asset" */
1013 asset: Asset;
14 /* The bundled text of a '.client.ts' script */
15 // TODO: track imports this has into `asset`
1116 script: string;
17 /* The bundled style tag contents. Keyed by 'css.styleKey' */
18 style: string;
19 /* Metadata about a static page */
20 pageMetadata: PageMetadata;
21 /* Metadata about a dynamic view */
22 viewMetadata: ViewMetadata;
1223}
13type ArtifactType = keyof ArtifactMap;
14interface Asset {
24type ArtifactKind = keyof ArtifactMap;
25export interface Asset {
1526 buffer: Buffer;
1627 headers: Record<string, string | undefined>;
1728 hash: string;
1829}
30/**
31 * This interface intentionally omits the *contents*
32 * of its scripts and styles for fine-grained rebuilds.
33 */
34export interface PageMetadata {
35 html: string;
36 meta: string;
37 cssImports: string[];
38 theme: css.Theme;
39 clientRefs: string[];
40}
41/**
42 * Like a page, this intentionally omits resources,
43 * but additionally omits the bundled server code.
44 */
45export interface ViewMetadata {
46 file: string;
47 // staticMeta: string | null; TODO
48 cssImports: string[];
49 theme: css.Theme;
50 clientRefs: string[];
51 hasLayout: boolean;
52}
53
54// -- incremental support types --
1955export interface PutBase {
2056 sources: SourceId[];
2157 key: string;
2258}
23export interface Put<T extends ArtifactType> extends PutBase {
24 type: T;
59export interface Put<T extends ArtifactKind> extends PutBase {
60 kind: T;
2561 value: ArtifactMap[T];
2662}
2763export interface Invalidations {
......@@ -37,6 +73,9 @@ export class Incremental {
3773 } = {
3874 asset: new Map(),
3975 script: new Map(),
76 style: new Map(),
77 pageMetadata: new Map(),
78 viewMetadata: new Map(),
4079 };
4180 /** Tracking filesystem entries to `srcId` */
4281 invals = new Map<SourceId, Invalidations>();
......@@ -53,9 +92,16 @@ export class Incremental {
5392 getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
5493 });
5594
56 /** Invalidation deletes build artifacts so the check is trivial. */
57 needsBuild(type: ArtifactType, key: string) {
58 return !this.out[type].has(key);
95 getArtifact<T extends ArtifactKind>(kind: T, key: string) {
96 return this.out[kind].get(key);
97 }
98
99 hasArtifact<T extends ArtifactKind>(kind: T, key: string) {
100 return this.out[kind].has(key);
101 }
102
103 sourcesFor(kind: ArtifactKind, key: string) {
104 return UNWRAP(this.sources.get(kind + "\0" + key));
59105 }
60106
61107 /*
......@@ -63,18 +109,19 @@ export class Incremental {
63109 * used to build this must be provided. 'Incremental' will trace JS
64110 * imports and file modification times tracked by 'hot.ts'.
65111 */
66 put<T extends ArtifactType>({
112 put<T extends ArtifactKind>({
67113 sources,
68 type,
114 kind,
69115 key,
70116 value,
71117 }: Put<T>) {
72 this.out[type].set(key, value);
118 console.log("put " + kind + ": " + key);
119 this.out[kind].set(key, value);
73120
74121 // Update sources information
75 ASSERT(sources.length > 0, "Missing sources for " + type + " " + key);
122 ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
76123 sources = sources.map((src) => path.normalize(src));
77 const fullKey = `${type}#${key}`;
124 const fullKey = `${kind}\0${key}`;
78125 const prevSources = this.sources.get(fullKey);
79126 const newSources = new Set(
80127 sources.map((file) =>
......@@ -155,8 +202,9 @@ export class Incremental {
155202 );
156203 const { files, outputs } = invalidations;
157204 for (const out of outputs) {
158 const [type, artifactKey] = out.split("#", 2);
159 this.out[type as ArtifactType].delete(artifactKey);
205 const [kind, artifactKey] = out.split("\0");
206 this.out[kind as ArtifactKind].delete(artifactKey);
207 console.log("stale " + kind + ": " + artifactKey);
160208 }
161209 invalidQueue.push(...files);
162210 }
......@@ -179,7 +227,7 @@ export class Incremental {
179227 },
180228 hash,
181229 };
182 const a = this.put({ ...info, type: "asset", value });
230 const a = this.put({ ...info, kind: "asset", value });
183231 if (!this.compress.has(hash)) {
184232 const label = info.key;
185233 this.compress.set(hash, {
......@@ -412,3 +460,4 @@ import * as hot from "./hot.ts";
412460import * as mime from "#sitegen/mime";
413461import * as path from "node:path";
414462import { Buffer } from "node:buffer";
463import * as css from "./css.ts";
framework/lib/view.ts+1-3
......@@ -42,10 +42,8 @@ export async function renderView(
4242 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
4343 sitegen: sg.initRender(),
4444 });
45 console.log(sitegen);
4645
4746 // -- join document and send --
48 console.log(scripts);
4947 return c.html(wrapDocument({
5048 body,
5149 head: await renderedMetaPromise,
......@@ -56,7 +54,7 @@ export async function renderView(
5654 }));
5755}
5856
59export function provideViews(v: typeof views, s: typeof scripts) {
57export function provideViewData(v: typeof views, s: typeof scripts) {
6058 views = v;
6159 scripts = s;
6260}
framework/watch.ts+2-2
......@@ -25,7 +25,7 @@ export async function main() {
2525 successText: generate.successText,
2626 failureText: () => "sitegen FAIL",
2727 }, async (spinner) => {
28 console.clear();
28 console.log("---");
2929 console.log(
3030 "Updated" +
3131 (changed.length === 1
......@@ -36,7 +36,7 @@ export async function main() {
3636 incr.toDisk(); // Allows picking up this state again
3737 for (const file of watch.files) {
3838 const relative = path.relative(hot.projectRoot, file);
39 if (!incr.invals.has(file)) watch.remove(file);
39 if (!incr.invals.has(relative)) watch.remove(file);
4040 }
4141 return result;
4242 }).catch((err) => {