authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-13 00:13:22-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
loge6cca067561965523a27ce63fabb551d3a6ef32f
tree0ff132f436e571b5d34c10e645dbf60934bb29dd
parent896eb7a6b0005ea9214fc7f59ee0a5ac5e5d5a6d
signature Commit is signed but in an unrecognized format.

incremental sitegen dev server!

8 files changed, 535 insertions(+), 233 deletions(-)

framework/bundle.ts+126-105
...@@ -1,8 +1,4 @@...@@ -1,8 +1,4 @@
1// This file implements client-side bundling, mostly wrapping esbuild.1// This file implements client-side bundling, mostly wrapping esbuild.
2const clientPlugins: esbuild.Plugin[] = [
3 // There are currently no plugins needed by 'paperclover.net'
4];
5
6export async function bundleClientJavaScript(2export async function bundleClientJavaScript(
7 referencedScripts: string[],3 referencedScripts: string[],
8 extraPublicScripts: string[],4 extraPublicScripts: string[],
...@@ -11,7 +7,7 @@ export async function bundleClientJavaScript(...@@ -11,7 +7,7 @@ export async function bundleClientJavaScript(
11) {7) {
12 const entryPoints = [8 const entryPoints = [
13 ...new Set([9 ...new Set([
14 ...referencedScripts,10 ...referencedScripts.map((file) => path.resolve(hot.projectSrc, file)),
15 ...extraPublicScripts,11 ...extraPublicScripts,
16 ]),12 ]),
17 ];13 ];
...@@ -26,6 +22,10 @@ export async function bundleClientJavaScript(...@@ -26,6 +22,10 @@ export async function bundleClientJavaScript(
26 );22 );
27 }23 }
2824
25 const clientPlugins: esbuild.Plugin[] = [
26 // There are currently no plugins needed by 'paperclover.net'
27 ];
28
29 const bundle = await esbuild.build({29 const bundle = await esbuild.build({
30 bundle: true,30 bundle: true,
31 chunkNames: "/js/c.[hash]",31 chunkNames: "/js/c.[hash]",
...@@ -51,7 +51,6 @@ export async function bundleClientJavaScript(...@@ -51,7 +51,6 @@ export async function bundleClientJavaScript(
51 );51 );
52 const { metafile } = bundle;52 const { metafile } = bundle;
53 const promises: Promise<void>[] = [];53 const promises: Promise<void>[] = [];
54 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
55 for (const file of bundle.outputFiles) {54 for (const file of bundle.outputFiles) {
56 const { text } = file;55 const { text } = file;
57 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");56 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
...@@ -61,7 +60,7 @@ export async function bundleClientJavaScript(...@@ -61,7 +60,7 @@ export async function bundleClientJavaScript(
61 // Register non-chunks as script entries.60 // Register non-chunks as script entries.
62 const chunk = route.startsWith("/js/c.");61 const chunk = route.startsWith("/js/c.");
63 if (!chunk) {62 if (!chunk) {
64 const key = hot.getScriptId(sources[0]);63 const key = hot.getScriptId(path.resolve(sources[0]));
65 route = "/js/" + key + ".js";64 route = "/js/" + key + ".js";
66 incr.put({65 incr.put({
67 sources,66 sources,
...@@ -82,41 +81,40 @@ export async function bundleClientJavaScript(...@@ -82,41 +81,40 @@ export async function bundleClientJavaScript(
82 await Promise.all(promises);81 await Promise.all(promises);
83}82}
8483
85type ServerPlatform = "node" | "passthru";84export type ServerPlatform = "node" | "passthru";
86export async function bundleServerJavaScript(85export async function bundleServerJavaScript(
87 /** Has 'export default app;' */86 incr: Incremental,
88 _: string,
89 /** Views for dynamic loading */
90 viewEntryPoints: FileItem[],
91 platform: ServerPlatform = "node",87 platform: ServerPlatform = "node",
92) {88) {
93 const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";89 if (incr.hasArtifact("backendBundle", platform)) return;
90
91 // Comment
92 const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
93
94 const viewSource = [94 const viewSource = [
95 ...viewEntryPoints.map((view, i) =>95 ...Array.from(
96 `import * as view${i} from ${JSON.stringify(view.file)}`96 incr.out.viewMetadata,
97 ([, view], i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
97 ),98 ),
98 `const styles = ${scriptMagic}[-2]`,99 `const styles = ${magicWord}[-2]`,
99 `export const scripts = ${scriptMagic}[-1]`,100 `export const scripts = ${magicWord}[-1]`,
100 "export const views = {",101 "export const views = {",
101 ...viewModules.flatMap(({ view, module }, i) => [102 ...Array.from(incr.out.viewMetadata, ([key, view], i) =>
102 ` ${JSON.stringify(view.id)}: {`,103 [
103 ` component: view${i}.default,`,104 ` ${JSON.stringify(key)}: {`,
104 ` meta: view${i}.meta,`,105 ` component: view${i}.default,`,
105 ` layout: ${106 // ` meta: ${
106 module.layout?.default ? `view${i}.layout?.default` : "undefined"107 // view.staticMeta ? JSON.stringify(view.staticMeta) : `view${i}.meta`
107 },`,108 // },`,
108 ` theme: ${109 ` meta: view${i}.meta,`,
109 module.layout?.theme110 ` layout: ${view.hasLayout ? `view${i}.layout?.default` : "null"},`,
110 ? `view${i}.layout?.theme`111 ` inlineCss: styles[${magicWord}[${i}]]`,
111 : module.theme112 ` },`,
112 ? `view${i}.theme`113 ].join("\n")),
113 : "undefined"
114 },`,
115 ` inlineCss: styles[${scriptMagic}[${i}]]`,
116 ` },`,
117 ]),
118 "}",114 "}",
119 ].join("\n");115 ].join("\n");
116
117 // -- plugins --
120 const serverPlugins: esbuild.Plugin[] = [118 const serverPlugins: esbuild.Plugin[] = [
121 virtualFiles({119 virtualFiles({
122 "$views": viewSource,120 "$views": viewSource,
...@@ -125,10 +123,21 @@ export async function bundleServerJavaScript(...@@ -125,10 +123,21 @@ export async function bundleServerJavaScript(
125 {123 {
126 name: "marko via build cache",124 name: "marko via build cache",
127 setup(b) {125 setup(b) {
128 b.onLoad({ filter: /\.marko$/ }, async ({ path: file }) => ({126 b.onLoad(
129 loader: "ts",127 { filter: /\.marko$/ },
130 contents: hot.getSourceCode(file),128 async ({ path: file }) => {
131 }));129 const key = path.relative(hot.projectRoot, file)
130 .replaceAll("\\", "/");
131 const cacheEntry = incr.out.serverMarko.get(key);
132 if (!cacheEntry) {
133 throw new Error("Marko file not in cache: " + file);
134 }
135 return ({
136 loader: "ts",
137 contents: cacheEntry.src,
138 });
139 },
140 );
132 },141 },
133 },142 },
134 {143 {
...@@ -145,10 +154,10 @@ export async function bundleServerJavaScript(...@@ -145,10 +154,10 @@ export async function bundleServerJavaScript(
145 },154 },
146 },155 },
147 ];156 ];
148 const bundle = await esbuild.build({157 const { metafile, outputFiles, warnings } = await esbuild.build({
149 bundle: true,158 bundle: true,
150 chunkNames: "c.[hash]",159 chunkNames: "c.[hash]",
151 entryNames: "[name]",160 entryNames: "server",
152 entryPoints: [161 entryPoints: [
153 path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),162 path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
154 ],163 ],
...@@ -161,81 +170,96 @@ export async function bundleServerJavaScript(...@@ -161,81 +170,96 @@ export async function bundleServerJavaScript(
161 write: false,170 write: false,
162 metafile: true,171 metafile: true,
163 });172 });
164 const viewModules = viewEntryPoints.map((view) => {173
165 const module = require(view.file);174 const files: Record<string, Buffer> = {};
166 if (!module.meta) {175 let fileWithMagicWord: string | null = null;
167 throw new Error(`${view.file} is missing 'export const meta'`);176 for (const output of outputFiles) {
168 }177 const basename = output.path.replace(/^.*?!/, "");
169 if (!module.default) {178 const key = "out!" + basename.replaceAll("\\", "/");
170 throw new Error(`${view.file} is missing a default export.`);179 // If this contains the generated "$views" file, then
180 // mark this file as the one for replacement. Because
181 // `splitting` is `true`, esbuild will not emit this
182 // file in more than one chunk.
183 if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
184 fileWithMagicWord = basename;
171 }185 }
172 return { module, view };186 files[basename] = Buffer.from(output.contents);
173 });187 }
174 const viewData = viewModules.map(({ module, view }) => {188 incr.put({
175 return {189 kind: "backendBundle",
176 id: view.id,190 key: platform,
177 theme: module.theme,191 value: {
178 cssImports: hot.getCssImports(view.file)192 magicWord,
179 .concat("src/global.css")193 files,
180 .map((file) => path.resolve(file)),194 fileWithMagicWord,
181 clientRefs: hot.getClientScriptRefs(view.file),195 },
182 };196 sources: Object.keys(metafile.inputs).filter((x) =>
197 !x.startsWith("vfs:") &&
198 !x.startsWith("dropped:") &&
199 !x.includes("node_modules")
200 ),
183 });201 });
184 return {
185 views: viewData,
186 bundle,
187 scriptMagic,
188 };
189}202}
190203
191type Await<T> = T extends Promise<infer R> ? R : T;204export async function finalizeServerJavaScript(
192
193export function finalizeServerJavaScript(
194 backend: Await<ReturnType<typeof bundleServerJavaScript>>,
195 viewCssBundles: css.Output[],
196 incr: Incremental,205 incr: Incremental,
206 platform: ServerPlatform,
197) {207) {
198 const { metafile, outputFiles } = backend.bundle;208 if (incr.hasArtifact("backendReplace", platform)) return;
209 const {
210 files,
211 fileWithMagicWord,
212 magicWord,
213 } = UNWRAP(incr.getArtifact("backendBundle", platform));
199214
200 // Only the reachable script files need to be inserted into the bundle.215 if (!fileWithMagicWord) return;
201 const reachableScriptKeys = new Set(216
202 backend.views.flatMap((view) => view.clientRefs),217 // Only the reachable resources need to be inserted into the bundle.
203 );218 const viewScriptsList = new Set(
204 const reachableScripts = Object.fromEntries(219 Array.from(incr.out.viewMetadata.values())
205 Array.from(incr.out.script)220 .flatMap((view) => view.clientRefs),
206 .filter(([k]) => reachableScriptKeys.has(k)),
207 );221 );
222 const viewStyleKeys = Array.from(incr.out.viewMetadata.values())
223 .map((view) => css.styleKey(view.cssImports, view.theme));
224 const viewCssBundles = viewStyleKeys
225 .map((key) => UNWRAP(incr.out.style.get(key), "Style key: " + key));
208226
209 // Deduplicate styles227 // Deduplicate styles
210 const styleList = Array.from(new Set(viewCssBundles));228 const styleList = Array.from(new Set(viewCssBundles));
211229
212 for (const output of outputFiles) {230 // Replace the magic word
213 const basename = output.path.replace(/^.*?!/, "");231 let text = files[fileWithMagicWord].toString("utf-8");
214 const key = "out!" + basename.replaceAll("\\", "/");232 text = text.replace(
233 new RegExp(magicWord + "\\[(-?\\d)\\]", "gs"),
234 (_, i) => {
235 i = Number(i);
236 // Inline the styling data
237 if (i === -2) {
238 return JSON.stringify(styleList.map((cssText) => cssText));
239 }
240 // Inline the script data
241 if (i === -1) {
242 return JSON.stringify(Object.fromEntries(incr.out.script));
243 }
244 // Reference an index into `styleList`
245 return `${styleList.indexOf(viewCssBundles[i])}`;
246 },
247 );
215248
216 // If this contains the generated "$views" file, then249 incr.put({
217 // replace the IDs with the bundled results.250 kind: "backendReplace",
218 let text = output.text;251 key: platform,
219 if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {252 sources: [
220 text = text.replace(253 // Backend input code (includes view code)
221 /CLOVER_CLIENT_SCRIPTS_DEFINITION\[(-?\d)\]/gs,254 ...incr.sourcesFor("backendBundle", platform),
222 (_, i) => {255 // Script
223 i = Number(i);256 ...Array.from(viewScriptsList)
224 // Inline the styling data257 .flatMap((key) => incr.sourcesFor("script", hot.getScriptId(key))),
225 if (i === -2) {258 // Style
226 return JSON.stringify(styleList.map((item) => item.text));259 ...viewStyleKeys.flatMap((key) => incr.sourcesFor("style", key)),
227 }260 ],
228 // Inline the script data261 value: Buffer.from(text),
229 if (i === -1) {262 });
230 return JSON.stringify(Object.fromEntries(incr.out.script));
231 }
232 // Reference an index into `styleList`
233 return `${styleList.indexOf(viewCssBundles[i])}`;
234 },
235 );
236 }
237 fs.writeMkdirSync(path.join(".clover/backend/" + basename), text);
238 }
239}263}
240264
241import * as esbuild from "esbuild";265import * as esbuild from "esbuild";
...@@ -248,7 +272,4 @@ import {...@@ -248,7 +272,4 @@ import {
248 virtualFiles,272 virtualFiles,
249} from "./esbuild-support.ts";273} from "./esbuild-support.ts";
250import { Incremental } from "./incremental.ts";274import { Incremental } from "./incremental.ts";
251import type { FileItem } from "#sitegen";
252import * as marko from "@marko/compiler";
253import * as css from "./css.ts";275import * as css from "./css.ts";
254import * as fs from "./lib/fs.ts";
framework/css.ts+5-5
...@@ -51,10 +51,12 @@ export function styleKey(...@@ -51,10 +51,12 @@ export function styleKey(
51) {51) {
52 cssImports = cssImports52 cssImports = cssImports
53 .map((file) =>53 .map((file) =>
54 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file54 (path.isAbsolute(file) ? path.relative(hot.projectSrc, file) : file)
55 .replaceAll("\\", "/")
55 )56 )
56 .sort();57 .sort();
57 return cssImports.join(":") + JSON.stringify(theme);58 return cssImports.join(":") + ":" +
59 Object.entries(theme).map(([k, v]) => `${k}=${v}`);
58}60}
5961
60export async function bundleCssFiles(62export async function bundleCssFiles(
...@@ -62,9 +64,7 @@ export async function bundleCssFiles(...@@ -62,9 +64,7 @@ export async function bundleCssFiles(
62 theme: Theme,64 theme: Theme,
63 dev: boolean = false,65 dev: boolean = false,
64): Promise<Output> {66): Promise<Output> {
65 cssImports = cssImports.map((file) =>67 cssImports = cssImports.map((file) => path.resolve(hot.projectSrc, file));
66 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
67 );
68 const plugin = {68 const plugin = {
69 name: "clover css",69 name: "clover css",
70 setup(b) {70 setup(b) {
framework/generate.ts+48-36
...@@ -2,23 +2,35 @@...@@ -2,23 +2,35 @@
2// By using `Incremental`'s ability to automatically purge stale2// By using `Incremental`'s ability to automatically purge stale
3// assets, the `sitegen` function performs partial rebuilds.3// assets, the `sitegen` function performs partial rebuilds.
44
5export function main(incremental?: Incremental) {5export function main() {
6 return withSpinner<Record<string, unknown>, any>({6 return withSpinner<Record<string, unknown>, any>({
7 text: "Recovering State",7 text: "Recovering State",
8 successText,8 successText,
9 failureText: () => "sitegen FAIL",9 failureText: () => "sitegen FAIL",
10 }, async (spinner) => {10 }, async (spinner) => {
11 // const incr = Incremental.fromDisk();11 const incr = Incremental.fromDisk();
12 // await incr.statAllFiles();12 await incr.statAllFiles();
13 const incr = new Incremental();13 // const incr = new Incremental();
14 const result = await sitegen(spinner, incr);14 const result = await sitegen(spinner, incr);
15 incr.toDisk(); // Allows picking up this state again15 incr.toDisk(); // Allows picking up this state again
16 return result;16 return result;
17 }) as ReturnType<typeof sitegen>;17 }) as ReturnType<typeof sitegen>;
18}18}
1919
20export function successText({ elapsed }: { elapsed: number }) {20export function successText({
21 return "sitegen! update in " + elapsed.toFixed(1) + "s";21 elapsed,
22 inserted,
23 referenced,
24 unreferenced,
25}: Awaited<ReturnType<typeof sitegen>>) {
26 const s = (array: unknown[]) => array.length === 1 ? "" : "s";
27 const kind = inserted.length === referenced.length ? "build" : "update";
28 const status = inserted.length > 0
29 ? `${kind} ${inserted.length} key${s(inserted)}`
30 : unreferenced.length > 0
31 ? `pruned ${unreferenced.length} key${s(unreferenced)}`
32 : `checked ${referenced.length} key${s(referenced)}`;
33 return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
22}34}
2335
24export async function sitegen(36export async function sitegen(
...@@ -159,7 +171,10 @@ export async function sitegen(...@@ -159,7 +171,10 @@ export async function sitegen(
159 ...css.defaultTheme,171 ...css.defaultTheme,
160 ...pageTheme,172 ...pageTheme,
161 };173 };
162 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];174 const cssImports = Array.from(
175 new Set([globalCssPath, ...hot.getCssImports(item.file)]),
176 (file) => path.relative(hot.projectSrc, file),
177 );
163 ensureCssGetsBuilt(cssImports, theme, item.id);178 ensureCssGetsBuilt(cssImports, theme, item.id);
164179
165 // -- metadata --180 // -- metadata --
...@@ -216,7 +231,7 @@ export async function sitegen(...@@ -216,7 +231,7 @@ export async function sitegen(
216 };231 };
217 const cssImports = hot.getCssImports(view.file)232 const cssImports = hot.getCssImports(view.file)
218 .concat("src/global.css")233 .concat("src/global.css")
219 .map((file) => path.relative(hot.projectRoot, path.resolve(file)));234 .map((file) => path.relative(hot.projectSrc, path.resolve(file)));
220 incr.put({235 incr.put({
221 kind: "viewMetadata",236 kind: "viewMetadata",
222 key: view.id,237 key: view.id,
...@@ -250,6 +265,9 @@ export async function sitegen(...@@ -250,6 +265,9 @@ export async function sitegen(
250 return !existing;265 return !existing;
251 });266 });
252267
268 // Load the marko cache before render modules are loaded
269 incr.loadMarkoCache();
270
253 // This is done in two passes so that a page that throws during evaluation271 // This is done in two passes so that a page that throws during evaluation
254 // will report "Load Render Module" instead of "Render Static Page".272 // will report "Load Render Module" instead of "Render Static Page".
255 const spinnerFormat = status.format;273 const spinnerFormat = status.format;
...@@ -281,22 +299,12 @@ export async function sitegen(...@@ -281,22 +299,12 @@ export async function sitegen(
281 await viewQueue.done({ method: "stop" });299 await viewQueue.done({ method: "stop" });
282 status.format = spinnerFormat;300 status.format = spinnerFormat;
283301
284 // -- bundle backend and views --302 // -- bundle server javascript (backend and views) --
285 // status.text = "Bundle backend code";303 status.text = "Bundle JavaScript";
286 // const backend = await bundle.bundleServerJavaScript(304 incr.snapshotMarkoCache();
287 // join("backend.ts"),305 const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
288 // views,
289 // );
290 // const viewCssPromise = await Promise.all(
291 // backend.views.map((view) =>
292 // cssOnce.get(
293 // view.cssImports.join(":") + JSON.stringify(view.theme),
294 // () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
295 // )
296 // ),
297 // );
298306
299 // -- bundle scripts --307 // -- bundle client javascript --
300 const referencedScripts = Array.from(308 const referencedScripts = Array.from(
301 new Set(309 new Set(
302 [310 [
...@@ -317,19 +325,17 @@ export async function sitegen(...@@ -317,19 +325,17 @@ export async function sitegen(
317 (script) => path.resolve(hot.projectSrc, script),325 (script) => path.resolve(hot.projectSrc, script),
318 ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));326 ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
319 const extraPublicScripts = scripts.map((entry) => entry.file);327 const extraPublicScripts = scripts.map((entry) => entry.file);
320 const uniqueCount = new Set([328 const clientJavaScriptPromise = bundle.bundleClientJavaScript(
321 ...referencedScripts,
322 ...extraPublicScripts,
323 ]).size;
324 status.text = `Bundle ${uniqueCount} Scripts`;
325 await bundle.bundleClientJavaScript(
326 referencedScripts,329 referencedScripts,
327 extraPublicScripts,330 extraPublicScripts,
328 incr,331 incr,
329 );332 );
330333 await Promise.all([
331 // -- finalize backend bundle --334 serverJavaScriptPromise,
332 // await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);335 clientJavaScriptPromise,
336 cssQueue.done({ method: "stop" }),
337 ]);
338 await bundle.finalizeServerJavaScript(incr, "node");
333339
334 // -- copy/compress static files --340 // -- copy/compress static files --
335 async function doStaticFile(item: FileItem) {341 async function doStaticFile(item: FileItem) {
...@@ -353,8 +359,6 @@ export async function sitegen(...@@ -353,8 +359,6 @@ export async function sitegen(
353 await staticQueue.done({ method: "stop" });359 await staticQueue.done({ method: "stop" });
354 status.format = spinnerFormat;360 status.format = spinnerFormat;
355361
356 await cssQueue.done({ method: "stop" });
357
358 // -- concatenate static rendered pages --362 // -- concatenate static rendered pages --
359 status.text = `Concat Pages`;363 status.text = `Concat Pages`;
360 await Promise.all(pages.map(async (page) => {364 await Promise.all(pages.map(async (page) => {
...@@ -399,11 +403,19 @@ export async function sitegen(...@@ -399,11 +403,19 @@ export async function sitegen(
399 // to this point have been left as dangling promises.403 // to this point have been left as dangling promises.
400 await incr.wait();404 await incr.wait();
401405
406 const { inserted, referenced, unreferenced } = incr.shake();
407
402 // Flush the site to disk.408 // Flush the site to disk.
403 status.format = spinnerFormat;409 status.format = spinnerFormat;
404 status.text = `Incremental Flush`;410 status.text = `Incremental Flush`;
405 incr.flush(); // Write outputs411 incr.flush("node"); // Write outputs
406 return { incr, elapsed: (performance.now() - startTime) / 1000 };412 return {
413 incr,
414 inserted,
415 referenced,
416 unreferenced,
417 elapsed: (performance.now() - startTime) / 1000,
418 };
407}419}
408420
409function getItemText({ file }: FileItem) {421function getItemText({ file }: FileItem) {
framework/hot.ts+54-40
...@@ -35,9 +35,10 @@ export interface FileStat {...@@ -35,9 +35,10 @@ export interface FileStat {
35 lastModified: number;35 lastModified: number;
36 imports: string[];36 imports: string[];
37}37}
38let fsGraph = new Map<string, FileStat>();38const fileStats = new Map<string, FileStat>();
39export function getFsGraph() {39
40 return fsGraph;40export function getFileStat(filepath: string) {
41 return fileStats.get(path.resolve(filepath));
41}42}
4243
43function shouldTrackPath(filename: string) {44function shouldTrackPath(filename: string) {
...@@ -63,18 +64,16 @@ Module.prototype._compile = function (...@@ -63,18 +64,16 @@ Module.prototype._compile = function (
63 const cssImportsMaybe: string[] = [];64 const cssImportsMaybe: string[] = [];
64 const imports: string[] = [];65 const imports: string[] = [];
65 for (const { filename: file } of this.children) {66 for (const { filename: file } of this.children) {
66 const relative = path.relative(projectRoot, file);67 if (file.endsWith(".css")) cssImportsMaybe.push(file);
67 if (file.endsWith(".css")) cssImportsMaybe.push(relative);
68 else {68 else {
69 const child = fsGraph.get(relative);69 const child = fileStats.get(file);
70 if (!child) continue;70 if (!child) continue;
71 const { cssImportsRecursive } = child;71 const { cssImportsRecursive } = child;
72 if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);72 if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
73 imports.push(relative);73 imports.push(file);
74 }74 }
75 }75 }
76 const relative = path.relative(projectRoot, filename);76 fileStats.set(filename, {
77 fsGraph.set(relative, {
78 cssImportsRecursive: cssImportsMaybe.length > 077 cssImportsRecursive: cssImportsMaybe.length > 0
79 ? Array.from(new Set(cssImportsMaybe))78 ? Array.from(new Set(cssImportsMaybe))
80 : null,79 : null,
...@@ -155,36 +154,46 @@ function resolveClientRef(sourcePath: string, ref: string) {...@@ -155,36 +154,46 @@ function resolveClientRef(sourcePath: string, ref: string) {
155 ) {154 ) {
156 throw new Error("addScript must be a .client.ts or .client.tsx");155 throw new Error("addScript must be a .client.ts or .client.tsx");
157 }156 }
158 return filePath;157 return path.relative(projectSrc, filePath);
159}158}
160159
160// TODO: extract the marko compilation tools out, lazy load them
161export interface MarkoCacheEntry {
162 src: string;
163 scannedClientRefs: string[];
164}
165export const markoCache = new Map<string, MarkoCacheEntry>();
161function loadMarko(module: NodeJS.Module, filepath: string) {166function loadMarko(module: NodeJS.Module, filepath: string) {
162 let src = fs.readFileSync(filepath, "utf8");167 let cache = markoCache.get(filepath);
163 // A non-standard thing here is Clover Sitegen implements168 if (!cache) {
164 // its own client side scripting stuff, so it overrides169 let src = fs.readFileSync(filepath, "utf8");
165 // bare client import statements to it's own usage.170 // A non-standard thing here is Clover Sitegen implements
166 const scannedClientRefs = new Set<string>();171 // its own client side scripting stuff, so it overrides
167 if (src.match(/^\s*client\s+import\s+["']/m)) {172 // bare client import statements to it's own usage.
168 src = src.replace(173 const scannedClientRefs = new Set<string>();
169 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,174 if (src.match(/^\s*client\s+import\s+["']/m)) {
170 (_, src) => {175 src = src.replace(
171 const ref = JSON.parse(`"${src.slice(1, -1)}"`);176 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
172 const resolved = resolveClientRef(filepath, ref);177 (_, src) => {
173 scannedClientRefs.add(resolved);178 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
174 return `<CloverScriptInclude=${179 const resolved = resolveClientRef(filepath, ref);
175 JSON.stringify(getScriptId(resolved))180 scannedClientRefs.add(resolved);
176 } />`;181 return `<CloverScriptInclude=${
177 },182 JSON.stringify(getScriptId(resolved))
178 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';183 } />`;
179 }184 },
180185 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
181 src = marko.compileSync(src, filepath).code;186 }
182 src = src.replace("marko/debug/html", "#ssr/marko");
183187
184 module.cloverSourceCode = src;188 src = marko.compileSync(src, filepath).code;
189 src = src.replace("marko/debug/html", "#ssr/marko");
190 cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
191 markoCache.set(filepath, cache);
192 }
185193
194 const { src, scannedClientRefs } = cache;
186 return loadEsbuildCode(module, filepath, src, {195 return loadEsbuildCode(module, filepath, src, {
187 scannedClientRefs: Array.from(scannedClientRefs),196 scannedClientRefs,
188 });197 });
189}198}
190199
...@@ -202,12 +211,19 @@ function loadCss(module: NodeJS.Module, _filepath: string) {...@@ -202,12 +211,19 @@ function loadCss(module: NodeJS.Module, _filepath: string) {
202export function reloadRecursive(filepath: string) {211export function reloadRecursive(filepath: string) {
203 filepath = path.resolve(filepath);212 filepath = path.resolve(filepath);
204 const existing = cache[filepath];213 const existing = cache[filepath];
205 if (existing) deleteRecursive(filepath, existing);214 if (existing) deleteRecursiveInner(filepath, existing);
206 fsGraph.clear();215 fileStats.clear();
207 return require(filepath);216 return require(filepath);
208}217}
209218
210function deleteRecursive(id: string, module: any) {219export function unload(filepath: string) {
220 filepath = path.resolve(filepath);
221 const existing = cache[filepath];
222 if (existing) delete cache[filepath];
223 fileStats.delete(filepath);
224}
225
226function deleteRecursiveInner(id: string, module: any) {
211 if (id.includes(path.sep + "node_modules" + path.sep)) {227 if (id.includes(path.sep + "node_modules" + path.sep)) {
212 return;228 return;
213 }229 }
...@@ -215,15 +231,14 @@ function deleteRecursive(id: string, module: any) {...@@ -215,15 +231,14 @@ function deleteRecursive(id: string, module: any) {
215 for (const child of module.children) {231 for (const child of module.children) {
216 if (child.filename.includes("/engine/")) return;232 if (child.filename.includes("/engine/")) return;
217 const existing = cache[child.filename];233 const existing = cache[child.filename];
218 if (existing === child) deleteRecursive(child.filename, existing);234 if (existing === child) deleteRecursiveInner(child.filename, existing);
219 }235 }
220}236}
221237
222export function getCssImports(filepath: string) {238export function getCssImports(filepath: string) {
223 filepath = path.resolve(filepath);239 filepath = path.resolve(filepath);
224 if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");240 if (!require.cache[filepath]) throw new Error(filepath + " was never loaded");
225 return fsGraph.get(path.relative(projectRoot, filepath))241 return fileStats.get(filepath)?.cssImportsRecursive ?? [];
226 ?.cssImportsRecursive ?? [];
227}242}
228243
229export function getClientScriptRefs(filepath: string) {244export function getClientScriptRefs(filepath: string) {
...@@ -296,7 +311,6 @@ export function resolveClientRefs(...@@ -296,7 +311,6 @@ export function resolveClientRefs(
296311
297export function getScriptId(file: string) {312export function getScriptId(file: string) {
298 return (path.isAbsolute(file) ? path.relative(projectSrc, file) : file)313 return (path.isAbsolute(file) ? path.relative(projectSrc, file) : file)
299 .replace(/^\/?src\//, "")
300 .replaceAll("\\", "/");314 .replaceAll("\\", "/");
301}315}
302316
framework/incremental.ts+236-38
...@@ -20,8 +20,19 @@ interface ArtifactMap {...@@ -20,8 +20,19 @@ interface ArtifactMap {
20 pageMetadata: PageMetadata;20 pageMetadata: PageMetadata;
21 /* Metadata about a dynamic view */21 /* Metadata about a dynamic view */
22 viewMetadata: ViewMetadata;22 viewMetadata: ViewMetadata;
23 /* Cached '.marko' server compilation */
24 serverMarko: hot.MarkoCacheEntry;
25 /* Backend source code, pre-replacement. Keyed by platform type. */
26 backendBundle: BackendBundle;
27 /* One file in the backend receives post-processing. */
28 backendReplace: Buffer;
23}29}
24type ArtifactKind = keyof ArtifactMap;30type ArtifactKind = keyof ArtifactMap;
31/* Automatic path tracing is performed to make it so that
32 * specifying 'sources: [file]' refers to it and everything it imports.
33 * These kinds do not have that behavior
34 */
35const exactDependencyKinds = ["serverMarko"];
25export interface Asset {36export interface Asset {
26 buffer: Buffer;37 buffer: Buffer;
27 headers: Record<string, string | undefined>;38 headers: Record<string, string | undefined>;
...@@ -50,6 +61,11 @@ export interface ViewMetadata {...@@ -50,6 +61,11 @@ export interface ViewMetadata {
50 clientRefs: string[];61 clientRefs: string[];
51 hasLayout: boolean;62 hasLayout: boolean;
52}63}
64export interface BackendBundle {
65 magicWord: string;
66 fileWithMagicWord: string | null;
67 files: Record<string, Buffer>;
68}
5369
54// -- incremental support types --70// -- incremental support types --
55export interface PutBase {71export interface PutBase {
...@@ -76,6 +92,9 @@ export class Incremental {...@@ -76,6 +92,9 @@ export class Incremental {
76 style: new Map(),92 style: new Map(),
77 pageMetadata: new Map(),93 pageMetadata: new Map(),
78 viewMetadata: new Map(),94 viewMetadata: new Map(),
95 serverMarko: new Map(),
96 backendBundle: new Map(),
97 backendReplace: new Map(),
79 };98 };
80 /** Tracking filesystem entries to `srcId` */99 /** Tracking filesystem entries to `srcId` */
81 invals = new Map<SourceId, Invalidations>();100 invals = new Map<SourceId, Invalidations>();
...@@ -92,16 +111,52 @@ export class Incremental {...@@ -92,16 +111,52 @@ export class Incremental {
92 getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,111 getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
93 });112 });
94113
114 /** Reset at the end of each update */
115 round = {
116 inserted: new Set<ArtifactId>(),
117 referenced: new Set<ArtifactId>(),
118 };
119
95 getArtifact<T extends ArtifactKind>(kind: T, key: string) {120 getArtifact<T extends ArtifactKind>(kind: T, key: string) {
121 this.round.referenced.add(`${kind}\0${key}`);
96 return this.out[kind].get(key);122 return this.out[kind].get(key);
97 }123 }
98124
99 hasArtifact<T extends ArtifactKind>(kind: T, key: string) {125 hasArtifact(kind: ArtifactKind, key: string) {
100 return this.out[kind].has(key);126 return this.getArtifact(kind, key) != null;
101 }127 }
102128
103 sourcesFor(kind: ArtifactKind, key: string) {129 sourcesFor(kind: ArtifactKind, key: string) {
104 return UNWRAP(this.sources.get(kind + "\0" + key));130 return UNWRAP(
131 this.sources.get(kind + "\0" + key),
132 `No artifact '${kind}' '${key}'`,
133 );
134 }
135
136 shake() {
137 const toPublic = (str: string) => {
138 const [kind, key] = str.split("\0");
139 return { kind: kind as ArtifactKind, key };
140 };
141 const inserted = Array.from(this.round.inserted, toPublic);
142 const referenced = Array.from(this.round.referenced, toPublic);
143 const unreferenced: { kind: ArtifactKind; key: string }[] = [];
144
145 for (const kind in this.out) {
146 const map = this.out[kind as keyof typeof this.out];
147 if (!map) continue;
148 for (const key of map.keys()) {
149 if (!this.round.referenced.has(`${kind}\0${key}`)) {
150 unreferenced.push({ kind: kind as ArtifactKind, key });
151 console.warn("unreferened " + kind + " : " + key);
152 }
153 }
154 }
155
156 this.round.inserted.clear();
157 this.round.referenced.clear();
158
159 return { inserted, referenced, unreferenced };
105 }160 }
106161
107 /*162 /*
...@@ -115,9 +170,25 @@ export class Incremental {...@@ -115,9 +170,25 @@ export class Incremental {
115 key,170 key,
116 value,171 value,
117 }: Put<T>) {172 }: Put<T>) {
118 console.log("put " + kind + ": " + key);173 // These three invariants affect incremental accuracy.
174 if (this.round.inserted.has(`${kind}\0${key}`)) {
175 console.error(
176 `Artifact ${kind}:${key} was inserted multiple times in the same round!`,
177 );
178 } else if (!this.round.referenced.has(`${kind}\0${key}`)) {
179 console.error(
180 `Artifact ${kind}:${key} was inserted without checking if (!hasArtifact())`,
181 );
182 } else if (this.out[kind].has(key)) {
183 console.error(
184 `Artifact ${kind}:${key} is not stale, but overwritten.`,
185 );
186 }
187
119 this.out[kind].set(key, value);188 this.out[kind].set(key, value);
120189
190 this.round.inserted.add(`${kind}\0${key}`);
191
121 // Update sources information192 // Update sources information
122 ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);193 ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
123 sources = sources.map((src) => path.normalize(src));194 sources = sources.map((src) => path.normalize(src));
...@@ -150,8 +221,7 @@ export class Incremental {...@@ -150,8 +221,7 @@ export class Incremental {
150 #getOrInitInvals(source: string) {221 #getOrInitInvals(source: string) {
151 let invals = this.invals.get(source);222 let invals = this.invals.get(source);
152 if (!invals) {223 if (!invals) {
153 const g = hot.getFsGraph().get(source);224 const lastModified = hot.getFileStat(source)?.lastModified ??
154 const lastModified = g?.lastModified ??
155 fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;225 fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;
156 this.invals.set(226 this.invals.set(
157 source,227 source,
...@@ -166,8 +236,7 @@ export class Incremental {...@@ -166,8 +236,7 @@ export class Incremental {
166 }236 }
167237
168 #followImports(file: string) {238 #followImports(file: string) {
169 const graph = hot.getFsGraph();239 const stat = hot.getFileStat(file);
170 const stat = graph.get(file);
171 if (!stat) return;240 if (!stat) return;
172 for (const i of stat.imports) {241 for (const i of stat.imports) {
173 const invals = this.#getOrInitInvals(i);242 const invals = this.#getOrInitInvals(i);
...@@ -178,19 +247,23 @@ export class Incremental {...@@ -178,19 +247,23 @@ export class Incremental {
178247
179 async statAllFiles() {248 async statAllFiles() {
180 for (const file of this.invals.keys()) {249 for (const file of this.invals.keys()) {
181 const mtime = fs.statSync(file).mtimeMs;250 try {
182 this.updateStat(file, mtime);251 const mtime = fs.statSync(file).mtimeMs;
252 this.updateStat(file, mtime);
253 } catch (err) {
254 }
183 }255 }
184 }256 }
185257
186 updateStat(file: string, newLastModified: number) {258 updateStat(file: string, newLastModified: number | null) {
187 file = path.relative(hot.projectRoot, file);259 file = path.relative(hot.projectRoot, file);
188 const stat = this.invals.get(file);260 const stat = this.invals.get(file);
189 ASSERT(stat, "Updated stat on untracked file " + file);261 ASSERT(stat, "Updated stat on untracked file " + file);
190 const hasUpdate = stat.lastModified < newLastModified;262 const hasUpdate = !newLastModified || stat.lastModified < newLastModified;
191 if (hasUpdate) {263 if (hasUpdate) {
192 // Invalidate264 // Invalidate
193 console.info(file + " updated");265 console.info(file + " " + (newLastModified ? "updated" : "deleted"));
266 hot.unload(file);
194 const invalidQueue = [file];267 const invalidQueue = [file];
195 let currentInvalid;268 let currentInvalid;
196 while (currentInvalid = invalidQueue.pop()) {269 while (currentInvalid = invalidQueue.pop()) {
...@@ -204,12 +277,15 @@ export class Incremental {...@@ -204,12 +277,15 @@ export class Incremental {
204 for (const out of outputs) {277 for (const out of outputs) {
205 const [kind, artifactKey] = out.split("\0");278 const [kind, artifactKey] = out.split("\0");
206 this.out[kind as ArtifactKind].delete(artifactKey);279 this.out[kind as ArtifactKind].delete(artifactKey);
207 console.log("stale " + kind + ": " + artifactKey);
208 }280 }
209 invalidQueue.push(...files);281 invalidQueue.push(...files);
210 }282 }
211 }283 }
212 stat.lastModified = newLastModified;284 if (newLastModified) {
285 stat.lastModified = newLastModified;
286 } else {
287 this.invals.delete(file);
288 }
213 return hasUpdate;289 return hasUpdate;
214 }290 }
215291
...@@ -266,6 +342,7 @@ export class Incremental {...@@ -266,6 +342,7 @@ export class Incremental {
266 serialize() {342 serialize() {
267 const writer = new BufferWriter();343 const writer = new BufferWriter();
268344
345 // -- artifact --
269 const asset = Array.from(346 const asset = Array.from(
270 this.out.asset,347 this.out.asset,
271 ([key, { buffer, hash, headers }]) => {348 ([key, { buffer, hash, headers }]) => {
...@@ -283,6 +360,31 @@ export class Incremental {...@@ -283,6 +360,31 @@ export class Incremental {
283 },360 },
284 );361 );
285 const script = Array.from(this.out.script);362 const script = Array.from(this.out.script);
363 const style = Array.from(this.out.style);
364 const pageMetadata = Array.from(this.out.pageMetadata);
365 const viewMetadata = Array.from(this.out.viewMetadata);
366 const serverMarko = Array.from(this.out.serverMarko);
367 const backendBundle = Array.from(this.out.backendBundle, ([k, v]) => {
368 return [k, {
369 magicWord: v.magicWord,
370 fileWithMagicWord: v.fileWithMagicWord,
371 files: Object.entries(v.files).map(
372 ([file, contents]) => [
373 file,
374 writer.write(contents, "backendBundle" + k + ":" + file),
375 ],
376 ),
377 }] satisfies SerializedMeta["backendBundle"][0];
378 });
379 const backendReplace = Array.from(
380 this.out.backendReplace,
381 ([k, v]) =>
382 [
383 k,
384 writer.write(v, "backendReplace" + k),
385 ] satisfies SerializedMeta["backendReplace"][0],
386 );
387 // -- incremental metadata --
286 const invals = Array.from(this.invals, ([key, value]) => {388 const invals = Array.from(this.invals, ([key, value]) => {
287 const { lastModified, files, outputs } = value;389 const { lastModified, files, outputs } = value;
288 return [key, {390 return [key, {
...@@ -299,6 +401,12 @@ export class Incremental {...@@ -299,6 +401,12 @@ export class Incremental {
299 script,401 script,
300 invals,402 invals,
301 sources,403 sources,
404 style,
405 pageMetadata,
406 viewMetadata,
407 serverMarko,
408 backendBundle,
409 backendReplace,
302 } satisfies SerializedMeta;410 } satisfies SerializedMeta;
303 const meta = Buffer.from(JSON.stringify(json), "utf-8");411 const meta = Buffer.from(JSON.stringify(json), "utf-8");
304412
...@@ -317,21 +425,39 @@ export class Incremental {...@@ -317,21 +425,39 @@ export class Incremental {
317 buffer.subarray(4 + metaLength + start, 4 + metaLength + end);425 buffer.subarray(4 + metaLength + start, 4 + metaLength + end);
318426
319 const incr = new Incremental();427 const incr = new Incremental();
320 incr.out.asset = new Map<string, Asset>(meta.asset.map(([key, value]) => {428 incr.out = {
321 const { hash, raw, gzip, zstd, headers } = value;429 asset: new Map(meta.asset.map(([key, value]) => {
322 if ((gzip || zstd) && !incr.compress.has(hash)) {430 const { hash, raw, gzip, zstd, headers } = value;
323 incr.compress.set(hash, {431 if ((gzip || zstd) && !incr.compress.has(hash)) {
324 gzip: gzip ? view(gzip) : undefined,432 incr.compress.set(hash, {
325 zstd: zstd ? view(zstd) : undefined,433 gzip: gzip ? view(gzip) : undefined,
326 });434 zstd: zstd ? view(zstd) : undefined,
327 }435 });
328 return [key, {436 }
329 buffer: view(raw),437 return [key, {
330 headers: headers,438 buffer: view(raw),
331 hash: hash,439 headers: headers,
332 }];440 hash: hash,
333 }));441 }];
334 incr.out.script = new Map(meta.script);442 })),
443 script: new Map(meta.script),
444 style: new Map(meta.style),
445 pageMetadata: new Map(meta.pageMetadata),
446 viewMetadata: new Map(meta.viewMetadata),
447 serverMarko: new Map(meta.serverMarko),
448 backendBundle: new Map(meta.backendBundle.map(([key, value]) => {
449 return [key, {
450 magicWord: value.magicWord,
451 fileWithMagicWord: value.fileWithMagicWord,
452 files: Object.fromEntries(
453 value.files.map(([file, contents]) => [file, view(contents)]),
454 ),
455 }];
456 })),
457 backendReplace: new Map(
458 meta.backendReplace.map(([key, contents]) => [key, view(contents)]),
459 ),
460 };
335 incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {461 incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {
336 return [key, {462 return [key, {
337 lastModified: m,463 lastModified: m,
...@@ -343,6 +469,37 @@ export class Incremental {...@@ -343,6 +469,37 @@ export class Incremental {
343 return incr;469 return incr;
344 }470 }
345471
472 /*
473 * Move the cached (server) marko transpilations from this incremental
474 * into the running process.
475 */
476 loadMarkoCache() {
477 hot.markoCache.clear();
478 for (const [key, value] of this.out.serverMarko) {
479 hot.markoCache.set(path.resolve(hot.projectRoot, key), value);
480 }
481 }
482
483 /*
484 * Move the cached (server) marko transpilations from this incremental
485 * into the running process.
486 */
487 snapshotMarkoCache() {
488 for (const [file, value] of hot.markoCache) {
489 const key = path.relative(hot.projectRoot, file).replaceAll("\\", "/");
490 // Only insert if it doesn't exist. Calling 'put' when it
491 // already exists would inform the user of extra calls to put.
492 if (!this.hasArtifact("serverMarko", key)) {
493 this.put({
494 kind: "serverMarko",
495 sources: [file],
496 key,
497 value,
498 });
499 }
500 }
501 }
502
346 toDisk(file = ".clover/incr.state") {503 toDisk(file = ".clover/incr.state") {
347 const buffer = this.serialize();504 const buffer = this.serialize();
348 fs.writeFileSync(file, buffer);505 fs.writeFileSync(file, buffer);
...@@ -362,23 +519,52 @@ export class Incremental {...@@ -362,23 +519,52 @@ export class Incremental {
362 await this.compressQueue.done({ method: "success" });519 await this.compressQueue.done({ method: "success" });
363 }520 }
364521
365 async flush() {522 async flush(
523 platform: bundle.ServerPlatform,
524 dir = path.resolve(".clover/out"),
525 ) {
366 ASSERT(!this.compressQueue.active);526 ASSERT(!this.compressQueue.active);
527 const join = (...args: string[]) => path.join(dir, ...args);
367 const writer = new BufferWriter();528 const writer = new BufferWriter();
368 // TODO: ensure all assets are actually compressed and not fake lying.529
530 // TODO: ensure all compressed got compressed
531
369 const asset = Object.fromEntries(532 const asset = Object.fromEntries(
370 Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {533 Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
371 const raw = writer.write(buffer, hash);534 const raw = writer.write(buffer, hash);
372 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};535 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
373 const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;536 const gzip = writer.write(UNWRAP(gzipBuf), hash + ".gz");
374 const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;537 const zstd = writer.write(UNWRAP(zstdBuf), hash + ".zstd");
375 return [key, { raw, gzip, zstd, headers }];538 return [key, { raw, gzip, zstd, headers }];
376 }),539 }),
377 );540 );
378 await Promise.all([541 const backendBundle = UNWRAP(this.out.backendBundle.get(platform));
379 fs.writeFile(".clover/static.json", JSON.stringify(asset)),542
380 fs.writeFile(".clover/static.blob", writer.get()),543 // Arrange output files
381 ]);544 const outFiles: Array<[file: string, contents: string | Buffer]> = [
545 // Asset manifest
546 ["static.json", JSON.stringify(asset)],
547 ["static.blob", writer.get()],
548
549 // Backend
550 ...Object.entries(backendBundle.files).map(([subPath, contents]) =>
551 [
552 subPath,
553 subPath === backendBundle.fileWithMagicWord
554 ? UNWRAP(this.out.backendReplace.get(platform))
555 : contents,
556 ] as [string, Buffer]
557 ),
558 ];
559
560 // TODO: check duplicates
561
562 // Perform all i/o
563 await Promise.all(
564 outFiles.map(([subPath, contents]) =>
565 fs.writeMkdir(join(subPath), contents, { flush: true })
566 ),
567 );
382 }568 }
383}569}
384570
...@@ -440,6 +626,17 @@ export interface SerializedMeta {...@@ -440,6 +626,17 @@ export interface SerializedMeta {
440 headers: Record<string, string>;626 headers: Record<string, string>;
441 }]>;627 }]>;
442 script: Array<[key: string, value: string]>;628 script: Array<[key: string, value: string]>;
629 style: Array<[key: string, value: string]>;
630 pageMetadata: Array<[key: string, PageMetadata]>;
631 viewMetadata: Array<[key: string, ViewMetadata]>;
632 serverMarko: Array<[key: string, hot.MarkoCacheEntry]>;
633 backendBundle: Array<[platform: string, {
634 magicWord: string;
635 fileWithMagicWord: string | null;
636 files: Array<[string, View]>;
637 }]>;
638 backendReplace: Array<[key: string, View]>;
639
443 invals: Array<[key: string, {640 invals: Array<[key: string, {
444 /** Modified */641 /** Modified */
445 m: number;642 m: number;
...@@ -461,3 +658,4 @@ import * as mime from "#sitegen/mime";...@@ -461,3 +658,4 @@ import * as mime from "#sitegen/mime";
461import * as path from "node:path";658import * as path from "node:path";
462import { Buffer } from "node:buffer";659import { Buffer } from "node:buffer";
463import * as css from "./css.ts";660import * as css from "./css.ts";
661import type * as bundle from "./bundle.ts";
framework/lib/assets.ts+14-4
...@@ -8,9 +8,10 @@ export type StaticPageId = string;...@@ -8,9 +8,10 @@ export type StaticPageId = string;
88
9export async function reload() {9export async function reload() {
10 const [map, buf] = await Promise.all([10 const [map, buf] = await Promise.all([
11 fs.readFile(".clover/static.json", "utf8"),11 fs.readFile(path.join(import.meta.dirname, "static.json"), "utf8"),
12 fs.readFile(".clover/static.blob"),12 fs.readFile(path.join(import.meta.dirname, "static.blob")),
13 ]);13 ]);
14 console.log("new buffer loaded");
14 assets = {15 assets = {
15 map: JSON.parse(map),16 map: JSON.parse(map),
16 buf,17 buf,
...@@ -18,8 +19,11 @@ export async function reload() {...@@ -18,8 +19,11 @@ export async function reload() {
18}19}
1920
20export async function reloadSync() {21export async function reloadSync() {
21 const map = fs.readFileSync(".clover/static.json", "utf8");22 const map = fs.readFileSync(
22 const buf = fs.readFileSync(".clover/static.blob");23 path.join(import.meta.dirname, "static.json"),
24 "utf8",
25 );
26 const buf = fs.readFileSync(path.join(import.meta.dirname, "static.blob"));
23 assets = {27 assets = {
24 map: JSON.parse(map),28 map: JSON.parse(map),
25 buf,29 buf,
...@@ -105,8 +109,14 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {...@@ -105,8 +109,14 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
105 return c.res = new Response(body, { headers, status });109 return c.res = new Response(body, { headers, status });
106}110}
107111
112process.on("message", (msg: any) => {
113 console.log({ msg });
114 if (msg?.type === "clover.assets.reload") reload();
115});
116
108import * as fs from "#sitegen/fs";117import * as fs from "#sitegen/fs";
109import type { Context, Next } from "hono";118import type { Context, Next } from "hono";
110import type { StatusCode } from "hono/utils/http-status";119import type { StatusCode } from "hono/utils/http-status";
111import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";120import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
112import { Buffer } from "node:buffer";121import { Buffer } from "node:buffer";
122import * as path from "node:path";
framework/lib/fs.ts+8-2
...@@ -22,9 +22,15 @@ export function mkdirSync(dir: string) {...@@ -22,9 +22,15 @@ export function mkdirSync(dir: string) {
22 return nodeMkdirSync(dir, { recursive: true });22 return nodeMkdirSync(dir, { recursive: true });
23}23}
2424
25export async function writeMkdir(file: string, contents: Buffer | string) {25export type WriteFileAsyncOptions = Parameters<typeof writeFile>[2];
26
27export async function writeMkdir(
28 file: string,
29 contents: Buffer | string,
30 options?: WriteFileAsyncOptions,
31) {
26 await mkdir(path.dirname(file));32 await mkdir(path.dirname(file));
27 return writeFile(file, contents);33 return writeFile(file, contents, options);
28}34}
2935
30export function writeMkdirSync(file: string, contents: Buffer | string) {36export function writeMkdirSync(file: string, contents: Buffer | string) {
framework/watch.ts+44-3
...@@ -3,12 +3,39 @@...@@ -3,12 +3,39 @@
3const debounceMilliseconds = 25; 3const debounceMilliseconds = 25;
4 4
5export async function main() { 5export async function main() {
6 let subprocess: child_process.ChildProcess | null = null;
7
6 // Catch up state by running a main build. 8 // Catch up state by running a main build.
7 const { incr } = await generate.main(); 9 const { incr } = await generate.main();
8 // ...and watch the files that cause invals. 10 // ...and watch the files that cause invals.
9 const watch = new Watch(rebuild); 11 const watch = new Watch(rebuild);
10 watch.add(...incr.invals.keys()); 12 watch.add(...incr.invals.keys());
11 statusLine(); 13 statusLine();
14 // ... an
15 serve();
16
17 function serve() {
18 if (subprocess) {
19 subprocess.removeListener("close", onSubprocessClose);
20 subprocess.kill();
21 }
22 subprocess = child_process.fork(".clover/out/server.js", [
23 "--development",
24 ], {
25 stdio: "inherit",
26 });
27 subprocess.on("close", onSubprocessClose);
28 }
29
30 function onSubprocessClose(code: number | null, signal: string | null) {
31 subprocess = null;
32 const status = code != null ? `code ${code}` : `signal ${signal}`;
33 console.error(`Backend process exited with ${status}`);
34 }
35
36 process.on("beforeExit", () => {
37 subprocess?.removeListener("close", onSubprocessClose);
38 });
12 39
13 function rebuild(files: string[]) { 40 function rebuild(files: string[]) {
14 files = files.map((file) => path.relative(hot.projectRoot, file)); 41 files = files.map((file) => path.relative(hot.projectRoot, file));
...@@ -20,7 +47,7 @@ export async function main() {...@@ -20,7 +47,7 @@ export async function main() {
20 console.warn("Files were modified but the 'modify' time did not change."); 47 console.warn("Files were modified but the 'modify' time did not change.");
21 return; 48 return;
22 } 49 }
23 withSpinner<Record<string, unknown>, any>({ 50 withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
24 text: "Rebuilding", 51 text: "Rebuilding",
25 successText: generate.successText, 52 successText: generate.successText,
26 failureText: () => "sitegen FAIL", 53 failureText: () => "sitegen FAIL",
...@@ -39,6 +66,20 @@ export async function main() {...@@ -39,6 +66,20 @@ export async function main() {
39 if (!incr.invals.has(relative)) watch.remove(file); 66 if (!incr.invals.has(relative)) watch.remove(file);
40 } 67 }
41 return result; 68 return result;
69 }).then((result) => {
70 // Restart the server if it was changed or not running.
71 if (
72 !subprocess ||
73 result.inserted.some(({ kind }) => kind === "backendReplace")
74 ) {
75 serve();
76 } else if (
77 subprocess &&
78 result.inserted.some(({ kind }) => kind === "asset")
79 ) {
80 subprocess.send({ type: "clover.assets.reload" });
81 }
82 return result;
42 }).catch((err) => { 83 }).catch((err) => {
43 console.error(util.inspect(err)); 84 console.error(util.inspect(err));
44 }).finally(statusLine); 85 }).finally(statusLine);
...@@ -142,10 +183,10 @@ class Watch {...@@ -142,10 +183,10 @@ class Watch {
142 } 183 }
143} 184}
144 185
145import { Incremental } from "./incremental.ts";
146import * as fs from "node:fs"; 186import * as fs from "node:fs";
147import { Spinner, withSpinner } from "@paperclover/console/Spinner"; 187import { withSpinner } from "@paperclover/console/Spinner";
148import * as generate from "./generate.ts"; 188import * as generate from "./generate.ts";
149import * as path from "node:path"; 189import * as path from "node:path";
150import * as util from "node:util"; 190import * as util from "node:util";
151import * as hot from "./hot.ts"; 191import * as hot from "./hot.ts";
192import * as child_process from "node:child_process";