authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-04-02 23:01:46-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-04-06 01:17:01-07:00
log223b5dc94bf1459d8a62aed61e447b8f3abafcd8
tree1474ed85958cda0d3fcd54edff9d05815fd8561f
parent22ebf23fcec9685339e4fa903ed984d374f06948
signature Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

fix: load the marko cache

helps #27

4 files changed, 138 insertions(+), 67 deletions(-)

framework/generate.ts+53-1
...@@ -40,7 +40,7 @@ export async function generate(p: progress.Ref) {...@@ -40,7 +40,7 @@ export async function generate(p: progress.Ref) {
4040
41 const builtFonts = fonts.buildFonts(p, siteConfig.fonts);41 const builtFonts = fonts.buildFonts(p, siteConfig.fonts);
4242
43 // TODO: loadMarkoCache43 await loadMarkoCache(p);
4444
45 const pageNode = p.start("generate static pages", {});45 const pageNode = p.start("generate static pages", {});
46 const viewNode = p.start("compile dynamic views", {});46 const viewNode = p.start("compile dynamic views", {});
...@@ -245,6 +245,57 @@ export async function scanSiteSection(io: Io, section: sg.Section) {...@@ -245,6 +245,57 @@ export async function scanSiteSection(io: Io, section: sg.Section) {
245 return { staticFiles, pages, views, scripts };245 return { staticFiles, pages, views, scripts };
246}246}
247247
248export async function loadMarkoCache(p: progress.Ref) {
249 using node = p.start("load marko cache");
250
251 markoCache.clear();
252 const files = fs.readDirRecOptionalSync(".clover/marko")
253 .filter((file) => file.endsWith(".js"));
254 let restored = 0;
255 let rebuilt = 0;
256 await Promise.all(
257 files.map(async (cachedPath) => {
258 const source = resolveMarkoCacheSource(cachedPath);
259 if (!source) return;
260
261 const cachePath = path.join(".clover/marko", cachedPath);
262 const [sourceStat, cacheStat] = await Promise.all([
263 fs.stat(source).catch(() => null),
264 fs.stat(cachePath).catch(() => null),
265 ]);
266 if (!sourceStat) return;
267
268 const absSource = path.resolve(source);
269 if (!cacheStat || cacheStat.mtimeMs < sourceStat.mtimeMs) {
270 rebuilt += 1;
271 markoCache.set(
272 absSource,
273 buildMarkoCacheEntry(absSource, source.endsWith(".mdo") ? "markdown" : "html"),
274 );
275 return;
276 }
277
278 restored += 1;
279 const sourceText = await fs.readFile(source, "utf-8");
280 markoCache.set(absSource, {
281 src: await fs.readFile(cachePath, "utf-8"),
282 scannedClientRefs: scanMarkoClientImports(sourceText, absSource)
283 .scannedClientRefs,
284 });
285 }),
286 );
287
288 node.text = `load marko cache (${restored} restored, ${rebuilt} rebuilt)`;
289}
290
291function resolveMarkoCacheSource(cachedPath: string) {
292 const source = cachedPath.slice(0, -".js".length);
293 if (fs.existsSync(source)) return source;
294 const srcSource = path.join("src", source);
295 if (fs.existsSync(srcSource)) return srcSource;
296 return null;
297}
298
248export async function preparePage(299export async function preparePage(
249 io: Io,300 io: Io,
250 { item, metaTemplate }: { item: sg.FileItem; metaTemplate: meta.Template },301 { item, metaTemplate }: { item: sg.FileItem; metaTemplate: meta.Template },
...@@ -428,3 +479,4 @@ import * as fonts from "./font.ts";...@@ -428,3 +479,4 @@ import * as fonts from "./font.ts";
428import * as hot from "./hot.ts";479import * as hot from "./hot.ts";
429import * as incr from "./incremental.ts";480import * as incr from "./incremental.ts";
430import { Io } from "./incremental.ts";481import { Io } from "./incremental.ts";
482import { buildMarkoCacheEntry, markoCache, scanMarkoClientImports } from "./marko.ts";
framework/hot.ts+2-1
...@@ -223,7 +223,8 @@ export function resolveFrom(src: string, dest: string) {...@@ -223,7 +223,8 @@ export function resolveFrom(src: string, dest: string) {
223 return createRequire(src).resolve(dest);223 return createRequire(src).resolve(dest);
224 } catch (err: any) {224 } catch (err: any) {
225 if (225 if (
226 err.code === "MODULE_NOT_FOUND" && err.requireStack226 err.code === "MODULE_NOT_FOUND"
227 && err.requireStack
227 && err.requireStack.length <= 1228 && err.requireStack.length <= 1
228 ) {229 ) {
229 err.message = err.message.split("\n")[0] + " from '" + src + "'";230 err.message = err.message.split("\n")[0] + " from '" + src + "'";
framework/marko.ts+79-64
...@@ -18,77 +18,92 @@ function loadMarkoExtra(...@@ -18,77 +18,92 @@ function loadMarkoExtra(
18 mode: "html" | "markdown",18 mode: "html" | "markdown",
19) {19) {
20 let cache = markoCache.get(filepath);20 let cache = markoCache.get(filepath);
21 if (!cache) {21 if (!cache) markoCache.set(filepath, cache = buildMarkoCacheEntry(filepath, mode));
22 let src = fs.readFileSync(filepath, "utf8");
23 if (mode === "markdown") {
24 let transform;
25 try {
26 transform = markodown.transform({
27 source: src,
28 selfImport: `./${path.basename(filepath)}`,
29 });
30 } catch (err) {
31 throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
32 cause: err,
33 });
34 }
35 if (!transform.success) {
36 // TODO: better error handling
37 throw new Error(
38 "Failed " + filepath + ": " + JSON.stringify(transform.errors),
39 );
40 }
41 src = transform.text;
4222
43 fs.writeMkdirSync(23 const { src, scannedClientRefs } = cache;
44 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")24 return hot.loadEsbuildCode(module, filepath, src, {
45 + ".marko",25 scannedClientRefs,
46 src,26 });
27}
28
29export function buildMarkoCacheEntry(
30 filepath: string,
31 mode: "html" | "markdown" = "html",
32): MarkoCacheEntry {
33 let src = fs.readFileSync(filepath, "utf8");
34 if (mode === "markdown") {
35 let transform;
36 try {
37 transform = markodown.transform({
38 source: src,
39 selfImport: `./${path.basename(filepath)}`,
40 });
41 } catch (err) {
42 throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
43 cause: err,
44 });
45 }
46 if (!transform.success) {
47 // TODO: better error handling
48 throw new Error(
49 "Failed " + filepath + ": " + JSON.stringify(transform.errors),
47 );50 );
48 }51 }
52 src = transform.text;
53 fs.writeMkdirSync(getMarkoCachePath(filepath, ".marko"), src);
54 }
4955
50 // A non-standard thing here is Clover Sitegen implements56 const resolved = resolveClientImports(src, filepath);
51 // its own client side scripting stuff, so it overrides57 let compiled = marko.compileSync(resolved.src, filepath, {
52 // bare client import statements to it's own usage.58 sourceMaps: true,
53 const scannedClientRefs = new Set<string>();59 });
54 if (src.match(/^\s*client\s+import\s+["']/m)) {60 // workaround an esbuild bug?
55 src = src.replace(61 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
56 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,62 compiled.map.sources[0] = filepath;
57 (_, src) => {63 const compiledSrc = compiled.code
58 const ref = JSON.parse(`"${src.slice(1, -1)}"`);64 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
59 const resolved = hot.resolveClientRef(filepath, ref);65 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");
60 scannedClientRefs.add(resolved);66 fs.writeMkdirSync(getMarkoCachePath(filepath, ".js"), compiledSrc);
61 return `<CloverScriptInclude=${67 return {
62 JSON.stringify(68 src: compiledSrc,
63 hot.getScriptId(resolved),69 scannedClientRefs: resolved.scannedClientRefs,
64 )70 };
65 } />`;71}
66 },72
67 ) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";73export function getMarkoCachePath(
68 }74 filepath: string,
75 ext: ".js" | ".marko",
76) {
77 return ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
78 + ext;
79}
6980
70 let compiled = marko.compileSync(src, filepath, {81export function scanMarkoClientImports(src: string, filepath: string) {
71 sourceMaps: true,82 // Clover Sitegen uses `client import` as a convenience shorthand
72 });83 // and rewrites it to the runtime script include component.
73 // workaround an esbuild bug?84 const scannedClientRefs = new Set<string>();
74 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";85 let found = false;
75 compiled.map.sources[0] = filepath;86 src = src.replace(
76 src = compiled.code87 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]*$/gm,
77 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"88 (_, rawRef) => {
78 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");89 found = true;
79 cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };90 const ref = JSON.parse(`"${rawRef.slice(1, -1)}"`);
80 markoCache.set(filepath, cache);91 const resolved = hot.resolveClientRef(filepath, ref);
81 fs.writeMkdirSync(92 scannedClientRefs.add(resolved);
82 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")93 return `<CloverScriptInclude=${JSON.stringify(hot.getScriptId(resolved))} />`;
83 + ".js",94 },
84 src,95 );
85 );96 if (found) {
97 src += "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
86 }98 }
99 return {
100 src,
101 scannedClientRefs: Array.from(scannedClientRefs),
102 };
103}
87104
88 const { src, scannedClientRefs } = cache;105function resolveClientImports(src: string, filepath: string) {
89 return hot.loadEsbuildCode(module, filepath, src, {106 return scanMarkoClientImports(src, filepath);
90 scannedClientRefs,
91 });
92}107}
93108
94import * as fs from "#sitegen/fs";109import * as fs from "#sitegen/fs";
framework/watch.ts+4-1
...@@ -45,7 +45,10 @@ async function rebuild(files: string[]) {...@@ -45,7 +45,10 @@ async function rebuild(files: string[]) {
45 if (isRunning) return;45 if (isRunning) return;
46 isRunning = true;46 isRunning = true;
47 try {47 try {
48 child_process.execSync("./node_modules/.bin/dprint fmt " + files.join(" "), { stdio: "inherit" });48 const formattingFiles = files.filter(x => !x.endsWith(".mdo"));
49 if (formattingFiles.length > 0) {
50 child_process.execSync("./node_modules/.bin/dprint fmt " + formattingFiles.join(" "), { stdio: "inherit" });
51 }
49 } catch (err) {52 } catch (err) {
50 isRunning = false;53 isRunning = false;
51 throw err;54 throw err;