authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-04-02 23:01:46-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-04-06 01:17:01-07:00
log223b5dc94bf1459d8a62aed61e447b8f3abafcd8
tree1474ed85958cda0d3fcd54edff9d05815fd8561f
parent22ebf23fcec9685339e4fa903ed984d374f06948
signature Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

fix: load the marko cache

helps #27

4 files changed, 138 insertions(+), 67 deletions(-)

framework/generate.ts+53-1
......@@ -40,7 +40,7 @@ export async function generate(p: progress.Ref) {
4040
4141 const builtFonts = fonts.buildFonts(p, siteConfig.fonts);
4242
43 // TODO: loadMarkoCache
43 await loadMarkoCache(p);
4444
4545 const pageNode = p.start("generate static pages", {});
4646 const viewNode = p.start("compile dynamic views", {});
......@@ -245,6 +245,57 @@ export async function scanSiteSection(io: Io, section: sg.Section) {
245245 return { staticFiles, pages, views, scripts };
246246}
247247
248export async function loadMarkoCache(p: progress.Ref) {
249 using node = p.start("load marko cache");
250
251 markoCache.clear();
252 const files = fs.readDirRecOptionalSync(".clover/marko")
253 .filter((file) => file.endsWith(".js"));
254 let restored = 0;
255 let rebuilt = 0;
256 await Promise.all(
257 files.map(async (cachedPath) => {
258 const source = resolveMarkoCacheSource(cachedPath);
259 if (!source) return;
260
261 const cachePath = path.join(".clover/marko", cachedPath);
262 const [sourceStat, cacheStat] = await Promise.all([
263 fs.stat(source).catch(() => null),
264 fs.stat(cachePath).catch(() => null),
265 ]);
266 if (!sourceStat) return;
267
268 const absSource = path.resolve(source);
269 if (!cacheStat || cacheStat.mtimeMs < sourceStat.mtimeMs) {
270 rebuilt += 1;
271 markoCache.set(
272 absSource,
273 buildMarkoCacheEntry(absSource, source.endsWith(".mdo") ? "markdown" : "html"),
274 );
275 return;
276 }
277
278 restored += 1;
279 const sourceText = await fs.readFile(source, "utf-8");
280 markoCache.set(absSource, {
281 src: await fs.readFile(cachePath, "utf-8"),
282 scannedClientRefs: scanMarkoClientImports(sourceText, absSource)
283 .scannedClientRefs,
284 });
285 }),
286 );
287
288 node.text = `load marko cache (${restored} restored, ${rebuilt} rebuilt)`;
289}
290
291function resolveMarkoCacheSource(cachedPath: string) {
292 const source = cachedPath.slice(0, -".js".length);
293 if (fs.existsSync(source)) return source;
294 const srcSource = path.join("src", source);
295 if (fs.existsSync(srcSource)) return srcSource;
296 return null;
297}
298
248299export async function preparePage(
249300 io: Io,
250301 { item, metaTemplate }: { item: sg.FileItem; metaTemplate: meta.Template },
......@@ -428,3 +479,4 @@ import * as fonts from "./font.ts";
428479import * as hot from "./hot.ts";
429480import * as incr from "./incremental.ts";
430481import { Io } from "./incremental.ts";
482import { buildMarkoCacheEntry, markoCache, scanMarkoClientImports } from "./marko.ts";
framework/hot.ts+2-1
......@@ -223,7 +223,8 @@ export function resolveFrom(src: string, dest: string) {
223223 return createRequire(src).resolve(dest);
224224 } catch (err: any) {
225225 if (
226 err.code === "MODULE_NOT_FOUND" && err.requireStack
226 err.code === "MODULE_NOT_FOUND"
227 && err.requireStack
227228 && err.requireStack.length <= 1
228229 ) {
229230 err.message = err.message.split("\n")[0] + " from '" + src + "'";
framework/marko.ts+79-64
......@@ -18,77 +18,92 @@ function loadMarkoExtra(
1818 mode: "html" | "markdown",
1919) {
2020 let cache = markoCache.get(filepath);
21 if (!cache) {
22 let src = fs.readFileSync(filepath, "utf8");
23 if (mode === "markdown") {
24 let transform;
25 try {
26 transform = markodown.transform({
27 source: src,
28 selfImport: `./${path.basename(filepath)}`,
29 });
30 } catch (err) {
31 throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
32 cause: err,
33 });
34 }
35 if (!transform.success) {
36 // TODO: better error handling
37 throw new Error(
38 "Failed " + filepath + ": " + JSON.stringify(transform.errors),
39 );
40 }
41 src = transform.text;
21 if (!cache) markoCache.set(filepath, cache = buildMarkoCacheEntry(filepath, mode));
4222
43 fs.writeMkdirSync(
44 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
45 + ".marko",
46 src,
23 const { src, scannedClientRefs } = cache;
24 return hot.loadEsbuildCode(module, filepath, src, {
25 scannedClientRefs,
26 });
27}
28
29export function buildMarkoCacheEntry(
30 filepath: string,
31 mode: "html" | "markdown" = "html",
32): MarkoCacheEntry {
33 let src = fs.readFileSync(filepath, "utf8");
34 if (mode === "markdown") {
35 let transform;
36 try {
37 transform = markodown.transform({
38 source: src,
39 selfImport: `./${path.basename(filepath)}`,
40 });
41 } catch (err) {
42 throw new Error(`Failed ${filepath}: Markodown Compiler Panic`, {
43 cause: err,
44 });
45 }
46 if (!transform.success) {
47 // TODO: better error handling
48 throw new Error(
49 "Failed " + filepath + ": " + JSON.stringify(transform.errors),
4750 );
4851 }
52 src = transform.text;
53 fs.writeMkdirSync(getMarkoCachePath(filepath, ".marko"), src);
54 }
4955
50 // A non-standard thing here is Clover Sitegen implements
51 // its own client side scripting stuff, so it overrides
52 // bare client import statements to it's own usage.
53 const scannedClientRefs = new Set<string>();
54 if (src.match(/^\s*client\s+import\s+["']/m)) {
55 src = src.replace(
56 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
57 (_, src) => {
58 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
59 const resolved = hot.resolveClientRef(filepath, ref);
60 scannedClientRefs.add(resolved);
61 return `<CloverScriptInclude=${
62 JSON.stringify(
63 hot.getScriptId(resolved),
64 )
65 } />`;
66 },
67 ) + "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
68 }
56 const resolved = resolveClientImports(src, filepath);
57 let compiled = marko.compileSync(resolved.src, filepath, {
58 sourceMaps: true,
59 });
60 // workaround an esbuild bug?
61 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
62 compiled.map.sources[0] = filepath;
63 const compiledSrc = compiled.code
64 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
65 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");
66 fs.writeMkdirSync(getMarkoCachePath(filepath, ".js"), compiledSrc);
67 return {
68 src: compiledSrc,
69 scannedClientRefs: resolved.scannedClientRefs,
70 };
71}
72
73export function getMarkoCachePath(
74 filepath: string,
75 ext: ".js" | ".marko",
76) {
77 return ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
78 + ext;
79}
6980
70 let compiled = marko.compileSync(src, filepath, {
71 sourceMaps: true,
72 });
73 // workaround an esbuild bug?
74 (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/";
75 compiled.map.sources[0] = filepath;
76 src = compiled.code
77 + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,"
78 + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64");
79 cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
80 markoCache.set(filepath, cache);
81 fs.writeMkdirSync(
82 ".clover/marko/" + path.relative(".", filepath).replace(/^src\//, "")
83 + ".js",
84 src,
85 );
81export function scanMarkoClientImports(src: string, filepath: string) {
82 // Clover Sitegen uses `client import` as a convenience shorthand
83 // and rewrites it to the runtime script include component.
84 const scannedClientRefs = new Set<string>();
85 let found = false;
86 src = src.replace(
87 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]*$/gm,
88 (_, rawRef) => {
89 found = true;
90 const ref = JSON.parse(`"${rawRef.slice(1, -1)}"`);
91 const resolved = hot.resolveClientRef(filepath, ref);
92 scannedClientRefs.add(resolved);
93 return `<CloverScriptInclude=${JSON.stringify(hot.getScriptId(resolved))} />`;
94 },
95 );
96 if (found) {
97 src += "\nimport { CloverScriptInclude } from \"#sitegen\";\n";
8698 }
99 return {
100 src,
101 scannedClientRefs: Array.from(scannedClientRefs),
102 };
103}
87104
88 const { src, scannedClientRefs } = cache;
89 return hot.loadEsbuildCode(module, filepath, src, {
90 scannedClientRefs,
91 });
105function resolveClientImports(src: string, filepath: string) {
106 return scanMarkoClientImports(src, filepath);
92107}
93108
94109import * as fs from "#sitegen/fs";
framework/watch.ts+4-1
......@@ -45,7 +45,10 @@ async function rebuild(files: string[]) {
4545 if (isRunning) return;
4646 isRunning = true;
4747 try {
48 child_process.execSync("./node_modules/.bin/dprint fmt " + files.join(" "), { stdio: "inherit" });
48 const formattingFiles = files.filter(x => !x.endsWith(".mdo"));
49 if (formattingFiles.length > 0) {
50 child_process.execSync("./node_modules/.bin/dprint fmt " + formattingFiles.join(" "), { stdio: "inherit" });
51 }
4952 } catch (err) {
5053 isRunning = false;
5154 throw err;