1// This file contains the main site generator build process.
2// By using `incr.work`'s ability to cache work between runs,
3// the site generator is very fast to re-run.
4//
5// See `watch.ts` for a live development environment.
6const { toRel, toAbs } = incr;
7const globalCssPath = toAbs("src/global.css");
8
9export async function main() {
10 if (!process.argv.includes("-f")) await incr.restore();
11 await incr.compile(generate);
12}
13
14export async function generate(p: progress.Ref) {
15 // -- read config and discover files --
16 const siteConfig = await incr.work(p, readManifest);
17 const metaTemplate: meta.Template = {
18 ...siteConfig.meta,
19 base: new URL(siteConfig.meta.base),
20 };
21 // -- start font work --
22 const {
23 staticFiles,
24 scripts,
25 views,
26 pages,
27 } = await discoverAllFiles(p, siteConfig);
28
29 const aliasingInputs = Array.from(
30 Map.groupBy([...staticFiles, ...pages], (x) => x.id),
31 ).filter((x) => x[1].length > 1);
32 if (
33 aliasingInputs.length > 0
34 ) {
35 throw new Error(
36 `source files alias:\n`
37 + aliasingInputs.map(([id, files]) => ` - ${id}\n` + files.map((x) => " - " + x.file).join("\n")).join("\n"),
38 );
39 }
40
41 const builtFonts = fonts.buildFonts(p, siteConfig.fonts, siteConfig.meta.base);
42
43 await loadMarkoCache(p);
44
45 const pageNode = p.start("generate static pages", {});
46 const viewNode = p.start("compile dynamic views", {});
47 const staticNode = p.start("compress static asset", {});
48
49 // -- perform build-time rendering --
50 const builtPages = pages.map((item) =>
51 incr.work(
52 pageNode.start(item.file, { passive: true }),
53 preparePage,
54 { item, metaTemplate },
55 )
56 );
57 const builtViews = views.map((item) =>
58 incr.work(
59 viewNode.start(item.file, { passive: true }),
60 prepareView,
61 item,
62 )
63 );
64 Promise.allSettled(builtViews).then(() => viewNode.end());
65 const builtStaticFiles = Promise.all(
66 staticFiles.map((item) =>
67 incr.work(staticNode, async (io, { id, file }) => {
68 using _ = io.start(file);
69 return void (await io.writeAsset({
70 pathname: id,
71 buffer: await io.readFile(file),
72 }));
73 }, item)
74 ),
75 ).then(() => staticNode.end());
76 const routes = await Promise.all([...builtViews, ...builtPages]);
77
78 const viewsAndDynPages: incr.Ref<PageOrView>[] = [
79 ...builtViews,
80 ...builtPages.filter((page) => UNWRAP(page.value).regenerate),
81 ];
82
83 // -- page resources --
84 const scriptMap = incr.work(p, bundle.bundleClientJavaScript, {
85 clientRefs: routes.flatMap((x) => x.clientRefs),
86 extraPublicScripts: scripts.map((entry) => entry.file),
87 dev: false,
88 });
89 const styleMap = prepareInlineCss(p, routes);
90
91 // -- backend --
92 const builtBackend = bundle.bundleServerJavaScript({
93 progress: p,
94 entries: siteConfig.backends,
95 platform: "node",
96 styleMap,
97 scriptMap,
98 viewItems: viewsAndDynPages.map((ref) => {
99 const { id, file, type } = UNWRAP(ref.value);
100 return { id: type === "page" ? `page:${id}` : id, file };
101 }),
102 viewRefs: viewsAndDynPages,
103 metaTemplate,
104 });
105
106 // -- assemble page assets --
107 const pAssemblePages = Promise.all(
108 builtPages.map((page) => assembleAndWritePage(page, styleMap, scriptMap)),
109 ).then(() => pageNode.end());
110
111 await Promise.all([
112 builtBackend,
113 builtStaticFiles,
114 pAssemblePages,
115 builtFonts,
116 ]);
117}
118
119export async function readManifest(io: Io) {
120 using _ = io.start("read site manifest");
121 const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
122 return {
123 siteSections: cfg.siteSections.map((section) => ({
124 root: toRel(section.root),
125 base: section.base ?? "/",
126 })),
127 backends: cfg.backends.map(toRel),
128 fonts: cfg.fonts,
129 meta: {
130 ...cfg.meta,
131 base: cfg.meta.base.toString(),
132 },
133 };
134}
135
136export async function discoverAllFiles(
137 p: progress.Ref,
138 siteConfig: Awaited<ReturnType<typeof readManifest>>,
139) {
140 using node = p.start("discover files");
141 return (
142 await Promise.all(
143 siteConfig.siteSections.map(({ root: root, base }) =>
144 incr.work(node, scanSiteSection, {
145 root: toAbs(root),
146 base: base.endsWith("/") ? base : base + "/",
147 })
148 ),
149 )
150 ).reduce((acc, next) => ({
151 staticFiles: acc.staticFiles.concat(next.staticFiles),
152 pages: acc.pages.concat(next.pages),
153 views: acc.views.concat(next.views),
154 scripts: acc.scripts.concat(next.scripts),
155 }));
156}
157
158export async function scanSiteSection(io: Io, section: sg.Section) {
159 using _ = io.start(toRel(section.root));
160 // Static files are compressed and served as-is.
161 // - "{section}/static/*.png"
162 let staticFiles: FileItem[] = [];
163 // Pages are rendered then served as static files.
164 // - "{section}/pages/*.
165 let pages: FileItem[] = [];
166 // Views are dynamically rendered pages called via backend code.
167 // - "{section}/views/*.tsx"
168 let views: FileItem[] = [];
169 // Public scripts are bundled for the client as static assets under "/js/[...]"
170 // This is used for the file viewer's canvases.
171 // Note that '.client.ts' can be placed anywhere in the file structure.
172 // - "{section}/scripts/*.client.ts"
173 let scripts: FileItem[] = [];
174
175 const sectionPath = (...sub: string[]) => path.join(section.root, ...sub);
176 const rootPrefix = hot.projectSrc === section.root
177 ? ""
178 : path.relative(hot.projectSrc, section.root) + "/";
179 const kinds = [
180 {
181 dir: sectionPath("pages"),
182 list: pages,
183 prefix: section.base,
184 include: [".tsx", ".marko", ".mdo"],
185 exclude: [".client.ts"],
186 page: true,
187 },
188 {
189 dir: sectionPath("static"),
190 list: staticFiles,
191 prefix: section.base,
192 ext: true,
193 page: true,
194 },
195 {
196 dir: sectionPath("scripts"),
197 list: scripts,
198 prefix: rootPrefix,
199 include: [".client.ts"],
200 page: false,
201 },
202 {
203 dir: sectionPath("views"),
204 list: views,
205 prefix: rootPrefix,
206 include: [".tsx", ".marko", ".mdo"],
207 exclude: [".client.ts"],
208 page: false,
209 },
210 ];
211 for (const kind of kinds) {
212 const {
213 dir,
214 list,
215 prefix,
216 include = [""],
217 exclude = [],
218 ext = false,
219 } = kind;
220
221 let items;
222 try {
223 items = await io.readDirRecursive(dir);
224 } catch (err: any) {
225 if (err.code === "ENOENT") continue;
226 throw err;
227 }
228 for (const subPath of items) {
229 const file = path.join(dir, subPath);
230 const stat = fs.statSync(file);
231 if (stat.isDirectory()) continue;
232 if (!include.some((e) => subPath.endsWith(e))) continue;
233 if (exclude.some((e) => subPath.endsWith(e))) continue;
234 const trim = ext
235 ? subPath
236 : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
237 let id = prefix + trim.replaceAll("\\", "/");
238 if (kind.page && id.endsWith("/index")) {
239 id = id.slice(0, -"/index".length) || "/";
240 }
241 list.push({ id, file: path.relative(hot.projectRoot, file) });
242 }
243 }
244
245 return { staticFiles, pages, views, scripts };
246}
247
248export async function loadMarkoCache(p: progress.Ref) {
249 using node = p.start("load marko cache");
250
251 markoCache.clear();
252 const files = fs.readDirRecOptionalSync(".clover/marko")
253 .filter((file) => file.endsWith(".js"));
254 let restored = 0;
255 let rebuilt = 0;
256 await Promise.all(
257 files.map(async (cachedPath) => {
258 const source = resolveMarkoCacheSource(cachedPath);
259 if (!source) return;
260
261 const cachePath = path.join(".clover/marko", cachedPath);
262 const [sourceStat, cacheStat] = await Promise.all([
263 fs.stat(source).catch(() => null),
264 fs.stat(cachePath).catch(() => null),
265 ]);
266 if (!sourceStat) return;
267
268 const absSource = path.resolve(source);
269 if (!cacheStat || cacheStat.mtimeMs < sourceStat.mtimeMs) {
270 rebuilt += 1;
271 markoCache.set(
272 absSource,
273 buildMarkoCacheEntry(absSource, source.endsWith(".mdo") ? "markdown" : "html"),
274 );
275 return;
276 }
277
278 restored += 1;
279 const sourceText = await fs.readFile(source, "utf-8");
280 markoCache.set(absSource, {
281 src: await fs.readFile(cachePath, "utf-8"),
282 scannedClientRefs: scanMarkoClientImports(sourceText, absSource)
283 .scannedClientRefs,
284 });
285 }),
286 );
287
288 node.text = `load marko cache (${restored} restored, ${rebuilt} rebuilt)`;
289}
290
291function resolveMarkoCacheSource(cachedPath: string) {
292 const source = cachedPath.slice(0, -".js".length);
293 if (fs.existsSync(source)) return source;
294 const srcSource = path.join("src", source);
295 if (fs.existsSync(srcSource)) return srcSource;
296 return null;
297}
298
299export async function preparePage(
300 io: Io,
301 { item, metaTemplate }: { item: sg.FileItem; metaTemplate: meta.Template },
302) {
303 try {
304 // -- load and validate module --
305 let {
306 default: Page,
307 meta: metadata,
308 theme: pageTheme,
309 layout,
310 regenerate,
311 markoOnly,
312 } = await io.import<sg.PageExports>(item.file);
313 if (!Page) throw new Error("Page is missing a 'default' export.");
314 if (!metadata) {
315 throw new Error("Page is missing 'meta' export with a title.");
316 }
317
318 // -- css --
319 if (layout?.theme) pageTheme = layout.theme;
320 const theme: css.Theme = {
321 ...css.defaultTheme,
322 ...pageTheme,
323 };
324 const cssImports = Array.from(
325 new Set([globalCssPath, ...hot.getCssImports(item.file)]),
326 (file) => path.relative(hot.projectSrc, file),
327 );
328
329 // -- metadata --
330 const renderedMeta = meta.toHtml(metaTemplate, {
331 canonical: item.id,
332 ...typeof metadata === "function"
333 ? await metadata({ ssr: true })
334 : metadata,
335 });
336 if (!renderedMeta.includes("<title>")) {
337 throw new Error(
338 "Page is missing 'meta.title'. " + "All pages need a title tag.",
339 );
340 }
341
342 // -- html --
343 let text: string;
344 const cloverScripts = new Set<string>();
345 text = await SitegenWrap.render({
346 cloverScripts,
347 page: Page,
348 attrs: {},
349 });
350
351 const styleKey = css.styleKey(cssImports, theme);
352 return {
353 type: "page",
354 id: item.id,
355 file: item.file,
356 regenerate,
357 html: text,
358 meta: renderedMeta,
359 cssImports,
360 theme: theme ?? null,
361 styleKey,
362 clientRefs: Array.from(cloverScripts),
363 } as const;
364 } catch (e) {
365 exception.rethrowWithMetadata(e, { page: item.id });
366 }
367}
368
369export async function prepareView(io: Io, item: sg.FileItem) {
370 const module = await io.import<any>(item.file);
371 if (!module.meta) throw new Error(`View is missing 'export const meta'`);
372 if (!module.default) throw new Error(`View is missing a default export.`);
373 const pageTheme = module.layout?.theme ?? module.theme;
374 const theme: css.Theme = { ...css.defaultTheme, ...pageTheme };
375 const cssImports = Array.from(
376 new Set([globalCssPath, ...hot.getCssImports(item.file)]),
377 (file) => path.relative(hot.projectSrc, file),
378 );
379 const styleKey = css.styleKey(cssImports, theme);
380 return {
381 type: "view",
382 id: item.id,
383 file: item.file,
384 cssImports,
385 theme,
386 clientRefs: hot.getClientScriptRefs(item.file),
387 hasLayout: !!module.layout?.default,
388 styleKey,
389 } as const;
390}
391export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
392
393export function prepareInlineCss(
394 p: progress.Ref,
395 items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,
396) {
397 const node = p.start("bundle css");
398 const map = new Map<string, incr.Ref<string>>();
399 for (const { styleKey, cssImports, theme } of items) {
400 if (map.has(styleKey)) continue;
401 map.set(
402 styleKey,
403 incr.work(node, css.bundleCssFiles, { cssImports, theme, dev: false }),
404 );
405 }
406 Promise.allSettled([...map.values()]).then(() => node.end());
407 return map;
408}
409
410export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
411export async function assembleAndWritePage(
412 pageWork: incr.Ref<PreparedPage>,
413 styleMap: Map<string, incr.Ref<string>>,
414 scriptWork: incr.Ref<Record<string, string>>,
415) {
416 const page = await pageWork;
417 ASSERT("passive" in pageWork.progress);
418 const pageNode = pageWork.progress;
419 return incr.work(
420 pageWork.progress,
421 async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
422 const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
423
424 const scriptIds = clientRefs.map(hot.getScriptId);
425 const scriptMap = await io.readWork(scriptWork);
426
427 pageNode.passive = false;
428
429 const scripts = sg.joinScripts(scriptIds
430 .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`)));
431
432 const buffer = sg.wrapDocument({
433 body: html,
434 head: meta,
435 inlineCss,
436 scripts,
437 });
438 await io.writeAsset({
439 pathname: id,
440 buffer,
441 headers: {
442 "Content-Type": "text/html;charset=utf8",
443 },
444 regenerative: !!regenerate,
445 });
446 pageNode.passive = true;
447 },
448 page,
449 );
450}
451
452export type PageOrView = PreparedPage | PreparedView;
453
454import * as path from "node:path";
455
456import * as sg from "#sitegen";
457import type { FileItem } from "#sitegen";
458import * as fs from "#sitegen/fs";
459import * as exception from "@clo/lib/exception";
460import * as meta from "@clo/lib/meta";
461import * as progress from "@clo/lib/progress";
462
463import SitegenWrap from "./sitegen-wrap.marko";
464
465import { ASSERT, UNWRAP } from "@clo/lib/assert";
466import * as bundle from "./bundle.ts";
467import * as css from "./css.ts";
468import * as fonts from "./font.ts";
469import * as hot from "./hot.ts";
470import * as incr from "./incremental.ts";
471import { Io } from "./incremental.ts";
472import { buildMarkoCacheEntry, markoCache, scanMarkoClientImports } from "./marko.ts";