| 1 | /** |
| 2 | * build script for paperclover.net. the root entry point for the site. |
| 3 | * @module |
| 4 | */ |
| 5 | const root = process.cwd(); |
| 6 | const out = ".clover/o"; |
| 7 | const globalCss = "src/global.css"; |
| 8 | const toRel = (str: string) => path.relative(root, str).replaceAll("\\", "/"); |
| 9 | |
| 10 | export async function main(graph: incremental.Graph) { |
| 11 | if (process.argv.includes("--build")) { |
| 12 | await graph.incrementalBuild({ buildAndWriteOutput }); |
| 13 | return; |
| 14 | } |
| 15 | |
| 16 | // in production nothing but a deploy changes the files, so they are read once and not watched |
| 17 | const server = await incremental.serve({ |
| 18 | graph, |
| 19 | buildAndWriteOutput, |
| 20 | watchFilesystem: !process.argv.includes("--prod"), |
| 21 | }); |
| 22 | |
| 23 | if (!process.argv.includes("--no-open") && !process.argv.includes("--prod")) { |
| 24 | // subprocess.launchBrowser(server.url); |
| 25 | } |
| 26 | |
| 27 | // requests in flight finish before the process leaves |
| 28 | const stop = () => void server.close().then(() => process.exit(0)); |
| 29 | process.on("SIGINT", stop); |
| 30 | process.on("SIGTERM", stop); |
| 31 | for await (const result of server.watch()) { |
| 32 | if (result.value) server.setHandler(loadBackend(server, result.value)); |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | type Respond = (request: Request) => Response | Promise<Response>; |
| 37 | |
| 38 | /** hand a build to the code that serves it, loaded by the graph so that it reloads with its code */ |
| 39 | async function loadBackend( |
| 40 | server: incremental.Server<Output>, |
| 41 | output: Output, |
| 42 | progressOverride?: progress.Ref, |
| 43 | ): Promise<Respond> { |
| 44 | const { graph } = server; |
| 45 | const graphNode = progressOverride ?? server.progress; |
| 46 | using _ = graphNode.start("load backend"); |
| 47 | const sitegen = await graph.import<typeof import("../framework/lib/sitegen.ts")>("framework/lib/sitegen.ts"); |
| 48 | sitegen.setGraph(graph); |
| 49 | const assets = await graph.import<typeof import("../framework/lib/assets.ts")>("framework/lib/assets.ts"); |
| 50 | await assets.load(output.assets, `${out}/asset`); |
| 51 | |
| 52 | // views are loaded before any request needs one, so serving never waits on the graph |
| 53 | const modules: Parameters<typeof view.load>[1] = {}; |
| 54 | for (const { id, file } of Object.values(output.views)) modules[id] = await graph.import(file); |
| 55 | const view = await graph.import<typeof import("../framework/lib/view.ts")>("framework/lib/view.ts"); |
| 56 | view.load(output, modules); |
| 57 | |
| 58 | const backend = await graph.import<{ default: { fetch: Respond } }>("src/backend.ts"); |
| 59 | return (req) => { |
| 60 | // deployments |
| 61 | if ( |
| 62 | new URL(req.url).pathname === "/.sitegen/rebuild" |
| 63 | && !req.headers.has("x-forwarded-for") |
| 64 | ) { |
| 65 | const root = new progress.Root(); |
| 66 | graph.incrementalBuild({ buildAndWriteOutput, progress: root }) |
| 67 | .then((output) => loadBackend(server, output, root)) |
| 68 | .then((backend) => { |
| 69 | server.setHandler(backend); |
| 70 | root.end(); |
| 71 | }) |
| 72 | .catch((e) => root.error(e)); |
| 73 | return new Response(progress.encodeByteStream(root), { |
| 74 | headers: { "Content-Type": progress.contentType }, |
| 75 | }); |
| 76 | } |
| 77 | |
| 78 | return backend.default.fetch(req); |
| 79 | }; |
| 80 | } |
| 81 | |
| 82 | export interface Output { |
| 83 | /** by pathname. the bytes are in `.clover/o/asset`, by hash, also as `.gz` and `.zst` */ |
| 84 | assets: Record<string, Asset>; |
| 85 | /** pages rendered per request, by id */ |
| 86 | views: Record<string, ts.DeepReadonly<PreparedView>>; |
| 87 | /** the step that bundled the client scripts a page can include, by script id */ |
| 88 | scripts: incremental.Step<{ scripts: Record<string, string> }>; |
| 89 | /** the steps that bundled stylesheets, by the style key of a page or view */ |
| 90 | styles: Record<string, incremental.Step<string>>; |
| 91 | /** what every page's metadata starts from */ |
| 92 | meta: SiteMeta; |
| 93 | } |
| 94 | |
| 95 | export interface Asset { |
| 96 | hash: string; |
| 97 | headers: Record<string, string>; |
| 98 | } |
| 99 | |
| 100 | export async function buildAndWriteOutput(ref: incremental.Ref, title = "sitegen!"): Promise<Output> { |
| 101 | using incr = ref.start(title); |
| 102 | await checkSpelling(incr); |
| 103 | |
| 104 | const site = await incr.step(async (b) => { |
| 105 | const { siteSections, fonts, meta } = await b.import<typeof import("./site.ts")>("./src/site.ts"); |
| 106 | // relative paths, so what a build caches still fits once the project has moved |
| 107 | const sections = siteSections.map(({ root, base = "/" }) => ({ root: toRel(root), base })); |
| 108 | return { sections, fonts, meta }; |
| 109 | }).read(); |
| 110 | const files = await discoverAllFiles(incr, site.sections); |
| 111 | |
| 112 | const aliasing = Array.from(Map.groupBy([...files.staticFiles, ...files.pages], (item) => item.id)) |
| 113 | .filter(([, items]) => items.length > 1); |
| 114 | if (aliasing.length > 0) { |
| 115 | throw new Error( |
| 116 | `source files alias:\n` |
| 117 | + aliasing.map(([id, items]) => ` - ${id}\n` + items.map((item) => " - " + item.file).join("\n")).join("\n"), |
| 118 | ); |
| 119 | } |
| 120 | |
| 121 | // -- perform build-time rendering -- |
| 122 | const fontFiles = site.fonts.flatMap((font) => { |
| 123 | const source = incr.step(fonts.fetchFont, font.name, font.sources); |
| 124 | return font.subsets.map((subset) => incr.step(fonts.subsetFont, source, subset, site.meta.base)); |
| 125 | }); |
| 126 | const staticFiles = files.staticFiles.map((item) => incr.step((b, file) => b.readFile(file).bytes(), item.file)); |
| 127 | const pages = files.pages.map((item) => incr.step(preparePage, item, site.meta)); |
| 128 | const views = await Promise.all(files.views.map((item) => incr.step(prepareView, item).read())); |
| 129 | const prepared = await Promise.all(pages.map((page) => page.read())); |
| 130 | const routes = [...views, ...prepared]; |
| 131 | |
| 132 | // -- page resources -- |
| 133 | const bundleScripts = incr.step(bundle.bundleClientJavaScript, { |
| 134 | clientRefs: Array.from(new Set(routes.flatMap((route) => route.clientRefs))).sort(), |
| 135 | publicScripts: files.scripts.map((item) => getScriptId(path.resolve(root, item.file))).sort(), |
| 136 | }); |
| 137 | const styles = new Map<string, incremental.Step<string>>(); |
| 138 | for (const { styleKey, cssImports, theme } of routes) { |
| 139 | if (styles.has(styleKey)) continue; |
| 140 | styles.set(styleKey, incr.step(css.bundleCssFiles, { cssImports: [...cssImports], theme })); |
| 141 | } |
| 142 | |
| 143 | // -- assemble page assets -- |
| 144 | const documents = pages.map((page, i) => |
| 145 | incr.step(assemblePage, page, UNWRAP(styles.get(prepared[i]!.styleKey)), bundleScripts) |
| 146 | ); |
| 147 | |
| 148 | // -- write assets, once for all the pathnames with the same bytes -- |
| 149 | // each asset's bytes are a step of their own, so an asset is hashed and |
| 150 | // written only when its bytes change. assets are listed in the order the |
| 151 | // files were found, so the same site lists them the same way. |
| 152 | const { assets: scriptAssets } = await bundleScripts.read(); |
| 153 | const fontAssets = await Promise.all(fontFiles.map((font) => font.read())); |
| 154 | const html = { "content-type": "text/html;charset=utf8" }; |
| 155 | const sources: Array<{ pathname: string; bytes: incremental.Step<string | Uint8Array>; headers?: typeof html }> = [ |
| 156 | ...Object.keys(scriptAssets).map((pathname) => ({ |
| 157 | pathname, |
| 158 | bytes: incr.step(scriptAsset, bundleScripts, pathname), |
| 159 | })), |
| 160 | ...fontFiles.flatMap((font, i) => |
| 161 | Object.keys(fontAssets[i]!).map((pathname) => ({ pathname, bytes: incr.step(fontAsset, font, pathname) })) |
| 162 | ), |
| 163 | ...files.staticFiles.map((item, i) => ({ pathname: item.id, bytes: staticFiles[i]! })), |
| 164 | ...files.pages.map((item, i) => ({ pathname: item.id, bytes: documents[i]!, headers: html })), |
| 165 | ]; |
| 166 | const hashes = await Promise.all(sources.map(({ bytes }) => incr.step(hashAsset, bytes).read())); |
| 167 | const assets: Output["assets"] = {}; |
| 168 | const writes = new Map<string, Promise<unknown>>(); |
| 169 | sources.forEach(({ pathname, bytes, headers }, i) => { |
| 170 | ASSERT(pathname.startsWith("/") && !(pathname in assets), () => `asset ${pathname} is written twice`); |
| 171 | const hash = hashes[i]!; |
| 172 | assets[pathname] = { |
| 173 | hash, |
| 174 | headers: { "content-type": mime.for(pathname), ...headers, etag: JSON.stringify(hash) }, |
| 175 | }; |
| 176 | if (!writes.has(hash)) writes.set(hash, incr.step(writeAsset, hash, bytes).read()); |
| 177 | }); |
| 178 | await Promise.all(writes.values()); |
| 179 | await incr.writeFile(`${out}/asset.json`, JSON.stringify(assets)); |
| 180 | |
| 181 | // -- types of what a backend can ask for by name -- |
| 182 | const union = (keys: string[]) => keys.map((key) => JSON.stringify(key)).join(" | ") || "never"; |
| 183 | const viewTypes = views.map((view) => |
| 184 | ` ${JSON.stringify(view.id)}: typeof import(${JSON.stringify("../../" + view.file)});` |
| 185 | ); |
| 186 | await Promise.all([ |
| 187 | incr.writeFile(".clover/ts/asset.d.ts", `export type AssetKey = ${union(Object.keys(assets))};\n`), |
| 188 | incr.writeFile(".clover/ts/view.d.ts", `export interface RegisteredViews {\n${viewTypes.join("\n")}\n}\n`), |
| 189 | ]); |
| 190 | |
| 191 | return { |
| 192 | assets, |
| 193 | views: Object.fromEntries(views.map((view) => [view.id, view])), |
| 194 | scripts: bundleScripts, |
| 195 | styles: Object.fromEntries(styles), |
| 196 | meta: site.meta, |
| 197 | }; |
| 198 | } |
| 199 | |
| 200 | async function discoverAllFiles( |
| 201 | ref: incremental.Ref, |
| 202 | sections: readonly Readonly<sg.Section>[], |
| 203 | ) { |
| 204 | using node = ref.start("discover files"); |
| 205 | const found = await Promise.all( |
| 206 | sections.map(({ root, base = "/" }) => |
| 207 | node.step(scanSiteSection, { |
| 208 | root, |
| 209 | base: base.endsWith("/") ? base : base + "/", |
| 210 | }).read() |
| 211 | ), |
| 212 | ); |
| 213 | return { |
| 214 | staticFiles: found.flatMap((section) => section.staticFiles), |
| 215 | pages: found.flatMap((section) => section.pages), |
| 216 | views: found.flatMap((section) => section.views), |
| 217 | scripts: found.flatMap((section) => section.scripts), |
| 218 | }; |
| 219 | } |
| 220 | |
| 221 | export async function scanSiteSection(b: incremental.Build, section: Readonly<sg.Section>) { |
| 222 | using _ = b.start(section.root); |
| 223 | // Static files are compressed and served as-is. |
| 224 | // - "{section}/static/*.png" |
| 225 | let staticFiles: sg.FileItem[] = []; |
| 226 | // Pages are rendered then served as static files. |
| 227 | // - "{section}/pages/*. |
| 228 | let pages: sg.FileItem[] = []; |
| 229 | // Views are dynamically rendered pages called via backend code. |
| 230 | // - "{section}/views/*.tsx" |
| 231 | let views: sg.FileItem[] = []; |
| 232 | // Public scripts are bundled for the client as static assets under "/js/[...]" |
| 233 | // This is used for the file viewer's canvases. |
| 234 | // Note that '.client.ts' can be placed anywhere in the file structure. |
| 235 | // - "{section}/scripts/*.client.ts" |
| 236 | let scripts: sg.FileItem[] = []; |
| 237 | |
| 238 | const sectionPath = (...sub: string[]) => path.join(section.root, ...sub); |
| 239 | const rootPrefix = section.root === "src" |
| 240 | ? "" |
| 241 | : path.relative("src", section.root) + "/"; |
| 242 | const kinds = [ |
| 243 | { |
| 244 | dir: sectionPath("pages"), |
| 245 | list: pages, |
| 246 | prefix: section.base, |
| 247 | include: [".marko", ".mdo"], |
| 248 | exclude: [".client.ts"], |
| 249 | page: true, |
| 250 | }, |
| 251 | { |
| 252 | dir: sectionPath("static"), |
| 253 | list: staticFiles, |
| 254 | prefix: section.base, |
| 255 | ext: true, |
| 256 | page: true, |
| 257 | }, |
| 258 | { |
| 259 | dir: sectionPath("scripts"), |
| 260 | list: scripts, |
| 261 | prefix: rootPrefix, |
| 262 | include: [".client.ts"], |
| 263 | page: false, |
| 264 | }, |
| 265 | { |
| 266 | dir: sectionPath("views"), |
| 267 | list: views, |
| 268 | prefix: rootPrefix, |
| 269 | include: [".marko", ".mdo"], |
| 270 | exclude: [".client.ts"], |
| 271 | page: false, |
| 272 | }, |
| 273 | ]; |
| 274 | for (const kind of kinds) { |
| 275 | const { |
| 276 | dir, |
| 277 | list, |
| 278 | prefix, |
| 279 | include = [""], |
| 280 | exclude = [], |
| 281 | ext = false, |
| 282 | } = kind; |
| 283 | |
| 284 | let items; |
| 285 | try { |
| 286 | items = await b.readDirRecursive(dir); |
| 287 | } catch (err) { |
| 288 | if (exception.code(err) === "ENOENT") continue; |
| 289 | throw err; |
| 290 | } |
| 291 | for (const subPath of items) { |
| 292 | const file = path.join(dir, subPath); |
| 293 | if (file.endsWith("/")) continue; |
| 294 | if (!include.some((e) => subPath.endsWith(e))) continue; |
| 295 | if (exclude.some((e) => subPath.endsWith(e))) continue; |
| 296 | const trim = ext |
| 297 | ? subPath |
| 298 | : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/"); |
| 299 | let id = prefix + trim.replaceAll("\\", "/"); |
| 300 | if (kind.page && id.endsWith("/index")) { |
| 301 | id = id.slice(0, -"/index".length) || "/"; |
| 302 | } |
| 303 | list.push({ id, file: toRel(file) }); |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | return { staticFiles, pages, views, scripts }; |
| 308 | } |
| 309 | |
| 310 | type Route = sg.ViewExports & { layout?: { default?: unknown; theme?: css.Theme } }; |
| 311 | // what a step can hold of a template: no url, and no function |
| 312 | type SiteMeta = Omit<meta.Template, "base" | "titleTemplate"> & { base: string }; |
| 313 | |
| 314 | /** render a page that is the same for every request */ |
| 315 | export async function preparePage(b: incremental.Build, item: Readonly<sg.FileItem>, template: SiteMeta) { |
| 316 | using _ = b.start(item.file); |
| 317 | try { |
| 318 | const { default: Page, meta: metadata, theme: pageTheme, layout } = await b.import<Route>(item.file); |
| 319 | if (!Page) throw new Error("Page is missing a 'default' export."); |
| 320 | if (!metadata) throw new Error("Page is missing 'meta' export with a title."); |
| 321 | |
| 322 | const theme: css.Theme = { ...css.defaultTheme, ...layout?.theme ?? pageTheme }; |
| 323 | const cssImports = await findCssImports(b, item.file); |
| 324 | |
| 325 | const renderedMeta = meta.toHtml({ ...template, base: new URL(template.base) }, { |
| 326 | canonical: item.id, |
| 327 | ...typeof metadata === "function" ? await metadata({ ssr: true }) : metadata, |
| 328 | }); |
| 329 | if (!renderedMeta.includes("<title>")) { |
| 330 | throw new Error("Page is missing 'meta.title'. " + "All pages need a title tag."); |
| 331 | } |
| 332 | |
| 333 | // components add the scripts they need while rendering. marko would pick a |
| 334 | // random id for the render, and a step has to return the same html every time |
| 335 | const cloverScripts = new Set<string>(); |
| 336 | const $global = { renderId: "_" }; |
| 337 | const html = String(await SitegenWrap.render({ $global, cloverScripts, page: Page, attrs: {} })); |
| 338 | |
| 339 | return { |
| 340 | id: item.id, |
| 341 | file: item.file, |
| 342 | html, |
| 343 | meta: renderedMeta, |
| 344 | cssImports, |
| 345 | theme, |
| 346 | styleKey: css.styleKey(cssImports, theme), |
| 347 | clientRefs: Array.from(cloverScripts), |
| 348 | }; |
| 349 | } catch (e) { |
| 350 | exception.rethrowWithMetadata(e, { page: item.id }); |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | /** what a server needs beside a view's module to render it per request */ |
| 355 | export async function prepareView(b: incremental.Build, item: Readonly<sg.FileItem>) { |
| 356 | using _ = b.start(item.file); |
| 357 | const module = await b.import<Partial<Route>>(item.file); |
| 358 | if (!module.meta) throw new Error(`View is missing 'export const meta'`); |
| 359 | if (!module.default) throw new Error(`View is missing a default export.`); |
| 360 | const theme: css.Theme = { ...css.defaultTheme, ...module.layout?.theme ?? module.theme }; |
| 361 | const cssImports = await findCssImports(b, item.file); |
| 362 | // a view renders with props no build knows, so it gets every script it could include |
| 363 | const clientRefs = (await b.trackModule(item.file)).flatMap((module) => |
| 364 | (module.data as { scannedClientRefs?: string[] } | null)?.scannedClientRefs ?? [] |
| 365 | ); |
| 366 | return { |
| 367 | id: item.id, |
| 368 | file: item.file, |
| 369 | cssImports, |
| 370 | theme, |
| 371 | clientRefs: Array.from(new Set(clientRefs)), |
| 372 | hasLayout: !!module.layout?.default, |
| 373 | styleKey: css.styleKey(cssImports, theme), |
| 374 | }; |
| 375 | } |
| 376 | export type PreparedView = Awaited<ReturnType<typeof prepareView>>; |
| 377 | |
| 378 | /** the stylesheets a module imports, in import order, after the one every page has */ |
| 379 | async function findCssImports(b: incremental.Build, file: string): Promise<string[]> { |
| 380 | const modules = await b.trackModule(file); |
| 381 | const imports = new Map(modules.map((module) => [module.url, module.imports])); |
| 382 | const found = new Set([globalCss]); |
| 383 | const seen = new Set<string>(); |
| 384 | const visit = (href: string) => { |
| 385 | if (seen.has(href)) return; |
| 386 | seen.add(href); |
| 387 | if (href.endsWith(".css")) return void found.add(toRel(url.fileURLToPath(href))); |
| 388 | imports.get(href)?.forEach(visit); |
| 389 | }; |
| 390 | visit(UNWRAP(modules[0]).url); |
| 391 | return Array.from(found); |
| 392 | } |
| 393 | |
| 394 | /** the whole html document of a page, with its styles and scripts inline */ |
| 395 | export async function assemblePage( |
| 396 | b: incremental.Build, |
| 397 | page: incremental.Step<Awaited<ReturnType<typeof preparePage>>>, |
| 398 | style: incremental.Step<string>, |
| 399 | scripts: incremental.Step<{ scripts: Record<string, string> }>, |
| 400 | ) { |
| 401 | const { html, meta, clientRefs } = await b.readStep(page); |
| 402 | const scriptMap = (await b.readStep(scripts)).scripts; |
| 403 | return sg.wrapDocument({ |
| 404 | body: html, |
| 405 | head: meta, |
| 406 | inlineCss: await b.readStep(style), |
| 407 | scripts: sg.joinScripts(clientRefs.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))), |
| 408 | }); |
| 409 | } |
| 410 | |
| 411 | /** one of the scripts the bundle made, so writing it follows that script alone */ |
| 412 | async function scriptAsset( |
| 413 | b: incremental.Build, |
| 414 | bundle: incremental.Step<{ assets: Record<string, string> }>, |
| 415 | pathname: string, |
| 416 | ) { |
| 417 | return UNWRAP((await b.readStep(bundle)).assets[pathname]); |
| 418 | } |
| 419 | |
| 420 | /** one of the files of a font subset */ |
| 421 | async function fontAsset(b: incremental.Build, font: incremental.Step<Record<string, Uint8Array>>, pathname: string) { |
| 422 | return UNWRAP((await b.readStep(font))[pathname]); |
| 423 | } |
| 424 | |
| 425 | /** the hash that an asset's file is named by */ |
| 426 | async function hashAsset(b: incremental.Build, bytes: incremental.Step<string | Uint8Array>) { |
| 427 | return crypto.createHash("sha1").update(await b.readStep(bytes)).digest("hex"); |
| 428 | } |
| 429 | |
| 430 | /** an asset's file and its compressed copies, named by its hash */ |
| 431 | async function writeAsset(b: incremental.Build, hash: string, bytes: incremental.Step<string | Uint8Array>) { |
| 432 | const content = await b.readStep(bytes); |
| 433 | const [gzipped, zstded] = await Promise.all([gzip(content), zstd(content)]); |
| 434 | await Promise.all([ |
| 435 | b.writeFile(`${out}/asset/${hash}`, content), |
| 436 | b.writeFile(`${out}/asset/${hash}.gz`, gzipped), |
| 437 | b.writeFile(`${out}/asset/${hash}.zst`, zstded), |
| 438 | ]); |
| 439 | } |
| 440 | |
| 441 | const gzip = util.promisify(zlib.gzip); |
| 442 | const zstd = util.promisify(zlib.zstdCompress); |
| 443 | |
| 444 | import * as sg from "#sitegen"; |
| 445 | import { ASSERT, UNWRAP } from "@clo/lib/assert"; |
| 446 | import * as exception from "@clo/lib/exception"; |
| 447 | import * as incremental from "@clo/lib/incremental"; |
| 448 | import * as meta from "@clo/lib/meta"; |
| 449 | import * as mime from "@clo/lib/mime"; |
| 450 | import * as progress from "@clo/lib/progress"; |
| 451 | import * as subprocess from "@clo/lib/subprocess"; |
| 452 | import type * as ts from "@clo/lib/ts"; |
| 453 | import * as crypto from "node:crypto"; |
| 454 | import path from "node:path"; |
| 455 | import * as url from "node:url"; |
| 456 | import * as util from "node:util"; |
| 457 | import * as zlib from "node:zlib"; |
| 458 | import * as bundle from "../framework/bundle.ts"; |
| 459 | import * as css from "../framework/css.ts"; |
| 460 | import * as fonts from "../framework/font.ts"; |
| 461 | import { getScriptId } from "../framework/marko.ts"; |
| 462 | import SitegenWrap from "../framework/sitegen-wrap.marko"; |
| 463 | import { checkSpelling } from "./spelling.ts"; |