| author | |
| committer | |
| log | 64a274c53ac58e86de04683d69da4a77151a835f |
| tree | 982b1bc272ccc113a8d52f3c2b0a1e30734b329d |
| parent | 6832e061d3ab21c081e593dc308f5087eacf377d |
| signature | Commit is signed but in an unrecognized format. |
25 files changed, 670 insertions(+), 510 deletions(-)
framework/assets.ts deleted-100| ... | @@ -1,100 +0,0 @@ | ||
| 1 | interface Loaded { | ||
| 2 | map: BuiltAssetMap; | ||
| 3 | buf: Buffer; | ||
| 4 | } | ||
| 5 | let assets: Loaded | null = null; | ||
| 6 | |||
| 7 | export type StaticPageId = string; | ||
| 8 | |||
| 9 | export async function reload() { | ||
| 10 | const [map, buf] = await Promise.all([ | ||
| 11 | fs.readFile(".clover/static.json", "utf8"), | ||
| 12 | fs.readFile(".clover/static.blob"), | ||
| 13 | ]); | ||
| 14 | assets = { | ||
| 15 | map: JSON.parse(map), | ||
| 16 | buf, | ||
| 17 | }; | ||
| 18 | } | ||
| 19 | |||
| 20 | export async function reloadSync() { | ||
| 21 | const map = fs.readFileSync(".clover/static.json", "utf8"); | ||
| 22 | const buf = fs.readFileSync(".clover/static.blob"); | ||
| 23 | assets = { | ||
| 24 | map: JSON.parse(map), | ||
| 25 | buf, | ||
| 26 | }; | ||
| 27 | } | ||
| 28 | |||
| 29 | export async function assetMiddleware(c: Context, next: Next) { | ||
| 30 | if (!assets) await reload(); | ||
| 31 | const asset = assets!.map[c.req.path]; | ||
| 32 | if (asset) { | ||
| 33 | return assetInner(c, asset, 200); | ||
| 34 | } | ||
| 35 | return next(); | ||
| 36 | } | ||
| 37 | |||
| 38 | export async function serveAsset( | ||
| 39 | c: Context, | ||
| 40 | id: StaticPageId, | ||
| 41 | status: StatusCode, | ||
| 42 | ) { | ||
| 43 | assets ?? await reload(); | ||
| 44 | return assetInner(c, assets!.map[id], status); | ||
| 45 | } | ||
| 46 | |||
| 47 | export function hasAsset(id: string) { | ||
| 48 | if (!assets) reloadSync(); | ||
| 49 | return assets!.map[id] !== undefined; | ||
| 50 | } | ||
| 51 | |||
| 52 | export function etagMatches(etag: string, ifNoneMatch: string) { | ||
| 53 | return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; | ||
| 54 | } | ||
| 55 | |||
| 56 | function subarrayAsset([start, end]: View) { | ||
| 57 | return assets!.buf.subarray(start, end); | ||
| 58 | } | ||
| 59 | |||
| 60 | function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { | ||
| 61 | const ifnonematch = c.req.header("If-None-Match"); | ||
| 62 | if (ifnonematch) { | ||
| 63 | const etag = asset.headers.ETag; | ||
| 64 | if (etagMatches(etag, ifnonematch)) { | ||
| 65 | c.res = new Response(null, { | ||
| 66 | status: 304, | ||
| 67 | statusText: "Not Modified", | ||
| 68 | headers: { | ||
| 69 | ETag: etag, | ||
| 70 | }, | ||
| 71 | }); | ||
| 72 | return; | ||
| 73 | } | ||
| 74 | } | ||
| 75 | const acceptEncoding = c.req.header("Accept-Encoding") ?? ""; | ||
| 76 | let body; | ||
| 77 | let headers = asset.headers; | ||
| 78 | if (acceptEncoding.includes("zstd") && asset.zstd) { | ||
| 79 | body = subarrayAsset(asset.zstd); | ||
| 80 | headers = { | ||
| 81 | ...asset.headers, | ||
| 82 | "Content-Encoding": "zstd", | ||
| 83 | }; | ||
| 84 | } else if (acceptEncoding.includes("gzip") && asset.gzip) { | ||
| 85 | body = subarrayAsset(asset.gzip); | ||
| 86 | headers = { | ||
| 87 | ...asset.headers, | ||
| 88 | "Content-Encoding": "gzip", | ||
| 89 | }; | ||
| 90 | } else { | ||
| 91 | body = subarrayAsset(asset.raw); | ||
| 92 | } | ||
| 93 | c.res = new Response(body, { headers, status }); | ||
| 94 | } | ||
| 95 | |||
| 96 | import * as fs from "./fs.ts"; | ||
| 97 | import type { Context, Next } from "hono"; | ||
| 98 | import type { StatusCode } from "hono/utils/http-status"; | ||
| 99 | import type { BuiltAsset, BuiltAssetMap, View } from "./incremental.ts"; | ||
| 100 | import { Buffer } from "node:buffer"; | ||
framework/backend/entry-node.ts created+32| ... | @@ -0,0 +1,32 @@ | ||
| 1 | import "@paperclover/console/inject"; | ||
| 2 | |||
| 3 | const protocol = "http"; | ||
| 4 | |||
| 5 | const server = serve(app, ({ address, port }) => { | ||
| 6 | if (address === "::") address = "::1"; | ||
| 7 | console.info(url.format({ | ||
| 8 | protocol, | ||
| 9 | hostname: address, | ||
| 10 | port, | ||
| 11 | })); | ||
| 12 | }); | ||
| 13 | |||
| 14 | process.on("SIGINT", () => { | ||
| 15 | server.close(); | ||
| 16 | process.exit(0); | ||
| 17 | }); | ||
| 18 | |||
| 19 | process.on("SIGTERM", () => { | ||
| 20 | server.close((err) => { | ||
| 21 | if (err) { | ||
| 22 | console.error(err); | ||
| 23 | process.exit(1); | ||
| 24 | } | ||
| 25 | process.exit(0); | ||
| 26 | }); | ||
| 27 | }); | ||
| 28 | |||
| 29 | import app from "#backend"; | ||
| 30 | import url from "node:url"; | ||
| 31 | import { serve } from "@hono/node-server"; | ||
| 32 | import process from "node:process"; | ||
framework/bundle.ts+17-4| ... | @@ -49,7 +49,7 @@ export async function bundleClientJavaScript( | ... | @@ -49,7 +49,7 @@ export async function bundleClientJavaScript( |
| 49 | const publicScriptRoutes = extraPublicScripts.map((file) => | 49 | const publicScriptRoutes = extraPublicScripts.map((file) => |
| 50 | path.basename(file).replace(/\.client\.[tj]sx?/, "") | 50 | path.basename(file).replace(/\.client\.[tj]sx?/, "") |
| 51 | ); | 51 | ); |
| 52 | const promises: Promise<unknown>[] = []; | 52 | const promises: Promise<void>[] = []; |
| 53 | // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes. | 53 | // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes. |
| 54 | for (const file of bundle.outputFiles) { | 54 | for (const file of bundle.outputFiles) { |
| 55 | let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/"); | 55 | let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/"); |
| ... | @@ -73,10 +73,23 @@ export async function bundleClientJavaScript( | ... | @@ -73,10 +73,23 @@ export async function bundleClientJavaScript( |
| 73 | })); | 73 | })); |
| 74 | } | 74 | } |
| 75 | } | 75 | } |
| 76 | await Promise.all(promises); | ||
| 77 | } | ||
| 76 | 78 | ||
| 77 | if (promises.length > 0) { | 79 | export async function bundleServerJavaScript(entryPoint: string) { |
| 78 | await Promise.all(promises); | 80 | const bundle = await esbuild.build({ |
| 79 | } | 81 | bundle: true, |
| 82 | chunkNames: "/js/c.[hash]", | ||
| 83 | entryNames: "/js/[name]", | ||
| 84 | assetNames: "/asset/[hash]", | ||
| 85 | entryPoints: [entryPoint], | ||
| 86 | format: "esm", | ||
| 87 | minify: true, | ||
| 88 | outdir: "/out!", | ||
| 89 | plugins, | ||
| 90 | splitting: true, | ||
| 91 | write: false, | ||
| 92 | }); | ||
| 80 | } | 93 | } |
| 81 | 94 | ||
| 82 | import * as esbuild from "esbuild"; | 95 | import * as esbuild from "esbuild"; |
framework/fs.ts+1-1| ... | @@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) { | ... | @@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) { |
| 34 | 34 | ||
| 35 | export function readDirRecOptionalSync(dir: string) { | 35 | export function readDirRecOptionalSync(dir: string) { |
| 36 | try { | 36 | try { |
| 37 | return readdirSync(dir, { withFileTypes: true }); | 37 | return readdirSync(dir, { recursive: true, withFileTypes: true }); |
| 38 | } catch (err: any) { | 38 | } catch (err: any) { |
| 39 | if (err.code === "ENOENT") return []; | 39 | if (err.code === "ENOENT") return []; |
| 40 | throw err; | 40 | throw err; |
framework/generate.tsx created+299| ... | @@ -0,0 +1,299 @@ | ||
| 1 | export function main() { | ||
| 2 | return withSpinner({ | ||
| 3 | text: "Recovering State", | ||
| 4 | successText: ({ elapsed }) => | ||
| 5 | "sitegen! update in " + elapsed.toFixed(1) + "s", | ||
| 6 | failureText: () => "sitegen FAIL", | ||
| 7 | }, sitegen); | ||
| 8 | } | ||
| 9 | |||
| 10 | /** | ||
| 11 | * A filesystem object associated with some ID, | ||
| 12 | * such as a page's route to it's source file. | ||
| 13 | */ | ||
| 14 | interface FileItem { | ||
| 15 | id: string; | ||
| 16 | file: string; | ||
| 17 | } | ||
| 18 | |||
| 19 | async function sitegen(status: Spinner) { | ||
| 20 | const startTime = performance.now(); | ||
| 21 | |||
| 22 | let root = path.resolve(import.meta.dirname, "../src"); | ||
| 23 | const join = (...sub: string[]) => path.join(root, ...sub); | ||
| 24 | const incr = new Incremental(); | ||
| 25 | |||
| 26 | // Sitegen reviews every defined section for resources to process | ||
| 27 | const sections: sg.Section[] = | ||
| 28 | require(path.join(root, "site.ts")).siteSections; | ||
| 29 | |||
| 30 | // Static files are compressed and served as-is. | ||
| 31 | // - "{section}/static/*.png" | ||
| 32 | let staticFiles: FileItem[] = []; | ||
| 33 | // Pages are rendered then served as static files. | ||
| 34 | // - "{section}/pages/*.marko" | ||
| 35 | let pages: FileItem[] = []; | ||
| 36 | // Views are dynamically rendered pages called via backend code. | ||
| 37 | // - "{section}/views/*.tsx" | ||
| 38 | let views: FileItem[] = []; | ||
| 39 | // Public scripts are bundled for the client as static assets under "/js/[...]" | ||
| 40 | // This is used for the file viewer's canvases. | ||
| 41 | // Note that '.client.ts' can be placed anywhere in the file structure. | ||
| 42 | // - "{section}/scripts/*.client.ts" | ||
| 43 | let scripts: FileItem[] = []; | ||
| 44 | |||
| 45 | // -- Scan for files -- | ||
| 46 | status.text = "Scanning Project"; | ||
| 47 | for (const section of sections) { | ||
| 48 | const { root: sectionRoot } = section; | ||
| 49 | const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); | ||
| 50 | const rootPrefix = root === sectionRoot | ||
| 51 | ? "" | ||
| 52 | : path.relative(root, sectionRoot) + "/"; | ||
| 53 | const kinds = [ | ||
| 54 | { | ||
| 55 | dir: sectionPath("pages"), | ||
| 56 | list: pages, | ||
| 57 | prefix: "/", | ||
| 58 | exclude: [".css", ".client.ts", ".client.tsx"], | ||
| 59 | }, | ||
| 60 | { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, | ||
| 61 | { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, | ||
| 62 | { | ||
| 63 | dir: sectionPath("views"), | ||
| 64 | list: views, | ||
| 65 | prefix: rootPrefix, | ||
| 66 | exclude: [".css", ".client.ts", ".client.tsx"], | ||
| 67 | }, | ||
| 68 | ]; | ||
| 69 | for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { | ||
| 70 | const items = fs.readDirRecOptionalSync(dir); | ||
| 71 | item: for (const item of items) { | ||
| 72 | if (item.isDirectory()) continue; | ||
| 73 | for (const e of exclude) { | ||
| 74 | if (item.name.endsWith(e)) continue item; | ||
| 75 | } | ||
| 76 | const file = path.relative(dir, item.parentPath + "/" + item.name); | ||
| 77 | const trim = ext | ||
| 78 | ? file | ||
| 79 | : file.slice(0, -path.extname(file).length).replaceAll(".", "/"); | ||
| 80 | let id = prefix + trim.replaceAll("\\", "/"); | ||
| 81 | if (prefix === "/" && id.endsWith("/index")) { | ||
| 82 | id = id.slice(0, -"/index".length) || "/"; | ||
| 83 | } | ||
| 84 | list.push({ id, file: path.join(item.parentPath, item.name) }); | ||
| 85 | } | ||
| 86 | } | ||
| 87 | } | ||
| 88 | scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); | ||
| 89 | const globalCssPath = join("global.css"); | ||
| 90 | |||
| 91 | // TODO: invalidate incremental resources | ||
| 92 | |||
| 93 | // -- server side render -- | ||
| 94 | status.text = "Building"; | ||
| 95 | const cssOnce = new OnceMap<string>(); | ||
| 96 | const cssQueue = new Queue<[string, string[], css.Theme], string>({ | ||
| 97 | name: "Bundle", | ||
| 98 | fn: ([, files, theme]) => css.bundleCssFiles(files, theme), | ||
| 99 | passive: true, | ||
| 100 | getItemText: ([id]) => id, | ||
| 101 | maxJobs: 2, | ||
| 102 | }); | ||
| 103 | interface RenderResult { | ||
| 104 | body: string; | ||
| 105 | head: string; | ||
| 106 | inlineCss: string; | ||
| 107 | scriptFiles: string[]; | ||
| 108 | item: FileItem; | ||
| 109 | } | ||
| 110 | const renderResults: RenderResult[] = []; | ||
| 111 | async function loadPageModule({ file }: FileItem) { | ||
| 112 | require(file); | ||
| 113 | } | ||
| 114 | async function renderPage(item: FileItem) { | ||
| 115 | // -- load and validate module -- | ||
| 116 | let { default: Page, meta: metadata, theme: pageTheme, layout } = require( | ||
| 117 | item.file, | ||
| 118 | ); | ||
| 119 | if (!Page) throw new Error("Page is missing a 'default' export."); | ||
| 120 | if (!metadata) { | ||
| 121 | throw new Error("Page is missing 'meta' export with a title."); | ||
| 122 | } | ||
| 123 | if (layout?.theme) pageTheme = layout.theme; | ||
| 124 | const theme = { | ||
| 125 | bg: "#fff", | ||
| 126 | fg: "#050505", | ||
| 127 | primary: "#2e7dab", | ||
| 128 | ...pageTheme, | ||
| 129 | }; | ||
| 130 | |||
| 131 | // -- metadata -- | ||
| 132 | const renderedMetaPromise = Promise.resolve( | ||
| 133 | typeof metadata === "function" ? metadata({ ssr: true }) : metadata, | ||
| 134 | ).then((m) => meta.renderMeta(m)); | ||
| 135 | // -- css -- | ||
| 136 | const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; | ||
| 137 | const cssPromise = cssOnce.get( | ||
| 138 | cssImports.join(":") + JSON.stringify(theme), | ||
| 139 | () => cssQueue.add([item.id, cssImports, theme]), | ||
| 140 | ); | ||
| 141 | // -- html -- | ||
| 142 | const bodyPromise = await ssr.ssrAsync(<Page />, { | ||
| 143 | sitegen: sg.initRender(), | ||
| 144 | }); | ||
| 145 | |||
| 146 | const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([ | ||
| 147 | bodyPromise, | ||
| 148 | cssPromise, | ||
| 149 | renderedMetaPromise, | ||
| 150 | ]); | ||
| 151 | if (!renderedMeta.includes("<title>")) { | ||
| 152 | throw new Error( | ||
| 153 | "Page is missing 'meta.title'. " + | ||
| 154 | "All pages need a title tag.", | ||
| 155 | ); | ||
| 156 | } | ||
| 157 | // The script content is not ready, allow another page to Render. The page | ||
| 158 | // contents will be rebuilt at the end. This is more convenient anyways | ||
| 159 | // because it means client scripts don't re-render the page. | ||
| 160 | renderResults.push({ | ||
| 161 | body: text, | ||
| 162 | head: renderedMeta, | ||
| 163 | inlineCss, | ||
| 164 | scriptFiles: Array.from(addon.sitegen.scripts), | ||
| 165 | item: item, | ||
| 166 | }); | ||
| 167 | } | ||
| 168 | // This is done in two passes so that a page that throws during evaluation | ||
| 169 | // will report "Load Render Module" instead of "Render Static Page". | ||
| 170 | const spinnerFormat = status.format; | ||
| 171 | status.format = () => ""; | ||
| 172 | const moduleLoadQueue = new Queue({ | ||
| 173 | name: "Load Render Module", | ||
| 174 | fn: loadPageModule, | ||
| 175 | getItemText, | ||
| 176 | maxJobs: 1, | ||
| 177 | }); | ||
| 178 | moduleLoadQueue.addMany(pages); | ||
| 179 | await moduleLoadQueue.done({ method: "stop" }); | ||
| 180 | const pageQueue = new Queue({ | ||
| 181 | name: "Render Static Page", | ||
| 182 | fn: renderPage, | ||
| 183 | getItemText, | ||
| 184 | maxJobs: 2, | ||
| 185 | }); | ||
| 186 | pageQueue.addMany(pages); | ||
| 187 | await pageQueue.done({ method: "stop" }); | ||
| 188 | status.format = spinnerFormat; | ||
| 189 | |||
| 190 | // -- bundle scripts -- | ||
| 191 | const referencedScripts = Array.from( | ||
| 192 | new Set(renderResults.flatMap((r) => r.scriptFiles)), | ||
| 193 | ); | ||
| 194 | const extraPublicScripts = scripts.map((entry) => entry.file); | ||
| 195 | const uniqueCount = new Set([ | ||
| 196 | ...referencedScripts, | ||
| 197 | ...extraPublicScripts, | ||
| 198 | ]).size; | ||
| 199 | status.text = `Bundle ${uniqueCount} Scripts`; | ||
| 200 | await bundle.bundleClientJavaScript( | ||
| 201 | referencedScripts, | ||
| 202 | extraPublicScripts, | ||
| 203 | incr, | ||
| 204 | ); | ||
| 205 | |||
| 206 | // -- copy/compress static files -- | ||
| 207 | async function doStaticFile(item: FileItem) { | ||
| 208 | const body = await fs.readFile(item.file); | ||
| 209 | await incr.putAsset({ | ||
| 210 | srcId: "static:" + item.file, | ||
| 211 | key: item.id, | ||
| 212 | body, | ||
| 213 | }); | ||
| 214 | } | ||
| 215 | const staticQueue = new Queue({ | ||
| 216 | name: "Load Static", | ||
| 217 | fn: doStaticFile, | ||
| 218 | getItemText, | ||
| 219 | maxJobs: 16, | ||
| 220 | }); | ||
| 221 | status.format = () => ""; | ||
| 222 | staticQueue.addMany(staticFiles); | ||
| 223 | await staticQueue.done({ method: "stop" }); | ||
| 224 | status.format = spinnerFormat; | ||
| 225 | |||
| 226 | // -- concatenate static rendered pages -- | ||
| 227 | status.text = `Concat ${renderResults.length} Pages`; | ||
| 228 | await Promise.all( | ||
| 229 | renderResults.map( | ||
| 230 | async ({ item: page, body, head, inlineCss, scriptFiles }) => { | ||
| 231 | const doc = wrapDocument({ | ||
| 232 | body, | ||
| 233 | head, | ||
| 234 | inlineCss, | ||
| 235 | scripts: scriptFiles.map( | ||
| 236 | (id) => | ||
| 237 | UNWRAP( | ||
| 238 | incr.out.script.get( | ||
| 239 | path.basename(id).replace(/\.client\.[jt]sx?$/, ""), | ||
| 240 | ), | ||
| 241 | ), | ||
| 242 | ).map((x) => `{${x}}`).join("\n"), | ||
| 243 | }); | ||
| 244 | await incr.putAsset({ | ||
| 245 | srcId: "page:" + page.file, | ||
| 246 | key: page.id, | ||
| 247 | body: doc, | ||
| 248 | headers: { | ||
| 249 | "Content-Type": "text/html", | ||
| 250 | }, | ||
| 251 | }); | ||
| 252 | }, | ||
| 253 | ), | ||
| 254 | ); | ||
| 255 | status.format = () => ""; | ||
| 256 | status.text = ``; | ||
| 257 | // This will wait for all compression jobs to finish, which up | ||
| 258 | // to this point have been left as dangling promises. | ||
| 259 | await incr.wait(); | ||
| 260 | |||
| 261 | // Flush the site to disk. | ||
| 262 | status.format = spinnerFormat; | ||
| 263 | status.text = `Incremental Flush`; | ||
| 264 | incr.flush(); | ||
| 265 | incr.serializeToDisk(); // Allows picking up this state again | ||
| 266 | return { elapsed: (performance.now() - startTime) / 1000 }; | ||
| 267 | } | ||
| 268 | |||
| 269 | function getItemText({ file }: FileItem) { | ||
| 270 | return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); | ||
| 271 | } | ||
| 272 | |||
| 273 | function wrapDocument({ | ||
| 274 | body, | ||
| 275 | head, | ||
| 276 | inlineCss, | ||
| 277 | scripts, | ||
| 278 | }: { | ||
| 279 | head: string; | ||
| 280 | body: string; | ||
| 281 | inlineCss: string; | ||
| 282 | scripts: string; | ||
| 283 | }) { | ||
| 284 | return `<!doctype html><head>${head}${ | ||
| 285 | inlineCss ? `<style>${inlineCss}</style>` : "" | ||
| 286 | }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`; | ||
| 287 | } | ||
| 288 | |||
| 289 | import { OnceMap, Queue } from "./queue.ts"; | ||
| 290 | import { Incremental } from "./incremental.ts"; | ||
| 291 | import * as bundle from "./bundle.ts"; | ||
| 292 | import * as css from "./css.ts"; | ||
| 293 | import * as fs from "./fs.ts"; | ||
| 294 | import { Spinner, withSpinner } from "@paperclover/console/Spinner"; | ||
| 295 | import * as meta from "./meta.ts"; | ||
| 296 | import * as ssr from "./engine/ssr.ts"; | ||
| 297 | import * as sg from "#sitegen"; | ||
| 298 | import * as hot from "./hot.ts"; | ||
| 299 | import * as path from "node:path"; | ||
framework/hot.ts+1-2| ... | @@ -146,6 +146,7 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { | ... | @@ -146,6 +146,7 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { |
| 146 | target: "esnext", | 146 | target: "esnext", |
| 147 | jsx: "automatic", | 147 | jsx: "automatic", |
| 148 | jsxImportSource: "#ssr", | 148 | jsxImportSource: "#ssr", |
| 149 | sourcefile: filepath, | ||
| 149 | }).code; | 150 | }).code; |
| 150 | return module._compile(src, filepath, "commonjs"); | 151 | return module._compile(src, filepath, "commonjs"); |
| 151 | } | 152 | } |
| ... | @@ -162,8 +163,6 @@ function loadMarko(module: NodeJS.Module, filepath: string) { | ... | @@ -162,8 +163,6 @@ function loadMarko(module: NodeJS.Module, filepath: string) { |
| 162 | ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n'; | 163 | ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n'; |
| 163 | } | 164 | } |
| 164 | 165 | ||
| 165 | console.log(src); | ||
| 166 | console.log("---"); | ||
| 167 | src = marko.compileSync(src, filepath).code; | 166 | src = marko.compileSync(src, filepath).code; |
| 168 | src = src.replace("marko/debug/html", "#ssr/marko"); | 167 | src = src.replace("marko/debug/html", "#ssr/marko"); |
| 169 | return loadEsbuildCode(module, filepath, src); | 168 | return loadEsbuildCode(module, filepath, src); |
framework/incremental.ts+10-13| ... | @@ -7,7 +7,6 @@ | ... | @@ -7,7 +7,6 @@ |
| 7 | // all its imports) to map to the build of a page, which produces an HTML file | 7 | // all its imports) to map to the build of a page, which produces an HTML file |
| 8 | // plus a list of scripts. | 8 | // plus a list of scripts. |
| 9 | 9 | ||
| 10 | import { Buffer } from "node:buffer"; | ||
| 11 | interface ArtifactMap { | 10 | interface ArtifactMap { |
| 12 | asset: Asset; | 11 | asset: Asset; |
| 13 | script: string; | 12 | script: string; |
| ... | @@ -126,18 +125,17 @@ export class Incremental { | ... | @@ -126,18 +125,17 @@ export class Incremental { |
| 126 | }, | 125 | }, |
| 127 | hash, | 126 | hash, |
| 128 | }; | 127 | }; |
| 128 | const a = this.put({ ...info, type: "asset", value }); | ||
| 129 | if (!this.compress.has(hash)) { | 129 | if (!this.compress.has(hash)) { |
| 130 | const label = info.key; | 130 | const label = info.key; |
| 131 | this.compress.set(hash, { | 131 | this.compress.set(hash, { |
| 132 | zstd: undefined, | 132 | zstd: undefined, |
| 133 | gzip: undefined, | 133 | gzip: undefined, |
| 134 | }); | 134 | }); |
| 135 | await Promise.all([ | 135 | this.compressQueue.add({ label, buffer, algo: "zstd", hash }); |
| 136 | this.compressQueue.add({ label, buffer, algo: "zstd", hash }), | 136 | this.compressQueue.add({ label, buffer, algo: "gzip", hash }); |
| 137 | this.compressQueue.add({ label, buffer, algo: "gzip", hash }), | ||
| 138 | ]); | ||
| 139 | } | 137 | } |
| 140 | return this.put({ ...info, type: "asset", value }); | 138 | return a; |
| 141 | } | 139 | } |
| 142 | 140 | ||
| 143 | async compressImpl({ algo, buffer, hash }: CompressJob) { | 141 | async compressImpl({ algo, buffer, hash }: CompressJob) { |
| ... | @@ -167,16 +165,18 @@ export class Incremental { | ... | @@ -167,16 +165,18 @@ export class Incremental { |
| 167 | } | 165 | } |
| 168 | 166 | ||
| 169 | async wait() { | 167 | async wait() { |
| 170 | await this.compressQueue.done({ method: "stop" }); | 168 | await this.compressQueue.done({ method: "success" }); |
| 171 | } | 169 | } |
| 170 | |||
| 172 | async flush() { | 171 | async flush() { |
| 172 | ASSERT(!this.compressQueue.active); | ||
| 173 | const writer = new BufferWriter(); | 173 | const writer = new BufferWriter(); |
| 174 | const asset = Object.fromEntries( | 174 | const asset = Object.fromEntries( |
| 175 | Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => { | 175 | Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => { |
| 176 | const raw = writer.write(buffer, hash); | 176 | const raw = writer.write(buffer, hash); |
| 177 | const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {}; | 177 | const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {}; |
| 178 | const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null; | 178 | const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null; |
| 179 | const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null; | 179 | const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null; |
| 180 | return [key, { | 180 | return [key, { |
| 181 | raw, | 181 | raw, |
| 182 | gzip, | 182 | gzip, |
| ... | @@ -252,10 +252,6 @@ export interface SerializedMeta { | ... | @@ -252,10 +252,6 @@ export interface SerializedMeta { |
| 252 | script: [key: string, value: string][]; | 252 | script: [key: string, value: string][]; |
| 253 | } | 253 | } |
| 254 | 254 | ||
| 255 | function never(): never { | ||
| 256 | throw new Error("Impossible"); | ||
| 257 | } | ||
| 258 | |||
| 259 | import * as path from "node:path"; | 255 | import * as path from "node:path"; |
| 260 | import * as fs from "./fs.ts"; | 256 | import * as fs from "./fs.ts"; |
| 261 | import * as zlib from "node:zlib"; | 257 | import * as zlib from "node:zlib"; |
| ... | @@ -263,3 +259,4 @@ import * as util from "node:util"; | ... | @@ -263,3 +259,4 @@ import * as util from "node:util"; |
| 263 | import { Queue } from "./queue.ts"; | 259 | import { Queue } from "./queue.ts"; |
| 264 | import * as hot from "./hot.ts"; | 260 | import * as hot from "./hot.ts"; |
| 265 | import * as mime from "./mime.ts"; | 261 | import * as mime from "./mime.ts"; |
| 262 | import { Buffer } from "node:buffer"; |
framework/lib/assets.ts created+112| ... | @@ -0,0 +1,112 @@ | ||
| 1 | interface Loaded { | ||
| 2 | map: BuiltAssetMap; | ||
| 3 | buf: Buffer; | ||
| 4 | } | ||
| 5 | let assets: Loaded | null = null; | ||
| 6 | |||
| 7 | export type StaticPageId = string; | ||
| 8 | |||
| 9 | export async function reload() { | ||
| 10 | const [map, buf] = await Promise.all([ | ||
| 11 | fs.readFile(".clover/static.json", "utf8"), | ||
| 12 | fs.readFile(".clover/static.blob"), | ||
| 13 | ]); | ||
| 14 | assets = { | ||
| 15 | map: JSON.parse(map), | ||
| 16 | buf, | ||
| 17 | }; | ||
| 18 | } | ||
| 19 | |||
| 20 | export async function reloadSync() { | ||
| 21 | const map = fs.readFileSync(".clover/static.json", "utf8"); | ||
| 22 | const buf = fs.readFileSync(".clover/static.blob"); | ||
| 23 | assets = { | ||
| 24 | map: JSON.parse(map), | ||
| 25 | buf, | ||
| 26 | }; | ||
| 27 | } | ||
| 28 | |||
| 29 | export async function middleware(c: Context, next: Next) { | ||
| 30 | if (!assets) await reload(); | ||
| 31 | const asset = assets!.map[c.req.path]; | ||
| 32 | if (asset) { | ||
| 33 | return assetInner(c, asset, 200); | ||
| 34 | } | ||
| 35 | return next(); | ||
| 36 | } | ||
| 37 | |||
| 38 | export async function notFound(c: Context) { | ||
| 39 | if (!assets) await reload(); | ||
| 40 | let pathname = c.req.path; | ||
| 41 | do { | ||
| 42 | const asset = assets!.map[pathname + "/404"]; | ||
| 43 | if (asset) return assetInner(c, asset, 404); | ||
| 44 | pathname = pathname.slice(0, pathname.lastIndexOf("/")); | ||
| 45 | } while (pathname); | ||
| 46 | const asset = assets!.map["/404"]; | ||
| 47 | if (asset) return assetInner(c, asset, 404); | ||
| 48 | return c.text("the 'Not Found' page was not found", 404); | ||
| 49 | } | ||
| 50 | |||
| 51 | export async function serveAsset( | ||
| 52 | c: Context, | ||
| 53 | id: StaticPageId, | ||
| 54 | status: StatusCode, | ||
| 55 | ) { | ||
| 56 | assets ?? await reload(); | ||
| 57 | return assetInner(c, assets!.map[id], status); | ||
| 58 | } | ||
| 59 | |||
| 60 | export function hasAsset(id: string) { | ||
| 61 | if (!assets) reloadSync(); | ||
| 62 | return assets!.map[id] !== undefined; | ||
| 63 | } | ||
| 64 | |||
| 65 | export function etagMatches(etag: string, ifNoneMatch: string) { | ||
| 66 | return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; | ||
| 67 | } | ||
| 68 | |||
| 69 | function subarrayAsset([start, end]: View) { | ||
| 70 | return assets!.buf.subarray(start, end); | ||
| 71 | } | ||
| 72 | |||
| 73 | function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { | ||
| 74 | const ifnonematch = c.req.header("If-None-Match"); | ||
| 75 | if (ifnonematch) { | ||
| 76 | const etag = asset.headers.ETag; | ||
| 77 | if (etagMatches(etag, ifnonematch)) { | ||
| 78 | return c.res = new Response(null, { | ||
| 79 | status: 304, | ||
| 80 | statusText: "Not Modified", | ||
| 81 | headers: { | ||
| 82 | ETag: etag, | ||
| 83 | }, | ||
| 84 | }); | ||
| 85 | } | ||
| 86 | } | ||
| 87 | const acceptEncoding = c.req.header("Accept-Encoding") ?? ""; | ||
| 88 | let body; | ||
| 89 | let headers = asset.headers; | ||
| 90 | if (acceptEncoding.includes("zstd") && asset.zstd) { | ||
| 91 | body = subarrayAsset(asset.zstd); | ||
| 92 | headers = { | ||
| 93 | ...asset.headers, | ||
| 94 | "Content-Encoding": "zstd", | ||
| 95 | }; | ||
| 96 | } else if (acceptEncoding.includes("gzip") && asset.gzip) { | ||
| 97 | body = subarrayAsset(asset.gzip); | ||
| 98 | headers = { | ||
| 99 | ...asset.headers, | ||
| 100 | "Content-Encoding": "gzip", | ||
| 101 | }; | ||
| 102 | } else { | ||
| 103 | body = subarrayAsset(asset.raw); | ||
| 104 | } | ||
| 105 | return c.res = new Response(body, { headers, status }); | ||
| 106 | } | ||
| 107 | |||
| 108 | import * as fs from "../fs.ts"; | ||
| 109 | import type { Context, Next } from "hono"; | ||
| 110 | import type { StatusCode } from "hono/utils/http-status"; | ||
| 111 | import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts"; | ||
| 112 | import { Buffer } from "node:buffer"; | ||
framework/lib/sitegen.ts created+52| ... | @@ -0,0 +1,52 @@ | ||
| 1 | // Import this file with 'import * as sg from "#sitegen";' | ||
| 2 | export type ScriptId = string; | ||
| 3 | |||
| 4 | const frameworkDir = path.dirname(import.meta.dirname); | ||
| 5 | |||
| 6 | export interface SitegenRender { | ||
| 7 | scripts: Set<ScriptId>; | ||
| 8 | } | ||
| 9 | |||
| 10 | export function initRender(): SitegenRender { | ||
| 11 | return { | ||
| 12 | scripts: new Set(), | ||
| 13 | }; | ||
| 14 | } | ||
| 15 | |||
| 16 | export function getRender() { | ||
| 17 | return ssr.getUserData<SitegenRender>("sitegen", () => { | ||
| 18 | throw new Error( | ||
| 19 | "This function can only be used in a page (static or view)", | ||
| 20 | ); | ||
| 21 | }); | ||
| 22 | } | ||
| 23 | |||
| 24 | /** Add a client-side script to the page. */ | ||
| 25 | export function addScript(id: ScriptId) { | ||
| 26 | const srcFile: string = util.getCallSites() | ||
| 27 | .find((site) => !site.scriptName.startsWith(frameworkDir))! | ||
| 28 | .scriptName; | ||
| 29 | const filePath = hot.resolveFrom(srcFile, id); | ||
| 30 | if ( | ||
| 31 | !filePath.endsWith(".client.ts") && | ||
| 32 | !filePath.endsWith(".client.tsx") | ||
| 33 | ) { | ||
| 34 | throw new Error("addScript must be a .client.ts or .client.tsx"); | ||
| 35 | } | ||
| 36 | getRender().scripts.add(filePath); | ||
| 37 | } | ||
| 38 | |||
| 39 | export function Script({ src }: { src: ScriptId }) { | ||
| 40 | if (!src) throw new Error("Missing 'src' attribute"); | ||
| 41 | addScript(src); | ||
| 42 | return null; | ||
| 43 | } | ||
| 44 | |||
| 45 | export interface Section { | ||
| 46 | root: string; | ||
| 47 | } | ||
| 48 | |||
| 49 | import * as ssr from "../engine/ssr.ts"; | ||
| 50 | import * as util from "node:util"; | ||
| 51 | import * as hot from "../hot.ts"; | ||
| 52 | import * as path from "node:path"; | ||
framework/queue.ts+4| ... | @@ -165,6 +165,10 @@ export class Queue<T, R> { | ... | @@ -165,6 +165,10 @@ export class Queue<T, R> { |
| 165 | 165 | ||
| 166 | if (bar) bar[method](); | 166 | if (bar) bar[method](); |
| 167 | } | 167 | } |
| 168 | |||
| 169 | get active(): boolean { | ||
| 170 | return this.#active.length !== 0; | ||
| 171 | } | ||
| 168 | } | 172 | } |
| 169 | 173 | ||
| 170 | const cwd = process.cwd(); | 174 | const cwd = process.cwd(); |
framework/sitegen-lib.ts deleted-49| ... | @@ -1,49 +0,0 @@ | ||
| 1 | // Import this file with 'import * as sg from "#sitegen";' | ||
| 2 | export type ScriptId = string; | ||
| 3 | |||
| 4 | export interface SitegenRender { | ||
| 5 | scripts: Set<ScriptId>; | ||
| 6 | } | ||
| 7 | |||
| 8 | export function initRender(): SitegenRender { | ||
| 9 | return { | ||
| 10 | scripts: new Set(), | ||
| 11 | }; | ||
| 12 | } | ||
| 13 | |||
| 14 | export function getRender() { | ||
| 15 | return ssr.getUserData<SitegenRender>("sitegen", () => { | ||
| 16 | throw new Error( | ||
| 17 | "This function can only be used in a page (static or view)", | ||
| 18 | ); | ||
| 19 | }); | ||
| 20 | } | ||
| 21 | |||
| 22 | /** Add a client-side script to the page. */ | ||
| 23 | export function addScript(id: ScriptId) { | ||
| 24 | const srcFile: string = util.getCallSites() | ||
| 25 | .find((site) => !site.scriptName.startsWith(import.meta.dirname))! | ||
| 26 | .scriptName; | ||
| 27 | const filePath = hot.resolveFrom(srcFile, id); | ||
| 28 | if ( | ||
| 29 | !filePath.endsWith(".client.ts") && | ||
| 30 | !filePath.endsWith(".client.tsx") | ||
| 31 | ) { | ||
| 32 | throw new Error("addScript must be a .client.ts or .client.tsx"); | ||
| 33 | } | ||
| 34 | getRender().scripts.add(filePath); | ||
| 35 | } | ||
| 36 | |||
| 37 | export function Script({ src }: { src: ScriptId }) { | ||
| 38 | if (!src) throw new Error("Missing 'src' attribute"); | ||
| 39 | addScript(src); | ||
| 40 | return null; | ||
| 41 | } | ||
| 42 | |||
| 43 | export interface Section { | ||
| 44 | root: string; | ||
| 45 | } | ||
| 46 | |||
| 47 | import * as ssr from "./engine/ssr.ts"; | ||
| 48 | import * as util from "node:util"; | ||
| 49 | import * as hot from "./hot.ts"; | ||
framework/sitegen.tsx deleted-309| ... | @@ -1,309 +0,0 @@ | ||
| 1 | // Sitegen! Clover's static site generator, built with love. | ||
| 2 | |||
| 3 | export function main() { | ||
| 4 | return withSpinner({ | ||
| 5 | text: "Recovering State", | ||
| 6 | successText: ({ elapsed }) => | ||
| 7 | "sitegen! update in " + elapsed.toFixed(1) + "s", | ||
| 8 | failureText: () => "sitegen FAIL", | ||
| 9 | }, sitegen); | ||
| 10 | } | ||
| 11 | |||
| 12 | /** | ||
| 13 | * A filesystem object associated with some ID, | ||
| 14 | * such as a page's route to it's source file. | ||
| 15 | */ | ||
| 16 | interface FileItem { | ||
| 17 | id: string; | ||
| 18 | file: string; | ||
| 19 | } | ||
| 20 | |||
| 21 | async function sitegen(status: Spinner) { | ||
| 22 | const startTime = performance.now(); | ||
| 23 | |||
| 24 | let root = path.resolve(import.meta.dirname, "../src"); | ||
| 25 | const join = (...sub: string[]) => path.join(root, ...sub); | ||
| 26 | const incr = new Incremental(); | ||
| 27 | |||
| 28 | // Sitegen reviews every defined section for resources to process | ||
| 29 | const sections: Section[] = | ||
| 30 | require(path.join(root, "sections.ts")).siteSections; | ||
| 31 | |||
| 32 | // Static files are compressed and served as-is. | ||
| 33 | // - "{section}/static/*.png" | ||
| 34 | let staticFiles: FileItem[] = []; | ||
| 35 | // Pages are rendered then served as static files. | ||
| 36 | // - "{section}/pages/*.marko" | ||
| 37 | let pages: FileItem[] = []; | ||
| 38 | // Views are dynamically rendered pages called via backend code. | ||
| 39 | // - "{section}/views/*.tsx" | ||
| 40 | let views: FileItem[] = []; | ||
| 41 | // Public scripts are bundled for the client as static assets under "/js/[...]" | ||
| 42 | // This is used for the file viewer's canvases. | ||
| 43 | // Note that '.client.ts' can be placed anywhere in the file structure. | ||
| 44 | // - "{section}/scripts/*.client.ts" | ||
| 45 | let scripts: FileItem[] = []; | ||
| 46 | // 'backend.ts' | ||
| 47 | const backendFiles = []; | ||
| 48 | |||
| 49 | // -- Scan for files -- | ||
| 50 | status.text = "Scanning Project"; | ||
| 51 | for (const section of sections) { | ||
| 52 | const { root: sectionRoot } = section; | ||
| 53 | const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); | ||
| 54 | const rootPrefix = root === sectionRoot | ||
| 55 | ? "" | ||
| 56 | : path.relative(root, sectionRoot) + "/"; | ||
| 57 | const kinds = [ | ||
| 58 | { | ||
| 59 | dir: sectionPath("pages"), | ||
| 60 | list: pages, | ||
| 61 | prefix: "/", | ||
| 62 | exclude: [".css", ".client.ts", ".client.tsx"], | ||
| 63 | }, | ||
| 64 | { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, | ||
| 65 | { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, | ||
| 66 | { | ||
| 67 | dir: sectionPath("views"), | ||
| 68 | list: views, | ||
| 69 | prefix: rootPrefix, | ||
| 70 | exclude: [".css", ".client.ts", ".client.tsx"], | ||
| 71 | }, | ||
| 72 | ]; | ||
| 73 | for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { | ||
| 74 | const items = fs.readDirRecOptionalSync(dir); | ||
| 75 | item: for (const item of items) { | ||
| 76 | if (item.isDirectory()) continue; | ||
| 77 | for (const e of exclude) { | ||
| 78 | if (item.name.endsWith(e)) continue item; | ||
| 79 | } | ||
| 80 | const file = path.relative(dir, item.parentPath + "/" + item.name); | ||
| 81 | const trim = ext | ||
| 82 | ? file | ||
| 83 | : file.slice(0, -path.extname(file).length).replaceAll(".", "/"); | ||
| 84 | let id = prefix + trim.replaceAll("\\", "/"); | ||
| 85 | if (prefix === "/" && id.endsWith("/index")) { | ||
| 86 | id = id.slice(0, -"/index".length) || "/"; | ||
| 87 | } | ||
| 88 | list.push({ id, file: path.join(item.parentPath, item.name) }); | ||
| 89 | } | ||
| 90 | } | ||
| 91 | let backendFile = [ | ||
| 92 | sectionPath("backend.ts"), | ||
| 93 | sectionPath("backend.tsx"), | ||
| 94 | ].find((file) => fs.existsSync(file)); | ||
| 95 | if (backendFile) backendFiles.push(backendFile); | ||
| 96 | } | ||
| 97 | scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); | ||
| 98 | const globalCssPath = join("global.css"); | ||
| 99 | |||
| 100 | // TODO: invalidate incremental resources | ||
| 101 | |||
| 102 | // -- server side render -- | ||
| 103 | status.text = "Building"; | ||
| 104 | const cssOnce = new OnceMap<string>(); | ||
| 105 | const cssQueue = new Queue<[string, string[], css.Theme], string>({ | ||
| 106 | name: "Bundle", | ||
| 107 | fn: ([, files, theme]) => css.bundleCssFiles(files, theme), | ||
| 108 | passive: true, | ||
| 109 | getItemText: ([id]) => id, | ||
| 110 | maxJobs: 2, | ||
| 111 | }); | ||
| 112 | interface RenderResult { | ||
| 113 | body: string; | ||
| 114 | head: string; | ||
| 115 | inlineCss: string; | ||
| 116 | scriptFiles: string[]; | ||
| 117 | item: FileItem; | ||
| 118 | } | ||
| 119 | const renderResults: RenderResult[] = []; | ||
| 120 | async function loadPageModule({ file }: FileItem) { | ||
| 121 | require(file); | ||
| 122 | } | ||
| 123 | async function renderPage(item: FileItem) { | ||
| 124 | // -- load and validate module -- | ||
| 125 | let { default: Page, meta: metadata, theme: pageTheme, layout } = require( | ||
| 126 | item.file, | ||
| 127 | ); | ||
| 128 | if (!Page) throw new Error("Page is missing a 'default' export."); | ||
| 129 | if (!metadata) { | ||
| 130 | throw new Error("Page is missing 'meta' export with a title."); | ||
| 131 | } | ||
| 132 | if (layout?.theme) pageTheme = layout.theme; | ||
| 133 | const theme = { | ||
| 134 | bg: "#fff", | ||
| 135 | fg: "#050505", | ||
| 136 | primary: "#2e7dab", | ||
| 137 | ...pageTheme, | ||
| 138 | }; | ||
| 139 | |||
| 140 | // -- metadata -- | ||
| 141 | const renderedMetaPromise = Promise.resolve( | ||
| 142 | typeof metadata === "function" ? metadata({ ssr: true }) : metadata, | ||
| 143 | ).then((m) => meta.renderMeta(m)); | ||
| 144 | // -- css -- | ||
| 145 | const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; | ||
| 146 | const cssPromise = cssOnce.get( | ||
| 147 | cssImports.join(":") + JSON.stringify(theme), | ||
| 148 | () => cssQueue.add([item.id, cssImports, theme]), | ||
| 149 | ); | ||
| 150 | // -- html -- | ||
| 151 | const bodyPromise = await ssr.ssrAsync(<Page />, { | ||
| 152 | sitegen: sg.initRender(), | ||
| 153 | }); | ||
| 154 | |||
| 155 | const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([ | ||
| 156 | bodyPromise, | ||
| 157 | cssPromise, | ||
| 158 | renderedMetaPromise, | ||
| 159 | ]); | ||
| 160 | if (!renderedMeta.includes("<title>")) { | ||
| 161 | throw new Error( | ||
| 162 | "Page is missing 'meta.title'. " + | ||
| 163 | "All pages need a title tag.", | ||
| 164 | ); | ||
| 165 | } | ||
| 166 | // The script content is not ready, allow another page to Render. The page | ||
| 167 | // contents will be rebuilt at the end. This is more convenient anyways | ||
| 168 | // because it means client scripts don't re-render the page. | ||
| 169 | renderResults.push({ | ||
| 170 | body: text, | ||
| 171 | head: renderedMeta, | ||
| 172 | inlineCss, | ||
| 173 | scriptFiles: Array.from(addon.sitegen.scripts), | ||
| 174 | item: item, | ||
| 175 | }); | ||
| 176 | } | ||
| 177 | // This is done in two passes so that a page that throws during evaluation | ||
| 178 | // will report "Load Render Module" instead of "Render Static Page". | ||
| 179 | const spinnerFormat = status.format; | ||
| 180 | status.format = () => ""; | ||
| 181 | const moduleLoadQueue = new Queue({ | ||
| 182 | name: "Load Render Module", | ||
| 183 | fn: loadPageModule, | ||
| 184 | getItemText, | ||
| 185 | maxJobs: 1, | ||
| 186 | }); | ||
| 187 | moduleLoadQueue.addMany(pages); | ||
| 188 | await moduleLoadQueue.done({ method: "stop" }); | ||
| 189 | const pageQueue = new Queue({ | ||
| 190 | name: "Render Static Page", | ||
| 191 | fn: renderPage, | ||
| 192 | getItemText, | ||
| 193 | maxJobs: 2, | ||
| 194 | }); | ||
| 195 | pageQueue.addMany(pages); | ||
| 196 | await pageQueue.done({ method: "stop" }); | ||
| 197 | status.format = spinnerFormat; | ||
| 198 | |||
| 199 | // -- bundle scripts -- | ||
| 200 | const referencedScripts = Array.from( | ||
| 201 | new Set(renderResults.flatMap((r) => r.scriptFiles)), | ||
| 202 | ); | ||
| 203 | const extraPublicScripts = scripts.map((entry) => entry.file); | ||
| 204 | const uniqueCount = new Set([ | ||
| 205 | ...referencedScripts, | ||
| 206 | ...extraPublicScripts, | ||
| 207 | ]).size; | ||
| 208 | status.text = `Bundle ${uniqueCount} Scripts`; | ||
| 209 | await bundle.bundleClientJavaScript( | ||
| 210 | referencedScripts, | ||
| 211 | extraPublicScripts, | ||
| 212 | incr, | ||
| 213 | ); | ||
| 214 | |||
| 215 | // -- copy/compress static files -- | ||
| 216 | async function doStaticFile(item: FileItem) { | ||
| 217 | const body = await fs.readFile(item.file); | ||
| 218 | incr.putAsset({ | ||
| 219 | srcId: "static:" + item.file, | ||
| 220 | key: item.id, | ||
| 221 | body, | ||
| 222 | }); | ||
| 223 | } | ||
| 224 | const staticQueue = new Queue({ | ||
| 225 | name: "Load Static", | ||
| 226 | fn: doStaticFile, | ||
| 227 | getItemText, | ||
| 228 | maxJobs: 16, | ||
| 229 | }); | ||
| 230 | status.format = () => ""; | ||
| 231 | staticQueue.addMany(staticFiles); | ||
| 232 | await staticQueue.done({ method: "stop" }); | ||
| 233 | status.format = spinnerFormat; | ||
| 234 | |||
| 235 | // -- concatenate static rendered pages -- | ||
| 236 | status.text = `Concat ${renderResults.length} Pages`; | ||
| 237 | await Promise.all( | ||
| 238 | renderResults.map( | ||
| 239 | async ({ item: page, body, head, inlineCss, scriptFiles }) => { | ||
| 240 | const doc = wrapDocument({ | ||
| 241 | body, | ||
| 242 | head, | ||
| 243 | inlineCss, | ||
| 244 | scripts: scriptFiles.map( | ||
| 245 | (id) => | ||
| 246 | UNWRAP( | ||
| 247 | incr.out.script.get( | ||
| 248 | path.basename(id).replace(/\.client\.[jt]sx?$/, ""), | ||
| 249 | ), | ||
| 250 | ), | ||
| 251 | ).map((x) => `{${x}}`).join("\n"), | ||
| 252 | }); | ||
| 253 | incr.putAsset({ | ||
| 254 | srcId: "page:" + page.file, | ||
| 255 | key: page.id, | ||
| 256 | body: doc, | ||
| 257 | headers: { | ||
| 258 | "Content-Type": "text/html", | ||
| 259 | }, | ||
| 260 | }); | ||
| 261 | }, | ||
| 262 | ), | ||
| 263 | ); | ||
| 264 | status.format = () => ""; | ||
| 265 | status.text = ``; | ||
| 266 | // This will wait for all compression jobs to finish, which up | ||
| 267 | // to this point have been left as dangling promises. | ||
| 268 | await incr.wait(); | ||
| 269 | |||
| 270 | // Flush the site to disk. | ||
| 271 | status.format = spinnerFormat; | ||
| 272 | status.text = `Incremental Flush`; | ||
| 273 | incr.flush(); | ||
| 274 | incr.serializeToDisk(); // Allows picking up this state again | ||
| 275 | return { elapsed: (performance.now() - startTime) / 1000 }; | ||
| 276 | } | ||
| 277 | |||
| 278 | function getItemText({ file }: FileItem) { | ||
| 279 | return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); | ||
| 280 | } | ||
| 281 | |||
| 282 | function wrapDocument({ | ||
| 283 | body, | ||
| 284 | head, | ||
| 285 | inlineCss, | ||
| 286 | scripts, | ||
| 287 | }: { | ||
| 288 | head: string; | ||
| 289 | body: string; | ||
| 290 | inlineCss: string; | ||
| 291 | scripts: string; | ||
| 292 | }) { | ||
| 293 | return `<!doctype html><head>${head}${ | ||
| 294 | inlineCss ? `<style>${inlineCss}</style>` : "" | ||
| 295 | }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`; | ||
| 296 | } | ||
| 297 | |||
| 298 | import type { Section } from "./sitegen-lib.ts"; | ||
| 299 | import { OnceMap, Queue } from "./queue.ts"; | ||
| 300 | import { Incremental } from "./incremental.ts"; | ||
| 301 | import * as bundle from "./bundle.ts"; | ||
| 302 | import * as css from "./css.ts"; | ||
| 303 | import * as fs from "./fs.ts"; | ||
| 304 | import { Spinner, withSpinner } from "@paperclover/console/Spinner"; | ||
| 305 | import * as meta from "./meta.ts"; | ||
| 306 | import * as ssr from "./engine/ssr.ts"; | ||
| 307 | import * as sg from "./sitegen-lib.ts"; | ||
| 308 | import * as hot from "./hot.ts"; | ||
| 309 | import * as path from "node:path"; | ||
framework/sqlite.ts+2-5| ... | @@ -41,11 +41,8 @@ export class WrappedDatabase { | ... | @@ -41,11 +41,8 @@ export class WrappedDatabase { |
| 41 | (key, version) values (?, ?); | 41 | (key, version) values (?, ?); |
| 42 | `), | 42 | `), |
| 43 | )); | 43 | )); |
| 44 | const { changes, lastInsertRowid } = s.run(name, 1); | 44 | const { changes } = s.run(name, 1); |
| 45 | console.log(changes, lastInsertRowid); | 45 | if (changes === 1) this.node.exec(schema); |
| 46 | if (changes === 1) { | ||
| 47 | this.node.exec(schema); | ||
| 48 | } | ||
| 49 | } | 46 | } |
| 50 | 47 | ||
| 51 | prepare<Args extends unknown[] = [], Result = unknown>( | 48 | prepare<Args extends unknown[] = [], Result = unknown>( |
package.json+4-1| ... | @@ -15,7 +15,9 @@ | ... | @@ -15,7 +15,9 @@ |
| 15 | "typescript": "^5.8.3" | 15 | "typescript": "^5.8.3" |
| 16 | }, | 16 | }, |
| 17 | "imports": { | 17 | "imports": { |
| 18 | "#sitegen": "./framework/sitegen-lib.ts", | 18 | "#backend": "./src/backend.ts", |
| 19 | "#sitegen": "./framework/lib/sitegen.ts", | ||
| 20 | "#sitegen/*": "./framework/lib/*.ts", | ||
| 19 | "#sqlite": "./framework/sqlite.ts", | 21 | "#sqlite": "./framework/sqlite.ts", |
| 20 | "#ssr": "./framework/engine/ssr.ts", | 22 | "#ssr": "./framework/engine/ssr.ts", |
| 21 | "#ssr/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts", | 23 | "#ssr/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts", |
| ... | @@ -26,6 +28,7 @@ | ... | @@ -26,6 +28,7 @@ |
| 26 | "production": "marko/production", | 28 | "production": "marko/production", |
| 27 | "node": "marko/debug/html" | 29 | "node": "marko/debug/html" |
| 28 | }, | 30 | }, |
| 31 | "#hono": "hono", | ||
| 29 | "#hono/platform": { | 32 | "#hono/platform": { |
| 30 | "bun": "hono/bun", | 33 | "bun": "hono/bun", |
| 31 | "deno": "hono/deno", | 34 | "deno": "hono/deno", |
readme.md created+1| ... | @@ -0,0 +1 @@ | ||
| 1 | # clover sitegen framework | ||
repl.js+3-4| ... | @@ -27,8 +27,7 @@ hot.load("node:repl").start({ | ... | @@ -27,8 +27,7 @@ hot.load("node:repl").start({ |
| 27 | }); | 27 | }); |
| 28 | 28 | ||
| 29 | setTimeout(() => { | 29 | setTimeout(() => { |
| 30 | hot.reloadRecursive("./framework/engine/ssr.ts"); | 30 | hot.reloadRecursive("./framework/generate.tsx"); |
| 31 | hot.reloadRecursive("./framework/bundle.ts"); | ||
| 32 | }, 100); | 31 | }, 100); |
| 33 | 32 | ||
| 34 | async function evaluate(code) { | 33 | async function evaluate(code) { |
| ... | @@ -41,11 +40,11 @@ async function evaluate(code) { | ... | @@ -41,11 +40,11 @@ async function evaluate(code) { |
| 41 | if (code[0] === "=") { | 40 | if (code[0] === "=") { |
| 42 | try { | 41 | try { |
| 43 | const result = await eval(code[1]); | 42 | const result = await eval(code[1]); |
| 44 | console.log(inspect(result)); | 43 | console.info(inspect(result)); |
| 45 | } catch (err) { | 44 | } catch (err) { |
| 46 | if (err instanceof SyntaxError) { | 45 | if (err instanceof SyntaxError) { |
| 47 | const result = await eval("(async() => { return " + code + " })()"); | 46 | const result = await eval("(async() => { return " + code + " })()"); |
| 48 | console.log(inspect(result)); | 47 | console.info(inspect(result)); |
| 49 | } else { | 48 | } else { |
| 50 | throw err; | 49 | throw err; |
| 51 | } | 50 | } |
run.js+1-1| ... | @@ -18,7 +18,7 @@ import process from "node:process"; | ... | @@ -18,7 +18,7 @@ import process from "node:process"; |
| 18 | const hot = await import("./framework/hot.ts"); | 18 | const hot = await import("./framework/hot.ts"); |
| 19 | 19 | ||
| 20 | const console = hot.load("@paperclover/console"); | 20 | const console = hot.load("@paperclover/console"); |
| 21 | globalThis.console.log = console.info; | 21 | globalThis.console["log"] = console.info; |
| 22 | globalThis.console.info = console.info; | 22 | globalThis.console.info = console.info; |
| 23 | globalThis.console.warn = console.warn; | 23 | globalThis.console.warn = console.warn; |
| 24 | globalThis.console.error = console.error; | 24 | globalThis.console.error = console.error; |
src/admin.ts created+64| ... | @@ -0,0 +1,64 @@ | ||
| 1 | const cookieAge = 60 * 60 * 24 * 365; // 1 year | ||
| 2 | |||
| 3 | let lastKnownToken: string | null = null; | ||
| 4 | function compareToken(token: string) { | ||
| 5 | if (token === lastKnownToken) return true; | ||
| 6 | lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); | ||
| 7 | return token === lastKnownToken; | ||
| 8 | } | ||
| 9 | |||
| 10 | export async function middleware(c: Context, next: Next) { | ||
| 11 | if (c.req.path.startsWith("/admin")) { | ||
| 12 | return adminInner(c, next); | ||
| 13 | } | ||
| 14 | return next(); | ||
| 15 | } | ||
| 16 | |||
| 17 | export function adminInner(c: Context, next: Next) { | ||
| 18 | const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; | ||
| 19 | |||
| 20 | if (c.req.path === "/admin/login") { | ||
| 21 | const key = c.req.query("key"); | ||
| 22 | if (key) { | ||
| 23 | if (compareToken(key)) { | ||
| 24 | return c.body(null, 303, { | ||
| 25 | "Location": "/admin", | ||
| 26 | "Set-Cookie": | ||
| 27 | `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, | ||
| 28 | }); | ||
| 29 | } | ||
| 30 | return serveAsset(c, "/admin/login/fail", 403); | ||
| 31 | } | ||
| 32 | if (token && compareToken(token)) { | ||
| 33 | return c.redirect("/admin", 303); | ||
| 34 | } | ||
| 35 | if (c.req.method === "POST") { | ||
| 36 | return serveAsset(c, "/admin/login/fail", 403); | ||
| 37 | } else { | ||
| 38 | return serveAsset(c, "/admin/login", 200); | ||
| 39 | } | ||
| 40 | } | ||
| 41 | |||
| 42 | if (c.req.path === "/admin/logout") { | ||
| 43 | return c.body(null, 303, { | ||
| 44 | "Location": "/admin/login", | ||
| 45 | "Set-Cookie": | ||
| 46 | `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`, | ||
| 47 | }); | ||
| 48 | } | ||
| 49 | |||
| 50 | if (token && compareToken(token)) { | ||
| 51 | return next(); | ||
| 52 | } | ||
| 53 | |||
| 54 | return c.redirect("/admin/login", 303); | ||
| 55 | } | ||
| 56 | |||
| 57 | export function hasAdminToken(c: Context) { | ||
| 58 | const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; | ||
| 59 | return token && compareToken(token); | ||
| 60 | } | ||
| 61 | |||
| 62 | import * as fs from "fs"; | ||
| 63 | import type { Context, Next } from "hono"; | ||
| 64 | import { serveAsset } from "#sitegen/assets"; | ||
src/backend.ts created+38| ... | @@ -0,0 +1,38 @@ | ||
| 1 | const logHttp = scoped("http", { color: "magenta" }); | ||
| 2 | |||
| 3 | const app = new Hono(); | ||
| 4 | |||
| 5 | app.notFound(assets.notFound); | ||
| 6 | |||
| 7 | app.use(trimTrailingSlash()); | ||
| 8 | app.use(removeDuplicateSlashes); | ||
| 9 | app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4)))); | ||
| 10 | app.use(admin.middleware); | ||
| 11 | |||
| 12 | app.route("", require("./q+a/backend.ts").app); | ||
| 13 | // app.route("", require("./file-viewer/backend.tsx").app); | ||
| 14 | // app.route("", require("./friends/backend.tsx").app); | ||
| 15 | |||
| 16 | app.use(assets.middleware); | ||
| 17 | |||
| 18 | export default app; | ||
| 19 | |||
| 20 | async function removeDuplicateSlashes(c: Context, next: Next) { | ||
| 21 | const path = c.req.path; | ||
| 22 | if (/\/\/+/.test(path)) { | ||
| 23 | const normalizedPath = path.replace(/\/\/+/g, "/"); | ||
| 24 | const query = c.req.query(); | ||
| 25 | const queryString = Object.keys(query).length > 0 | ||
| 26 | ? "?" + new URLSearchParams(query).toString() | ||
| 27 | : ""; | ||
| 28 | return c.redirect(normalizedPath + queryString, 301); | ||
| 29 | } | ||
| 30 | await next(); | ||
| 31 | } | ||
| 32 | |||
| 33 | import { Hono } from "#hono"; | ||
| 34 | import { logger } from "hono/logger"; | ||
| 35 | import { trimTrailingSlash } from "hono/trailing-slash"; | ||
| 36 | import * as assets from "#sitegen/assets"; | ||
| 37 | import * as admin from "./admin.ts"; | ||
| 38 | import { scoped } from "@paperclover/console"; | ||
src/pages/404.mdx created+6| ... | @@ -0,0 +1,6 @@ | ||
| 1 | export const meta = { title: 'oh no,,,' }; | ||
| 2 | |||
| 3 | # oh dear | ||
| 4 | |||
| 5 | sound the alarms | ||
| 6 | |||
src/pages/index.marko-1| ... | @@ -16,7 +16,6 @@ export const meta: Meta = { | ... | @@ -16,7 +16,6 @@ export const meta: Meta = { |
| 16 | type: "website", | 16 | type: "website", |
| 17 | url: "https://paperclover.net", | 17 | url: "https://paperclover.net", |
| 18 | }, | 18 | }, |
| 19 | generator: "clover", | ||
| 20 | alternates: { | 19 | alternates: { |
| 21 | canonical: "https://paperclover.net", | 20 | canonical: "https://paperclover.net", |
| 22 | types: { | 21 | types: { |
src/q+a/backend.ts created+3| ... | @@ -0,0 +1,3 @@ | ||
| 1 | export const app = new Hono(); | ||
| 2 | |||
| 3 | import { Hono } from "#hono"; | ||
src/q+a/components/Question.marko+1-1| ... | @@ -30,7 +30,7 @@ static const transitionDate = 1735639200000; | ... | @@ -30,7 +30,7 @@ static const transitionDate = 1735639200000; |
| 30 | </> | 30 | </> |
| 31 | 31 | ||
| 32 | // this singleton script will make all the '<time>' tags clickable. | 32 | // this singleton script will make all the '<time>' tags clickable. |
| 33 | client import "./clickable-links.ts"; | 33 | client import "./clickable-links.client.ts"; |
| 34 | 34 | ||
| 35 | import type { Question } from "@/q+a/models/Question.ts"; | 35 | import type { Question } from "@/q+a/models/Question.ts"; |
| 36 | import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts"; | 36 | import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts"; |
src/sections.ts deleted-19| ... | @@ -1,19 +0,0 @@ | ||
| 1 | // This file defines the different "Sections" of the website. The sections act | ||
| 2 | // as separate codebases, but hosted as one. This allows me to have | ||
| 3 | // sub-projects like the file viewer in 'file', or the question answer system | ||
| 4 | // in 'q+a'. Each section can define configuration, pages, backend routes, and | ||
| 5 | // contain other files. | ||
| 6 | interface Section { | ||
| 7 | root: string; | ||
| 8 | } | ||
| 9 | |||
| 10 | const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); | ||
| 11 | |||
| 12 | export const siteSections: Section[] = [ | ||
| 13 | { root: join("./") }, | ||
| 14 | { root: join("q+a/") }, | ||
| 15 | { root: join("file-viewer/") }, | ||
| 16 | { root: join("friends/") }, | ||
| 17 | ]; | ||
| 18 | |||
| 19 | import * as path from "node:path"; | ||
src/site.ts created+19| ... | @@ -0,0 +1,19 @@ | ||
| 1 | // This file defines the different "Sections" of the website. The sections act | ||
| 2 | // as separate codebases, but hosted as one. This allows me to have | ||
| 3 | // sub-projects like the file viewer in 'file', or the question answer system | ||
| 4 | // in 'q+a'. Each section can define configuration, pages, backend routes, and | ||
| 5 | // contain other files. | ||
| 6 | interface Section { | ||
| 7 | root: string; | ||
| 8 | } | ||
| 9 | |||
| 10 | const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); | ||
| 11 | |||
| 12 | export const siteSections: Section[] = [ | ||
| 13 | { root: join("./") }, | ||
| 14 | { root: join("q+a/") }, | ||
| 15 | { root: join("file-viewer/") }, | ||
| 16 | { root: join("friends/") }, | ||
| 17 | ]; | ||
| 18 | |||
| 19 | import * as path from "node:path"; | ||