diff --git a/framework/assets.ts b/framework/assets.ts deleted file mode 100644 index 109e727d29da14b670b00461c476b3b5dbd58ef6..0000000000000000000000000000000000000000 --- a/framework/assets.ts +++ /dev/null @@ -1,100 +0,0 @@ -interface Loaded { - map: BuiltAssetMap; - buf: Buffer; -} -let assets: Loaded | null = null; - -export type StaticPageId = string; - -export async function reload() { - const [map, buf] = await Promise.all([ - fs.readFile(".clover/static.json", "utf8"), - fs.readFile(".clover/static.blob"), - ]); - assets = { - map: JSON.parse(map), - buf, - }; -} - -export async function reloadSync() { - const map = fs.readFileSync(".clover/static.json", "utf8"); - const buf = fs.readFileSync(".clover/static.blob"); - assets = { - map: JSON.parse(map), - buf, - }; -} - -export async function assetMiddleware(c: Context, next: Next) { - if (!assets) await reload(); - const asset = assets!.map[c.req.path]; - if (asset) { - return assetInner(c, asset, 200); - } - return next(); -} - -export async function serveAsset( - c: Context, - id: StaticPageId, - status: StatusCode, -) { - assets ?? await reload(); - return assetInner(c, assets!.map[id], status); -} - -export function hasAsset(id: string) { - if (!assets) reloadSync(); - return assets!.map[id] !== undefined; -} - -export function etagMatches(etag: string, ifNoneMatch: string) { - return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; -} - -function subarrayAsset([start, end]: View) { - return assets!.buf.subarray(start, end); -} - -function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { - const ifnonematch = c.req.header("If-None-Match"); - if (ifnonematch) { - const etag = asset.headers.ETag; - if (etagMatches(etag, ifnonematch)) { - c.res = new Response(null, { - status: 304, - statusText: "Not Modified", - headers: { - ETag: etag, - }, - }); - return; - } - } - const acceptEncoding = c.req.header("Accept-Encoding") ?? ""; - let body; - let headers = asset.headers; - if (acceptEncoding.includes("zstd") && asset.zstd) { - body = subarrayAsset(asset.zstd); - headers = { - ...asset.headers, - "Content-Encoding": "zstd", - }; - } else if (acceptEncoding.includes("gzip") && asset.gzip) { - body = subarrayAsset(asset.gzip); - headers = { - ...asset.headers, - "Content-Encoding": "gzip", - }; - } else { - body = subarrayAsset(asset.raw); - } - c.res = new Response(body, { headers, status }); -} - -import * as fs from "./fs.ts"; -import type { Context, Next } from "hono"; -import type { StatusCode } from "hono/utils/http-status"; -import type { BuiltAsset, BuiltAssetMap, View } from "./incremental.ts"; -import { Buffer } from "node:buffer"; diff --git a/framework/backend/entry-node.ts b/framework/backend/entry-node.ts new file mode 100644 index 0000000000000000000000000000000000000000..035a433c88f1e75753c5c05aa69e8cf723fc7995 --- /dev/null +++ b/framework/backend/entry-node.ts @@ -0,0 +1,32 @@ +import "@paperclover/console/inject"; + +const protocol = "http"; + +const server = serve(app, ({ address, port }) => { + if (address === "::") address = "::1"; + console.info(url.format({ + protocol, + hostname: address, + port, + })); +}); + +process.on("SIGINT", () => { + server.close(); + process.exit(0); +}); + +process.on("SIGTERM", () => { + server.close((err) => { + if (err) { + console.error(err); + process.exit(1); + } + process.exit(0); + }); +}); + +import app from "#backend"; +import url from "node:url"; +import { serve } from "@hono/node-server"; +import process from "node:process"; diff --git a/framework/bundle.ts b/framework/bundle.ts index 865150a05d583828bfaadae482c1abdc3c684abb..300d3d240268cf3f153da1075461db0fd8f88da9 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -49,7 +49,7 @@ export async function bundleClientJavaScript( const publicScriptRoutes = extraPublicScripts.map((file) => path.basename(file).replace(/\.client\.[tj]sx?/, "") ); - const promises: Promise[] = []; + const promises: Promise[] = []; // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes. for (const file of bundle.outputFiles) { let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/"); @@ -73,10 +73,23 @@ export async function bundleClientJavaScript( })); } } + await Promise.all(promises); +} - if (promises.length > 0) { - await Promise.all(promises); - } +export async function bundleServerJavaScript(entryPoint: string) { + const bundle = await esbuild.build({ + bundle: true, + chunkNames: "/js/c.[hash]", + entryNames: "/js/[name]", + assetNames: "/asset/[hash]", + entryPoints: [entryPoint], + format: "esm", + minify: true, + outdir: "/out!", + plugins, + splitting: true, + write: false, + }); } import * as esbuild from "esbuild"; diff --git a/framework/fs.ts b/framework/fs.ts index 30554c47909b0c7e34ebfc74298fc45727f72f46..4761f37f8d31b262d723161853230dc0e29946d7 100644 --- a/framework/fs.ts +++ b/framework/fs.ts @@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) { export function readDirRecOptionalSync(dir: string) { try { - return readdirSync(dir, { withFileTypes: true }); + return readdirSync(dir, { recursive: true, withFileTypes: true }); } catch (err: any) { if (err.code === "ENOENT") return []; throw err; diff --git a/framework/generate.tsx b/framework/generate.tsx new file mode 100644 index 0000000000000000000000000000000000000000..cbe24f7a1c6b98288d3615b2c87acb80995ecf8f --- /dev/null +++ b/framework/generate.tsx @@ -0,0 +1,299 @@ +export function main() { + return withSpinner({ + text: "Recovering State", + successText: ({ elapsed }) => + "sitegen! update in " + elapsed.toFixed(1) + "s", + failureText: () => "sitegen FAIL", + }, sitegen); +} + +/** + * A filesystem object associated with some ID, + * such as a page's route to it's source file. + */ +interface FileItem { + id: string; + file: string; +} + +async function sitegen(status: Spinner) { + const startTime = performance.now(); + + let root = path.resolve(import.meta.dirname, "../src"); + const join = (...sub: string[]) => path.join(root, ...sub); + const incr = new Incremental(); + + // Sitegen reviews every defined section for resources to process + const sections: sg.Section[] = + require(path.join(root, "site.ts")).siteSections; + + // Static files are compressed and served as-is. + // - "{section}/static/*.png" + let staticFiles: FileItem[] = []; + // Pages are rendered then served as static files. + // - "{section}/pages/*.marko" + let pages: FileItem[] = []; + // Views are dynamically rendered pages called via backend code. + // - "{section}/views/*.tsx" + let views: FileItem[] = []; + // Public scripts are bundled for the client as static assets under "/js/[...]" + // This is used for the file viewer's canvases. + // Note that '.client.ts' can be placed anywhere in the file structure. + // - "{section}/scripts/*.client.ts" + let scripts: FileItem[] = []; + + // -- Scan for files -- + status.text = "Scanning Project"; + for (const section of sections) { + const { root: sectionRoot } = section; + const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); + const rootPrefix = root === sectionRoot + ? "" + : path.relative(root, sectionRoot) + "/"; + const kinds = [ + { + dir: sectionPath("pages"), + list: pages, + prefix: "/", + exclude: [".css", ".client.ts", ".client.tsx"], + }, + { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, + { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, + { + dir: sectionPath("views"), + list: views, + prefix: rootPrefix, + exclude: [".css", ".client.ts", ".client.tsx"], + }, + ]; + for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { + const items = fs.readDirRecOptionalSync(dir); + item: for (const item of items) { + if (item.isDirectory()) continue; + for (const e of exclude) { + if (item.name.endsWith(e)) continue item; + } + const file = path.relative(dir, item.parentPath + "/" + item.name); + const trim = ext + ? file + : file.slice(0, -path.extname(file).length).replaceAll(".", "/"); + let id = prefix + trim.replaceAll("\\", "/"); + if (prefix === "/" && id.endsWith("/index")) { + id = id.slice(0, -"/index".length) || "/"; + } + list.push({ id, file: path.join(item.parentPath, item.name) }); + } + } + } + scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); + const globalCssPath = join("global.css"); + + // TODO: invalidate incremental resources + + // -- server side render -- + status.text = "Building"; + const cssOnce = new OnceMap(); + const cssQueue = new Queue<[string, string[], css.Theme], string>({ + name: "Bundle", + fn: ([, files, theme]) => css.bundleCssFiles(files, theme), + passive: true, + getItemText: ([id]) => id, + maxJobs: 2, + }); + interface RenderResult { + body: string; + head: string; + inlineCss: string; + scriptFiles: string[]; + item: FileItem; + } + const renderResults: RenderResult[] = []; + async function loadPageModule({ file }: FileItem) { + require(file); + } + async function renderPage(item: FileItem) { + // -- load and validate module -- + let { default: Page, meta: metadata, theme: pageTheme, layout } = require( + item.file, + ); + if (!Page) throw new Error("Page is missing a 'default' export."); + if (!metadata) { + throw new Error("Page is missing 'meta' export with a title."); + } + if (layout?.theme) pageTheme = layout.theme; + const theme = { + bg: "#fff", + fg: "#050505", + primary: "#2e7dab", + ...pageTheme, + }; + + // -- metadata -- + const renderedMetaPromise = Promise.resolve( + typeof metadata === "function" ? metadata({ ssr: true }) : metadata, + ).then((m) => meta.renderMeta(m)); + // -- css -- + const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; + const cssPromise = cssOnce.get( + cssImports.join(":") + JSON.stringify(theme), + () => cssQueue.add([item.id, cssImports, theme]), + ); + // -- html -- + const bodyPromise = await ssr.ssrAsync(, { + sitegen: sg.initRender(), + }); + + const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([ + bodyPromise, + cssPromise, + renderedMetaPromise, + ]); + if (!renderedMeta.includes("")) { + throw new Error( + "Page is missing 'meta.title'. " + + "All pages need a title tag.", + ); + } + // The script content is not ready, allow another page to Render. The page + // contents will be rebuilt at the end. This is more convenient anyways + // because it means client scripts don't re-render the page. + renderResults.push({ + body: text, + head: renderedMeta, + inlineCss, + scriptFiles: Array.from(addon.sitegen.scripts), + item: item, + }); + } + // This is done in two passes so that a page that throws during evaluation + // will report "Load Render Module" instead of "Render Static Page". + const spinnerFormat = status.format; + status.format = () => ""; + const moduleLoadQueue = new Queue({ + name: "Load Render Module", + fn: loadPageModule, + getItemText, + maxJobs: 1, + }); + moduleLoadQueue.addMany(pages); + await moduleLoadQueue.done({ method: "stop" }); + const pageQueue = new Queue({ + name: "Render Static Page", + fn: renderPage, + getItemText, + maxJobs: 2, + }); + pageQueue.addMany(pages); + await pageQueue.done({ method: "stop" }); + status.format = spinnerFormat; + + // -- bundle scripts -- + const referencedScripts = Array.from( + new Set(renderResults.flatMap((r) => r.scriptFiles)), + ); + const extraPublicScripts = scripts.map((entry) => entry.file); + const uniqueCount = new Set([ + ...referencedScripts, + ...extraPublicScripts, + ]).size; + status.text = `Bundle ${uniqueCount} Scripts`; + await bundle.bundleClientJavaScript( + referencedScripts, + extraPublicScripts, + incr, + ); + + // -- copy/compress static files -- + async function doStaticFile(item: FileItem) { + const body = await fs.readFile(item.file); + await incr.putAsset({ + srcId: "static:" + item.file, + key: item.id, + body, + }); + } + const staticQueue = new Queue({ + name: "Load Static", + fn: doStaticFile, + getItemText, + maxJobs: 16, + }); + status.format = () => ""; + staticQueue.addMany(staticFiles); + await staticQueue.done({ method: "stop" }); + status.format = spinnerFormat; + + // -- concatenate static rendered pages -- + status.text = `Concat ${renderResults.length} Pages`; + await Promise.all( + renderResults.map( + async ({ item: page, body, head, inlineCss, scriptFiles }) => { + const doc = wrapDocument({ + body, + head, + inlineCss, + scripts: scriptFiles.map( + (id) => + UNWRAP( + incr.out.script.get( + path.basename(id).replace(/\.client\.[jt]sx?$/, ""), + ), + ), + ).map((x) => `{${x}}`).join("\n"), + }); + await incr.putAsset({ + srcId: "page:" + page.file, + key: page.id, + body: doc, + headers: { + "Content-Type": "text/html", + }, + }); + }, + ), + ); + status.format = () => ""; + status.text = ``; + // This will wait for all compression jobs to finish, which up + // to this point have been left as dangling promises. + await incr.wait(); + + // Flush the site to disk. + status.format = spinnerFormat; + status.text = `Incremental Flush`; + incr.flush(); + incr.serializeToDisk(); // Allows picking up this state again + return { elapsed: (performance.now() - startTime) / 1000 }; +} + +function getItemText({ file }: FileItem) { + return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); +} + +function wrapDocument({ + body, + head, + inlineCss, + scripts, +}: { + head: string; + body: string; + inlineCss: string; + scripts: string; +}) { + return `<!doctype html><head>${head}${ + inlineCss ? `<style>${inlineCss}</style>` : "" + }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`; +} + +import { OnceMap, Queue } from "./queue.ts"; +import { Incremental } from "./incremental.ts"; +import * as bundle from "./bundle.ts"; +import * as css from "./css.ts"; +import * as fs from "./fs.ts"; +import { Spinner, withSpinner } from "@paperclover/console/Spinner"; +import * as meta from "./meta.ts"; +import * as ssr from "./engine/ssr.ts"; +import * as sg from "#sitegen"; +import * as hot from "./hot.ts"; +import * as path from "node:path"; diff --git a/framework/hot.ts b/framework/hot.ts index 5078e5c35f174822d4b61212c2f1f6a45c586067..8e60aacff161b89d9eab8f868dfa744d64d86b15 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -146,6 +146,7 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { target: "esnext", jsx: "automatic", jsxImportSource: "#ssr", + sourcefile: filepath, }).code; return module._compile(src, filepath, "commonjs"); } @@ -162,8 +163,6 @@ function loadMarko(module: NodeJS.Module, filepath: string) { ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n'; } - console.log(src); - console.log("---"); src = marko.compileSync(src, filepath).code; src = src.replace("marko/debug/html", "#ssr/marko"); return loadEsbuildCode(module, filepath, src); diff --git a/framework/incremental.ts b/framework/incremental.ts index 12bbc3ee1f15909be951de609194bac553cfca13..793cf07dd1967602128fac66faad4a44e1c41d4d 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -7,7 +7,6 @@ // all its imports) to map to the build of a page, which produces an HTML file // plus a list of scripts. -import { Buffer } from "node:buffer"; interface ArtifactMap { asset: Asset; script: string; @@ -126,18 +125,17 @@ export class Incremental { }, hash, }; + const a = this.put({ ...info, type: "asset", value }); if (!this.compress.has(hash)) { const label = info.key; this.compress.set(hash, { zstd: undefined, gzip: undefined, }); - await Promise.all([ - this.compressQueue.add({ label, buffer, algo: "zstd", hash }), - this.compressQueue.add({ label, buffer, algo: "gzip", hash }), - ]); + this.compressQueue.add({ label, buffer, algo: "zstd", hash }); + this.compressQueue.add({ label, buffer, algo: "gzip", hash }); } - return this.put({ ...info, type: "asset", value }); + return a; } async compressImpl({ algo, buffer, hash }: CompressJob) { @@ -167,16 +165,18 @@ export class Incremental { } async wait() { - await this.compressQueue.done({ method: "stop" }); + await this.compressQueue.done({ method: "success" }); } + async flush() { + ASSERT(!this.compressQueue.active); const writer = new BufferWriter(); const asset = Object.fromEntries( Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => { const raw = writer.write(buffer, hash); const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {}; - const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null; - const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null; + const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null; + const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null; return [key, { raw, gzip, @@ -252,10 +252,6 @@ export interface SerializedMeta { script: [key: string, value: string][]; } -function never(): never { - throw new Error("Impossible"); -} - import * as path from "node:path"; import * as fs from "./fs.ts"; import * as zlib from "node:zlib"; @@ -263,3 +259,4 @@ import * as util from "node:util"; import { Queue } from "./queue.ts"; import * as hot from "./hot.ts"; import * as mime from "./mime.ts"; +import { Buffer } from "node:buffer"; diff --git a/framework/lib/assets.ts b/framework/lib/assets.ts new file mode 100644 index 0000000000000000000000000000000000000000..9dc295bb040c0cd6c9bec141a22b481a23c7150a --- /dev/null +++ b/framework/lib/assets.ts @@ -0,0 +1,112 @@ +interface Loaded { + map: BuiltAssetMap; + buf: Buffer; +} +let assets: Loaded | null = null; + +export type StaticPageId = string; + +export async function reload() { + const [map, buf] = await Promise.all([ + fs.readFile(".clover/static.json", "utf8"), + fs.readFile(".clover/static.blob"), + ]); + assets = { + map: JSON.parse(map), + buf, + }; +} + +export async function reloadSync() { + const map = fs.readFileSync(".clover/static.json", "utf8"); + const buf = fs.readFileSync(".clover/static.blob"); + assets = { + map: JSON.parse(map), + buf, + }; +} + +export async function middleware(c: Context, next: Next) { + if (!assets) await reload(); + const asset = assets!.map[c.req.path]; + if (asset) { + return assetInner(c, asset, 200); + } + return next(); +} + +export async function notFound(c: Context) { + if (!assets) await reload(); + let pathname = c.req.path; + do { + const asset = assets!.map[pathname + "/404"]; + if (asset) return assetInner(c, asset, 404); + pathname = pathname.slice(0, pathname.lastIndexOf("/")); + } while (pathname); + const asset = assets!.map["/404"]; + if (asset) return assetInner(c, asset, 404); + return c.text("the 'Not Found' page was not found", 404); +} + +export async function serveAsset( + c: Context, + id: StaticPageId, + status: StatusCode, +) { + assets ?? await reload(); + return assetInner(c, assets!.map[id], status); +} + +export function hasAsset(id: string) { + if (!assets) reloadSync(); + return assets!.map[id] !== undefined; +} + +export function etagMatches(etag: string, ifNoneMatch: string) { + return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; +} + +function subarrayAsset([start, end]: View) { + return assets!.buf.subarray(start, end); +} + +function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { + const ifnonematch = c.req.header("If-None-Match"); + if (ifnonematch) { + const etag = asset.headers.ETag; + if (etagMatches(etag, ifnonematch)) { + return c.res = new Response(null, { + status: 304, + statusText: "Not Modified", + headers: { + ETag: etag, + }, + }); + } + } + const acceptEncoding = c.req.header("Accept-Encoding") ?? ""; + let body; + let headers = asset.headers; + if (acceptEncoding.includes("zstd") && asset.zstd) { + body = subarrayAsset(asset.zstd); + headers = { + ...asset.headers, + "Content-Encoding": "zstd", + }; + } else if (acceptEncoding.includes("gzip") && asset.gzip) { + body = subarrayAsset(asset.gzip); + headers = { + ...asset.headers, + "Content-Encoding": "gzip", + }; + } else { + body = subarrayAsset(asset.raw); + } + return c.res = new Response(body, { headers, status }); +} + +import * as fs from "../fs.ts"; +import type { Context, Next } from "hono"; +import type { StatusCode } from "hono/utils/http-status"; +import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts"; +import { Buffer } from "node:buffer"; diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts new file mode 100644 index 0000000000000000000000000000000000000000..5404b8c243c9b5e82f25eaafbd6b160694d51ab6 --- /dev/null +++ b/framework/lib/sitegen.ts @@ -0,0 +1,52 @@ +// Import this file with 'import * as sg from "#sitegen";' +export type ScriptId = string; + +const frameworkDir = path.dirname(import.meta.dirname); + +export interface SitegenRender { + scripts: Set<ScriptId>; +} + +export function initRender(): SitegenRender { + return { + scripts: new Set(), + }; +} + +export function getRender() { + return ssr.getUserData<SitegenRender>("sitegen", () => { + throw new Error( + "This function can only be used in a page (static or view)", + ); + }); +} + +/** Add a client-side script to the page. */ +export function addScript(id: ScriptId) { + const srcFile: string = util.getCallSites() + .find((site) => !site.scriptName.startsWith(frameworkDir))! + .scriptName; + const filePath = hot.resolveFrom(srcFile, id); + if ( + !filePath.endsWith(".client.ts") && + !filePath.endsWith(".client.tsx") + ) { + throw new Error("addScript must be a .client.ts or .client.tsx"); + } + getRender().scripts.add(filePath); +} + +export function Script({ src }: { src: ScriptId }) { + if (!src) throw new Error("Missing 'src' attribute"); + addScript(src); + return null; +} + +export interface Section { + root: string; +} + +import * as ssr from "../engine/ssr.ts"; +import * as util from "node:util"; +import * as hot from "../hot.ts"; +import * as path from "node:path"; diff --git a/framework/queue.ts b/framework/queue.ts index 532a24928734b26077d25dee6559f1e17a4e02d2..08bfcfb5039455288958fb1a310ead0d4ff4d631 100644 --- a/framework/queue.ts +++ b/framework/queue.ts @@ -165,6 +165,10 @@ export class Queue<T, R> { if (bar) bar[method](); } + + get active(): boolean { + return this.#active.length !== 0; + } } const cwd = process.cwd(); diff --git a/framework/sitegen-lib.ts b/framework/sitegen-lib.ts deleted file mode 100644 index e9ea2d5cbe7928cd191f1b1b9f022035a7fc2cde..0000000000000000000000000000000000000000 --- a/framework/sitegen-lib.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Import this file with 'import * as sg from "#sitegen";' -export type ScriptId = string; - -export interface SitegenRender { - scripts: Set<ScriptId>; -} - -export function initRender(): SitegenRender { - return { - scripts: new Set(), - }; -} - -export function getRender() { - return ssr.getUserData<SitegenRender>("sitegen", () => { - throw new Error( - "This function can only be used in a page (static or view)", - ); - }); -} - -/** Add a client-side script to the page. */ -export function addScript(id: ScriptId) { - const srcFile: string = util.getCallSites() - .find((site) => !site.scriptName.startsWith(import.meta.dirname))! - .scriptName; - const filePath = hot.resolveFrom(srcFile, id); - if ( - !filePath.endsWith(".client.ts") && - !filePath.endsWith(".client.tsx") - ) { - throw new Error("addScript must be a .client.ts or .client.tsx"); - } - getRender().scripts.add(filePath); -} - -export function Script({ src }: { src: ScriptId }) { - if (!src) throw new Error("Missing 'src' attribute"); - addScript(src); - return null; -} - -export interface Section { - root: string; -} - -import * as ssr from "./engine/ssr.ts"; -import * as util from "node:util"; -import * as hot from "./hot.ts"; diff --git a/framework/sitegen.tsx b/framework/sitegen.tsx deleted file mode 100644 index 0a03f1b619a6fa0082f66cefbc58f1e2cb969aa4..0000000000000000000000000000000000000000 --- a/framework/sitegen.tsx +++ /dev/null @@ -1,309 +0,0 @@ -// Sitegen! Clover's static site generator, built with love. - -export function main() { - return withSpinner({ - text: "Recovering State", - successText: ({ elapsed }) => - "sitegen! update in " + elapsed.toFixed(1) + "s", - failureText: () => "sitegen FAIL", - }, sitegen); -} - -/** - * A filesystem object associated with some ID, - * such as a page's route to it's source file. - */ -interface FileItem { - id: string; - file: string; -} - -async function sitegen(status: Spinner) { - const startTime = performance.now(); - - let root = path.resolve(import.meta.dirname, "../src"); - const join = (...sub: string[]) => path.join(root, ...sub); - const incr = new Incremental(); - - // Sitegen reviews every defined section for resources to process - const sections: Section[] = - require(path.join(root, "sections.ts")).siteSections; - - // Static files are compressed and served as-is. - // - "{section}/static/*.png" - let staticFiles: FileItem[] = []; - // Pages are rendered then served as static files. - // - "{section}/pages/*.marko" - let pages: FileItem[] = []; - // Views are dynamically rendered pages called via backend code. - // - "{section}/views/*.tsx" - let views: FileItem[] = []; - // Public scripts are bundled for the client as static assets under "/js/[...]" - // This is used for the file viewer's canvases. - // Note that '.client.ts' can be placed anywhere in the file structure. - // - "{section}/scripts/*.client.ts" - let scripts: FileItem[] = []; - // 'backend.ts' - const backendFiles = []; - - // -- Scan for files -- - status.text = "Scanning Project"; - for (const section of sections) { - const { root: sectionRoot } = section; - const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); - const rootPrefix = root === sectionRoot - ? "" - : path.relative(root, sectionRoot) + "/"; - const kinds = [ - { - dir: sectionPath("pages"), - list: pages, - prefix: "/", - exclude: [".css", ".client.ts", ".client.tsx"], - }, - { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, - { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, - { - dir: sectionPath("views"), - list: views, - prefix: rootPrefix, - exclude: [".css", ".client.ts", ".client.tsx"], - }, - ]; - for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { - const items = fs.readDirRecOptionalSync(dir); - item: for (const item of items) { - if (item.isDirectory()) continue; - for (const e of exclude) { - if (item.name.endsWith(e)) continue item; - } - const file = path.relative(dir, item.parentPath + "/" + item.name); - const trim = ext - ? file - : file.slice(0, -path.extname(file).length).replaceAll(".", "/"); - let id = prefix + trim.replaceAll("\\", "/"); - if (prefix === "/" && id.endsWith("/index")) { - id = id.slice(0, -"/index".length) || "/"; - } - list.push({ id, file: path.join(item.parentPath, item.name) }); - } - } - let backendFile = [ - sectionPath("backend.ts"), - sectionPath("backend.tsx"), - ].find((file) => fs.existsSync(file)); - if (backendFile) backendFiles.push(backendFile); - } - scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); - const globalCssPath = join("global.css"); - - // TODO: invalidate incremental resources - - // -- server side render -- - status.text = "Building"; - const cssOnce = new OnceMap<string>(); - const cssQueue = new Queue<[string, string[], css.Theme], string>({ - name: "Bundle", - fn: ([, files, theme]) => css.bundleCssFiles(files, theme), - passive: true, - getItemText: ([id]) => id, - maxJobs: 2, - }); - interface RenderResult { - body: string; - head: string; - inlineCss: string; - scriptFiles: string[]; - item: FileItem; - } - const renderResults: RenderResult[] = []; - async function loadPageModule({ file }: FileItem) { - require(file); - } - async function renderPage(item: FileItem) { - // -- load and validate module -- - let { default: Page, meta: metadata, theme: pageTheme, layout } = require( - item.file, - ); - if (!Page) throw new Error("Page is missing a 'default' export."); - if (!metadata) { - throw new Error("Page is missing 'meta' export with a title."); - } - if (layout?.theme) pageTheme = layout.theme; - const theme = { - bg: "#fff", - fg: "#050505", - primary: "#2e7dab", - ...pageTheme, - }; - - // -- metadata -- - const renderedMetaPromise = Promise.resolve( - typeof metadata === "function" ? metadata({ ssr: true }) : metadata, - ).then((m) => meta.renderMeta(m)); - // -- css -- - const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; - const cssPromise = cssOnce.get( - cssImports.join(":") + JSON.stringify(theme), - () => cssQueue.add([item.id, cssImports, theme]), - ); - // -- html -- - const bodyPromise = await ssr.ssrAsync(<Page />, { - sitegen: sg.initRender(), - }); - - const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([ - bodyPromise, - cssPromise, - renderedMetaPromise, - ]); - if (!renderedMeta.includes("<title>")) { - throw new Error( - "Page is missing 'meta.title'. " + - "All pages need a title tag.", - ); - } - // The script content is not ready, allow another page to Render. The page - // contents will be rebuilt at the end. This is more convenient anyways - // because it means client scripts don't re-render the page. - renderResults.push({ - body: text, - head: renderedMeta, - inlineCss, - scriptFiles: Array.from(addon.sitegen.scripts), - item: item, - }); - } - // This is done in two passes so that a page that throws during evaluation - // will report "Load Render Module" instead of "Render Static Page". - const spinnerFormat = status.format; - status.format = () => ""; - const moduleLoadQueue = new Queue({ - name: "Load Render Module", - fn: loadPageModule, - getItemText, - maxJobs: 1, - }); - moduleLoadQueue.addMany(pages); - await moduleLoadQueue.done({ method: "stop" }); - const pageQueue = new Queue({ - name: "Render Static Page", - fn: renderPage, - getItemText, - maxJobs: 2, - }); - pageQueue.addMany(pages); - await pageQueue.done({ method: "stop" }); - status.format = spinnerFormat; - - // -- bundle scripts -- - const referencedScripts = Array.from( - new Set(renderResults.flatMap((r) => r.scriptFiles)), - ); - const extraPublicScripts = scripts.map((entry) => entry.file); - const uniqueCount = new Set([ - ...referencedScripts, - ...extraPublicScripts, - ]).size; - status.text = `Bundle ${uniqueCount} Scripts`; - await bundle.bundleClientJavaScript( - referencedScripts, - extraPublicScripts, - incr, - ); - - // -- copy/compress static files -- - async function doStaticFile(item: FileItem) { - const body = await fs.readFile(item.file); - incr.putAsset({ - srcId: "static:" + item.file, - key: item.id, - body, - }); - } - const staticQueue = new Queue({ - name: "Load Static", - fn: doStaticFile, - getItemText, - maxJobs: 16, - }); - status.format = () => ""; - staticQueue.addMany(staticFiles); - await staticQueue.done({ method: "stop" }); - status.format = spinnerFormat; - - // -- concatenate static rendered pages -- - status.text = `Concat ${renderResults.length} Pages`; - await Promise.all( - renderResults.map( - async ({ item: page, body, head, inlineCss, scriptFiles }) => { - const doc = wrapDocument({ - body, - head, - inlineCss, - scripts: scriptFiles.map( - (id) => - UNWRAP( - incr.out.script.get( - path.basename(id).replace(/\.client\.[jt]sx?$/, ""), - ), - ), - ).map((x) => `{${x}}`).join("\n"), - }); - incr.putAsset({ - srcId: "page:" + page.file, - key: page.id, - body: doc, - headers: { - "Content-Type": "text/html", - }, - }); - }, - ), - ); - status.format = () => ""; - status.text = ``; - // This will wait for all compression jobs to finish, which up - // to this point have been left as dangling promises. - await incr.wait(); - - // Flush the site to disk. - status.format = spinnerFormat; - status.text = `Incremental Flush`; - incr.flush(); - incr.serializeToDisk(); // Allows picking up this state again - return { elapsed: (performance.now() - startTime) / 1000 }; -} - -function getItemText({ file }: FileItem) { - return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); -} - -function wrapDocument({ - body, - head, - inlineCss, - scripts, -}: { - head: string; - body: string; - inlineCss: string; - scripts: string; -}) { - return `<!doctype html><head>${head}${ - inlineCss ? `<style>${inlineCss}</style>` : "" - }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`; -} - -import type { Section } from "./sitegen-lib.ts"; -import { OnceMap, Queue } from "./queue.ts"; -import { Incremental } from "./incremental.ts"; -import * as bundle from "./bundle.ts"; -import * as css from "./css.ts"; -import * as fs from "./fs.ts"; -import { Spinner, withSpinner } from "@paperclover/console/Spinner"; -import * as meta from "./meta.ts"; -import * as ssr from "./engine/ssr.ts"; -import * as sg from "./sitegen-lib.ts"; -import * as hot from "./hot.ts"; -import * as path from "node:path"; diff --git a/framework/sqlite.ts b/framework/sqlite.ts index 0343a0de76c4367195f480f256fddf30ed0b3073..1646932f64c99e59c25c24614ece6a81f72a194d 100644 --- a/framework/sqlite.ts +++ b/framework/sqlite.ts @@ -41,11 +41,8 @@ export class WrappedDatabase { (key, version) values (?, ?); `), )); - const { changes, lastInsertRowid } = s.run(name, 1); - console.log(changes, lastInsertRowid); - if (changes === 1) { - this.node.exec(schema); - } + const { changes } = s.run(name, 1); + if (changes === 1) this.node.exec(schema); } prepare<Args extends unknown[] = [], Result = unknown>( diff --git a/package.json b/package.json index a4002aa05ddc9cbfababb944dbbee8bca5d7c8d2..ef8de70c34aea6e9962c5aed9dbae4b131d3c943 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "typescript": "^5.8.3" }, "imports": { - "#sitegen": "./framework/sitegen-lib.ts", + "#backend": "./src/backend.ts", + "#sitegen": "./framework/lib/sitegen.ts", + "#sitegen/*": "./framework/lib/*.ts", "#sqlite": "./framework/sqlite.ts", "#ssr": "./framework/engine/ssr.ts", "#ssr/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts", @@ -26,6 +28,7 @@ "production": "marko/production", "node": "marko/debug/html" }, + "#hono": "hono", "#hono/platform": { "bun": "hono/bun", "deno": "hono/deno", diff --git a/readme.md b/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..efc8523db2744e7de33dbba9dd99217f9e76cdf5 --- /dev/null +++ b/readme.md @@ -0,0 +1 @@ +# clover sitegen framework diff --git a/repl.js b/repl.js index 10b4f1efa0749ee8aaa2d5bf7c99c8d2df3954e6..1fb7330c7c8e108d27658658a29098c7222c6d4f 100644 --- a/repl.js +++ b/repl.js @@ -27,8 +27,7 @@ hot.load("node:repl").start({ }); setTimeout(() => { - hot.reloadRecursive("./framework/engine/ssr.ts"); - hot.reloadRecursive("./framework/bundle.ts"); + hot.reloadRecursive("./framework/generate.tsx"); }, 100); async function evaluate(code) { @@ -41,11 +40,11 @@ async function evaluate(code) { if (code[0] === "=") { try { const result = await eval(code[1]); - console.log(inspect(result)); + console.info(inspect(result)); } catch (err) { if (err instanceof SyntaxError) { const result = await eval("(async() => { return " + code + " })()"); - console.log(inspect(result)); + console.info(inspect(result)); } else { throw err; } diff --git a/run.js b/run.js index 80abcecceb1c246db1effba5b0a46ec489590494..2a0cba8434bcbe5c48e933e4d8da6547673bd54c 100644 --- a/run.js +++ b/run.js @@ -18,7 +18,7 @@ import process from "node:process"; const hot = await import("./framework/hot.ts"); const console = hot.load("@paperclover/console"); -globalThis.console.log = console.info; +globalThis.console["log"] = console.info; globalThis.console.info = console.info; globalThis.console.warn = console.warn; globalThis.console.error = console.error; diff --git a/src/admin.ts b/src/admin.ts new file mode 100644 index 0000000000000000000000000000000000000000..96691e6a59aea94e0436bee5afeae5a156bdada3 --- /dev/null +++ b/src/admin.ts @@ -0,0 +1,64 @@ +const cookieAge = 60 * 60 * 24 * 365; // 1 year + +let lastKnownToken: string | null = null; +function compareToken(token: string) { + if (token === lastKnownToken) return true; + lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + return token === lastKnownToken; +} + +export async function middleware(c: Context, next: Next) { + if (c.req.path.startsWith("/admin")) { + return adminInner(c, next); + } + return next(); +} + +export function adminInner(c: Context, next: Next) { + const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; + + if (c.req.path === "/admin/login") { + const key = c.req.query("key"); + if (key) { + if (compareToken(key)) { + return c.body(null, 303, { + "Location": "/admin", + "Set-Cookie": + `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, + }); + } + return serveAsset(c, "/admin/login/fail", 403); + } + if (token && compareToken(token)) { + return c.redirect("/admin", 303); + } + if (c.req.method === "POST") { + return serveAsset(c, "/admin/login/fail", 403); + } else { + return serveAsset(c, "/admin/login", 200); + } + } + + if (c.req.path === "/admin/logout") { + return c.body(null, 303, { + "Location": "/admin/login", + "Set-Cookie": + `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`, + }); + } + + if (token && compareToken(token)) { + return next(); + } + + return c.redirect("/admin/login", 303); +} + +export function hasAdminToken(c: Context) { + const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; + return token && compareToken(token); +} + +import * as fs from "fs"; +import type { Context, Next } from "hono"; +import { serveAsset } from "#sitegen/assets"; diff --git a/src/backend.ts b/src/backend.ts new file mode 100644 index 0000000000000000000000000000000000000000..5cbda382711d713315caedd0a1ab1fec29bb68a3 --- /dev/null +++ b/src/backend.ts @@ -0,0 +1,38 @@ +const logHttp = scoped("http", { color: "magenta" }); + +const app = new Hono(); + +app.notFound(assets.notFound); + +app.use(trimTrailingSlash()); +app.use(removeDuplicateSlashes); +app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4)))); +app.use(admin.middleware); + +app.route("", require("./q+a/backend.ts").app); +// app.route("", require("./file-viewer/backend.tsx").app); +// app.route("", require("./friends/backend.tsx").app); + +app.use(assets.middleware); + +export default app; + +async function removeDuplicateSlashes(c: Context, next: Next) { + const path = c.req.path; + if (/\/\/+/.test(path)) { + const normalizedPath = path.replace(/\/\/+/g, "/"); + const query = c.req.query(); + const queryString = Object.keys(query).length > 0 + ? "?" + new URLSearchParams(query).toString() + : ""; + return c.redirect(normalizedPath + queryString, 301); + } + await next(); +} + +import { Hono } from "#hono"; +import { logger } from "hono/logger"; +import { trimTrailingSlash } from "hono/trailing-slash"; +import * as assets from "#sitegen/assets"; +import * as admin from "./admin.ts"; +import { scoped } from "@paperclover/console"; diff --git a/src/pages/404.mdx b/src/pages/404.mdx new file mode 100644 index 0000000000000000000000000000000000000000..d5b52e1be51d08521590615922497a65926f90f4 --- /dev/null +++ b/src/pages/404.mdx @@ -0,0 +1,6 @@ +export const meta = { title: 'oh no,,,' }; + +# oh dear + +sound the alarms + diff --git a/src/pages/index.marko b/src/pages/index.marko index 7caf6b523ccd8ba249ebf71a20f7ccf3620d4997..50d72b141eede9c8bf2cfda31f19c2b2264d1f32 100644 --- a/src/pages/index.marko +++ b/src/pages/index.marko @@ -16,7 +16,6 @@ export const meta: Meta = { type: "website", url: "https://paperclover.net", }, - generator: "clover", alternates: { canonical: "https://paperclover.net", types: { diff --git a/src/q+a/backend.ts b/src/q+a/backend.ts new file mode 100644 index 0000000000000000000000000000000000000000..95dec0ca3415362e58a23da8a97bd356c8d115db --- /dev/null +++ b/src/q+a/backend.ts @@ -0,0 +1,3 @@ +export const app = new Hono(); + +import { Hono } from "#hono"; diff --git a/src/q+a/components/Question.marko b/src/q+a/components/Question.marko index a5debf131ad384bc2d2a8ef25414203d1e8b1c5a..063708f72646ac0ed50758d5b30d7c62128c2018 100644 --- a/src/q+a/components/Question.marko +++ b/src/q+a/components/Question.marko @@ -30,7 +30,7 @@ static const transitionDate = 1735639200000; </> // this singleton script will make all the '<time>' tags clickable. -client import "./clickable-links.ts"; +client import "./clickable-links.client.ts"; import type { Question } from "@/q+a/models/Question.ts"; import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts"; diff --git a/src/sections.ts b/src/sections.ts deleted file mode 100644 index a5a6df57047574f71c3edbeb8cc2c0db2abb5b3f..0000000000000000000000000000000000000000 --- a/src/sections.ts +++ /dev/null @@ -1,19 +0,0 @@ -// This file defines the different "Sections" of the website. The sections act -// as separate codebases, but hosted as one. This allows me to have -// sub-projects like the file viewer in 'file', or the question answer system -// in 'q+a'. Each section can define configuration, pages, backend routes, and -// contain other files. -interface Section { - root: string; -} - -const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); - -export const siteSections: Section[] = [ - { root: join("./") }, - { root: join("q+a/") }, - { root: join("file-viewer/") }, - { root: join("friends/") }, -]; - -import * as path from "node:path"; diff --git a/src/site.ts b/src/site.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5a6df57047574f71c3edbeb8cc2c0db2abb5b3f --- /dev/null +++ b/src/site.ts @@ -0,0 +1,19 @@ +// This file defines the different "Sections" of the website. The sections act +// as separate codebases, but hosted as one. This allows me to have +// sub-projects like the file viewer in 'file', or the question answer system +// in 'q+a'. Each section can define configuration, pages, backend routes, and +// contain other files. +interface Section { + root: string; +} + +const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); + +export const siteSections: Section[] = [ + { root: join("./") }, + { root: join("q+a/") }, + { root: join("file-viewer/") }, + { root: join("friends/") }, +]; + +import * as path from "node:path";