diff --git a/framework/bundle.ts b/framework/bundle.ts index 8bc42957bf92feeada2ad1d4a15f3279dff4dcee..78f72d891160120e2989142a1058327b5358ca92 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -1,5 +1,5 @@ // This file implements client-side bundling, mostly wrapping esbuild. -const plugins: esbuild.Plugin[] = [ +const clientPlugins: esbuild.Plugin[] = [ // There are currently no plugins needed by 'paperclover.net' ]; @@ -35,7 +35,7 @@ export async function bundleClientJavaScript( format: "esm", minify: !dev, outdir: "/out!", - plugins, + plugins: clientPlugins, splitting: true, write: false, metafile: true, @@ -82,23 +82,85 @@ export async function bundleClientJavaScript( await Promise.all(promises); } -export async function bundleServerJavaScript(entryPoint: string) { +type ServerPlatform = "node" | "passthru"; +export async function bundleServerJavaScript( + /** Has 'export default app;' */ + backendEntryPoint: string, + /** Views for dynamic loading */ + viewEntryPoints: FileItem[], + platform: ServerPlatform = "node", +) { + const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION"; + const viewSource = [ + ...viewEntryPoints.map((view, i) => + `import * as view${i} from ${JSON.stringify(view.file)}` + ), + `const scripts = ${scriptMagic}[-1]`, + "export const views = {", + ...viewEntryPoints.flatMap((view, i) => [ + ` ${JSON.stringify(view.id)}: {`, + ` component: view${i}.default,`, + ` meta: view${i}.meta,`, + ` layout: view${i}.layout?.default,`, + ` theme: view${i}.layout?.theme ?? view${i}.theme,`, + ` scripts: ${scriptMagic}[${i}]`, + ` },`, + ]), + "}", + ].join("\n"); + const serverPlugins: esbuild.Plugin[] = [ + virtualFiles({ + "$views": viewSource, + }), + banFiles([ + "hot.ts", + "incremental.ts", + "bundle.ts", + "generate.ts", + "css.ts", + ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))), + { + name: "marko", + setup(b) { + b.onLoad({ filter: /\.marko$/ }, async ({ path }) => { + const src = await fs.readFile(path); + const result = await marko.compile(src, path, { + output: "html", + }); + return { + loader: "ts", + contents: result.code, + }; + }); + }, + }, + ]; const bundle = await esbuild.build({ bundle: true, chunkNames: "/js/c.[hash]", entryNames: "/js/[name]", assetNames: "/asset/[hash]", - entryPoints: [entryPoint], + entryPoints: [backendEntryPoint], + platform: "node", format: "esm", - minify: true, - outdir: "/out!", - plugins, + minify: false, + // outdir: "/out!", + outdir: ".clover/wah", + plugins: serverPlugins, splitting: true, - write: false, + write: true, + external: ["@babel/preset-typescript"], }); + console.log(bundle); + throw new Error("wahhh"); } import * as esbuild from "esbuild"; import * as path from "node:path"; import process from "node:process"; +import * as hot from "./hot.ts"; +import { banFiles, virtualFiles } from "./esbuild-support.ts"; import { Incremental } from "./incremental.ts"; +import type { FileItem } from "#sitegen"; +import * as marko from "@marko/compiler"; +import * as fs from "./lib/fs.ts"; diff --git a/framework/css.ts b/framework/css.ts index eda4d3255b480075153c852a7e1ee0c66ea1ea93..2192456da6fb190b02928624a49895e82da46e22 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -48,22 +48,8 @@ export async function bundleCssFiles( path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file ); const plugin = { - name: "clover", + name: "clover css", setup(b) { - b.onResolve( - { filter: /^\$input\$$/ }, - () => ({ path: ".", namespace: "input" }), - ); - b.onLoad( - { filter: /./, namespace: "input" }, - () => ({ - loader: "css", - contents: - cssImports.map((path) => `@import url(${JSON.stringify(path)});`) - .join("\n") + stringifyTheme(theme), - resolveDir: ".", - }), - ); b.onLoad( { filter: /\.css$/ }, async ({ path: file }) => ({ @@ -79,7 +65,18 @@ export async function bundleCssFiles( external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"], metafile: true, minify: !dev, - plugins: [plugin], + plugins: [ + virtualFiles({ + "$input$": { + contents: cssImports.map((path) => + `@import url(${JSON.stringify(path)});` + ) + .join("\n") + stringifyTheme(theme), + loader: "css", + }, + }), + plugin, + ], target: ["ie11"], write: false, }); @@ -102,4 +99,5 @@ import * as esbuild from "esbuild"; import * as fs from "#sitegen/fs"; import * as hot from "./hot.ts"; import * as path from "node:path"; +import { virtualFiles } from "./esbuild-support.ts"; import { Incremental } from "./incremental.ts"; diff --git a/framework/esbuild-support.ts b/framework/esbuild-support.ts new file mode 100644 index 0000000000000000000000000000000000000000..db91ef58ef60d434b6754a9111fe3d1fb1029043 --- /dev/null +++ b/framework/esbuild-support.ts @@ -0,0 +1,58 @@ +export function virtualFiles( + map: Record, +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + // TODO: Proper Escape + `\\$`, + ), + }, + ({ path }) => { + console.log({ path }); + return ({ path, namespace: "vfs" }); + }, + ); + b.onLoad( + { filter: /./, namespace: "vfs" }, + ({ path }) => { + const entry = map[path]; + return ({ + resolveDir: ".", + loader: "ts", + ...typeof entry === "string" ? { contents: entry } : entry, + }); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +export function banFiles( + files: string[], +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + "^(?:" + files.map((file) => string.escapeRegExp(file)).join("|") + + ")$", + ), + }, + ({ path, importer }) => { + throw new Error( + `Loading ${path} (from ${importer}) is banned!`, + ); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +import * as esbuild from "esbuild"; +import * as string from "#sitegen/string"; diff --git a/framework/generate.ts b/framework/generate.ts new file mode 100644 index 0000000000000000000000000000000000000000..66e0b4ce8836897957c858daa4e8a9f6a2fbfc69 --- /dev/null +++ b/framework/generate.ts @@ -0,0 +1,300 @@ +export function main() { + return withSpinner({ + text: "Recovering State", + successText: ({ elapsed }) => + "sitegen! update in " + elapsed.toFixed(1) + "s", + failureText: () => "sitegen FAIL", + }, sitegen); +} + +async function sitegen(status: Spinner) { + const startTime = performance.now(); + + let root = path.resolve(import.meta.dirname, "../src"); + const join = (...sub: string[]) => path.join(root, ...sub); + const incr = Incremental.fromDisk(); + await incr.statAllFiles(); + + // Sitegen reviews every defined section for resources to process + const sections: sg.Section[] = + require(path.join(root, "site.ts")).siteSections; + + // Static files are compressed and served as-is. + // - "{section}/static/*.png" + let staticFiles: FileItem[] = []; + // Pages are rendered then served as static files. + // - "{section}/pages/*.marko" + let pages: FileItem[] = []; + // Views are dynamically rendered pages called via backend code. + // - "{section}/views/*.tsx" + let views: FileItem[] = []; + // Public scripts are bundled for the client as static assets under "/js/[...]" + // This is used for the file viewer's canvases. + // Note that '.client.ts' can be placed anywhere in the file structure. + // - "{section}/scripts/*.client.ts" + let scripts: FileItem[] = []; + + // -- Scan for files -- + status.text = "Scanning Project"; + for (const section of sections) { + const { root: sectionRoot } = section; + const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); + const rootPrefix = root === sectionRoot + ? "" + : path.relative(root, sectionRoot) + "/"; + const kinds = [ + { + dir: sectionPath("pages"), + list: pages, + prefix: "/", + exclude: [".css", ".client.ts", ".client.tsx"], + }, + { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, + { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, + { + dir: sectionPath("views"), + list: views, + prefix: rootPrefix, + exclude: [".css", ".client.ts", ".client.tsx"], + }, + ]; + for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { + const items = fs.readDirRecOptionalSync(dir); + item: for (const subPath of items) { + const file = path.join(dir, subPath); + const stat = fs.statSync(file); + if (stat.isDirectory()) continue; + for (const e of exclude) { + if (subPath.endsWith(e)) continue item; + } + const trim = ext + ? subPath + : subPath.slice(0, -path.extname(subPath).length).replaceAll( + ".", + "/", + ); + let id = prefix + trim.replaceAll("\\", "/"); + if (prefix === "/" && id.endsWith("/index")) { + id = id.slice(0, -"/index".length) || "/"; + } + list.push({ id, file: file }); + } + } + } + scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); + const globalCssPath = join("global.css"); + + // TODO: invalidate incremental resources + + // -- server side render -- + status.text = "Building"; + const cssOnce = new OnceMap(); + const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({ + name: "Bundle", + fn: ([, files, theme]) => css.bundleCssFiles(files, theme), + passive: true, + getItemText: ([id]) => id, + maxJobs: 2, + }); + interface RenderResult { + body: string; + head: string; + css: css.Output; + scriptFiles: string[]; + item: FileItem; + } + const renderResults: RenderResult[] = []; + async function loadPageModule({ file }: FileItem) { + require(file); + } + async function renderPage(item: FileItem) { + // -- load and validate module -- + let { + default: Page, + meta: metadata, + theme: pageTheme, + layout, + } = require(item.file); + if (!Page) throw new Error("Page is missing a 'default' export."); + if (!metadata) { + throw new Error("Page is missing 'meta' export with a title."); + } + if (layout?.theme) pageTheme = layout.theme; + const theme = { + bg: "#fff", + fg: "#050505", + primary: "#2e7dab", + ...pageTheme, + }; + + // -- metadata -- + const renderedMetaPromise = Promise.resolve( + typeof metadata === "function" ? metadata({ ssr: true }) : metadata, + ).then((m) => meta.renderMeta(m)); + // -- css -- + const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; + const cssPromise = cssOnce.get( + cssImports.join(":") + JSON.stringify(theme), + () => cssQueue.add([item.id, cssImports, theme]), + ); + // -- html -- + let page = [engine.kElement, Page, {}]; + if (layout?.default) { + page = [engine.kElement, layout.default, { children: page }]; + } + const bodyPromise = engine.ssrAsync(page, { + sitegen: sg.initRender(), + }); + + const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([ + bodyPromise, + cssPromise, + renderedMetaPromise, + ]); + if (!renderedMeta.includes("")) { + throw new Error( + "Page is missing 'meta.title'. " + + "All pages need a title tag.", + ); + } + // The script content is not ready, allow another page to Render. The page + // contents will be rebuilt at the end. This is more convenient anyways + // because it means client scripts don't re-render the page. + renderResults.push({ + body: text, + head: renderedMeta, + css: cssBundle, + scriptFiles: Array.from(addon.sitegen.scripts), + item: item, + }); + } + // This is done in two passes so that a page that throws during evaluation + // will report "Load Render Module" instead of "Render Static Page". + const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id)); + const spinnerFormat = status.format; + status.format = () => ""; + const moduleLoadQueue = new Queue({ + name: "Load Render Module", + fn: loadPageModule, + getItemText, + maxJobs: 1, + }); + moduleLoadQueue.addMany(neededPages); + await moduleLoadQueue.done({ method: "stop" }); + const pageQueue = new Queue({ + name: "Render Static Page", + fn: renderPage, + getItemText, + maxJobs: 2, + }); + pageQueue.addMany(neededPages); + await pageQueue.done({ method: "stop" }); + status.format = spinnerFormat; + + // -- bundle backend and views -- + status.text = "Bundle backend code"; + const {} = await bundle.bundleServerJavaScript( + join("backend.ts"), + views, + ); + + // -- bundle scripts -- + const referencedScripts = Array.from( + new Set(renderResults.flatMap((r) => r.scriptFiles)), + ); + const extraPublicScripts = scripts.map((entry) => entry.file); + const uniqueCount = new Set([ + ...referencedScripts, + ...extraPublicScripts, + ]).size; + status.text = `Bundle ${uniqueCount} Scripts`; + await bundle.bundleClientJavaScript( + referencedScripts, + extraPublicScripts, + incr, + ); + + // -- copy/compress static files -- + async function doStaticFile(item: FileItem) { + const body = await fs.readFile(item.file); + await incr.putAsset({ + sources: [item.file], + key: item.id, + body, + }); + } + const staticQueue = new Queue({ + name: "Load Static", + fn: doStaticFile, + getItemText, + maxJobs: 16, + }); + status.format = () => ""; + staticQueue.addMany( + staticFiles.filter((file) => incr.needsBuild("asset", file.id)), + ); + await staticQueue.done({ method: "stop" }); + status.format = spinnerFormat; + + // -- concatenate static rendered pages -- + status.text = `Concat ${renderResults.length} Pages`; + await Promise.all( + renderResults.map( + async ( + { item: page, body, head, css, scriptFiles }, + ) => { + const doc = wrapDocument({ + body, + head, + inlineCss: css.text, + scripts: scriptFiles.map( + (id) => + UNWRAP( + incr.out.script.get( + path.basename(id).replace(/\.client\.[jt]sx?$/, ""), + ), + ), + ).map((x) => `{${x}}`).join("\n"), + }); + await incr.putAsset({ + sources: [page.file, ...css.sources], + key: page.id, + body: doc, + headers: { + "Content-Type": "text/html", + }, + }); + }, + ), + ); + status.format = () => ""; + status.text = ``; + // This will wait for all compression jobs to finish, which up + // to this point have been left as dangling promises. + await incr.wait(); + + // Flush the site to disk. + status.format = spinnerFormat; + status.text = `Incremental Flush`; + incr.flush(); // Write outputs + incr.toDisk(); // Allows picking up this state again + return { elapsed: (performance.now() - startTime) / 1000 }; +} + +function getItemText({ file }: FileItem) { + return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); +} + +import { OnceMap, Queue } from "./queue.ts"; +import { Incremental } from "./incremental.ts"; +import * as bundle from "./bundle.ts"; +import * as css from "./css.ts"; +import * as engine from "./engine/ssr.ts"; +import * as hot from "./hot.ts"; +import * as fs from "#sitegen/fs"; +import * as sg from "#sitegen"; +import type { FileItem } from "#sitegen"; +import * as path from "node:path"; +import * as meta from "#sitegen/meta"; +import { Spinner, withSpinner } from "@paperclover/console/Spinner"; +import { wrapDocument } from "./lib/view.ts"; diff --git a/framework/generate.tsx b/framework/generate.tsx deleted file mode 100644 index 63964b6558bb1ece9beec60de6fa4304886f7290..0000000000000000000000000000000000000000 --- a/framework/generate.tsx +++ /dev/null @@ -1,316 +0,0 @@ -export function main() { - return withSpinner({ - text: "Recovering State", - successText: ({ elapsed }) => - "sitegen! update in " + elapsed.toFixed(1) + "s", - failureText: () => "sitegen FAIL", - }, sitegen); -} - -/** - * A filesystem object associated with some ID, - * such as a page's route to it's source file. - */ -interface FileItem { - id: string; - file: string; -} - -async function sitegen(status: Spinner) { - const startTime = performance.now(); - - let root = path.resolve(import.meta.dirname, "../src"); - const join = (...sub: string[]) => path.join(root, ...sub); - const incr = Incremental.fromDisk(); - await incr.statAllFiles(); - - // Sitegen reviews every defined section for resources to process - const sections: sg.Section[] = - require(path.join(root, "site.ts")).siteSections; - - // Static files are compressed and served as-is. - // - "{section}/static/*.png" - let staticFiles: FileItem[] = []; - // Pages are rendered then served as static files. - // - "{section}/pages/*.marko" - let pages: FileItem[] = []; - // Views are dynamically rendered pages called via backend code. - // - "{section}/views/*.tsx" - let views: FileItem[] = []; - // Public scripts are bundled for the client as static assets under "/js/[...]" - // This is used for the file viewer's canvases. - // Note that '.client.ts' can be placed anywhere in the file structure. - // - "{section}/scripts/*.client.ts" - let scripts: FileItem[] = []; - - // -- Scan for files -- - status.text = "Scanning Project"; - for (const section of sections) { - const { root: sectionRoot } = section; - const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); - const rootPrefix = root === sectionRoot - ? "" - : path.relative(root, sectionRoot) + "/"; - const kinds = [ - { - dir: sectionPath("pages"), - list: pages, - prefix: "/", - exclude: [".css", ".client.ts", ".client.tsx"], - }, - { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true }, - { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix }, - { - dir: sectionPath("views"), - list: views, - prefix: rootPrefix, - exclude: [".css", ".client.ts", ".client.tsx"], - }, - ]; - for (const { dir, list, prefix, exclude = [], ext = false } of kinds) { - const items = fs.readDirRecOptionalSync(dir); - item: for (const subPath of items) { - const file = path.join(dir, subPath); - const stat = fs.statSync(file); - if (stat.isDirectory()) continue; - for (const e of exclude) { - if (subPath.endsWith(e)) continue item; - } - const trim = ext - ? subPath - : subPath.slice(0, -path.extname(subPath).length).replaceAll( - ".", - "/", - ); - let id = prefix + trim.replaceAll("\\", "/"); - if (prefix === "/" && id.endsWith("/index")) { - id = id.slice(0, -"/index".length) || "/"; - } - list.push({ id, file: file }); - } - } - } - scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); - const globalCssPath = join("global.css"); - - // TODO: invalidate incremental resources - - // -- server side render -- - status.text = "Building"; - const cssOnce = new OnceMap<css.Output>(); - const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({ - name: "Bundle", - fn: ([, files, theme]) => css.bundleCssFiles(files, theme), - passive: true, - getItemText: ([id]) => id, - maxJobs: 2, - }); - interface RenderResult { - body: string; - head: string; - css: css.Output; - scriptFiles: string[]; - item: FileItem; - } - const renderResults: RenderResult[] = []; - async function loadPageModule({ file }: FileItem) { - require(file); - } - async function renderPage(item: FileItem) { - // -- load and validate module -- - let { default: Page, meta: metadata, theme: pageTheme, layout } = require( - item.file, - ); - if (!Page) throw new Error("Page is missing a 'default' export."); - if (!metadata) { - throw new Error("Page is missing 'meta' export with a title."); - } - if (layout?.theme) pageTheme = layout.theme; - const theme = { - bg: "#fff", - fg: "#050505", - primary: "#2e7dab", - ...pageTheme, - }; - - // -- metadata -- - const renderedMetaPromise = Promise.resolve( - typeof metadata === "function" ? metadata({ ssr: true }) : metadata, - ).then((m) => meta.renderMeta(m)); - // -- css -- - const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; - const cssPromise = cssOnce.get( - cssImports.join(":") + JSON.stringify(theme), - () => cssQueue.add([item.id, cssImports, theme]), - ); - // -- html -- - let page = <Page />; - if (layout?.default) { - const Layout = layout.default; - page = <Layout>{page}</Layout>; - } - const bodyPromise = await ssr.ssrAsync(page, { - sitegen: sg.initRender(), - }); - - const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([ - bodyPromise, - cssPromise, - renderedMetaPromise, - ]); - if (!renderedMeta.includes("<title>")) { - throw new Error( - "Page is missing 'meta.title'. " + - "All pages need a title tag.", - ); - } - // The script content is not ready, allow another page to Render. The page - // contents will be rebuilt at the end. This is more convenient anyways - // because it means client scripts don't re-render the page. - renderResults.push({ - body: text, - head: renderedMeta, - css: cssBundle, - scriptFiles: Array.from(addon.sitegen.scripts), - item: item, - }); - } - // This is done in two passes so that a page that throws during evaluation - // will report "Load Render Module" instead of "Render Static Page". - const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id)); - const spinnerFormat = status.format; - status.format = () => ""; - const moduleLoadQueue = new Queue({ - name: "Load Render Module", - fn: loadPageModule, - getItemText, - maxJobs: 1, - }); - moduleLoadQueue.addMany(neededPages); - await moduleLoadQueue.done({ method: "stop" }); - const pageQueue = new Queue({ - name: "Render Static Page", - fn: renderPage, - getItemText, - maxJobs: 2, - }); - pageQueue.addMany(neededPages); - await pageQueue.done({ method: "stop" }); - status.format = spinnerFormat; - - // -- bundle scripts -- - const referencedScripts = Array.from( - new Set(renderResults.flatMap((r) => r.scriptFiles)), - ); - const extraPublicScripts = scripts.map((entry) => entry.file); - const uniqueCount = new Set([ - ...referencedScripts, - ...extraPublicScripts, - ]).size; - status.text = `Bundle ${uniqueCount} Scripts`; - await bundle.bundleClientJavaScript( - referencedScripts, - extraPublicScripts, - incr, - ); - - // -- copy/compress static files -- - async function doStaticFile(item: FileItem) { - const body = await fs.readFile(item.file); - await incr.putAsset({ - sources: [item.file], - key: item.id, - body, - }); - } - const staticQueue = new Queue({ - name: "Load Static", - fn: doStaticFile, - getItemText, - maxJobs: 16, - }); - status.format = () => ""; - staticQueue.addMany( - staticFiles.filter((file) => incr.needsBuild("asset", file.id)), - ); - await staticQueue.done({ method: "stop" }); - status.format = spinnerFormat; - - // -- concatenate static rendered pages -- - status.text = `Concat ${renderResults.length} Pages`; - await Promise.all( - renderResults.map( - async ( - { item: page, body, head, css, scriptFiles }, - ) => { - const doc = wrapDocument({ - body, - head, - inlineCss: css.text, - scripts: scriptFiles.map( - (id) => - UNWRAP( - incr.out.script.get( - path.basename(id).replace(/\.client\.[jt]sx?$/, ""), - ), - ), - ).map((x) => `{${x}}`).join("\n"), - }); - await incr.putAsset({ - sources: [page.file, ...css.sources], - key: page.id, - body: doc, - headers: { - "Content-Type": "text/html", - }, - }); - }, - ), - ); - status.format = () => ""; - status.text = ``; - // This will wait for all compression jobs to finish, which up - // to this point have been left as dangling promises. - await incr.wait(); - - // Flush the site to disk. - status.format = spinnerFormat; - status.text = `Incremental Flush`; - incr.flush(); - incr.toDisk(); // Allows picking up this state again - return { elapsed: (performance.now() - startTime) / 1000 }; -} - -function getItemText({ file }: FileItem) { - return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); -} - -function wrapDocument({ - body, - head, - inlineCss, - scripts, -}: { - head: string; - body: string; - inlineCss: string; - scripts: string; -}) { - return `<!doctype html><html lang=en><head>${head}${ - inlineCss ? `<style>${inlineCss}</style>` : "" - }</head><body>${body}${ - scripts ? `<script>${scripts}</script>` : "" - }</body></html>`; -} - -import { OnceMap, Queue } from "./queue.ts"; -import { Incremental } from "./incremental.ts"; -import * as bundle from "./bundle.ts"; -import * as css from "./css.ts"; -import * as ssr from "./engine/ssr.ts"; -import * as hot from "./hot.ts"; -import * as fs from "#sitegen/fs"; -import * as sg from "#sitegen"; -import * as path from "node:path"; -import * as meta from "#sitegen/meta"; -import { Spinner, withSpinner } from "@paperclover/console/Spinner"; diff --git a/framework/hot.ts b/framework/hot.ts index 728845eb13b383b4c22efc109f053d7f3fe1e734..666701cf4176131367562b09bb2fc5130c1c9cd1 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -36,12 +36,6 @@ export interface FileStat { imports: string[]; } let fsGraph = new Map<string, FileStat>(); -export function setFsGraph(g: Map<string, FileStat>) { - if (fsGraph.size > 0) { - throw new Error("Cannot restore fsGraph when it has been written into"); - } - fsGraph = g; -} export function getFsGraph() { return fsGraph; } @@ -108,11 +102,18 @@ Module._resolveFilename = (...args) => { }; function loadEsbuild(module: NodeJS.Module, filepath: string) { - let src = fs.readFileSync(filepath, "utf8"); - return loadEsbuildCode(module, filepath, src); + return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8")); } -function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { +interface LoadOptions { + scannedClientRefs?: string[]; +} +function loadEsbuildCode( + module: NodeJS.Module, + filepath: string, + src: string, + opt: LoadOptions = {}, +) { if (filepath === import.meta.filename) { module.exports = self; return; @@ -122,12 +123,13 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { if (filepath.endsWith(".ts")) loader = "ts"; else if (filepath.endsWith(".jsx")) loader = "jsx"; else if (filepath.endsWith(".js")) loader = "js"; + module.cloverClientRefs = opt.scannedClientRefs ?? extractClientScripts(src); if (src.includes("import.meta")) { src = ` import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())}; import.meta.dirname = ${JSON.stringify(path.dirname(filepath))}; import.meta.filename = ${JSON.stringify(filepath)}; - ` + src; + `.trim().replace(/\n/g, "") + src; } src = esbuild.transformSync(src, { loader, @@ -148,7 +150,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) { if (src.match(/^\s*client\s+import\s+["']/m)) { src = src.replace( /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m, - "<CloverScriptInclude src=$1 />", + (_, src) => `<CloverScriptInclude src=${src} />`, ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n'; } @@ -206,9 +208,40 @@ export function resolveFrom(src: string, dest: string) { } } +const importRegExp = + /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s; +const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))/; +export function extractClientScripts(source: string): string[] { + // This match finds a call to 'import ... from "#sitegen"' + const importMatch = source.match(importRegExp); + if (!importMatch) return []; + const items = importMatch[1]; + let identifier = ""; + if (items.startsWith("{")) { + const clauseMatch = items.match(getSitegenAddScriptRegExp); + if (!clauseMatch) return []; // did not import + identifier = clauseMatch[1] || "addScript"; + } else if (items.startsWith("*")) { + identifier = importMatch[2] + "\\s*\\.\\s*addScript"; + } else { + throw new Error("Impossible"); + } + identifier = identifier.replaceAll("$", "\\$"); // only needed escape + const findCallsRegExp = new RegExp( + `\\b${identifier}\\s*\\(("[^"]+"|'[^']+')\\)`, + "gs", + ); + const calls = source.matchAll(findCallsRegExp); + return [...calls].map((call) => { + return JSON.parse(`"${call[1].slice(1, -1)}"`) as string; + }); +} + declare global { namespace NodeJS { interface Module { + cloverClientRefs?: string[]; + _compile( this: NodeJS.Module, content: string, diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index 5404b8c243c9b5e82f25eaafbd6b160694d51ab6..dbe30f1f35157a6a84e4ae5050238652869eaf8d 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -1,6 +1,15 @@ // Import this file with 'import * as sg from "#sitegen";' export type ScriptId = string; +/** + * A filesystem object associated with some ID, + * such as a page's route to it's source file. + */ +export interface FileItem { + id: string; + file: string; +} + const frameworkDir = path.dirname(import.meta.dirname); export interface SitegenRender { diff --git a/framework/lib/string.ts b/framework/lib/string.ts new file mode 100644 index 0000000000000000000000000000000000000000..239e02ebc2b2bb17fd60200403d365b82d7a030a --- /dev/null +++ b/framework/lib/string.ts @@ -0,0 +1,3 @@ +export function escapeRegExp(source: string) { + return source.replace(/[\$\\]/g, "\\$&"); +} diff --git a/framework/lib/view.ts b/framework/lib/view.ts new file mode 100644 index 0000000000000000000000000000000000000000..a133529e1fb2b71811eef8d83f5ed5128616d75e --- /dev/null +++ b/framework/lib/view.ts @@ -0,0 +1,87 @@ +// This import is generated by code 'bundle.ts' +export interface View { + component: engine.Component; + meta: + | meta.Meta + | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta); + layout?: engine.Component; + theme?: css.Theme; + inlineCss: string; + scripts: Record<string, string>; +} +let views: Record<string, View> = {}; + +// An older version of the Clover Engine supported streaming suspense +// boundaries, but those were never used. Pages will wait until they +// are fully rendered before sending. +async function renderView( + c: hono.Context, + id: string, + props: Record<string, unknown>, +) { + views = require("$views").views; + // The view contains pre-bundled CSS and scripts, but keeps the scripts + // separate for run-time dynamic scripts. For example, the file viewer + // includes the canvas for the current page, but only the current page. + const { + component, + inlineCss, + layout, + meta: metadata, + scripts, + theme, + }: View = UNWRAP(views[id]); + + // -- metadata -- + const renderedMetaPromise = Promise.resolve( + typeof metadata === "function" ? metadata({ context: c }) : metadata, + ).then((m) => meta.renderMeta(m)); + + // -- html -- + let page: engine.Element = [engine.kElement, component, props]; + if (layout) page = [engine.kElement, layout, { children: page }]; + const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { + sitegen: sg.initRender(), + }); + + // -- join document and send -- + return c.html(wrapDocument({ + body, + head: await renderedMetaPromise, + inlineCss, + scripts: joinScripts( + Array.from(sitegen.scripts, (script) => scripts[script]), + ), + })); +} + +export function joinScripts(scriptSources: string[]) { + const { length } = scriptSources; + if (length === 0) return ""; + if (length === 1) return scriptSources[0]; + return scriptSources.map((source) => `{${source}}`).join(";"); +} + +export function wrapDocument({ + body, + head, + inlineCss, + scripts, +}: { + head: string; + body: string; + inlineCss: string; + scripts: string; +}) { + return `<!doctype html><html lang=en><head>${head}${ + inlineCss ? `<style>${inlineCss}</style>` : "" + }</head><body>${body}${ + scripts ? `<script>${scripts}</script>` : "" + }</body></html>`; +} + +import * as meta from "./meta.ts"; +import type * as hono from "#hono"; +import * as engine from "../engine/ssr.ts"; +import type * as css from "../css.ts"; +import * as sg from "./sitegen.ts"; diff --git a/readme.md b/readme.md index e31405fb7fd4369fffaed954c8d3343ca07df818..05ce24f09900f925db1e375b1255984b09038a73 100644 --- a/readme.md +++ b/readme.md @@ -36,7 +36,9 @@ Included is `src`, which contains `paperclover.net`. Website highlights: minimum system requirements: - a cpu with at least 1 core. - random access memory. -- windows 7 or later, macos, or linux operating system. +- windows 7 or later, macos, or other operating system. + +my development machine, for example, is Dell Inspiron 7348 with Core i7 ``` npm install diff --git a/repl.js b/repl.js index 1fb7330c7c8e108d27658658a29098c7222c6d4f..f23342dce533325232e313c455e8632e43df88fc 100644 --- a/repl.js +++ b/repl.js @@ -27,7 +27,7 @@ hot.load("node:repl").start({ }); setTimeout(() => { - hot.reloadRecursive("./framework/generate.tsx"); + hot.reloadRecursive("./framework/generate.ts"); }, 100); async function evaluate(code) { diff --git a/src/backend.ts b/src/backend.ts index 3e6c20fee5870a9daa9ba18cd580222cc695ce4c..025239971eae5d5fd0421b1f748c9f3d0c9680d9 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,3 +1,5 @@ +import * as v from "#sitegen/view"; +console.log(v); const logHttp = scoped("http", { color: "magenta" }); const app = new Hono(); diff --git a/src/components/Video.tsx b/src/components/Video.tsx index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8c4ce75ad5e59e641ebc532669a7e92a522d9169 100644 --- a/src/components/Video.tsx +++ b/src/components/Video.tsx @@ -0,0 +1,57 @@ +import * as path from "node:path"; +import { addScript } from "#sitegen"; +import { PrecomputedBlurhash } from "./blurhash.tsx"; +import "./Video.css"; +export namespace Video { + export interface Props { + title: string; + width: number; + height: number; + sources: string[]; + downloads: string[]; + poster?: string; + posterHash?: string; + borderless?: boolean; + } +} + function PrecomputedBlurhash({ hash }: { hash: string }) { +export function Video( + { title, sources, height, poster, posterHash, width, borderless }: + Video.Props, +) { + addScript("./video.client.ts"); + return ( + <figure class={`video ${borderless ? "borderless" : ""}`}> + <figcaption>{title}</figcaption> + {/* posterHash && <PrecomputedBlurhash hash={posterHash} /> */} + {poster && <img src={poster} alt="waterfalls" />} + <video + controls + preload="none" + style={`width:100%;background:transparent;aspect-ratio:${ + simplifyFraction(width, height) + }`} + poster="data:null" + > + {sources.map((src) => ( + <source + src={src} + type={contentTypeFromExt(src)} + /> + ))} + </video> + </figure> + ); +} +export function contentTypeFromExt(src: string) { + if (src.endsWith(".m3u8")) return "application/x-mpegURL"; + if (src.endsWith(".webm")) return "video/webm"; + if (src.endsWith(".mp4")) return "video/mp4"; + if (src.endsWith(".ogg")) return "video/ogg"; + throw new Error("Unknown video extension: " + path.extname(src)); +} +const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a; +function simplifyFraction(n: number, d: number) { + const divisor = gcd(n, d); + return `${n / divisor}/${d / divisor}`; +} diff --git a/src/q+a/views/backend-inbox.marko b/src/q+a/views/backend-inbox.marko index cddd372640eb976d0dab116438f44ef19dfdd950..aa3d62a91820b3565b0572fa15c27613a2eeb7b4 100644 --- a/src/q+a/views/backend-inbox.marko +++ b/src/q+a/views/backend-inbox.marko @@ -8,7 +8,6 @@ import { } from "../q+a/QuestionRender"; <const/questions = PendingQuestion.getAll() /> -<Script src="backend-inbox.client.ts" /> <h1>inbox</h1> <for|q| of=questions> @@ -16,7 +15,7 @@ import { data-q=q.id style="border-bottom: 2px solid #fff7; margin-bottom: 1rem" > - <time datetime={formatQuestionISOTimestamp(q.date)}> + <time datetime=formatQuestionISOTimestamp(q.date)> ${formatQuestionTimestamp(q.date)} ${q.id} </time> <div style="color: dodgerblue; margin-bottom: 0.25rem"> @@ -26,7 +25,7 @@ import { </div> <p style="white-space: pre-wrap">${q.prompt}</p> <p> - <button onclick=`onReply("${q.id}") style="color: lime"> + <button onclick=`onReply("${q.id}")` style="color: lime"> reply </button> <button onclick=`onDelete("${q.id}")` style="color: red"> @@ -38,3 +37,5 @@ import { </p> </div> </> + +client import "backend-inbox.client.ts"; diff --git a/src/q+a/views/fail.marko b/src/q+a/views/fail.marko index a68cfd5a2ccfa057cc1e077c3642cc15fe556cfa..1c45e399b331913da305425f205e4acf3505c013 100644 --- a/src/q+a/views/fail.marko +++ b/src/q+a/views/fail.marko @@ -1,4 +1,4 @@ -import * as layout from "../layouts/questions.tsx"; +import * as layout from "../layout.tsx"; export { layout }; export const theme = { ...layout.theme, @@ -10,18 +10,14 @@ export const theme = { <p style="color: red"> ${error} </p> -{ - content && ( - <> - <br /> - <br /> - <p> - here is a copy of what you wrote, if you want to try again: - </p> - <pre style="white-space: pre-wrap"><code>${content}</code></pre> - </> - ) -} +<if=content> + <br /> + <br /> + <p> + here is a copy of what you wrote, if you want to try again: + </p> + <pre style="white-space: pre-wrap"><code>${content}</code></pre> +</> <br /> <br /> <p> diff --git a/src/q+a/views/success.marko b/src/q+a/views/success.marko index 26beec95c8f95223b06666ea68d9fe1895145894..f9b12386feafe13937658cb068aa8fb3e637f848 100644 --- a/src/q+a/views/success.marko +++ b/src/q+a/views/success.marko @@ -3,7 +3,7 @@ export interface Input { } <const/{ permalink }=input/> -import * as layout from "../layouts/questions.tsx"; +import * as layout from "../layout.tsx"; export { layout }; export const theme = { ...layout.theme,