authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-09 21:13:51-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log93498bf84ee5dd9ad221152af00a7da852ed406c
tree644e661f13812cc584b3ad7f6ff422be5a4c6204
parent2fa9b4eb6ee7d21505a0def86bd0c36801869e9a
signature Commit is signed but in an unrecognized format.

stuff

16 files changed, 662 insertions(+), 370 deletions(-)

framework/bundle.ts+70-8
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1// This file implements client-side bundling, mostly wrapping esbuild.1// This file implements client-side bundling, mostly wrapping esbuild.
2const plugins: esbuild.Plugin[] = [2const clientPlugins: esbuild.Plugin[] = [
3 // There are currently no plugins needed by 'paperclover.net'3 // There are currently no plugins needed by 'paperclover.net'
4];4];
55
...@@ -35,7 +35,7 @@ export async function bundleClientJavaScript(...@@ -35,7 +35,7 @@ export async function bundleClientJavaScript(
35 format: "esm",35 format: "esm",
36 minify: !dev,36 minify: !dev,
37 outdir: "/out!",37 outdir: "/out!",
38 plugins,38 plugins: clientPlugins,
39 splitting: true,39 splitting: true,
40 write: false,40 write: false,
41 metafile: true,41 metafile: true,
...@@ -82,23 +82,85 @@ export async function bundleClientJavaScript(...@@ -82,23 +82,85 @@ export async function bundleClientJavaScript(
82 await Promise.all(promises);82 await Promise.all(promises);
83}83}
8484
85export async function bundleServerJavaScript(entryPoint: string) {85type ServerPlatform = "node" | "passthru";
86export async function bundleServerJavaScript(
87 /** Has 'export default app;' */
88 backendEntryPoint: string,
89 /** Views for dynamic loading */
90 viewEntryPoints: FileItem[],
91 platform: ServerPlatform = "node",
92) {
93 const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
94 const viewSource = [
95 ...viewEntryPoints.map((view, i) =>
96 `import * as view${i} from ${JSON.stringify(view.file)}`
97 ),
98 `const scripts = ${scriptMagic}[-1]`,
99 "export const views = {",
100 ...viewEntryPoints.flatMap((view, i) => [
101 ` ${JSON.stringify(view.id)}: {`,
102 ` component: view${i}.default,`,
103 ` meta: view${i}.meta,`,
104 ` layout: view${i}.layout?.default,`,
105 ` theme: view${i}.layout?.theme ?? view${i}.theme,`,
106 ` scripts: ${scriptMagic}[${i}]`,
107 ` },`,
108 ]),
109 "}",
110 ].join("\n");
111 const serverPlugins: esbuild.Plugin[] = [
112 virtualFiles({
113 "$views": viewSource,
114 }),
115 banFiles([
116 "hot.ts",
117 "incremental.ts",
118 "bundle.ts",
119 "generate.ts",
120 "css.ts",
121 ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))),
122 {
123 name: "marko",
124 setup(b) {
125 b.onLoad({ filter: /\.marko$/ }, async ({ path }) => {
126 const src = await fs.readFile(path);
127 const result = await marko.compile(src, path, {
128 output: "html",
129 });
130 return {
131 loader: "ts",
132 contents: result.code,
133 };
134 });
135 },
136 },
137 ];
86 const bundle = await esbuild.build({138 const bundle = await esbuild.build({
87 bundle: true,139 bundle: true,
88 chunkNames: "/js/c.[hash]",140 chunkNames: "/js/c.[hash]",
89 entryNames: "/js/[name]",141 entryNames: "/js/[name]",
90 assetNames: "/asset/[hash]",142 assetNames: "/asset/[hash]",
91 entryPoints: [entryPoint],143 entryPoints: [backendEntryPoint],
144 platform: "node",
92 format: "esm",145 format: "esm",
93 minify: true,146 minify: false,
94 outdir: "/out!",147 // outdir: "/out!",
95 plugins,148 outdir: ".clover/wah",
149 plugins: serverPlugins,
96 splitting: true,150 splitting: true,
97 write: false,151 write: true,
152 external: ["@babel/preset-typescript"],
98 });153 });
154 console.log(bundle);
155 throw new Error("wahhh");
99}156}
100157
101import * as esbuild from "esbuild";158import * as esbuild from "esbuild";
102import * as path from "node:path";159import * as path from "node:path";
103import process from "node:process";160import process from "node:process";
161import * as hot from "./hot.ts";
162import { banFiles, virtualFiles } from "./esbuild-support.ts";
104import { Incremental } from "./incremental.ts";163import { Incremental } from "./incremental.ts";
164import type { FileItem } from "#sitegen";
165import * as marko from "@marko/compiler";
166import * as fs from "./lib/fs.ts";
framework/css.ts+14-16
...@@ -48,22 +48,8 @@ export async function bundleCssFiles(...@@ -48,22 +48,8 @@ export async function bundleCssFiles(
48 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file48 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
49 );49 );
50 const plugin = {50 const plugin = {
51 name: "clover",51 name: "clover css",
52 setup(b) {52 setup(b) {
53 b.onResolve(
54 { filter: /^\$input\$$/ },
55 () => ({ path: ".", namespace: "input" }),
56 );
57 b.onLoad(
58 { filter: /./, namespace: "input" },
59 () => ({
60 loader: "css",
61 contents:
62 cssImports.map((path) => `@import url(${JSON.stringify(path)});`)
63 .join("\n") + stringifyTheme(theme),
64 resolveDir: ".",
65 }),
66 );
67 b.onLoad(53 b.onLoad(
68 { filter: /\.css$/ },54 { filter: /\.css$/ },
69 async ({ path: file }) => ({55 async ({ path: file }) => ({
...@@ -79,7 +65,18 @@ export async function bundleCssFiles(...@@ -79,7 +65,18 @@ export async function bundleCssFiles(
79 external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"],65 external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"],
80 metafile: true,66 metafile: true,
81 minify: !dev,67 minify: !dev,
82 plugins: [plugin],68 plugins: [
69 virtualFiles({
70 "$input$": {
71 contents: cssImports.map((path) =>
72 `@import url(${JSON.stringify(path)});`
73 )
74 .join("\n") + stringifyTheme(theme),
75 loader: "css",
76 },
77 }),
78 plugin,
79 ],
83 target: ["ie11"],80 target: ["ie11"],
84 write: false,81 write: false,
85 });82 });
...@@ -102,4 +99,5 @@ import * as esbuild from "esbuild";...@@ -102,4 +99,5 @@ import * as esbuild from "esbuild";
102import * as fs from "#sitegen/fs";99import * as fs from "#sitegen/fs";
103import * as hot from "./hot.ts";100import * as hot from "./hot.ts";
104import * as path from "node:path";101import * as path from "node:path";
102import { virtualFiles } from "./esbuild-support.ts";
105import { Incremental } from "./incremental.ts";103import { Incremental } from "./incremental.ts";
framework/esbuild-support.ts created+58
...@@ -0,0 +1,58 @@
1export function virtualFiles(
2 map: Record<string, string | esbuild.OnLoadResult>,
3) {
4 return {
5 name: "clover vfs",
6 setup(b) {
7 b.onResolve(
8 {
9 filter: new RegExp(
10 // TODO: Proper Escape
11 `\\$`,
12 ),
13 },
14 ({ path }) => {
15 console.log({ path });
16 return ({ path, namespace: "vfs" });
17 },
18 );
19 b.onLoad(
20 { filter: /./, namespace: "vfs" },
21 ({ path }) => {
22 const entry = map[path];
23 return ({
24 resolveDir: ".",
25 loader: "ts",
26 ...typeof entry === "string" ? { contents: entry } : entry,
27 });
28 },
29 );
30 },
31 } satisfies esbuild.Plugin;
32}
33
34export function banFiles(
35 files: string[],
36) {
37 return {
38 name: "clover vfs",
39 setup(b) {
40 b.onResolve(
41 {
42 filter: new RegExp(
43 "^(?:" + files.map((file) => string.escapeRegExp(file)).join("|") +
44 ")$",
45 ),
46 },
47 ({ path, importer }) => {
48 throw new Error(
49 `Loading ${path} (from ${importer}) is banned!`,
50 );
51 },
52 );
53 },
54 } satisfies esbuild.Plugin;
55}
56
57import * as esbuild from "esbuild";
58import * as string from "#sitegen/string";
framework/generate.ts created+300
...@@ -0,0 +1,300 @@
1export function main() {
2 return withSpinner({
3 text: "Recovering State",
4 successText: ({ elapsed }) =>
5 "sitegen! update in " + elapsed.toFixed(1) + "s",
6 failureText: () => "sitegen FAIL",
7 }, sitegen);
8}
9
10async function sitegen(status: Spinner) {
11 const startTime = performance.now();
12
13 let root = path.resolve(import.meta.dirname, "../src");
14 const join = (...sub: string[]) => path.join(root, ...sub);
15 const incr = Incremental.fromDisk();
16 await incr.statAllFiles();
17
18 // Sitegen reviews every defined section for resources to process
19 const sections: sg.Section[] =
20 require(path.join(root, "site.ts")).siteSections;
21
22 // Static files are compressed and served as-is.
23 // - "{section}/static/*.png"
24 let staticFiles: FileItem[] = [];
25 // Pages are rendered then served as static files.
26 // - "{section}/pages/*.marko"
27 let pages: FileItem[] = [];
28 // Views are dynamically rendered pages called via backend code.
29 // - "{section}/views/*.tsx"
30 let views: FileItem[] = [];
31 // Public scripts are bundled for the client as static assets under "/js/[...]"
32 // This is used for the file viewer's canvases.
33 // Note that '.client.ts' can be placed anywhere in the file structure.
34 // - "{section}/scripts/*.client.ts"
35 let scripts: FileItem[] = [];
36
37 // -- Scan for files --
38 status.text = "Scanning Project";
39 for (const section of sections) {
40 const { root: sectionRoot } = section;
41 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
42 const rootPrefix = root === sectionRoot
43 ? ""
44 : path.relative(root, sectionRoot) + "/";
45 const kinds = [
46 {
47 dir: sectionPath("pages"),
48 list: pages,
49 prefix: "/",
50 exclude: [".css", ".client.ts", ".client.tsx"],
51 },
52 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
53 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
54 {
55 dir: sectionPath("views"),
56 list: views,
57 prefix: rootPrefix,
58 exclude: [".css", ".client.ts", ".client.tsx"],
59 },
60 ];
61 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
62 const items = fs.readDirRecOptionalSync(dir);
63 item: for (const subPath of items) {
64 const file = path.join(dir, subPath);
65 const stat = fs.statSync(file);
66 if (stat.isDirectory()) continue;
67 for (const e of exclude) {
68 if (subPath.endsWith(e)) continue item;
69 }
70 const trim = ext
71 ? subPath
72 : subPath.slice(0, -path.extname(subPath).length).replaceAll(
73 ".",
74 "/",
75 );
76 let id = prefix + trim.replaceAll("\\", "/");
77 if (prefix === "/" && id.endsWith("/index")) {
78 id = id.slice(0, -"/index".length) || "/";
79 }
80 list.push({ id, file: file });
81 }
82 }
83 }
84 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
85 const globalCssPath = join("global.css");
86
87 // TODO: invalidate incremental resources
88
89 // -- server side render --
90 status.text = "Building";
91 const cssOnce = new OnceMap<css.Output>();
92 const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
93 name: "Bundle",
94 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
95 passive: true,
96 getItemText: ([id]) => id,
97 maxJobs: 2,
98 });
99 interface RenderResult {
100 body: string;
101 head: string;
102 css: css.Output;
103 scriptFiles: string[];
104 item: FileItem;
105 }
106 const renderResults: RenderResult[] = [];
107 async function loadPageModule({ file }: FileItem) {
108 require(file);
109 }
110 async function renderPage(item: FileItem) {
111 // -- load and validate module --
112 let {
113 default: Page,
114 meta: metadata,
115 theme: pageTheme,
116 layout,
117 } = require(item.file);
118 if (!Page) throw new Error("Page is missing a 'default' export.");
119 if (!metadata) {
120 throw new Error("Page is missing 'meta' export with a title.");
121 }
122 if (layout?.theme) pageTheme = layout.theme;
123 const theme = {
124 bg: "#fff",
125 fg: "#050505",
126 primary: "#2e7dab",
127 ...pageTheme,
128 };
129
130 // -- metadata --
131 const renderedMetaPromise = Promise.resolve(
132 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
133 ).then((m) => meta.renderMeta(m));
134 // -- css --
135 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
136 const cssPromise = cssOnce.get(
137 cssImports.join(":") + JSON.stringify(theme),
138 () => cssQueue.add([item.id, cssImports, theme]),
139 );
140 // -- html --
141 let page = [engine.kElement, Page, {}];
142 if (layout?.default) {
143 page = [engine.kElement, layout.default, { children: page }];
144 }
145 const bodyPromise = engine.ssrAsync(page, {
146 sitegen: sg.initRender(),
147 });
148
149 const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
150 bodyPromise,
151 cssPromise,
152 renderedMetaPromise,
153 ]);
154 if (!renderedMeta.includes("<title>")) {
155 throw new Error(
156 "Page is missing 'meta.title'. " +
157 "All pages need a title tag.",
158 );
159 }
160 // The script content is not ready, allow another page to Render. The page
161 // contents will be rebuilt at the end. This is more convenient anyways
162 // because it means client scripts don't re-render the page.
163 renderResults.push({
164 body: text,
165 head: renderedMeta,
166 css: cssBundle,
167 scriptFiles: Array.from(addon.sitegen.scripts),
168 item: item,
169 });
170 }
171 // This is done in two passes so that a page that throws during evaluation
172 // will report "Load Render Module" instead of "Render Static Page".
173 const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
174 const spinnerFormat = status.format;
175 status.format = () => "";
176 const moduleLoadQueue = new Queue({
177 name: "Load Render Module",
178 fn: loadPageModule,
179 getItemText,
180 maxJobs: 1,
181 });
182 moduleLoadQueue.addMany(neededPages);
183 await moduleLoadQueue.done({ method: "stop" });
184 const pageQueue = new Queue({
185 name: "Render Static Page",
186 fn: renderPage,
187 getItemText,
188 maxJobs: 2,
189 });
190 pageQueue.addMany(neededPages);
191 await pageQueue.done({ method: "stop" });
192 status.format = spinnerFormat;
193
194 // -- bundle backend and views --
195 status.text = "Bundle backend code";
196 const {} = await bundle.bundleServerJavaScript(
197 join("backend.ts"),
198 views,
199 );
200
201 // -- bundle scripts --
202 const referencedScripts = Array.from(
203 new Set(renderResults.flatMap((r) => r.scriptFiles)),
204 );
205 const extraPublicScripts = scripts.map((entry) => entry.file);
206 const uniqueCount = new Set([
207 ...referencedScripts,
208 ...extraPublicScripts,
209 ]).size;
210 status.text = `Bundle ${uniqueCount} Scripts`;
211 await bundle.bundleClientJavaScript(
212 referencedScripts,
213 extraPublicScripts,
214 incr,
215 );
216
217 // -- copy/compress static files --
218 async function doStaticFile(item: FileItem) {
219 const body = await fs.readFile(item.file);
220 await incr.putAsset({
221 sources: [item.file],
222 key: item.id,
223 body,
224 });
225 }
226 const staticQueue = new Queue({
227 name: "Load Static",
228 fn: doStaticFile,
229 getItemText,
230 maxJobs: 16,
231 });
232 status.format = () => "";
233 staticQueue.addMany(
234 staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
235 );
236 await staticQueue.done({ method: "stop" });
237 status.format = spinnerFormat;
238
239 // -- concatenate static rendered pages --
240 status.text = `Concat ${renderResults.length} Pages`;
241 await Promise.all(
242 renderResults.map(
243 async (
244 { item: page, body, head, css, scriptFiles },
245 ) => {
246 const doc = wrapDocument({
247 body,
248 head,
249 inlineCss: css.text,
250 scripts: scriptFiles.map(
251 (id) =>
252 UNWRAP(
253 incr.out.script.get(
254 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
255 ),
256 ),
257 ).map((x) => `{${x}}`).join("\n"),
258 });
259 await incr.putAsset({
260 sources: [page.file, ...css.sources],
261 key: page.id,
262 body: doc,
263 headers: {
264 "Content-Type": "text/html",
265 },
266 });
267 },
268 ),
269 );
270 status.format = () => "";
271 status.text = ``;
272 // This will wait for all compression jobs to finish, which up
273 // to this point have been left as dangling promises.
274 await incr.wait();
275
276 // Flush the site to disk.
277 status.format = spinnerFormat;
278 status.text = `Incremental Flush`;
279 incr.flush(); // Write outputs
280 incr.toDisk(); // Allows picking up this state again
281 return { elapsed: (performance.now() - startTime) / 1000 };
282}
283
284function getItemText({ file }: FileItem) {
285 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
286}
287
288import { OnceMap, Queue } from "./queue.ts";
289import { Incremental } from "./incremental.ts";
290import * as bundle from "./bundle.ts";
291import * as css from "./css.ts";
292import * as engine from "./engine/ssr.ts";
293import * as hot from "./hot.ts";
294import * as fs from "#sitegen/fs";
295import * as sg from "#sitegen";
296import type { FileItem } from "#sitegen";
297import * as path from "node:path";
298import * as meta from "#sitegen/meta";
299import { Spinner, withSpinner } from "@paperclover/console/Spinner";
300import { wrapDocument } from "./lib/view.ts";
framework/generate.tsx deleted-316
...@@ -1,316 +0,0 @@
1export function main() {
2 return withSpinner({
3 text: "Recovering State",
4 successText: ({ elapsed }) =>
5 "sitegen! update in " + elapsed.toFixed(1) + "s",
6 failureText: () => "sitegen FAIL",
7 }, sitegen);
8}
9
10/**
11 * A filesystem object associated with some ID,
12 * such as a page's route to it's source file.
13 */
14interface FileItem {
15 id: string;
16 file: string;
17}
18
19async function sitegen(status: Spinner) {
20 const startTime = performance.now();
21
22 let root = path.resolve(import.meta.dirname, "../src");
23 const join = (...sub: string[]) => path.join(root, ...sub);
24 const incr = Incremental.fromDisk();
25 await incr.statAllFiles();
26
27 // Sitegen reviews every defined section for resources to process
28 const sections: sg.Section[] =
29 require(path.join(root, "site.ts")).siteSections;
30
31 // Static files are compressed and served as-is.
32 // - "{section}/static/*.png"
33 let staticFiles: FileItem[] = [];
34 // Pages are rendered then served as static files.
35 // - "{section}/pages/*.marko"
36 let pages: FileItem[] = [];
37 // Views are dynamically rendered pages called via backend code.
38 // - "{section}/views/*.tsx"
39 let views: FileItem[] = [];
40 // Public scripts are bundled for the client as static assets under "/js/[...]"
41 // This is used for the file viewer's canvases.
42 // Note that '.client.ts' can be placed anywhere in the file structure.
43 // - "{section}/scripts/*.client.ts"
44 let scripts: FileItem[] = [];
45
46 // -- Scan for files --
47 status.text = "Scanning Project";
48 for (const section of sections) {
49 const { root: sectionRoot } = section;
50 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
51 const rootPrefix = root === sectionRoot
52 ? ""
53 : path.relative(root, sectionRoot) + "/";
54 const kinds = [
55 {
56 dir: sectionPath("pages"),
57 list: pages,
58 prefix: "/",
59 exclude: [".css", ".client.ts", ".client.tsx"],
60 },
61 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
62 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
63 {
64 dir: sectionPath("views"),
65 list: views,
66 prefix: rootPrefix,
67 exclude: [".css", ".client.ts", ".client.tsx"],
68 },
69 ];
70 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
71 const items = fs.readDirRecOptionalSync(dir);
72 item: for (const subPath of items) {
73 const file = path.join(dir, subPath);
74 const stat = fs.statSync(file);
75 if (stat.isDirectory()) continue;
76 for (const e of exclude) {
77 if (subPath.endsWith(e)) continue item;
78 }
79 const trim = ext
80 ? subPath
81 : subPath.slice(0, -path.extname(subPath).length).replaceAll(
82 ".",
83 "/",
84 );
85 let id = prefix + trim.replaceAll("\\", "/");
86 if (prefix === "/" && id.endsWith("/index")) {
87 id = id.slice(0, -"/index".length) || "/";
88 }
89 list.push({ id, file: file });
90 }
91 }
92 }
93 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
94 const globalCssPath = join("global.css");
95
96 // TODO: invalidate incremental resources
97
98 // -- server side render --
99 status.text = "Building";
100 const cssOnce = new OnceMap<css.Output>();
101 const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
102 name: "Bundle",
103 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
104 passive: true,
105 getItemText: ([id]) => id,
106 maxJobs: 2,
107 });
108 interface RenderResult {
109 body: string;
110 head: string;
111 css: css.Output;
112 scriptFiles: string[];
113 item: FileItem;
114 }
115 const renderResults: RenderResult[] = [];
116 async function loadPageModule({ file }: FileItem) {
117 require(file);
118 }
119 async function renderPage(item: FileItem) {
120 // -- load and validate module --
121 let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
122 item.file,
123 );
124 if (!Page) throw new Error("Page is missing a 'default' export.");
125 if (!metadata) {
126 throw new Error("Page is missing 'meta' export with a title.");
127 }
128 if (layout?.theme) pageTheme = layout.theme;
129 const theme = {
130 bg: "#fff",
131 fg: "#050505",
132 primary: "#2e7dab",
133 ...pageTheme,
134 };
135
136 // -- metadata --
137 const renderedMetaPromise = Promise.resolve(
138 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
139 ).then((m) => meta.renderMeta(m));
140 // -- css --
141 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
142 const cssPromise = cssOnce.get(
143 cssImports.join(":") + JSON.stringify(theme),
144 () => cssQueue.add([item.id, cssImports, theme]),
145 );
146 // -- html --
147 let page = <Page />;
148 if (layout?.default) {
149 const Layout = layout.default;
150 page = <Layout>{page}</Layout>;
151 }
152 const bodyPromise = await ssr.ssrAsync(page, {
153 sitegen: sg.initRender(),
154 });
155
156 const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
157 bodyPromise,
158 cssPromise,
159 renderedMetaPromise,
160 ]);
161 if (!renderedMeta.includes("<title>")) {
162 throw new Error(
163 "Page is missing 'meta.title'. " +
164 "All pages need a title tag.",
165 );
166 }
167 // The script content is not ready, allow another page to Render. The page
168 // contents will be rebuilt at the end. This is more convenient anyways
169 // because it means client scripts don't re-render the page.
170 renderResults.push({
171 body: text,
172 head: renderedMeta,
173 css: cssBundle,
174 scriptFiles: Array.from(addon.sitegen.scripts),
175 item: item,
176 });
177 }
178 // This is done in two passes so that a page that throws during evaluation
179 // will report "Load Render Module" instead of "Render Static Page".
180 const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
181 const spinnerFormat = status.format;
182 status.format = () => "";
183 const moduleLoadQueue = new Queue({
184 name: "Load Render Module",
185 fn: loadPageModule,
186 getItemText,
187 maxJobs: 1,
188 });
189 moduleLoadQueue.addMany(neededPages);
190 await moduleLoadQueue.done({ method: "stop" });
191 const pageQueue = new Queue({
192 name: "Render Static Page",
193 fn: renderPage,
194 getItemText,
195 maxJobs: 2,
196 });
197 pageQueue.addMany(neededPages);
198 await pageQueue.done({ method: "stop" });
199 status.format = spinnerFormat;
200
201 // -- bundle scripts --
202 const referencedScripts = Array.from(
203 new Set(renderResults.flatMap((r) => r.scriptFiles)),
204 );
205 const extraPublicScripts = scripts.map((entry) => entry.file);
206 const uniqueCount = new Set([
207 ...referencedScripts,
208 ...extraPublicScripts,
209 ]).size;
210 status.text = `Bundle ${uniqueCount} Scripts`;
211 await bundle.bundleClientJavaScript(
212 referencedScripts,
213 extraPublicScripts,
214 incr,
215 );
216
217 // -- copy/compress static files --
218 async function doStaticFile(item: FileItem) {
219 const body = await fs.readFile(item.file);
220 await incr.putAsset({
221 sources: [item.file],
222 key: item.id,
223 body,
224 });
225 }
226 const staticQueue = new Queue({
227 name: "Load Static",
228 fn: doStaticFile,
229 getItemText,
230 maxJobs: 16,
231 });
232 status.format = () => "";
233 staticQueue.addMany(
234 staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
235 );
236 await staticQueue.done({ method: "stop" });
237 status.format = spinnerFormat;
238
239 // -- concatenate static rendered pages --
240 status.text = `Concat ${renderResults.length} Pages`;
241 await Promise.all(
242 renderResults.map(
243 async (
244 { item: page, body, head, css, scriptFiles },
245 ) => {
246 const doc = wrapDocument({
247 body,
248 head,
249 inlineCss: css.text,
250 scripts: scriptFiles.map(
251 (id) =>
252 UNWRAP(
253 incr.out.script.get(
254 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
255 ),
256 ),
257 ).map((x) => `{${x}}`).join("\n"),
258 });
259 await incr.putAsset({
260 sources: [page.file, ...css.sources],
261 key: page.id,
262 body: doc,
263 headers: {
264 "Content-Type": "text/html",
265 },
266 });
267 },
268 ),
269 );
270 status.format = () => "";
271 status.text = ``;
272 // This will wait for all compression jobs to finish, which up
273 // to this point have been left as dangling promises.
274 await incr.wait();
275
276 // Flush the site to disk.
277 status.format = spinnerFormat;
278 status.text = `Incremental Flush`;
279 incr.flush();
280 incr.toDisk(); // Allows picking up this state again
281 return { elapsed: (performance.now() - startTime) / 1000 };
282}
283
284function getItemText({ file }: FileItem) {
285 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
286}
287
288function wrapDocument({
289 body,
290 head,
291 inlineCss,
292 scripts,
293}: {
294 head: string;
295 body: string;
296 inlineCss: string;
297 scripts: string;
298}) {
299 return `<!doctype html><html lang=en><head>${head}${
300 inlineCss ? `<style>${inlineCss}</style>` : ""
301 }</head><body>${body}${
302 scripts ? `<script>${scripts}</script>` : ""
303 }</body></html>`;
304}
305
306import { OnceMap, Queue } from "./queue.ts";
307import { Incremental } from "./incremental.ts";
308import * as bundle from "./bundle.ts";
309import * as css from "./css.ts";
310import * as ssr from "./engine/ssr.ts";
311import * as hot from "./hot.ts";
312import * as fs from "#sitegen/fs";
313import * as sg from "#sitegen";
314import * as path from "node:path";
315import * as meta from "#sitegen/meta";
316import { Spinner, withSpinner } from "@paperclover/console/Spinner";
framework/hot.ts+44-11
...@@ -36,12 +36,6 @@ export interface FileStat {...@@ -36,12 +36,6 @@ export interface FileStat {
36 imports: string[];36 imports: string[];
37}37}
38let fsGraph = new Map<string, FileStat>();38let fsGraph = new Map<string, FileStat>();
39export function setFsGraph(g: Map<string, FileStat>) {
40 if (fsGraph.size > 0) {
41 throw new Error("Cannot restore fsGraph when it has been written into");
42 }
43 fsGraph = g;
44}
45export function getFsGraph() {39export function getFsGraph() {
46 return fsGraph;40 return fsGraph;
47}41}
...@@ -108,11 +102,18 @@ Module._resolveFilename = (...args) => {...@@ -108,11 +102,18 @@ Module._resolveFilename = (...args) => {
108};102};
109103
110function loadEsbuild(module: NodeJS.Module, filepath: string) {104function loadEsbuild(module: NodeJS.Module, filepath: string) {
111 let src = fs.readFileSync(filepath, "utf8");105 return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
112 return loadEsbuildCode(module, filepath, src);
113}106}
114107
115function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {108interface LoadOptions {
109 scannedClientRefs?: string[];
110}
111function loadEsbuildCode(
112 module: NodeJS.Module,
113 filepath: string,
114 src: string,
115 opt: LoadOptions = {},
116) {
116 if (filepath === import.meta.filename) {117 if (filepath === import.meta.filename) {
117 module.exports = self;118 module.exports = self;
118 return;119 return;
...@@ -122,12 +123,13 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {...@@ -122,12 +123,13 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {
122 if (filepath.endsWith(".ts")) loader = "ts";123 if (filepath.endsWith(".ts")) loader = "ts";
123 else if (filepath.endsWith(".jsx")) loader = "jsx";124 else if (filepath.endsWith(".jsx")) loader = "jsx";
124 else if (filepath.endsWith(".js")) loader = "js";125 else if (filepath.endsWith(".js")) loader = "js";
126 module.cloverClientRefs = opt.scannedClientRefs ?? extractClientScripts(src);
125 if (src.includes("import.meta")) {127 if (src.includes("import.meta")) {
126 src = `128 src = `
127 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};129 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
128 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};130 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
129 import.meta.filename = ${JSON.stringify(filepath)};131 import.meta.filename = ${JSON.stringify(filepath)};
130 ` + src;132 `.trim().replace(/\n/g, "") + src;
131 }133 }
132 src = esbuild.transformSync(src, {134 src = esbuild.transformSync(src, {
133 loader,135 loader,
...@@ -148,7 +150,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) {...@@ -148,7 +150,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) {
148 if (src.match(/^\s*client\s+import\s+["']/m)) {150 if (src.match(/^\s*client\s+import\s+["']/m)) {
149 src = src.replace(151 src = src.replace(
150 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,152 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
151 "<CloverScriptInclude src=$1 />",153 (_, src) => `<CloverScriptInclude src=${src} />`,
152 ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';154 ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';
153 }155 }
154156
...@@ -206,9 +208,40 @@ export function resolveFrom(src: string, dest: string) {...@@ -206,9 +208,40 @@ export function resolveFrom(src: string, dest: string) {
206 }208 }
207}209}
208210
211const importRegExp =
212 /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;
213const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))/;
214export function extractClientScripts(source: string): string[] {
215 // This match finds a call to 'import ... from "#sitegen"'
216 const importMatch = source.match(importRegExp);
217 if (!importMatch) return [];
218 const items = importMatch[1];
219 let identifier = "";
220 if (items.startsWith("{")) {
221 const clauseMatch = items.match(getSitegenAddScriptRegExp);
222 if (!clauseMatch) return []; // did not import
223 identifier = clauseMatch[1] || "addScript";
224 } else if (items.startsWith("*")) {
225 identifier = importMatch[2] + "\\s*\\.\\s*addScript";
226 } else {
227 throw new Error("Impossible");
228 }
229 identifier = identifier.replaceAll("$", "\\$"); // only needed escape
230 const findCallsRegExp = new RegExp(
231 `\\b${identifier}\\s*\\(("[^"]+"|'[^']+')\\)`,
232 "gs",
233 );
234 const calls = source.matchAll(findCallsRegExp);
235 return [...calls].map((call) => {
236 return JSON.parse(`"${call[1].slice(1, -1)}"`) as string;
237 });
238}
239
209declare global {240declare global {
210 namespace NodeJS {241 namespace NodeJS {
211 interface Module {242 interface Module {
243 cloverClientRefs?: string[];
244
212 _compile(245 _compile(
213 this: NodeJS.Module,246 this: NodeJS.Module,
214 content: string,247 content: string,
framework/lib/sitegen.ts+9
...@@ -1,6 +1,15 @@...@@ -1,6 +1,15 @@
1// Import this file with 'import * as sg from "#sitegen";'1// Import this file with 'import * as sg from "#sitegen";'
2export type ScriptId = string;2export type ScriptId = string;
33
4/**
5 * A filesystem object associated with some ID,
6 * such as a page's route to it's source file.
7 */
8export interface FileItem {
9 id: string;
10 file: string;
11}
12
4const frameworkDir = path.dirname(import.meta.dirname);13const frameworkDir = path.dirname(import.meta.dirname);
514
6export interface SitegenRender {15export interface SitegenRender {
framework/lib/string.ts created+3
...@@ -0,0 +1,3 @@
1export function escapeRegExp(source: string) {
2 return source.replace(/[\$\\]/g, "\\$&");
3}
framework/lib/view.ts created+87
...@@ -0,0 +1,87 @@
1// This import is generated by code 'bundle.ts'
2export interface View {
3 component: engine.Component;
4 meta:
5 | meta.Meta
6 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
7 layout?: engine.Component;
8 theme?: css.Theme;
9 inlineCss: string;
10 scripts: Record<string, string>;
11}
12let views: Record<string, View> = {};
13
14// An older version of the Clover Engine supported streaming suspense
15// boundaries, but those were never used. Pages will wait until they
16// are fully rendered before sending.
17async function renderView(
18 c: hono.Context,
19 id: string,
20 props: Record<string, unknown>,
21) {
22 views = require("$views").views;
23 // The view contains pre-bundled CSS and scripts, but keeps the scripts
24 // separate for run-time dynamic scripts. For example, the file viewer
25 // includes the canvas for the current page, but only the current page.
26 const {
27 component,
28 inlineCss,
29 layout,
30 meta: metadata,
31 scripts,
32 theme,
33 }: View = UNWRAP(views[id]);
34
35 // -- metadata --
36 const renderedMetaPromise = Promise.resolve(
37 typeof metadata === "function" ? metadata({ context: c }) : metadata,
38 ).then((m) => meta.renderMeta(m));
39
40 // -- html --
41 let page: engine.Element = [engine.kElement, component, props];
42 if (layout) page = [engine.kElement, layout, { children: page }];
43 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
44 sitegen: sg.initRender(),
45 });
46
47 // -- join document and send --
48 return c.html(wrapDocument({
49 body,
50 head: await renderedMetaPromise,
51 inlineCss,
52 scripts: joinScripts(
53 Array.from(sitegen.scripts, (script) => scripts[script]),
54 ),
55 }));
56}
57
58export function joinScripts(scriptSources: string[]) {
59 const { length } = scriptSources;
60 if (length === 0) return "";
61 if (length === 1) return scriptSources[0];
62 return scriptSources.map((source) => `{${source}}`).join(";");
63}
64
65export function wrapDocument({
66 body,
67 head,
68 inlineCss,
69 scripts,
70}: {
71 head: string;
72 body: string;
73 inlineCss: string;
74 scripts: string;
75}) {
76 return `<!doctype html><html lang=en><head>${head}${
77 inlineCss ? `<style>${inlineCss}</style>` : ""
78 }</head><body>${body}${
79 scripts ? `<script>${scripts}</script>` : ""
80 }</body></html>`;
81}
82
83import * as meta from "./meta.ts";
84import type * as hono from "#hono";
85import * as engine from "../engine/ssr.ts";
86import type * as css from "../css.ts";
87import * as sg from "./sitegen.ts";
readme.md+3-1
...@@ -36,7 +36,9 @@ Included is `src`, which contains `paperclover.net`. Website highlights:...@@ -36,7 +36,9 @@ Included is `src`, which contains `paperclover.net`. Website highlights:
36minimum system requirements: 36minimum system requirements:
37- a cpu with at least 1 core. 37- a cpu with at least 1 core.
38- random access memory. 38- random access memory.
39- windows 7 or later, macos, or linux operating system. 39- windows 7 or later, macos, or other operating system.
40
41my development machine, for example, is Dell Inspiron 7348 with Core i7
40 42
41``` 43```
42npm install 44npm install
repl.js+1-1
...@@ -27,7 +27,7 @@ hot.load("node:repl").start({...@@ -27,7 +27,7 @@ hot.load("node:repl").start({
27});27});
2828
29setTimeout(() => {29setTimeout(() => {
30 hot.reloadRecursive("./framework/generate.tsx");30 hot.reloadRecursive("./framework/generate.ts");
31}, 100);31}, 100);
3232
33async function evaluate(code) {33async function evaluate(code) {
src/backend.ts+2
...@@ -1,3 +1,5 @@...@@ -1,3 +1,5 @@
1import * as v from "#sitegen/view";
2console.log(v);
1const logHttp = scoped("http", { color: "magenta" }); 3const logHttp = scoped("http", { color: "magenta" });
2 4
3const app = new Hono(); 5const app = new Hono();
src/components/Video.tsx+57
...@@ -0,0 +1,57 @@...@@ -0,0 +1,57 @@
1import * as path from "node:path";
2import { addScript } from "#sitegen";
3import { PrecomputedBlurhash } from "./blurhash.tsx";
4import "./Video.css";
5export namespace Video {
6 export interface Props {
7 title: string;
8 width: number;
9 height: number;
10 sources: string[];
11 downloads: string[];
12 poster?: string;
13 posterHash?: string;
14 borderless?: boolean;
15 }
16}
17 function PrecomputedBlurhash({ hash }: { hash: string }) {
18export function Video(
19 { title, sources, height, poster, posterHash, width, borderless }:
20 Video.Props,
21) {
22 addScript("./video.client.ts");
23 return (
24 <figure class={`video ${borderless ? "borderless" : ""}`}>
25 <figcaption>{title}</figcaption>
26 {/* posterHash && <PrecomputedBlurhash hash={posterHash} /> */}
27 {poster && <img src={poster} alt="waterfalls" />}
28 <video
29 controls
30 preload="none"
31 style={`width:100%;background:transparent;aspect-ratio:${
32 simplifyFraction(width, height)
33 }`}
34 poster="data:null"
35 >
36 {sources.map((src) => (
37 <source
38 src={src}
39 type={contentTypeFromExt(src)}
40 />
41 ))}
42 </video>
43 </figure>
44 );
45}
46export function contentTypeFromExt(src: string) {
47 if (src.endsWith(".m3u8")) return "application/x-mpegURL";
48 if (src.endsWith(".webm")) return "video/webm";
49 if (src.endsWith(".mp4")) return "video/mp4";
50 if (src.endsWith(".ogg")) return "video/ogg";
51 throw new Error("Unknown video extension: " + path.extname(src));
52}
53const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a;
54function simplifyFraction(n: number, d: number) {
55 const divisor = gcd(n, d);
56 return `${n / divisor}/${d / divisor}`;
57}
src/q+a/views/backend-inbox.marko+4-3
...@@ -8,7 +8,6 @@ import {...@@ -8,7 +8,6 @@ import {
8} from "../q+a/QuestionRender";8} from "../q+a/QuestionRender";
99
10<const/questions = PendingQuestion.getAll() />10<const/questions = PendingQuestion.getAll() />
11<Script src="backend-inbox.client.ts" />
1211
13<h1>inbox</h1>12<h1>inbox</h1>
14<for|q| of=questions>13<for|q| of=questions>
...@@ -16,7 +15,7 @@ import {...@@ -16,7 +15,7 @@ import {
16 data-q=q.id15 data-q=q.id
17 style="border-bottom: 2px solid #fff7; margin-bottom: 1rem"16 style="border-bottom: 2px solid #fff7; margin-bottom: 1rem"
18 >17 >
19 <time datetime={formatQuestionISOTimestamp(q.date)}>18 <time datetime=formatQuestionISOTimestamp(q.date)>
20 ${formatQuestionTimestamp(q.date)} ${q.id}19 ${formatQuestionTimestamp(q.date)} ${q.id}
21 </time>20 </time>
22 <div style="color: dodgerblue; margin-bottom: 0.25rem">21 <div style="color: dodgerblue; margin-bottom: 0.25rem">
...@@ -26,7 +25,7 @@ import {...@@ -26,7 +25,7 @@ import {
26 </div>25 </div>
27 <p style="white-space: pre-wrap">${q.prompt}</p>26 <p style="white-space: pre-wrap">${q.prompt}</p>
28 <p>27 <p>
29 <button onclick=`onReply("${q.id}") style="color: lime">28 <button onclick=`onReply("${q.id}")` style="color: lime">
30 reply29 reply
31 </button>30 </button>
32 <button onclick=`onDelete("${q.id}")` style="color: red">31 <button onclick=`onDelete("${q.id}")` style="color: red">
...@@ -38,3 +37,5 @@ import {...@@ -38,3 +37,5 @@ import {
38 </p>37 </p>
39 </div>38 </div>
40</>39</>
40
41client import "backend-inbox.client.ts";
src/q+a/views/fail.marko+9-13
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1import * as layout from "../layouts/questions.tsx";1import * as layout from "../layout.tsx";
2export { layout };2export { layout };
3export const theme = {3export const theme = {
4 ...layout.theme,4 ...layout.theme,
...@@ -10,18 +10,14 @@ export const theme = {...@@ -10,18 +10,14 @@ export const theme = {
10<p style="color: red">10<p style="color: red">
11 ${error}11 ${error}
12</p>12</p>
13{13<if=content>
14 content && (14 <br />
15 <>15 <br />
16 <br />16 <p>
17 <br />17 here is a copy of what you wrote, if you want to try again:
18 <p>18 </p>
19 here is a copy of what you wrote, if you want to try again:19 <pre style="white-space: pre-wrap"><code>${content}</code></pre>
20 </p>20</>
21 <pre style="white-space: pre-wrap"><code>${content}</code></pre>
22 </>
23 )
24}
25<br />21<br />
26<br />22<br />
27<p>23<p>
src/q+a/views/success.marko+1-1
...@@ -3,7 +3,7 @@ export interface Input {...@@ -3,7 +3,7 @@ export interface Input {
3}3}
4<const/{ permalink }=input/>4<const/{ permalink }=input/>
55
6import * as layout from "../layouts/questions.tsx";6import * as layout from "../layout.tsx";
7export { layout };7export { layout };
8export const theme = {8export const theme = {
9 ...layout.theme,9 ...layout.theme,