authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-07 02:25:06-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log0a8336e50495e659d9cfd113e258cca17e8148ac
tree4da062a577e455124467812f7a4389a7b9b70b8e
parent69f8d0a5f1709f154d141d6585af5e1af99baa81
signature Commit is signed but in an unrecognized format.

restore sitegen.tsx

5 files changed, 245 insertions(+), 100 deletions(-)

framework/definitions.d.ts created+2
...@@ -0,0 +1,2 @@
1declare function UNWRAP<T>(value: T | null | undefined): T;
2declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
framework/fs.ts+45-9
...@@ -1,12 +1,18 @@...@@ -1,12 +1,18 @@
1// File System APIs1// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in
2import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync as nodeMkdirSync, rmSync } from 'node:fs';2// Node.js sync+promise fs methods. For convenince.
3import { readFile, writeFile, readdir, stat, mkdir as nodeMkdir, rm } from 'node:fs/promises';
4
5// Re-export a mix of built-in Node.js sync+promise fs methods.
6export {3export {
7 readFileSync, writeFileSync, readdirSync, statSync, existsSync, rmSync ,4 existsSync,
8 readFile, writeFile, readdir, stat, mkdir, rm 5 readdir,
9}6 readdirSync,
7 readFile,
8 readFileSync,
9 rm,
10 rmSync,
11 stat,
12 statSync,
13 writeFile,
14 writeFileSync,
15};
1016
11export function mkdir(dir: string) {17export function mkdir(dir: string) {
12 return nodeMkdir(dir, { recursive: true });18 return nodeMkdir(dir, { recursive: true });
...@@ -21,5 +27,35 @@ export async function writeMkdir(file: string, contents: Buffer | string) {...@@ -21,5 +27,35 @@ export async function writeMkdir(file: string, contents: Buffer | string) {
21 return writeFile(file, contents);27 return writeFile(file, contents);
22}28}
2329
24import * as path from 'node:path';30export function writeMkdirSync(file: string, contents: Buffer | string) {
31 mkdirSync(path.dirname(file));
32 return writeFileSync(file, contents);
33}
34
35export function readDirRecOptionalSync(dir: string) {
36 try {
37 return readdirSync(dir, { withFileTypes: true });
38 } catch (err: any) {
39 if (err.code === "ENOENT") return [];
40 throw err;
41 }
42}
2543
44import * as path from "node:path";
45import {
46 existsSync,
47 mkdirSync as nodeMkdirSync,
48 readdirSync,
49 readFileSync,
50 rmSync,
51 statSync,
52 writeFileSync,
53} from "node:fs";
54import {
55 mkdir as nodeMkdir,
56 readdir,
57 readFile,
58 rm,
59 stat,
60 writeFile,
61} from "node:fs/promises";
framework/sitegen.tsx+167-79
...@@ -9,20 +9,44 @@ function main() {...@@ -9,20 +9,44 @@ function main() {
9 }, sitegen);9 }, sitegen);
10}10}
1111
12async function sitegen(status) {12/**
13 * A filesystem object associated with some ID,
14 * such as a page's route to it's source file.
15 */
16interface FileItem {
17 id: string;
18 file: string;
19}
20
21async function sitegen(status: Spinner) {
13 const startTime = performance.now();22 const startTime = performance.now();
1423
15 let root = path.resolve(import_meta.dirname, "../src");24 let root = path.resolve(import.meta.dirname, "../src");
16 const join = (...sub) => path.join(root, ...sub);25 const join = (...sub: string[]) => path.join(root, ...sub);
17 const incr = new Incremental();26 const incr = new Incremental();
27
28 // Sitegen reviews every defined section for resources to process
18 const sections: Section[] =29 const sections: Section[] =
19 require(path.join(root, "sections.ts")).siteSections;30 require(path.join(root, "sections.ts")).siteSections;
20 const views = [];31
21 const staticFiles = [];32 // Static files are compressed and served as-is.
22 let pages = [];33 // - "{section}/static/*.png"
23 let scripts = [];34 let staticFiles: FileItem[] = [];
35 // Pages are rendered then served as static files.
36 // - "{section}/pages/*.marko"
37 let pages: FileItem[] = [];
38 // Views are dynamically rendered pages called via backend code.
39 // - "{section}/views/*.tsx"
40 let views: FileItem[] = [];
41 // Public scripts are bundled for the client as static assets under "/js/[...]"
42 // This is used for the file viewer's canvases.
43 // Note that '.client.ts' can be placed anywhere in the file structure.
44 // - "{section}/scripts/*.client.ts"
45 let scripts: FileItem[] = [];
46 // 'backend.ts'
24 const backendFiles = [];47 const backendFiles = [];
2548
49 // -- Scan for files --
26 status.text = "Scanning Project";50 status.text = "Scanning Project";
27 for (const section of sections) {51 for (const section of sections) {
28 const { root: sectionRoot } = section;52 const { root: sectionRoot } = section;
...@@ -39,16 +63,21 @@ async function sitegen(status) {...@@ -39,16 +63,21 @@ async function sitegen(status) {
39 },63 },
40 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },64 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
41 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },65 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
42 { dir: sectionPath("views"), list: views, prefix: rootPrefix },66 {
67 dir: sectionPath("views"),
68 list: views,
69 prefix: rootPrefix,
70 exclude: [".css", ".client.ts", ".client.tsx"],
71 },
43 ];72 ];
44 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {73 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
45 const pages2 = fs.readDirRecOptional(dir);74 const items = fs.readDirRecOptionalSync(dir);
46 page: for (const page of pages2) {75 item: for (const item of items) {
47 if (page.isDirectory()) continue;76 if (item.isDirectory()) continue;
48 for (const ext2 of exclude) {77 for (const e of exclude) {
49 if (page.name.endsWith(ext2)) continue page;78 if (item.name.endsWith(e)) continue item;
50 }79 }
51 const file = path.relative(dir, page.parentPath + "/" + page.name);80 const file = path.relative(dir, item.parentPath + "/" + item.name);
52 const trim = ext81 const trim = ext
53 ? file82 ? file
54 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");83 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
...@@ -56,7 +85,7 @@ async function sitegen(status) {...@@ -56,7 +85,7 @@ async function sitegen(status) {
56 if (prefix === "/" && id.endsWith("/index")) {85 if (prefix === "/" && id.endsWith("/index")) {
57 id = id.slice(0, -"/index".length) || "/";86 id = id.slice(0, -"/index".length) || "/";
58 }87 }
59 list.push({ id, file: path.join(page.parentPath, page.name) });88 list.push({ id, file: path.join(item.parentPath, item.name) });
60 }89 }
61 }90 }
62 let backendFile = [91 let backendFile = [
...@@ -67,86 +96,110 @@ async function sitegen(status) {...@@ -67,86 +96,110 @@ async function sitegen(status) {
67 }96 }
68 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));97 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
69 const globalCssPath = join("global.css");98 const globalCssPath = join("global.css");
99
100 // TODO: invalidate incremental resources
101
102 // -- server side render --
70 status.text = "Building";103 status.text = "Building";
71 const cssOnce = new OnceMap();104 const cssOnce = new OnceMap();
72 const cssQueue = new Queue({105 const cssQueue = new Queue<[string, string[], css.Theme], string>({
73 name: "Bundle",106 name: "Bundle",
74 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),107 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
75 passive: true,108 passive: true,
76 getItemText: ([id]) => id,109 getItemText: ([id]) => id,
77 maxJobs: 2,110 maxJobs: 2,
78 });111 });
79 const ssrResults = [];112 interface RenderResult {
80 function loadSsrModule(page) {113 body: string;
81 require(page.file);114 head: string;
115 inlineCss: string;
116 scriptFiles: string[];
117 item: FileItem;
82 }118 }
83 async function doSsrPage(page) {119 const renderResults: RenderResult[] = [];
84 const module2 = require(page.file);120 async function loadPageModule({ file }: FileItem) {
85 const Page = module2.default;121 require(file);
86 if (!Page) {122 }
87 throw new Error("Page is missing a 'default' export.");123 async function renderPage(item: FileItem) {
88 }124 // -- load and validate module --
89 const metadata = module2.meta;125 let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
126 item.file,
127 );
128 if (!Page) throw new Error("Page is missing a 'default' export.");
90 if (!metadata) {129 if (!metadata) {
91 throw new Error("Page is missing 'meta' attribute with a title.");130 throw new Error("Page is missing 'meta' export with a title.");
92 }131 }
132 if (layout?.theme) pageTheme = layout.theme;
93 const theme = {133 const theme = {
94 bg: "#fff",134 bg: "#fff",
95 fg: "#050505",135 fg: "#050505",
96 primary: "#2e7dab",136 primary: "#2e7dab",
97 ...module2.theme,137 ...pageTheme,
98 };138 };
139
140 // -- metadata --
99 const renderedMetaPromise = Promise.resolve(141 const renderedMetaPromise = Promise.resolve(
100 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,142 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
101 ).then((m) => meta.resolveAndRenderMetadata(m));143 ).then((m) => meta.resolveAndRenderMetadata(m));
102 const cssImports = [globalCssPath, ...hot.getCssImports(page.file)];144 // -- css --
145 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
103 const cssPromise = cssOnce.get(146 const cssPromise = cssOnce.get(
104 cssImports.join(":") + JSON.stringify(theme),147 cssImports.join(":") + JSON.stringify(theme),
105 () => cssQueue.add([page.id, cssImports, theme]),148 () => cssQueue.add([item.id, cssImports, theme]),
106 );149 );
150 // -- html --
107 const sitegenApi = sg.initRender();151 const sitegenApi = sg.initRender();
108 const body = await (0, import_ssr.ssrAsync)(152 const bodyPromise = await ssr.ssrAsync(<Page />, {
109 /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Page, {}),153 sitegen: sitegenApi,
110 {154 });
111 sitegen: sitegenApi,155
112 },156 const [body, inlineCss, renderedMeta] = await Promise.all([
113 );157 bodyPromise,
114 const inlineCss = await cssPromise;158 cssPromise,
115 const renderedMeta = await renderedMetaPromise;159 renderedMetaPromise,
160 ]);
116 if (!renderedMeta.includes("<title>")) {161 if (!renderedMeta.includes("<title>")) {
117 throw new Error(162 throw new Error(
118 "Page is missing 'meta.title'. All pages need a title tag.",163 "Page is missing 'meta.title'. " +
164 "All pages need a title tag.",
119 );165 );
120 }166 }
121 ssrResults.push({167 // The script content is not ready, allow another page to Render. The page
168 // contents will be rebuilt at the end. This is more convenient anyways
169 // because it means client scripts don't re-render the page.
170 renderResults.push({
122 body,171 body,
123 head: renderedMeta,172 head: renderedMeta,
124 inlineCss,173 inlineCss,
125 scriptFiles: Array.from(sitegenApi.scripts),174 scriptFiles: Array.from(sitegenApi.scripts),
126 page,175 item: item,
127 });176 });
128 }177 }
178 // This is done in two passes so that a page that throws during evaluation
179 // will report "Load Render Module" instead of "Render Static Page".
129 const spinnerFormat = status.format;180 const spinnerFormat = status.format;
130 status.format = () => "";181 status.format = () => "";
131 const moduleLoadQueue = new import_queue.Queue({182 const moduleLoadQueue = new Queue({
132 name: "Load Render Module",183 name: "Load Render Module",
133 fn: loadSSRModule,184 fn: loadPageModule,
134 getItemText,185 getItemText,
135 maxJobs: 1,186 maxJobs: 1,
136 });187 });
137 moduleLoadQueue.addMany(pages);188 moduleLoadQueue.addMany(pages);
138 await moduleLoadQueue.done({ method: "stop" });189 await moduleLoadQueue.done({ method: "stop" });
139 const pageQueue = new import_queue.Queue({190 const pageQueue = new Queue({
140 name: "Render",191 name: "Render Static Page",
141 fn: doSsrPage,192 fn: renderPage,
142 getItemText,193 getItemText,
143 maxJobs: 2,194 maxJobs: 2,
144 });195 });
145 pageQueue.addMany(pages);196 pageQueue.addMany(pages);
146 await pageQueue.done({ method: "stop" });197 await pageQueue.done({ method: "stop" });
147 status.format = spinnerFormat;198 status.format = spinnerFormat;
199
200 // -- bundle scripts --
148 const referencedScripts = Array.from(201 const referencedScripts = Array.from(
149 new Set(ssrResults.flatMap((r) => r.scriptFiles)),202 new Set(renderResults.flatMap((r) => r.scriptFiles)),
150 );203 );
151 const extraPublicScripts = scripts.map((entry) => entry.file);204 const extraPublicScripts = scripts.map((entry) => entry.file);
152 const uniqueCount = new Set([205 const uniqueCount = new Set([
...@@ -159,15 +212,17 @@ async function sitegen(status) {...@@ -159,15 +212,17 @@ async function sitegen(status) {
159 extraPublicScripts,212 extraPublicScripts,
160 incr,213 incr,
161 );214 );
162 async function doStaticFile(page) {215
163 const body = await fs.readFile(page.file);216 // -- copy/compress static files --
217 async function doStaticFile(item: FileItem) {
218 const body = await fs.readFile(item.file);
164 incr.putAsset({219 incr.putAsset({
165 srcId: "static:" + page.file,220 srcId: "static:" + item.file,
166 key: page.id,221 key: item.id,
167 body,222 body,
168 });223 });
169 }224 }
170 const staticQueue = new import_queue.Queue({225 const staticQueue = new Queue({
171 name: "Load Static",226 name: "Load Static",
172 fn: doStaticFile,227 fn: doStaticFile,
173 getItemText,228 getItemText,
...@@ -177,40 +232,68 @@ async function sitegen(status) {...@@ -177,40 +232,68 @@ async function sitegen(status) {
177 staticQueue.addMany(staticFiles);232 staticQueue.addMany(staticFiles);
178 await staticQueue.done({ method: "stop" });233 await staticQueue.done({ method: "stop" });
179 status.format = spinnerFormat;234 status.format = spinnerFormat;
180 status.text = `Concat ${ssrResults.length} Pages`;235
236 // -- concatenate static rendered pages --
237 status.text = `Concat ${renderResults.length} Pages`;
181 await Promise.all(238 await Promise.all(
182 ssrResults.map(async ({ page, body, head, inlineCss, scriptFiles }) => {239 renderResults.map(
183 const doc = wrapDocument({240 async ({ item: page, body, head, inlineCss, scriptFiles }) => {
184 body,241 const doc = wrapDocument({
185 head,242 body,
186 inlineCss,243 head,
187 scripts: scriptFiles.map(244 inlineCss,
188 (id) =>245 scripts: scriptFiles.map(
189 UNWRAP(246 (id) =>
190 incr.out.script.get(247 UNWRAP(
191 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),248 incr.out.script.get(
249 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
250 ),
192 ),251 ),
193 ),252 ).map((x) => `{${x}}`).join("\n"),
194 ).map((x) => `{${x}}`).join(""),253 });
195 });254 incr.putAsset({
196 incr.putAsset({255 srcId: "page:" + page.file,
197 srcId: "page:" + page.file,256 key: page.id,
198 key: page.id,257 body: doc,
199 body: doc,258 headers: {
200 headers: {259 "Content-Type": "text/html",
201 "Content-Type": "text/html",260 },
202 },261 });
203 });262 },
204 }),263 ),
205 );264 );
206 status.format = () => "";265 status.format = () => "";
207 status.text = ``;266 status.text = ``;
267 // This will wait for all compression jobs to finish, which up
268 // to this point have been left as dangling promises.
208 await incr.wait();269 await incr.wait();
270
271 // Flush the site to disk.
209 status.format = spinnerFormat;272 status.format = spinnerFormat;
210 status.text = `Incremental Flush`;273 status.text = `Incremental Flush`;
211 incr.flush();274 incr.flush();
212 incr.serializeToDisk();275 incr.serializeToDisk(); // Allows picking up this state again
213 return { elapsed: (performance.now() - startTime) / 1e3 };276 return { elapsed: (performance.now() - startTime) / 1000 };
277}
278
279function getItemText({ file }: FileItem) {
280 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
281}
282
283function wrapDocument({
284 body,
285 head,
286 inlineCss,
287 scripts,
288}: {
289 head: string;
290 body: string;
291 inlineCss: string;
292 scripts: string;
293}) {
294 return `<!doctype html><head>${head}${
295 inlineCss ? `<style>${inlineCss}</style>` : ""
296 }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
214}297}
215298
216import type { Section } from "./sitegen-lib.ts";299import type { Section } from "./sitegen-lib.ts";
...@@ -218,5 +301,10 @@ import { OnceMap, Queue } from "./queue.ts";...@@ -218,5 +301,10 @@ import { OnceMap, Queue } from "./queue.ts";
218import { Incremental } from "./incremental.ts";301import { Incremental } from "./incremental.ts";
219import * as bundle from "./bundle.ts";302import * as bundle from "./bundle.ts";
220import * as css from "./css.ts";303import * as css from "./css.ts";
221import * as fs from './fs.ts';304import * as fs from "./fs.ts";
222import { withSpinner, Spinner } from "@paperclover/console/Spinner";305import { Spinner, withSpinner } from "@paperclover/console/Spinner";
306import * as meta from "./meta";
307import * as ssr from "./engine/ssr.ts";
308import * as sg from "./sitegen-lib.ts";
309import * as hot from "./hot.ts";
310import * as path from "node:path";
run.js+18-1
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1// This file allows using Node.js in combination with1// This file allows using Node.js in combination with
2// available plugins. Usage: "node run <script>"2// all available plugins. Usage: "node run <script>"
3import * as path from "node:path";3import * as path from "node:path";
4import * as util from "node:util";
4import process from "node:process";5import process from "node:process";
56
6// Disable experimental warnings (Type Stripping, etc)7// Disable experimental warnings (Type Stripping, etc)
...@@ -24,6 +25,22 @@ globalThis.console.warn = console.warn;...@@ -24,6 +25,22 @@ globalThis.console.warn = console.warn;
24globalThis.console.error = console.error;25globalThis.console.error = console.error;
25globalThis.console.debug = console.scoped("dbg");26globalThis.console.debug = console.scoped("dbg");
2627
28globalThis.UNWRAP = (t, ...args) => {
29 if (t == null) {
30 throw new Error(
31 args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")",
32 );
33 }
34 return t;
35};
36globalThis.ASSERT = (t, ...args) => {
37 if (!t) {
38 throw new Error(
39 args.length > 0 ? util.format(...args) : "Assertion Failed",
40 );
41 }
42};
43
27// Load with hooks44// Load with hooks
28if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {45if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
29 if (process.argv.length == 2) {46 if (process.argv.length == 2) {
tsconfig.json+13-11
...@@ -1,13 +1,15 @@...@@ -1,13 +1,15 @@
1{1{
2 "allowImportingTSExtensions": true,2 "compilerOptions": {
3 "baseDir": ".",3 "allowImportingTSExtensions": true,
4 "incremental": true,4 "baseDir": ".",
5 "lib": ["dom", "esnext"],5 "incremental": true,
6 "moduleResolution": "node18",6 "lib": ["dom", "esnext", "esnext.iterator"],
7 "outdir": ".clover/ts",7 "module": "nodenext",
8 "paths": { "@/*": ["src/*"] },8 "outdir": ".clover/ts",
9 "strict": true,9 "paths": { "@/*": ["src/*"] },
10 "target": "es2022",10 "strict": true,
11 "jsxImportSource": "#ssr",11 "target": "es2022",
12 "jsx": "react-jsx",12 "jsxImportSource": "#ssr",
13 "jsx": "react-jsx"
14 }
13}15}