authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-07 02:25:06-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log0a8336e50495e659d9cfd113e258cca17e8148ac
tree4da062a577e455124467812f7a4389a7b9b70b8e
parent69f8d0a5f1709f154d141d6585af5e1af99baa81
signature Commit is signed but in an unrecognized format.

restore sitegen.tsx

5 files changed, 245 insertions(+), 100 deletions(-)

framework/definitions.d.ts created+2
......@@ -0,0 +1,2 @@
1declare function UNWRAP<T>(value: T | null | undefined): T;
2declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
framework/fs.ts+45-9
......@@ -1,12 +1,18 @@
1// File System APIs
2import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync as nodeMkdirSync, rmSync } from 'node:fs';
3import { readFile, writeFile, readdir, stat, mkdir as nodeMkdir, rm } from 'node:fs/promises';
4
5// Re-export a mix of built-in Node.js sync+promise fs methods.
1// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in
2// Node.js sync+promise fs methods. For convenince.
63export {
7 readFileSync, writeFileSync, readdirSync, statSync, existsSync, rmSync ,
8 readFile, writeFile, readdir, stat, mkdir, rm
9}
4 existsSync,
5 readdir,
6 readdirSync,
7 readFile,
8 readFileSync,
9 rm,
10 rmSync,
11 stat,
12 statSync,
13 writeFile,
14 writeFileSync,
15};
1016
1117export function mkdir(dir: string) {
1218 return nodeMkdir(dir, { recursive: true });
......@@ -21,5 +27,35 @@ export async function writeMkdir(file: string, contents: Buffer | string) {
2127 return writeFile(file, contents);
2228}
2329
24import * as path from 'node:path';
30export function writeMkdirSync(file: string, contents: Buffer | string) {
31 mkdirSync(path.dirname(file));
32 return writeFileSync(file, contents);
33}
34
35export function readDirRecOptionalSync(dir: string) {
36 try {
37 return readdirSync(dir, { withFileTypes: true });
38 } catch (err: any) {
39 if (err.code === "ENOENT") return [];
40 throw err;
41 }
42}
2543
44import * as path from "node:path";
45import {
46 existsSync,
47 mkdirSync as nodeMkdirSync,
48 readdirSync,
49 readFileSync,
50 rmSync,
51 statSync,
52 writeFileSync,
53} from "node:fs";
54import {
55 mkdir as nodeMkdir,
56 readdir,
57 readFile,
58 rm,
59 stat,
60 writeFile,
61} from "node:fs/promises";
framework/sitegen.tsx+167-79
......@@ -9,20 +9,44 @@ function main() {
99 }, sitegen);
1010}
1111
12async function sitegen(status) {
12/**
13 * A filesystem object associated with some ID,
14 * such as a page's route to it's source file.
15 */
16interface FileItem {
17 id: string;
18 file: string;
19}
20
21async function sitegen(status: Spinner) {
1322 const startTime = performance.now();
1423
15 let root = path.resolve(import_meta.dirname, "../src");
16 const join = (...sub) => path.join(root, ...sub);
24 let root = path.resolve(import.meta.dirname, "../src");
25 const join = (...sub: string[]) => path.join(root, ...sub);
1726 const incr = new Incremental();
27
28 // Sitegen reviews every defined section for resources to process
1829 const sections: Section[] =
1930 require(path.join(root, "sections.ts")).siteSections;
20 const views = [];
21 const staticFiles = [];
22 let pages = [];
23 let scripts = [];
31
32 // Static files are compressed and served as-is.
33 // - "{section}/static/*.png"
34 let staticFiles: FileItem[] = [];
35 // Pages are rendered then served as static files.
36 // - "{section}/pages/*.marko"
37 let pages: FileItem[] = [];
38 // Views are dynamically rendered pages called via backend code.
39 // - "{section}/views/*.tsx"
40 let views: FileItem[] = [];
41 // Public scripts are bundled for the client as static assets under "/js/[...]"
42 // This is used for the file viewer's canvases.
43 // Note that '.client.ts' can be placed anywhere in the file structure.
44 // - "{section}/scripts/*.client.ts"
45 let scripts: FileItem[] = [];
46 // 'backend.ts'
2447 const backendFiles = [];
2548
49 // -- Scan for files --
2650 status.text = "Scanning Project";
2751 for (const section of sections) {
2852 const { root: sectionRoot } = section;
......@@ -39,16 +63,21 @@ async function sitegen(status) {
3963 },
4064 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
4165 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
42 { dir: sectionPath("views"), list: views, prefix: rootPrefix },
66 {
67 dir: sectionPath("views"),
68 list: views,
69 prefix: rootPrefix,
70 exclude: [".css", ".client.ts", ".client.tsx"],
71 },
4372 ];
4473 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
45 const pages2 = fs.readDirRecOptional(dir);
46 page: for (const page of pages2) {
47 if (page.isDirectory()) continue;
48 for (const ext2 of exclude) {
49 if (page.name.endsWith(ext2)) continue page;
74 const items = fs.readDirRecOptionalSync(dir);
75 item: for (const item of items) {
76 if (item.isDirectory()) continue;
77 for (const e of exclude) {
78 if (item.name.endsWith(e)) continue item;
5079 }
51 const file = path.relative(dir, page.parentPath + "/" + page.name);
80 const file = path.relative(dir, item.parentPath + "/" + item.name);
5281 const trim = ext
5382 ? file
5483 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
......@@ -56,7 +85,7 @@ async function sitegen(status) {
5685 if (prefix === "/" && id.endsWith("/index")) {
5786 id = id.slice(0, -"/index".length) || "/";
5887 }
59 list.push({ id, file: path.join(page.parentPath, page.name) });
88 list.push({ id, file: path.join(item.parentPath, item.name) });
6089 }
6190 }
6291 let backendFile = [
......@@ -67,86 +96,110 @@ async function sitegen(status) {
6796 }
6897 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
6998 const globalCssPath = join("global.css");
99
100 // TODO: invalidate incremental resources
101
102 // -- server side render --
70103 status.text = "Building";
71104 const cssOnce = new OnceMap();
72 const cssQueue = new Queue({
105 const cssQueue = new Queue<[string, string[], css.Theme], string>({
73106 name: "Bundle",
74107 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
75108 passive: true,
76109 getItemText: ([id]) => id,
77110 maxJobs: 2,
78111 });
79 const ssrResults = [];
80 function loadSsrModule(page) {
81 require(page.file);
112 interface RenderResult {
113 body: string;
114 head: string;
115 inlineCss: string;
116 scriptFiles: string[];
117 item: FileItem;
82118 }
83 async function doSsrPage(page) {
84 const module2 = require(page.file);
85 const Page = module2.default;
86 if (!Page) {
87 throw new Error("Page is missing a 'default' export.");
88 }
89 const metadata = module2.meta;
119 const renderResults: RenderResult[] = [];
120 async function loadPageModule({ file }: FileItem) {
121 require(file);
122 }
123 async function renderPage(item: FileItem) {
124 // -- load and validate module --
125 let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
126 item.file,
127 );
128 if (!Page) throw new Error("Page is missing a 'default' export.");
90129 if (!metadata) {
91 throw new Error("Page is missing 'meta' attribute with a title.");
130 throw new Error("Page is missing 'meta' export with a title.");
92131 }
132 if (layout?.theme) pageTheme = layout.theme;
93133 const theme = {
94134 bg: "#fff",
95135 fg: "#050505",
96136 primary: "#2e7dab",
97 ...module2.theme,
137 ...pageTheme,
98138 };
139
140 // -- metadata --
99141 const renderedMetaPromise = Promise.resolve(
100142 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
101143 ).then((m) => meta.resolveAndRenderMetadata(m));
102 const cssImports = [globalCssPath, ...hot.getCssImports(page.file)];
144 // -- css --
145 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
103146 const cssPromise = cssOnce.get(
104147 cssImports.join(":") + JSON.stringify(theme),
105 () => cssQueue.add([page.id, cssImports, theme]),
148 () => cssQueue.add([item.id, cssImports, theme]),
106149 );
150 // -- html --
107151 const sitegenApi = sg.initRender();
108 const body = await (0, import_ssr.ssrAsync)(
109 /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Page, {}),
110 {
111 sitegen: sitegenApi,
112 },
113 );
114 const inlineCss = await cssPromise;
115 const renderedMeta = await renderedMetaPromise;
152 const bodyPromise = await ssr.ssrAsync(<Page />, {
153 sitegen: sitegenApi,
154 });
155
156 const [body, inlineCss, renderedMeta] = await Promise.all([
157 bodyPromise,
158 cssPromise,
159 renderedMetaPromise,
160 ]);
116161 if (!renderedMeta.includes("<title>")) {
117162 throw new Error(
118 "Page is missing 'meta.title'. All pages need a title tag.",
163 "Page is missing 'meta.title'. " +
164 "All pages need a title tag.",
119165 );
120166 }
121 ssrResults.push({
167 // The script content is not ready, allow another page to Render. The page
168 // contents will be rebuilt at the end. This is more convenient anyways
169 // because it means client scripts don't re-render the page.
170 renderResults.push({
122171 body,
123172 head: renderedMeta,
124173 inlineCss,
125174 scriptFiles: Array.from(sitegenApi.scripts),
126 page,
175 item: item,
127176 });
128177 }
178 // This is done in two passes so that a page that throws during evaluation
179 // will report "Load Render Module" instead of "Render Static Page".
129180 const spinnerFormat = status.format;
130181 status.format = () => "";
131 const moduleLoadQueue = new import_queue.Queue({
182 const moduleLoadQueue = new Queue({
132183 name: "Load Render Module",
133 fn: loadSSRModule,
184 fn: loadPageModule,
134185 getItemText,
135186 maxJobs: 1,
136187 });
137188 moduleLoadQueue.addMany(pages);
138189 await moduleLoadQueue.done({ method: "stop" });
139 const pageQueue = new import_queue.Queue({
140 name: "Render",
141 fn: doSsrPage,
190 const pageQueue = new Queue({
191 name: "Render Static Page",
192 fn: renderPage,
142193 getItemText,
143194 maxJobs: 2,
144195 });
145196 pageQueue.addMany(pages);
146197 await pageQueue.done({ method: "stop" });
147198 status.format = spinnerFormat;
199
200 // -- bundle scripts --
148201 const referencedScripts = Array.from(
149 new Set(ssrResults.flatMap((r) => r.scriptFiles)),
202 new Set(renderResults.flatMap((r) => r.scriptFiles)),
150203 );
151204 const extraPublicScripts = scripts.map((entry) => entry.file);
152205 const uniqueCount = new Set([
......@@ -159,15 +212,17 @@ async function sitegen(status) {
159212 extraPublicScripts,
160213 incr,
161214 );
162 async function doStaticFile(page) {
163 const body = await fs.readFile(page.file);
215
216 // -- copy/compress static files --
217 async function doStaticFile(item: FileItem) {
218 const body = await fs.readFile(item.file);
164219 incr.putAsset({
165 srcId: "static:" + page.file,
166 key: page.id,
220 srcId: "static:" + item.file,
221 key: item.id,
167222 body,
168223 });
169224 }
170 const staticQueue = new import_queue.Queue({
225 const staticQueue = new Queue({
171226 name: "Load Static",
172227 fn: doStaticFile,
173228 getItemText,
......@@ -177,40 +232,68 @@ async function sitegen(status) {
177232 staticQueue.addMany(staticFiles);
178233 await staticQueue.done({ method: "stop" });
179234 status.format = spinnerFormat;
180 status.text = `Concat ${ssrResults.length} Pages`;
235
236 // -- concatenate static rendered pages --
237 status.text = `Concat ${renderResults.length} Pages`;
181238 await Promise.all(
182 ssrResults.map(async ({ page, body, head, inlineCss, scriptFiles }) => {
183 const doc = wrapDocument({
184 body,
185 head,
186 inlineCss,
187 scripts: scriptFiles.map(
188 (id) =>
189 UNWRAP(
190 incr.out.script.get(
191 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
239 renderResults.map(
240 async ({ item: page, body, head, inlineCss, scriptFiles }) => {
241 const doc = wrapDocument({
242 body,
243 head,
244 inlineCss,
245 scripts: scriptFiles.map(
246 (id) =>
247 UNWRAP(
248 incr.out.script.get(
249 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
250 ),
192251 ),
193 ),
194 ).map((x) => `{${x}}`).join(""),
195 });
196 incr.putAsset({
197 srcId: "page:" + page.file,
198 key: page.id,
199 body: doc,
200 headers: {
201 "Content-Type": "text/html",
202 },
203 });
204 }),
252 ).map((x) => `{${x}}`).join("\n"),
253 });
254 incr.putAsset({
255 srcId: "page:" + page.file,
256 key: page.id,
257 body: doc,
258 headers: {
259 "Content-Type": "text/html",
260 },
261 });
262 },
263 ),
205264 );
206265 status.format = () => "";
207266 status.text = ``;
267 // This will wait for all compression jobs to finish, which up
268 // to this point have been left as dangling promises.
208269 await incr.wait();
270
271 // Flush the site to disk.
209272 status.format = spinnerFormat;
210273 status.text = `Incremental Flush`;
211274 incr.flush();
212 incr.serializeToDisk();
213 return { elapsed: (performance.now() - startTime) / 1e3 };
275 incr.serializeToDisk(); // Allows picking up this state again
276 return { elapsed: (performance.now() - startTime) / 1000 };
277}
278
279function getItemText({ file }: FileItem) {
280 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
281}
282
283function wrapDocument({
284 body,
285 head,
286 inlineCss,
287 scripts,
288}: {
289 head: string;
290 body: string;
291 inlineCss: string;
292 scripts: string;
293}) {
294 return `<!doctype html><head>${head}${
295 inlineCss ? `<style>${inlineCss}</style>` : ""
296 }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
214297}
215298
216299import type { Section } from "./sitegen-lib.ts";
......@@ -218,5 +301,10 @@ import { OnceMap, Queue } from "./queue.ts";
218301import { Incremental } from "./incremental.ts";
219302import * as bundle from "./bundle.ts";
220303import * as css from "./css.ts";
221import * as fs from './fs.ts';
222import { withSpinner, Spinner } from "@paperclover/console/Spinner";
304import * as fs from "./fs.ts";
305import { Spinner, withSpinner } from "@paperclover/console/Spinner";
306import * as meta from "./meta";
307import * as ssr from "./engine/ssr.ts";
308import * as sg from "./sitegen-lib.ts";
309import * as hot from "./hot.ts";
310import * as path from "node:path";
run.js+18-1
......@@ -1,6 +1,7 @@
11// This file allows using Node.js in combination with
2// available plugins. Usage: "node run <script>"
2// all available plugins. Usage: "node run <script>"
33import * as path from "node:path";
4import * as util from "node:util";
45import process from "node:process";
56
67// Disable experimental warnings (Type Stripping, etc)
......@@ -24,6 +25,22 @@ globalThis.console.warn = console.warn;
2425globalThis.console.error = console.error;
2526globalThis.console.debug = console.scoped("dbg");
2627
28globalThis.UNWRAP = (t, ...args) => {
29 if (t == null) {
30 throw new Error(
31 args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")",
32 );
33 }
34 return t;
35};
36globalThis.ASSERT = (t, ...args) => {
37 if (!t) {
38 throw new Error(
39 args.length > 0 ? util.format(...args) : "Assertion Failed",
40 );
41 }
42};
43
2744// Load with hooks
2845if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
2946 if (process.argv.length == 2) {
tsconfig.json+13-11
......@@ -1,13 +1,15 @@
11{
2 "allowImportingTSExtensions": true,
3 "baseDir": ".",
4 "incremental": true,
5 "lib": ["dom", "esnext"],
6 "moduleResolution": "node18",
7 "outdir": ".clover/ts",
8 "paths": { "@/*": ["src/*"] },
9 "strict": true,
10 "target": "es2022",
11 "jsxImportSource": "#ssr",
12 "jsx": "react-jsx",
2 "compilerOptions": {
3 "allowImportingTSExtensions": true,
4 "baseDir": ".",
5 "incremental": true,
6 "lib": ["dom", "esnext", "esnext.iterator"],
7 "module": "nodenext",
8 "outdir": ".clover/ts",
9 "paths": { "@/*": ["src/*"] },
10 "strict": true,
11 "target": "es2022",
12 "jsxImportSource": "#ssr",
13 "jsx": "react-jsx"
14 }
1315}