authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-08 15:12:04-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log64a274c53ac58e86de04683d69da4a77151a835f
tree982b1bc272ccc113a8d52f3c2b0a1e30734b329d
parent6832e061d3ab21c081e593dc308f5087eacf377d
signature Commit is signed but in an unrecognized format.

primative backend support

25 files changed, 670 insertions(+), 510 deletions(-)

framework/assets.ts deleted-100
......@@ -1,100 +0,0 @@
1interface Loaded {
2 map: BuiltAssetMap;
3 buf: Buffer;
4}
5let assets: Loaded | null = null;
6
7export type StaticPageId = string;
8
9export async function reload() {
10 const [map, buf] = await Promise.all([
11 fs.readFile(".clover/static.json", "utf8"),
12 fs.readFile(".clover/static.blob"),
13 ]);
14 assets = {
15 map: JSON.parse(map),
16 buf,
17 };
18}
19
20export async function reloadSync() {
21 const map = fs.readFileSync(".clover/static.json", "utf8");
22 const buf = fs.readFileSync(".clover/static.blob");
23 assets = {
24 map: JSON.parse(map),
25 buf,
26 };
27}
28
29export async function assetMiddleware(c: Context, next: Next) {
30 if (!assets) await reload();
31 const asset = assets!.map[c.req.path];
32 if (asset) {
33 return assetInner(c, asset, 200);
34 }
35 return next();
36}
37
38export async function serveAsset(
39 c: Context,
40 id: StaticPageId,
41 status: StatusCode,
42) {
43 assets ?? await reload();
44 return assetInner(c, assets!.map[id], status);
45}
46
47export function hasAsset(id: string) {
48 if (!assets) reloadSync();
49 return assets!.map[id] !== undefined;
50}
51
52export function etagMatches(etag: string, ifNoneMatch: string) {
53 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
54}
55
56function subarrayAsset([start, end]: View) {
57 return assets!.buf.subarray(start, end);
58}
59
60function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
61 const ifnonematch = c.req.header("If-None-Match");
62 if (ifnonematch) {
63 const etag = asset.headers.ETag;
64 if (etagMatches(etag, ifnonematch)) {
65 c.res = new Response(null, {
66 status: 304,
67 statusText: "Not Modified",
68 headers: {
69 ETag: etag,
70 },
71 });
72 return;
73 }
74 }
75 const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
76 let body;
77 let headers = asset.headers;
78 if (acceptEncoding.includes("zstd") && asset.zstd) {
79 body = subarrayAsset(asset.zstd);
80 headers = {
81 ...asset.headers,
82 "Content-Encoding": "zstd",
83 };
84 } else if (acceptEncoding.includes("gzip") && asset.gzip) {
85 body = subarrayAsset(asset.gzip);
86 headers = {
87 ...asset.headers,
88 "Content-Encoding": "gzip",
89 };
90 } else {
91 body = subarrayAsset(asset.raw);
92 }
93 c.res = new Response(body, { headers, status });
94}
95
96import * as fs from "./fs.ts";
97import type { Context, Next } from "hono";
98import type { StatusCode } from "hono/utils/http-status";
99import type { BuiltAsset, BuiltAssetMap, View } from "./incremental.ts";
100import { Buffer } from "node:buffer";
framework/backend/entry-node.ts created+32
......@@ -0,0 +1,32 @@
1import "@paperclover/console/inject";
2
3const protocol = "http";
4
5const server = serve(app, ({ address, port }) => {
6 if (address === "::") address = "::1";
7 console.info(url.format({
8 protocol,
9 hostname: address,
10 port,
11 }));
12});
13
14process.on("SIGINT", () => {
15 server.close();
16 process.exit(0);
17});
18
19process.on("SIGTERM", () => {
20 server.close((err) => {
21 if (err) {
22 console.error(err);
23 process.exit(1);
24 }
25 process.exit(0);
26 });
27});
28
29import app from "#backend";
30import url from "node:url";
31import { serve } from "@hono/node-server";
32import process from "node:process";
framework/bundle.ts+17-4
......@@ -49,7 +49,7 @@ export async function bundleClientJavaScript(
4949 const publicScriptRoutes = extraPublicScripts.map((file) =>
5050 path.basename(file).replace(/\.client\.[tj]sx?/, "")
5151 );
52 const promises: Promise<unknown>[] = [];
52 const promises: Promise<void>[] = [];
5353 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
5454 for (const file of bundle.outputFiles) {
5555 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
......@@ -73,10 +73,23 @@ export async function bundleClientJavaScript(
7373 }));
7474 }
7575 }
76 await Promise.all(promises);
77}
7678
77 if (promises.length > 0) {
78 await Promise.all(promises);
79 }
79export async function bundleServerJavaScript(entryPoint: string) {
80 const bundle = await esbuild.build({
81 bundle: true,
82 chunkNames: "/js/c.[hash]",
83 entryNames: "/js/[name]",
84 assetNames: "/asset/[hash]",
85 entryPoints: [entryPoint],
86 format: "esm",
87 minify: true,
88 outdir: "/out!",
89 plugins,
90 splitting: true,
91 write: false,
92 });
8093}
8194
8295import * as esbuild from "esbuild";
framework/fs.ts+1-1
......@@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) {
3434
3535export function readDirRecOptionalSync(dir: string) {
3636 try {
37 return readdirSync(dir, { withFileTypes: true });
37 return readdirSync(dir, { recursive: true, withFileTypes: true });
3838 } catch (err: any) {
3939 if (err.code === "ENOENT") return [];
4040 throw err;
framework/generate.tsx created+299
......@@ -0,0 +1,299 @@
1export function main() {
2 return withSpinner({
3 text: "Recovering State",
4 successText: ({ elapsed }) =>
5 "sitegen! update in " + elapsed.toFixed(1) + "s",
6 failureText: () => "sitegen FAIL",
7 }, sitegen);
8}
9
10/**
11 * A filesystem object associated with some ID,
12 * such as a page's route to it's source file.
13 */
14interface FileItem {
15 id: string;
16 file: string;
17}
18
19async function sitegen(status: Spinner) {
20 const startTime = performance.now();
21
22 let root = path.resolve(import.meta.dirname, "../src");
23 const join = (...sub: string[]) => path.join(root, ...sub);
24 const incr = new Incremental();
25
26 // Sitegen reviews every defined section for resources to process
27 const sections: sg.Section[] =
28 require(path.join(root, "site.ts")).siteSections;
29
30 // Static files are compressed and served as-is.
31 // - "{section}/static/*.png"
32 let staticFiles: FileItem[] = [];
33 // Pages are rendered then served as static files.
34 // - "{section}/pages/*.marko"
35 let pages: FileItem[] = [];
36 // Views are dynamically rendered pages called via backend code.
37 // - "{section}/views/*.tsx"
38 let views: FileItem[] = [];
39 // Public scripts are bundled for the client as static assets under "/js/[...]"
40 // This is used for the file viewer's canvases.
41 // Note that '.client.ts' can be placed anywhere in the file structure.
42 // - "{section}/scripts/*.client.ts"
43 let scripts: FileItem[] = [];
44
45 // -- Scan for files --
46 status.text = "Scanning Project";
47 for (const section of sections) {
48 const { root: sectionRoot } = section;
49 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
50 const rootPrefix = root === sectionRoot
51 ? ""
52 : path.relative(root, sectionRoot) + "/";
53 const kinds = [
54 {
55 dir: sectionPath("pages"),
56 list: pages,
57 prefix: "/",
58 exclude: [".css", ".client.ts", ".client.tsx"],
59 },
60 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
61 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
62 {
63 dir: sectionPath("views"),
64 list: views,
65 prefix: rootPrefix,
66 exclude: [".css", ".client.ts", ".client.tsx"],
67 },
68 ];
69 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
70 const items = fs.readDirRecOptionalSync(dir);
71 item: for (const item of items) {
72 if (item.isDirectory()) continue;
73 for (const e of exclude) {
74 if (item.name.endsWith(e)) continue item;
75 }
76 const file = path.relative(dir, item.parentPath + "/" + item.name);
77 const trim = ext
78 ? file
79 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
80 let id = prefix + trim.replaceAll("\\", "/");
81 if (prefix === "/" && id.endsWith("/index")) {
82 id = id.slice(0, -"/index".length) || "/";
83 }
84 list.push({ id, file: path.join(item.parentPath, item.name) });
85 }
86 }
87 }
88 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
89 const globalCssPath = join("global.css");
90
91 // TODO: invalidate incremental resources
92
93 // -- server side render --
94 status.text = "Building";
95 const cssOnce = new OnceMap<string>();
96 const cssQueue = new Queue<[string, string[], css.Theme], string>({
97 name: "Bundle",
98 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
99 passive: true,
100 getItemText: ([id]) => id,
101 maxJobs: 2,
102 });
103 interface RenderResult {
104 body: string;
105 head: string;
106 inlineCss: string;
107 scriptFiles: string[];
108 item: FileItem;
109 }
110 const renderResults: RenderResult[] = [];
111 async function loadPageModule({ file }: FileItem) {
112 require(file);
113 }
114 async function renderPage(item: FileItem) {
115 // -- load and validate module --
116 let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
117 item.file,
118 );
119 if (!Page) throw new Error("Page is missing a 'default' export.");
120 if (!metadata) {
121 throw new Error("Page is missing 'meta' export with a title.");
122 }
123 if (layout?.theme) pageTheme = layout.theme;
124 const theme = {
125 bg: "#fff",
126 fg: "#050505",
127 primary: "#2e7dab",
128 ...pageTheme,
129 };
130
131 // -- metadata --
132 const renderedMetaPromise = Promise.resolve(
133 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
134 ).then((m) => meta.renderMeta(m));
135 // -- css --
136 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
137 const cssPromise = cssOnce.get(
138 cssImports.join(":") + JSON.stringify(theme),
139 () => cssQueue.add([item.id, cssImports, theme]),
140 );
141 // -- html --
142 const bodyPromise = await ssr.ssrAsync(<Page />, {
143 sitegen: sg.initRender(),
144 });
145
146 const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([
147 bodyPromise,
148 cssPromise,
149 renderedMetaPromise,
150 ]);
151 if (!renderedMeta.includes("<title>")) {
152 throw new Error(
153 "Page is missing 'meta.title'. " +
154 "All pages need a title tag.",
155 );
156 }
157 // The script content is not ready, allow another page to Render. The page
158 // contents will be rebuilt at the end. This is more convenient anyways
159 // because it means client scripts don't re-render the page.
160 renderResults.push({
161 body: text,
162 head: renderedMeta,
163 inlineCss,
164 scriptFiles: Array.from(addon.sitegen.scripts),
165 item: item,
166 });
167 }
168 // This is done in two passes so that a page that throws during evaluation
169 // will report "Load Render Module" instead of "Render Static Page".
170 const spinnerFormat = status.format;
171 status.format = () => "";
172 const moduleLoadQueue = new Queue({
173 name: "Load Render Module",
174 fn: loadPageModule,
175 getItemText,
176 maxJobs: 1,
177 });
178 moduleLoadQueue.addMany(pages);
179 await moduleLoadQueue.done({ method: "stop" });
180 const pageQueue = new Queue({
181 name: "Render Static Page",
182 fn: renderPage,
183 getItemText,
184 maxJobs: 2,
185 });
186 pageQueue.addMany(pages);
187 await pageQueue.done({ method: "stop" });
188 status.format = spinnerFormat;
189
190 // -- bundle scripts --
191 const referencedScripts = Array.from(
192 new Set(renderResults.flatMap((r) => r.scriptFiles)),
193 );
194 const extraPublicScripts = scripts.map((entry) => entry.file);
195 const uniqueCount = new Set([
196 ...referencedScripts,
197 ...extraPublicScripts,
198 ]).size;
199 status.text = `Bundle ${uniqueCount} Scripts`;
200 await bundle.bundleClientJavaScript(
201 referencedScripts,
202 extraPublicScripts,
203 incr,
204 );
205
206 // -- copy/compress static files --
207 async function doStaticFile(item: FileItem) {
208 const body = await fs.readFile(item.file);
209 await incr.putAsset({
210 srcId: "static:" + item.file,
211 key: item.id,
212 body,
213 });
214 }
215 const staticQueue = new Queue({
216 name: "Load Static",
217 fn: doStaticFile,
218 getItemText,
219 maxJobs: 16,
220 });
221 status.format = () => "";
222 staticQueue.addMany(staticFiles);
223 await staticQueue.done({ method: "stop" });
224 status.format = spinnerFormat;
225
226 // -- concatenate static rendered pages --
227 status.text = `Concat ${renderResults.length} Pages`;
228 await Promise.all(
229 renderResults.map(
230 async ({ item: page, body, head, inlineCss, scriptFiles }) => {
231 const doc = wrapDocument({
232 body,
233 head,
234 inlineCss,
235 scripts: scriptFiles.map(
236 (id) =>
237 UNWRAP(
238 incr.out.script.get(
239 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
240 ),
241 ),
242 ).map((x) => `{${x}}`).join("\n"),
243 });
244 await incr.putAsset({
245 srcId: "page:" + page.file,
246 key: page.id,
247 body: doc,
248 headers: {
249 "Content-Type": "text/html",
250 },
251 });
252 },
253 ),
254 );
255 status.format = () => "";
256 status.text = ``;
257 // This will wait for all compression jobs to finish, which up
258 // to this point have been left as dangling promises.
259 await incr.wait();
260
261 // Flush the site to disk.
262 status.format = spinnerFormat;
263 status.text = `Incremental Flush`;
264 incr.flush();
265 incr.serializeToDisk(); // Allows picking up this state again
266 return { elapsed: (performance.now() - startTime) / 1000 };
267}
268
269function getItemText({ file }: FileItem) {
270 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
271}
272
273function wrapDocument({
274 body,
275 head,
276 inlineCss,
277 scripts,
278}: {
279 head: string;
280 body: string;
281 inlineCss: string;
282 scripts: string;
283}) {
284 return `<!doctype html><head>${head}${
285 inlineCss ? `<style>${inlineCss}</style>` : ""
286 }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
287}
288
289import { OnceMap, Queue } from "./queue.ts";
290import { Incremental } from "./incremental.ts";
291import * as bundle from "./bundle.ts";
292import * as css from "./css.ts";
293import * as fs from "./fs.ts";
294import { Spinner, withSpinner } from "@paperclover/console/Spinner";
295import * as meta from "./meta.ts";
296import * as ssr from "./engine/ssr.ts";
297import * as sg from "#sitegen";
298import * as hot from "./hot.ts";
299import * as path from "node:path";
framework/hot.ts+1-2
......@@ -146,6 +146,7 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {
146146 target: "esnext",
147147 jsx: "automatic",
148148 jsxImportSource: "#ssr",
149 sourcefile: filepath,
149150 }).code;
150151 return module._compile(src, filepath, "commonjs");
151152}
......@@ -162,8 +163,6 @@ function loadMarko(module: NodeJS.Module, filepath: string) {
162163 ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';
163164 }
164165
165 console.log(src);
166 console.log("---");
167166 src = marko.compileSync(src, filepath).code;
168167 src = src.replace("marko/debug/html", "#ssr/marko");
169168 return loadEsbuildCode(module, filepath, src);
framework/incremental.ts+10-13
......@@ -7,7 +7,6 @@
77// all its imports) to map to the build of a page, which produces an HTML file
88// plus a list of scripts.
99
10import { Buffer } from "node:buffer";
1110interface ArtifactMap {
1211 asset: Asset;
1312 script: string;
......@@ -126,18 +125,17 @@ export class Incremental {
126125 },
127126 hash,
128127 };
128 const a = this.put({ ...info, type: "asset", value });
129129 if (!this.compress.has(hash)) {
130130 const label = info.key;
131131 this.compress.set(hash, {
132132 zstd: undefined,
133133 gzip: undefined,
134134 });
135 await Promise.all([
136 this.compressQueue.add({ label, buffer, algo: "zstd", hash }),
137 this.compressQueue.add({ label, buffer, algo: "gzip", hash }),
138 ]);
135 this.compressQueue.add({ label, buffer, algo: "zstd", hash });
136 this.compressQueue.add({ label, buffer, algo: "gzip", hash });
139137 }
140 return this.put({ ...info, type: "asset", value });
138 return a;
141139 }
142140
143141 async compressImpl({ algo, buffer, hash }: CompressJob) {
......@@ -167,16 +165,18 @@ export class Incremental {
167165 }
168166
169167 async wait() {
170 await this.compressQueue.done({ method: "stop" });
168 await this.compressQueue.done({ method: "success" });
171169 }
170
172171 async flush() {
172 ASSERT(!this.compressQueue.active);
173173 const writer = new BufferWriter();
174174 const asset = Object.fromEntries(
175175 Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
176176 const raw = writer.write(buffer, hash);
177177 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
178 const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null;
179 const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null;
178 const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
179 const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
180180 return [key, {
181181 raw,
182182 gzip,
......@@ -252,10 +252,6 @@ export interface SerializedMeta {
252252 script: [key: string, value: string][];
253253}
254254
255function never(): never {
256 throw new Error("Impossible");
257}
258
259255import * as path from "node:path";
260256import * as fs from "./fs.ts";
261257import * as zlib from "node:zlib";
......@@ -263,3 +259,4 @@ import * as util from "node:util";
263259import { Queue } from "./queue.ts";
264260import * as hot from "./hot.ts";
265261import * as mime from "./mime.ts";
262import { Buffer } from "node:buffer";
framework/lib/assets.ts created+112
......@@ -0,0 +1,112 @@
1interface Loaded {
2 map: BuiltAssetMap;
3 buf: Buffer;
4}
5let assets: Loaded | null = null;
6
7export type StaticPageId = string;
8
9export async function reload() {
10 const [map, buf] = await Promise.all([
11 fs.readFile(".clover/static.json", "utf8"),
12 fs.readFile(".clover/static.blob"),
13 ]);
14 assets = {
15 map: JSON.parse(map),
16 buf,
17 };
18}
19
20export async function reloadSync() {
21 const map = fs.readFileSync(".clover/static.json", "utf8");
22 const buf = fs.readFileSync(".clover/static.blob");
23 assets = {
24 map: JSON.parse(map),
25 buf,
26 };
27}
28
29export async function middleware(c: Context, next: Next) {
30 if (!assets) await reload();
31 const asset = assets!.map[c.req.path];
32 if (asset) {
33 return assetInner(c, asset, 200);
34 }
35 return next();
36}
37
38export async function notFound(c: Context) {
39 if (!assets) await reload();
40 let pathname = c.req.path;
41 do {
42 const asset = assets!.map[pathname + "/404"];
43 if (asset) return assetInner(c, asset, 404);
44 pathname = pathname.slice(0, pathname.lastIndexOf("/"));
45 } while (pathname);
46 const asset = assets!.map["/404"];
47 if (asset) return assetInner(c, asset, 404);
48 return c.text("the 'Not Found' page was not found", 404);
49}
50
51export async function serveAsset(
52 c: Context,
53 id: StaticPageId,
54 status: StatusCode,
55) {
56 assets ?? await reload();
57 return assetInner(c, assets!.map[id], status);
58}
59
60export function hasAsset(id: string) {
61 if (!assets) reloadSync();
62 return assets!.map[id] !== undefined;
63}
64
65export function etagMatches(etag: string, ifNoneMatch: string) {
66 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
67}
68
69function subarrayAsset([start, end]: View) {
70 return assets!.buf.subarray(start, end);
71}
72
73function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
74 const ifnonematch = c.req.header("If-None-Match");
75 if (ifnonematch) {
76 const etag = asset.headers.ETag;
77 if (etagMatches(etag, ifnonematch)) {
78 return c.res = new Response(null, {
79 status: 304,
80 statusText: "Not Modified",
81 headers: {
82 ETag: etag,
83 },
84 });
85 }
86 }
87 const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
88 let body;
89 let headers = asset.headers;
90 if (acceptEncoding.includes("zstd") && asset.zstd) {
91 body = subarrayAsset(asset.zstd);
92 headers = {
93 ...asset.headers,
94 "Content-Encoding": "zstd",
95 };
96 } else if (acceptEncoding.includes("gzip") && asset.gzip) {
97 body = subarrayAsset(asset.gzip);
98 headers = {
99 ...asset.headers,
100 "Content-Encoding": "gzip",
101 };
102 } else {
103 body = subarrayAsset(asset.raw);
104 }
105 return c.res = new Response(body, { headers, status });
106}
107
108import * as fs from "../fs.ts";
109import type { Context, Next } from "hono";
110import type { StatusCode } from "hono/utils/http-status";
111import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
112import { Buffer } from "node:buffer";
framework/lib/sitegen.ts created+52
......@@ -0,0 +1,52 @@
1// Import this file with 'import * as sg from "#sitegen";'
2export type ScriptId = string;
3
4const frameworkDir = path.dirname(import.meta.dirname);
5
6export interface SitegenRender {
7 scripts: Set<ScriptId>;
8}
9
10export function initRender(): SitegenRender {
11 return {
12 scripts: new Set(),
13 };
14}
15
16export function getRender() {
17 return ssr.getUserData<SitegenRender>("sitegen", () => {
18 throw new Error(
19 "This function can only be used in a page (static or view)",
20 );
21 });
22}
23
24/** Add a client-side script to the page. */
25export function addScript(id: ScriptId) {
26 const srcFile: string = util.getCallSites()
27 .find((site) => !site.scriptName.startsWith(frameworkDir))!
28 .scriptName;
29 const filePath = hot.resolveFrom(srcFile, id);
30 if (
31 !filePath.endsWith(".client.ts") &&
32 !filePath.endsWith(".client.tsx")
33 ) {
34 throw new Error("addScript must be a .client.ts or .client.tsx");
35 }
36 getRender().scripts.add(filePath);
37}
38
39export function Script({ src }: { src: ScriptId }) {
40 if (!src) throw new Error("Missing 'src' attribute");
41 addScript(src);
42 return null;
43}
44
45export interface Section {
46 root: string;
47}
48
49import * as ssr from "../engine/ssr.ts";
50import * as util from "node:util";
51import * as hot from "../hot.ts";
52import * as path from "node:path";
framework/queue.ts+4
......@@ -165,6 +165,10 @@ export class Queue<T, R> {
165165
166166 if (bar) bar[method]();
167167 }
168
169 get active(): boolean {
170 return this.#active.length !== 0;
171 }
168172}
169173
170174const cwd = process.cwd();
framework/sitegen-lib.ts deleted-49
......@@ -1,49 +0,0 @@
1// Import this file with 'import * as sg from "#sitegen";'
2export type ScriptId = string;
3
4export interface SitegenRender {
5 scripts: Set<ScriptId>;
6}
7
8export function initRender(): SitegenRender {
9 return {
10 scripts: new Set(),
11 };
12}
13
14export function getRender() {
15 return ssr.getUserData<SitegenRender>("sitegen", () => {
16 throw new Error(
17 "This function can only be used in a page (static or view)",
18 );
19 });
20}
21
22/** Add a client-side script to the page. */
23export function addScript(id: ScriptId) {
24 const srcFile: string = util.getCallSites()
25 .find((site) => !site.scriptName.startsWith(import.meta.dirname))!
26 .scriptName;
27 const filePath = hot.resolveFrom(srcFile, id);
28 if (
29 !filePath.endsWith(".client.ts") &&
30 !filePath.endsWith(".client.tsx")
31 ) {
32 throw new Error("addScript must be a .client.ts or .client.tsx");
33 }
34 getRender().scripts.add(filePath);
35}
36
37export function Script({ src }: { src: ScriptId }) {
38 if (!src) throw new Error("Missing 'src' attribute");
39 addScript(src);
40 return null;
41}
42
43export interface Section {
44 root: string;
45}
46
47import * as ssr from "./engine/ssr.ts";
48import * as util from "node:util";
49import * as hot from "./hot.ts";
framework/sitegen.tsx deleted-309
......@@ -1,309 +0,0 @@
1// Sitegen! Clover's static site generator, built with love.
2
3export function main() {
4 return withSpinner({
5 text: "Recovering State",
6 successText: ({ elapsed }) =>
7 "sitegen! update in " + elapsed.toFixed(1) + "s",
8 failureText: () => "sitegen FAIL",
9 }, sitegen);
10}
11
12/**
13 * A filesystem object associated with some ID,
14 * such as a page's route to it's source file.
15 */
16interface FileItem {
17 id: string;
18 file: string;
19}
20
21async function sitegen(status: Spinner) {
22 const startTime = performance.now();
23
24 let root = path.resolve(import.meta.dirname, "../src");
25 const join = (...sub: string[]) => path.join(root, ...sub);
26 const incr = new Incremental();
27
28 // Sitegen reviews every defined section for resources to process
29 const sections: Section[] =
30 require(path.join(root, "sections.ts")).siteSections;
31
32 // Static files are compressed and served as-is.
33 // - "{section}/static/*.png"
34 let staticFiles: FileItem[] = [];
35 // Pages are rendered then served as static files.
36 // - "{section}/pages/*.marko"
37 let pages: FileItem[] = [];
38 // Views are dynamically rendered pages called via backend code.
39 // - "{section}/views/*.tsx"
40 let views: FileItem[] = [];
41 // Public scripts are bundled for the client as static assets under "/js/[...]"
42 // This is used for the file viewer's canvases.
43 // Note that '.client.ts' can be placed anywhere in the file structure.
44 // - "{section}/scripts/*.client.ts"
45 let scripts: FileItem[] = [];
46 // 'backend.ts'
47 const backendFiles = [];
48
49 // -- Scan for files --
50 status.text = "Scanning Project";
51 for (const section of sections) {
52 const { root: sectionRoot } = section;
53 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
54 const rootPrefix = root === sectionRoot
55 ? ""
56 : path.relative(root, sectionRoot) + "/";
57 const kinds = [
58 {
59 dir: sectionPath("pages"),
60 list: pages,
61 prefix: "/",
62 exclude: [".css", ".client.ts", ".client.tsx"],
63 },
64 { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
65 { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
66 {
67 dir: sectionPath("views"),
68 list: views,
69 prefix: rootPrefix,
70 exclude: [".css", ".client.ts", ".client.tsx"],
71 },
72 ];
73 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
74 const items = fs.readDirRecOptionalSync(dir);
75 item: for (const item of items) {
76 if (item.isDirectory()) continue;
77 for (const e of exclude) {
78 if (item.name.endsWith(e)) continue item;
79 }
80 const file = path.relative(dir, item.parentPath + "/" + item.name);
81 const trim = ext
82 ? file
83 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
84 let id = prefix + trim.replaceAll("\\", "/");
85 if (prefix === "/" && id.endsWith("/index")) {
86 id = id.slice(0, -"/index".length) || "/";
87 }
88 list.push({ id, file: path.join(item.parentPath, item.name) });
89 }
90 }
91 let backendFile = [
92 sectionPath("backend.ts"),
93 sectionPath("backend.tsx"),
94 ].find((file) => fs.existsSync(file));
95 if (backendFile) backendFiles.push(backendFile);
96 }
97 scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
98 const globalCssPath = join("global.css");
99
100 // TODO: invalidate incremental resources
101
102 // -- server side render --
103 status.text = "Building";
104 const cssOnce = new OnceMap<string>();
105 const cssQueue = new Queue<[string, string[], css.Theme], string>({
106 name: "Bundle",
107 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
108 passive: true,
109 getItemText: ([id]) => id,
110 maxJobs: 2,
111 });
112 interface RenderResult {
113 body: string;
114 head: string;
115 inlineCss: string;
116 scriptFiles: string[];
117 item: FileItem;
118 }
119 const renderResults: RenderResult[] = [];
120 async function loadPageModule({ file }: FileItem) {
121 require(file);
122 }
123 async function renderPage(item: FileItem) {
124 // -- load and validate module --
125 let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
126 item.file,
127 );
128 if (!Page) throw new Error("Page is missing a 'default' export.");
129 if (!metadata) {
130 throw new Error("Page is missing 'meta' export with a title.");
131 }
132 if (layout?.theme) pageTheme = layout.theme;
133 const theme = {
134 bg: "#fff",
135 fg: "#050505",
136 primary: "#2e7dab",
137 ...pageTheme,
138 };
139
140 // -- metadata --
141 const renderedMetaPromise = Promise.resolve(
142 typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
143 ).then((m) => meta.renderMeta(m));
144 // -- css --
145 const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
146 const cssPromise = cssOnce.get(
147 cssImports.join(":") + JSON.stringify(theme),
148 () => cssQueue.add([item.id, cssImports, theme]),
149 );
150 // -- html --
151 const bodyPromise = await ssr.ssrAsync(<Page />, {
152 sitegen: sg.initRender(),
153 });
154
155 const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([
156 bodyPromise,
157 cssPromise,
158 renderedMetaPromise,
159 ]);
160 if (!renderedMeta.includes("<title>")) {
161 throw new Error(
162 "Page is missing 'meta.title'. " +
163 "All pages need a title tag.",
164 );
165 }
166 // The script content is not ready, allow another page to Render. The page
167 // contents will be rebuilt at the end. This is more convenient anyways
168 // because it means client scripts don't re-render the page.
169 renderResults.push({
170 body: text,
171 head: renderedMeta,
172 inlineCss,
173 scriptFiles: Array.from(addon.sitegen.scripts),
174 item: item,
175 });
176 }
177 // This is done in two passes so that a page that throws during evaluation
178 // will report "Load Render Module" instead of "Render Static Page".
179 const spinnerFormat = status.format;
180 status.format = () => "";
181 const moduleLoadQueue = new Queue({
182 name: "Load Render Module",
183 fn: loadPageModule,
184 getItemText,
185 maxJobs: 1,
186 });
187 moduleLoadQueue.addMany(pages);
188 await moduleLoadQueue.done({ method: "stop" });
189 const pageQueue = new Queue({
190 name: "Render Static Page",
191 fn: renderPage,
192 getItemText,
193 maxJobs: 2,
194 });
195 pageQueue.addMany(pages);
196 await pageQueue.done({ method: "stop" });
197 status.format = spinnerFormat;
198
199 // -- bundle scripts --
200 const referencedScripts = Array.from(
201 new Set(renderResults.flatMap((r) => r.scriptFiles)),
202 );
203 const extraPublicScripts = scripts.map((entry) => entry.file);
204 const uniqueCount = new Set([
205 ...referencedScripts,
206 ...extraPublicScripts,
207 ]).size;
208 status.text = `Bundle ${uniqueCount} Scripts`;
209 await bundle.bundleClientJavaScript(
210 referencedScripts,
211 extraPublicScripts,
212 incr,
213 );
214
215 // -- copy/compress static files --
216 async function doStaticFile(item: FileItem) {
217 const body = await fs.readFile(item.file);
218 incr.putAsset({
219 srcId: "static:" + item.file,
220 key: item.id,
221 body,
222 });
223 }
224 const staticQueue = new Queue({
225 name: "Load Static",
226 fn: doStaticFile,
227 getItemText,
228 maxJobs: 16,
229 });
230 status.format = () => "";
231 staticQueue.addMany(staticFiles);
232 await staticQueue.done({ method: "stop" });
233 status.format = spinnerFormat;
234
235 // -- concatenate static rendered pages --
236 status.text = `Concat ${renderResults.length} Pages`;
237 await Promise.all(
238 renderResults.map(
239 async ({ item: page, body, head, inlineCss, scriptFiles }) => {
240 const doc = wrapDocument({
241 body,
242 head,
243 inlineCss,
244 scripts: scriptFiles.map(
245 (id) =>
246 UNWRAP(
247 incr.out.script.get(
248 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
249 ),
250 ),
251 ).map((x) => `{${x}}`).join("\n"),
252 });
253 incr.putAsset({
254 srcId: "page:" + page.file,
255 key: page.id,
256 body: doc,
257 headers: {
258 "Content-Type": "text/html",
259 },
260 });
261 },
262 ),
263 );
264 status.format = () => "";
265 status.text = ``;
266 // This will wait for all compression jobs to finish, which up
267 // to this point have been left as dangling promises.
268 await incr.wait();
269
270 // Flush the site to disk.
271 status.format = spinnerFormat;
272 status.text = `Incremental Flush`;
273 incr.flush();
274 incr.serializeToDisk(); // Allows picking up this state again
275 return { elapsed: (performance.now() - startTime) / 1000 };
276}
277
278function getItemText({ file }: FileItem) {
279 return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
280}
281
282function wrapDocument({
283 body,
284 head,
285 inlineCss,
286 scripts,
287}: {
288 head: string;
289 body: string;
290 inlineCss: string;
291 scripts: string;
292}) {
293 return `<!doctype html><head>${head}${
294 inlineCss ? `<style>${inlineCss}</style>` : ""
295 }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
296}
297
298import type { Section } from "./sitegen-lib.ts";
299import { OnceMap, Queue } from "./queue.ts";
300import { Incremental } from "./incremental.ts";
301import * as bundle from "./bundle.ts";
302import * as css from "./css.ts";
303import * as fs from "./fs.ts";
304import { Spinner, withSpinner } from "@paperclover/console/Spinner";
305import * as meta from "./meta.ts";
306import * as ssr from "./engine/ssr.ts";
307import * as sg from "./sitegen-lib.ts";
308import * as hot from "./hot.ts";
309import * as path from "node:path";
framework/sqlite.ts+2-5
......@@ -41,11 +41,8 @@ export class WrappedDatabase {
4141 (key, version) values (?, ?);
4242 `),
4343 ));
44 const { changes, lastInsertRowid } = s.run(name, 1);
45 console.log(changes, lastInsertRowid);
46 if (changes === 1) {
47 this.node.exec(schema);
48 }
44 const { changes } = s.run(name, 1);
45 if (changes === 1) this.node.exec(schema);
4946 }
5047
5148 prepare<Args extends unknown[] = [], Result = unknown>(
package.json+4-1
......@@ -15,7 +15,9 @@
1515 "typescript": "^5.8.3"
1616 },
1717 "imports": {
18 "#sitegen": "./framework/sitegen-lib.ts",
18 "#backend": "./src/backend.ts",
19 "#sitegen": "./framework/lib/sitegen.ts",
20 "#sitegen/*": "./framework/lib/*.ts",
1921 "#sqlite": "./framework/sqlite.ts",
2022 "#ssr": "./framework/engine/ssr.ts",
2123 "#ssr/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts",
......@@ -26,6 +28,7 @@
2628 "production": "marko/production",
2729 "node": "marko/debug/html"
2830 },
31 "#hono": "hono",
2932 "#hono/platform": {
3033 "bun": "hono/bun",
3134 "deno": "hono/deno",
readme.md created+1
......@@ -0,0 +1 @@
1# clover sitegen framework
repl.js+3-4
......@@ -27,8 +27,7 @@ hot.load("node:repl").start({
2727});
2828
2929setTimeout(() => {
30 hot.reloadRecursive("./framework/engine/ssr.ts");
31 hot.reloadRecursive("./framework/bundle.ts");
30 hot.reloadRecursive("./framework/generate.tsx");
3231}, 100);
3332
3433async function evaluate(code) {
......@@ -41,11 +40,11 @@ async function evaluate(code) {
4140 if (code[0] === "=") {
4241 try {
4342 const result = await eval(code[1]);
44 console.log(inspect(result));
43 console.info(inspect(result));
4544 } catch (err) {
4645 if (err instanceof SyntaxError) {
4746 const result = await eval("(async() => { return " + code + " })()");
48 console.log(inspect(result));
47 console.info(inspect(result));
4948 } else {
5049 throw err;
5150 }
run.js+1-1
......@@ -18,7 +18,7 @@ import process from "node:process";
1818const hot = await import("./framework/hot.ts");
1919
2020const console = hot.load("@paperclover/console");
21globalThis.console.log = console.info;
21globalThis.console["log"] = console.info;
2222globalThis.console.info = console.info;
2323globalThis.console.warn = console.warn;
2424globalThis.console.error = console.error;
src/admin.ts created+64
......@@ -0,0 +1,64 @@
1const cookieAge = 60 * 60 * 24 * 365; // 1 year
2
3let lastKnownToken: string | null = null;
4function compareToken(token: string) {
5 if (token === lastKnownToken) return true;
6 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
7 return token === lastKnownToken;
8}
9
10export async function middleware(c: Context, next: Next) {
11 if (c.req.path.startsWith("/admin")) {
12 return adminInner(c, next);
13 }
14 return next();
15}
16
17export function adminInner(c: Context, next: Next) {
18 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
19
20 if (c.req.path === "/admin/login") {
21 const key = c.req.query("key");
22 if (key) {
23 if (compareToken(key)) {
24 return c.body(null, 303, {
25 "Location": "/admin",
26 "Set-Cookie":
27 `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
28 });
29 }
30 return serveAsset(c, "/admin/login/fail", 403);
31 }
32 if (token && compareToken(token)) {
33 return c.redirect("/admin", 303);
34 }
35 if (c.req.method === "POST") {
36 return serveAsset(c, "/admin/login/fail", 403);
37 } else {
38 return serveAsset(c, "/admin/login", 200);
39 }
40 }
41
42 if (c.req.path === "/admin/logout") {
43 return c.body(null, 303, {
44 "Location": "/admin/login",
45 "Set-Cookie":
46 `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`,
47 });
48 }
49
50 if (token && compareToken(token)) {
51 return next();
52 }
53
54 return c.redirect("/admin/login", 303);
55}
56
57export function hasAdminToken(c: Context) {
58 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
59 return token && compareToken(token);
60}
61
62import * as fs from "fs";
63import type { Context, Next } from "hono";
64import { serveAsset } from "#sitegen/assets";
src/backend.ts created+38
......@@ -0,0 +1,38 @@
1const logHttp = scoped("http", { color: "magenta" });
2
3const app = new Hono();
4
5app.notFound(assets.notFound);
6
7app.use(trimTrailingSlash());
8app.use(removeDuplicateSlashes);
9app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4))));
10app.use(admin.middleware);
11
12app.route("", require("./q+a/backend.ts").app);
13// app.route("", require("./file-viewer/backend.tsx").app);
14// app.route("", require("./friends/backend.tsx").app);
15
16app.use(assets.middleware);
17
18export default app;
19
20async function removeDuplicateSlashes(c: Context, next: Next) {
21 const path = c.req.path;
22 if (/\/\/+/.test(path)) {
23 const normalizedPath = path.replace(/\/\/+/g, "/");
24 const query = c.req.query();
25 const queryString = Object.keys(query).length > 0
26 ? "?" + new URLSearchParams(query).toString()
27 : "";
28 return c.redirect(normalizedPath + queryString, 301);
29 }
30 await next();
31}
32
33import { Hono } from "#hono";
34import { logger } from "hono/logger";
35import { trimTrailingSlash } from "hono/trailing-slash";
36import * as assets from "#sitegen/assets";
37import * as admin from "./admin.ts";
38import { scoped } from "@paperclover/console";
src/pages/404.mdx created+6
......@@ -0,0 +1,6 @@
1export const meta = { title: 'oh no,,,' };
2
3# oh dear
4
5sound the alarms
6
src/pages/index.marko-1
......@@ -16,7 +16,6 @@ export const meta: Meta = {
1616 type: "website",
1717 url: "https://paperclover.net",
1818 },
19 generator: "clover",
2019 alternates: {
2120 canonical: "https://paperclover.net",
2221 types: {
src/q+a/backend.ts created+3
......@@ -0,0 +1,3 @@
1export const app = new Hono();
2
3import { Hono } from "#hono";
src/q+a/components/Question.marko+1-1
......@@ -30,7 +30,7 @@ static const transitionDate = 1735639200000;
3030</>
3131
3232// this singleton script will make all the '<time>' tags clickable.
33client import "./clickable-links.ts";
33client import "./clickable-links.client.ts";
3434
3535import type { Question } from "@/q+a/models/Question.ts";
3636import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";
src/sections.ts deleted-19
......@@ -1,19 +0,0 @@
1// This file defines the different "Sections" of the website. The sections act
2// as separate codebases, but hosted as one. This allows me to have
3// sub-projects like the file viewer in 'file', or the question answer system
4// in 'q+a'. Each section can define configuration, pages, backend routes, and
5// contain other files.
6interface Section {
7 root: string;
8}
9
10const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
11
12export const siteSections: Section[] = [
13 { root: join("./") },
14 { root: join("q+a/") },
15 { root: join("file-viewer/") },
16 { root: join("friends/") },
17];
18
19import * as path from "node:path";
src/site.ts created+19
......@@ -0,0 +1,19 @@
1// This file defines the different "Sections" of the website. The sections act
2// as separate codebases, but hosted as one. This allows me to have
3// sub-projects like the file viewer in 'file', or the question answer system
4// in 'q+a'. Each section can define configuration, pages, backend routes, and
5// contain other files.
6interface Section {
7 root: string;
8}
9
10const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
11
12export const siteSections: Section[] = [
13 { root: join("./") },
14 { root: join("q+a/") },
15 { root: join("file-viewer/") },
16 { root: join("friends/") },
17];
18
19import * as path from "node:path";