authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-08-11 22:43:27-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
log98be3e2bd40a40c6595de522d087092df63cdbb7
treeac5bf4b195a15f8e87222dad3ccceeff0592e8b4
parent8d0985c06cb96067dc084cb17f5c3bfe8c43f56d
signature Commit is signed but in an unrecognized format.

feat: dynamic page regeneration (#24)

the asset system is reworked to support "dynamic" entries, where each entry is a separate file on disk containing the latest generation's headers+raw+gzip+zstd. when calling view.regenerate, it will look for pages that had "export const regenerate" during generation, and render those pages using the view system, but then store the results as assets instead of sending as a response. pages configured as regenerable are also bundled as views, using the non-aliasing key "page:${page.id}". this cannot alias because file paths may not contain a colon.

42 files changed, 1415 insertions(+), 912 deletions(-)

flake.nix+23-23
...@@ -3,27 +3,27 @@...@@ -3,27 +3,27 @@
3 nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";3 nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
4 utils.url = "github:numtide/flake-utils";4 utils.url = "github:numtide/flake-utils";
5 };5 };
6 outputs =6 outputs = inputs: inputs.utils.lib.eachDefaultSystem (system:
7 { nixpkgs, utils, ... }:7 with inputs.nixpkgs.legacyPackages.${system}; {
8 utils.lib.eachDefaultSystem (8 devShells.default = pkgs.mkShell {
9 system:9 buildInputs = [
10 let10 pkgs.nodejs_24 # runtime
11 pkgs = nixpkgs.legacyPackages.${system};11 pkgs.deno # formatter
12 in12 (pkgs.ffmpeg.override {
13 {13 withOpus = true;
14 devShells.default = pkgs.mkShell {14 withSvtav1 = true;
15 buildInputs = [15 withJxl = true;
16 pkgs.nodejs_24 # runtime16 withWebp = true;
17 pkgs.deno # formatter17 })
18 (pkgs.ffmpeg.override {18 pkgs.rsync
19 withOpus = true;19 ];
20 withSvtav1 = true;20 };
21 withJxl = true;21 devShells.min = pkgs.mkShell {
22 withWebp = true;22 buildInputs = [
23 })23 pkgs.nodejs_24 # runtime
24 pkgs.rsync24 pkgs.deno # formatter
25 ];25 pkgs.rsync
26 };26 ];
27 }27 };
28 );28 });
29}29}
framework/bundle.ts+189-139
...@@ -88,19 +88,19 @@ export async function bundleClientJavaScript(...@@ -88,19 +88,19 @@ export async function bundleClientJavaScript(
88 for (const file of outputFiles) {88 for (const file of outputFiles) {
89 const { text } = file;89 const { text } = file;
90 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");90 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
91 const { inputs } = UNWRAP(metafile.outputs["out!" + route]);91 const { entryPoint } = UNWRAP(metafile.outputs["out!" + route]);
92 const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x));
9392
94 // Register non-chunks as script entries.93 // Register non-chunks as script entries.
95 const chunk = route.startsWith("/js/c.");94 const chunk = route.startsWith("/js/c.");
96 if (!chunk) {95 if (!chunk) {
97 const key = hot.getScriptId(path.resolve(sources[sources.length - 1]));96 const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
97 console.log(route, key);
98 route = "/js/" + key.replace(/\.client\.tsx?/, ".js");98 route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
99 scripts[key] = text;99 scripts[key] = text;
100 }100 }
101 // Register chunks and public scripts as assets.101 // Register chunks and public scripts as assets.
102 if (chunk || publicScriptRoutes.includes(route)) {102 if (chunk || publicScriptRoutes.includes(route)) {
103 p.push(io.writeAsset(route, text));103 p.push(io.writeAsset({ pathname: route, buffer: text }));
104 }104 }
105 }105 }
106 await Promise.all(p);106 await Promise.all(p);
...@@ -111,7 +111,7 @@ export type ServerPlatform = "node" | "passthru";...@@ -111,7 +111,7 @@ export type ServerPlatform = "node" | "passthru";
111export interface ServerSideOptions {111export interface ServerSideOptions {
112 entries: string[];112 entries: string[];
113 viewItems: sg.FileItem[];113 viewItems: sg.FileItem[];
114 viewRefs: incr.Ref<PreparedView>[];114 viewRefs: incr.Ref<PageOrView>[];
115 styleMap: Map<string, incr.Ref<string>>;115 styleMap: Map<string, incr.Ref<string>>;
116 scriptMap: incr.Ref<Record<string, string>>;116 scriptMap: incr.Ref<Record<string, string>>;
117 platform: ServerPlatform;117 platform: ServerPlatform;
...@@ -124,144 +124,192 @@ export async function bundleServerJavaScript({...@@ -124,144 +124,192 @@ export async function bundleServerJavaScript({
124 entries,124 entries,
125 platform,125 platform,
126}: ServerSideOptions) {126}: ServerSideOptions) {
127 const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => {127 const regenKeys: Record<string, string[]> = {};
128 const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");128 const regenTtls: view.Ttl[] = [];
129 return {129 for (const ref of viewRefs) {
130 magicWord,130 const value = UNWRAP(ref.value);
131 file: [131 if (value.type === "page" && (value.regenerate?.tags?.length ?? 0) > 0) {
132 ...viewItems.map(132 for (const tag of value.regenerate!.tags!) {
133 (view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`,133 (regenKeys[tag] ??= []).push(`page:${value.id}`);
134 ),134 }
135 `const styles = ${magicWord}[-2]`,135 }
136 `export const scripts = ${magicWord}[-1]`,136 if (value.type === "page" && (value.regenerate?.seconds ?? 0) > 0) {
137 "export const views = {",137 regenTtls.push({
138 ...viewItems.map((view, i) =>138 key: `page:${value.id}` as view.Key,
139 [139 seconds: value.regenerate!.seconds!,
140 ` ${JSON.stringify(view.id)}: {`,140 });
141 ` component: view${i}.default,`,141 }
142 ` meta: view${i}.meta,`,142 }
143 ` layout: view${i}.layout?.default ?? null,`,
144 ` inlineCss: styles[${magicWord}[${i}]]`,
145 ` },`,
146 ].join("\n"),
147 ),
148 "}",
149 ].join("\n"),
150 };
151 }, viewItems);
152
153 const wBundles = entries.map(
154 (entry) =>
155 [
156 entry,
157 incr.work(async (io, entry) => {
158 const pkg = await io.readJson<{
159 dependencies: Record<string, string>;
160 }>("package.json");
161143
162 let magicWord = null as string | null;144 const wViewSource = incr.work(
163 // -- plugins --145 async (
164 const serverPlugins: esbuild.Plugin[] = [146 _,
165 virtualFiles({147 { viewItems, regenKeys, regenTtls }: {
166 // only add dependency when imported.148 viewItems: sg.FileItem[];
167 $views: async () => {149 regenKeys: Record<string, string[]>;
168 const view = await io.readWork(wViewSource);150 regenTtls: view.Ttl[];
169 ({ magicWord } = view);151 },
170 return view.file;152 ) => {
171 },153 const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
172 }),154 return {
173 projectRelativeResolution(),155 magicWord,
174 markoViaBuildCache(),156 file: [
175 {157 ...viewItems.map(
176 name: "replace client references",158 (view, i) =>
177 setup(b) {159 `import * as view${i} from ${JSON.stringify(view.file)}`,
178 b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({160 ),
179 contents: hot.resolveClientRefs(161 `const styles = ${magicWord}[-2]`,
180 await fs.readFile(file, "utf-8"),162 `export const scripts = ${magicWord}[-1]`,
181 file,163 "export const views = {",
182 ).code,164 ...viewItems.map((view, i) =>
183 loader: path.extname(file).slice(1) as esbuild.Loader,165 [
184 }));166 ` ${JSON.stringify(view.id)}: {`,
185 },167 ` component: view${i}.default,`,
186 },168 ` meta: view${i}.meta,`,
187 {169 ` layout: view${i}.layout?.default ?? null,`,
188 name: "mark css external",170 ` inlineCss: styles[${magicWord}[${i}]]`,
189 setup(b) {171 ` },`,
190 b.onResolve({ filter: /\.css$/ }, () => ({172 ].join("\n")
191 path: ".",173 ),
192 namespace: "dropped",174 "}",
193 }));175 `export const regenTags = ${JSON.stringify(regenKeys)};`,
194 b.onLoad({ filter: /./, namespace: "dropped" }, () => ({176 `export const regenTtls = ${JSON.stringify(regenTtls)};`,
195 contents: "",177 ].join("\n"),
196 }));178 };
197 },179 },
198 },180 { viewItems, regenKeys, regenTtls },
199 ];181 );
200182
201 const { metafile, outputFiles } = await esbuild.build({183 await incr.work(
202 bundle: true,184 async (io, { regenKeys, viewItems }) => {
203 chunkNames: "c.[hash]",185 io.writeFile(
204 entryNames: path.basename(entry, path.extname(entry)),186 "../ts/view.d.ts",
205 entryPoints: [187 [
206 path.join(188 "export interface RegisteredViews {",
207 import.meta.dirname,189 ...viewItems
208 "backend/entry-" + platform + ".ts",190 .filter((view) => !view.id.startsWith("page:"))
209 ),191 .map(
210 ],192 (view) =>
211 platform: "node",193 ` ${JSON.stringify(view.id)}: ` +
212 format: "esm",194 `typeof import(${
213 minify: false,195 JSON.stringify(path.relative(".clover/ts", toAbs(view.file)))
214 outdir: "out!",196 }),`,
215 plugins: serverPlugins,
216 splitting: true,
217 logLevel: "silent",
218 write: false,
219 metafile: true,
220 jsx: "automatic",
221 jsxImportSource: "#engine",
222 jsxDev: false,
223 define: {
224 MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
225 "globalThis.CLOVER_SERVER_ENTRY": JSON.stringify(entry),
226 },
227 external: Object.keys(pkg.dependencies).filter(
228 (x) => !x.startsWith("@paperclover"),
229 ),197 ),
230 });198 "}",
231 await trackEsbuild(io, metafile);199 "export type RegenKey = " +
200 (regenKeys.map((key) => JSON.stringify(key)).join(" | ") ||
201 "never"),
202 ].join("\n"),
203 );
204 },
205 { regenKeys: Object.keys(regenKeys), viewItems },
206 );
232207
233 let fileWithMagicWord: {208 const wBundles = entries.map((entry) =>
234 bytes: Buffer;209 incr.work(async (io, entry) => {
235 basename: string;210 const pkg = await io.readJson<{
236 magicWord: string;211 dependencies: Record<string, string>;
237 } | null = null;212 }>("package.json");
238 for (const output of outputFiles) {213
239 const basename = output.path.replace(/^.*?!(?:\/|\\)/, "");214 let magicWord = null as string | null;
240 const key = "out!/" + basename.replaceAll("\\", "/");215 // -- plugins --
241 // If this contains the generated "$views" file, then216 const serverPlugins: esbuild.Plugin[] = [
242 // mark this file as the one for replacement. Because217 virtualFiles({
243 // `splitting` is `true`, esbuild will not emit this218 // only add dependency when imported.
244 // file in more than one chunk.219 $views: async () => {
245 if (220 const view = await io.readWork(wViewSource);
246 magicWord &&221 ({ magicWord } = view);
247 metafile.outputs[key].inputs["framework/lib/view.ts"]222 return view.file;
248 ) {223 },
249 ASSERT(!fileWithMagicWord);224 }),
250 fileWithMagicWord = {225 projectRelativeResolution(),
251 basename,226 markoViaBuildCache(),
252 bytes: Buffer.from(output.contents),227 {
253 magicWord,228 name: "replace client references",
254 };229 setup(b) {
255 } else {230 b.onLoad({ filter: /\.tsx?$/ }, async ({ path: file }) => ({
256 io.writeFile(basename, Buffer.from(output.contents));231 contents: hot.resolveClientRefs(
257 }232 await fs.readFile(file, "utf-8"),
258 }233 file,
259 return fileWithMagicWord;234 ).code,
260 }, entry),235 loader: path.extname(file).slice(1) as esbuild.Loader,
261 ] as const,236 }));
237 },
238 },
239 {
240 name: "mark css external",
241 setup(b) {
242 b.onResolve({ filter: /\.css$/ }, () => ({
243 path: ".",
244 namespace: "dropped",
245 }));
246 b.onLoad({ filter: /./, namespace: "dropped" }, () => ({
247 contents: "",
248 }));
249 },
250 },
251 ];
252
253 const { metafile, outputFiles } = await esbuild.build({
254 bundle: true,
255 chunkNames: "c.[hash]",
256 entryNames: path.basename(entry, path.extname(entry)),
257 entryPoints: [
258 path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
259 ],
260 platform: "node",
261 format: "esm",
262 minify: false,
263 outdir: "out!",
264 plugins: serverPlugins,
265 splitting: true,
266 logLevel: "silent",
267 write: false,
268 metafile: true,
269 jsx: "automatic",
270 jsxImportSource: "#engine",
271 jsxDev: false,
272 define: {
273 MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
274 "globalThis.CLOVER_SERVER_ENTRY": JSON.stringify(entry),
275 },
276 external: Object.keys(pkg.dependencies).filter(
277 (x) => !x.startsWith("@paperclover"),
278 ),
279 });
280 await trackEsbuild(io, metafile);
281
282 let fileWithMagicWord: {
283 bytes: Buffer;
284 basename: string;
285 magicWord: string;
286 } | null = null;
287 for (const output of outputFiles) {
288 const basename = output.path.replace(/^.*?!(?:\/|\\)/, "");
289 const key = "out!/" + basename.replaceAll("\\", "/");
290 // If this contains the generated "$views" file, then
291 // mark this file as the one for replacement. Because
292 // `splitting` is `true`, esbuild will not emit this
293 // file in more than one chunk.
294 if (
295 magicWord &&
296 UNWRAP(metafile.outputs[key]).inputs["framework/lib/view.ts"]
297 ) {
298 ASSERT(!fileWithMagicWord);
299 fileWithMagicWord = {
300 basename,
301 bytes: Buffer.from(output.contents),
302 magicWord,
303 };
304 } else {
305 io.writeFile(basename, Buffer.from(output.contents));
306 }
307 }
308 return fileWithMagicWord;
309 }, entry)
262 );310 );
263311
264 const wProcessed = wBundles.map(async ([entry, wBundle]) => {312 const wProcessed = wBundles.map(async (wBundle) => {
265 if (!(await wBundle)) return;313 if (!(await wBundle)) return;
266 await incr.work(async (io) => {314 await incr.work(async (io) => {
267 // Only the reachable resources need to be read and inserted into the bundle.315 // Only the reachable resources need to be read and inserted into the bundle.
...@@ -278,7 +326,7 @@ export async function bundleServerJavaScript({...@@ -278,7 +326,7 @@ export async function bundleServerJavaScript({
278 const viewStyleKeys = views.map((view) => view.styleKey);326 const viewStyleKeys = views.map((view) => view.styleKey);
279 const viewCssBundles = await Promise.all(327 const viewCssBundles = await Promise.all(
280 viewStyleKeys.map((key) =>328 viewStyleKeys.map((key) =>
281 io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key)),329 io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key))
282 ),330 ),
283 );331 );
284 const styleList = Array.from(new Set(viewCssBundles));332 const styleList = Array.from(new Set(viewCssBundles));
...@@ -297,7 +345,7 @@ export async function bundleServerJavaScript({...@@ -297,7 +345,7 @@ export async function bundleServerJavaScript({
297 return JSON.stringify(Object.fromEntries(neededScripts));345 return JSON.stringify(Object.fromEntries(neededScripts));
298 }346 }
299 // Reference an index into `styleList`347 // Reference an index into `styleList`
300 return `${styleList.indexOf(viewCssBundles[i])}`;348 return `${styleList.indexOf(UNWRAP(viewCssBundles[i]))}`;
301 });349 });
302350
303 io.writeFile(basename, text);351 io.writeFile(basename, text);
...@@ -322,3 +370,5 @@ import * as fs from "#sitegen/fs";...@@ -322,3 +370,5 @@ import * as fs from "#sitegen/fs";
322import * as mime from "#sitegen/mime";370import * as mime from "#sitegen/mime";
323import * as incr from "./incremental.ts";371import * as incr from "./incremental.ts";
324import * as sg from "#sitegen";372import * as sg from "#sitegen";
373import type { PageOrView } from "./generate.ts";
374import type * as view from "#sitegen/view";
framework/css.ts+9-6
...@@ -57,12 +57,14 @@ export function styleKey(...@@ -57,12 +57,14 @@ export function styleKey(
57export async function bundleCssFiles(57export async function bundleCssFiles(
58 io: Io,58 io: Io,
59 { cssImports, theme, dev }: {59 { cssImports, theme, dev }: {
60 cssImports: string[],60 cssImports: string[];
61 theme: Theme,61 theme: Theme;
62 dev: boolean,62 dev: boolean;
63 }63 },
64) {64) {
65 cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file)));65 cssImports = await Promise.all(
66 cssImports.map((file) => io.trackFile("src/" + file)),
67 );
66 const plugin = {68 const plugin = {
67 name: "clover css",69 name: "clover css",
68 setup(b) {70 setup(b) {
...@@ -111,4 +113,5 @@ import * as esbuild from "esbuild";...@@ -111,4 +113,5 @@ import * as esbuild from "esbuild";
111import * as fs from "#sitegen/fs";113import * as fs from "#sitegen/fs";
112import * as hot from "./hot.ts";114import * as hot from "./hot.ts";
113import * as path from "node:path";115import * as path from "node:path";
114import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts";116import { virtualFiles } from "./esbuild-support.ts";
117import type { Io } from "./incremental.ts";
framework/debug.safe.ts+1-1
...@@ -9,4 +9,4 @@ globalThis.UNWRAP = (t, ...args) => {...@@ -9,4 +9,4 @@ globalThis.UNWRAP = (t, ...args) => {
9globalThis.ASSERT = assert.ok;9globalThis.ASSERT = assert.ok;
1010
11import * as util from "node:util";11import * as util from "node:util";
12import * as assert from 'node:assert'12import * as assert from "node:assert";
framework/engine/jsx-runtime.ts+1-1
...@@ -51,4 +51,4 @@ declare global {...@@ -51,4 +51,4 @@ declare global {
51 }51 }
52}52}
5353
54import * as render from "./render.ts";54import * as render from "#engine/render";
framework/engine/marko-runtime.ts+1
...@@ -18,6 +18,7 @@ export const createTemplate = (...@@ -18,6 +18,7 @@ export const createTemplate = (
18 const r = render.current;18 const r = render.current;
19 // Support using Marko outside of Clover SSR19 // Support using Marko outside of Clover SSR
20 if (!r) return renderer(props, n);20 if (!r) return renderer(props, n);
21 render.setCurrent(null);
21 const markoResult = renderFn.call(renderer, {22 const markoResult = renderFn.call(renderer, {
22 ...props,23 ...props,
23 $global: { clover: r, cloverAsyncMarker },24 $global: { clover: r, cloverAsyncMarker },
framework/engine/render.test.tsx+1-1
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1import { test } from "node:test";1import { test } from "node:test";
2import * as render from "./render.ts";2import * as render from "#engine/render";
33
4test("sanity", (t) => t.assert.equal(render.sync("gm <3").text, "gm &lt;3"));4test("sanity", (t) => t.assert.equal(render.sync("gm <3").text, "gm &lt;3"));
5test("simple tree", (t) =>5test("simple tree", (t) =>
framework/engine/render.ts+30-8
...@@ -118,8 +118,11 @@ export function resolveNode(r: State, node: unknown): ResolvedNode {...@@ -118,8 +118,11 @@ export function resolveNode(r: State, node: unknown): ResolvedNode {
118 if (!node && node !== 0) return ""; // falsy, non numeric118 if (!node && node !== 0) return ""; // falsy, non numeric
119 if (typeof node !== "object") {119 if (typeof node !== "object") {
120 if (node === true) return ""; // booleans are ignored120 if (node === true) return ""; // booleans are ignored
121 if (typeof node === "string") return escapeHtml(node);121 if (typeof node === "string") return escapeHtmlContent(node);
122 if (typeof node === "number") return String(node); // no escaping ever122 if (typeof node === "number") return String(node); // no escaping ever
123 if (typeof node === "symbol" && node.toString() === kElement.toString()) {
124 throw new Error(`There are two instances of Clover SSR loaded!`);
125 }
123 throw new Error(`Cannot render ${inspect(node)} to HTML`);126 throw new Error(`Cannot render ${inspect(node)} to HTML`);
124 }127 }
125 if (node instanceof Promise) {128 if (node instanceof Promise) {
...@@ -217,12 +220,14 @@ function stringifyElement(element: ResolvedElement) {...@@ -217,12 +220,14 @@ function stringifyElement(element: ResolvedElement) {
217 let attr;220 let attr;
218 switch (prop) {221 switch (prop) {
219 default:222 default:
220 attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`;223 attr = `${prop}=${quoteIfNeeded(escapeAttribute(String(value)))}`;
221 break;224 break;
222 case "className":225 case "className":
223 // Legacy React Compat226 // Legacy React Compat
224 case "class":227 case "class":
225 attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`;228 attr = `class=${
229 quoteIfNeeded(escapeAttribute(clsx(value as ClsxInput)))
230 }`;
226 break;231 break;
227 case "htmlFor":232 case "htmlFor":
228 throw new Error("Do not use the `htmlFor` attribute. Use `for`");233 throw new Error("Do not use the `htmlFor` attribute. Use `for`");
...@@ -233,7 +238,7 @@ function stringifyElement(element: ResolvedElement) {...@@ -233,7 +238,7 @@ function stringifyElement(element: ResolvedElement) {
233 case "key":238 case "key":
234 continue;239 continue;
235 }240 }
236 if (needSpace) (out += " "), (needSpace = !attr.endsWith('"'));241 if (needSpace) ((out += " "), (needSpace = !attr.endsWith('"')));
237 out += attr;242 out += attr;
238 }243 }
239 out += ">";244 out += ">";
...@@ -254,14 +259,16 @@ export function stringifyStyleAttribute(style: Record<string, string>) {...@@ -254,14 +259,16 @@ export function stringifyStyleAttribute(style: Record<string, string>) {
254 let out = ``;259 let out = ``;
255 for (const styleName in style) {260 for (const styleName in style) {
256 if (out) out += ";";261 if (out) out += ";";
257 out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${escapeHtml(262 out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${
258 String(style[styleName]),263 escapeAttribute(
259 )}`;264 String(style[styleName]),
265 )
266 }`;
260 }267 }
261 return "style=" + quoteIfNeeded(out);268 return "style=" + quoteIfNeeded(out);
262}269}
263export function quoteIfNeeded(text: string) {270export function quoteIfNeeded(text: string) {
264 if (text.includes(" ")) return '"' + text + '"';271 if (text.match(/["/>]/)) return '"' + text + '"';
265 return text;272 return text;
266}273}
267274
...@@ -303,6 +310,21 @@ export function clsx(mix: ClsxInput) {...@@ -303,6 +310,21 @@ export function clsx(mix: ClsxInput) {
303 return str;310 return str;
304}311}
305312
313export const escapeHtmlContent = (unsafeText: string) =>
314 String(unsafeText)
315 .replace(/&/g, "&amp;")
316 .replace(/</g, "&lt;")
317 .replace(/>/g, "&gt;")
318 .replace(/"/g, "&quot;");
319// TODO: combine into one function which decides if an attribute needs quotes
320// and escapes it correctly depending on the context.
321const escapeAttribute = (unsafeText: string) =>
322 String(unsafeText)
323 .replace(/&/g, "&amp;")
324 .replace(/</g, "&lt;")
325 .replace(/>/g, "&gt;")
326 .replace(/"/g, "&quot;");
327/** @deprecated */
306export const escapeHtml = (unsafeText: string) =>328export const escapeHtml = (unsafeText: string) =>
307 String(unsafeText)329 String(unsafeText)
308 .replace(/&/g, "&amp;")330 .replace(/&/g, "&amp;")
framework/engine/suspense.ts+2-2
...@@ -42,7 +42,7 @@ export function Suspense({ children, fallback }: SuspenseProps): render.Node {...@@ -42,7 +42,7 @@ export function Suspense({ children, fallback }: SuspenseProps): render.Node {
42 r.asyncDone = () => {42 r.asyncDone = () => {
43 const rejections = r.rejections;43 const rejections = r.rejections;
44 if (rejections && rejections.length > 0) throw new Error("TODO");44 if (rejections && rejections.length > 0) throw new Error("TODO");
45 state.pushChunk?.(name, (ip[0] = resolved));45 state.pushChunk?.(name, ip[0] = resolved);
46 };46 };
47 return render.raw(ip);47 return render.raw(ip);
48}48}
...@@ -99,4 +99,4 @@ export async function* renderStreaming<...@@ -99,4 +99,4 @@ export async function* renderStreaming<
99 return addonOutput as unknown as T;99 return addonOutput as unknown as T;
100}100}
101101
102import * as render from "./render.ts";102import * as render from "#engine/render";
framework/esbuild-support.ts+10-5
...@@ -1,7 +1,12 @@...@@ -1,7 +1,12 @@
1type Awaitable<T> = T | Promise<T>;1type Awaitable<T> = T | Promise<T>;
22
3export function virtualFiles(3export function virtualFiles(
4 map: Record<string, string | esbuild.OnLoadResult | (() => Awaitable<string | esbuild.OnLoadResult>)>,4 map: Record<
5 string,
6 | string
7 | esbuild.OnLoadResult
8 | (() => Awaitable<string | esbuild.OnLoadResult>)
9 >,
5) {10) {
6 return {11 return {
7 name: "clover vfs",12 name: "clover vfs",
...@@ -22,7 +27,7 @@ export function virtualFiles(...@@ -22,7 +27,7 @@ export function virtualFiles(
22 { filter: /./, namespace: "vfs" },27 { filter: /./, namespace: "vfs" },
23 async ({ path }) => {28 async ({ path }) => {
24 let entry = map[path];29 let entry = map[path];
25 if (typeof entry === 'function') entry = await entry();30 if (typeof entry === "function") entry = await entry();
26 return ({31 return ({
27 resolveDir: ".",32 resolveDir: ".",
28 loader: "ts",33 loader: "ts",
...@@ -88,7 +93,6 @@ export function markoViaBuildCache(): esbuild.Plugin {...@@ -88,7 +93,6 @@ export function markoViaBuildCache(): esbuild.Plugin {
88 if (!fs.existsSync(file)) {93 if (!fs.existsSync(file)) {
89 console.warn(`File does not exist: ${file}`);94 console.warn(`File does not exist: ${file}`);
90 }95 }
91 console.log(markoCache.keys());
92 throw new Error("Marko file not in cache: " + file);96 throw new Error("Marko file not in cache: " + file);
93 }97 }
94 return ({98 return ({
...@@ -106,7 +110,7 @@ export function isIgnoredSource(source: string) {...@@ -106,7 +110,7 @@ export function isIgnoredSource(source: string) {
106 return source.includes("<define:") ||110 return source.includes("<define:") ||
107 source.startsWith("vfs:") ||111 source.startsWith("vfs:") ||
108 source.startsWith("dropped:") ||112 source.startsWith("dropped:") ||
109 source.includes("node_modules")113 source.includes("node_modules");
110}114}
111115
112import * as esbuild from "esbuild";116import * as esbuild from "esbuild";
...@@ -114,4 +118,5 @@ import * as string from "#sitegen/string";...@@ -114,4 +118,5 @@ import * as string from "#sitegen/string";
114import * as path from "node:path";118import * as path from "node:path";
115import * as fs from "#sitegen/fs";119import * as fs from "#sitegen/fs";
116import * as incr from "./incremental.ts";120import * as incr from "./incremental.ts";
117import * as hot from "./hot.ts";import { markoCache } from "./marko.ts";121import * as hot from "./hot.ts";
122import { markoCache } from "./marko.ts";
framework/generate.ts+63-36
...@@ -7,15 +7,16 @@ const { toRel, toAbs } = incr;...@@ -7,15 +7,16 @@ const { toRel, toAbs } = incr;
7const globalCssPath = toAbs("src/global.css");7const globalCssPath = toAbs("src/global.css");
88
9export async function main() {9export async function main() {
10 await incr.restore();10 if (!process.argv.includes("-f")) await incr.restore();
11 await incr.compile(generate);11 await incr.compile(generate);
12}12}
1313
14export async function generate() {14export async function generate() {
15 // -- read config and discover files --15 // -- read config and discover files --
16 const siteConfig = await incr.work(readManifest);16 const siteConfig = await incr.work(readManifest);
17 const { staticFiles, scripts, views, pages } =17 const { staticFiles, scripts, views, pages } = await discoverAllFiles(
18 await discoverAllFiles(siteConfig);18 siteConfig,
19 );
1920
20 // TODO: make sure that `static` and `pages` does not overlap21 // TODO: make sure that `static` and `pages` does not overlap
2122
...@@ -28,13 +29,21 @@ export async function generate() {...@@ -28,13 +29,21 @@ export async function generate() {
28 staticFiles.map((item) =>29 staticFiles.map((item) =>
29 incr.work(30 incr.work(
30 async (io, { id, file }) =>31 async (io, { id, file }) =>
31 void (await io.writeAsset(id, await io.readFile(file))),32 void (await io.writeAsset({
33 pathname: id,
34 buffer: await io.readFile(file),
35 })),
32 item,36 item,
33 ),37 )
34 ),38 ),
35 );39 );
36 const routes = await Promise.all([...builtViews, ...builtPages]);40 const routes = await Promise.all([...builtViews, ...builtPages]);
3741
42 const viewsAndDynPages: incr.Ref<PageOrView>[] = [
43 ...builtViews,
44 ...builtPages.filter((page) => UNWRAP(page.value).regenerate),
45 ];
46
38 // -- page resources --47 // -- page resources --
39 const scriptMap = incr.work(bundle.bundleClientJavaScript, {48 const scriptMap = incr.work(bundle.bundleClientJavaScript, {
40 clientRefs: routes.flatMap((x) => x.clientRefs),49 clientRefs: routes.flatMap((x) => x.clientRefs),
...@@ -49,13 +58,16 @@ export async function generate() {...@@ -49,13 +58,16 @@ export async function generate() {
49 platform: "node",58 platform: "node",
50 styleMap,59 styleMap,
51 scriptMap,60 scriptMap,
52 viewItems: views,61 viewItems: viewsAndDynPages.map((ref) => {
53 viewRefs: builtViews,62 const { id, file, type } = UNWRAP(ref.value);
63 return { id: type === "page" ? `page:${id}` : id, file };
64 }),
65 viewRefs: viewsAndDynPages,
54 });66 });
5567
56 // -- assemble page assets --68 // -- assemble page assets --
57 const pAssemblePages = builtPages.map((page) =>69 const pAssemblePages = builtPages.map((page) =>
58 assembleAndWritePage(page, styleMap, scriptMap),70 assembleAndWritePage(page, styleMap, scriptMap)
59 );71 );
6072
61 await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);73 await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
...@@ -77,7 +89,7 @@ export async function discoverAllFiles(...@@ -77,7 +89,7 @@ export async function discoverAllFiles(
77 return (89 return (
78 await Promise.all(90 await Promise.all(
79 siteConfig.siteSections.map(({ root: sectionRoot }) =>91 siteConfig.siteSections.map(({ root: sectionRoot }) =>
80 incr.work(scanSiteSection, toAbs(sectionRoot)),92 incr.work(scanSiteSection, toAbs(sectionRoot))
81 ),93 ),
82 )94 )
83 ).reduce((acc, next) => ({95 ).reduce((acc, next) => ({
...@@ -105,10 +117,9 @@ export async function scanSiteSection(io: Io, sectionRoot: string) {...@@ -105,10 +117,9 @@ export async function scanSiteSection(io: Io, sectionRoot: string) {
105 let scripts: FileItem[] = [];117 let scripts: FileItem[] = [];
106118
107 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);119 const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
108 const rootPrefix =120 const rootPrefix = hot.projectSrc === sectionRoot
109 hot.projectSrc === sectionRoot121 ? ""
110 ? ""122 : path.relative(hot.projectSrc, sectionRoot) + "/";
111 : path.relative(hot.projectSrc, sectionRoot) + "/";
112 const kinds = [123 const kinds = [
113 {124 {
114 dir: sectionPath("pages"),125 dir: sectionPath("pages"),
...@@ -181,7 +192,8 @@ export async function preparePage(io: Io, item: sg.FileItem) {...@@ -181,7 +192,8 @@ export async function preparePage(io: Io, item: sg.FileItem) {
181 meta: metadata,192 meta: metadata,
182 theme: pageTheme,193 theme: pageTheme,
183 layout,194 layout,
184 } = await io.import<any>(item.file);195 regenerate,
196 } = await io.import<sg.PageExports>(item.file);
185 if (!Page) throw new Error("Page is missing a 'default' export.");197 if (!Page) throw new Error("Page is missing a 'default' export.");
186 if (!metadata) throw new Error("Page is missing 'meta' export with a title.");198 if (!metadata) throw new Error("Page is missing 'meta' export with a title.");
187199
...@@ -219,16 +231,20 @@ export async function preparePage(io: Io, item: sg.FileItem) {...@@ -219,16 +231,20 @@ export async function preparePage(io: Io, item: sg.FileItem) {
219 "Page is missing 'meta.title'. " + "All pages need a title tag.",231 "Page is missing 'meta.title'. " + "All pages need a title tag.",
220 );232 );
221 }233 }
234
222 const styleKey = css.styleKey(cssImports, theme);235 const styleKey = css.styleKey(cssImports, theme);
223 return {236 return {
237 type: "page",
224 id: item.id,238 id: item.id,
239 file: item.file,
240 regenerate,
225 html: text,241 html: text,
226 meta: renderedMeta,242 meta: renderedMeta,
227 cssImports,243 cssImports,
228 theme: theme ?? null,244 theme: theme ?? null,
229 styleKey,245 styleKey,
230 clientRefs: Array.from(addon[sg.userData.key].scripts),246 clientRefs: Array.from(addon[sg.userData.key].scripts),
231 };247 } as const;
232}248}
233249
234export async function prepareView(io: Io, item: sg.FileItem) {250export async function prepareView(io: Io, item: sg.FileItem) {
...@@ -243,13 +259,15 @@ export async function prepareView(io: Io, item: sg.FileItem) {...@@ -243,13 +259,15 @@ export async function prepareView(io: Io, item: sg.FileItem) {
243 );259 );
244 const styleKey = css.styleKey(cssImports, theme);260 const styleKey = css.styleKey(cssImports, theme);
245 return {261 return {
246 file: path.relative(hot.projectRoot, item.file),262 type: "view",
263 id: item.id,
264 file: item.file,
247 cssImports,265 cssImports,
248 theme,266 theme,
249 clientRefs: hot.getClientScriptRefs(item.file),267 clientRefs: hot.getClientScriptRefs(item.file),
250 hasLayout: !!module.layout?.default,268 hasLayout: !!module.layout?.default,
251 styleKey,269 styleKey,
252 };270 } as const;
253}271}
254export type PreparedView = Awaited<ReturnType<typeof prepareView>>;272export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
255273
...@@ -274,28 +292,38 @@ export async function assembleAndWritePage(...@@ -274,28 +292,38 @@ export async function assembleAndWritePage(
274 scriptWork: incr.Ref<Record<string, string>>,292 scriptWork: incr.Ref<Record<string, string>>,
275) {293) {
276 const page = await pageWork;294 const page = await pageWork;
277 return incr.work(async (io, { id, html, meta, styleKey, clientRefs }) => {295 return incr.work(
278 const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));296 async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
297 const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
279298
280 const scriptIds = clientRefs.map(hot.getScriptId);299 const scriptIds = clientRefs.map(hot.getScriptId);
281 const scriptMap = await io.readWork(scriptWork);300 const scriptMap = await io.readWork(scriptWork);
282 const scripts = scriptIds301 const scripts = scriptIds
283 .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))302 .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
284 .map((x) => `{${x}}`)303 .map((x) => `{${x}}`)
285 .join("\n");304 .join("\n");
286305
287 const doc = wrapDocument({306 const buffer = sg.wrapDocument({
288 body: html,307 body: html,
289 head: meta,308 head: meta,
290 inlineCss,309 inlineCss,
291 scripts,310 scripts,
292 });311 });
293 await io.writeAsset(id, doc, {312 await io.writeAsset({
294 "Content-Type": "text/html",313 pathname: id,
295 });314 buffer,
296 }, page);315 headers: {
316 "Content-Type": "text/html",
317 },
318 regenerative: !!regenerate,
319 });
320 },
321 page,
322 );
297}323}
298324
325export type PageOrView = PreparedPage | PreparedView;
326
299import * as sg from "#sitegen";327import * as sg from "#sitegen";
300import * as incr from "./incremental.ts";328import * as incr from "./incremental.ts";
301import { Io } from "./incremental.ts";329import { Io } from "./incremental.ts";
...@@ -307,4 +335,3 @@ import * as fs from "#sitegen/fs";...@@ -307,4 +335,3 @@ import * as fs from "#sitegen/fs";
307import type { FileItem } from "#sitegen";335import type { FileItem } from "#sitegen";
308import * as path from "node:path";336import * as path from "node:path";
309import * as meta from "#sitegen/meta";337import * as meta from "#sitegen/meta";
310import { wrapDocument } from "./lib/view.ts";
framework/hot.ts+6-8
...@@ -78,10 +78,9 @@ Module.prototype._compile = function (...@@ -78,10 +78,9 @@ Module.prototype._compile = function (
78 }78 }
79 }79 }
80 fileStats.set(filename, {80 fileStats.set(filename, {
81 cssImportsRecursive:81 cssImportsRecursive: cssImportsMaybe.length > 0
82 cssImportsMaybe.length > 082 ? Array.from(new Set(cssImportsMaybe))
83 ? Array.from(new Set(cssImportsMaybe))83 : null,
84 : null,
85 imports,84 imports,
86 lastModified: Math.floor(stat.mtimeMs),85 lastModified: Math.floor(stat.mtimeMs),
87 });86 });
...@@ -137,14 +136,13 @@ export function loadEsbuildCode(...@@ -137,14 +136,13 @@ export function loadEsbuildCode(
137 src = code;136 src = code;
138 }137 }
139 if (src.includes("import.meta")) {138 if (src.includes("import.meta")) {
140 src =139 src = `
141 `
142 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};140 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
143 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};141 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
144 import.meta.filename = ${JSON.stringify(filepath)};142 import.meta.filename = ${JSON.stringify(filepath)};
145 `143 `
146 .trim()144 .trim()
147 .replace(/[\n\s]/g, "") + src;145 .replace(/[\n\s]/g, "") + src;
148 }146 }
149 src = esbuild.transformSync(src, {147 src = esbuild.transformSync(src, {
150 loader,148 loader,
framework/incremental.test.ts+11-11
...@@ -7,38 +7,38 @@ test("trivial case", async () => {...@@ -7,38 +7,38 @@ test("trivial case", async () => {
7 async function compilation() {7 async function compilation() {
8 const first = incr.work({8 const first = incr.work({
9 label: "first compute",9 label: "first compute",
10 async run (io) {10 async run(io) {
11 await setTimeout(1000);11 await setTimeout(1000);
12 const contents = await io.readFile(file1.path);12 const contents = await io.readFile(file1.path);
13 return [contents, Math.random()] as const;13 return [contents, Math.random()] as const;
14 }14 },
15 });15 });
16 const second = incr.work({16 const second = incr.work({
17 label: "second compute",17 label: "second compute",
18 wait: first,18 wait: first,
19 async run (io) {19 async run(io) {
20 await setTimeout(1000);20 await setTimeout(1000);
21 return io.readWork(first)[0].toUpperCase();21 return io.readWork(first)[0].toUpperCase();
22 }22 },
23 });23 });
24 const third = incr.work({24 const third = incr.work({
25 label: "third compute",25 label: "third compute",
26 wait: first,26 wait: first,
27 async run (io) {27 async run(io) {
28 await setTimeout(1000);28 await setTimeout(1000);
29 return io.readWork(first)[1] * 1000;29 return io.readWork(first)[1] * 1000;
30 }30 },
31 });31 });
32 return incr.work({32 return incr.work({
33 label: "last compute",33 label: "last compute",
34 wait: [second, third],34 wait: [second, third],
35 async run (io) {35 async run(io) {
36 await setTimeout(1000);36 await setTimeout(1000);
37 return {37 return {
38 second: io.readWork(second),38 second: io.readWork(second),
39 third: io.readWork(third),39 third: io.readWork(third),
40 }40 };
41 }41 },
42 });42 });
43 }43 }
44 const { value: first } = await incr.compile(compilation);44 const { value: first } = await incr.compile(compilation);
...@@ -52,5 +52,5 @@ test("trivial case", async () => {...@@ -52,5 +52,5 @@ test("trivial case", async () => {
5252
53import * as incr from "./incremental2.ts";53import * as incr from "./incremental2.ts";
54import { beforeEach, test } from "node:test";54import { beforeEach, test } from "node:test";
55import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises";55import { tmpFile } from "#sitegen/testing";
5656import { setTimeout } from "node:timers/promises";
framework/incremental.ts+186-142
...@@ -2,8 +2,7 @@...@@ -2,8 +2,7 @@
2// See `work()`, `compile()`, and `invalidate()` for details.2// See `work()`, `compile()`, and `invalidate()` for details.
3//3//
4// All state is serializable to allow recovering state across sessions.4// All state is serializable to allow recovering state across sessions.
5// This framework special-cases the asset map, but is otherwise5// This library special-cases the asset map, but is otherwise agnostic.
6// agnostic of what it is a compiler for.
7let running = false;6let running = false;
8let jobs = 0;7let jobs = 0;
9let newKeys = 0;8let newKeys = 0;
...@@ -15,27 +14,25 @@ let writes = new Map<string, FileWrite>();...@@ -15,27 +14,25 @@ let writes = new Map<string, FileWrite>();
15let assets = new Map<string, Asset>(); // keyed by hash14let assets = new Map<string, Asset>(); // keyed by hash
1615
17export interface Ref<T> {16export interface Ref<T> {
18 /** This method is compatible with `await` syntax */
19 then(
20 onFulfilled: (value: T) => void,
21 onRejected: (error: unknown) => void,
22 ): void;
23 key: string;17 key: string;
18 /** This method is compatible with `await` syntax */
19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20 get value(): T | null;
24}21}
25type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;22type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
2623
27/**24/**
28 * Declare and a unit of work. Return value is memoized and25 * Declare and a unit of work. Return value is memoized and only rebuilt when
29 * only rebuilt when inputs (declared via `Io`) change. Outputs26 * inputs change. Inputs are tracked via the `io` interface, as well as a hash
30 * are written at the end of a compilation (see `compile`).27 * of the `input` param and caller source code. Outputs are written at the end
28 * of a compilation (see `compile`).
31 *29 *
32 * If the returned `Ref` is not awaited or read30 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
33 * via io.readWork, the job is never started.
34 */31 */
35export function work<O>(job: Job<void, O>): Ref<O>;32export function work<O>(job: Job<void, O>): Ref<O>;
36export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;33export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;
37export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {34export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
38 const source = JSON.stringify(util.getCallSites(2)[1]);35 const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1]));
39 const keySource = [source, util.inspect(input)].join(":");36 const keySource = [source, util.inspect(input)].join(":");
40 const key = crypto.createHash("sha1").update(keySource).digest("base64url");37 const key = crypto.createHash("sha1").update(keySource).digest("base64url");
41 ASSERT(running);38 ASSERT(running);
...@@ -48,7 +45,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {...@@ -48,7 +45,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
4845
49 const prev = works.get(key) as Work<O> | null;46 const prev = works.get(key) as Work<O> | null;
50 if (prev) {47 if (prev) {
51 return { key, then: (done) => done(prev.value) };48 return { key, then: (done) => done(prev.value), value: prev.value };
52 }49 }
5350
54 async function perform() {51 async function perform() {
...@@ -59,13 +56,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {...@@ -59,13 +56,7 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
59 const value = await job(io, input);56 const value = await job(io, input);
60 validateSerializable(value, "");57 validateSerializable(value, "");
61 const { reads, writes } = io;58 const { reads, writes } = io;
62 works.set(key, {59 works.set(key, { value, affects: [], reads, writes });
63 value,
64 affects: [],
65 reads,
66 writes,
67 debug: source,
68 });
69 for (const add of reads.files) {60 for (const add of reads.files) {
70 const { affects } = UNWRAP(files.get(add));61 const { affects } = UNWRAP(files.get(add));
71 ASSERT(!affects.includes(key));62 ASSERT(!affects.includes(key));
...@@ -85,7 +76,12 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {...@@ -85,7 +76,12 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
85 let cached: Promise<O>;76 let cached: Promise<O>;
86 return {77 return {
87 key,78 key,
88 then: (fufill, reject) => void (cached ??= perform()).then(fufill, reject),79 then(fufill, reject) {
80 (cached ??= perform()).then(fufill, reject);
81 },
82 get value() {
83 return (works.get(this.key)?.value as O) ?? null;
84 },
89 };85 };
90}86}
9187
...@@ -97,11 +93,7 @@ export async function compile<T>(compiler: () => Promise<T>) {...@@ -97,11 +93,7 @@ export async function compile<T>(compiler: () => Promise<T>) {
97 const start = performance.now();93 const start = performance.now();
98 const timerSpinner = new Spinner({94 const timerSpinner = new Spinner({
99 text: () =>95 text: () =>
100 `sitegen! [${96 `sitegen! [${((performance.now() - start) / 1000).toFixed(1)}s]`,
101 ((performance.now() - start) / 1000).toFixed(
102 1,
103 )
104 }s]`,
105 fps: 10,97 fps: 10,
106 });98 });
107 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };99 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
...@@ -114,8 +106,10 @@ export async function compile<T>(compiler: () => Promise<T>) {...@@ -114,8 +106,10 @@ export async function compile<T>(compiler: () => Promise<T>) {
114 return {106 return {
115 value,107 value,
116 watchFiles: new Set(files.keys()),108 watchFiles: new Set(files.keys()),
117 newOutputs: Array.from(seenWrites).filter(x => x.startsWith('f:')).map(x => x.slice(2)),109 newOutputs: Array.from(seenWrites)
118 newAssets: !Array.from(seenWrites).some(x => x.startsWith('a:')),110 .filter((x) => x.startsWith("f:"))
111 .map((x) => x.slice(2)),
112 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
119 };113 };
120 } finally {114 } finally {
121 running = false;115 running = false;
...@@ -127,9 +121,10 @@ export async function compile<T>(compiler: () => Promise<T>) {...@@ -127,9 +121,10 @@ export async function compile<T>(compiler: () => Promise<T>) {
127121
128export async function flush(start: number) {122export async function flush(start: number) {
129 // Trim123 // Trim
130 const detachedFiles = new Set<string>;124 const detachedFiles = new Set<string>();
131 const referencedAssets = new Set<string>;125 const referencedAssets = new Set<string>();
132 for (const [k, { writes: { assets } }] of works) {126 for (const [k, v] of works) {
127 const assets = v.writes.assets;
133 if (seenWorks.has(k)) {128 if (seenWorks.has(k)) {
134 for (const asset of assets.values()) referencedAssets.add(asset.hash);129 for (const asset of assets.values()) referencedAssets.add(asset.hash);
135 continue;130 continue;
...@@ -141,12 +136,9 @@ export async function flush(start: number) {...@@ -141,12 +136,9 @@ export async function flush(start: number) {
141 files.delete(k);136 files.delete(k);
142 detachedFiles.add(k);137 detachedFiles.add(k);
143 }138 }
144 for (const k of assets.keys()) {139 for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k);
145 if (!referencedAssets.has(k))
146 assets.delete(k);
147 }
148140
149 const p = [];141 const p: Promise<void>[] = [];
150 // File writes142 // File writes
151 let dist = 0;143 let dist = 0;
152 for (const [key, { buffer, size }] of writes) {144 for (const [key, { buffer, size }] of writes) {
...@@ -155,10 +147,14 @@ export async function flush(start: number) {...@@ -155,10 +147,14 @@ export async function flush(start: number) {
155 }147 }
156 // Asset map148 // Asset map
157 {149 {
158 const { json, blob } = getAssetManifest();150 const { json, blob, dynamic, dts } = getAssetManifest();
159 const jsonString = Buffer.from(JSON.stringify(json));151 const jsonString = Buffer.from(JSON.stringify(json));
160 p.push(fs.writeMkdir(".clover/o/static.json", jsonString));152 p.push(fs.writeMkdir(".clover/o/asset.json", jsonString));
161 p.push(fs.writeMkdir(".clover/o/static.blob", blob));153 p.push(fs.writeMkdir(".clover/o/asset.blob", blob));
154 p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts));
155 for (const [k, v] of dynamic) {
156 p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v));
157 }
162 dist += blob.byteLength + jsonString.byteLength;158 dist += blob.byteLength + jsonString.byteLength;
163 }159 }
164 await Promise.all(p);160 await Promise.all(p);
...@@ -171,9 +167,8 @@ export async function flush(start: number) {...@@ -171,9 +167,8 @@ export async function flush(start: number) {
171 console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);167 console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);
172 console.writeLine(` - ${assets.size} static assets`);168 console.writeLine(` - ${assets.size} static assets`);
173 console.writeLine(169 console.writeLine(
174 ` - dist: ${formatSize(dist)}, incremental: ${170 ` - dist: ${formatSize(dist)},` +
175 formatSize(serialized.byteLength)171 ` incremental: ${formatSize(serialized.byteLength)}`,
176 }`,
177 );172 );
178}173}
179174
...@@ -212,13 +207,14 @@ function deleteWork(key: string) {...@@ -212,13 +207,14 @@ function deleteWork(key: string) {
212 affects.splice(affects.indexOf(key), 1);207 affects.splice(affects.indexOf(key), 1);
213 }208 }
214 for (const remove of affects) {209 for (const remove of affects) {
215 const { reads: { works: list } } = UNWRAP(works.get(remove), remove);210 const {
211 reads: { works: list },
212 } = UNWRAP(works.get(remove), remove);
216 ASSERT(list.has(key));213 ASSERT(list.has(key));
217 list.delete(key);214 list.delete(key);
218 }215 }
219 for (const file of w.files) {216 for (const file of w.files) {
220 if (UNWRAP(writes.get(file)).work === key)217 if (UNWRAP(writes.get(file)).work === key) writes.delete(file);
221 writes.delete(file);
222 }218 }
223 // Assets are temporarily kept, trimmed via manual GC after compilation.219 // Assets are temporarily kept, trimmed via manual GC after compilation.
224220
...@@ -234,23 +230,32 @@ export function reset() {...@@ -234,23 +230,32 @@ export function reset() {
234}230}
235231
236export function serialize() {232export function serialize() {
237 const fileEntries = Array.from(files, ([k, v]) =>233 const fileEntries = Array.from(
238 [234 files,
239 k,235 ([k, v]) =>
240 v.type,236 [
241 v.type === 'f' ? v.lastModified : v.type === 'd' ? v.contentHash : null,237 k,
242 ...v.affects,238 v.type,
243 ] as const);239 v.type === "f" ? v.lastModified : v.type === "d" ? v.contentHash : null,
244 const workEntries = Array.from(works, ([k, v]) =>240 ...v.affects,
245 [241 ] as const,
246 k,242 );
247 v.value,243 const workEntries = Array.from(
248 Array.from(v.reads.files),244 works,
249 Array.from(v.reads.works),245 ([k, v]) =>
250 Array.from(v.writes.files),246 [
251 Array.from(v.writes.assets, ([k, { headers }]) => [k, headers] as const),247 k,
252 v.affects,248 v.value,
253 ] as const);249 Array.from(v.reads.files),
250 Array.from(v.reads.works),
251 Array.from(v.writes.files),
252 Array.from(
253 v.writes.assets,
254 ([k, { headers }]) => [k, headers] as const,
255 ),
256 v.affects,
257 ] as const,
258 );
254 const expectedFilesOnDisk = Array.from(259 const expectedFilesOnDisk = Array.from(
255 writes,260 writes,
256 ([k, { size, work }]) => [k, size, work] as const,261 ([k, { size, work }]) => [k, size, work] as const,
...@@ -280,7 +285,7 @@ async function deserialize(buffer: Buffer) {...@@ -280,7 +285,7 @@ async function deserialize(buffer: Buffer) {
280 if (type === "f") {285 if (type === "f") {
281 ASSERT(typeof content === "number");286 ASSERT(typeof content === "number");
282 files.set(k, { type, affects, lastModified: content });287 files.set(k, { type, affects, lastModified: content });
283 } else if (type === 'd') {288 } else if (type === "d") {
284 ASSERT(typeof content === "string");289 ASSERT(typeof content === "string");
285 files.set(k, { type, affects, contentHash: content, contents: [] });290 files.set(k, { type, affects, contentHash: content, contents: [] });
286 } else {291 } else {
...@@ -288,15 +293,8 @@ async function deserialize(buffer: Buffer) {...@@ -288,15 +293,8 @@ async function deserialize(buffer: Buffer) {
288 }293 }
289 }294 }
290 for (const entry of workEntries) {295 for (const entry of workEntries) {
291 const [296 const [k, value, readFiles, readWorks, writeFiles, writeAssets, affects] =
292 k,297 entry;
293 value,
294 readFiles,
295 readWorks,
296 writeFiles,
297 writeAssets,
298 affects,
299 ] = entry;
300 works.set(k, {298 works.set(k, {
301 value,299 value,
302 reads: {300 reads: {
...@@ -305,23 +303,30 @@ async function deserialize(buffer: Buffer) {...@@ -305,23 +303,30 @@ async function deserialize(buffer: Buffer) {
305 },303 },
306 writes: {304 writes: {
307 files: new Set(writeFiles),305 files: new Set(writeFiles),
308 assets: new Map(Array.from(writeAssets, ([k, headers]) => [k, {306 assets: new Map(
309 hash: JSON.parse(UNWRAP(headers.etag)),307 Array.from(writeAssets, ([k, headers]) => [
310 headers,308 k,
311 }])),309 {
310 hash: JSON.parse(UNWRAP(headers.etag)),
311 headers,
312 },
313 ]),
314 ),
312 },315 },
313 affects,316 affects,
314 });317 });
315 }318 }
316 const statFiles = await Promise.all(expectedFilesOnDisk319 const statFiles = await Promise.all(
317 .map(([k, size, work]) =>320 expectedFilesOnDisk.map(([k, size, work]) =>
318 fs.stat(path.join(".clover/o", k))321 fs
322 .stat(path.join(".clover/o", k))
319 .catch((err) => {323 .catch((err) => {
320 if (err.code === "ENOENT") return null;324 if (err.code === "ENOENT") return null;
321 throw err;325 throw err;
322 })326 })
323 .then((stat) => ({ k, size, work, stat }))327 .then((stat) => ({ k, size, work, stat }))
324 ));328 ),
329 );
325 for (const { k, stat, work, size } of statFiles) {330 for (const { k, stat, work, size } of statFiles) {
326 if (stat?.size === size) {331 if (stat?.size === size) {
327 writes.set(k, {332 writes.set(k, {
...@@ -337,19 +342,28 @@ async function deserialize(buffer: Buffer) {...@@ -337,19 +342,28 @@ async function deserialize(buffer: Buffer) {
337 assets.set(hash, { raw, gzip, zstd });342 assets.set(hash, { raw, gzip, zstd });
338 }343 }
339344
340 await Promise.all(Array.from(files, ([key, file]) => invalidateEntry(key, file)));345 await Promise.all(
346 Array.from(files, ([key, file]) => invalidateEntry(key, file, false)),
347 );
341}348}
342349
343export async function invalidate(filePath: string): Promise<boolean> {350export async function invalidate(
351 filePath: string,
352 unloadModule: boolean = true,
353): Promise<boolean> {
344 const key = toRel(toAbs(filePath));354 const key = toRel(toAbs(filePath));
345 const file = UNWRAP(files.get(key), `Untracked file '${key}'`)355 const file = UNWRAP(files.get(key), `Untracked file '${key}'`);
346 return invalidateEntry(key, file)356 return invalidateEntry(key, file, unloadModule);
347}357}
348358
349export async function invalidateEntry(key: string, file: TrackedFile): Promise<boolean> {359export async function invalidateEntry(
360 key: string,
361 file: TrackedFile,
362 unloadModule: boolean,
363): Promise<boolean> {
350 try {364 try {
351 if (file.type === "d") {365 if (file.type === "d") {
352 const contents = file.contents = await fs.readdir(key);366 const contents = (file.contents = await fs.readdir(key));
353 contents.sort();367 contents.sort();
354 const contentHash = crypto368 const contentHash = crypto
355 .createHash("sha1")369 .createHash("sha1")
...@@ -359,48 +373,79 @@ export async function invalidateEntry(key: string, file: TrackedFile): Promise<b...@@ -359,48 +373,79 @@ export async function invalidateEntry(key: string, file: TrackedFile): Promise<b
359 file.contentHash = contentHash;373 file.contentHash = contentHash;
360 throw new Error();374 throw new Error();
361 }375 }
362 } else if (file.type === 'f') {376 } else if (file.type === "f") {
363 const lastModified = await fs.stat(key)377 const lastModified = await fs.stat(key).then(
364 .then(x => Math.floor(x.mtimeMs), () => 0);378 (x) => Math.floor(x.mtimeMs),
379 () => 0,
380 );
365 if (file.lastModified !== lastModified) {381 if (file.lastModified !== lastModified) {
366 file.lastModified = lastModified;382 file.lastModified = lastModified;
367 throw new Error();383 throw new Error();
368 }384 }
369 } else {385 } else {
370 file.type satisfies 'null';386 file.type satisfies "null";
371 const stat = await fs.stat(key).catch(() => null);387 const stat = await fs.stat(key).catch(() => null);
372 if (stat) throw new Error();388 if (stat) throw new Error();
373 }389 }
374 return false;390 return false;
375 } catch (e) {391 } catch (e) {
376 forceInvalidate(file);392 forceInvalidate(file);
377 hot.unload(toAbs(key));393 if (unloadModule) {
378 if (file.type === 'null') files.delete(key);394 // TODO: handle when this triggers unloading of `generate.ts`
395 hot.unload(toAbs(key));
396 }
397 if (file.type === "null") files.delete(key);
379 return true;398 return true;
380 }399 }
381}400}
382401
383export function getAssetManifest() {402export function getAssetManifest() {
403 const dynamic = new Map<string, Buffer>();
384 const writer = new BufferWriter();404 const writer = new BufferWriter();
385 const asset = Object.fromEntries(405 const assetMap = Object.fromEntries(
386 Array.from(works, (work) => work[1].writes.assets)406 Array.from(works, (work) => work[1].writes.assets)
387 .filter((map) => map.size > 0)407 .filter((map) => map.size > 0)
388 .flatMap((map) =>408 .flatMap((map) =>
389 Array.from(map, ([key, { hash, headers }]) => {409 Array.from(map, ([key, { hash, headers, regenerative }]) => {
390 const { raw, gzip, zstd } = UNWRAP(410 const { raw, gzip, zstd } = UNWRAP(
391 assets.get(hash),411 assets.get(hash),
392 `Asset ${key} (${hash})`,412 `Asset ${key} (${hash})`,
393 );413 );
394 return [key, {414 if (regenerative) {
395 raw: writer.write(raw, "raw:" + hash),415 const id = crypto
396 gzip: writer.write(gzip, "gzip:" + hash),416 .createHash("sha1")
397 zstd: writer.write(zstd, "zstd:" + hash),417 .update(key)
398 headers,418 .digest("hex")
399 }] as const;419 .slice(0, 16); /* TODO */
420 dynamic.set(
421 id,
422 manifest.packDynamicBuffer(raw, gzip, zstd, headers),
423 );
424 return [key, { type: 1, id }] as const;
425 }
426 return [
427 key,
428 {
429 type: 0,
430 raw: writer.write(raw, "raw:" + hash),
431 gzip: writer.write(gzip, "gzip:" + hash),
432 zstd: writer.write(zstd, "zstd:" + hash),
433 headers,
434 },
435 ] as const;
400 })436 })
401 ),437 ),
402 ) satisfies BuiltAssetMap;438 ) satisfies manifest.Manifest;
403 return { json: asset, blob: writer.get() };439 return {
440 json: assetMap,
441 blob: writer.get(),
442 dynamic,
443 dts: "export type AssetKey = " +
444 Object.keys(assetMap)
445 .map((key) => JSON.stringify(key))
446 .join(" | ") +
447 "\n",
448 };
404}449}
405450
406/* Input/Output with automatic tracking.451/* Input/Output with automatic tracking.
...@@ -444,9 +489,9 @@ export class Io {...@@ -444,9 +489,9 @@ export class Io {
444 }489 }
445 async readDir(dir: string) {490 async readDir(dir: string) {
446 const { key, resolved } = this.#trackFs(dir);491 const { key, resolved } = this.#trackFs(dir);
447 const existing = files.get(key);492 const existing = files.get(key);
448 try {493 try {
449 if (existing?.type === 'd') return existing.contents;494 if (existing?.type === "d") return existing.contents;
450 const contents = await fs.readdir(resolved);495 const contents = await fs.readdir(resolved);
451 contents.sort();496 contents.sort();
452 const contentHash = crypto497 const contentHash = crypto
...@@ -512,26 +557,27 @@ export class Io {...@@ -512,26 +557,27 @@ export class Io {
512 }557 }
513 }558 }
514 }559 }
515 async writeAsset(560 async writeAsset(asset: {
516 pathname: string,561 pathname: string;
517 blob: string | Buffer,562 buffer: string | Buffer;
518 headersOption?: HeadersInit,563 regenerative?: boolean;
519 ) {564 headers?: HeadersInit;
520 ASSERT(pathname.startsWith("/"));565 }) {
521 ASSERT(!seenWrites.has("a:" + pathname));566 ASSERT(asset.pathname.startsWith("/"));
522567 ASSERT(!seenWrites.has("a:" + asset.pathname));
523 const buffer = typeof blob === "string" ? Buffer.from(blob) : blob;568 const buffer = typeof asset.buffer === "string"
524569 ? Buffer.from(asset.buffer)
525 const headers = new Headers(headersOption ?? {});570 : asset.buffer;
571 const headers = new Headers(asset.headers ?? {});
526 const hash = crypto.createHash("sha1").update(buffer).digest("hex");572 const hash = crypto.createHash("sha1").update(buffer).digest("hex");
527 if (!headers.has("Content-Type")) {573 if (!headers.has("Content-Type")) {
528 headers.set("Content-Type", mime.contentTypeFor(pathname));574 headers.set("Content-Type", mime.contentTypeFor(asset.pathname));
529 }575 }
530 headers.set("ETag", JSON.stringify(hash));576 headers.set("etag", JSON.stringify(hash));
531 this.writes.assets.set(pathname, {577 this.writes.assets.set(asset.pathname, {
532 hash,578 hash,
533 // @ts-expect-error TODO
534 headers: Object.fromEntries(headers),579 headers: Object.fromEntries(headers),
580 regenerative: !!asset.regenerative,
535 });581 });
536 if (!assets.has(hash)) {582 if (!assets.has(hash)) {
537 jobs += 1;583 jobs += 1;
...@@ -593,7 +639,10 @@ export function validateSerializable(value: unknown, key: string) {...@@ -593,7 +639,10 @@ export function validateSerializable(value: unknown, key: string) {
593 } else if (value && typeof value === "object") {639 } else if (value && typeof value === "object") {
594 if (Array.isArray(value)) {640 if (Array.isArray(value)) {
595 value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));641 value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`));
596 } else if (Object.getPrototypeOf(value) === Object.prototype || Buffer.isBuffer(value)) {642 } else if (
643 Object.getPrototypeOf(value) === Object.prototype ||
644 Buffer.isBuffer(value)
645 ) {
597 Object.entries(value).forEach(([k, v]) =>646 Object.entries(value).forEach(([k, v]) =>
598 validateSerializable(v, `${key}.${k}`)647 validateSerializable(v, `${key}.${k}`)
599 );648 );
...@@ -631,53 +680,48 @@ interface FileWrite {...@@ -631,53 +680,48 @@ interface FileWrite {
631}680}
632interface Writes {681interface Writes {
633 files: Set<string>;682 files: Set<string>;
634 assets: Map<string, {683 assets: Map<string, AssetWrite>;
635 hash: string;
636 headers: Record<string, string>;
637 }>;
638}684}
639interface Asset {685interface Asset {
640 raw: Buffer;686 raw: Buffer;
641 gzip: Buffer;687 gzip: Buffer;
642 zstd: Buffer;688 zstd: Buffer;
643}689}
690interface AssetWrite {
691 hash: string;
692 headers: Record<string, string>;
693 regenerative: boolean;
694}
644interface Work<T = unknown> {695interface Work<T = unknown> {
645 debug?: string;
646 value: T;696 value: T;
647 reads: Reads;697 reads: Reads;
648 writes: Writes;698 writes: Writes;
649 affects: string[];699 affects: string[];
650}700}
651type TrackedFile =701type TrackedFile =
652 & {702 & { affects: string[] }
653 affects: string[];
654 }
655 & (703 & (
656 | { type: "f"; lastModified: number }704 | { type: "f"; lastModified: number }
657 | { type: "d"; contentHash: string; contents: string[] }705 | { type: "d"; contentHash: string; contents: string[] }
658 | { type: "null"; }706 | { type: "null" }
659 );707 );
660export interface BuiltAssetMap {
661 [route: string]: BuiltAsset;
662}
663export interface BuiltAsset {
664 raw: BufferView;
665 gzip: BufferView;
666 zstd: BufferView;
667 headers: Record<string, string>;
668}
669708
670const gzip = util.promisify(zlib.gzip);709const gzip = util.promisify(zlib.gzip);
671const zstdCompress = util.promisify(zlib.zstdCompress);710const zstdCompress = util.promisify(zlib.zstdCompress);
672711
712import * as hot from "./hot.ts";
713
673import * as fs from "#sitegen/fs";714import * as fs from "#sitegen/fs";
715import * as mime from "#sitegen/mime";
716import * as manifest from "#sitegen/assets";
717
674import * as path from "node:path";718import * as path from "node:path";
675import * as hot from "./hot.ts";
676import * as util from "node:util";719import * as util from "node:util";
677import * as crypto from "node:crypto";720import * as crypto from "node:crypto";
678import * as mime from "#sitegen/mime";
679import * as zlib from "node:zlib";721import * as zlib from "node:zlib";
722
680import * as console from "@paperclover/console";723import * as console from "@paperclover/console";
681import { Spinner } from "@paperclover/console/Spinner";724import { Spinner } from "@paperclover/console/Spinner";
682import { formatSize } from "@/file-viewer/format.ts";725import { formatSize } from "@/file-viewer/format.ts";
726
683import * as msgpackr from "msgpackr";727import * as msgpackr from "msgpackr";
framework/lib/assets.ts+167-43
...@@ -1,73 +1,114 @@...@@ -1,73 +1,114 @@
1// Static and dynamic assets are built alongside the server binary.
2// This module implements decoding and serving of the asset blobs,
3// but also implements patching of dynamic assets. The `Manifest`
4// is generated by `incremental.ts`
5const root = import.meta.dirname;
6let current: Loaded | null = null;
7
8// TODO: rename all these types
9type DynamicId = string;
10export type { Key };
11export type Manifest =
12 & {
13 [K in Key]: StaticAsset | DynamicAsset;
14 }
15 & {
16 [string: string]: StaticAsset | DynamicAsset;
17 };
18export interface StaticAsset extends AssetBase {
19 type: 0;
20}
21interface AssetBase {
22 headers: Record<string, string> & { etag: string };
23 raw: BufferView;
24 gzip: BufferView;
25 zstd: BufferView;
26}
27export interface DynamicAsset {
28 type: 1;
29 id: DynamicId;
30}
1interface Loaded {31interface Loaded {
2 map: BuiltAssetMap;32 map: Manifest;
3 buf: Buffer;33 static: Buffer;
34 dynamic: Map<DynamicId, DynamicEntry>;
4}35}
5let assets: Loaded | null = null;36export interface DynamicEntry extends AssetBase {
637 buffer: Buffer;
7export type StaticPageId = string;
8
9export async function reload() {
10 const [map, buf] = await Promise.all([
11 fs.readFile(path.join(import.meta.dirname, "static.json"), "utf8"),
12 fs.readFile(path.join(import.meta.dirname, "static.blob")),
13 ]);
14 return (assets = { map: JSON.parse(map), buf });
15}38}
1639
17export function reloadSync() {40export async function reload() {
18 const map = fs.readFileSync(41 const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
19 path.join(import.meta.dirname, "static.json"),42 const statics = await fs.readFile(path.join(root, "asset.blob"));
20 "utf8",43 const dynamic = new Map(
44 await Promise.all(
45 Object.entries(map)
46 .filter((entry): entry is [string, DynamicAsset] => entry[1].type === 1)
47 .map(async ([k, v]) =>
48 [
49 v.id,
50 await fs.readFile(path.join(root, "dynamic", v.id))
51 .then(loadRegenerative),
52 ] as const
53 ),
54 ),
21 );55 );
22 const buf = fs.readFileSync(path.join(import.meta.dirname, "static.blob"));56 return (current = { map, static: statics, dynamic });
23 return (assets = { map: JSON.parse(map), buf });
24}57}
2558
26export async function middleware(c: Context, next: Next) {59export async function middleware(c: Context, next: Next) {
27 if (!assets) await reload();60 if (!current) current = await reload();
28 const asset = assets!.map[c.req.path];61 const asset = current.map[c.req.path];
29 if (asset) return assetInner(c, asset, 200);62 if (asset) return assetInner(c, asset, 200);
30 return next();63 return next();
31}64}
3265
33export async function notFound(c: Context) {66export async function notFound(c: Context) {
34 if (!assets) await reload();67 if (!current) current = await reload();
35 let pathname = c.req.path;68 let pathname = c.req.path;
36 do {69 do {
37 const asset = assets!.map[pathname + "/404"];70 const asset = current.map[pathname + "/404"];
38 if (asset) return assetInner(c, asset, 404);71 if (asset) return assetInner(c, asset, 404);
39 pathname = pathname.slice(0, pathname.lastIndexOf("/"));72 pathname = pathname.slice(0, pathname.lastIndexOf("/"));
40 } while (pathname);73 } while (pathname);
41 const asset = assets!.map["/404"];74 const asset = current.map["/404"];
42 if (asset) return assetInner(c, asset, 404);75 if (asset) return assetInner(c, asset, 404);
43 return c.text("the 'Not Found' page was not found", 404);76 return c.text("the 'Not Found' page was not found", 404);
44}77}
4578
46export async function serveAsset(79export async function serveAsset(c: Context, id: Key, status: StatusCode) {
47 c: Context,80 return assetInner(c, (current ?? (await reload())).map[id], status);
48 id: StaticPageId,
49 status: StatusCode,
50) {
51 return assetInner(c, (assets ?? (await reload())).map[id], status);
52}81}
5382
83/** @deprecated */
54export function hasAsset(id: string) {84export function hasAsset(id: string) {
55 return (assets ?? reloadSync()).map[id] !== undefined;85 return UNWRAP(current).map[id] !== undefined;
56}86}
5787
58export function etagMatches(etag: string, ifNoneMatch: string) {88export function etagMatches(etag: string, ifNoneMatch: string) {
59 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;89 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
60}90}
6191
62function subarrayAsset([start, end]: BufferView) {92function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {
63 return assets!.buf.subarray(start, end);93 ASSERT(current);
94 if (asset.type === 0) {
95 return respondWithBufferAndViews(c, current.static, asset, status);
96 } else {
97 const entry = UNWRAP(current.dynamic.get(asset.id));
98 return respondWithBufferAndViews(c, entry.buffer, entry, status);
99 }
64}100}
65101
66function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {102function respondWithBufferAndViews(
67 const ifnonematch = c.req.header("If-None-Match");103 c: Context,
68 if (ifnonematch) {104 buffer: Buffer,
69 const etag = asset.headers.ETag;105 asset: AssetBase,
70 if (etagMatches(etag, ifnonematch)) {106 status: StatusCode,
107) {
108 const ifNoneMatch = c.req.header("If-None-Match");
109 if (ifNoneMatch) {
110 const etag = asset.headers.etag;
111 if (etagMatches(etag, ifNoneMatch)) {
71 return (c.res = new Response(null, {112 return (c.res = new Response(null, {
72 status: 304,113 status: 304,
73 statusText: "Not Modified",114 statusText: "Not Modified",
...@@ -80,24 +121,103 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {...@@ -80,24 +121,103 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
80 const acceptEncoding = c.req.header("Accept-Encoding") ?? "";121 const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
81 let body;122 let body;
82 let headers = asset.headers;123 let headers = asset.headers;
83 if (acceptEncoding.includes("zstd") && asset.zstd) {124 if (acceptEncoding.includes("zstd")) {
84 body = subarrayAsset(asset.zstd);125 body = buffer.subarray(...asset.zstd);
85 headers = {126 headers = {
86 ...asset.headers,127 ...asset.headers,
87 "Content-Encoding": "zstd",128 "Content-Encoding": "zstd",
88 };129 };
89 } else if (acceptEncoding.includes("gzip") && asset.gzip) {130 } else if (acceptEncoding.includes("gzip")) {
90 body = subarrayAsset(asset.gzip);131 body = buffer.subarray(...asset.gzip);
91 headers = {132 headers = {
92 ...asset.headers,133 ...asset.headers,
93 "Content-Encoding": "gzip",134 "Content-Encoding": "gzip",
94 };135 };
95 } else {136 } else {
96 body = subarrayAsset(asset.raw);137 body = buffer.subarray(...asset.raw);
97 }138 }
98 return (c.res = new Response(body, { headers, status }));139 return (c.res = new Response(body, { headers, status }));
99}140}
100141
142export function packDynamicBuffer(
143 raw: Buffer,
144 gzip: Buffer,
145 zstd: Buffer,
146 headers: Record<string, string>,
147) {
148 const headersBuffer = Buffer.from(
149 Object.entries(headers)
150 .map((entry) => entry.join(":"))
151 .join("\n"),
152 "utf-8",
153 );
154 const header = new Uint32Array(3);
155 header[0] = headersBuffer.byteLength + header.byteLength;
156 header[1] = header[0] + raw.byteLength;
157 header[2] = header[1] + gzip.byteLength;
158 return Buffer.concat([
159 Buffer.from(header.buffer),
160 headersBuffer,
161 raw,
162 gzip,
163 zstd,
164 ]);
165}
166
167function loadRegenerative(buffer: Buffer): DynamicEntry {
168 const headersEnd = buffer.readUInt32LE(0);
169 const headers = Object.fromEntries(
170 buffer
171 .subarray(3 * 4, headersEnd)
172 .toString("utf-8")
173 .split("\n")
174 .map((line) => {
175 const i = line.indexOf(":");
176 return [line.slice(0, i), line.slice(i + 1)];
177 }),
178 );
179 const raw = buffer.readUInt32LE(4);
180 const gzip = buffer.readUInt32LE(8);
181 const hasEtag = (v: object): v is typeof v & { etag: string } =>
182 "etag" in v && typeof v.etag === "string";
183 ASSERT(hasEtag(headers));
184 return {
185 headers,
186 buffer,
187 raw: [headersEnd, raw],
188 gzip: [raw, gzip],
189 zstd: [gzip, buffer.byteLength],
190 };
191}
192
193const gzip = util.promisify(zlib.gzip);
194const zstdCompress = util.promisify(zlib.zstdCompress);
195
196export async function overwriteDynamic(
197 key: Key,
198 value: string | Buffer,
199 headers: Record<string, string>,
200) {
201 if (!current) current = await reload();
202 const asset = UNWRAP(current.map[key]);
203 ASSERT(asset.type === 1);
204 UNWRAP(current.dynamic.has(asset.id));
205 const buffer = Buffer.from(value);
206 const etag = JSON.stringify(
207 crypto.createHash("sha1").update(buffer).digest("hex"),
208 );
209 const [gzipBuffer, zstdBuffer] = await Promise.all([
210 gzip(buffer),
211 zstdCompress(buffer),
212 ]);
213 const packed = packDynamicBuffer(buffer, gzipBuffer, zstdBuffer, {
214 ...headers,
215 etag,
216 });
217 current.dynamic.set(asset.id, loadRegenerative(packed));
218 await fs.writeFile(path.join(root, "dynamic", asset.id), packed);
219}
220
101process.on("message", (msg: any) => {221process.on("message", (msg: any) => {
102 if (msg?.type === "clover.assets.reload") reload();222 if (msg?.type === "clover.assets.reload") reload();
103});223});
...@@ -105,6 +225,10 @@ process.on("message", (msg: any) => {...@@ -105,6 +225,10 @@ process.on("message", (msg: any) => {
105import * as fs from "#sitegen/fs";225import * as fs from "#sitegen/fs";
106import type { Context, Next } from "hono";226import type { Context, Next } from "hono";
107import type { StatusCode } from "hono/utils/http-status";227import type { StatusCode } from "hono/utils/http-status";
108import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts";228import type { BufferView } from "../incremental.ts";
109import { Buffer } from "node:buffer";229import { Buffer } from "node:buffer";
110import * as path from "node:path";230import * as path from "node:path";
231import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
232import * as crypto from "node:crypto";
233import * as zlib from "node:zlib";
234import * as util from "node:util";
framework/lib/async.ts+2-2
...@@ -51,8 +51,8 @@ export class Queue<T, R> {...@@ -51,8 +51,8 @@ export class Queue<T, R> {
51 let n = 0;51 let n = 0;
52 for (const item of active) {52 for (const item of active) {
53 let itemText = "- " + item.format(now);53 let itemText = "- " + item.format(now);
54 text +=54 text += `\n` +
55 `\n` + itemText.slice(0, Math.max(0, process.stdout.columns - 1));55 itemText.slice(0, Math.max(0, process.stdout.columns - 1));
56 if (n > 10) {56 if (n > 10) {
57 text += `\n ... + ${active.length - n} more`;57 text += `\n ... + ${active.length - n} more`;
58 break;58 break;
framework/lib/fs.ts+2-2
...@@ -4,6 +4,7 @@ export {...@@ -4,6 +4,7 @@ export {
4 createReadStream,4 createReadStream,
5 createWriteStream,5 createWriteStream,
6 existsSync,6 existsSync,
7 type FileHandle,
7 open,8 open,
8 readdir,9 readdir,
9 readdirSync,10 readdirSync,
...@@ -15,7 +16,6 @@ export {...@@ -15,7 +16,6 @@ export {
15 statSync,16 statSync,
16 writeFile,17 writeFile,
17 writeFileSync,18 writeFileSync,
18 type FileHandle,
19};19};
2020
21export function mkdir(dir: string) {21export function mkdir(dir: string) {
...@@ -98,6 +98,7 @@ import {...@@ -98,6 +98,7 @@ import {
98 writeFileSync,98 writeFileSync,
99} from "node:fs";99} from "node:fs";
100import {100import {
101 type FileHandle,
101 mkdir as nodeMkdir,102 mkdir as nodeMkdir,
102 open,103 open,
103 readdir,104 readdir,
...@@ -106,6 +107,5 @@ import {...@@ -106,6 +107,5 @@ import {
106 rmdir,107 rmdir,
107 stat,108 stat,
108 writeFile,109 writeFile,
109 type FileHandle,
110} from "node:fs/promises";110} from "node:fs/promises";
111export { Stats } from "node:fs";111export { Stats } from "node:fs";
framework/lib/markdown.tsx+12-8
...@@ -1,9 +1,11 @@...@@ -1,9 +1,11 @@
1/* Impementation of CommonMark specification for markdown with support1/* Implementation of [CommonMark] specification for markdown with support
2 * for custom syntax extensions via the parser options. Instead of2 * for custom syntax extensions via the parser options. Instead of
3 * returning an AST that has a second conversion pass to JSX, the3 * returning an AST that has a second conversion pass to JSX, the
4 * returned value of 'parse' is 'engine.Node' which can be stringified4 * returned value of 'parse' is 'engine.Node' which can be stringified
5 * via clover's SSR engine. This way, generation optimizations, async5 * via Clover's SSR engine. This way, generation optimizations, async
6 * components, and other features are gained for free here.6 * components, and other features are gained for free here.
7 *
8 * [CommonMark]: https://spec.commonmark.org/0.31.2/
7 */9 */
8function parse(src: string, options: Partial<ParseOpts> = {}) {}10function parse(src: string, options: Partial<ParseOpts> = {}) {}
911
...@@ -16,6 +18,9 @@ export function Markdown({...@@ -16,6 +18,9 @@ export function Markdown({
16 return parse(src, options);18 return parse(src, options);
17}19}
1820
21// TODO: This implementation is flawed because it is impossible to sanely handle
22// emphasis and strong emphasis, and all their edge cases. Instead of making these
23// using extensions interface, they should be special cased.
19function parseInline(src: string, options: Partial<InlineOpts> = {}) {24function parseInline(src: string, options: Partial<InlineOpts> = {}) {
20 const { rules = inlineRules, links = new Map() } = options;25 const { rules = inlineRules, links = new Map() } = options;
21 const opts: InlineOpts = { rules, links };26 const opts: InlineOpts = { rules, links };
...@@ -110,12 +115,11 @@ export const inlineRules: Record<string, InlineRule> = {...@@ -110,12 +115,11 @@ export const inlineRules: Record<string, InlineRule> = {
110 } else if (afterText[0] === "[") {115 } else if (afterText[0] === "[") {
111 const splitTarget = splitFirst(afterText.slice(1), /]/);116 const splitTarget = splitFirst(afterText.slice(1), /]/);
112 if (!splitTarget) return null;117 if (!splitTarget) return null;
113 const name =118 const name = splitTarget.first.trim().length === 0
114 splitTarget.first.trim().length === 0119 // Collapsed reference link
115 ? // Collapsed reference link120 ? textSrc.trim()
116 textSrc.trim()121 // Full Reference Link
117 : // Full Reference Link122 : splitTarget.first.trim();
118 splitTarget.first.trim();
119 const target = opts.links.get(name);123 const target = opts.links.get(name);
120 if (!target) return null;124 if (!target) return null;
121 ({ href, title } = target);125 ({ href, title } = target);
framework/lib/sitegen.ts+40-1
...@@ -1,6 +1,25 @@...@@ -1,6 +1,25 @@
1// Import this file with 'import * as sg from "#sitegen";'
2export type ScriptId = string;1export type ScriptId = string;
32
3export interface PageExports extends ViewExports {
4 regenerate?: PageRegenerateOptions;
5}
6export interface ViewExports {
7 default: render.Component;
8 meta: meta.Meta | ((props: { ssr: true }) => Promise<meta.Meta> | meta.Meta);
9 theme?: css.Theme;
10 layout?: Layout;
11}
12export interface Layout {
13 default: render.Component;
14 theme?: css.Theme;
15 // TODO: nested layout
16}
17export interface PageRegenerateOptions {
18 tags?: string[];
19 seconds?: number;
20 debounce?: number;
21}
22
4/**23/**
5 * A filesystem object associated with some ID,24 * A filesystem object associated with some ID,
6 * such as a page's route to it's source file.25 * such as a page's route to it's source file.
...@@ -30,4 +49,24 @@ export function addScript(id: ScriptId | { value: ScriptId }) {...@@ -30,4 +49,24 @@ export function addScript(id: ScriptId | { value: ScriptId }) {
30 userData.get().scripts.add(typeof id === "string" ? id : id.value);49 userData.get().scripts.add(typeof id === "string" ? id : id.value);
31}50}
3251
52export function wrapDocument({
53 body,
54 head,
55 inlineCss,
56 scripts,
57}: {
58 head: string;
59 body: string;
60 inlineCss: string;
61 scripts: string;
62}) {
63 return `<!doctype html><html lang=en><head>${head}${
64 inlineCss ? `<style>${inlineCss}</style>` : ""
65 }</head><body>${body}${
66 scripts ? `<script>${scripts}</script>` : ""
67 }</body></html>`;
68}
69
33import * as render from "#engine/render";70import * as render from "#engine/render";
71import type * as meta from "./meta.ts";
72import type * as css from "../css.ts";
framework/lib/sqlite.ts+3-1
...@@ -10,7 +10,9 @@ export function getDb(file: string) {...@@ -10,7 +10,9 @@ export function getDb(file: string) {
10 if (db) return db;10 if (db) return db;
11 const fileWithExt = file.includes(".") ? file : file + ".sqlite";11 const fileWithExt = file.includes(".") ? file : file + ".sqlite";
12 db = new WrappedDatabase(12 db = new WrappedDatabase(
13 new DatabaseSync(path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt)),13 new DatabaseSync(
14 path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
15 ),
14 );16 );
15 map.set(file, db);17 map.set(file, db);
16 return db;18 return db;
framework/lib/testing.ts+7-3
...@@ -1,5 +1,9 @@...@@ -1,5 +1,9 @@
1export function tmpFile(basename: string) {1export function tmpFile(basename: string) {
2 const file = path.join(import.meta.dirname, '../../.clover/testing', basename);2 const file = path.join(
3 import.meta.dirname,
4 "../../.clover/testing",
5 basename,
6 );
3 return {7 return {
4 path: file,8 path: file,
5 read: fs.readFile.bind(fs, file),9 read: fs.readFile.bind(fs, file),
...@@ -7,5 +11,5 @@ export function tmpFile(basename: string) {...@@ -7,5 +11,5 @@ export function tmpFile(basename: string) {
7 };11 };
8}12}
913
10import * as path from 'node:path';14import * as path from "node:path";
11import * as fs from './fs.ts';15import * as fs from "./fs.ts";
framework/lib/view.ts+70-44
...@@ -1,38 +1,64 @@...@@ -1,38 +1,64 @@
1export interface View {1// The "view" system allows rendering dynamic pages within backends.
2// This is done by scanning all `views` dirs, bundling their client
3// resources, and then providing `serve` which renders a page.
4//
5// This system also implements page regeneration.
6let codegen: Codegen;
7try {
8 codegen = require("$views");
9} catch {
10 throw new Error("Can only import '#sitegen/view' in backends.");
11}
12
13// Generated in `bundle.ts`
14export interface Codegen {
15 views: { [K in Key]: View<PropsFromModule<ViewMap[K]>> };
16 scripts: Record<string, string>;
17 regenTtls: Ttl[];
18 regenTags: Record<RegenKey, Key[]>;
19}
20// The view contains pre-bundled CSS and scripts, but keeps the scripts
21// separate for run-time dynamic scripts. For example, the file viewer
22// includes the canvas for the current page, but only the current page.
23export interface View<Props extends Record<string, unknown>> {
2 component: render.Component;24 component: render.Component;
3 meta:25 meta: meta.Meta | ((props: Props) => Promise<meta.Meta> | meta.Meta);
4 | meta.Meta
5 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
6 layout?: render.Component;26 layout?: render.Component;
7 inlineCss: string;27 inlineCss: string;
8 scripts: Record<string, string>;28 scripts: Record<string, string>;
9}29}
30export interface Ttl {
31 seconds: number;
32 key: Key;
33}
34export type Key = keyof ViewMap;
1035
11let views: Record<string, View> = null!;36export async function serve<K extends Key>(
12let scripts: Record<string, string> = null!;
13
14export async function renderView(
15 context: hono.Context,37 context: hono.Context,
16 id: string,38 id: K,
17 props: Record<string, unknown>,39 props: PropsFromModule<ViewMap[K]>,
18) {40) {
19 return context.html(await renderViewToString(id, { context, ...props }));41 return context.html(await renderToString(id, { context, ...props }));
20}42}
2143
22export async function renderViewToString(44type PropsFromModule<M extends any> = M extends {
23 id: string,45 default: (props: infer T) => render.Node;
24 props: Record<string, unknown>,46} ? T
47 : never;
48
49export async function renderToString<K extends Key>(
50 id: K,
51 props: PropsFromModule<ViewMap[K]>,
25) {52) {
26 views ?? ({ views, scripts } = require("$views"));
27 // The view contains pre-bundled CSS and scripts, but keeps the scripts
28 // separate for run-time dynamic scripts. For example, the file viewer
29 // includes the canvas for the current page, but only the current page.
30 const {53 const {
31 component,54 component,
32 inlineCss,55 inlineCss,
33 layout,56 layout,
34 meta: metadata,57 meta: metadata,
35 }: View = UNWRAP(views[id], `Missing view ${id}`);58 }: View<PropsFromModule<ViewMap[K]>> = UNWRAP(
59 codegen.views[id],
60 `Missing view ${id}`,
61 );
3662
37 // -- metadata --63 // -- metadata --
38 const renderedMetaPromise = Promise.resolve(64 const renderedMetaPromise = Promise.resolve(
...@@ -48,48 +74,48 @@ export async function renderViewToString(...@@ -48,48 +74,48 @@ export async function renderViewToString(
48 } = await render.async(page, { [sg.userData.key]: sg.initRender() });74 } = await render.async(page, { [sg.userData.key]: sg.initRender() });
4975
50 // -- join document and send --76 // -- join document and send --
51 return wrapDocument({77 return sg.wrapDocument({
52 body,78 body,
53 head: await renderedMetaPromise,79 head: await renderedMetaPromise,
54 inlineCss,80 inlineCss,
55 scripts: joinScripts(81 scripts: joinScripts(
56 Array.from(sitegen.scripts, (id) =>82 Array.from(
57 UNWRAP(scripts[id], `Missing script ${id}`),83 sitegen!.scripts,
84 (id) => UNWRAP(codegen.scripts[id], `Missing script ${id}`),
58 ),85 ),
59 ),86 ),
60 });87 });
61}88}
6289
63export function provideViewData(v: typeof views, s: typeof scripts) {90export function regenerate(tag: RegenKey) {
64 (views = v), (scripts = s);91 for (const view of codegen.regenTags[tag]) {
92 const key = view.slice("page:".length);
93 renderToString(view, {})
94 .then((result) => {
95 console.info(`regenerate ${key}`);
96 asset.overwriteDynamic(key as asset.Key, result, {
97 "content-type": "text/html",
98 });
99 })
100 .catch((e) => {
101 console.error(`Failed regenerating ${view} from tag ${tag}`, e);
102 });
103 }
65}104}
66105
67export function joinScripts(scriptSources: string[]) {106function joinScripts(scriptSources: string[]) {
68 const { length } = scriptSources;107 const { length } = scriptSources;
69 if (length === 0) return "";108 if (length === 0) return "";
70 if (length === 1) return scriptSources[0];109 if (0 in scriptSources) return scriptSources[0];
71 return scriptSources.map((source) => `{${source}}`).join(";");110 return scriptSources.map((source) => `{${source}}`).join(";");
72}111}
73112
74export function wrapDocument({
75 body,
76 head,
77 inlineCss,
78 scripts,
79}: {
80 head: string;
81 body: string;
82 inlineCss: string;
83 scripts: string;
84}) {
85 return `<!doctype html><html lang=en><head>${head}${
86 inlineCss ? `<style>${inlineCss}</style>` : ""
87 }</head><body>${body}${
88 scripts ? `<script>${scripts}</script>` : ""
89 }</body></html>`;
90}
91
92import * as meta from "./meta.ts";113import * as meta from "./meta.ts";
93import type * as hono from "#hono";114import type * as hono from "#hono";
94import * as render from "#engine/render";115import * as render from "#engine/render";
95import * as sg from "./sitegen.ts";116import * as sg from "./sitegen.ts";
117import * as asset from "./assets.ts";
118import type {
119 RegenKey,
120 RegisteredViews as ViewMap,
121} from "../../.clover/ts/view.d.ts";
framework/marko.ts+12-11
...@@ -14,18 +14,19 @@ export function loadMarko(module: NodeJS.Module, filepath: string) {...@@ -14,18 +14,19 @@ export function loadMarko(module: NodeJS.Module, filepath: string) {
14 // bare client import statements to it's own usage.14 // bare client import statements to it's own usage.
15 const scannedClientRefs = new Set<string>();15 const scannedClientRefs = new Set<string>();
16 if (src.match(/^\s*client\s+import\s+["']/m)) {16 if (src.match(/^\s*client\s+import\s+["']/m)) {
17 src =17 src = src.replace(
18 src.replace(18 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
19 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,19 (_, src) => {
20 (_, src) => {20 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
21 const ref = JSON.parse(`"${src.slice(1, -1)}"`);21 const resolved = hot.resolveClientRef(filepath, ref);
22 const resolved = hot.resolveClientRef(filepath, ref);22 scannedClientRefs.add(resolved);
23 scannedClientRefs.add(resolved);23 return `<CloverScriptInclude=${
24 return `<CloverScriptInclude=${JSON.stringify(24 JSON.stringify(
25 hot.getScriptId(resolved),25 hot.getScriptId(resolved),
26 )} />`;26 )
27 },27 } />`;
28 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';28 },
29 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
29 }30 }
3031
31 src = marko.compileSync(src, filepath).code;32 src = marko.compileSync(src, filepath).code;
framework/watch.ts+10-9
...@@ -11,7 +11,7 @@ let watch: Watch;...@@ -11,7 +11,7 @@ let watch: Watch;
1111
12export async function main() {12export async function main() {
13 // Catch up state by running a main build.13 // Catch up state by running a main build.
14 await incr.restore();14 if (!process.argv.includes("-f")) await incr.restore();
15 watch = new Watch(rebuild);15 watch = new Watch(rebuild);
16 rebuild([]);16 rebuild([]);
17}17}
...@@ -36,15 +36,16 @@ function onSubprocessClose(code: number | null, signal: string | null) {...@@ -36,15 +36,16 @@ function onSubprocessClose(code: number | null, signal: string | null) {
36}36}
3737
38async function rebuild(files: string[]) {38async function rebuild(files: string[]) {
39 const hasInvalidated = files.length === 039 const hasInvalidated = files.length === 0 ||
40 || (await Promise.all(files.map(incr.invalidate))).some(Boolean);40 (await Promise.all(files.map((file) => incr.invalidate(file))))
41 .some(Boolean);
41 if (!hasInvalidated) return;42 if (!hasInvalidated) return;
42 incr.compile(generate.generate).then(({43 incr.compile(generate.generate).then(({
43 watchFiles,44 watchFiles,
44 newOutputs,45 newOutputs,
45 newAssets46 newAssets,
46 }) => {47 }) => {
47 const removeWatch = [...watch.files].filter(x => !watchFiles.has(x))48 const removeWatch = [...watch.files].filter((x) => !watchFiles.has(x));
48 for (const file of removeWatch) watch.remove(file);49 for (const file of removeWatch) watch.remove(file);
49 watch.add(...watchFiles);50 watch.add(...watchFiles);
50 // Restart the server if it was changed or not running.51 // Restart the server if it was changed or not running.
...@@ -60,8 +61,8 @@ async function rebuild(files: string[]) {...@@ -60,8 +61,8 @@ async function rebuild(files: string[]) {
6061
61function statusLine() {62function statusLine() {
62 console.info(63 console.info(
63 `Watching ${watch.files.size} files `64 `Watching ${watch.files.size} files ` +
64 + `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,65 `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
65 );66 );
66}67}
6768
...@@ -142,7 +143,7 @@ class Watch {...@@ -142,7 +143,7 @@ class Watch {
142 #getFiles(absPath: string, event: fs.WatchEventType) {143 #getFiles(absPath: string, event: fs.WatchEventType) {
143 const files = [];144 const files = [];
144 if (this.files.has(absPath)) files.push(absPath);145 if (this.files.has(absPath)) files.push(absPath);
145 if (event === 'rename') {146 if (event === "rename") {
146 const dir = path.dirname(absPath);147 const dir = path.dirname(absPath);
147 if (this.files.has(dir)) files.push(dir);148 if (this.files.has(dir)) files.push(dir);
148 }149 }
...@@ -153,7 +154,7 @@ class Watch {...@@ -153,7 +154,7 @@ class Watch {
153 if (!subPath) return;154 if (!subPath) return;
154 const files = this.#getFiles(path.join(root, subPath), event);155 const files = this.#getFiles(path.join(root, subPath), event);
155 if (files.length === 0) return;156 if (files.length === 0) return;
156 for(const file of files) this.stale.add(file);157 for (const file of files) this.stale.add(file);
157 const { debounce } = this;158 const { debounce } = this;
158 if (debounce !== null) clearTimeout(debounce);159 if (debounce !== null) clearTimeout(debounce);
159 this.debounce = setTimeout(() => {160 this.debounce = setTimeout(() => {
src/backend.ts+3-3
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1// This is the main file for the backend1// This is the main file for paperclover.net's server.
2const app = new Hono();2const app = new Hono();
3const logHttp = scoped("http", { color: "magenta" });3const logHttp = console.scoped("http", { color: "magenta" });
44
5// Middleware5// Middleware
6app.use(trimTrailingSlash());6app.use(trimTrailingSlash());
...@@ -38,4 +38,4 @@ import { logger } from "hono/logger";...@@ -38,4 +38,4 @@ import { logger } from "hono/logger";
38import { trimTrailingSlash } from "hono/trailing-slash";38import { trimTrailingSlash } from "hono/trailing-slash";
39import * as assets from "#sitegen/assets";39import * as assets from "#sitegen/assets";
40import * as admin from "./admin.ts";40import * as admin from "./admin.ts";
41import { scoped } from "@paperclover/console";41import * as console from "@paperclover/console";
src/blog/pages/25/file-viewer-internals.mdx deleted-10
...@@ -1,10 +0,0 @@
1export const blog: BlogMeta = {
2 title: "Marko is the coziest HTML templating language",
3 desc: "...todo...",
4 date: "2025-07-08",
5 draft: true,
6};
7export const meta = formatBlogMeta(blob);
8export * as layout from "@/blog/layout.tsx";
9
10
src/blog/pages/25/marko-intro.markodown deleted-290
...@@ -1,290 +0,0 @@
1export const blog: BlogMeta = {
2 title: "Marko is the coziest HTML templating language",
3 desc: "...todo...",
4 date: "2025-06-13",
5 draft: true,
6};
7export const meta = formatBlogMeta(blob);
8export * as layout from "@/blog/layout.tsx";
9
10I've been recently playing around [Marko][1], and after adding limited support
11for it in my website generator, [sitegen][2], I instantly fell in love with how
12minimalistic it is in comparison to JSX, Astro components, and Svelte.
13
14## Introduction
15
16If JSX was taking HTML and shoving its syntax into JavaScript, Marko is shoving
17JavaScript into HTML. Attributes are JavaScript expressions.
18
19```marko
20<div>
21 // `input` is like props, but given in the top-level scope
22 <time datetime=input.date.toISOString()>
23 // Interpolation with JS template string syntax
24 ${formatTimeNicely(input.date)}
25 </time>
26 <div>
27 <a href=`/users/${input.user.id}`>${input.user.name}</a>
28 </div>
29
30 // Capital letter variables for imported components
31 <MarkdownContent message=input.message />
32
33 // Components also can be auto-imported by lowercase.
34 // This will look upwards for a `tags/` folder containing
35 // "custom-footer.marko", similar to how Node.js finds
36 // package names in all upwards `node_modules` folders.
37 <custom-footer />
38</div>
39
40// ESM `import` / `export` just work as expected.
41// I prefer my imports at the end, to highlight the markup.
42import MarkdownContent from "./MarkdownContent.marko";
43import { formatTimeNicely } from "../date-helpers.ts";
44```
45
46Tags with the `value` attribute have a shorthand, which is used by the built-in
47`<if>` for conditional rendering.
48
49```marko
50// Sugar for <input value="string" />
51<input="string" />
52
53// and it composes amazingly to the 'if' built-in
54<if=input.user>
55 <UserProfile=input.user />
56</if>
57```
58
59Tags can also return values into the scope for use in the template using `/`, such as `<id>` for unique ID generation. This is available to components that `<return=output/>`.
60
61```
62<id/uniqueId />
63
64<input id=uniqueId type="checkbox" name="allow_trans_rights" />
65<label for=uniqueId>click me!</>
66// ^ oh, you can also omit the
67// closing tag name if you want.
68```
69
70It's important that I started with the two forms of "Tag I/O": `=` for input
71and `/` for output. With those building blocks, we introduce local variables
72with `const`
73
74```
75<const/rendered = markdownToHtml(input.value) />
76
77// This is how you insert raw HTML to the document
78<inline-html=rendered />
79
80// It supports all of the cozy destructuring syntax JS has
81<const/{ id, name } = user />
82```
83
84Unlike JSX, when you pass content within a tag (`input.content` instead of
85JSX's `children`), instead of it being a JSX element, it is actually a
86function. This means that the `for` tag can render the content multiple times.
87
88```
89<ul>
90 <for from=1 to=10>
91 // Renders a new random number for each iteration.
92 <li>${Math.random()}</li>
93 </>
94</ul>
95```
96
97Since `content` is a function, it can take arguments. This is done with `|`
98
99```
100<h1>my friends</h1>
101<ul>
102 // I tend to omit the closing tag names for the built-in control
103 // flow tags, but I keep them for HTML tags. It's kinda like how
104 // in JavaScript you just write `}` to close your `if`s and loops.
105 //
106 // Anyways <for> also has 'of'
107 <for|item| of=user.friends>
108 <li class="friend">${item.name}</li>
109 </>
110
111 // They support the same syntax JavaScript function params allows,
112 // so you can have destructuring here too, and multiple params.
113 <for|{ name }, index| of=user.friends>
114 // By the way you can also use emmet-style class and ID shorthands.
115 <li.friend>My #${index + 1} friend is ${name}</li>
116 </>
117</ul>
118```
119
120Instead of named slots, Marko has attribute tags. These are more powerful than
121slots since they are functions, and can also act as sugar for more complicated
122attributes.
123
124```
125<Layout title="Welcome">
126 <@header variant="big">
127 <h1>the next big thing</h1>
128 </@header>
129
130 <p>body text...</p>
131</Layout>
132
133// The `input` variable inside of <Layout /> is:
134//
135// {
136// title: "Welcome",
137// header: {
138// content: /* function rendering "<h1>the next big thing</h1>" */,
139// variant: "big",
140// },
141// content: /* function rendering "<p>body text</p>" */
142// }
143```
144
145This layout could be implemented as such:
146
147```marko
148<main>
149 <if=input.header />
150 <const/{ ...headerProps, content }=input.header />
151 <header ...headerProps>
152 // Instead of assigning to a variable with a capital letter,
153 // template interpolation works on tag names. This can also
154 // be a string to render the native HTML tag of that kind.
155 <${content} />
156 </header>
157 <hr />
158 </>
159
160 <${input.content} />
161</main>
162```
163
164The last syntax feature missing is calling a tag with parameters. That is done
165just like a regular function call, with '('.
166
167```
168<Something(item, index) />
169```
170
171In fact, attributes can just be sugar over this syntax; _this technically isn't
172true but it's close enough for the example_
173
174```
175<SpecialButton type="submit" class="red" />
176
177// is equal to
178
179<SpecialButton({ type: "submit", class: "red" }) />
180```
181
182All of the above is about how Marko's syntax works, and how it performs HTML
183generation with components. Marko also allows interactive components, but an
184explaination of that is beyond the scope of this page, mostly since I have not
185used it. A brief example of it, modified from their documentation.
186
187```marko
188// Reactive variables with <let/> just work...
189<let/basicCounter=0 />
190<button onClick() { basicCounter += 1 }>${basicCounter}</button>
191// ...but a counter is boring.
192
193<let/todos=[
194 { id: 0, text: "Learn Marko" },
195 { id: 1, text: "Make a Website" },
196]/>
197
198// 'by' is like React JSX's "key" property, but it's optional.
199<ul><for|todo, i| of=todos by=(todo => todo.id)>
200 <li.todo>
201 // this variable remains stable even if the list
202 // re-orders, because 'by' was specified.
203 <let/done=false/>
204 <label>
205 <span>${todo.text}</span>
206 // ':=' creates a two-way reactive binding,
207 // (it passes a callback for `checkedChanged`)
208 <input type="checkbox" checked:=done />
209 </label>
210 <button
211 title="delete"
212 disabled=!done
213 onClick() {
214 todos = todos.toSpliced(i, 1);
215 }
216 > &times; </button>
217 </li>
218</></ul>
219
220// Form example
221<let/nextId=2/>
222<form onSubmit(e) {
223 e.preventDefault();
224 todos = todos.concat({
225 id: nextId++,
226 // HTMLFormElement exposes all its named input
227 // elements as extra properties on the object.
228 text: e.target.text.value,
229 });
230 // And you can clear it with 'reset()'
231 e.target.reset();
232}>
233 // We don't 'onChange' like a React loser. The form
234 // value can be read in the submit event like normal.
235 <input name="text" placeholder="Another Item">
236 <button type="submit">Add</button>
237</form>
238```
239
240## Usage on `paperclover.net`
241
242TODO: document a lot of feedback, how i embedded Marko
243
244My website uses statically generated HTML. That is why I have not needed to use
245reactive variables. My generator doesn't even try compiling components
246client-side.
247
248Here is the actual component used to render [questions on the clover q+a][/q+a].
249
250```marko
251// Renders a `Question` entry including its markdown body.
252export interface Input {
253 question: Question;
254 admin?: boolean;
255}
256
257// 2024-12-31 05:00:00 EST
258export const transitionDate = 1735639200000;
259
260<const/{ question, admin } = input />
261<const/{ id, date, text } = question/>
262
263<${"e-"}
264 f=(date > transitionDate ? true : undefined)
265 id=admin ? `q${id}` : undefined
266>
267 <if=admin>
268 <a
269 style="margin-right: 0.5rem"
270 href=`/admin/q+a/${id}`
271 >[EDIT]</a>
272 </>
273 <a>
274 <time
275 datetime=formatQuestionISOTimestamp(date)
276 >${formatQuestionTimestamp(date)}</time>
277 </a>
278
279 <CloverMarkdown ...{ text } />
280</>
281
282// this singleton script will make all the '<time>' tags clickable.
283client import "./clickable-links.client.ts";
284
285import type { Question } from "@/q+a/models/Question.ts";
286import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";
287import { CloverMarkdown } from "@/q+a/clover-markdown.tsx";
288```
289
290import { type BlogMeta, formatBlogMeta } from '@/blog/helpers.ts';
src/blog/pages/webdev/deranged-typescript.md created+87
...@@ -0,0 +1,87 @@
1- imports at the bottom
2 - order your file by importance.
3 - 'G' to jump to imports, etc
4 - prefer namespace imports
5 - easier to type and refactor. easier to read.
6- large files are okay
7 - all files are their own library
8 - split files up by making components modular, not by "oh it's too big"
9 - engine/render.ts is a standalone library, in order to split JSX, Suspense,
10 and Marko out, the main file was made modular.
11- lowercase
12- name objects ultra-concisely
13 - filenames are often one word describing what they contain
14 - avoid useless descriptors like "utils", "helpers", and "data"
15 - examples
16 - async.ts contains all the async library functions.
17 - watch.ts contains the file watcher and watch-reload mode.
18 - render.*, Io
19- be ultra-concise in comments
20- no "discarded" variables, embrace `void x`
21 - makes code more readable
22 - note how i want to write a lint for this
23 - note the one proposal i want about void
24- push the ts inference engine (as const, ReturnType, etc)
25 - reduces how much you repeat yourself making it easier to refactor things
26 - use the code as the source of truth
27- push the ts inference engine (generics)
28 - do not implement crazy things with the TS engine, instead use generic input
29 types, and then use regular control to narrow and transform the return type.
30 source of truth is your code.
31- UNWRAP, ASSERT utility globals are amazing
32 - ban postfix '!'
33 - stripped for production frontend builds
34- destructure often
35 - use the one example from work lol
36- package.json "imports" are amazing
37 - remapping
38 - implementation switching
39 - testing
40- embrace the web and node.js APIs
41 - sitegen relies on so many node features that bun and deno fail to run it.
42 - overlay modules are great
43- avoid dependencies
44 - once you build your own mini standard library you win
45 - talk about regrets with mdx
46
47## imports at the bottom
48
49Here is an abridged version of my website's `backend.ts`. When reading it from
50top to bottom it is immediately obvious that it is a Hono web server.
51
52```ts
53// This is the main file for paperclover.net's server.
54const app = new Hono();
55const logHttp = console.scoped("http", { color: "magenta" });
56
57// Middleware
58app.use(...);
59...
60
61// Backends
62app.route("", require("./q+a/backend.ts").app);
63...
64
65export default app;
66
67...
68
69import { type Context, Hono, type Next } from "#hono";
70import { logger } from "hono/logger";
71import { trimTrailingSlash } from "hono/trailing-slash";
72import * as assets from "#sitegen/assets";
73import * as admin from "./admin.ts";
74import * as console from "@paperclover/console";
75```
76
77Since `import`/`export` statements are hoisted like `var` and `function`, the
78position of these statements within the file does not matter. The imported
79modules have to be loaded first before this file can start. With this, I've
80found it nicer to sort the file by _importance_ rather than by arbitrary rules
81dictated by how C-style `#include`s worked.
82
83Start with a documentation comment, then the most important
84functions/variables/types, sort the file by importance. Imports are not really
85important since you very quickly get to know where common namespaces come from.
86And since they're at the bottom, you can just press `G` in Vim or `CMD+Down` on
87the Mac to scroll to the end of the file.
src/blog/pages/webdev/marko-intro.markodown created+339
...@@ -0,0 +1,339 @@
1export const blog: BlogMeta = {
2 title: "Marko is the coziest HTML templating language",
3 desc: "...todo...",
4 created: "2025-06-13",
5 draft: true,
6};
7export const meta = formatBlogMeta(blob);
8export * as layout from "@/blog/layout.tsx";
9
10I've been recently playing around [Marko], and after adding limited support
11for it in my website generator, [sitegen], I instantly fell in love with how
12minimalistic it is in comparison to JSX, Astro components, and Svelte.
13
14[Marko]: https://next.markojs.com
15[sitegen]: https://paperclover.dev/clo/sitegen
16
17## Introduction to Marko
18
19If JSX was taking HTML and shoving its syntax into JavaScript, Marko is shoving
20JavaScript into HTML. Attributes are JavaScript expressions.
21
22```marko
23<div>
24 // `input` is like props, but given in the top-level scope
25 <time datetime=input.date.toISOString()>
26 // Interpolation with JS template string syntax
27 ${formatTimeNicely(input.date)}
28 </time>
29 <div>
30 <a href=`/users/${input.user.id}`>${input.user.name}</a>
31 </div>
32
33 // Capital letter variables for imported components
34 <MarkdownContent message=input.message />
35
36 // Components also can be auto-imported by lowercase.
37 // This will look upwards for a `tags/` folder containing
38 // "custom-footer.marko", similar to how Node.js finds
39 // package names in all upwards `node_modules` folders.
40 <custom-footer />
41</div>
42
43// ESM `import` / `export` just work as expected.
44// I prefer my imports at the end, to highlight the markup.
45import MarkdownContent from "./MarkdownContent.marko";
46import { formatTimeNicely } from "../date-helpers.ts";
47```
48
49Tags with the `value` attribute have a shorthand, which is used by the built-in
50`<if>` for conditional rendering.
51
52```marko
53// Sugar for <input value="string" />
54<input="string" />
55
56// and it composes amazingly to the 'if' built-in
57<if=input.user>
58 <UserProfile=input.user />
59</if>
60```
61
62Tags can also return values into the scope for use in the template using `/`, such as `<id>` for unique ID generation. This is available to components that `<return=output/>`.
63
64```
65<id/uniqueId />
66
67<input id=uniqueId type="checkbox" name="allow_trans_rights" />
68<label for=uniqueId>click me!</>
69// ^ oh, you can also omit the
70// closing tag name if you want.
71```
72
73It's important that I started with the two forms of "Tag I/O": `=` for input
74and `/` for output. With those building blocks, we introduce local variables
75with `const`
76
77```
78<const/rendered = markdownToHtml(input.value) />
79
80// This is how you insert raw HTML to the document
81<inline-html=rendered />
82
83// It supports all of the cozy destructuring syntax JS has
84<const/{ id, name } = user />
85```
86
87Unlike JSX, when you pass content within a tag (`input.content` instead of
88JSX's `children`), instead of it being a JSX element, it is actually a
89function. This means that the `for` tag can render the content multiple times.
90
91```
92<ul>
93 <for from=1 to=10>
94 // Renders a new random number for each iteration.
95 <li>${Math.random()}</li>
96 </>
97</ul>
98```
99
100Since `content` is a function, it can take arguments. This is done with `|`
101
102```
103<h1>my friends</h1>
104<ul>
105 // I tend to omit the closing tag names for the built-in control
106 // flow tags, but I keep them for HTML tags. It's kinda like how
107 // in JavaScript you just write `}` to close your `if`s and loops.
108 //
109 // Anyways <for> also has 'of'
110 <for|item| of=user.friends>
111 <li class="friend">${item.name}</li>
112 </>
113
114 // They support the same syntax JavaScript function params allows,
115 // so you can have destructuring here too, and multiple params.
116 <for|{ name }, index| of=user.friends>
117 // By the way you can also use emmet-style class and ID shorthands.
118 <li.friend>My #${index + 1} friend is ${name}</li>
119 </>
120</ul>
121```
122
123Instead of named slots, Marko has attribute tags. These are more powerful than
124slots since they are functions, and can also act as sugar for more complicated
125attributes.
126
127```
128<Layout title="Welcome">
129 <@header variant="big">
130 <h1>the next big thing</h1>
131 </@header>
132
133 <p>body text...</p>
134</Layout>
135
136// The `input` variable inside of <Layout /> is:
137//
138// {
139// title: "Welcome",
140// header: {
141// content: /* function rendering "<h1>the next big thing</h1>" */,
142// variant: "big",
143// },
144// content: /* function rendering "<p>body text</p>" */
145// }
146```
147
148This layout could be implemented as such:
149
150```marko
151<main>
152 <if=input.header />
153 <const/{ ...headerProps, content }=input.header />
154 <header ...headerProps>
155 // Instead of assigning to a variable with a capital letter,
156 // template interpolation works on tag names. This can also
157 // be a string to render the native HTML tag of that kind.
158 <${content} />
159 </header>
160 <hr />
161 </>
162
163 <${input.content} />
164</main>
165```
166
167The last syntax feature missing is calling a tag with parameters. That is done
168just like a regular function call, with '('.
169
170```
171<Something(item, index) />
172```
173
174In fact, attributes can just be sugar over this syntax. (this technically isn't
175true but it's close enough for the example)
176
177```
178<SpecialButton type="submit" class="red" />
179
180// is equal to
181
182<SpecialButton({ type: "submit", class: "red" }) />
183```
184
185All of the above is about how Marko's syntax works, and how it performs HTML
186generation with components. Marko also allows interactive components, but an
187explaination of that is beyond the scope of this page, mostly since I have not
188used it. A brief example of it, modified from their documentation.
189
190```marko
191// Reactive variables with <let/> just work...
192<let/basicCounter=0 />
193<button onClick() { basicCounter += 1 }>${basicCounter}</button>
194// ...but a counter is boring.
195
196<let/todos=[
197 { id: 0, text: "Learn Marko" },
198 { id: 1, text: "Make a Website" },
199]/>
200
201// 'by' is like React JSX's "key" property, but it's optional.
202<ul><for|todo, i| of=todos by=(todo => todo.id)>
203 <li.todo>
204 // this variable remains stable even if the list
205 // re-orders, because 'by' was specified.
206 <let/done=false/>
207 <label>
208 <span>${todo.text}</span>
209 // ':=' creates a two-way reactive binding,
210 // (it passes a callback for `checkedChanged`)
211 <input type="checkbox" checked:=done />
212 </label>
213 <button
214 title="delete"
215 disabled=!done
216 onClick() {
217 todos = todos.toSpliced(i, 1);
218 }
219 > &times; </button>
220 </li>
221</></ul>
222
223// Form example
224<let/nextId=2/>
225<form onSubmit(e) {
226 e.preventDefault();
227 todos = todos.concat({
228 id: nextId++,
229 // HTMLFormElement exposes all its named input
230 // elements as extra properties on the object.
231 text: e.target.text.value,
232 });
233 // And you can clear it with 'reset()'
234 e.target.reset();
235}>
236 // We don't 'onChange' like a React loser. The form
237 // value can be read in the submit event like normal.
238 <input name="text" placeholder="Another Item">
239 <button type="submit">Add</button>
240</form>
241```
242
243<SectionHeader updated="2025-08-11">Usage on `paperclover.net`</Section>
244
245Using Marko for HTML generation is quite easy. `.marko` files can be compiled
246into `.js` using the `@marko/compiler` library.
247
248```ts
249const src = fs.readFileSync("page.marko", "utf8");
250const compile = marko.compileSync(src, filepath);
251fs.writeFileSync("page.js", compile.code);
252
253const page = require("./page.js");
254console.info(page);
255
256import * as fs from "node:fs";
257import * as marko from "@marko/compiler";
258```
259
260To get client side JavaScript, an option can be passed to the Marko compiler to
261generate the client side code. While it is a big selling point of Marko, I do
262not use any of their client side features, instead deferring to manually-written
263frontend scripts. This is because that is how my website has been for years,
264statically generated. And for websites like mine that are content focused, this
265is the correct way to do things.
266
267Since I have a custom HTML generation library (built on JSX and some React-like
268patterns), I have written a simple integration for it to utilize Marko
269components, which is loaded by replacing the generated import to `marko/html`,
270which lets me overwrite functions like `createTemplate` (to change the signature
271of a component), `dynamicTag` (to allow Marko to render non-Marko components),
272and `fork` (to enable async integration with the rendering framework). An
273additional feature of this is I have a Node.js loader hook to allow importing
274these files directly.
275
276```tsx
277function Page() {
278 const q = Question.getByDate(new Date("2025-06-07 12:12 EST"));
279 return <div>
280 <h1>example question</h1>
281 <QuestionRender question={q} />
282 </div>;
283}
284
285// The synchronous render can be used because `Page` and `question.marko`
286// do not await any promises (SQLite runs synchronously)
287console.info(render.sync(<Page />).text);
288
289import * as render from "#engine/render";
290import QuestionRender from "@/q+a/tags/question.marko";
291import { Question } from "@/q+a/models/Question.ts";
292```
293
294Here is the `question.marko` tag used to render [questions on the clover q+a](/q+a).
295
296```marko
297// Renders a `Question` entry including its markdown body.
298export interface Input {
299 question: Question;
300 admin?: boolean;
301}
302
303// 2024-12-31 05:00:00 EST
304export const transitionDate = 1735639200000;
305
306<const/{ question, admin } = input />
307<const/{ id, date, text } = question/>
308
309<${"e-"}
310 f=(date > transitionDate ? true : undefined)
311 id=admin ? `q${id}` : undefined
312>
313 <if=admin>
314 <a
315 style="margin-right: 0.5rem"
316 href=`/admin/q+a/${id}`
317 >[EDIT]</a>
318 </>
319 <a>
320 <time
321 datetime=formatQuestionISOTimestamp(date)
322 >${formatQuestionTimestamp(date)}</time>
323 </a>
324
325 <CloverMarkdown ...{ text } />
326</>
327
328// this singleton script will make all the '<time>' tags clickable.
329client import "./clickable-links.client.ts";
330
331import type { Question } from "@/q+a/models/Question.ts";
332import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";
333import { CloverMarkdown } from "@/q+a/clover-markdown.tsx";
334```
335
336The integration is great, `client import` is quite a magical concept, and I've
337tuned it to do the expected thing in my framework.
338
339import { type BlogMeta, formatBlogMeta } from '@/blog/helpers.ts';
src/file-viewer/backend.tsx+11-13
...@@ -93,7 +93,7 @@ app.get("/file/*", async (c, next) => {...@@ -93,7 +93,7 @@ app.get("/file/*", async (c, next) => {
93 } satisfies APIDirectoryList;93 } satisfies APIDirectoryList;
94 return c.json(json);94 return c.json(json);
95 }95 }
96 c.res = await renderView(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {96 c.res = await view.serve(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {
97 file,97 file,
98 hasCotyledonCookie,98 hasCotyledonCookie,
99 });99 });
...@@ -111,7 +111,7 @@ app.get("/file/*", async (c, next) => {...@@ -111,7 +111,7 @@ app.get("/file/*", async (c, next) => {
111 !lofi111 !lofi
112 ) {112 ) {
113 prefetchFile(file.path);113 prefetchFile(file.path);
114 c.res = await renderView(c, "file-viewer/clofi", {114 c.res = await view.serve(c, "file-viewer/clofi", {
115 file,115 file,
116 hasCotyledonCookie,116 hasCotyledonCookie,
117 });117 });
...@@ -125,11 +125,10 @@ app.get("/file/*", async (c, next) => {...@@ -125,11 +125,10 @@ app.get("/file/*", async (c, next) => {
125125
126 let encoding = decideEncoding(c.req.header("Accept-Encoding"));126 let encoding = decideEncoding(c.req.header("Accept-Encoding"));
127127
128 let sizeHeader =128 let sizeHeader = encoding === "raw"
129 encoding === "raw"129 ? expectedSize
130 ? expectedSize130 // Size cannot be known because of compression modes
131 : // Size cannot be known because of compression modes131 : undefined;
132 undefined;
133132
134 // Etag133 // Etag
135 {134 {
...@@ -217,7 +216,7 @@ app.get("/canvas/:script", async (c, next) => {...@@ -217,7 +216,7 @@ app.get("/canvas/:script", async (c, next) => {
217 if (!hasAsset(`/js/canvas/${script}.js`)) {216 if (!hasAsset(`/js/canvas/${script}.js`)) {
218 return next();217 return next();
219 }218 }
220 return renderView(c, "file-viewer/canvas", {219 return view.serve(c, "file-viewer/canvas", {
221 script,220 script,
222 });221 });
223});222});
...@@ -295,10 +294,9 @@ function handleRanges(...@@ -295,10 +294,9 @@ function handleRanges(
295): Response {294): Response {
296 // TODO: multiple ranges295 // TODO: multiple ranges
297 const rangeSize = ranges.reduce((a, b) => a + (b[1] - b[0] + 1), 0);296 const rangeSize = ranges.reduce((a, b) => a + (b[1] - b[0] + 1), 0);
298 const rangeBody =297 const rangeBody = streamOrBuffer instanceof ReadableStream
299 streamOrBuffer instanceof ReadableStream298 ? applySingleRangeToStream(streamOrBuffer, ranges)
300 ? applySingleRangeToStream(streamOrBuffer, ranges)299 : applyRangesToBuffer(streamOrBuffer, ranges, rangeSize);
301 : applyRangesToBuffer(streamOrBuffer, ranges, rangeSize);
302 return new Response(rangeBody, {300 return new Response(rangeBody, {
303 status: 206,301 status: 206,
304 headers: {302 headers: {
...@@ -420,7 +418,7 @@ import { type Context, Hono } from "hono";...@@ -420,7 +418,7 @@ import { type Context, Hono } from "hono";
420418
421import * as render from "#engine/render";419import * as render from "#engine/render";
422import { etagMatches, hasAsset, serveAsset } from "#sitegen/assets";420import { etagMatches, hasAsset, serveAsset } from "#sitegen/assets";
423import { renderView } from "#sitegen/view";421import * as view from "#sitegen/view";
424import { contentTypeFor } from "#sitegen/mime";422import { contentTypeFor } from "#sitegen/mime";
425423
426import { requireFriendAuth } from "@/friend-auth.ts";424import { requireFriendAuth } from "@/friend-auth.ts";
src/file-viewer/bin/scan3.ts+20-14
...@@ -17,9 +17,11 @@ export async function main() {...@@ -17,9 +17,11 @@ export async function main() {
17 const start = performance.now();17 const start = performance.now();
18 const timerSpinner = new Spinner({18 const timerSpinner = new Spinner({
19 text: () =>19 text: () =>
20 `paper clover's scan3 [${((performance.now() - start) / 1000).toFixed(20 `paper clover's scan3 [${
21 1,21 ((performance.now() - start) / 1000).toFixed(
22 )}s]`,22 1,
23 )
24 }s]`,
23 fps: 10,25 fps: 10,
24 });26 });
25 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };27 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
...@@ -44,7 +46,7 @@ export async function main() {...@@ -44,7 +46,7 @@ export async function main() {
44 .flatMap((child) =>46 .flatMap((child) =>
45 child.kind === MediaFileKind.directory47 child.kind === MediaFileKind.directory
46 ? child.getRecursiveFileChildren()48 ? child.getRecursiveFileChildren()
47 : child,49 : child
48 );50 );
4951
50 qMeta.addMany(52 qMeta.addMany(
...@@ -103,9 +105,11 @@ export async function main() {...@@ -103,9 +105,11 @@ export async function main() {
103 ) {105 ) {
104 date = mediaFile.date;106 date = mediaFile.date;
105 console.warn(107 console.warn(
106 `M-time on ${publicPath} was likely corrupted. ${formatDate(108 `M-time on ${publicPath} was likely corrupted. ${
107 mediaFile.date,109 formatDate(
108 )} -> ${formatDate(stat.mtime)}`,110 mediaFile.date,
111 )
112 } -> ${formatDate(stat.mtime)}`,
109 );113 );
110 }114 }
111 mediaFile = MediaFile.createFile({115 mediaFile = MediaFile.createFile({
...@@ -162,7 +166,7 @@ export async function main() {...@@ -162,7 +166,7 @@ export async function main() {
162 }: Omit<ProcessFileArgs, "spin">) {166 }: Omit<ProcessFileArgs, "spin">) {
163 const ext = mediaFile.extensionNonEmpty.toLowerCase();167 const ext = mediaFile.extensionNonEmpty.toLowerCase();
164 let possible = processors.filter((p) =>168 let possible = processors.filter((p) =>
165 p.include ? p.include.has(ext) : !p.exclude?.has(ext),169 p.include ? p.include.has(ext) : !p.exclude?.has(ext)
166 );170 );
167 if (possible.length === 0) return;171 if (possible.length === 0) return;
168172
...@@ -196,7 +200,7 @@ export async function main() {...@@ -196,7 +200,7 @@ export async function main() {
196 );200 );
197 } else {201 } else {
198 possible = order.map(({ id }) =>202 possible = order.map(({ id }) =>
199 UNWRAP(possible.find((p) => p.id === id)),203 UNWRAP(possible.find((p) => p.id === id))
200 );204 );
201 }205 }
202206
...@@ -260,8 +264,9 @@ export async function main() {...@@ -260,8 +264,9 @@ export async function main() {
260 const children = dir.getChildren();264 const children = dir.getChildren();
261265
262 // readme.txt266 // readme.txt
263 const readmeContent =267 const readmeContent = children.find((x) =>
264 children.find((x) => x.basename === "readme.txt")?.contents ?? "";268 x.basename === "readme.txt"
269 )?.contents ?? "";
265270
266 // dirsort271 // dirsort
267 let dirsort: string[] | null = null;272 let dirsort: string[] | null = null;
...@@ -354,7 +359,9 @@ export async function main() {...@@ -354,7 +359,9 @@ export async function main() {
354 console.info(359 console.info(
355 "Global Stats:\n" +360 "Global Stats:\n" +
356 `- File Count: \x1b[1m${count}\x1b[0m\n` +361 `- File Count: \x1b[1m${count}\x1b[0m\n` +
357 `- Canonical Size: \x1b[1m${formatSize(MediaFile.getByPath("/")!.size)}\x1b[0m\n` +362 `- Canonical Size: \x1b[1m${
363 formatSize(MediaFile.getByPath("/")!.size)
364 }\x1b[0m\n` +
358 `- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n`,365 `- Media Duration: \x1b[1m${formatDurationLong(duration)}\x1b[0m\n`,
359 );366 );
360}367}
...@@ -459,7 +466,6 @@ const procLoadTextContents: Process = {...@@ -459,7 +466,6 @@ const procLoadTextContents: Process = {
459 async run({ absPath, mediaFile, stat }) {466 async run({ absPath, mediaFile, stat }) {
460 if (stat.size > 1_000_000) return;467 if (stat.size > 1_000_000) return;
461 const text = await fs.readFile(absPath, "utf-8");468 const text = await fs.readFile(absPath, "utf-8");
462 console.log({ text });
463 mediaFile.setContents(text);469 mediaFile.setContents(text);
464 },470 },
465};471};
...@@ -696,7 +702,7 @@ async function unproduceAsset(key: string) {...@@ -696,7 +702,7 @@ async function unproduceAsset(key: string) {
696 const ref = AssetRef.get(key);702 const ref = AssetRef.get(key);
697 if (ref) {703 if (ref) {
698 ref.unref();704 ref.unref();
699 console.log(`unref ${key}`);705 console.warn(`TODO: unref ${key}`);
700 // TODO: remove associated files from target706 // TODO: remove associated files from target
701 }707 }
702}708}
src/file-viewer/format.ts+34-20
...@@ -29,7 +29,9 @@ export function formatDurationLong(seconds: number) {...@@ -29,7 +29,9 @@ export function formatDurationLong(seconds: number) {
29 const hours = Math.floor(seconds / 3600);29 const hours = Math.floor(seconds / 3600);
30 const minutes = Math.floor((seconds % 3600) / 60);30 const minutes = Math.floor((seconds % 3600) / 60);
31 const remainingSeconds = seconds % 60;31 const remainingSeconds = seconds % 60;
32 return `${hours}:${minutes.toString().padStart(2, "0")}:${remainingSeconds.toString().padStart(2, "0")}`;32 return `${hours}:${minutes.toString().padStart(2, "0")}:${
33 remainingSeconds.toString().padStart(2, "0")
34 }`;
33}35}
3436
35export function escapeUri(uri: string) {37export function escapeUri(uri: string) {
...@@ -100,21 +102,27 @@ export function highlightLinksInTextView(...@@ -100,21 +102,27 @@ export function highlightLinksInTextView(
100 // Case 1: https:// or http:// URLs102 // Case 1: https:// or http:// URLs
101 if (match.startsWith("http")) {103 if (match.startsWith("http")) {
102 if (match.includes(findDomain)) {104 if (match.includes(findDomain)) {
103 return `<a href="${match105 return `<a href="${
104 .replace(/https?:\/\/paperclover\.net\/+/, "/")106 match
105 .replace(/\/\/+/g, "/")}">${match}</a>`;107 .replace(/https?:\/\/paperclover\.net\/+/, "/")
108 .replace(/\/\/+/g, "/")
109 }">${match}</a>`;
106 }110 }
107 return `<a href="${match.replace(111 return `<a href="${
108 /\/\/+/g,112 match.replace(
109 "/",113 /\/\/+/g,
110 )}" target="_blank" rel="noopener noreferrer">${match}</a>`;114 "/",
115 )
116 }" target="_blank" rel="noopener noreferrer">${match}</a>`;
111 }117 }
112118
113 // Case 2: domain URLs without protocol119 // Case 2: domain URLs without protocol
114 if (match.startsWith(findDomain)) {120 if (match.startsWith(findDomain)) {
115 return `<a href="${match121 return `<a href="${
116 .replace(findDomain + "/", "/")122 match
117 .replace(/\/\/+/g, "/")}">${match}</a>`;123 .replace(findDomain + "/", "/")
124 .replace(/\/\/+/g, "/")
125 }">${match}</a>`;
118 }126 }
119127
120 // Case 3: /file/ URLs128 // Case 3: /file/ URLs
...@@ -145,7 +153,7 @@ export function highlightLinksInTextView(...@@ -145,7 +153,7 @@ export function highlightLinksInTextView(
145 // Match sibling file names (only if they're not already part of a link)153 // Match sibling file names (only if they're not already part of a link)
146 if (siblingFiles.length > 0) {154 if (siblingFiles.length > 0) {
147 const escapedBasenames = siblingFiles.map((f) =>155 const escapedBasenames = siblingFiles.map((f) =>
148 f.basename.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),156 f.basename.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
149 );157 );
150 const pattern = new RegExp(`\\b(${escapedBasenames.join("|")})\\b`, "g");158 const pattern = new RegExp(`\\b(${escapedBasenames.join("|")})\\b`, "g");
151 const parts = processedText.split(/(<[^>]*>)/);159 const parts = processedText.split(/(<[^>]*>)/);
...@@ -155,9 +163,11 @@ export function highlightLinksInTextView(...@@ -155,9 +163,11 @@ export function highlightLinksInTextView(
155 parts[i] = parts[i].replace(pattern, (match: string) => {163 parts[i] = parts[i].replace(pattern, (match: string) => {
156 const file = siblingLookup[match];164 const file = siblingLookup[match];
157 if (file) {165 if (file) {
158 return `<a href="/file/${file.path166 return `<a href="/file/${
159 .replace(/^\//, "")167 file.path
160 .replace(/\/\/+/g, "/")}">${match}</a>`;168 .replace(/^\//, "")
169 .replace(/\/\/+/g, "/")
170 }">${match}</a>`;
161 }171 }
162 return match;172 return match;
163 });173 });
...@@ -240,9 +250,11 @@ export function highlightConvo(text: string) {...@@ -240,9 +250,11 @@ export function highlightConvo(text: string) {
240250
241 return paras251 return paras
242 .map(({ speaker, lines }) => {252 .map(({ speaker, lines }) => {
243 return `<div class="s-${speaker}">${lines253 return `<div class="s-${speaker}">${
244 .map((line) => `<div class="line">${line}</div>`)254 lines
245 .join("\n")}</div>`;255 .map((line) => `<div class="line">${line}</div>`)
256 .join("\n")
257 }</div>`;
246 })258 })
247 .join("\n");259 .join("\n");
248}260}
...@@ -267,10 +279,12 @@ export function formatDate(dateTime: Date) {...@@ -267,10 +279,12 @@ export function formatDate(dateTime: Date) {
267 ? dateTime < unknownDate279 ? dateTime < unknownDate
268 ? "??.??.??"280 ? "??.??.??"
269 : `xx.xx.${21 + Math.floor(dateTime.getTime() / 86400000)}`281 : `xx.xx.${21 + Math.floor(dateTime.getTime() / 86400000)}`
270 : `${(dateTime.getMonth() + 1).toString().padStart(2, "0")}.${dateTime282 : `${(dateTime.getMonth() + 1).toString().padStart(2, "0")}.${
283 dateTime
271 .getDate()284 .getDate()
272 .toString()285 .toString()
273 .padStart(2, "0")}.${dateTime.getFullYear().toString().slice(2)}`;286 .padStart(2, "0")
287 }.${dateTime.getFullYear().toString().slice(2)}`;
274}288}
275289
276import type { MediaFile } from "@/file-viewer/models/MediaFile.ts";290import type { MediaFile } from "@/file-viewer/models/MediaFile.ts";
src/file-viewer/rsync.ts+6-2
...@@ -131,8 +131,12 @@ export class Parse {...@@ -131,8 +131,12 @@ export class Parse {
131 percentage: Number(percentageStr),131 percentage: Number(percentageStr),
132 timeElapsed,132 timeElapsed,
133 transferNumber: this.currentTransfer,133 transferNumber: this.currentTransfer,
134 filesToCheck: toCheckStr ? this.toCheck = Number(toCheckStr) : this.toCheck,134 filesToCheck: toCheckStr
135 totalFiles: totalStr ? this.totalFiles = Number(totalStr) : this.totalFiles,135 ? this.toCheck = Number(toCheckStr)
136 : this.toCheck,
137 totalFiles: totalStr
138 ? this.totalFiles = Number(totalStr)
139 : this.totalFiles,
136 speed: speed || null,140 speed: speed || null,
137 };141 };
138 }142 }
src/global.css+15-11
...@@ -4,9 +4,7 @@...@@ -4,9 +4,7 @@
4 font-weight: 400 750;4 font-weight: 400 750;
5 font-style: normal;5 font-style: normal;
6 font-display: swap;6 font-display: swap;
7 font-variation-settings:7 font-variation-settings: "CASL" 0.25, "MONO" 0;
8 "CASL" 0.25,
9 "MONO" 0;
10 font-style: oblique -15deg 0deg;8 font-style: oblique -15deg 0deg;
11 unicode-range: U+0020-007E;9 unicode-range: U+0020-007E;
12}10}
...@@ -16,9 +14,7 @@...@@ -16,9 +14,7 @@
16 font-weight: 400 800;14 font-weight: 400 800;
17 font-style: normal;15 font-style: normal;
18 font-display: swap;16 font-display: swap;
19 font-variation-settings:17 font-variation-settings: "CASL" 0.25, "MONO" 1;
20 "CASL" 0.25,
21 "MONO" 1;
22 font-style: oblique -15deg 0deg;18 font-style: oblique -15deg 0deg;
23 unicode-range: U+0020-007E;19 unicode-range: U+0020-007E;
24}20}
...@@ -28,13 +24,21 @@...@@ -28,13 +24,21 @@
28 font-weight: 400 800;24 font-weight: 400 800;
29 font-style: normal;25 font-style: normal;
30 font-display: swap;26 font-display: swap;
31 font-variation-settings:27 font-variation-settings: "CASL" 0.25, "MONO" 1;
32 "CASL" 0.25,
33 "MONO" 1;
34 font-style: oblique -15deg 0deg;28 font-style: oblique -15deg 0deg;
35 unicode-range:29 unicode-range:
36 U+00C0-00FF, U+00A9, U+2190-2193, U+2018, U+2019, U+201C, U+201D, U+2022,30 U+00C0-00FF,
37 U+00A0-00A8, U+00AA-00BF, U+2194-2199, U+0100-017F;31 U+00A9,
32 U+2190-2193,
33 U+2018,
34 U+2019,
35 U+201C,
36 U+201D,
37 U+2022,
38 U+00A0-00A8,
39 U+00AA-00BF,
40 U+2194-2199,
41 U+0100-017F;
38}42}
3943
40*,44*,
src/q+a/backend.ts+20-22
...@@ -45,7 +45,7 @@ app.post("/q+a", async (c) => {...@@ -45,7 +45,7 @@ app.post("/q+a", async (c) => {
45 return sendSuccess(c, new Date());45 return sendSuccess(c, new Date());
46 }46 }
4747
48 const ipAddr = c.req.header("cf-connecting-ip");48 const ipAddr = c.req.header("x-forwarded-for");
49 if (ipAddr) {49 if (ipAddr) {
50 input.sourceName = uniqueNamesGenerator({50 input.sourceName = uniqueNamesGenerator({
51 dictionaries: [adjectives, colors, animals],51 dictionaries: [adjectives, colors, animals],
...@@ -54,11 +54,6 @@ app.post("/q+a", async (c) => {...@@ -54,11 +54,6 @@ app.post("/q+a", async (c) => {
54 });54 });
55 }55 }
5656
57 const cfIPCountry = c.req.header("cf-ipcountry");
58 if (cfIPCountry) {
59 input.sourceLocation = cfIPCountry;
60 }
61
62 if (ipAddr && PROXYCHECK_API_KEY) {57 if (ipAddr && PROXYCHECK_API_KEY) {
63 const proxyCheck = await fetch(58 const proxyCheck = await fetch(
64 `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,59 `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,
...@@ -77,17 +72,19 @@ app.post("/q+a", async (c) => {...@@ -77,17 +72,19 @@ app.post("/q+a", async (c) => {
77 proxyCheck[ipAddr].organisation ??72 proxyCheck[ipAddr].organisation ??
78 proxyCheck[ipAddr].provider ?? "unknown";73 proxyCheck[ipAddr].provider ?? "unknown";
79 }74 }
80 if (Number(proxyCheck[ipAddr].risk) > 72) {75 if (Number(proxyCheck[ipAddr].risk) > 78) {
81 return questionFailure(76 return questionFailure(
82 c,77 c,
83 403,78 403,
84 "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.",79 "This IP address has been flagged as a high risk IP address. If " +
80 "you are using a VPN/Proxy, please disable it and try again.",
85 text,81 text,
86 );82 );
87 }83 }
88 }84 }
89 }85 }
9086
87 view.regenerate("q+a inbox");
91 const date = Question.create(88 const date = Question.create(
92 QuestionType.pending,89 QuestionType.pending,
93 JSON.stringify(input),90 JSON.stringify(input),
...@@ -104,7 +101,7 @@ async function sendSuccess(c: Context, date: Date) {...@@ -104,7 +101,7 @@ async function sendSuccess(c: Context, date: Date) {
104 id: formatQuestionId(date),101 id: formatQuestionId(date),
105 }, { status: 200 });102 }, { status: 200 });
106 }103 }
107 c.res = await renderView(c, "q+a/success", {104 c.res = await view.serve(c, "q+a/success", {
108 permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,105 permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,
109 });106 });
110}107}
...@@ -130,7 +127,7 @@ app.get("/q+a/:id", async (c, next) => {...@@ -130,7 +127,7 @@ app.get("/q+a/:id", async (c, next) => {
130 if (image) {127 if (image) {
131 return getQuestionImage(question, c.req.method === "HEAD");128 return getQuestionImage(question, c.req.method === "HEAD");
132 }129 }
133 return renderView(c, "q+a/permalink", { question });130 return view.serve(c, "q+a/permalink", { question });
134});131});
135132
136// Admin133// Admin
...@@ -138,7 +135,7 @@ app.get("/admin/q+a", async (c) => {...@@ -138,7 +135,7 @@ app.get("/admin/q+a", async (c) => {
138 return serveAsset(c, "/admin/q+a", 200);135 return serveAsset(c, "/admin/q+a", 200);
139});136});
140app.get("/admin/q+a/inbox", async (c) => {137app.get("/admin/q+a/inbox", async (c) => {
141 return renderView(c, "q+a/backend-inbox", {});138 return view.serve(c, "q+a/backend-inbox", {});
142});139});
143app.delete("/admin/q+a/:id", async (c, next) => {140app.delete("/admin/q+a/:id", async (c, next) => {
144 const id = c.req.param("id");141 const id = c.req.param("id");
...@@ -152,6 +149,7 @@ app.delete("/admin/q+a/:id", async (c, next) => {...@@ -152,6 +149,7 @@ app.delete("/admin/q+a/:id", async (c, next) => {
152 } else {149 } else {
153 Question.rejectByQmid(question.qmid);150 Question.rejectByQmid(question.qmid);
154 }151 }
152 view.regenerate("q+a");
155 return c.json({ success: true, message: "ok" });153 return c.json({ success: true, message: "ok" });
156});154});
157app.patch("/admin/q+a/:id", async (c, next) => {155app.patch("/admin/q+a/:id", async (c, next) => {
...@@ -165,6 +163,7 @@ app.patch("/admin/q+a/:id", async (c, next) => {...@@ -165,6 +163,7 @@ app.patch("/admin/q+a/:id", async (c, next) => {
165 return questionFailure(c, 400, "Bad Request");163 return questionFailure(c, 400, "Bad Request");
166 }164 }
167 Question.updateByQmid(question.qmid, form.text, form.type);165 Question.updateByQmid(question.qmid, form.text, form.type);
166 view.regenerate("q+a");
168 return c.json({ success: true, message: "ok" });167 return c.json({ success: true, message: "ok" });
169});168});
170app.get("/admin/q+a/:id", async (c, next) => {169app.get("/admin/q+a/:id", async (c, next) => {
...@@ -177,20 +176,22 @@ app.get("/admin/q+a/:id", async (c, next) => {...@@ -177,20 +176,22 @@ app.get("/admin/q+a/:id", async (c, next) => {
177 let pendingInfo: null | PendingQuestionData = null;176 let pendingInfo: null | PendingQuestionData = null;
178 if (question.type === QuestionType.pending) {177 if (question.type === QuestionType.pending) {
179 pendingInfo = JSON.parse(question.text) as PendingQuestionData;178 pendingInfo = JSON.parse(question.text) as PendingQuestionData;
180 question.text = pendingInfo.prompt.trim().split("\n").map((line) =>179 question.text = pendingInfo.prompt
181 line.trim().length === 0 ? "" : `q: ${line.trim()}`180 .trim()
182 ).join("\n") + "\n\n";181 .split("\n")
182 .map((line) => (line.trim().length === 0 ? "" : `q: ${line.trim()}`))
183 .join("\n") + "\n\n";
183 question.type = QuestionType.normal;184 question.type = QuestionType.normal;
184 }185 }
185186
186 return renderView(c, "q+a/editor", {187 return view.serve(c, "q+a/editor", {
187 pendingInfo,188 pendingInfo,
188 question,189 question,
189 });190 });
190});191});
191192
192app.get("/q+a/things/random", async (c) => {193app.get("/q+a/things/random", async (c) => {
193 c.res = await renderView(c, "q+a/things-random", {});194 c.res = await view.serve(c, "q+a/things-random", {});
194});195});
195196
196async function questionFailure(197async function questionFailure(
...@@ -202,7 +203,7 @@ async function questionFailure(...@@ -202,7 +203,7 @@ async function questionFailure(
202 if (c.req.header("Accept")?.includes("application/json")) {203 if (c.req.header("Accept")?.includes("application/json")) {
203 return c.json({ success: false, message, id: null }, { status });204 return c.json({ success: false, message, id: null }, { status });
204 }205 }
205 return await renderView(c, "q+a/fail", {206 return await view.serve(c, "q+a/fail", {
206 error: message,207 error: message,
207 content,208 content,
208 });209 });
...@@ -218,11 +219,8 @@ import {...@@ -218,11 +219,8 @@ import {
218} from "unique-names-generator";219} from "unique-names-generator";
219import { hasAdminToken } from "../admin.ts";220import { hasAdminToken } from "../admin.ts";
220import { serveAsset } from "#sitegen/assets";221import { serveAsset } from "#sitegen/assets";
221import {222import type { PendingQuestionData } from "./models/PendingQuestion.ts";
222 PendingQuestion,
223 PendingQuestionData,
224} from "./models/PendingQuestion.ts";
225import { Question, QuestionType } from "./models/Question.ts";223import { Question, QuestionType } from "./models/Question.ts";
226import { renderView } from "#sitegen/view";224import * as view from "#sitegen/view";
227import { getQuestionImage } from "./image.tsx";225import { getQuestionImage } from "./image.tsx";
228import { formatQuestionId, questionIdToTimestamp } from "./format.ts";226import { formatQuestionId, questionIdToTimestamp } from "./format.ts";
src/q+a/clover-markdown.tsx+1-3
...@@ -144,9 +144,7 @@ function ListRenderer(node: ASTNode, children: any[]) {...@@ -144,9 +144,7 @@ function ListRenderer(node: ASTNode, children: any[]) {
144 const T = node.ordered ? "ol" : "ul";144 const T = node.ordered ? "ol" : "ul";
145 return (145 return (
146 <T>146 <T>
147 {children.map((child) => (147 {children.map((child) => <li>{child}</li>)}
148 <li>{child}</li>
149 ))}
150 </T>148 </T>
151 );149 );
152}150}
src/q+a/image.tsx+5-4
...@@ -11,7 +11,7 @@ const getBrowser = RefCountedExpirable(...@@ -11,7 +11,7 @@ const getBrowser = RefCountedExpirable(
11);11);
1212
13export async function renderQuestionImage(question: Question) {13export async function renderQuestionImage(question: Question) {
14 const html = await renderViewToString("q+a/image-embed", { question });14 const html = await view.renderToString("q+a/image-embed", { question });
1515
16 // this browser session will be reused if multiple images are generated16 // this browser session will be reused if multiple images are generated
17 // either at the same time or within a 5-minute time span. the dispose17 // either at the same time or within a 5-minute time span. the dispose
...@@ -44,14 +44,15 @@ export async function getQuestionImage(...@@ -44,14 +44,15 @@ export async function getQuestionImage(
44 question: Question,44 question: Question,
45 headOnly: boolean,45 headOnly: boolean,
46): Promise<Response> {46): Promise<Response> {
47 const hash = crypto.createHash("sha1")47 const hash = crypto
48 .createHash("sha1")
48 .update(question.qmid + question.type + question.text)49 .update(question.qmid + question.type + question.text)
49 .digest("hex");50 .digest("hex");
5051
51 const headers = {52 const headers = {
52 "Content-Type": "image/png",53 "Content-Type": "image/png",
53 "Cache-Control": "public, max-age=31536000",54 "Cache-Control": "public, max-age=31536000",
54 "ETag": `"${hash}"`,55 ETag: `"${hash}"`,
55 "Last-Modified": question.date.toUTCString(),56 "Last-Modified": question.date.toUTCString(),
56 };57 };
5758
...@@ -78,4 +79,4 @@ import * as path from "node:path";...@@ -78,4 +79,4 @@ import * as path from "node:path";
78import * as puppeteer from "puppeteer";79import * as puppeteer from "puppeteer";
79import { Question } from "@/q+a/models/Question.ts";80import { Question } from "@/q+a/models/Question.ts";
80import { RefCountedExpirable } from "#sitegen/async";81import { RefCountedExpirable } from "#sitegen/async";
81import { renderViewToString } from "#sitegen/view";82import * as view from "#sitegen/view";
src/q+a/pages/admin.q+a.marko+1-1
...@@ -7,7 +7,7 @@ export const meta: Metadata = {...@@ -7,7 +7,7 @@ export const meta: Metadata = {
7 description: "ask clover a question",7 description: "ask clover a question",
8};8};
9export const regenerate = {9export const regenerate = {
10 manual: true,10 tags: ["q+a", "q+a inbox"]
11};11};
1212
13<const/inboxSize = PendingQuestion.getAll().length />13<const/inboxSize = PendingQuestion.getAll().length />
src/q+a/pages/q+a.marko+4-6
...@@ -1,11 +1,9 @@...@@ -1,11 +1,9 @@
1export * as layout from "../layout.tsx";
2export const regenerate = {
3 manual: true,
4};
5
6export interface Input {1export interface Input {
7 admin?: boolean;2 admin?: boolean;
8}3}
4
5export * as layout from "../layout.tsx";
6export const regenerate = { tags: ["q+a"] };
9export const meta: Metadata = {7export const meta: Metadata = {
10 title: "paper clover q+a",8 title: "paper clover q+a",
11 description: "ask clover a question",9 description: "ask clover a question",
...@@ -14,7 +12,7 @@ export const meta: Metadata = {...@@ -14,7 +12,7 @@ export const meta: Metadata = {
14<const/{ admin = false } = input />12<const/{ admin = false } = input />
15<const/questions = [...Question.getAll()] />13<const/questions = [...Question.getAll()] />
1614
17<if=true>15<if=!admin>
18 <question-form />16 <question-form />
19</>17</>
20<for|question| of=questions>18<for|question| of=questions>
src/q+a/views/backend-inbox.marko+2-2
...@@ -19,8 +19,8 @@ export const meta = { title: 'question answer inbox' };...@@ -19,8 +19,8 @@ export const meta = { title: 'question answer inbox' };
19 </time>19 </time>
20 <div style="color: dodgerblue; margin-bottom: 0.25rem">20 <div style="color: dodgerblue; margin-bottom: 0.25rem">
21 ${q.sourceName}21 ${q.sourceName}
22 ${q.sourceLocation !== "unknown" ? `from ${q.sourceLocation}` : null}22 ${q.sourceLocation !== "unknown" ? `from ${q.sourceLocation}` : ''}
23 ${q.sourceVPN ? `(VPN: ${q.sourceVPN})` : null}23 ${q.sourceVPN ? `(VPN: ${q.sourceVPN})` : ''}
24 </div>24 </div>
25 <p style="white-space: pre-wrap">${q.prompt}</p>25 <p style="white-space: pre-wrap">${q.prompt}</p>
26 <p>26 <p>
src/source-of-truth.ts+6-2
...@@ -33,8 +33,12 @@ const rawFileRoot = process.env.CLOVER_FILE_RAW ??...@@ -33,8 +33,12 @@ const rawFileRoot = process.env.CLOVER_FILE_RAW ??
33const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??33const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
34 path.join(nasRoot, "Documents/Config/paperclover/derived");34 path.join(nasRoot, "Documents/Config/paperclover/derived");
3535
36if (!fs.existsSync(rawFileRoot)) throw new Error(`${rawFileRoot} does not exist`);36if (!fs.existsSync(rawFileRoot)) {
37if (!fs.existsSync(derivedFileRoot)) throw new Error(`${derivedFileRoot} does not exist`);37 throw new Error(`${rawFileRoot} does not exist`);
38}
39if (!fs.existsSync(derivedFileRoot)) {
40 throw new Error(`${derivedFileRoot} does not exist`);
41}
3842
39type Awaitable<T> = T | Promise<T>;43type Awaitable<T> = T | Promise<T>;
4044
tsconfig.json+3-2
...@@ -5,7 +5,7 @@...@@ -5,7 +5,7 @@
5 "incremental": true,5 "incremental": true,
6 "jsx": "react-jsxdev",6 "jsx": "react-jsxdev",
7 "jsxImportSource": "#engine",7 "jsxImportSource": "#engine",
8 "lib": ["dom", "esnext", "esnext.iterator"],8 "lib": ["esnext", "dom", "dom.iterable"],
9 "module": "nodenext",9 "module": "nodenext",
10 "noEmit": true,10 "noEmit": true,
11 "outDir": ".clover/ts",11 "outDir": ".clover/ts",
...@@ -14,7 +14,8 @@...@@ -14,7 +14,8 @@
14 "skipLibCheck": true,14 "skipLibCheck": true,
15 "strict": true,15 "strict": true,
16 "verbatimModuleSyntax": true,16 "verbatimModuleSyntax": true,
17 "target": "es2022"17 "target": "es2022",
18 "noUncheckedIndexedAccess": true
18 },19 },
19 "include": ["framework/**/*", "src/**/*"]20 "include": ["framework/**/*", "src/**/*"]
20}21}