authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-10 01:13:59-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log73cb20561b7cc88cebc4d2be5bff899efa8d80d5
tree6036855ff360c891f25c1db1116f75e71fa61200
parent93498bf84ee5dd9ad221152af00a7da852ed406c
signature Commit is signed but in an unrecognized format.

almost implement views

16 files changed, 514 insertions(+), 115 deletions(-)

framework/bundle.ts+127-31
...@@ -50,7 +50,6 @@ export async function bundleClientJavaScript(...@@ -50,7 +50,6 @@ export async function bundleClientJavaScript(
50 path.basename(file).replace(/\.client\.[tj]sx?/, "")50 path.basename(file).replace(/\.client\.[tj]sx?/, "")
51 );51 );
52 const { metafile } = bundle;52 const { metafile } = bundle;
53 console.log(metafile);
54 const promises: Promise<void>[] = [];53 const promises: Promise<void>[] = [];
55 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.54 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
56 for (const file of bundle.outputFiles) {55 for (const file of bundle.outputFiles) {
...@@ -85,25 +84,44 @@ export async function bundleClientJavaScript(...@@ -85,25 +84,44 @@ export async function bundleClientJavaScript(
85type ServerPlatform = "node" | "passthru";84type ServerPlatform = "node" | "passthru";
86export async function bundleServerJavaScript(85export async function bundleServerJavaScript(
87 /** Has 'export default app;' */86 /** Has 'export default app;' */
88 backendEntryPoint: string,87 _: string,
89 /** Views for dynamic loading */88 /** Views for dynamic loading */
90 viewEntryPoints: FileItem[],89 viewEntryPoints: FileItem[],
91 platform: ServerPlatform = "node",90 platform: ServerPlatform = "node",
92) {91) {
93 const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";92 const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
93 const viewModules = viewEntryPoints.map((view) => {
94 const module = require(view.file);
95 if (!module.meta) {
96 throw new Error(`${view.file} is missing 'export const meta'`);
97 }
98 if (!module.default) {
99 throw new Error(`${view.file} is missing a default export.`);
100 }
101 return { module, view };
102 });
94 const viewSource = [103 const viewSource = [
95 ...viewEntryPoints.map((view, i) =>104 ...viewEntryPoints.map((view, i) =>
96 `import * as view${i} from ${JSON.stringify(view.file)}`105 `import * as view${i} from ${JSON.stringify(view.file)}`
97 ),106 ),
98 `const scripts = ${scriptMagic}[-1]`,107 `const styles = ${scriptMagic}[-2]`,
108 `export const scripts = ${scriptMagic}[-1]`,
99 "export const views = {",109 "export const views = {",
100 ...viewEntryPoints.flatMap((view, i) => [110 ...viewModules.flatMap(({ view, module }, i) => [
101 ` ${JSON.stringify(view.id)}: {`,111 ` ${JSON.stringify(view.id)}: {`,
102 ` component: view${i}.default,`,112 ` component: view${i}.default,`,
103 ` meta: view${i}.meta,`,113 ` meta: view${i}.meta,`,
104 ` layout: view${i}.layout?.default,`,114 ` layout: ${
105 ` theme: view${i}.layout?.theme ?? view${i}.theme,`,115 module.layout?.default ? `view${i}.layout?.default` : "undefined"
106 ` scripts: ${scriptMagic}[${i}]`,116 },`,
117 ` theme: ${
118 module.layout?.theme
119 ? `view${i}.layout?.theme`
120 : module.theme
121 ? `view${i}.theme`
122 : "undefined"
123 },`,
124 ` inlineCss: styles[${scriptMagic}[${i}]]`,
107 ` },`,125 ` },`,
108 ]),126 ]),
109 "}",127 "}",
...@@ -112,55 +130,133 @@ export async function bundleServerJavaScript(...@@ -112,55 +130,133 @@ export async function bundleServerJavaScript(
112 virtualFiles({130 virtualFiles({
113 "$views": viewSource,131 "$views": viewSource,
114 }),132 }),
115 banFiles([133 // banFiles([
116 "hot.ts",134 // "hot.ts",
117 "incremental.ts",135 // "incremental.ts",
118 "bundle.ts",136 // "bundle.ts",
119 "generate.ts",137 // "generate.ts",
120 "css.ts",138 // "css.ts",
121 ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))),139 // ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))),
140 projectRelativeResolution(),
122 {141 {
123 name: "marko",142 name: "marko",
124 setup(b) {143 setup(b) {
125 b.onLoad({ filter: /\.marko$/ }, async ({ path }) => {144 b.onLoad({ filter: /\.marko$/ }, async ({ path: file }) => {
126 const src = await fs.readFile(path);
127 const result = await marko.compile(src, path, {
128 output: "html",
129 });
130 return {145 return {
131 loader: "ts",146 loader: "ts",
132 contents: result.code,147 contents: hot.getSourceCode(file),
133 };148 };
134 });149 });
135 },150 },
136 },151 },
152 {
153 name: "mark css external",
154 setup(b) {
155 b.onResolve(
156 { filter: /\.css$/ },
157 () => ({ path: ".", namespace: "dropped" }),
158 );
159 b.onLoad(
160 { filter: /./, namespace: "dropped" },
161 () => ({ contents: "" }),
162 );
163 },
164 },
137 ];165 ];
138 const bundle = await esbuild.build({166 const bundle = await esbuild.build({
139 bundle: true,167 bundle: true,
140 chunkNames: "/js/c.[hash]",168 chunkNames: "c.[hash]",
141 entryNames: "/js/[name]",169 entryNames: "[name]",
142 assetNames: "/asset/[hash]",170 entryPoints: [
143 entryPoints: [backendEntryPoint],171 path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
172 ],
144 platform: "node",173 platform: "node",
145 format: "esm",174 format: "esm",
146 minify: false,175 minify: false,
147 // outdir: "/out!",176 outdir: "/out!",
148 outdir: ".clover/wah",
149 plugins: serverPlugins,177 plugins: serverPlugins,
150 splitting: true,178 splitting: true,
151 write: true,179 write: false,
152 external: ["@babel/preset-typescript"],180 metafile: true,
181 });
182 const viewData = viewModules.map(({ module, view }) => {
183 return {
184 id: view.id,
185 theme: module.theme,
186 cssImports: hot.getCssImports(view.file)
187 .concat("src/global.css")
188 .map((file) => path.resolve(file)),
189 clientRefs: hot.getClientScriptRefs(view.file),
190 };
153 });191 });
154 console.log(bundle);192 return {
155 throw new Error("wahhh");193 views: viewData,
194 bundle,
195 scriptMagic,
196 };
197}
198
199type Await<T> = T extends Promise<infer R> ? R : T;
200
201export function finalizeServerJavaScript(
202 backend: Await<ReturnType<typeof bundleServerJavaScript>>,
203 viewCssBundles: css.Output[],
204 incr: Incremental,
205) {
206 const { metafile, outputFiles } = backend.bundle;
207
208 // Only the reachable script files need to be inserted into the bundle.
209 const reachableScriptKeys = new Set(
210 backend.views.flatMap((view) => view.clientRefs),
211 );
212 const reachableScripts = Object.fromEntries(
213 Array.from(incr.out.script)
214 .filter(([k]) => reachableScriptKeys.has(k)),
215 );
216
217 // Deduplicate styles
218 const styleList = Array.from(new Set(viewCssBundles));
219
220 for (const output of outputFiles) {
221 const basename = output.path.replace(/^.*?!/, "");
222 const key = "out!" + basename.replaceAll("\\", "/");
223
224 // If this contains the generated "$views" file, then
225 // replace the IDs with the bundled results.
226 let text = output.text;
227 if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
228 text = text.replace(
229 /CLOVER_CLIENT_SCRIPTS_DEFINITION\[(-?\d)\]/gs,
230 (_, i) => {
231 i = Number(i);
232 // Inline the styling data
233 if (i === -2) {
234 return JSON.stringify(styleList.map((item) => item.text));
235 }
236 // Inline the script data
237 if (i === -1) {
238 return JSON.stringify(Object.fromEntries(incr.out.script));
239 }
240 // Reference an index into `styleList`
241 return `${styleList.indexOf(viewCssBundles[i])}`;
242 },
243 );
244 }
245 fs.writeMkdirSync(path.join(".clover/backend/" + basename), text);
246 }
156}247}
157248
158import * as esbuild from "esbuild";249import * as esbuild from "esbuild";
159import * as path from "node:path";250import * as path from "node:path";
160import process from "node:process";251import process from "node:process";
161import * as hot from "./hot.ts";252import * as hot from "./hot.ts";
162import { banFiles, virtualFiles } from "./esbuild-support.ts";253import {
254 banFiles,
255 projectRelativeResolution,
256 virtualFiles,
257} from "./esbuild-support.ts";
163import { Incremental } from "./incremental.ts";258import { Incremental } from "./incremental.ts";
164import type { FileItem } from "#sitegen";259import type { FileItem } from "#sitegen";
165import * as marko from "@marko/compiler";260import * as marko from "@marko/compiler";
261import * as css from "./css.ts";
166import * as fs from "./lib/fs.ts";262import * as fs from "./lib/fs.ts";
framework/css.ts+1-2
...@@ -91,7 +91,7 @@ export async function bundleCssFiles(...@@ -91,7 +91,7 @@ export async function bundleCssFiles(
91 return {91 return {
92 text: outputFiles[0].text,92 text: outputFiles[0].text,
93 sources: Object.keys(metafile.outputs["$input$.css"].inputs)93 sources: Object.keys(metafile.outputs["$input$.css"].inputs)
94 .filter((x) => x !== "input:."),94 .filter((x) => !x.startsWith("vfs:")),
95 };95 };
96}96}
9797
...@@ -100,4 +100,3 @@ import * as fs from "#sitegen/fs";...@@ -100,4 +100,3 @@ import * as fs from "#sitegen/fs";
100import * as hot from "./hot.ts";100import * as hot from "./hot.ts";
101import * as path from "node:path";101import * as path from "node:path";
102import { virtualFiles } from "./esbuild-support.ts";102import { virtualFiles } from "./esbuild-support.ts";
103import { Incremental } from "./incremental.ts";
framework/esbuild-support.ts+23-8
...@@ -7,14 +7,14 @@ export function virtualFiles(...@@ -7,14 +7,14 @@ export function virtualFiles(
7 b.onResolve( 7 b.onResolve(
8 { 8 {
9 filter: new RegExp( 9 filter: new RegExp(
10 // TODO: Proper Escape 10 `^(?:${
11 `\\$`, 11 Object.keys(map).map((file) => string.escapeRegExp(file)).join(
12 "|",
13 )
14 })\$`,
12 ), 15 ),
13 }, 16 },
14 ({ path }) => { 17 ({ path }) => ({ path, namespace: "vfs" }),
15 console.log({ path });
16 return ({ path, namespace: "vfs" });
17 },
18 ); 18 );
19 b.onLoad( 19 b.onLoad(
20 { filter: /./, namespace: "vfs" }, 20 { filter: /./, namespace: "vfs" },
...@@ -40,8 +40,9 @@ export function banFiles(...@@ -40,8 +40,9 @@ export function banFiles(
40 b.onResolve( 40 b.onResolve(
41 { 41 {
42 filter: new RegExp( 42 filter: new RegExp(
43 "^(?:" + files.map((file) => string.escapeRegExp(file)).join("|") + 43 `^(?:${
44 ")$", 44 files.map((file) => string.escapeRegExp(file)).join("|")
45 })\$`,
45 ), 46 ),
46 }, 47 },
47 ({ path, importer }) => { 48 ({ path, importer }) => {
...@@ -54,5 +55,19 @@ export function banFiles(...@@ -54,5 +55,19 @@ export function banFiles(
54 } satisfies esbuild.Plugin; 55 } satisfies esbuild.Plugin;
55} 56}
56 57
58export function projectRelativeResolution(root = process.cwd() + "/src") {
59 return {
60 name: "project relative resolution ('@/' prefix)",
61 setup(b) {
62 b.onResolve({ filter: /^@\// }, ({ path: id }) => {
63 return {
64 path: path.resolve(root, id.slice(2)),
65 };
66 });
67 },
68 } satisfies esbuild.Plugin;
69}
70
57import * as esbuild from "esbuild"; 71import * as esbuild from "esbuild";
58import * as string from "#sitegen/string"; 72import * as string from "#sitegen/string";
73import * as path from "node:path";
framework/generate.ts+22-6
...@@ -12,7 +12,8 @@ async function sitegen(status: Spinner) {...@@ -12,7 +12,8 @@ async function sitegen(status: Spinner) {
1212
13 let root = path.resolve(import.meta.dirname, "../src");13 let root = path.resolve(import.meta.dirname, "../src");
14 const join = (...sub: string[]) => path.join(root, ...sub);14 const join = (...sub: string[]) => path.join(root, ...sub);
15 const incr = Incremental.fromDisk();15 const incr = new Incremental();
16 // const incr = Incremental.fromDisk();
16 await incr.statAllFiles();17 await incr.statAllFiles();
1718
18 // Sitegen reviews every defined section for resources to process19 // Sitegen reviews every defined section for resources to process
...@@ -100,7 +101,7 @@ async function sitegen(status: Spinner) {...@@ -100,7 +101,7 @@ async function sitegen(status: Spinner) {
100 body: string;101 body: string;
101 head: string;102 head: string;
102 css: css.Output;103 css: css.Output;
103 scriptFiles: string[];104 clientRefs: string[];
104 item: FileItem;105 item: FileItem;
105 }106 }
106 const renderResults: RenderResult[] = [];107 const renderResults: RenderResult[] = [];
...@@ -164,7 +165,7 @@ async function sitegen(status: Spinner) {...@@ -164,7 +165,7 @@ async function sitegen(status: Spinner) {
164 body: text,165 body: text,
165 head: renderedMeta,166 head: renderedMeta,
166 css: cssBundle,167 css: cssBundle,
167 scriptFiles: Array.from(addon.sitegen.scripts),168 clientRefs: Array.from(addon.sitegen.scripts),
168 item: item,169 item: item,
169 });170 });
170 }171 }
...@@ -193,14 +194,26 @@ async function sitegen(status: Spinner) {...@@ -193,14 +194,26 @@ async function sitegen(status: Spinner) {
193194
194 // -- bundle backend and views --195 // -- bundle backend and views --
195 status.text = "Bundle backend code";196 status.text = "Bundle backend code";
196 const {} = await bundle.bundleServerJavaScript(197 const backend = await bundle.bundleServerJavaScript(
197 join("backend.ts"),198 join("backend.ts"),
198 views,199 views,
199 );200 );
201 const viewCssPromise = await Promise.all(
202 backend.views.map((view) =>
203 cssOnce.get(
204 view.cssImports.join(":") + JSON.stringify(view.theme),
205 () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
206 )
207 ),
208 );
200209
201 // -- bundle scripts --210 // -- bundle scripts --
202 const referencedScripts = Array.from(211 const referencedScripts = Array.from(
203 new Set(renderResults.flatMap((r) => r.scriptFiles)),212 new Set([
213 ...renderResults.flatMap((r) => r.clientRefs),
214 ...backend.views.flatMap((r) => r.clientRefs),
215 ]),
216 (script) => path.join(hot.projectSrc, script),
204 );217 );
205 const extraPublicScripts = scripts.map((entry) => entry.file);218 const extraPublicScripts = scripts.map((entry) => entry.file);
206 const uniqueCount = new Set([219 const uniqueCount = new Set([
...@@ -214,6 +227,9 @@ async function sitegen(status: Spinner) {...@@ -214,6 +227,9 @@ async function sitegen(status: Spinner) {
214 incr,227 incr,
215 );228 );
216229
230 // -- finalize backend bundle --
231 await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
232
217 // -- copy/compress static files --233 // -- copy/compress static files --
218 async function doStaticFile(item: FileItem) {234 async function doStaticFile(item: FileItem) {
219 const body = await fs.readFile(item.file);235 const body = await fs.readFile(item.file);
...@@ -241,7 +257,7 @@ async function sitegen(status: Spinner) {...@@ -241,7 +257,7 @@ async function sitegen(status: Spinner) {
241 await Promise.all(257 await Promise.all(
242 renderResults.map(258 renderResults.map(
243 async (259 async (
244 { item: page, body, head, css, scriptFiles },260 { item: page, body, head, css, clientRefs: scriptFiles },
245 ) => {261 ) => {
246 const doc = wrapDocument({262 const doc = wrapDocument({
247 body,263 body,
framework/hot.ts+70-14
...@@ -118,12 +118,17 @@ function loadEsbuildCode(...@@ -118,12 +118,17 @@ function loadEsbuildCode(
118 module.exports = self;118 module.exports = self;
119 return;119 return;
120 }120 }
121
122 let loader: any = "tsx";121 let loader: any = "tsx";
123 if (filepath.endsWith(".ts")) loader = "ts";122 if (filepath.endsWith(".ts")) loader = "ts";
124 else if (filepath.endsWith(".jsx")) loader = "jsx";123 else if (filepath.endsWith(".jsx")) loader = "jsx";
125 else if (filepath.endsWith(".js")) loader = "js";124 else if (filepath.endsWith(".js")) loader = "js";
126 module.cloverClientRefs = opt.scannedClientRefs ?? extractClientScripts(src);125 if (opt.scannedClientRefs) {
126 module.cloverClientRefs = opt.scannedClientRefs;
127 } else {
128 let { code, refs } = resolveClientRefs(src, filepath);
129 module.cloverClientRefs = refs;
130 src = code;
131 }
127 if (src.includes("import.meta")) {132 if (src.includes("import.meta")) {
128 src = `133 src = `
129 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};134 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
...@@ -142,21 +147,43 @@ function loadEsbuildCode(...@@ -142,21 +147,43 @@ function loadEsbuildCode(
142 return module._compile(src, filepath, "commonjs");147 return module._compile(src, filepath, "commonjs");
143}148}
144149
150function resolveClientRef(sourcePath: string, ref: string) {
151 const filePath = resolveFrom(sourcePath, ref);
152 if (
153 !filePath.endsWith(".client.ts") &&
154 !filePath.endsWith(".client.tsx")
155 ) {
156 throw new Error("addScript must be a .client.ts or .client.tsx");
157 }
158 return path.relative(projectSrc, filePath);
159}
160
145function loadMarko(module: NodeJS.Module, filepath: string) {161function loadMarko(module: NodeJS.Module, filepath: string) {
146 let src = fs.readFileSync(filepath, "utf8");162 let src = fs.readFileSync(filepath, "utf8");
147 // A non-standard thing here is Clover Sitegen implements163 // A non-standard thing here is Clover Sitegen implements
148 // its own client side scripting stuff, so it overrides164 // its own client side scripting stuff, so it overrides
149 // bare client import statements to it's own usage.165 // bare client import statements to it's own usage.
166 const scannedClientRefs = new Set<string>();
150 if (src.match(/^\s*client\s+import\s+["']/m)) {167 if (src.match(/^\s*client\s+import\s+["']/m)) {
151 src = src.replace(168 src = src.replace(
152 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,169 /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
153 (_, src) => `<CloverScriptInclude src=${src} />`,170 (_, src) => {
154 ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';171 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
172 const resolved = resolveClientRef(filepath, ref);
173 scannedClientRefs.add(resolved);
174 return `<CloverScriptInclude=${JSON.stringify(resolved)} />`;
175 },
176 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
155 }177 }
156178
157 src = marko.compileSync(src, filepath).code;179 src = marko.compileSync(src, filepath).code;
158 src = src.replace("marko/debug/html", "#ssr/marko");180 src = src.replace("marko/debug/html", "#ssr/marko");
159 return loadEsbuildCode(module, filepath, src);181
182 module.cloverSourceCode = src;
183
184 return loadEsbuildCode(module, filepath, src, {
185 scannedClientRefs: Array.from(scannedClientRefs),
186 });
160}187}
161188
162function loadMdx(module: NodeJS.Module, filepath: string) {189function loadMdx(module: NodeJS.Module, filepath: string) {
...@@ -197,6 +224,23 @@ export function getCssImports(filepath: string) {...@@ -197,6 +224,23 @@ export function getCssImports(filepath: string) {
197 ?.cssImportsRecursive ?? [];224 ?.cssImportsRecursive ?? [];
198}225}
199226
227export function getClientScriptRefs(filepath: string) {
228 filepath = path.resolve(filepath);
229 const module = require.cache[filepath];
230 if (!module) throw new Error(filepath + " was never loaded");
231 return module.cloverClientRefs ?? [];
232}
233
234export function getSourceCode(filepath: string) {
235 filepath = path.resolve(filepath);
236 const module = require.cache[filepath];
237 if (!module) throw new Error(filepath + " was never loaded");
238 if (!module.cloverSourceCode) {
239 throw new Error(filepath + " did not record source code");
240 }
241 return module.cloverSourceCode;
242}
243
200export function resolveFrom(src: string, dest: string) {244export function resolveFrom(src: string, dest: string) {
201 try {245 try {
202 return createRequire(src).resolve(dest);246 return createRequire(src).resolve(dest);
...@@ -210,16 +254,23 @@ export function resolveFrom(src: string, dest: string) {...@@ -210,16 +254,23 @@ export function resolveFrom(src: string, dest: string) {
210254
211const importRegExp =255const importRegExp =
212 /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;256 /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;
213const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))/;257const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))?/;
214export function extractClientScripts(source: string): string[] {258interface ResolvedClientRefs {
259 code: string;
260 refs: string[];
261}
262export function resolveClientRefs(
263 code: string,
264 filepath: string,
265): ResolvedClientRefs {
215 // This match finds a call to 'import ... from "#sitegen"'266 // This match finds a call to 'import ... from "#sitegen"'
216 const importMatch = source.match(importRegExp);267 const importMatch = code.match(importRegExp);
217 if (!importMatch) return [];268 if (!importMatch) return { code, refs: [] };
218 const items = importMatch[1];269 const items = importMatch[1];
219 let identifier = "";270 let identifier = "";
220 if (items.startsWith("{")) {271 if (items.startsWith("{")) {
221 const clauseMatch = items.match(getSitegenAddScriptRegExp);272 const clauseMatch = items.match(getSitegenAddScriptRegExp);
222 if (!clauseMatch) return []; // did not import273 if (!clauseMatch) return { code, refs: [] }; // did not import
223 identifier = clauseMatch[1] || "addScript";274 identifier = clauseMatch[1] || "addScript";
224 } else if (items.startsWith("*")) {275 } else if (items.startsWith("*")) {
225 identifier = importMatch[2] + "\\s*\\.\\s*addScript";276 identifier = importMatch[2] + "\\s*\\.\\s*addScript";
...@@ -228,19 +279,24 @@ export function extractClientScripts(source: string): string[] {...@@ -228,19 +279,24 @@ export function extractClientScripts(source: string): string[] {
228 }279 }
229 identifier = identifier.replaceAll("$", "\\$"); // only needed escape280 identifier = identifier.replaceAll("$", "\\$"); // only needed escape
230 const findCallsRegExp = new RegExp(281 const findCallsRegExp = new RegExp(
231 `\\b${identifier}\\s*\\(("[^"]+"|'[^']+')\\)`,282 `\\b(${identifier})\\s*\\(("[^"]+"|'[^']+')\\)`,
232 "gs",283 "gs",
233 );284 );
234 const calls = source.matchAll(findCallsRegExp);285 const scannedClientRefs = new Set<string>();
235 return [...calls].map((call) => {286 code = code.replace(findCallsRegExp, (_, call, arg) => {
236 return JSON.parse(`"${call[1].slice(1, -1)}"`) as string;287 const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
288 const resolved = resolveClientRef(filepath, ref);
289 scannedClientRefs.add(resolved);
290 return `${call}(${JSON.stringify(resolved)})`;
237 });291 });
292 return { code, refs: Array.from(scannedClientRefs) };
238}293}
239294
240declare global {295declare global {
241 namespace NodeJS {296 namespace NodeJS {
242 interface Module {297 interface Module {
243 cloverClientRefs?: string[];298 cloverClientRefs?: string[];
299 cloverSourceCode?: string;
244300
245 _compile(301 _compile(
246 this: NodeJS.Module,302 this: NodeJS.Module,
framework/incremental.ts+1-1
...@@ -141,7 +141,7 @@ export class Incremental {...@@ -141,7 +141,7 @@ export class Incremental {
141 ASSERT(stat, "Updated stat on untracked file " + fileKey);141 ASSERT(stat, "Updated stat on untracked file " + fileKey);
142 if (stat.lastModified < newLastModified) {142 if (stat.lastModified < newLastModified) {
143 // Invalidate143 // Invalidate
144 console.log(fileKey + " updated");144 console.info(fileKey + " updated");
145 const invalidQueue = [fileKey];145 const invalidQueue = [fileKey];
146 let currentInvalid;146 let currentInvalid;
147 while (currentInvalid = invalidQueue.pop()) {147 while (currentInvalid = invalidQueue.pop()) {
framework/lib/fs.ts+8
...@@ -41,6 +41,14 @@ export function readDirRecOptionalSync(dir: string) {...@@ -41,6 +41,14 @@ export function readDirRecOptionalSync(dir: string) {
41 }41 }
42}42}
4343
44export async function readJson<T>(file: string) {
45 return JSON.parse(await readFile(file, "utf-8")) as T;
46}
47
48export function readJsonSync<T>(file: string) {
49 return JSON.parse(readFileSync(file, "utf-8")) as T;
50}
51
44import * as path from "node:path";52import * as path from "node:path";
45import {53import {
46 existsSync,54 existsSync,
framework/lib/sitegen.ts+3-24
...@@ -10,10 +10,8 @@ export interface FileItem {...@@ -10,10 +10,8 @@ export interface FileItem {
10 file: string;10 file: string;
11}11}
1212
13const frameworkDir = path.dirname(import.meta.dirname);
14
15export interface SitegenRender {13export interface SitegenRender {
16 scripts: Set<ScriptId>;14 scripts: Set<string>;
17}15}
1816
19export function initRender(): SitegenRender {17export function initRender(): SitegenRender {
...@@ -31,24 +29,8 @@ export function getRender() {...@@ -31,24 +29,8 @@ export function getRender() {
31}29}
3230
33/** Add a client-side script to the page. */31/** Add a client-side script to the page. */
34export function addScript(id: ScriptId) {32export function addScript(id: ScriptId | { value: ScriptId }) {
35 const srcFile: string = util.getCallSites()33 getRender().scripts.add(typeof id === "string" ? id : id.value);
36 .find((site) => !site.scriptName.startsWith(frameworkDir))!
37 .scriptName;
38 const filePath = hot.resolveFrom(srcFile, id);
39 if (
40 !filePath.endsWith(".client.ts") &&
41 !filePath.endsWith(".client.tsx")
42 ) {
43 throw new Error("addScript must be a .client.ts or .client.tsx");
44 }
45 getRender().scripts.add(filePath);
46}
47
48export function Script({ src }: { src: ScriptId }) {
49 if (!src) throw new Error("Missing 'src' attribute");
50 addScript(src);
51 return null;
52}34}
5335
54export interface Section {36export interface Section {
...@@ -56,6 +38,3 @@ export interface Section {...@@ -56,6 +38,3 @@ export interface Section {
56}38}
5739
58import * as ssr from "../engine/ssr.ts";40import * as ssr from "../engine/ssr.ts";
59import * as util from "node:util";
60import * as hot from "../hot.ts";
61import * as path from "node:path";
framework/lib/view.ts+5-7
...@@ -5,21 +5,19 @@ export interface View {...@@ -5,21 +5,19 @@ export interface View {
5 | meta.Meta 5 | meta.Meta
6 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta); 6 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
7 layout?: engine.Component; 7 layout?: engine.Component;
8 theme?: css.Theme;
9 inlineCss: string; 8 inlineCss: string;
10 scripts: Record<string, string>; 9 scripts: Record<string, string>;
11} 10}
12let views: Record<string, View> = {};
13 11
14// An older version of the Clover Engine supported streaming suspense 12// An older version of the Clover Engine supported streaming suspense
15// boundaries, but those were never used. Pages will wait until they 13// boundaries, but those were never used. Pages will wait until they
16// are fully rendered before sending. 14// are fully rendered before sending.
17async function renderView( 15export async function renderView(
18 c: hono.Context, 16 c: hono.Context,
19 id: string, 17 id: string,
20 props: Record<string, unknown>, 18 props: Record<string, unknown>,
21) { 19) {
22 views = require("$views").views; 20 const { views, scripts } = require("$views");
23 // The view contains pre-bundled CSS and scripts, but keeps the scripts 21 // The view contains pre-bundled CSS and scripts, but keeps the scripts
24 // separate for run-time dynamic scripts. For example, the file viewer 22 // separate for run-time dynamic scripts. For example, the file viewer
25 // includes the canvas for the current page, but only the current page. 23 // includes the canvas for the current page, but only the current page.
...@@ -28,9 +26,7 @@ async function renderView(...@@ -28,9 +26,7 @@ async function renderView(
28 inlineCss, 26 inlineCss,
29 layout, 27 layout,
30 meta: metadata, 28 meta: metadata,
31 scripts, 29 }: View = views[id];
32 theme,
33 }: View = UNWRAP(views[id]);
34 30
35 // -- metadata -- 31 // -- metadata --
36 const renderedMetaPromise = Promise.resolve( 32 const renderedMetaPromise = Promise.resolve(
...@@ -43,8 +39,10 @@ async function renderView(...@@ -43,8 +39,10 @@ async function renderView(
43 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { 39 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
44 sitegen: sg.initRender(), 40 sitegen: sg.initRender(),
45 }); 41 });
42 console.log(sitegen);
46 43
47 // -- join document and send -- 44 // -- join document and send --
45 console.log(scripts);
48 return c.html(wrapDocument({ 46 return c.html(wrapDocument({
49 body, 47 body,
50 head: await renderedMetaPromise, 48 head: await renderedMetaPromise,
src/backend.ts-2
...@@ -1,5 +1,3 @@...@@ -1,5 +1,3 @@
1import * as v from "#sitegen/view";
2console.log(v);
3const logHttp = scoped("http", { color: "magenta" }); 1const logHttp = scoped("http", { color: "magenta" });
4 2
5const app = new Hono(); 3const app = new Hono();
src/q+a/backend.ts+231-1
...@@ -1,3 +1,233 @@...@@ -1,3 +1,233 @@
1const PROXYCHECK_API_KEY = process.env.PROXYCHECK_API_KEY;
2
1export const app = new Hono(); 3export const app = new Hono();
2 4
3import { Hono } from "#hono"; 5// Main page
6app.get("/q+a", async (c) => {
7 if (hasAdminToken(c)) {
8 return serveAsset(c, "/admin/q+a", 200);
9 }
10 return serveAsset(c, "/q+a", 200);
11});
12
13// Submit form
14app.post("/q+a", async (c) => {
15 const form = await c.req.formData();
16 let text = form.get("text");
17 if (typeof text !== "string") {
18 return questionFailure(c, 400, "Bad Request");
19 }
20 text = text.trim();
21 const input = {
22 date: new Date(),
23 prompt: text,
24 sourceName: "unknown",
25 sourceLocation: "unknown",
26 sourceVPN: null,
27 };
28
29 input.date.setMilliseconds(0);
30
31 if (text.length <= 0) {
32 return questionFailure(c, 400, "Content is too short", text);
33 }
34
35 if (text.length > 16000) {
36 return questionFailure(c, 400, "Content is too long", text);
37 }
38
39 // Ban patterns
40 if (
41 text.includes("hams" + "terko" + "mbat" + ".expert") // 2025-02-18. 3 occurrences. All VPN
42 ) {
43 // To prevent known automatic spam-bots from noticing something automatic is
44 // happening, pretend that the question was successfully submitted.
45 return sendSuccess(c, new Date());
46 }
47
48 const ipAddr = c.req.header("cf-connecting-ip");
49 if (ipAddr) {
50 input.sourceName = uniqueNamesGenerator({
51 dictionaries: [adjectives, colors, animals],
52 separator: "-",
53 seed: ipAddr + PROXYCHECK_API_KEY,
54 });
55 }
56
57 const cfIPCountry = c.req.header("cf-ipcountry");
58 if (cfIPCountry) {
59 input.sourceLocation = cfIPCountry;
60 }
61
62 if (ipAddr && PROXYCHECK_API_KEY) {
63 const proxyCheck = await fetch(
64 `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,
65 {
66 method: "POST",
67 body: "ips=" + ipAddr,
68 headers: {
69 "Content-Type": "application/x-www-form-urlencoded",
70 },
71 },
72 ).then((res) => res.json());
73
74 if (ipAddr && proxyCheck[ipAddr]) {
75 if (proxyCheck[ipAddr].proxy === "yes") {
76 input.sourceVPN = proxyCheck[ipAddr].operator?.name ??
77 proxyCheck[ipAddr].organisation ??
78 proxyCheck[ipAddr].provider ?? "unknown";
79 }
80 if (Number(proxyCheck[ipAddr].risk) > 72) {
81 return questionFailure(
82 c,
83 403,
84 "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.",
85 text,
86 );
87 }
88 }
89 }
90
91 const date = Question.create(
92 QuestionType.pending,
93 JSON.stringify(input),
94 input.date,
95 );
96 await sendSuccess(c, date);
97});
98async function sendSuccess(c: Context, date: Date) {
99 if (c.req.header("Accept")?.includes("application/json")) {
100 return c.json({
101 success: true,
102 message: "ok",
103 date: date.getTime(),
104 id: formatQuestionId(date),
105 }, { status: 200 });
106 }
107 c.res = await renderView(c, "qa_success", {
108 permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,
109 });
110}
111// Question Permalink
112app.get("/q+a/:id", (c, next) => {
113 // from deadname era, the seconds used to be in the url.
114 // this was removed so that the url can be crafted by hand.
115 let id = c.req.param("id");
116 if (id.length === 12 && /^\d+$/.test(id)) {
117 return c.redirect(`/q+a/${id.slice(0, 10)}`);
118 }
119 let image = false;
120 if (id.endsWith(".png")) {
121 image = true;
122 id = id.slice(0, -4);
123 }
124
125 const timestamp = questionIdToTimestamp(id);
126 if (!timestamp) return next();
127 const question = Question.getByDate(timestamp);
128 if (!question) return next();
129
130 // if (image) {
131 // return getQuestionImage(question, c.req.method === "HEAD");
132 // }
133 return renderView(c, "q+a/permalink", { question });
134});
135
136// Admin
137app.get("/admin/q+a", async (c) => {
138 return serveAsset(c, "/admin/q+a", 200);
139});
140app.get("/admin/q+a/inbox", async (c) => {
141 return renderView(c, "qa_backend_inbox", {});
142});
143app.delete("/admin/q+a/:id", async (c, next) => {
144 const id = c.req.param("id");
145 const timestamp = questionIdToTimestamp(id);
146 if (!timestamp) return next();
147 const question = Question.getByDate(timestamp);
148 if (!question) return next();
149 const deleteFull = c.req.header("X-Delete-Full") === "true";
150 if (deleteFull) {
151 Question.deleteByQmid(question.qmid);
152 } else {
153 Question.rejectByQmid(question.qmid);
154 }
155 return c.json({ success: true, message: "ok" });
156});
157app.patch("/admin/q+a/:id", async (c, next) => {
158 const id = c.req.param("id");
159 const timestamp = questionIdToTimestamp(id);
160 if (!timestamp) return next();
161 const question = Question.getByDate(timestamp);
162 if (!question) return next();
163 const form = await c.req.raw.json();
164 if (typeof form.text !== "string" || typeof form.type !== "number") {
165 return questionFailure(c, 400, "Bad Request");
166 }
167 Question.updateByQmid(question.qmid, form.text, form.type);
168 return c.json({ success: true, message: "ok" });
169});
170app.get("/admin/q+a/:id", async (c, next) => {
171 const id = c.req.param("id");
172 const timestamp = questionIdToTimestamp(id);
173 if (!timestamp) return next();
174 const question = Question.getByDate(timestamp);
175 if (!question) return next();
176
177 let pendingInfo: null | PendingQuestionData = null;
178 if (question.type === QuestionType.pending) {
179 pendingInfo = JSON.parse(question.text) as PendingQuestionData;
180 question.text = pendingInfo.prompt.trim().split("\n").map((line) =>
181 line.trim().length === 0 ? "" : `q: ${line.trim()}`
182 ).join("\n") + "\n\n";
183 question.type = QuestionType.normal;
184 }
185
186 return renderView(c, "q+a/editor", {
187 pendingInfo,
188 question,
189 });
190});
191
192app.get("/q+a/things/random", async (c) => {
193 c.res = await renderView(c, "q+a/things-random", {});
194});
195
196// 404
197app.get("/q+a/*", async (c) => {
198 return serveAsset(c, "/q+a/404", 404);
199});
200
201async function questionFailure(
202 c: Context,
203 status: ContentfulStatusCode,
204 message: string,
205 content?: string,
206) {
207 if (c.req.header("Accept")?.includes("application/json")) {
208 return c.json({ success: false, message, id: null }, { status });
209 }
210 return await renderView(c, "q+a/fail", {
211 error: message,
212 content,
213 });
214}
215
216import { type Context, Hono } from "#hono";
217import type { ContentfulStatusCode } from "hono/utils/http-status";
218import {
219 adjectives,
220 animals,
221 colors,
222 uniqueNamesGenerator,
223} from "unique-names-generator";
224import { hasAdminToken } from "../admin.ts";
225import { serveAsset } from "#sitegen/assets";
226import {
227 PendingQuestion,
228 PendingQuestionData,
229} from "./models/PendingQuestion.ts";
230import { Question, QuestionType } from "./models/Question.ts";
231import { renderView } from "#sitegen/view";
232// import { getQuestionImage } from "./question_image";
233import { formatQuestionId, questionIdToTimestamp } from "./format.ts";
src/q+a/views/backend-inbox.client.ts created
src/q+a/views/backend-inbox.marko+5-6
...@@ -1,11 +1,10 @@...@@ -1,11 +1,10 @@
1---1export { minimal as layout } from "../layout.tsx";
2export { minimal as layout } from "../layouts/questions.tsx";2import { PendingQuestion } from "@/q+a/models/PendingQuestion.ts";
3import { PendingQuestion } from "../db.ts";
4import { useInlineScript } from "../framework/page-resources";
5import {3import {
6 formatQuestionISOTimestamp,4 formatQuestionISOTimestamp,
7 formatQuestionTimestamp,5 formatQuestionTimestamp,
8} from "../q+a/QuestionRender";6} from "@/q+a/format.ts";
7export const meta = { title: 'question answer inbox' };
98
10<const/questions = PendingQuestion.getAll() />9<const/questions = PendingQuestion.getAll() />
1110
...@@ -38,4 +37,4 @@ import {...@@ -38,4 +37,4 @@ import {
38 </div>37 </div>
39</>38</>
4039
41client import "backend-inbox.client.ts";40client import "./backend-inbox.client.ts";
src/q+a/views/fail.marko+2
...@@ -4,6 +4,8 @@ export const theme = {...@@ -4,6 +4,8 @@ export const theme = {
4 ...layout.theme,4 ...layout.theme,
5 primary: "#58ffee",5 primary: "#58ffee",
6};6};
7export const meta = { title: 'oh no' };
8
7<const/{ error, content }=input/>9<const/{ error, content }=input/>
810
9<h2 style="color: red">:(</h2>11<h2 style="color: red">:(</h2>
src/q+a/views/success.marko+10-7
...@@ -1,14 +1,9 @@...@@ -1,14 +1,9 @@
1export interface Input {1export interface Input {
2 permalink: string;2 permalink: string;
3}3}
4<const/{ permalink }=input/>4export const meta = { title: 'question submitted!!!' };
55
6import * as layout from "../layout.tsx";6<const/{ permalink }=input/>
7export { layout };
8export const theme = {
9 ...layout.theme,
10 primary: "#58ffee",
11};
127
13<h2 style="color: #8c78ff; margin-bottom: 0">thank you</h2>8<h2 style="color: #8c78ff; margin-bottom: 0">thank you</h2>
14<p style="color: #8c78ff">9<p style="color: #8c78ff">
...@@ -28,3 +23,11 @@ export const theme = {...@@ -28,3 +23,11 @@ export const theme = {
28<p>23<p>
29 <a href="/q+a">return to the questions list</a>24 <a href="/q+a">return to the questions list</a>
30</p>25</p>
26
27import * as layout from "../layout.tsx";
28export { layout };
29export const theme = {
30 ...layout.theme,
31 primary: "#58ffee",
32};
33
src/q+a/views/things-random.marko+6-6
...@@ -1,10 +1,10 @@...@@ -1,10 +1,10 @@
1---1export const meta = { title: 'random number' };
2import { useInlineScript } from "../framework/page-resources";2client import "./things-random.client.ts";
3
4<const/number = Math.floor(Math.random() * 999999) + 1 />
35
4const number = Math.floor(Math.random() * 999999) + 1;
5useInlineScript("qa_things_random");
6---
7<main>6<main>
8 <div>{number}</div>7 <div>${number}</div>
9 <button>another</button>8 <button>another</button>
10</main>9</main>
10