authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-09 00:12:41-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
log2fa9b4eb6ee7d21505a0def86bd0c36801869e9a
treeaa4f1b43e028347a8a1242ef73f78136feb1ce63
parentad50869c177bec4aa2eb956e55acbee3ba04ad27
signature Commit is signed but in an unrecognized format.

incremental generator

12 files changed, 435 insertions(+), 337 deletions(-)

framework/bundle.ts+10-4
...@@ -38,6 +38,7 @@ export async function bundleClientJavaScript(...@@ -38,6 +38,7 @@ export async function bundleClientJavaScript(
38 plugins,38 plugins,
39 splitting: true,39 splitting: true,
40 write: false,40 write: false,
41 metafile: true,
41 });42 });
42 if (bundle.errors.length || bundle.warnings.length) {43 if (bundle.errors.length || bundle.warnings.length) {
43 throw new AggregateError(44 throw new AggregateError(
...@@ -45,29 +46,34 @@ export async function bundleClientJavaScript(...@@ -45,29 +46,34 @@ export async function bundleClientJavaScript(
45 "JS bundle failed",46 "JS bundle failed",
46 );47 );
47 }48 }
48 incr.invalidate("bundle-script");
49 const publicScriptRoutes = extraPublicScripts.map((file) =>49 const publicScriptRoutes = extraPublicScripts.map((file) =>
50 path.basename(file).replace(/\.client\.[tj]sx?/, "")50 path.basename(file).replace(/\.client\.[tj]sx?/, "")
51 );51 );
52 const { metafile } = bundle;
53 console.log(metafile);
52 const promises: Promise<void>[] = [];54 const promises: Promise<void>[] = [];
53 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.55 // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
54 for (const file of bundle.outputFiles) {56 for (const file of bundle.outputFiles) {
57 const { text } = file;
55 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");58 let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
56 const text = file.text;59 const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
60 const sources = Object.keys(inputs);
61
57 // Register non-chunks as script entries.62 // Register non-chunks as script entries.
58 const chunk = route.startsWith("/js/c.");63 const chunk = route.startsWith("/js/c.");
59 if (!chunk) {64 if (!chunk) {
60 route = route.replace(".client.js", ".js");65 route = route.replace(".client.js", ".js");
61 incr.put({66 incr.put({
62 srcId: "bundle-script",67 sources,
63 type: "script",68 type: "script",
64 key: route.slice("/js/".length, -".js".length),69 key: route.slice("/js/".length, -".js".length),
65 value: text,70 value: text,
66 });71 });
67 }72 }
73 // Register chunks and public scripts as assets.
68 if (chunk || publicScriptRoutes.includes(route)) {74 if (chunk || publicScriptRoutes.includes(route)) {
69 promises.push(incr.putAsset({75 promises.push(incr.putAsset({
70 srcId: "bundle-script",76 sources,
71 key: route,77 key: route,
72 body: text,78 body: text,
73 }));79 }));
framework/css.ts+23-7
...@@ -34,11 +34,19 @@ export function preprocess(css: string, theme: Theme): string {...@@ -34,11 +34,19 @@ export function preprocess(css: string, theme: Theme): string {
34 );34 );
35}35}
3636
37export interface Output {
38 text: string;
39 sources: string[];
40}
41
37export async function bundleCssFiles(42export async function bundleCssFiles(
38 cssImports: string[],43 cssImports: string[],
39 theme: Theme,44 theme: Theme,
40 dev: boolean = false,45 dev: boolean = false,
41): Promise<string> {46): Promise<Output> {
47 cssImports = cssImports.map((file) =>
48 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
49 );
42 const plugin = {50 const plugin = {
43 name: "clover",51 name: "clover",
44 setup(b) {52 setup(b) {
...@@ -68,13 +76,14 @@ export async function bundleCssFiles(...@@ -68,13 +76,14 @@ export async function bundleCssFiles(
68 const build = await esbuild.build({76 const build = await esbuild.build({
69 bundle: true,77 bundle: true,
70 entryPoints: ["$input$"],78 entryPoints: ["$input$"],
71 write: false,79 external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"],
72 external: ["*.woff2"],80 metafile: true,
73 target: ["ie11"],
74 plugins: [plugin],
75 minify: !dev,81 minify: !dev,
82 plugins: [plugin],
83 target: ["ie11"],
84 write: false,
76 });85 });
77 const { errors, warnings, outputFiles } = build;86 const { errors, warnings, outputFiles, metafile } = build;
78 if (errors.length > 0) {87 if (errors.length > 0) {
79 throw new AggregateError(errors, "CSS Build Failed");88 throw new AggregateError(errors, "CSS Build Failed");
80 }89 }
...@@ -82,8 +91,15 @@ export async function bundleCssFiles(...@@ -82,8 +91,15 @@ export async function bundleCssFiles(
82 throw new AggregateError(warnings, "CSS Build Failed");91 throw new AggregateError(warnings, "CSS Build Failed");
83 }92 }
84 if (outputFiles.length > 1) throw new Error("Too many output files");93 if (outputFiles.length > 1) throw new Error("Too many output files");
85 return outputFiles[0].text;94 return {
95 text: outputFiles[0].text,
96 sources: Object.keys(metafile.outputs["$input$.css"].inputs)
97 .filter((x) => x !== "input:."),
98 };
86}99}
87100
88import * as esbuild from "esbuild";101import * as esbuild from "esbuild";
89import * as fs from "#sitegen/fs";102import * as fs from "#sitegen/fs";
103import * as hot from "./hot.ts";
104import * as path from "node:path";
105import { Incremental } from "./incremental.ts";
framework/generate.tsx+41-24
...@@ -21,7 +21,8 @@ async function sitegen(status: Spinner) {...@@ -21,7 +21,8 @@ async function sitegen(status: Spinner) {
2121
22 let root = path.resolve(import.meta.dirname, "../src");22 let root = path.resolve(import.meta.dirname, "../src");
23 const join = (...sub: string[]) => path.join(root, ...sub);23 const join = (...sub: string[]) => path.join(root, ...sub);
24 const incr = new Incremental();24 const incr = Incremental.fromDisk();
25 await incr.statAllFiles();
2526
26 // Sitegen reviews every defined section for resources to process27 // Sitegen reviews every defined section for resources to process
27 const sections: sg.Section[] =28 const sections: sg.Section[] =
...@@ -68,20 +69,24 @@ async function sitegen(status: Spinner) {...@@ -68,20 +69,24 @@ async function sitegen(status: Spinner) {
68 ];69 ];
69 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {70 for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
70 const items = fs.readDirRecOptionalSync(dir);71 const items = fs.readDirRecOptionalSync(dir);
71 item: for (const item of items) {72 item: for (const subPath of items) {
72 if (item.isDirectory()) continue;73 const file = path.join(dir, subPath);
74 const stat = fs.statSync(file);
75 if (stat.isDirectory()) continue;
73 for (const e of exclude) {76 for (const e of exclude) {
74 if (item.name.endsWith(e)) continue item;77 if (subPath.endsWith(e)) continue item;
75 }78 }
76 const file = path.relative(dir, item.parentPath + "/" + item.name);
77 const trim = ext79 const trim = ext
78 ? file80 ? subPath
79 : file.slice(0, -path.extname(file).length).replaceAll(".", "/");81 : subPath.slice(0, -path.extname(subPath).length).replaceAll(
82 ".",
83 "/",
84 );
80 let id = prefix + trim.replaceAll("\\", "/");85 let id = prefix + trim.replaceAll("\\", "/");
81 if (prefix === "/" && id.endsWith("/index")) {86 if (prefix === "/" && id.endsWith("/index")) {
82 id = id.slice(0, -"/index".length) || "/";87 id = id.slice(0, -"/index".length) || "/";
83 }88 }
84 list.push({ id, file: path.join(item.parentPath, item.name) });89 list.push({ id, file: file });
85 }90 }
86 }91 }
87 }92 }
...@@ -92,8 +97,8 @@ async function sitegen(status: Spinner) {...@@ -92,8 +97,8 @@ async function sitegen(status: Spinner) {
9297
93 // -- server side render --98 // -- server side render --
94 status.text = "Building";99 status.text = "Building";
95 const cssOnce = new OnceMap<string>();100 const cssOnce = new OnceMap<css.Output>();
96 const cssQueue = new Queue<[string, string[], css.Theme], string>({101 const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
97 name: "Bundle",102 name: "Bundle",
98 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),103 fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
99 passive: true,104 passive: true,
...@@ -103,7 +108,7 @@ async function sitegen(status: Spinner) {...@@ -103,7 +108,7 @@ async function sitegen(status: Spinner) {
103 interface RenderResult {108 interface RenderResult {
104 body: string;109 body: string;
105 head: string;110 head: string;
106 inlineCss: string;111 css: css.Output;
107 scriptFiles: string[];112 scriptFiles: string[];
108 item: FileItem;113 item: FileItem;
109 }114 }
...@@ -139,11 +144,16 @@ async function sitegen(status: Spinner) {...@@ -139,11 +144,16 @@ async function sitegen(status: Spinner) {
139 () => cssQueue.add([item.id, cssImports, theme]),144 () => cssQueue.add([item.id, cssImports, theme]),
140 );145 );
141 // -- html --146 // -- html --
142 const bodyPromise = await ssr.ssrAsync(<Page />, {147 let page = <Page />;
148 if (layout?.default) {
149 const Layout = layout.default;
150 page = <Layout>{page}</Layout>;
151 }
152 const bodyPromise = await ssr.ssrAsync(page, {
143 sitegen: sg.initRender(),153 sitegen: sg.initRender(),
144 });154 });
145155
146 const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([156 const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
147 bodyPromise,157 bodyPromise,
148 cssPromise,158 cssPromise,
149 renderedMetaPromise,159 renderedMetaPromise,
...@@ -160,13 +170,14 @@ async function sitegen(status: Spinner) {...@@ -160,13 +170,14 @@ async function sitegen(status: Spinner) {
160 renderResults.push({170 renderResults.push({
161 body: text,171 body: text,
162 head: renderedMeta,172 head: renderedMeta,
163 inlineCss,173 css: cssBundle,
164 scriptFiles: Array.from(addon.sitegen.scripts),174 scriptFiles: Array.from(addon.sitegen.scripts),
165 item: item,175 item: item,
166 });176 });
167 }177 }
168 // This is done in two passes so that a page that throws during evaluation178 // This is done in two passes so that a page that throws during evaluation
169 // will report "Load Render Module" instead of "Render Static Page".179 // will report "Load Render Module" instead of "Render Static Page".
180 const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
170 const spinnerFormat = status.format;181 const spinnerFormat = status.format;
171 status.format = () => "";182 status.format = () => "";
172 const moduleLoadQueue = new Queue({183 const moduleLoadQueue = new Queue({
...@@ -175,7 +186,7 @@ async function sitegen(status: Spinner) {...@@ -175,7 +186,7 @@ async function sitegen(status: Spinner) {
175 getItemText,186 getItemText,
176 maxJobs: 1,187 maxJobs: 1,
177 });188 });
178 moduleLoadQueue.addMany(pages);189 moduleLoadQueue.addMany(neededPages);
179 await moduleLoadQueue.done({ method: "stop" });190 await moduleLoadQueue.done({ method: "stop" });
180 const pageQueue = new Queue({191 const pageQueue = new Queue({
181 name: "Render Static Page",192 name: "Render Static Page",
...@@ -183,7 +194,7 @@ async function sitegen(status: Spinner) {...@@ -183,7 +194,7 @@ async function sitegen(status: Spinner) {
183 getItemText,194 getItemText,
184 maxJobs: 2,195 maxJobs: 2,
185 });196 });
186 pageQueue.addMany(pages);197 pageQueue.addMany(neededPages);
187 await pageQueue.done({ method: "stop" });198 await pageQueue.done({ method: "stop" });
188 status.format = spinnerFormat;199 status.format = spinnerFormat;
189200
...@@ -207,7 +218,7 @@ async function sitegen(status: Spinner) {...@@ -207,7 +218,7 @@ async function sitegen(status: Spinner) {
207 async function doStaticFile(item: FileItem) {218 async function doStaticFile(item: FileItem) {
208 const body = await fs.readFile(item.file);219 const body = await fs.readFile(item.file);
209 await incr.putAsset({220 await incr.putAsset({
210 srcId: "static:" + item.file,221 sources: [item.file],
211 key: item.id,222 key: item.id,
212 body,223 body,
213 });224 });
...@@ -219,7 +230,9 @@ async function sitegen(status: Spinner) {...@@ -219,7 +230,9 @@ async function sitegen(status: Spinner) {
219 maxJobs: 16,230 maxJobs: 16,
220 });231 });
221 status.format = () => "";232 status.format = () => "";
222 staticQueue.addMany(staticFiles);233 staticQueue.addMany(
234 staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
235 );
223 await staticQueue.done({ method: "stop" });236 await staticQueue.done({ method: "stop" });
224 status.format = spinnerFormat;237 status.format = spinnerFormat;
225238
...@@ -227,11 +240,13 @@ async function sitegen(status: Spinner) {...@@ -227,11 +240,13 @@ async function sitegen(status: Spinner) {
227 status.text = `Concat ${renderResults.length} Pages`;240 status.text = `Concat ${renderResults.length} Pages`;
228 await Promise.all(241 await Promise.all(
229 renderResults.map(242 renderResults.map(
230 async ({ item: page, body, head, inlineCss, scriptFiles }) => {243 async (
244 { item: page, body, head, css, scriptFiles },
245 ) => {
231 const doc = wrapDocument({246 const doc = wrapDocument({
232 body,247 body,
233 head,248 head,
234 inlineCss,249 inlineCss: css.text,
235 scripts: scriptFiles.map(250 scripts: scriptFiles.map(
236 (id) =>251 (id) =>
237 UNWRAP(252 UNWRAP(
...@@ -242,7 +257,7 @@ async function sitegen(status: Spinner) {...@@ -242,7 +257,7 @@ async function sitegen(status: Spinner) {
242 ).map((x) => `{${x}}`).join("\n"),257 ).map((x) => `{${x}}`).join("\n"),
243 });258 });
244 await incr.putAsset({259 await incr.putAsset({
245 srcId: "page:" + page.file,260 sources: [page.file, ...css.sources],
246 key: page.id,261 key: page.id,
247 body: doc,262 body: doc,
248 headers: {263 headers: {
...@@ -262,7 +277,7 @@ async function sitegen(status: Spinner) {...@@ -262,7 +277,7 @@ async function sitegen(status: Spinner) {
262 status.format = spinnerFormat;277 status.format = spinnerFormat;
263 status.text = `Incremental Flush`;278 status.text = `Incremental Flush`;
264 incr.flush();279 incr.flush();
265 incr.serializeToDisk(); // Allows picking up this state again280 incr.toDisk(); // Allows picking up this state again
266 return { elapsed: (performance.now() - startTime) / 1000 };281 return { elapsed: (performance.now() - startTime) / 1000 };
267}282}
268283
...@@ -281,9 +296,11 @@ function wrapDocument({...@@ -281,9 +296,11 @@ function wrapDocument({
281 inlineCss: string;296 inlineCss: string;
282 scripts: string;297 scripts: string;
283}) {298}) {
284 return `<!doctype html><head>${head}${299 return `<!doctype html><html lang=en><head>${head}${
285 inlineCss ? `<style>${inlineCss}</style>` : ""300 inlineCss ? `<style>${inlineCss}</style>` : ""
286 }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;301 }</head><body>${body}${
302 scripts ? `<script>${scripts}</script>` : ""
303 }</body></html>`;
287}304}
288305
289import { OnceMap, Queue } from "./queue.ts";306import { OnceMap, Queue } from "./queue.ts";
framework/hot.ts-11
...@@ -34,8 +34,6 @@ export interface FileStat {...@@ -34,8 +34,6 @@ export interface FileStat {
34 cssImportsRecursive: string[] | null;34 cssImportsRecursive: string[] | null;
35 lastModified: number;35 lastModified: number;
36 imports: string[];36 imports: string[];
37 /* Used by 'incremental.ts' */
38 srcIds: string[];
39}37}
40let fsGraph = new Map<string, FileStat>();38let fsGraph = new Map<string, FileStat>();
41export function setFsGraph(g: Map<string, FileStat>) {39export function setFsGraph(g: Map<string, FileStat>) {
...@@ -60,14 +58,6 @@ Module.prototype._compile = function (...@@ -60,14 +58,6 @@ Module.prototype._compile = function (
60 filename: string,58 filename: string,
61 format: "module" | "commonjs",59 format: "module" | "commonjs",
62) {60) {
63 fs.writeMkdirSync(
64 ".clover/debug-transpilation/" +
65 path.relative(projectRoot, filename).replaceAll("\\", "/").replaceAll(
66 "../",
67 "_/",
68 ).replaceAll("/", "."),
69 content,
70 );
71 const result = ModulePrototypeUnderscoreCompile.call(61 const result = ModulePrototypeUnderscoreCompile.call(
72 this,62 this,
73 content,63 content,
...@@ -96,7 +86,6 @@ Module.prototype._compile = function (...@@ -96,7 +86,6 @@ Module.prototype._compile = function (
96 : null,86 : null,
97 imports,87 imports,
98 lastModified: stat.mtimeMs,88 lastModified: stat.mtimeMs,
99 srcIds: [],
100 });89 });
101 }90 }
102 return result;91 return result;
framework/incremental.ts+223-73
...@@ -1,54 +1,50 @@...@@ -1,54 +1,50 @@
1// `Incremental` contains multiple maps for the different parts of a site1// `Incremental` contains multiple maps for the different parts of a site
2// build, and tracks reused items across builds. It also handles emitting and2// build, and tracks reused items across builds. It also handles emitting and
3// updating the built site. This structure is self contained and serializable.3// updating the built site. This structure is self contained and serializable.
4//4
5// Tracking is simple: Files map to one or more 'source IDs', which map to one5type SourceId = string; // relative to project root, e.g. 'src/global.css'
6// or more 'artifact'. This two layer approach allows many files (say a page +6type ArtifactId = string; // `${ArtifactType}#${string}`
7// all its imports) to map to the build of a page, which produces an HTML file7type Sha1Id = string; // Sha1 hex string
8// plus a list of scripts.
98
10interface ArtifactMap {9interface ArtifactMap {
11 asset: Asset;10 asset: Asset;
12 script: string;11 script: string;
13}12}
14type AllArtifactMaps = {
15 [K in keyof ArtifactMap]: Map<string, ArtifactMap[K]>;
16};
17type ArtifactType = keyof ArtifactMap;13type ArtifactType = keyof ArtifactMap;
18
19interface Asset {14interface Asset {
20 buffer: Buffer;15 buffer: Buffer;
21 headers: Record<string, string | undefined>;16 headers: Record<string, string | undefined>;
22 hash: string;17 hash: string;
23}18}
24
25export interface PutBase {19export interface PutBase {
26 srcTag?: string; // deprecated20 sources: SourceId[];
27 srcId: string;
28 key: string;21 key: string;
29}22}
30
31export interface Put<T extends ArtifactType> extends PutBase {23export interface Put<T extends ArtifactType> extends PutBase {
32 type: T;24 type: T;
33 value: ArtifactMap[T];25 value: ArtifactMap[T];
34}26}
3527export interface Invalidations {
36export interface Output {28 lastModified: number;
37 type: ArtifactType;29 outputs: Set<ArtifactId>;
38 key: string;30 files: Set<SourceId>;
39}31}
4032
41const gzip = util.promisify(zlib.gzip);
42const zstd = util.promisify(zlib.zstdCompress);
43
44export class Incremental {33export class Incremental {
45 /** The generated artifacts */34 /** The generated artifacts */
46 out: AllArtifactMaps = {35 out: {
36 [K in keyof ArtifactMap]: Map<string, ArtifactMap[K]>;
37 } = {
47 asset: new Map(),38 asset: new Map(),
48 script: new Map(),39 script: new Map(),
49 };40 };
41 /** Tracking filesystem entries to `srcId` */
42 invals = new Map<SourceId, Invalidations>();
43 /** Tracking output keys to files */
44 sources = new Map<ArtifactId, SourceId[]>();
45
50 /** Compressed resources */46 /** Compressed resources */
51 compress = new Map<string, Compressed>();47 compress = new Map<Sha1Id, Compressed>();
52 compressQueue = new Queue<CompressJob, void>({48 compressQueue = new Queue<CompressJob, void>({
53 name: "Compress",49 name: "Compress",
54 maxJobs: 5,50 maxJobs: 5,
...@@ -56,59 +52,114 @@ export class Incremental {...@@ -56,59 +52,114 @@ export class Incremental {
56 passive: true,52 passive: true,
57 getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,53 getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
58 });54 });
59 /** Tracking filesystem entries to `srcId` */
60 files = new Map<string, hot.FileStat>();
61 srcIds = new Map<string, Output[]>();
6255
63 static fromSerialized() {56 /** Invalidation deletes build artifacts so the check is trivial. */
57 needsBuild(type: ArtifactType, key: string) {
58 return !this.out[type].has(key);
64 }59 }
65 serialize() {
66 const writer = new BufferWriter();
6760
68 const asset = Array.from(61 /*
69 this.out.asset,62 * Put built artifacts into the incremental cache. The entry points
70 ([key, { buffer, hash, headers }]) => {63 * used to build this must be provided. 'Incremental' will trace JS
71 const raw = writer.write(buffer, hash);64 * imports and file modification times tracked by 'hot.ts'.
72 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};65 */
73 const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null;66 put<T extends ArtifactType>({
74 const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null;67 sources,
75 return [key, {68 type,
76 raw,69 key,
77 gzip,70 value,
78 zstd,71 }: Put<T>) {
79 hash,72 this.out[type].set(key, value);
80 headers,
81 }];
82 },
83 );
84 const script = Array.from(this.out.script);
8573
86 const meta = Buffer.from(74 // Update sources information
87 JSON.stringify({75 ASSERT(sources.length > 0, "Missing sources for " + type + " " + key);
88 asset,76 sources = sources.map((src) => path.normalize(src));
89 script,77 const fullKey = `${type}#${key}`;
90 }),78 const prevSources = this.sources.get(fullKey);
91 "utf-8",79 const newSources = new Set(
80 sources.map((file) =>
81 path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
82 ),
92 );83 );
84 this.sources.set(fullKey, [...newSources]);
85 for (const source of prevSources ?? []) {
86 if (sources.includes(source)) {
87 newSources.delete(source);
88 continue;
89 }
90 const invals = UNWRAP(this.invals.get(source));
91 ASSERT(invals.outputs.has(fullKey));
92 invals.outputs.delete(fullKey);
93 }
94 // Use reflection from the plugin system to get imports.
95 for (const source of newSources) {
96 const invals = this.#getOrInitInvals(source);
97 invals.outputs.add(fullKey);
98 this.#followImports(source);
99 }
100 }
93101
94 const lengthBuffer = Buffer.alloc(4);102 // TODO: this doesnt remove stuff when it disappeary
95 lengthBuffer.writeUInt32LE(meta.byteLength, 0);103 #getOrInitInvals(source: string) {
104 let invals = this.invals.get(source);
105 if (!invals) {
106 const g = hot.getFsGraph().get(source);
107 const lastModified = g?.lastModified ??
108 fs.statSync(path.resolve(hot.projectRoot, source)).mtimeMs;
109 this.invals.set(
110 source,
111 invals = {
112 lastModified,
113 files: new Set(),
114 outputs: new Set(),
115 },
116 );
117 }
118 return invals;
119 }
96120
97 return Buffer.concat([meta, lengthBuffer, ...writer.buffers]);121 #followImports(file: string) {
122 const graph = hot.getFsGraph();
123 const stat = graph.get(file);
124 if (!stat) return;
125 for (const i of stat.imports) {
126 const invals = this.#getOrInitInvals(i);
127 invals.files.add(file);
128 this.#followImports(i);
129 }
98 }130 }
99131
100 serializeToDisk(file = ".clover/incr.state") {132 async statAllFiles() {
101 const buffer = this.serialize();133 for (const file of this.invals.keys()) {
102 fs.writeFileSync(file, buffer);134 const mtime = fs.statSync(file).mtimeMs;
135 this.updateStat(file, mtime);
136 }
103 }137 }
104138
105 put<T extends ArtifactType>({139 updateStat(fileKey: string, newLastModified: number) {
106 srcId,140 const stat = this.invals.get(fileKey);
107 type,141 ASSERT(stat, "Updated stat on untracked file " + fileKey);
108 key,142 if (stat.lastModified < newLastModified) {
109 value,143 // Invalidate
110 }: Put<T>) {144 console.log(fileKey + " updated");
111 this.out[type].set(key, value);145 const invalidQueue = [fileKey];
146 let currentInvalid;
147 while (currentInvalid = invalidQueue.pop()) {
148 const invalidations = this.invals.get(currentInvalid);
149 ASSERT(
150 invalidations,
151 "No reason to track file '" + currentInvalid +
152 "' if it has no invalidations",
153 );
154 const { files, outputs } = invalidations;
155 for (const out of outputs) {
156 const [type, artifactKey] = out.split("#", 2);
157 this.out[type as ArtifactType].delete(artifactKey);
158 }
159 invalidQueue.push(...files);
160 }
161 }
162 stat.lastModified = newLastModified;
112 }163 }
113164
114 async putAsset(info: PutAsset) {165 async putAsset(info: PutAsset) {
...@@ -161,7 +212,99 @@ export class Incremental {...@@ -161,7 +212,99 @@ export class Incremental {
161 entry![algo] = out;212 entry![algo] = out;
162 }213 }
163214
164 invalidate(srcId: string) {215 serialize() {
216 const writer = new BufferWriter();
217
218 const asset = Array.from(
219 this.out.asset,
220 ([key, { buffer, hash, headers }]) => {
221 const raw = writer.write(buffer, hash);
222 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
223 const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
224 const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
225 return [key, {
226 raw,
227 gzip,
228 zstd,
229 hash,
230 headers: headers as Record<string, string>,
231 }] satisfies SerializedMeta["asset"][0];
232 },
233 );
234 const script = Array.from(this.out.script);
235 const invals = Array.from(this.invals, ([key, value]) => {
236 const { lastModified, files, outputs } = value;
237 return [key, {
238 m: lastModified,
239 f: [...files],
240 o: [...outputs],
241 }] satisfies SerializedMeta["invals"][0];
242 });
243 const sources = Array.from(this.sources, ([key, value]) => {
244 return [key, ...value] as [string, ...string[]];
245 });
246 const json = {
247 asset,
248 script,
249 invals,
250 sources,
251 } satisfies SerializedMeta;
252 const meta = Buffer.from(JSON.stringify(json), "utf-8");
253
254 const lengthBuffer = Buffer.alloc(4);
255 lengthBuffer.writeUInt32LE(meta.byteLength, 0);
256
257 return Buffer.concat([lengthBuffer, meta, ...writer.buffers]);
258 }
259
260 static fromSerialized(buffer: Buffer): Incremental {
261 const metaLength = buffer.readUint32LE(0);
262 const meta: SerializedMeta = JSON.parse(
263 buffer.subarray(4, 4 + metaLength).toString("utf8"),
264 );
265 const view = ([start, end]: View) =>
266 buffer.subarray(4 + metaLength + start, 4 + metaLength + end);
267
268 const incr = new Incremental();
269 incr.out.asset = new Map<string, Asset>(meta.asset.map(([key, value]) => {
270 const { hash, raw, gzip, zstd, headers } = value;
271 if ((gzip || zstd) && !incr.compress.has(hash)) {
272 incr.compress.set(hash, {
273 gzip: gzip ? view(gzip) : undefined,
274 zstd: zstd ? view(zstd) : undefined,
275 });
276 }
277 return [key, {
278 buffer: view(raw),
279 headers: headers,
280 hash: hash,
281 }];
282 }));
283 incr.out.script = new Map(meta.script);
284 incr.invals = new Map(meta.invals.map(([key, { m, f, o }]) => {
285 return [key, {
286 lastModified: m,
287 files: new Set(f),
288 outputs: new Set(o),
289 }];
290 }));
291 incr.sources = new Map(meta.sources.map(([key, ...value]) => [key, value]));
292 return incr;
293 }
294
295 toDisk(file = ".clover/incr.state") {
296 const buffer = this.serialize();
297 fs.writeFileSync(file, buffer);
298 }
299
300 static fromDisk(file = ".clover/incr.state"): Incremental {
301 try {
302 const buffer = fs.readFileSync(file);
303 return Incremental.fromSerialized(buffer);
304 } catch (err: any) {
305 if (err.code === "ENOENT") return new Incremental();
306 throw err;
307 }
165 }308 }
166309
167 async wait() {310 async wait() {
...@@ -171,18 +314,14 @@ export class Incremental {...@@ -171,18 +314,14 @@ export class Incremental {
171 async flush() {314 async flush() {
172 ASSERT(!this.compressQueue.active);315 ASSERT(!this.compressQueue.active);
173 const writer = new BufferWriter();316 const writer = new BufferWriter();
317 // TODO: ensure all assets are actually compressed and not fake lying.
174 const asset = Object.fromEntries(318 const asset = Object.fromEntries(
175 Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {319 Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
176 const raw = writer.write(buffer, hash);320 const raw = writer.write(buffer, hash);
177 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};321 const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
178 const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;322 const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
179 const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;323 const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
180 return [key, {324 return [key, { raw, gzip, zstd, headers }];
181 raw,
182 gzip,
183 zstd,
184 headers,
185 }];
186 }),325 }),
187 );326 );
188 await Promise.all([327 await Promise.all([
...@@ -230,10 +369,10 @@ class BufferWriter {...@@ -230,10 +369,10 @@ class BufferWriter {
230369
231export type View = [start: number, end: number];370export type View = [start: number, end: number];
232371
233// Alongside this type is a byte buffer, containing all the assets.
234export interface BuiltAssetMap {372export interface BuiltAssetMap {
235 [route: string]: BuiltAsset;373 [route: string]: BuiltAsset;
236}374}
375
237export interface BuiltAsset {376export interface BuiltAsset {
238 raw: View;377 raw: View;
239 gzip: View;378 gzip: View;
...@@ -249,13 +388,24 @@ export interface SerializedMeta {...@@ -249,13 +388,24 @@ export interface SerializedMeta {
249 hash: string;388 hash: string;
250 headers: Record<string, string>;389 headers: Record<string, string>;
251 }]>;390 }]>;
252 script: [key: string, value: string][];391 script: Array<[key: string, value: string]>;
392 invals: Array<[key: string, {
393 /** Modified */
394 m: number;
395 f: SourceId[];
396 o: ArtifactId[];
397 }]>;
398 sources: Array<[string, ...string[]]>;
253}399}
254400
401const gzip = util.promisify(zlib.gzip);
402const zstd = util.promisify(zlib.zstdCompress);
403
255import * as fs from "#sitegen/fs";404import * as fs from "#sitegen/fs";
256import * as zlib from "node:zlib";405import * as zlib from "node:zlib";
257import * as util from "node:util";406import * as util from "node:util";
258import { Queue } from "./queue.ts";407import { Queue } from "./queue.ts";
259import * as hot from "./hot.ts";408import * as hot from "./hot.ts";
260import * as mime from "#sitegen/mime";409import * as mime from "#sitegen/mime";
410import * as path from "node:path";
261import { Buffer } from "node:buffer";411import { Buffer } from "node:buffer";
framework/lib/fs.ts+1-1
...@@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) {...@@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) {
3434
35export function readDirRecOptionalSync(dir: string) {35export function readDirRecOptionalSync(dir: string) {
36 try {36 try {
37 return readdirSync(dir, { recursive: true, withFileTypes: true });37 return readdirSync(dir, { recursive: true, encoding: "utf8" });
38 } catch (err: any) {38 } catch (err: any) {
39 if (err.code === "ENOENT") return [];39 if (err.code === "ENOENT") return [];
40 throw err;40 throw err;
readme.md+5
...@@ -33,6 +33,11 @@ Included is `src`, which contains `paperclover.net`. Website highlights:...@@ -33,6 +33,11 @@ Included is `src`, which contains `paperclover.net`. Website highlights:
33 33
34## Development 34## Development
35 35
36minimum system requirements:
37- a cpu with at least 1 core.
38- random access memory.
39- windows 7 or later, macos, or linux operating system.
40
36``` 41```
37npm install 42npm install
38 43
src/q+a/components/QuestionForm.marko created+13
...@@ -0,0 +1,13 @@
1<form action="/q+a" method="POST">
2 <textarea
3 name="text"
4 placeholder="ask clover a question..."
5 required
6 minlength="1"
7 maxlength="10000"
8 />
9 <div aria-hidden class="title">ask a question</div>
10 <button type="submit">send</button>
11 <div class="disabled-button">send</div>
12</form>
13
src/q+a/pages/q+a.marko+5-14
...@@ -3,7 +3,6 @@ export * as layout from "../layout.tsx";...@@ -3,7 +3,6 @@ export * as layout from "../layout.tsx";
3export interface Input {3export interface Input {
4 admin?: boolean;4 admin?: boolean;
5}5}
6
7export const meta: Metadata = {6export const meta: Metadata = {
8 title: "paper clover q+a",7 title: "paper clover q+a",
9 description: "ask clover a question",8 description: "ask clover a question",
...@@ -12,24 +11,16 @@ export const meta: Metadata = {...@@ -12,24 +11,16 @@ export const meta: Metadata = {
12<const/{ admin = false } = input />11<const/{ admin = false } = input />
13<const/questions = [...Question.getAll()] />12<const/questions = [...Question.getAll()] />
1413
15<form action="/q+a" method="POST">14<if=!admin>
16 <textarea15 <QuestionForm />
17 name="text"16</>
18 placeholder="ask clover a question..."
19 required
20 minlength="1"
21 maxlength="10000"
22 />
23 <div aria-hidden class="title">ask a question</div>
24 <button type="submit">send</button>
25 <div class="disabled-button">send</div>
26</form>
27<for|q| of=questions>17<for|q| of=questions>
28 <QuestionRender question=q admin=admin />;18 <QuestionRender question=q admin=admin />
29</>19</>
30<footer>20<footer>
31 fun fact: clover has answered ${questions.length} questions21 fun fact: clover has answered ${questions.length} questions
32</footer>22</footer>
3323
34import { Question } from "@/q+a/models/Question.ts";24import { Question } from "@/q+a/models/Question.ts";
25import { QuestionForm } from "@/q+a/components/QuestionForm.marko";
35import QuestionRender from '@/q+a/components/Question.marko';26import QuestionRender from '@/q+a/components/Question.marko';
src/static/robots.txt+5-58
...@@ -1,59 +1,6 @@...@@ -1,59 +1,6 @@
1// __esModule1User-agent: *
2true2Allow: /
33
4// getEnd4# Restrict access to media files to ensure bots do not unintentionally
5function getEnd(file, node) {5# break their license agreement (https://paperclover.net/license)
6 // Restore if merged: https://github.com/babel/babel/pull/168496Disallow: /file
7 // if (node.end != null) {
8 // return node.end;
9 // }
10
11 if (node.loc) {
12 return locToIndex(file, node.loc.end);
13 }
14
15 return null;
16}
17
18// getLoc
19function getLoc(file, index) {
20 return findLoc(getLineIndexes(file), 0, index);
21}
22
23// getLocRange
24function getLocRange(file, start, end) {
25 const lineIndexes = getLineIndexes(file);
26 const startLoc = findLoc(lineIndexes, 0, start);
27
28 if (startLoc) {
29 const endLoc =
30 start === end ? startLoc : findLoc(lineIndexes, startLoc.line - 1, end);
31
32 return {
33 start: startLoc,
34 end: endLoc
35 };
36 }
37}
38
39// getStart
40function getStart(file, node) {
41 // Restore if merged: https://github.com/babel/babel/pull/16849
42 // if (node.start != null) {
43 // return node.start;
44 // }
45
46 if (node.loc) {
47 return locToIndex(file, node.loc.start);
48 }
49
50 return null;
51}
52
53// withLoc
54function withLoc(file, node, start, end) {
55 node.loc = getLocRange(file, start, end);
56 node.start = start;
57 node.end = end;
58 return node;
59}
src/static/rss.xml+43-115
...@@ -1,115 +1,43 @@...@@ -1,115 +1,43 @@
1// __esModule1<?xml version="1.0" encoding="UTF-8" ?>
2true2<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
33
4// parseArgs4<channel>
5function parseArgs(file, str, sourceStart, sourceEnd) {5 <title>paper clover</title>
6 const parsed = parseExpression(file, `_(${str})`, sourceStart, sourceEnd, 2);6 <description>art</description>
77 <link>https://paperclover.net</link>
8 if (parsed.type === "CallExpression") {8 <item>
9 return parsed.arguments;9 <title>clo's files &amp; cotyledon</title>
10 }10 <link>https://paperclover.net/file</link>
1111 <description>
12 return [ensureParseError(file, parsed, sourceStart, sourceEnd)];12 i rewrote the file viewer to be nicer to use. as well as uploading about
13}13 10gb of old content. there's over a dozen hours worth of shit to explore.
1414 </description>
15// parseExpression15 <pubDate>Wed, 30 Apr 2025 00:00:00 -0000</pubDate>
16function parseExpression(16 <guid>https://paperclover.net/file</guid>
17file,17 </item>
18str,18 <item>
19sourceStart,19 <title>waterfalls and gender identity update</title>
20sourceEnd,20 <link>https://paperclover.net/waterfalls</link>
21sourceOffset)21 <description>
22{22 this is a song about identity, jealousness, and the seasons. more
23 return tryParse(file, true, str, sourceStart, sourceEnd, sourceOffset);23 specifically, it's a personal account of how it felt realizing i
24}24 was a transgender woman. enjoy.
2525 </description>
26// parseParams26 <pubDate>Wed, 01 Jan 2025 00:00:00 -0000</pubDate>
27function parseParams(file, str, sourceStart, sourceEnd) {27 <guid>https://paperclover.net/waterfalls</guid>
28 const parsed = parseExpression(28
29 file,29 <enclosure url="https://paperclover.net/file/2025/waterfalls/waterfalls.mp4"
30 `(${str})=>{}`,30 type="video/mp4"/>
31 sourceStart,31 <media:content url="https://paperclover.net/file/2025/waterfalls/waterfalls.mp4"
32 sourceEnd,32 type="video/mp4"
33 133 width="1920"
34 );34 height="1080">
3535 </media:content>
36 if (parsed.type === "ArrowFunctionExpression") {36 <media:thumbnail url="https://paperclover.net/file/2025/waterfalls/thumbnail.jpeg"
37 return parsed.params;37 width="1920"
38 }38 height="1080"
3939 />
40 return [ensureParseError(file, parsed, sourceStart, sourceEnd)];40 </item>
41}41</channel>
4242
43// parseStatements43</rss>
44function parseStatements(
45file,
46str,
47sourceStart,
48sourceEnd,
49sourceOffset)
50{
51 return tryParse(file, false, str, sourceStart, sourceEnd, sourceOffset);
52}
53
54// parseTemplateLiteral
55function parseTemplateLiteral(file, str, sourceStart, sourceEnd) {
56 const parsed = parseExpression(
57 file,
58 "`" + str + "`",
59 sourceStart,
60 sourceEnd,
61 1
62 );
63
64 if (parsed.type === "TemplateLiteral") {
65 return _compiler.types.templateLiteral(parsed.quasis, parsed.expressions);
66 }
67
68 return ensureParseError(file, parsed, sourceStart, sourceEnd);
69}
70
71// parseTypeArgs
72function parseTypeArgs(file, str, sourceStart, sourceEnd) {
73 const parsed = parseExpression(file, `_<${str}>`, sourceStart, sourceEnd, 2);
74
75 if (parsed.type === "TSInstantiationExpression") {
76 // typeArguments is Flow only (not TS), we need to use typeParameters
77 return parsed.typeParameters;
78 }
79
80 return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
81}
82
83// parseTypeParams
84function parseTypeParams(file, str, sourceStart, sourceEnd) {
85 const parsed = parseExpression(
86 file,
87 `<${str}>()=>{}`,
88 sourceStart,
89 sourceEnd,
90 1
91 );
92
93 if (parsed.type === "ArrowFunctionExpression") {
94 return parsed.typeParameters;
95 }
96
97 return [ensureParseError(file, parsed, sourceStart, sourceEnd)];
98}
99
100// parseVar
101function parseVar(file, str, sourceStart, sourceEnd) {
102 const parsed = parseExpression(
103 file,
104 `(${str})=>{}`,
105 sourceStart,
106 sourceEnd,
107 1
108 );
109
110 if (parsed.type === "ArrowFunctionExpression" && parsed.params.length === 1) {
111 return parsed.params[0];
112 }
113
114 return ensureParseError(file, parsed, sourceStart, sourceEnd);
115}
src/static/wurtzhome.html+66-30
...@@ -1,30 +1,66 @@...@@ -1,30 +1,66 @@
1// __esModule1<html lang="en">
2true2 <head>
33 <title>paper clover</title>
4// getTagDefForTagName4 </head>
5function getTagDefForTagName(file, tagName) {5 <body bgcolor="black" style="word-wrap: initial">
6 const tagDef = getTaglibLookup(file).getTag(tagName);6 <main style="display: flex; flex-direction: column; justify-content: center; align-items: center; height: 100vh">
77 <div>
8 if (tagDef) {8 <p style="margin: 0.5rem 0">
9 let seen = file.metadata.marko[SEEN_TAGS_KEY];9 <a
10 if (!seen) {10 href="/waterfalls"
11 seen = file.metadata.marko[SEEN_TAGS_KEY] = new Set();11 style="text-decoration: none; font-family: times; font-size: 48px"
12 }12 >
1313 <font color="#FFB5E8">water</font><font color="#FFE3F6">falls</font>
14 if (!seen.has(tagDef)) {14 </a>
15 seen.add(tagName);15 </p>
16 const { filePath } = tagDef;16 <p style="margin: 0.5rem 0">
17 const len = filePath.length;17 <a
1818 href="/q+a"
19 if (filePath[len - 14] === "m" && filePath.endsWith("marko-tag.json")) {19 style="text-decoration: none; font-family: times; font-size: 48px"
20 file.metadata.marko.watchFiles.push(filePath);20 >
21 }21 <font color="#97E5B9">questions</font>
22 }22 <font color="#D4F5E2">and answers</font>
23 }23 </a>
24 return tagDef;24 </p>
25}25 <p style="margin: 0.5rem 0">
2626 <a
27// getTaglibLookup27 href="/file"
28function getTaglibLookup(file) {28 style="text-decoration: none; font-family: times; font-size: 48px"
29 return file.___taglibLookup;29 >
30}30 <font color="#B28DFF">file</font>
31 <font color="#DDC6FF">browser</font>
32 </a>
33 </p>
34 <p style="margin: 0.5rem 0">
35 <a
36 href="/discord"
37 style="text-decoration: none; font-family: times; font-size: 48px"
38 >
39 <font color="#5865F2">dis</font><font color="#7289DA">cord</font>
40 </a>
41 </p>
42 <p style="margin: 0.5rem 0">
43 <a
44 href="/github"
45 style="text-decoration: none; font-family: times; font-size: 48px"
46 >
47 <font color="#ffffff">git</font><font color="#959DA5">hub</font>
48 </a>
49 </p>
50 <p style="margin: 0.5rem 0">
51 <a
52 href="/rss.xml"
53 style="text-decoration: none; font-family: times; font-size: 48px"
54 >
55 <font color="#F26522">rss</font>
56 <font color="#FF8147">feed</font>
57 </a>
58 </p>
59 <h1 style="margin: -1.5rem 0 3rem 0; font-size: 7rem; font-weight: 400; font-family: times">
60 <font color="#B8E1FF">paper</font>
61 <font color="#E8F4FF">clover</font>
62 </h1>
63 </div>
64 </main>
65 </body>
66</html>