authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-10 20:06:32-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:44-07:00
loga176037c3c37afa39eac946515d3d5a533076674
tree34f54f82f86c0052cd9ec9822deff416d30593aa
parent73cb20561b7cc88cebc4d2be5bff899efa8d80d5
signature Commit is signed but in an unrecognized format.

add a file watcher, live rebuild.

this is only verified functional on windows 7

8 files changed, 190 insertions(+), 47 deletions(-)

framework/bundle.ts+11-19
...@@ -21,8 +21,8 @@ export async function bundleClientJavaScript(...@@ -21,8 +21,8 @@ export async function bundleClientJavaScript(
21 if (invalidFiles.length > 0) {21 if (invalidFiles.length > 0) {
22 const cwd = process.cwd();22 const cwd = process.cwd();
23 throw new Error(23 throw new Error(
24 "All client-side scripts should be named like '.client.ts'. Exceptions: " +24 "All client-side scripts should be named like '.client.ts'. Exceptions: \n" +
25 invalidFiles.map((x) => path.join(cwd, x)).join(","),25 invalidFiles.map((x) => path.join(cwd, x)).join("\n"),
26 );26 );
27 }27 }
2828
...@@ -36,9 +36,9 @@ export async function bundleClientJavaScript(...@@ -36,9 +36,9 @@ export async function bundleClientJavaScript(
36 minify: !dev,36 minify: !dev,
37 outdir: "/out!",37 outdir: "/out!",
38 plugins: clientPlugins,38 plugins: clientPlugins,
39 splitting: true,
40 write: false,39 write: false,
41 metafile: true,40 metafile: true,
41 external: ["node_modules/"],
42 });42 });
43 if (bundle.errors.length || bundle.warnings.length) {43 if (bundle.errors.length || bundle.warnings.length) {
44 throw new AggregateError(44 throw new AggregateError(
...@@ -61,11 +61,12 @@ export async function bundleClientJavaScript(...@@ -61,11 +61,12 @@ export async function bundleClientJavaScript(
61 // Register non-chunks as script entries.61 // Register non-chunks as script entries.
62 const chunk = route.startsWith("/js/c.");62 const chunk = route.startsWith("/js/c.");
63 if (!chunk) {63 if (!chunk) {
64 route = route.replace(".client.js", ".js");64 const key = hot.getScriptId(sources[0]);
65 route = "/js/" + key + ".js";
65 incr.put({66 incr.put({
66 sources,67 sources,
67 type: "script",68 type: "script",
68 key: route.slice("/js/".length, -".js".length),69 key,
69 value: text,70 value: text,
70 });71 });
71 }72 }
...@@ -130,23 +131,14 @@ export async function bundleServerJavaScript(...@@ -130,23 +131,14 @@ export async function bundleServerJavaScript(
130 virtualFiles({131 virtualFiles({
131 "$views": viewSource,132 "$views": viewSource,
132 }),133 }),
133 // banFiles([
134 // "hot.ts",
135 // "incremental.ts",
136 // "bundle.ts",
137 // "generate.ts",
138 // "css.ts",
139 // ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))),
140 projectRelativeResolution(),134 projectRelativeResolution(),
141 {135 {
142 name: "marko",136 name: "marko via build cache",
143 setup(b) {137 setup(b) {
144 b.onLoad({ filter: /\.marko$/ }, async ({ path: file }) => {138 b.onLoad({ filter: /\.marko$/ }, async ({ path: file }) => ({
145 return {139 loader: "ts",
146 loader: "ts",140 contents: hot.getSourceCode(file),
147 contents: hot.getSourceCode(file),141 }));
148 };
149 });
150 },142 },
151 },143 },
152 {144 {
framework/generate.ts+21-18
...@@ -1,20 +1,29 @@...@@ -1,20 +1,29 @@
1export function main() {1export function main(incremental?: Incremental) {
2 return withSpinner({2 return withSpinner<Record<string, unknown>, any>({
3 text: "Recovering State",3 text: "Recovering State",
4 successText: ({ elapsed }) =>4 successText,
5 "sitegen! update in " + elapsed.toFixed(1) + "s",
6 failureText: () => "sitegen FAIL",5 failureText: () => "sitegen FAIL",
7 }, sitegen);6 }, async (spinner) => {
7 const incr = Incremental.fromDisk();
8 await incr.statAllFiles();
9 const result = await sitegen(spinner, incr);
10 incr.toDisk(); // Allows picking up this state again
11 return result;
12 }) as ReturnType<typeof sitegen>;
8}13}
914
10async function sitegen(status: Spinner) {15export function successText({ elapsed }: { elapsed: number }) {
16 return "sitegen! update in " + elapsed.toFixed(1) + "s";
17}
18
19export async function sitegen(
20 status: Spinner,
21 incr: Incremental,
22) {
11 const startTime = performance.now();23 const startTime = performance.now();
1224
13 let root = path.resolve(import.meta.dirname, "../src");25 let root = path.resolve(import.meta.dirname, "../src");
14 const join = (...sub: string[]) => path.join(root, ...sub);26 const join = (...sub: string[]) => path.join(root, ...sub);
15 const incr = new Incremental();
16 // const incr = Incremental.fromDisk();
17 await incr.statAllFiles();
1827
19 // Sitegen reviews every defined section for resources to process28 // Sitegen reviews every defined section for resources to process
20 const sections: sg.Section[] =29 const sections: sg.Section[] =
...@@ -213,7 +222,7 @@ async function sitegen(status: Spinner) {...@@ -213,7 +222,7 @@ async function sitegen(status: Spinner) {
213 ...renderResults.flatMap((r) => r.clientRefs),222 ...renderResults.flatMap((r) => r.clientRefs),
214 ...backend.views.flatMap((r) => r.clientRefs),223 ...backend.views.flatMap((r) => r.clientRefs),
215 ]),224 ]),
216 (script) => path.join(hot.projectSrc, script),225 (script) => path.resolve(hot.projectSrc, script),
217 );226 );
218 const extraPublicScripts = scripts.map((entry) => entry.file);227 const extraPublicScripts = scripts.map((entry) => entry.file);
219 const uniqueCount = new Set([228 const uniqueCount = new Set([
...@@ -264,12 +273,7 @@ async function sitegen(status: Spinner) {...@@ -264,12 +273,7 @@ async function sitegen(status: Spinner) {
264 head,273 head,
265 inlineCss: css.text,274 inlineCss: css.text,
266 scripts: scriptFiles.map(275 scripts: scriptFiles.map(
267 (id) =>276 (file) => UNWRAP(incr.out.script.get(hot.getScriptId(file))),
268 UNWRAP(
269 incr.out.script.get(
270 path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
271 ),
272 ),
273 ).map((x) => `{${x}}`).join("\n"),277 ).map((x) => `{${x}}`).join("\n"),
274 });278 });
275 await incr.putAsset({279 await incr.putAsset({
...@@ -293,8 +297,7 @@ async function sitegen(status: Spinner) {...@@ -293,8 +297,7 @@ async function sitegen(status: Spinner) {
293 status.format = spinnerFormat;297 status.format = spinnerFormat;
294 status.text = `Incremental Flush`;298 status.text = `Incremental Flush`;
295 incr.flush(); // Write outputs299 incr.flush(); // Write outputs
296 incr.toDisk(); // Allows picking up this state again300 return { incr, elapsed: (performance.now() - startTime) / 1000 };
297 return { elapsed: (performance.now() - startTime) / 1000 };
298}301}
299302
300function getItemText({ file }: FileItem) {303function getItemText({ file }: FileItem) {
framework/hot.ts+11-3
...@@ -155,7 +155,7 @@ function resolveClientRef(sourcePath: string, ref: string) {...@@ -155,7 +155,7 @@ function resolveClientRef(sourcePath: string, ref: string) {
155 ) {155 ) {
156 throw new Error("addScript must be a .client.ts or .client.tsx");156 throw new Error("addScript must be a .client.ts or .client.tsx");
157 }157 }
158 return path.relative(projectSrc, filePath);158 return filePath;
159}159}
160160
161function loadMarko(module: NodeJS.Module, filepath: string) {161function loadMarko(module: NodeJS.Module, filepath: string) {
...@@ -171,7 +171,9 @@ function loadMarko(module: NodeJS.Module, filepath: string) {...@@ -171,7 +171,9 @@ function loadMarko(module: NodeJS.Module, filepath: string) {
171 const ref = JSON.parse(`"${src.slice(1, -1)}"`);171 const ref = JSON.parse(`"${src.slice(1, -1)}"`);
172 const resolved = resolveClientRef(filepath, ref);172 const resolved = resolveClientRef(filepath, ref);
173 scannedClientRefs.add(resolved);173 scannedClientRefs.add(resolved);
174 return `<CloverScriptInclude=${JSON.stringify(resolved)} />`;174 return `<CloverScriptInclude=${
175 JSON.stringify(getScriptId(resolved))
176 } />`;
175 },177 },
176 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';178 ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
177 }179 }
...@@ -287,11 +289,17 @@ export function resolveClientRefs(...@@ -287,11 +289,17 @@ export function resolveClientRefs(
287 const ref = JSON.parse(`"${arg.slice(1, -1)}"`);289 const ref = JSON.parse(`"${arg.slice(1, -1)}"`);
288 const resolved = resolveClientRef(filepath, ref);290 const resolved = resolveClientRef(filepath, ref);
289 scannedClientRefs.add(resolved);291 scannedClientRefs.add(resolved);
290 return `${call}(${JSON.stringify(resolved)})`;292 return `${call}(${JSON.stringify(getScriptId(resolved))})`;
291 });293 });
292 return { code, refs: Array.from(scannedClientRefs) };294 return { code, refs: Array.from(scannedClientRefs) };
293}295}
294296
297export function getScriptId(file: string) {
298 return (path.isAbsolute(file) ? path.relative(projectSrc, file) : file)
299 .replace(/^\/?src\//, "")
300 .replaceAll("\\", "/");
301}
302
295declare global {303declare global {
296 namespace NodeJS {304 namespace NodeJS {
297 interface Module {305 interface Module {
framework/incremental.ts+9-6
...@@ -136,13 +136,15 @@ export class Incremental {...@@ -136,13 +136,15 @@ export class Incremental {
136 }136 }
137 }137 }
138138
139 updateStat(fileKey: string, newLastModified: number) {139 updateStat(file: string, newLastModified: number) {
140 const stat = this.invals.get(fileKey);140 file = path.relative(hot.projectRoot, file);
141 ASSERT(stat, "Updated stat on untracked file " + fileKey);141 const stat = this.invals.get(file);
142 if (stat.lastModified < newLastModified) {142 ASSERT(stat, "Updated stat on untracked file " + file);
143 const hasUpdate = stat.lastModified < newLastModified;
144 if (hasUpdate) {
143 // Invalidate145 // Invalidate
144 console.info(fileKey + " updated");146 console.info(file + " updated");
145 const invalidQueue = [fileKey];147 const invalidQueue = [file];
146 let currentInvalid;148 let currentInvalid;
147 while (currentInvalid = invalidQueue.pop()) {149 while (currentInvalid = invalidQueue.pop()) {
148 const invalidations = this.invals.get(currentInvalid);150 const invalidations = this.invals.get(currentInvalid);
...@@ -160,6 +162,7 @@ export class Incremental {...@@ -160,6 +162,7 @@ export class Incremental {
160 }162 }
161 }163 }
162 stat.lastModified = newLastModified;164 stat.lastModified = newLastModified;
165 return hasUpdate;
163 }166 }
164167
165 async putAsset(info: PutAsset) {168 async putAsset(info: PutAsset) {
framework/lib/view.ts+4-1
...@@ -9,6 +9,9 @@ export interface View {...@@ -9,6 +9,9 @@ export interface View {
9 scripts: Record<string, string>; 9 scripts: Record<string, string>;
10} 10}
11 11
12let views: Record<string, View> = null!;
13let scripts: Record<string, string> = null!;
14
12// An older version of the Clover Engine supported streaming suspense 15// An older version of the Clover Engine supported streaming suspense
13// boundaries, but those were never used. Pages will wait until they 16// boundaries, but those were never used. Pages will wait until they
14// are fully rendered before sending. 17// are fully rendered before sending.
...@@ -17,7 +20,7 @@ export async function renderView(...@@ -17,7 +20,7 @@ export async function renderView(
17 id: string, 20 id: string,
18 props: Record<string, unknown>, 21 props: Record<string, unknown>,
19) { 22) {
20 const { views, scripts } = require("$views"); 23 views ?? ({ views, scripts } = require("$views"));
21 // The view contains pre-bundled CSS and scripts, but keeps the scripts 24 // The view contains pre-bundled CSS and scripts, but keeps the scripts
22 // separate for run-time dynamic scripts. For example, the file viewer 25 // separate for run-time dynamic scripts. For example, the file viewer
23 // includes the canvas for the current page, but only the current page. 26 // includes the canvas for the current page, but only the current page.
framework/watch.ts created+132
...@@ -0,0 +1,132 @@
1// File watcher and live reloading site generator
2
3const debounceMilliseconds = 25;
4
5export async function main() {
6 // Catch up state
7 const { incr } = await generate.main();
8
9 // Initialize a watch
10 const watch = new Watch(rebuild);
11 statusLine();
12
13 function rebuild(files: string[]) {
14 files = files.map((file) => path.relative(hot.projectRoot, file));
15 const changed: string[] = [];
16 for (const file of files) {
17 if (incr.updateStat(file, fs.statSync(file).mtimeMs)) changed.push(file);
18 }
19 if (changed.length === 0) {
20 console.warn("Files were modified but the 'modify' time did not change.");
21 return;
22 }
23 withSpinner<Record<string, unknown>, any>({
24 text: "Recovering State",
25 successText: generate.successText,
26 failureText: () => "sitegen FAIL",
27 }, async (spinner) => {
28 console.clear();
29 console.log(
30 "Updated" +
31 (changed.length === 1
32 ? " " + changed[0]
33 : changed.map((file) => "\n- " + file)),
34 );
35 const result = await generate.sitegen(spinner, incr);
36 incr.toDisk(); // Allows picking up this state again
37 return result;
38 }).catch((err) => {
39 console.error(util.inspect(err));
40 }).finally(statusLine);
41 }
42
43 function statusLine() {
44 watch.add(...incr.invals.keys());
45 console.info(
46 `Watching ${incr.invals.size} files \x1b[36m[last change: ${
47 new Date().toLocaleTimeString()
48 }]\x1b[39m`,
49 );
50 }
51}
52
53class Watch {
54 files = new Set<string>();
55 stale = new Set<string>();
56 onChange: (files: string[]) => void;
57 watchers: fs.FSWatcher[] = [];
58 /** Has a trailing slash */
59 roots: string[] = [];
60 debounce: ReturnType<typeof setTimeout> | null = null;
61
62 constructor(onChange: Watch["onChange"]) {
63 this.onChange = onChange;
64 }
65
66 add(...files: string[]) {
67 const { roots, watchers } = this;
68 let newRoots: string[] = [];
69 for (let file of files) {
70 file = path.resolve(file);
71 if (this.files.has(file)) continue;
72 this.files.add(file);
73 // Find an existing watcher
74 if (roots.some((root) => file.startsWith(root))) continue;
75 if (newRoots.some((root) => file.startsWith(root))) continue;
76 newRoots.push(path.dirname(file) + path.sep);
77 }
78 if (newRoots.length === 0) return;
79 // Filter out directories that are already specified
80 newRoots = newRoots
81 .sort((a, b) => a.length - b.length)
82 .filter((dir, i, a) => {
83 for (let j = 0; j < i; j++) if (dir.startsWith(a[j])) return false;
84 return true;
85 });
86 // Append Watches
87 let i = roots.length;
88 for (const root of newRoots) {
89 this.watchers.push(fs.watch(
90 root,
91 { recursive: true, encoding: "utf-8" },
92 this.#handleEvent.bind(this, root),
93 ));
94 this.roots.push(root);
95 }
96 // If any new roots shadow over and old one, delete it!
97 while (i > 0) {
98 i -= 1;
99 const root = roots[i];
100 if (newRoots.some((newRoot) => root.startsWith(newRoot))) {
101 watchers.splice(i, 1)[0].close();
102 roots.splice(i, 1);
103 }
104 }
105 }
106
107 stop() {
108 for (const w of this.watchers) w.close();
109 }
110
111 #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) {
112 if (!subPath) return;
113 const file = path.join(root, subPath);
114 if (!this.files.has(file)) return;
115 this.stale.add(file);
116 const { debounce } = this;
117 if (debounce !== null) clearTimeout(debounce);
118 this.debounce = setTimeout(() => {
119 this.debounce = null;
120 this.onChange(Array.from(this.stale));
121 this.stale.clear();
122 }, debounceMilliseconds);
123 }
124}
125
126import { Incremental } from "./incremental.ts";
127import * as fs from "node:fs";
128import { Spinner, withSpinner } from "@paperclover/console/Spinner";
129import * as generate from "./generate.ts";
130import * as path from "node:path";
131import * as util from "node:util";
132import * as hot from "./hot.ts";
src/global.css+1
...@@ -118,3 +118,4 @@ code {...@@ -118,3 +118,4 @@ code {
118 font-family: "rmo", monospace;118 font-family: "rmo", monospace;
119 font-size: inherit;119 font-size: inherit;
120}120}
121
tsconfig.json+1
...@@ -13,6 +13,7 @@...@@ -13,6 +13,7 @@
13 "rootDir": ".",13 "rootDir": ".",
14 "skipLibCheck": true,14 "skipLibCheck": true,
15 "strict": true,15 "strict": true,
16 "verbaitimModuleSyntax": true,
16 "target": "es2022"17 "target": "es2022"
17 }18 }
18}19}