authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-09-27 18:13:11-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:47-07:00
log27b17df57b1d672f6fee971407b0e4d9916bbf11
treec7c0531a642031b5bcd5bec15db73ca59af17c8a
parent7773a878828806850398f80dfa514df399b2deed
signature Commit is signed but in an unrecognized format.

feat(framework): use progress in sitegen / revive sitegen

fixes #30 fixes #32

15 files changed, 213 insertions(+), 106 deletions(-)

framework/bundle.ts+17-5
...@@ -41,6 +41,8 @@ export async function bundleClientJavaScript(...@@ -41,6 +41,8 @@ export async function bundleClientJavaScript(
41 markoViaBuildCache(),41 markoViaBuildCache(),
42 ];42 ];
4343
44 using _ = io.start(`bundle ${entryPoints.length} files`);
45
44 const bundle = await esbuild46 const bundle = await esbuild
45 .build({47 .build({
46 assetNames: "/asset/[hash]",48 assetNames: "/asset/[hash]",
...@@ -108,6 +110,7 @@ export async function bundleClientJavaScript(...@@ -108,6 +110,7 @@ export async function bundleClientJavaScript(
108110
109export type ServerPlatform = "node" | "passthru";111export type ServerPlatform = "node" | "passthru";
110export interface ServerSideOptions {112export interface ServerSideOptions {
113 progress: progress.Ref;
111 entries: string[];114 entries: string[];
112 viewItems: sg.FileItem[];115 viewItems: sg.FileItem[];
113 viewRefs: incr.Ref<PageOrView>[];116 viewRefs: incr.Ref<PageOrView>[];
...@@ -117,6 +120,7 @@ export interface ServerSideOptions {...@@ -117,6 +120,7 @@ export interface ServerSideOptions {
117 metaTemplate: meta.Template;120 metaTemplate: meta.Template;
118}121}
119export async function bundleServerJavaScript({122export async function bundleServerJavaScript({
123 progress,
120 viewItems,124 viewItems,
121 viewRefs,125 viewRefs,
122 styleMap,126 styleMap,
...@@ -143,6 +147,7 @@ export async function bundleServerJavaScript({...@@ -143,6 +147,7 @@ export async function bundleServerJavaScript({
143 }147 }
144148
145 const wViewSource = incr.work(149 const wViewSource = incr.work(
150 progress,
146 async (151 async (
147 _,152 _,
148 { viewItems, regenKeys, regenTtls }: {153 { viewItems, regenKeys, regenTtls }: {
...@@ -188,6 +193,7 @@ export async function bundleServerJavaScript({...@@ -188,6 +193,7 @@ export async function bundleServerJavaScript({
188 );193 );
189194
190 await incr.work(195 await incr.work(
196 progress,
191 async (io, { regenKeys, viewItems }) => {197 async (io, { regenKeys, viewItems }) => {
192 io.writeFile(198 io.writeFile(
193 "../ts/view.d.ts",199 "../ts/view.d.ts",
...@@ -212,8 +218,11 @@ export async function bundleServerJavaScript({...@@ -212,8 +218,11 @@ export async function bundleServerJavaScript({
212 { regenKeys: Object.keys(regenKeys), viewItems },218 { regenKeys: Object.keys(regenKeys), viewItems },
213 );219 );
214220
215 const wBundles = entries.map((entry) =>221 const wBundles = entries.map((entry) => {
216 incr.work(async (io, entry) => {222 const node = progress.start(incr.toRel(entry), { passive: true });
223 return incr.work(node, async (io, entry) => {
224 using _ = io.start("esbuild");
225
217 const pkg = await io.readJson<{226 const pkg = await io.readJson<{
218 dependencies: Record<string, string>;227 dependencies: Record<string, string>;
219 }>("package.json");228 }>("package.json");
...@@ -313,12 +322,14 @@ export async function bundleServerJavaScript({...@@ -313,12 +322,14 @@ export async function bundleServerJavaScript({
313 }322 }
314 }323 }
315 return fileWithMagicWord;324 return fileWithMagicWord;
316 }, entry)325 }, entry);
317 );326 });
318327
319 const wProcessed = wBundles.map(async (wBundle) => {328 const wProcessed = wBundles.map(async (wBundle) => {
320 if (!(await wBundle)) return;329 if (!(await wBundle)) return;
321 await incr.work(async (io) => {330 await incr.work(wBundle.progress, async (io) => {
331 using _ = io.start("postprocess");
332
322 // Only the reachable resources need to be read and inserted into the bundle.333 // Only the reachable resources need to be read and inserted into the bundle.
323 // This is what Map<string, incr.Ref> is for334 // This is what Map<string, incr.Ref> is for
324 const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));335 const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));
...@@ -380,3 +391,4 @@ import * as sg from "#sitegen";...@@ -380,3 +391,4 @@ import * as sg from "#sitegen";
380import type { PageOrView } from "./generate.ts";391import type { PageOrView } from "./generate.ts";
381import type * as view from "#sitegen/view";392import type * as view from "#sitegen/view";
382import * as meta from "#sitegen/meta";393import * as meta from "#sitegen/meta";
394import * as progress from "lib/log/progress.ts";
framework/css.ts+1
...@@ -62,6 +62,7 @@ export async function bundleCssFiles(...@@ -62,6 +62,7 @@ export async function bundleCssFiles(
62 dev: boolean;62 dev: boolean;
63 },63 },
64) {64) {
65 using _ = io.start(styleKey(cssImports, theme));
65 cssImports = await Promise.all(66 cssImports = await Promise.all(
66 cssImports.map((file) => io.trackFile("src/" + file)),67 cssImports.map((file) => io.trackFile("src/" + file)),
67 );68 );
framework/font.ts+36-19
...@@ -1,14 +1,17 @@...@@ -1,14 +1,17 @@
1export async function buildFonts(fonts: sg.Font[]) {1export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) {
2 if (fonts.length === 0) return;2 if (fonts.length === 0) return;
3 using node = p.start("build fonts");
34
4 const temp = path.resolve(".clover/font");5 const temp = path.resolve(".clover/font");
5 const venv = path.join(temp, "venv");6 const venv = path.join(temp, "venv");
6 const bin = path.join(venv, "bin");7 const bin = path.join(venv, "bin");
78
8 if (!fs.existsSync(venv)) {9 if (!fs.existsSync(venv)) {
10 using _ = node.start("setup python env");
9 await subprocess.exec("python", ["-m", "venv", venv]);11 await subprocess.exec("python", ["-m", "venv", venv]);
10 }12 }
11 if (!fs.existsSync(path.join(bin, "pyftsubset"))) {13 if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
14 using _ = node.start("install pyftsubset");
12 await subprocess.exec(path.join(bin, "pip"), [15 await subprocess.exec(path.join(bin, "pip"), [
13 "install",16 "install",
14 "fonttools==4.38.0",17 "fonttools==4.38.0",
...@@ -17,19 +20,27 @@ export async function buildFonts(fonts: sg.Font[]) {...@@ -17,19 +20,27 @@ export async function buildFonts(fonts: sg.Font[]) {
17 }20 }
1821
19 const instances = new async.OnceMap<void>();22 const instances = new async.OnceMap<void>();
20 async function makeInstance(input: string, vars: sg.FontVars) {23 function getArgs(vars: sg.FontVars) {
21 const args = Object.entries(vars).map((x) => {24 return Object.entries(vars).map((x) => {
22 const lower = x[0].toLowerCase();25 const lower = x[0].toLowerCase();
23 const k = lower === "wght" || lower === "slnt" ? lower : x[0];26 const k = lower === "wght" || lower === "slnt" ? lower : x[0];
24 const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];27 const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
25 return `${k}=${v}`;28 return `${k}=${v}`;
26 }).sort();29 }).sort();
27 const hash = crypto30 }
31
32 function getHash(input: string, args: string[]) {
33 return crypto
28 .createHash("sha256")34 .createHash("sha256")
29 .update(`${input}${args.join(" ")}`)35 .update(`${input}${args.join(" ")}`)
30 .digest("hex")36 .digest("hex")
31 .slice(0, 16)37 .slice(0, 16)
32 .toLowerCase();38 .toLowerCase();
39 }
40
41 async function makeInstance(input: string, vars: sg.FontVars) {
42 const args = getArgs(vars);
43 const hash = getHash(input, args);
33 const outfile = path.join(temp, `${hash}${path.extname(input)}`);44 const outfile = path.join(temp, `${hash}${path.extname(input)}`);
34 await instances.get(outfile, async () => {45 await instances.get(outfile, async () => {
35 await fs.rm(outfile, { force: true });46 await fs.rm(outfile, { force: true });
...@@ -45,32 +56,37 @@ export async function buildFonts(fonts: sg.Font[]) {...@@ -45,32 +56,37 @@ export async function buildFonts(fonts: sg.Font[]) {
45 }56 }
4657
47 await Promise.all(fonts.map((font) =>58 await Promise.all(fonts.map((font) =>
48 incr.work(async (io, font) => {59 incr.work(node, async (io, font) => {
60 using node = io.start(font.name);
49 const baseFile = await fetchFont(font.name, font.sources);61 const baseFile = await fetchFont(font.name, font.sources);
50 await Promise.all(font.subsets.map(async (subset) => {62 await Promise.all(font.subsets.map(async (subset) => {
51 let file = baseFile;63 const args = getArgs(subset.vars ?? {});
52 if (subset.vars) {
53 file = await makeInstance(baseFile, subset.vars);
54 }
55 const unicodes = fontRangesToString(64 const unicodes = fontRangesToString(
56 Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],65 Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
57 );66 );
58 const hash = crypto67 const hash = crypto
59 .createHash("sha256")68 .createHash("sha256")
60 .update(`${file}${unicodes}`)69 .update(`${baseFile}${args.join()}${unicodes}`)
61 .digest("hex")70 .digest("hex")
62 .slice(0, 16)71 .slice(0, 16)
63 .toLowerCase();72 .toLowerCase();
64 const woff = path.join(temp, hash + ".woff2");73 const woff = path.join(temp, hash + ".woff2");
65 await subprocess.exec(path.join(bin, "pyftsubset"), [74 if (!fs.existsSync(woff)) {
66 file,75 using _ = node.start(hash);
67 "--flavor=woff2",76 let file = baseFile;
68 `--output-file=${woff}`,77 if (subset.vars) {
69 ...subset.layoutFeatures78 file = await makeInstance(baseFile, subset.vars);
70 ? [`--layout-features=${subset.layoutFeatures.join(",")}`]79 }
71 : [],80 await subprocess.exec(path.join(bin, "pyftsubset"), [
72 `--unicodes=${unicodes}`,81 file,
73 ]);82 "--flavor=woff2",
83 `--output-file=${woff}`,
84 ...subset.layoutFeatures
85 ? [`--layout-features=${subset.layoutFeatures.join(",")}`]
86 : [],
87 `--unicodes=${unicodes}`,
88 ]);
89 }
74 await io.writeAsset({90 await io.writeAsset({
75 pathname: subset.asset.replace("[ext]", "woff2"),91 pathname: subset.asset.replace("[ext]", "woff2"),
76 buffer: await fs.readFile(woff),92 buffer: await fs.readFile(woff),
...@@ -195,3 +211,4 @@ import * as incr from "./incremental.ts";...@@ -195,3 +211,4 @@ import * as incr from "./incremental.ts";
195import * as crypto from "node:crypto";211import * as crypto from "node:crypto";
196import * as async from "#sitegen/async";212import * as async from "#sitegen/async";
197import * as error from "#sitegen/error";213import * as error from "#sitegen/error";
214import * as progress from "../lib/log/progress.ts";
framework/generate.ts+68-26
...@@ -11,44 +11,70 @@ export async function main() {...@@ -11,44 +11,70 @@ export async function main() {
11 await incr.compile(generate);11 await incr.compile(generate);
12}12}
1313
14export async function generate() {14export async function generate(p: progress.Ref) {
15 // -- read config and discover files --15 // -- read config and discover files --
16 const siteConfig = await incr.work(readManifest);16 const siteConfig = await incr.work(p, readManifest);
17 const metaTemplate: meta.Template = {17 const metaTemplate: meta.Template = {
18 ...siteConfig.meta,18 ...siteConfig.meta,
19 base: new URL(siteConfig.meta.base),19 base: new URL(siteConfig.meta.base),
20 };20 };
21 // -- start font work --
21 const {22 const {
22 staticFiles,23 staticFiles,
23 scripts,24 scripts,
24 views,25 views,
25 pages,26 pages,
26 } = await discoverAllFiles(siteConfig);27 } = await discoverAllFiles(p, siteConfig);
28
29 const aliasingInputs = Array.from(
30 Map.groupBy([...staticFiles, ...pages], (x) => x.id),
31 ).filter((x) => x[1].length > 1);
32 if (
33 aliasingInputs.length > 0
34 ) {
35 throw new Error(
36 `source files alias:\n` +
37 aliasingInputs.map(([id, files]) =>
38 ` - ${id}\n` + files.map((x) => " - " + x.file).join("\n")
39 ).join("\n"),
40 );
41 }
2742
28 // TODO: make sure that `static` and `pages` does not overlap43 const builtFonts = fonts.buildFonts(p, siteConfig.fonts);
2944
30 // TODO: loadMarkoCache45 // TODO: loadMarkoCache
3146
32 // -- start font work --47 const pageNode = p.start("generate static pages", {});
33 const builtFonts = fonts.buildFonts(siteConfig.fonts);48 const viewNode = p.start("compile dynamic views", {});
49 const staticNode = p.start("compress static asset", {});
3450
35 // -- perform build-time rendering --51 // -- perform build-time rendering --
36 const builtPages = pages.map((item) =>52 const builtPages = pages.map((item) =>
37 incr.work(preparePage, { item, metaTemplate })53 incr.work(
54 pageNode.start(item.file, { passive: true }),
55 preparePage,
56 { item, metaTemplate },
57 )
38 );58 );
39 const builtViews = views.map((item) => incr.work(prepareView, item));59 const builtViews = views.map((item) =>
60 incr.work(
61 viewNode.start(item.file, { passive: true }),
62 prepareView,
63 item,
64 )
65 );
66 Promise.allSettled(builtViews).then(() => viewNode.end());
40 const builtStaticFiles = Promise.all(67 const builtStaticFiles = Promise.all(
41 staticFiles.map((item) =>68 staticFiles.map((item) =>
42 incr.work(69 incr.work(staticNode, async (io, { id, file }) => {
43 async (io, { id, file }) =>70 using _ = io.start(file);
44 void (await io.writeAsset({71 return void (await io.writeAsset({
45 pathname: id,72 pathname: id,
46 buffer: await io.readFile(file),73 buffer: await io.readFile(file),
47 })),74 }));
48 item,75 }, item)
49 )
50 ),76 ),
51 );77 ).then(() => staticNode.end());
52 const routes = await Promise.all([...builtViews, ...builtPages]);78 const routes = await Promise.all([...builtViews, ...builtPages]);
5379
54 const viewsAndDynPages: incr.Ref<PageOrView>[] = [80 const viewsAndDynPages: incr.Ref<PageOrView>[] = [
...@@ -57,15 +83,16 @@ export async function generate() {...@@ -57,15 +83,16 @@ export async function generate() {
57 ];83 ];
5884
59 // -- page resources --85 // -- page resources --
60 const scriptMap = incr.work(bundle.bundleClientJavaScript, {86 const scriptMap = incr.work(p, bundle.bundleClientJavaScript, {
61 clientRefs: routes.flatMap((x) => x.clientRefs),87 clientRefs: routes.flatMap((x) => x.clientRefs),
62 extraPublicScripts: scripts.map((entry) => entry.file),88 extraPublicScripts: scripts.map((entry) => entry.file),
63 dev: false,89 dev: false,
64 });90 });
65 const styleMap = prepareInlineCss(routes);91 const styleMap = prepareInlineCss(p, routes);
6692
67 // -- backend --93 // -- backend --
68 const builtBackend = bundle.bundleServerJavaScript({94 const builtBackend = bundle.bundleServerJavaScript({
95 progress: p,
69 entries: siteConfig.backends,96 entries: siteConfig.backends,
70 platform: "node",97 platform: "node",
71 styleMap,98 styleMap,
...@@ -79,19 +106,20 @@ export async function generate() {...@@ -79,19 +106,20 @@ export async function generate() {
79 });106 });
80107
81 // -- assemble page assets --108 // -- assemble page assets --
82 const pAssemblePages = builtPages.map((page) =>109 const pAssemblePages = Promise.all(
83 assembleAndWritePage(page, styleMap, scriptMap)110 builtPages.map((page) => assembleAndWritePage(page, styleMap, scriptMap)),
84 );111 ).then(() => pageNode.end());
85112
86 await Promise.all([113 await Promise.all([
87 builtBackend,114 builtBackend,
88 builtStaticFiles,115 builtStaticFiles,
89 ...pAssemblePages,116 pAssemblePages,
90 builtFonts,117 builtFonts,
91 ]);118 ]);
92}119}
93120
94export async function readManifest(io: Io) {121export async function readManifest(io: Io) {
122 using _ = io.start("read site manifest");
95 const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");123 const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
96 return {124 return {
97 siteSections: cfg.siteSections.map((section) => ({125 siteSections: cfg.siteSections.map((section) => ({
...@@ -107,12 +135,14 @@ export async function readManifest(io: Io) {...@@ -107,12 +135,14 @@ export async function readManifest(io: Io) {
107}135}
108136
109export async function discoverAllFiles(137export async function discoverAllFiles(
138 p: progress.Ref,
110 siteConfig: Awaited<ReturnType<typeof readManifest>>,139 siteConfig: Awaited<ReturnType<typeof readManifest>>,
111) {140) {
141 using node = p.start("discover files");
112 return (142 return (
113 await Promise.all(143 await Promise.all(
114 siteConfig.siteSections.map(({ root: sectionRoot }) =>144 siteConfig.siteSections.map(({ root: sectionRoot }) =>
115 incr.work(scanSiteSection, toAbs(sectionRoot))145 incr.work(node, scanSiteSection, toAbs(sectionRoot))
116 ),146 ),
117 )147 )
118 ).reduce((acc, next) => ({148 ).reduce((acc, next) => ({
...@@ -124,6 +154,7 @@ export async function discoverAllFiles(...@@ -124,6 +154,7 @@ export async function discoverAllFiles(
124}154}
125155
126export async function scanSiteSection(io: Io, sectionRoot: string) {156export async function scanSiteSection(io: Io, sectionRoot: string) {
157 using _ = io.start(toRel(sectionRoot));
127 // Static files are compressed and served as-is.158 // Static files are compressed and served as-is.
128 // - "{section}/static/*.png"159 // - "{section}/static/*.png"
129 let staticFiles: FileItem[] = [];160 let staticFiles: FileItem[] = [];
...@@ -274,7 +305,7 @@ export async function preparePage(...@@ -274,7 +305,7 @@ export async function preparePage(
274 cssImports,305 cssImports,
275 theme: theme ?? null,306 theme: theme ?? null,
276 styleKey,307 styleKey,
277 clientRefs: Array.from(addon[sg.userData.key].scripts),308 clientRefs: Array.from(UNWRAP(addon[sg.userData.key]).scripts),
278 } as const;309 } as const;
279}310}
280311
...@@ -303,16 +334,19 @@ export async function prepareView(io: Io, item: sg.FileItem) {...@@ -303,16 +334,19 @@ export async function prepareView(io: Io, item: sg.FileItem) {
303export type PreparedView = Awaited<ReturnType<typeof prepareView>>;334export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
304335
305export function prepareInlineCss(336export function prepareInlineCss(
337 p: progress.Ref,
306 items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,338 items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,
307) {339) {
340 const node = p.start("bundle css");
308 const map = new Map<string, incr.Ref<string>>();341 const map = new Map<string, incr.Ref<string>>();
309 for (const { styleKey, cssImports, theme } of items) {342 for (const { styleKey, cssImports, theme } of items) {
310 if (map.has(styleKey)) continue;343 if (map.has(styleKey)) continue;
311 map.set(344 map.set(
312 styleKey,345 styleKey,
313 incr.work(css.bundleCssFiles, { cssImports, theme, dev: false }),346 incr.work(node, css.bundleCssFiles, { cssImports, theme, dev: false }),
314 );347 );
315 }348 }
349 Promise.allSettled([...map.values()]).then(() => node.end());
316 return map;350 return map;
317}351}
318352
...@@ -323,12 +357,18 @@ export async function assembleAndWritePage(...@@ -323,12 +357,18 @@ export async function assembleAndWritePage(
323 scriptWork: incr.Ref<Record<string, string>>,357 scriptWork: incr.Ref<Record<string, string>>,
324) {358) {
325 const page = await pageWork;359 const page = await pageWork;
360 ASSERT("passive" in pageWork.progress);
361 const pageNode = pageWork.progress;
326 return incr.work(362 return incr.work(
363 pageWork.progress,
327 async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {364 async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
328 const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));365 const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
329366
330 const scriptIds = clientRefs.map(hot.getScriptId);367 const scriptIds = clientRefs.map(hot.getScriptId);
331 const scriptMap = await io.readWork(scriptWork);368 const scriptMap = await io.readWork(scriptWork);
369
370 pageNode.passive = false;
371
332 const scripts = scriptIds372 const scripts = scriptIds
333 .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))373 .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
334 .map((x) => `{${x}}`)374 .map((x) => `{${x}}`)
...@@ -348,6 +388,7 @@ export async function assembleAndWritePage(...@@ -348,6 +388,7 @@ export async function assembleAndWritePage(
348 },388 },
349 regenerative: !!regenerate,389 regenerative: !!regenerate,
350 });390 });
391 pageNode.passive = true;
351 },392 },
352 page,393 page,
353 );394 );
...@@ -362,6 +403,7 @@ import * as meta from "#sitegen/meta";...@@ -362,6 +403,7 @@ import * as meta from "#sitegen/meta";
362import * as render from "#engine/render";403import * as render from "#engine/render";
363import * as sg from "#sitegen";404import * as sg from "#sitegen";
364import type { FileItem } from "#sitegen";405import type { FileItem } from "#sitegen";
406import * as progress from "../lib/log/progress.ts";
365407
366import * as bundle from "./bundle.ts";408import * as bundle from "./bundle.ts";
367import * as css from "./css.ts";409import * as css from "./css.ts";
framework/incremental.ts+40-26
...@@ -18,6 +18,8 @@ export interface Ref<T> {...@@ -18,6 +18,8 @@ export interface Ref<T> {
18 /** This method is compatible with `await` syntax */18 /** This method is compatible with `await` syntax */
19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;19 then(resolve: (value: T) => void, reject: (error: unknown) => void): void;
20 get value(): T | null;20 get value(): T | null;
21 /** The progress node passed into the input */
22 progress: progress.Node | progress.Ref;
21}23}
22type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;24type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
2325
...@@ -29,9 +31,13 @@ type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;...@@ -29,9 +31,13 @@ type Job<I = any, O = any> = (io: Io, input: I) => Promise<O>;
29 *31 *
30 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.32 * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed.
31 */33 */
32export function work<O>(job: Job<void, O>): Ref<O>;34export function work<O>(p: progress.Ref, job: Job<void, O>): Ref<O>;
33export function work<I, O>(job: Job<I, O>, input: I): Ref<O>;35export function work<I, O>(p: progress.Ref, job: Job<I, O>, input: I): Ref<O>;
34export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {36export function work<I, O>(
37 p: progress.Ref,
38 job: Job<I, O>,
39 input?: I,
40): Ref<O> {
35 const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1]));41 const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1]));
36 const keySource = [source, util.inspect(input)].join(":");42 const keySource = [source, util.inspect(input)].join(":");
37 const key = crypto.createHash("sha1").update(keySource).digest("base64url");43 const key = crypto.createHash("sha1").update(keySource).digest("base64url");
...@@ -45,15 +51,21 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {...@@ -45,15 +51,21 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
4551
46 const prev = works.get(key) as Work<O> | null;52 const prev = works.get(key) as Work<O> | null;
47 if (prev) {53 if (prev) {
48 return { key, then: (done) => done(prev.value), value: prev.value };54 return {
55 key,
56 then: (done) => done(prev.value),
57 value: prev.value,
58 progress: p,
59 };
49 }60 }
5061
51 async function perform() {62 async function perform() {
52 const io = new Io(key);63 const io = new Io(key, p);
53 jobs += 1;64 jobs += 1;
54 newKeys += 1;65 newKeys += 1;
66 await timers.setTimeout(200);
55 try {67 try {
56 const value = await job(io, input);68 const value = await job(io, input as I);
57 validateSerializable(value, "");69 validateSerializable(value, "");
58 const { reads, writes } = io;70 const { reads, writes } = io;
59 works.set(key, { value, affects: [], reads, writes });71 works.set(key, { value, affects: [], reads, writes });
...@@ -82,27 +94,24 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {...@@ -82,27 +94,24 @@ export function work<I, O>(job: Job<I, O>, input: I = null as I): Ref<O> {
82 get value() {94 get value() {
83 return (works.get(this.key)?.value as O) ?? null;95 return (works.get(this.key)?.value as O) ?? null;
84 },96 },
97 progress: p,
85 };98 };
86}99}
87100
88export async function compile<T>(compiler: () => Promise<T>) {101export async function compile<T>(
102 compiler: (node: progress.Ref) => Promise<T>,
103 p: progress.Ref = progress,
104) {
89 ASSERT(!running, `Cannot run twice at the same time.`);105 ASSERT(!running, `Cannot run twice at the same time.`);
106 const node = p.start("sitegen!");
90 try {107 try {
91 running = true;108 running = true;
92 ASSERT(jobs === 0);109 ASSERT(jobs === 0);
93 const start = performance.now();110 const start = performance.now();
94 const timerSpinner = new Spinner({
95 text: () =>
96 `sitegen! [${((performance.now() - start) / 1000).toFixed(1)}s]`,
97 fps: 10,
98 });
99 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
100111
101 const value = await compiler();112 const value = await compiler(node);
102 ASSERT(jobs === 0);113 ASSERT(jobs === 0);
103 timerSpinner.text = "incremental flush";114 await flush(node, start);
104 await flush(start);
105 timerSpinner.stop();
106 return {115 return {
107 value,116 value,
108 watchFiles: new Set(files.keys()),117 watchFiles: new Set(files.keys()),
...@@ -112,6 +121,7 @@ export async function compile<T>(compiler: () => Promise<T>) {...@@ -112,6 +121,7 @@ export async function compile<T>(compiler: () => Promise<T>) {
112 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),121 newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")),
113 };122 };
114 } finally {123 } finally {
124 node.end();
115 running = false;125 running = false;
116 newKeys = 0;126 newKeys = 0;
117 seenWrites.clear();127 seenWrites.clear();
...@@ -119,7 +129,8 @@ export async function compile<T>(compiler: () => Promise<T>) {...@@ -119,7 +129,8 @@ export async function compile<T>(compiler: () => Promise<T>) {
119 }129 }
120}130}
121131
122export async function flush(start: number) {132export async function flush(progress: progress.Node, start: number) {
133 using _ = progress.start("incremental flush");
123 // Trim134 // Trim
124 const detachedFiles = new Set<string>();135 const detachedFiles = new Set<string>();
125 const referencedAssets = new Set<string>();136 const referencedAssets = new Set<string>();
...@@ -163,11 +174,11 @@ export async function flush(start: number) {...@@ -163,11 +174,11 @@ export async function flush(start: number) {
163 const serialized = msgpackr.pack(serialize());174 const serialized = msgpackr.pack(serialize());
164 await fs.writeMkdir(".clover/incr.state", serialized);175 await fs.writeMkdir(".clover/incr.state", serialized);
165 const time = (performance.now() - start).toFixed(0);176 const time = (performance.now() - start).toFixed(0);
166 console.success(`sitegen! in ${time} ms`);177 progress.log.info(
167 console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`);178 `sitegen! in ${time} ms\n` +
168 console.writeLine(` - ${assets.size} static assets`);179 ` - ${works.size} keys (${works.size - newKeys} cached)\n` +
169 console.writeLine(180 ` - ${assets.size} static assets\n` +
170 ` - dist: ${formatSize(dist)},` +181 ` - dist: ${formatSize(dist)},` +
171 ` incremental: ${formatSize(serialized.byteLength)}`,182 ` incremental: ${formatSize(serialized.byteLength)}`,
172 );183 );
173}184}
...@@ -453,10 +464,14 @@ export function getAssetManifest() {...@@ -453,10 +464,14 @@ export function getAssetManifest() {
453 * - Outputs written with Io are deleted when abandoned.464 * - Outputs written with Io are deleted when abandoned.
454 */465 */
455export class Io {466export class Io {
456 constructor(public key: string) {}467 constructor(public key: string, private node: progress.Ref) {}
457 reads: Reads = { files: new Set(), works: new Set() };468 reads: Reads = { files: new Set(), works: new Set() };
458 writes: Writes = { files: new Set(), assets: new Map() };469 writes: Writes = { files: new Set(), assets: new Map() };
459470
471 start(text: string, options?: progress.StartOptions) {
472 return this.node.start(text, options);
473 }
474
460 #trackFs(file: string) {475 #trackFs(file: string) {
461 const resolved = toAbs(file);476 const resolved = toAbs(file);
462 const key = toRel(resolved);477 const key = toRel(resolved);
...@@ -720,8 +735,7 @@ import * as util from "node:util";...@@ -720,8 +735,7 @@ import * as util from "node:util";
720import * as crypto from "node:crypto";735import * as crypto from "node:crypto";
721import * as zlib from "node:zlib";736import * as zlib from "node:zlib";
722737
723import * as console from "@paperclover/console";738import * as progress from "../lib/log/progress.ts";
724import { Spinner } from "@paperclover/console/Spinner";
725import { formatSize } from "@/file-viewer/format.ts";739import { formatSize } from "@/file-viewer/format.ts";
726740
727import * as msgpackr from "msgpackr";741import * as msgpackr from "msgpackr";
lib/log.ts+1-4
...@@ -66,10 +66,7 @@ export interface Widget {...@@ -66,10 +66,7 @@ export interface Widget {
66 * may get called more often than the specified `fps`.66 * may get called more often than the specified `fps`.
67 * supports color codes but not ansi cursor movements.67 * supports color codes but not ansi cursor movements.
68 */68 */
69 format(69 format(now: ReturnType<typeof performance.now>): string | null;
70 now: ReturnType<typeof performance.now>,
71 focus: boolean,
72 ): string | null;
73 /** 'null' to never update (use 'onChange'). defaults to 12 fps */70 /** 'null' to never update (use 'onChange'). defaults to 12 fps */
74 fps?: number | null;71 fps?: number | null;
75 /** Subscribe to manual widget updates. Call `rerender` when needed. */72 /** Subscribe to manual widget updates. Call `rerender` when needed. */
lib/log/progress.ts+8-4
...@@ -233,7 +233,7 @@ function newNode(...@@ -233,7 +233,7 @@ function newNode(
233function formatProgress(now: number, list: Internal[]): string {233function formatProgress(now: number, list: Internal[]): string {
234 let out = "";234 let out = "";
235 for (const top of list) {235 for (const top of list) {
236 if (top.passive && top.children.length === 0) continue;236 if (top.passive && !hasChildren(top)) continue;
237 out += renderMainLine(top, now, 0) + "\n";237 out += renderMainLine(top, now, 0) + "\n";
238 out += renderChildren(top, now, []);238 out += renderChildren(top, now, []);
239 }239 }
...@@ -283,15 +283,19 @@ function renderMainLine(state: Internal, now: number, depth: number) {...@@ -283,15 +283,19 @@ function renderMainLine(state: Internal, now: number, depth: number) {
283 (state.value > 0 ? `[${state.value}] ` : "") + state.text;283 (state.value > 0 ? `[${state.value}] ` : "") + state.text;
284}284}
285285
286function hasChildren(states: Internal): boolean {
287 return states.children.some((x) => !x.passive || hasChildren(x));
288}
289
286function renderChildren(state: Internal, now: number, depth: boolean[]) {290function renderChildren(state: Internal, now: number, depth: boolean[]) {
287 let maxHeight = 25; // TODO:291 let maxHeight = 50; // TODO:
288 let truncated = 0;292 let truncated = 0;
289 let out = "";293 let out = "";
290 const { children } = state;294 let { children } = state;
295 children = children.filter((x) => !x.passive || hasChildren(x));
291 if (children.length === 0) return "";296 if (children.length === 0) return "";
292 for (let i = 0, { length } = children; i < length; i += 1) {297 for (let i = 0, { length } = children; i < length; i += 1) {
293 const child = UNWRAP(children[i]);298 const child = UNWRAP(children[i]);
294 if (child.passive && child.children.length === 0) continue;
295 let item = "";299 let item = "";
296 const left = depth.map((x) => x ? box.line : " ").join("");300 const left = depth.map((x) => x ? box.line : " ").join("");
297 item += left + (i === length - 1 && !truncated ? box.langle : box.tee);301 item += left + (i === length - 1 && !truncated ? box.langle : box.tee);
lib/packaging.ts+3-4
...@@ -33,12 +33,11 @@ export async function transpileForNpm(...@@ -33,12 +33,11 @@ export async function transpileForNpm(
33 packageJson: Record<string, unknown>,33 packageJson: Record<string, unknown>,
34 tsToJsdoc?: TsToJsdoc,34 tsToJsdoc?: TsToJsdoc,
35) {35) {
36 const libName = await Promise.resolve("ts-to-jsdoc");
36 try {37 try {
37 // @ts-expect-error optional dependency38 tsToJsdoc ??= await import(libName) as TsToJsdoc;
38 tsToJsdoc ??= await import("ts-to-jsdoc") as TsToJsdoc;
39 } catch {39 } catch {
40 // @ts-expect-error unsupported import specifier40 tsToJsdoc ??= await import("npm:" + libName + "@2.4.0") as TsToJsdoc;
41 tsToJsdoc ??= await import("npm:ts-to-jsdoc@2.4.0") as TsToJsdoc;
42 }41 }
43 const { transpileFile } = tsToJsdoc;42 const { transpileFile } = tsToJsdoc;
4443
lib/string.ts+8
...@@ -155,4 +155,12 @@ function segmentWidth(segment: string) {...@@ -155,4 +155,12 @@ function segmentWidth(segment: string) {
155 );155 );
156}156}
157157
158export function escapeShellArgument(s: string): string {
159 if (s.length === 0) return "''";
160 if (/^[a-zA-Z0-9_\-.\/@,+=%:]+$/.test(s) && s[0] !== ":") return s;
161 if (!s.includes("'")) return `'${s}'`;
162 if (!/["$`\\]/.test(s)) return `"${s}"`;
163 return `'${s.replace(/'/g, "'\\''")}'`;
164}
165
158import { UNWRAP } from "lib/assert.ts";166import { UNWRAP } from "lib/assert.ts";
readme.md+10-6
...@@ -1,7 +1,9 @@...@@ -1,7 +1,9 @@
1# clover sitegen framework1# clover sitegen framework and typescript library
22
3this repository contains clover's "sitegen" framework, which is a set of tools3this repository contains clover's "sitegen" framework, which is a set of tools
4that assist building websites. these tools power <https://paperclover.net>.4that assist building websites. additionally, this is the home of her typescript
5library `lib`, containing many high quality standalone sub-projects. all these
6tools power <https://paperclover.net>.
57
6- **HTML ("Server Side Rendering") engine written from scratch.** (~500 lines)8- **HTML ("Server Side Rendering") engine written from scratch.** (~500 lines)
7 - A more practical JSX runtime (`class` instead of `className`, built-in9 - A more practical JSX runtime (`class` instead of `className`, built-in
...@@ -18,12 +20,14 @@ that assist building websites. these tools power <https://paperclover.net>....@@ -18,12 +20,14 @@ that assist building websites. these tools power <https://paperclover.net>.
18 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs20 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
19 checks when the files change. For example, changing a component re-tests21 checks when the files change. For example, changing a component re-tests
20 only pages that use that component and re-lints only the changed file.22 only pages that use that component and re-lints only the changed file.
21- **Integrated libraries for building complex, content heavy web sites.**
22 - Static asset serving with automatic ETag and build-time compression.23 - Static asset serving with automatic ETag and build-time compression.
23 - Dynamic pages. Static pages can be regenerated (and cached) at runtime.24 - Dynamic pages. Static pages can be regenerated (and cached) at runtime.
24 - Databases with a minimal SQLite wrapper.
25 - Meta Tags and Open Graph generation.
26 - Font subsetting to reduce page bandwidth.25 - Font subsetting to reduce page bandwidth.
26- **General purpose typescript libraries.** (See [`./lib`](./lib/readme.md))
27 - [Logging](./lib/log.ts) and [progress](./lib/log/progress.ts) indicators.
28 - [Priority Queue](./lib/queue.ts) with automatic reschedulability.
29 - [Meta Tags](./framework/lib/meta.ts) and Open Graph generation.
30 - Databases with a minimal SQLite wrapper.
27- **Built on the stable and battle-tested Node.js runtime.**31- **Built on the stable and battle-tested Node.js runtime.**
2832
29none of these tools are complex or revolutionary. rather, this project is the33none of these tools are complex or revolutionary. rather, this project is the
...@@ -32,7 +36,7 @@ example on how other over-complicate other frameworks....@@ -32,7 +36,7 @@ example on how other over-complicate other frameworks.
3236
33[Marko]: https://next.markojs.com37[Marko]: https://next.markojs.com
3438
35Included is `src`, which contains `paperclover.net`. Website highlights:39Included is `src`, which contains `paperclover.net`'s source code. Highlights:
3640
37- TODO: flashy homepage.41- TODO: flashy homepage.
38- [Question/Answer board, custom markdown parser and components][q+a].42- [Question/Answer board, custom markdown parser and components][q+a].
run.js-2
...@@ -53,8 +53,6 @@ process.on("uncaughtException", (error) => {...@@ -53,8 +53,6 @@ process.on("uncaughtException", (error) => {
53 process.exit(1);53 process.exit(1);
54});54});
5555
56hot.load("node:fs").readFileSync(undefined);
57
58// Load with hooks56// Load with hooks
59if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {57if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
60 if (process.argv.length == 2) {58 if (process.argv.length == 2) {
src/file-viewer/backend.tsx+1-1
...@@ -126,7 +126,7 @@ app.get("/file/*", async (c, next) => {...@@ -126,7 +126,7 @@ app.get("/file/*", async (c, next) => {
126 rangeHeader = null;126 rangeHeader = null;
127 }127 }
128128
129 const found = await cache.read(filePath129 const found = await cache.read(filePath);
130 // foundFile = await cache.read(filePath, encoding, {130 // foundFile = await cache.read(filePath, encoding, {
131 // start: rangeHeader ? rangeHeader[0] : 0,131 // start: rangeHeader ? rangeHeader[0] : 0,
132 // end: rangeHeader ? rangeHeader[1] : Infinity,132 // end: rangeHeader ? rangeHeader[1] : Infinity,
src/file-viewer/bin/scan3.ts+6-1
...@@ -22,6 +22,7 @@ export async function main() {...@@ -22,6 +22,7 @@ export async function main() {
22 const walkQueue = new queue.PriorityQueue(10);22 const walkQueue = new queue.PriorityQueue(10);
2323
24 const dirsNode = progress.start("Walk Tree", { estimate: 1 });24 const dirsNode = progress.start("Walk Tree", { estimate: 1 });
25 dirsNode.passive = true;
25 const fileNode = progress.start("Process File", { estimate: 0 });26 const fileNode = progress.start("Process File", { estimate: 0 });
26 fileNode.sortChildren = (a, b) => {27 fileNode.sortChildren = (a, b) => {
27 const ac = a.children.length > 0 ? 1 : 0;28 const ac = a.children.length > 0 ? 1 : 0;
...@@ -260,7 +261,11 @@ export async function main() {...@@ -260,7 +261,11 @@ export async function main() {
260 }261 }
261262
262 // Add the root & recursively iterate!263 // Add the root & recursively iterate!
263 scanDirectory(Path.resolve(root));264 const rootPath = Path.resolve(root);
265 if (!rootPath.ifExistsSync()) {
266 throw new Error(`file store ${rootPath} is not mounted`);
267 }
268 scanDirectory(rootPath);
264 // await qList.done();269 // await qList.done();
265 // await qMeta.done();270 // await qMeta.done();
266 // await qProcess.done();271 // await qProcess.done();
src/file-viewer/ffmpeg.ts+3-2
...@@ -61,7 +61,7 @@ export async function spawn(options: SpawnOptions) {...@@ -61,7 +61,7 @@ export async function spawn(options: SpawnOptions) {
61 if (code !== 0) {61 if (code !== 0) {
62 const fmt = code ? `code ${code}` : `signal ${signal}`;62 const fmt = code ? `code ${code}` : `signal ${signal}`;
63 const e: any = new Error(`ffmpeg failed with ${fmt}`);63 const e: any = new Error(`ffmpeg failed with ${fmt}`);
64 e.args = [ffmpeg, ...args].join(" ");64 e.args = [ffmpeg, ...args].map(string.escapeShellArgument).join(" ");
65 e.code = code;65 e.code = code;
66 e.signal = signal;66 e.signal = signal;
67 throw e;67 throw e;
...@@ -155,7 +155,7 @@ export class Parse {...@@ -155,7 +155,7 @@ export class Parse {
155 }155 }
156 }156 }
157157
158 const hlsMatch = line.match(/^\[hls @ .*Opening '(.+)' for writing/);158 const hlsMatch = line.match(/.*Opening '(.+)' for writing/);
159 if (hlsMatch) {159 if (hlsMatch) {
160 if (!hlsMatch[1].endsWith(".tmp")) {160 if (!hlsMatch[1].endsWith(".tmp")) {
161 this.hlsFile = path.basename(hlsMatch[1]);161 this.hlsFile = path.basename(hlsMatch[1]);
...@@ -179,3 +179,4 @@ import events from "node:events";...@@ -179,3 +179,4 @@ import events from "node:events";
179import * as path from "node:path";179import * as path from "node:path";
180import * as progress from "../../lib/log/progress.ts";180import * as progress from "../../lib/log/progress.ts";
181import type { Path } from "#sitegen/path";181import type { Path } from "#sitegen/path";
182import * as string from "lib/string.ts";
src/file-viewer/transcode-rules.ts+11-6
...@@ -220,12 +220,17 @@ export function getMpegDashArgs(...@@ -220,12 +220,17 @@ export function getMpegDashArgs(
220 cmd.push("-use_template", "1");220 cmd.push("-use_template", "1");
221221
222 // build adaptation sets string222 // build adaptation sets string
223 const videoIndices = videos.map((_, i) => i).join(",");223 const videoAdaptationSets = videos.map((_, i) => `id=${i},streams=${i}`)
224 const audioIndices = audios.map((_, i) => videos.length + i).join(",");224 .join(" ");
225 cmd.push(225 let adaptationSets = videoAdaptationSets;
226 "-adaptation_sets",226
227 `id=0,streams=${videoIndices} id=1,streams=${audioIndices}`,227 if (audios.length > 0) {
228 );228 const audioId = videos.length;
229 const audioIndices = audios.map((_, i) => videos.length + i).join(",");
230 adaptationSets += ` id=${audioId},streams=${audioIndices}`;
231 }
232
233 cmd.push("-adaptation_sets", adaptationSets);
229234
230 cmd.push("-y");235 cmd.push("-y");
231 cmd.push(outDir.join("dash.mpd").toString());236 cmd.push(outDir.join("dash.mpd").toString());