authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-08-14 20:35:33-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
log952e415c67491ff213182c226cf0a4efa870b928
tree27d73a9cdcf9d93ff9f5b368ce4aabf2ee3d7dae
parentbc5017b0e58626345037c3bd2e32e6a2a1b576b4
signature Commit is signed but in an unrecognized format.

feat: font subsetting

25 files changed, 685 insertions(+), 191 deletions(-)

flake.nix+4
......@@ -7,8 +7,12 @@
77 with inputs.nixpkgs.legacyPackages.${system}; {
88 devShells.default = pkgs.mkShell {
99 buildInputs = [
10 # clover sitegen v3
1011 pkgs.nodejs_24 # runtime
1112 pkgs.deno # formatter
13 pkgs.python3 # for font subsetting
14
15 # paperclover.net
1216 (pkgs.ffmpeg.override {
1317 withOpus = true;
1418 withSvtav1 = true;
framework/backend/entry-node.ts+1-1
......@@ -9,7 +9,7 @@ const server = serve({
99 fetch: app.default.fetch,
1010 port: Number(process.env.PORT ?? 3000),
1111}, ({ address, port }) => {
12 if (address === "::") address = "::1";
12 if (address === "::") address = "localhost";
1313 console.info(url.format({
1414 protocol,
1515 hostname: address,
framework/bundle.ts-1
......@@ -94,7 +94,6 @@ export async function bundleClientJavaScript(
9494 const chunk = route.startsWith("/js/c.");
9595 if (!chunk) {
9696 const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
97 console.log(route, key);
9897 route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
9998 scripts[key] = text;
10099 }
framework/font.ts created+192
......@@ -0,0 +1,192 @@
1export async function buildFonts(fonts: sg.Font[]) {
2 if (fonts.length === 0) return;
3
4 const temp = path.resolve(".clover/font");
5 const venv = path.join(temp, "venv");
6 const bin = path.join(venv, "bin");
7
8 if (!fs.existsSync(venv)) {
9 await subprocess.exec("python", ["-m", "venv", venv]);
10 }
11 if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
12 await subprocess.exec(path.join(bin, "pip"), [
13 "install",
14 "fonttools==4.38.0",
15 "brotli==1.0.7",
16 ]);
17 }
18
19 const instances = new async.OnceMap<void>();
20 const subsets = new async.OnceMap<void>();
21 async function makeInstance(input: string, vars: sg.FontVars) {
22 const args = Object.entries(vars).map((x) => {
23 const lower = x[0].toLowerCase();
24 const k = lower === "wght" || lower === "slnt" ? lower : x[0];
25 const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
26 return `${k}=${v}`;
27 }).sort();
28 const hash = crypto
29 .createHash("sha256")
30 .update(`${input}${args.join(" ")}`)
31 .digest("hex")
32 .slice(0, 16)
33 .toLowerCase();
34 const outfile = path.join(temp, `${hash}${path.extname(input)}`);
35 await instances.get(outfile, async () => {
36 await fs.rm(outfile, { force: true });
37 await subprocess.exec(path.join(bin, "fonttools"), [
38 "varLib.instancer",
39 input,
40 ...args,
41 `--output=${outfile}`,
42 ]);
43 ASSERT(fs.existsSync(outfile));
44 });
45 return outfile;
46 }
47
48 await Promise.all(fonts.map((font) =>
49 incr.work(async (io, font) => {
50 const baseFile = await fetchFont(font.name, font.sources);
51 await Promise.all(font.subsets.map(async (subset) => {
52 let file = baseFile;
53 if (subset.vars) {
54 file = await makeInstance(baseFile, subset.vars);
55 }
56 const unicodes = fontRangesToString(
57 Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
58 );
59 const hash = crypto
60 .createHash("sha256")
61 .update(`${file}${unicodes}`)
62 .digest("hex")
63 .slice(0, 16)
64 .toLowerCase();
65 const woff = path.join(temp, hash + ".woff2");
66 await subprocess.exec(path.join(bin, "pyftsubset"), [
67 file,
68 "--flavor=woff2",
69 `--output-file=${woff}`,
70 ...subset.layoutFeatures
71 ? [`--layout-features=${subset.layoutFeatures.join(",")}`]
72 : [],
73 `--unicodes=${unicodes}`,
74 ]);
75 await io.writeAsset({
76 pathname: subset.asset.replace("[ext]", "woff2"),
77 buffer: await fs.readFile(woff),
78 });
79 }));
80 }, font)
81 ));
82}
83
84export async function fetchFont(name: string, sources: string[]) {
85 const errs = [];
86
87 for (const source of sources) {
88 const cacheName = path.join("", name + path.extname(source));
89 if (fs.existsSync(cacheName)) return cacheName;
90
91 if (source.startsWith("https://")) {
92 const response = await fetch(source);
93 if (response.ok) {
94 await fs.writeMkdir(
95 cacheName,
96 Buffer.from(await response.arrayBuffer()),
97 );
98 } else {
99 errs.push(
100 new Error(
101 `Fetching from ${source} failed: ${response.status} ${response.statusText}`,
102 ),
103 );
104 continue;
105 }
106 }
107
108 if (path.isAbsolute(source)) {
109 if (fs.existsSync(source)) return source;
110 errs.push(new Error(`Font not available at absolute path ${source}`));
111 } else if (!source.includes("/") && !source.includes("\\")) {
112 const home = os.homedir();
113 const bases = process.platform === "win32"
114 ? [
115 "\\Windows\\Fonts",
116 path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"),
117 ]
118 : process.platform === "darwin"
119 ? [
120 "/Library/Fonts",
121 path.posix.join(home, "Library/Fonts"),
122 ]
123 : [
124 "/usr/share/fonts",
125 "/usr/local/share/fonts",
126 path.posix.join(home, ".local/share/fonts"),
127 ];
128 for (const base of bases) {
129 const found = fs.readDirRecOptionalSync(base)
130 .find((file) => path.basename(file) === source);
131 if (found) {
132 return path.join(base, found);
133 }
134 }
135 errs.push(
136 new Error(
137 `Font file ${source} not found in the following directories: ${
138 bases.join(", ")
139 }`,
140 ),
141 );
142 }
143 }
144
145 throw new AggregateError(errs, `Failed to fetch the ${name} font`);
146}
147
148function fontRangesToString(ranges: sg.FontRange[]): string {
149 const segments: Array<{ start: number; end: number }> = [];
150
151 ranges.forEach((range) => {
152 if (typeof range === "string") {
153 for (let i = 0; i < range.length; i++) {
154 const cp = range.codePointAt(i) || 0;
155 segments.push({ start: cp, end: cp });
156 }
157 } else if (typeof range === "number") {
158 segments.push({ start: range, end: range });
159 } else {
160 segments.push({ start: range.start, end: range.end });
161 }
162 });
163
164 segments.sort((a, b) => a.start - b.start);
165
166 const merged: Array<{ start: number; end: number }> = [];
167 for (const seg of segments) {
168 const last = merged[merged.length - 1];
169 if (last && seg.start <= last.end + 1) {
170 last.end = Math.max(last.end, seg.end);
171 } else {
172 merged.push({ ...seg });
173 }
174 }
175
176 return merged.map(({ start, end }) =>
177 start === end
178 ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}`
179 : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${
180 end.toString(16).toUpperCase().padStart(4, "0")
181 }`
182 ).join(",");
183}
184
185import * as fs from "#sitegen/fs";
186import * as os from "node:os";
187import * as path from "node:path";
188import * as sg from "#sitegen";
189import * as subprocess from "#sitegen/subprocess";
190import * as incr from "./incremental.ts";
191import * as crypto from "node:crypto";
192import * as async from "#sitegen/async";
framework/generate.ts+20-8
......@@ -22,6 +22,9 @@ export async function generate() {
2222
2323 // TODO: loadMarkoCache
2424
25 // -- start font work --
26 const builtFonts = fonts.buildFonts(siteConfig.fonts);
27
2528 // -- perform build-time rendering --
2629 const builtPages = pages.map((item) => incr.work(preparePage, item));
2730 const builtViews = views.map((item) => incr.work(prepareView, item));
......@@ -70,7 +73,12 @@ export async function generate() {
7073 assembleAndWritePage(page, styleMap, scriptMap)
7174 );
7275
73 await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
76 await Promise.all([
77 builtBackend,
78 builtStaticFiles,
79 ...pAssemblePages,
80 builtFonts,
81 ]);
7482}
7583
7684export async function readManifest(io: Io) {
......@@ -80,6 +88,7 @@ export async function readManifest(io: Io) {
8088 root: toRel(section.root),
8189 })),
8290 backends: cfg.backends.map(toRel),
91 fonts: cfg.fonts,
8392 };
8493}
8594
......@@ -324,14 +333,17 @@ export async function assembleAndWritePage(
324333
325334export type PageOrView = PreparedPage | PreparedView;
326335
336import * as path from "node:path";
337
338import * as fs from "#sitegen/fs";
339import * as meta from "#sitegen/meta";
340import * as render from "#engine/render";
327341import * as sg from "#sitegen";
328import * as incr from "./incremental.ts";
329import { Io } from "./incremental.ts";
342import type { FileItem } from "#sitegen";
343
330344import * as bundle from "./bundle.ts";
331345import * as css from "./css.ts";
332import * as render from "#engine/render";
346import * as fonts from "./font.ts";
333347import * as hot from "./hot.ts";
334import * as fs from "#sitegen/fs";
335import type { FileItem } from "#sitegen";
336import * as path from "node:path";
337import * as meta from "#sitegen/meta";
348import * as incr from "./incremental.ts";
349import { Io } from "./incremental.ts";
framework/lib/assets.ts+67-8
......@@ -2,6 +2,7 @@
22// This module implements decoding and serving of the asset blobs,
33// but also implements patching of dynamic assets. The `Manifest`
44// is generated by `incremental.ts`
5const debug = console.scoped("assets");
56const root = import.meta.dirname;
67let current: Loaded | null = null;
78
......@@ -36,6 +37,7 @@ interface Loaded {
3637export interface DynamicEntry extends AssetBase {
3738 buffer: Buffer;
3839}
40type CacheMode = "auto" | "long-term" | "temporary" | "no-cache";
3941
4042export async function reload() {
4143 const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
......@@ -59,7 +61,7 @@ export async function reload() {
5961export async function middleware(c: Context, next: Next) {
6062 if (!current) current = await reload();
6163 const asset = current.map[c.req.path];
62 if (asset) return assetInner(c, asset, 200);
64 if (asset) return assetInner(c, c.req.path, asset, 200);
6365 return next();
6466}
6567
......@@ -68,16 +70,27 @@ export async function notFound(c: Context) {
6870 let pathname = c.req.path;
6971 do {
7072 const asset = current.map[pathname + "/404"];
71 if (asset) return assetInner(c, asset, 404);
73 if (asset) return assetInner(c, pathname + "/404", asset, 404);
7274 pathname = pathname.slice(0, pathname.lastIndexOf("/"));
7375 } while (pathname);
7476 const asset = current.map["/404"];
75 if (asset) return assetInner(c, asset, 404);
77 if (asset) return assetInner(c, "/404", asset, 404);
7678 return c.text("the 'Not Found' page was not found", 404);
7779}
7880
79export async function serveAsset(c: Context, id: Key, status: StatusCode) {
80 return assetInner(c, (current ?? (await reload())).map[id], status);
81export async function serveAsset(
82 c: Context,
83 id: Key,
84 status: StatusCode = 200,
85 cache: CacheMode = 'auto',
86) {
87 return assetInner(
88 c,
89 id,
90 (current ?? (await reload())).map[id],
91 status,
92 cache,
93 );
8194}
8295
8396/** @deprecated */
......@@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
89102 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
90103}
91104
92function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {
105function assetInner(
106 c: Context,
107 key: string,
108 asset: Manifest[Key],
109 status: StatusCode,
110 cache: CacheMode = "auto",
111) {
93112 ASSERT(current);
94113 if (asset.type === 0) {
95 return respondWithBufferAndViews(c, current.static, asset, status);
114 return respondWithBufferAndViews(
115 c,
116 key,
117 current.static,
118 asset,
119 status,
120 cache,
121 );
96122 } else {
97123 const entry = UNWRAP(current.dynamic.get(asset.id));
98 return respondWithBufferAndViews(c, entry.buffer, entry, status);
124 return respondWithBufferAndViews(
125 c,
126 key,
127 entry.buffer,
128 entry,
129 status,
130 cache,
131 );
99132 }
100133}
101134
102135function respondWithBufferAndViews(
103136 c: Context,
137 key: string,
104138 buffer: Buffer,
105139 asset: AssetBase,
106140 status: StatusCode,
141 cache: CacheMode,
107142) {
108143 const ifNoneMatch = c.req.header("If-None-Match");
109144 if (ifNoneMatch) {
......@@ -136,6 +171,29 @@ function respondWithBufferAndViews(
136171 } else {
137172 body = buffer.subarray(...asset.raw);
138173 }
174 debug(
175 `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`,
176 );
177 if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) {
178 if (cache === "auto") {
179 if (status < 200 || status >= 300) {
180 cache = 'no-cache';
181 } else if (headers['content-type']?.includes('text/html')) {
182 cache = 'temporary';
183 } else {
184 cache = 'long-term';
185 }
186 }
187 if (cache === "no-cache") {
188 headers["Cache-Control"] = "no-store";
189 } else if (cache === "long-term") {
190 headers["Cache-Control"] =
191 "public, max-age=7200, stale-while-revalidate=7200, immutable";
192 } else if (cache === "temporary") {
193 headers["Cache-Control"] =
194 "public, max-age=7200, stale-while-revalidate=7200";
195 }
196 }
139197 return (c.res = new Response(body, { headers, status }));
140198}
141199
......@@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
232290import * as crypto from "node:crypto";
233291import * as zlib from "node:zlib";
234292import * as util from "node:util";
293import * as console from "@paperclover/console";
framework/lib/error.ts created+47
......@@ -0,0 +1,47 @@
1declare global {
2 interface Error {
3 /* Extra fields on errors are extremely helpful for debugging. */
4 [metadata: string]: unknown;
5 }
6}
7
8/** Retrieve an error message from any value */
9export function message(error: unknown): string {
10 const message = (error as { message: unknown })?.message ?? error;
11 try {
12 return typeof message === "string" ? message : JSON.stringify(message);
13 } catch {}
14 try {
15 return String(message);
16 } catch {}
17 return `Could not stringify error message ${typeof message}`;
18}
19
20/** Retrieve an error-like object from any value. Useful to check fields. */
21export function obj(error: unknown): { message: string } & Partial<Error> {
22 if (error instanceof Error) return error;
23 return {
24 message: message(error),
25 ...(typeof error === "object" && error),
26 };
27}
28
29/* Node.js error codes are strings */
30export function code(error: unknown): NodeErrorCode | null {
31 const code = (error as { code: unknown })?.code;
32 return typeof code === "string" ? code : null;
33}
34
35/** Attach extra fields and throw */
36export function rethrowWithMetadata(
37 err: unknown,
38 meta: Record<string, unknown>,
39): never {
40 const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/
41 Object.assign(error, meta);
42 throw error;
43}
44
45export type NodeErrorCode =
46 | keyof typeof import("node:os").constants.errno
47 | (string & {});
framework/lib/meta.ts+5-2
......@@ -4,6 +4,9 @@ export interface Meta {
44 openGraph?: OpenGraph;
55 alternates?: Alternates;
66}
7export interface Template extends Omit<Meta, "title"> {
8 titleTemplate?: (title: string) => string,
9}
710export interface OpenGraph {
811 title?: string;
912 description?: string | undefined;
......@@ -19,6 +22,6 @@ export interface AlternateType {
1922 title: string;
2023}
2124export function renderMeta({ title }: Meta): string {
22 return `<title>${esc(title)}</title>`;
25 return `<title>${esc(title)}</title><link rel="icon" type="image/x-icon" href="/favicon.ico">`;
2326}
24import { escapeHtml as esc } from "#engine/render";
27import { escapeHtmlContent as esc } from "#engine/render";
framework/lib/mime.txt+100-99
......@@ -1,99 +1,100 @@
1# media types
2# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types
3.aac audio/x-aac
4.aif audio/x-aiff
5.aifc audio/x-aiff
6.aiff audio/x-aiff
7.asm text/x-asm
8.avi video/x-msvideo
9.bat application/x-msdownload
10.c text/x-c
11.chat text/x-clover-chatlog
12.class application/java-vm
13.cmd application/x-msdownload
14.com application/x-msdownload
15.conf text/plain
16.cpp text/x-c
17.css text/css
18.csv text/csv
19.cxx text/x-c
20.def text/plain
21.diff text/plain
22.dll application/x-msdownload
23.dmg application/octet-stream
24.doc application/msword
25.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document
26.epub application/epub+zip
27.exe application/x-msdownload
28.flv video/x-flv
29.fbx application/fbx
30.gz application/x-gzip
31.h text/x-c
32.h264 video/h264
33.hh text/x-c
34.htm text/html;charset=utf-8
35.html text/html;charset=utf-8
36.ico image/x-icon
37.ics text/calendar
38.in text/plain
39.jar application/java-archive
40.java text/x-java-source
41.jpeg image/jpeg
42.jpg image/jpeg
43.jpgv video/jpeg
44.jxl image/jxl
45.js application/javascript
46.json application/json
47.latex application/x-latex
48.list text/plain
49.log text/plain
50.m4a audio/mp4
51.man text/troff
52.mid audio/midi
53.midi audio/midi
54.mov video/quicktime
55.mp3 audio/mpeg
56.mp4 video/mp4
57.msh model/mesh
58.msi application/x-msdownload
59.obj application/octet-stream
60.ogg audio/ogg
61.otf application/x-font-otf
62.pdf application/pdf
63.png image/png
64.ppt application/vnd.ms-powerpoint
65.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation
66.psd image/vnd.adobe.photoshop
67.py text/x-python
68.rar application/x-rar-compressed
69.rss application/rss+xml
70.rtf application/rtf
71.rtx text/richtext
72.s text/x-asm
73.pem application/x-pem-file"
74.ser application/java-serialized-object
75.sh application/x-sh
76.sig application/pgp-signature
77.silo model/mesh
78.svg image/svg+xml
79.t text/troff
80.tar application/x-tar
81.text text/plain
82.tgz application/x-gzip
83.tif image/tiff
84.tiff image/tiff
85.torrent application/x-bittorrent
86.ttc application/x-font-ttf
87.ttf application/x-font-ttf
88.txt text/plain
89.urls text/uri-list
90.v text/x-v
91.wav audio/x-wav
92.wmv video/x-ms-wmv
93.xls application/vnd.ms-excel
94.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
95.xml application/xml
96.xps application/vnd.ms-xpsdocument
97
98# special cased based on file name
99rss.xml application/rss+xml
1# media types
2# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types
3.aac audio/x-aac
4.aif audio/x-aiff
5.aifc audio/x-aiff
6.aiff audio/x-aiff
7.asm text/x-asm
8.avi video/x-msvideo
9.bat application/x-msdownload
10.c text/x-c
11.chat text/x-clover-chatlog
12.class application/java-vm
13.cmd application/x-msdownload
14.com application/x-msdownload
15.conf text/plain
16.cpp text/x-c
17.css text/css
18.csv text/csv
19.cxx text/x-c
20.def text/plain
21.diff text/plain
22.dll application/x-msdownload
23.dmg application/octet-stream
24.doc application/msword
25.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document
26.epub application/epub+zip
27.exe application/x-msdownload
28.flv video/x-flv
29.fbx application/fbx
30.gz application/x-gzip
31.h text/x-c
32.h264 video/h264
33.hh text/x-c
34.htm text/html;charset=utf-8
35.html text/html;charset=utf-8
36.ico image/x-icon
37.ics text/calendar
38.in text/plain
39.jar application/java-archive
40.java text/x-java-source
41.jpeg image/jpeg
42.jpg image/jpeg
43.jpgv video/jpeg
44.jxl image/jxl
45.js application/javascript
46.json application/json
47.latex application/x-latex
48.list text/plain
49.log text/plain
50.m4a audio/mp4
51.man text/troff
52.mid audio/midi
53.midi audio/midi
54.mov video/quicktime
55.mp3 audio/mpeg
56.mp4 video/mp4
57.msh model/mesh
58.msi application/x-msdownload
59.obj application/octet-stream
60.ogg audio/ogg
61.otf application/x-font-otf
62.pdf application/pdf
63.png image/png
64.ppt application/vnd.ms-powerpoint
65.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation
66.psd image/vnd.adobe.photoshop
67.py text/x-python
68.rar application/x-rar-compressed
69.rss application/rss+xml
70.rtf application/rtf
71.rtx text/richtext
72.s text/x-asm
73.pem application/x-pem-file"
74.ser application/java-serialized-object
75.sh application/x-sh
76.sig application/pgp-signature
77.silo model/mesh
78.svg image/svg+xml
79.t text/troff
80.tar application/x-tar
81.text text/plain
82.tgz application/x-gzip
83.tif image/tiff
84.tiff image/tiff
85.torrent application/x-bittorrent
86.ttc application/x-font-ttf
87.ttf application/x-font-ttf
88.txt text/plain
89.urls text/uri-list
90.v text/x-v
91.wav audio/x-wav
92.wmv video/x-ms-wmv
93.woff2 font/woff2
94.xls application/vnd.ms-excel
95.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
96.xml application/xml
97.xps application/vnd.ms-xpsdocument
98
99# special cased based on file name
100rss.xml application/rss+xml
framework/lib/sitegen.ts+18
......@@ -31,6 +31,24 @@ export interface FileItem {
3131export interface Section {
3232 root: string;
3333}
34export interface Font {
35 name: string;
36 /**
37 * Specify either font name, file path, or URL to fetch it.
38 * Private fonts do not have a URL and will fail to build if missing.
39 */
40 sources: string[];
41 subsets: Array<{
42 vars?: Record<string, FontVariableAxis>;
43 layoutFeatures?: string[];
44 unicodes: FontRange | FontRange[];
45 /** Include [ext] to autofill 'woff2' */
46 asset: string;
47 }>;
48}
49export type FontVars = Record<string, FontVariableAxis>;
50export type FontVariableAxis = number | [min: number, max: number];
51export type FontRange = string | number | { start: number, end: number };
3452
3553export const userData = render.userData<SitegenRender>(() => {
3654 throw new Error("This function can only be used in a page (static or view)");
framework/lib/sqlite.ts+5-4
......@@ -11,9 +11,9 @@ export function getDb(file: string) {
1111 let db = map.get(file);
1212 if (db) return db;
1313 const fileWithExt = file.includes(".") ? file : file + ".sqlite";
14 db = new WrappedDatabase(
15 path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
16 );
14 const dir = process.env.CLOVER_DB ?? ".clover";
15 fs.mkdirSync(dir);
16 db = new WrappedDatabase(path.join(dir, fileWithExt));
1717 map.set(file, db);
1818 return db;
1919}
......@@ -21,7 +21,7 @@ export function getDb(file: string) {
2121export class WrappedDatabase {
2222 file: string;
2323 node: DatabaseSync;
24 stmts: Stmt[];
24 stmts: Stmt[] = [];
2525 stmtTableMigrate: WeakRef<StatementSync> | null = null;
2626
2727 constructor(file: string) {
......@@ -153,3 +153,4 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
153153
154154import { DatabaseSync, StatementSync } from "node:sqlite";
155155import * as path from "node:path";
156import * as fs from "#sitegen/fs";
framework/lib/subprocess.ts created+15
......@@ -0,0 +1,15 @@
1const execFileRaw = util.promisify(child_process.execFile);
2export const exec: typeof execFileRaw = ((
3 ...args: Parameters<typeof execFileRaw>
4) =>
5 execFileRaw(...args).catch((e: any) => {
6 if (e?.message?.startsWith?.("Command failed")) {
7 if (e.code > 2 ** 31) e.code |= 0;
8 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
9 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
10 }
11 throw e;
12 })) as any;
13
14import * as util from 'node:util';
15import * as child_process from 'node:child_process';
framework/watch.ts+1
......@@ -25,6 +25,7 @@ function serve() {
2525 "--development",
2626 ], {
2727 stdio: "inherit",
28 execArgv: ['--no-warnings'],
2829 });
2930 subprocess.on("close", onSubprocessClose);
3031}
readme.md+37-17
......@@ -11,23 +11,21 @@ that assist building websites. these tools power <https://paperclover.net>.
1111 - Different languages can be used at the same time. Supports `async function`
1212 components, `<Suspense />`, and custom extensions.
1313- **Incremental static site generator and build system.**
14 - Build entire production site at start, incremental updates when pages
15 change; Build system state survives coding sessions.
16 - The only difference in development and production mode is hidden source-maps
17 and stripped `console.debug` calls. The site you see locally is the same
18 site you see deployed.
14 - Build both development and production sites on startup start. Incremental
15 generator rebuilds changed pages; Build system state survives coding sessions.
16 - Multiple backend support.
1917 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
2018 checks when the files change. For example, changing a component re-tests
2119 only pages that use that component and re-lints only the changed file.
2220- **Integrated libraries for building complex, content heavy web sites.**
2321 - Static asset serving with ETag and build-time compression.
24 - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)
22 - Dynamicly rendered pages with static client. (`import "#sitegen/view"`)
2523 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
2624 - TODO: Meta and Open Graph generation. (`export const meta`)
27 - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
28- **Built on the battle-tested Node.js runtime.**
25 - Font subsetting to reduce page bandwidth.
26- **Built on the stable and battle-tested Node.js runtime.**
2927
30None of these tools are complex or revolutionary. Rather, this project is the
28none of these tools are complex or revolutionary. rather, this project is the
3129sum of many years of experience on managing content heavy websites, and an
3230example on how other over-complicate other frameworks.
3331
......@@ -35,9 +33,11 @@ example on how other over-complicate other frameworks.
3533
3634Included is `src`, which contains `paperclover.net`. Website highlights:
3735
36- TODO: flashy homepage.
3837- [Question/Answer board, custom markdown parser and components][q+a].
3938- [File viewer with fast ui/ux + optimized media streaming][file].
4039- [Personal, friends-only blog with password protection][friends].
40- TODO: digital garden styled blog.
4141
4242[q+a]: https://paperclover.net/q+a
4343[file]: https://paperclover.net/file
......@@ -51,15 +51,21 @@ minimum system requirements:
5151- random access memory.
5252- windows 7 or later, macos, or other operating system.
5353
54required software:
55
56- node.js v24
57- python v3
58
5459my development machine, for example, is Dell Inspiron 7348 with Core i7
5560
5661 npm install
5762
58 # production generation
63 # build site using 'run.js' to enable runtime plugins
5964 node run generate
60 node .clover/o/backend
65 # the built site runs in regular node.js
66 node --enable-source-maps .clover/o/backend
6167
62 # "development" watch mode
68 # watch-rebuild mode
6369 node run watch
6470
6571for unix systems, the provided `flake.nix` can be used with `nix develop` to
......@@ -67,7 +73,7 @@ open a shell with all needed system dependencies.
6773
6874## Deployment
6975
70There are two primary server components to be deployed: the web server and the
76there are two primary server components to be deployed: the web server and the
7177sourth of truth server. The latter is a singleton that runs on Clover's NAS,
7278which holds the full contents of the file storage. The web server pulls data
7379from the source of truth and renders web pages, and can be duplicated to
......@@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose:
7985 backend:
8086 container_name: backend
8187 build:
82 # this uses loopback to hit the self-hosted git server
88 # this uses loopback to hit the self-hosted git server,
89 # docker will cache the image to not re-fetch on reboot.
8390 context: http://127.0.0.1:3000/clo/sitegen.git
8491 dockerfile: src/source-of-truth.dockerfile
8592 environment:
......@@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose:
96103 - /mnt/storage1/clover/Documents/Config/paperclover:/data
97104 - /mnt/storage1/clover/Published:/published
98105
99Due to caching, one may need to manually purge images via
100`docker image rm ix-clover-backend -f` when an update is desired
106Due to caching, images may need to be purged via `docker image rm {image} -f`
107when an update is desired. Some docker GUIs support force pulls, some are buggy.
101108
102TODO: deployment instructions for a web node
109The web server performs rendering. A `Dockerfile` for it is present in
110`src/web.dockerfile` but it is currently unused. Deployments are done by
111building the project locally, and then using `rsync` to copy files.
112
113 node run generate
114 rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \
115 --delete-after --progress --human-readable
116 ssh "$REMOTE_USER" /bin/bash << EOF
117 set -e
118 cd ~/paperclover
119 npm ci
120 pm2 restart site
121 echo "-> https://paperclover.net"
122 EOF
103123
104124## Contributions
105125
run.js+22-13
......@@ -3,7 +3,6 @@
33import * as util from "node:util";
44import * as zlib from "node:zlib";
55import * as url from "node:url";
6import * as module from "node:module";
76import process from "node:process";
87
98// Disable experimental warnings (Type Stripping, etc)
......@@ -28,8 +27,8 @@ try {
2827 const brand = process.versions.bun
2928 ? `bun ${process.versions.bun}`
3029 : process.versions.deno
31 ? `deno ${process.versions.deno}`
32 : null;
30 ? `deno ${process.versions.deno}`
31 : null;
3332 const brandText = brand ? ` (${brand})` : "";
3433 globalThis.console.error(
3534 `sitegen depends on a node.js v24. your runtime is missing features.\n` +
......@@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
7574 else if (mod.default?.fetch) {
7675 const protocol = "http";
7776 const { serve } = hot.load("@hono/node-server");
78 serve({
79 fetch: mod.default.fetch,
80 }, ({ address, port }) => {
81 if (address === "::") address = "::1";
82 console.info(url.format({
83 protocol,
84 hostname: address,
85 port,
86 }));
87 });
77 serve(
78 {
79 fetch: mod.default.fetch,
80 },
81 ({ address, port }) => {
82 if (address === "::") address = "::1";
83 console.info(
84 url.format({
85 protocol,
86 hostname: address,
87 port,
88 }),
89 );
90 },
91 );
92 } else {
93 console.warn(
94 hot.load("path").relative('.', found),
95 'does not export a "main" function',
96 );
8897 }
8998 } catch (e) {
9099 console.error(util.inspect(e));
src/admin.ts+5-1
......@@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year
33let lastKnownToken: string | null = null;
44function compareToken(token: string) {
55 if (token === lastKnownToken) return true;
6 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
6 try {
7 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
8 } catch {
9 return false;
10 }
711 return token === lastKnownToken;
812}
913
src/backend.ts+14
......@@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app);
1515// Asset middleware has least precedence
1616app.use(assets.middleware);
1717
18// Asset Aliases
19app.get(
20 "/apple-touch-icon-precomposed.png",
21 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
22);
23app.get(
24 "/apple-touch-icon-57x57.png",
25 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
26);
27app.get(
28 "/apple-touch-icon-57x57-precomposed.png",
29 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
30);
31
1832// Handlers
1933app.notFound(assets.notFound);
2034
src/file-viewer/bin/scan3.ts+3-15
......@@ -10,7 +10,6 @@
1010// This is the third iteration of the scanner, hence its name "scan3";
1111// Remember that any software you want to be maintainable and high
1212// quality cannot be written with AI.
13const root = path.resolve("/Volumes/clover/Published");
1413const workDir = path.resolve(".clover/derived");
1514const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
1615
......@@ -404,18 +403,6 @@ interface Process {
404403 undo?(mediaFile: MediaFile): Promise<void>;
405404}
406405
407const execFileRaw = util.promisify(child_process.execFile);
408const execFile: typeof execFileRaw = ((
409 ...args: Parameters<typeof execFileRaw>
410) =>
411 execFileRaw(...args).catch((e: any) => {
412 if (e?.message?.startsWith?.("Command failed")) {
413 if (e.code > 2 ** 31) e.code |= 0;
414 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
415 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
416 }
417 throw e;
418 })) as any;
419406const ffprobeBin = testProgram("ffprobe", "--help");
420407const ffmpegBin = testProgram("ffmpeg", "--help");
421408
......@@ -426,7 +413,7 @@ const procDuration: Process = {
426413 enable: ffprobeBin !== null,
427414 include: rules.extsDuration,
428415 async run({ absPath, mediaFile }) {
429 const { stdout } = await execFile(ffprobeBin!, [
416 const { stdout } = await subprocess.exec(ffprobeBin!, [
430417 "-v",
431418 "error",
432419 "-show_entries",
......@@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress";
793780import { Spinner } from "@paperclover/console/Spinner";
794781import * as async from "#sitegen/async";
795782import * as fs from "#sitegen/fs";
783import * as subprocess from "#sitegen/subprocess";
796784
797785import * as path from "node:path";
798786import * as zlib from "node:zlib";
799787import * as child_process from "node:child_process";
800import * as util from "node:util";
801788import * as crypto from "node:crypto";
802789import * as stream from "node:stream";
803790
......@@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts";
814801import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
815802import * as rsync from "@/file-viewer/rsync.ts";
816803import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
804import { rawFileRoot as root } from "../paths.ts";
src/file-viewer/cache.ts+4-4
......@@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts";
1212declare const Deno: any;
1313
1414const sourceOfTruth = "https://nas.paperclover.net:43250";
15const caCert = fs.readFileSync("src/file-viewer/cert.pem");
15// const caCert = fs.readFileSync("src/file-viewer/cert.pem");
1616
17const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/");
17const diskCacheRoot = path.join(import.meta.dirname, ".filecache/");
1818const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB
1919const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB
2020const loadInProgress = new Map<
......@@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined"
338338 // Bun has two non-standard fetch extensions
339339 decompress: false,
340340 tls: {
341 ca: caCert,
341 // ca: caCert,
342342 },
343343 }
344344 // TODO: https://github.com/denoland/deno/issues/12291
......@@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined"
351351 // }
352352 // Node.js supports node:http
353353 : new Agent({
354 ca: caCert,
354 // ca: caCert,
355355 });
356356
357357function fetchFileNode(pathname: string): Promise<ReadableStream> {
src/file-viewer/paths.ts created+12
......@@ -0,0 +1,12 @@
1export const nasRoot = process.platform === "win32"
2 ? "\\\\zenith\\clover"
3 : process.platform === "darwin"
4 ? "/Volumes/clover"
5 : "/media/clover";
6
7export const rawFileRoot = process.env.CLOVER_FILE_RAW ??
8 path.join(nasRoot, "Published");
9export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
10 path.join(nasRoot, "Documents/Config/paperclover/derived");
11
12import * as path from "node:path";
src/site.ts+93-7
......@@ -1,11 +1,11 @@
1// This file defines the different "Sections" of the website. The sections act
2// as separate codebases, but hosted as one. This allows me to have
3// sub-projects like the file viewer in 'file', or the question answer system
4// in 'q+a'. Each section can define configuration, pages, backend routes, and
5// contain other files.
61const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
72
8export const siteSections: Section[] = [
3// Different sections of the website are split into their own folders. Acting as
4// as separate codebases, they are hosted as one. This allows me to have
5// sub-projects like the file viewer in 'file/', and the question answer system
6// in 'q+a', but maintain clear boundaries. Each section can define
7// configuration, pages, backend routes, and contain other files.
8export const siteSections: sg.Section[] = [
99 { root: join(".") },
1010 { root: join("q+a/") },
1111 { root: join("file-viewer/") },
......@@ -14,10 +14,96 @@ export const siteSections: Section[] = [
1414 // { root: join("fiction/"), pageBase: "/fiction" },
1515];
1616
17// All backends are bundled. The backend named "backend" is run by "node run watch"
1718export const backends: string[] = [
1819 join("backend.ts"),
1920 join("source-of-truth.ts"),
2021];
2122
23// Font subsets reduce bandwidth and protect against proprietary font theft.
24const fontRoot = path.join(nasRoot, 'Documents/Font');
25const ascii = { start: 0x20, end: 0x7E };
26const nonAscii: sg.FontRange[] = [
27 { start: 0xC0, end: 0xFF },
28 { start: 0x2190, end: 0x2193 },
29 { start: 0xA0, end: 0xA8 },
30 { start: 0xAA, end: 0xBF },
31 { start: 0x2194, end: 0x2199 },
32 { start: 0x100, end: 0x17F },
33 0xA9,
34 0x2018,
35 0x2019,
36 0x201C,
37 0x201D,
38 0x2022,
39];
40const recursiveVars: sg.FontVars = {
41 WGHT: [400, 750],
42 SLNT: [-15, 0],
43 CASL: 0.25,
44 CRSV: 0.5,
45 MONO: 0,
46};
47const recursiveVarsMono = { ...recursiveVars, MONO: 1 };
48const recursiveVarsQuestion: sg.FontVars = {
49 ...recursiveVars,
50 MONO: [0, 1],
51 WGHT: [400, 1000],
52};
53const layoutFeatures = ["numr", "dnom", "frac"];
54export const fonts: sg.Font[] = [
55 {
56 name: "Recursive",
57 sources: [
58 "Recursive_VF_1.085.ttf",
59 path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
60 "https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf",
61 ],
62 subsets: [
63 {
64 asset: "/recultramin.[ext]",
65 layoutFeatures,
66 vars: recursiveVars,
67 unicodes: ascii,
68 },
69 {
70 asset: "/recmono.[ext]",
71 vars: recursiveVarsMono,
72 unicodes: ascii,
73 },
74 {
75 asset: "/recqa.[ext]",
76 vars: recursiveVarsQuestion,
77 unicodes: ascii,
78 },
79 {
80 asset: "/recexotic.[ext]",
81 vars: recursiveVarsQuestion,
82 unicodes: nonAscii,
83 },
84 ],
85 },
86 {
87 name: "AT Name Sans Display Hairline",
88 sources: [
89 "ATNameSansDisplay-Hairline.woff2",
90 path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
91 ],
92 subsets: [
93 {
94 asset: "/cydn_header.[ext]",
95 layoutFeatures,
96 unicodes: "cotlyedon",
97 },
98 ],
99 },
100];
101
102export async function main() {
103 await font.buildFonts(fonts);
104}
105
22106import * as path from "node:path";
23import type { Section } from "#sitegen";
107import * as font from "../framework/font.ts";
108import type * as sg from "#sitegen";
109import { nasRoot } from "./file-viewer/paths.ts";
src/source-of-truth.ts+1-11
......@@ -22,17 +22,6 @@ export default app;
2222
2323const token = UNWRAP(process.env.CLOVER_SOT_KEY);
2424
25const nasRoot = process.platform === "win32"
26 ? "\\\\zenith\\clover"
27 : process.platform === "darwin"
28 ? "/Volumes/clover"
29 : "/media/clover";
30
31const rawFileRoot = process.env.CLOVER_FILE_RAW ??
32 path.join(nasRoot, "Published");
33const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
34 path.join(nasRoot, "Documents/Config/paperclover/derived");
35
3625if (!fs.existsSync(rawFileRoot)) {
3726 throw new Error(`${rawFileRoot} does not exist`);
3827}
......@@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs";
125114import * as util from "node:util";
126115import * as stream from "node:stream";
127116import * as mime from "#sitegen/mime";
117import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts";
src/static/apple-touch-icon.png created
Binary files /dev/null and b/src/static/apple-touch-icon.png differ
src/static/favicon.ico created
Binary files /dev/null and b/src/static/favicon.ico differ
src/web.dockerfile created+19
......@@ -0,0 +1,19 @@
1from node:24 as builder
2
3run apt install -y git
4
5workdir /tmp/builder
6copy package*.json ./
7run npm ci
8copy . ./
9run node run generate
10run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/
11
12from node:24 as runtime
13
14workdir /app
15copy --from=builder /app/ ./
16
17env PORT=80
18
19cmd ["node", "--enable-source-maps", "/app/backend.js"]