authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-08-14 20:35:33-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:46-07:00
log952e415c67491ff213182c226cf0a4efa870b928
tree27d73a9cdcf9d93ff9f5b368ce4aabf2ee3d7dae
parentbc5017b0e58626345037c3bd2e32e6a2a1b576b4
signature Commit is signed but in an unrecognized format.

feat: font subsetting


25 files changed, 685 insertions(+), 191 deletions(-)

flake.nix+4
...@@ -7,8 +7,12 @@...@@ -7,8 +7,12 @@
7 with inputs.nixpkgs.legacyPackages.${system}; {7 with inputs.nixpkgs.legacyPackages.${system}; {
8 devShells.default = pkgs.mkShell {8 devShells.default = pkgs.mkShell {
9 buildInputs = [9 buildInputs = [
10 # clover sitegen v3
10 pkgs.nodejs_24 # runtime11 pkgs.nodejs_24 # runtime
11 pkgs.deno # formatter12 pkgs.deno # formatter
13 pkgs.python3 # for font subsetting
14
15 # paperclover.net
12 (pkgs.ffmpeg.override {16 (pkgs.ffmpeg.override {
13 withOpus = true;17 withOpus = true;
14 withSvtav1 = true;18 withSvtav1 = true;
framework/backend/entry-node.ts+1-1
...@@ -9,7 +9,7 @@ const server = serve({...@@ -9,7 +9,7 @@ const server = serve({
9 fetch: app.default.fetch,9 fetch: app.default.fetch,
10 port: Number(process.env.PORT ?? 3000),10 port: Number(process.env.PORT ?? 3000),
11}, ({ address, port }) => {11}, ({ address, port }) => {
12 if (address === "::") address = "::1";12 if (address === "::") address = "localhost";
13 console.info(url.format({13 console.info(url.format({
14 protocol,14 protocol,
15 hostname: address,15 hostname: address,
framework/bundle.ts-1
...@@ -94,7 +94,6 @@ export async function bundleClientJavaScript(...@@ -94,7 +94,6 @@ export async function bundleClientJavaScript(
94 const chunk = route.startsWith("/js/c.");94 const chunk = route.startsWith("/js/c.");
95 if (!chunk) {95 if (!chunk) {
96 const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));96 const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
97 console.log(route, key);
98 route = "/js/" + key.replace(/\.client\.tsx?/, ".js");97 route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
99 scripts[key] = text;98 scripts[key] = text;
100 }99 }
framework/font.ts created+192
...@@ -0,0 +1,192 @@
1export async function buildFonts(fonts: sg.Font[]) {
2 if (fonts.length === 0) return;
3
4 const temp = path.resolve(".clover/font");
5 const venv = path.join(temp, "venv");
6 const bin = path.join(venv, "bin");
7
8 if (!fs.existsSync(venv)) {
9 await subprocess.exec("python", ["-m", "venv", venv]);
10 }
11 if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
12 await subprocess.exec(path.join(bin, "pip"), [
13 "install",
14 "fonttools==4.38.0",
15 "brotli==1.0.7",
16 ]);
17 }
18
19 const instances = new async.OnceMap<void>();
20 const subsets = new async.OnceMap<void>();
21 async function makeInstance(input: string, vars: sg.FontVars) {
22 const args = Object.entries(vars).map((x) => {
23 const lower = x[0].toLowerCase();
24 const k = lower === "wght" || lower === "slnt" ? lower : x[0];
25 const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
26 return `${k}=${v}`;
27 }).sort();
28 const hash = crypto
29 .createHash("sha256")
30 .update(`${input}${args.join(" ")}`)
31 .digest("hex")
32 .slice(0, 16)
33 .toLowerCase();
34 const outfile = path.join(temp, `${hash}${path.extname(input)}`);
35 await instances.get(outfile, async () => {
36 await fs.rm(outfile, { force: true });
37 await subprocess.exec(path.join(bin, "fonttools"), [
38 "varLib.instancer",
39 input,
40 ...args,
41 `--output=${outfile}`,
42 ]);
43 ASSERT(fs.existsSync(outfile));
44 });
45 return outfile;
46 }
47
48 await Promise.all(fonts.map((font) =>
49 incr.work(async (io, font) => {
50 const baseFile = await fetchFont(font.name, font.sources);
51 await Promise.all(font.subsets.map(async (subset) => {
52 let file = baseFile;
53 if (subset.vars) {
54 file = await makeInstance(baseFile, subset.vars);
55 }
56 const unicodes = fontRangesToString(
57 Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
58 );
59 const hash = crypto
60 .createHash("sha256")
61 .update(`${file}${unicodes}`)
62 .digest("hex")
63 .slice(0, 16)
64 .toLowerCase();
65 const woff = path.join(temp, hash + ".woff2");
66 await subprocess.exec(path.join(bin, "pyftsubset"), [
67 file,
68 "--flavor=woff2",
69 `--output-file=${woff}`,
70 ...subset.layoutFeatures
71 ? [`--layout-features=${subset.layoutFeatures.join(",")}`]
72 : [],
73 `--unicodes=${unicodes}`,
74 ]);
75 await io.writeAsset({
76 pathname: subset.asset.replace("[ext]", "woff2"),
77 buffer: await fs.readFile(woff),
78 });
79 }));
80 }, font)
81 ));
82}
83
84export async function fetchFont(name: string, sources: string[]) {
85 const errs = [];
86
87 for (const source of sources) {
88 const cacheName = path.join("", name + path.extname(source));
89 if (fs.existsSync(cacheName)) return cacheName;
90
91 if (source.startsWith("https://")) {
92 const response = await fetch(source);
93 if (response.ok) {
94 await fs.writeMkdir(
95 cacheName,
96 Buffer.from(await response.arrayBuffer()),
97 );
98 } else {
99 errs.push(
100 new Error(
101 `Fetching from ${source} failed: ${response.status} ${response.statusText}`,
102 ),
103 );
104 continue;
105 }
106 }
107
108 if (path.isAbsolute(source)) {
109 if (fs.existsSync(source)) return source;
110 errs.push(new Error(`Font not available at absolute path ${source}`));
111 } else if (!source.includes("/") && !source.includes("\\")) {
112 const home = os.homedir();
113 const bases = process.platform === "win32"
114 ? [
115 "\\Windows\\Fonts",
116 path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"),
117 ]
118 : process.platform === "darwin"
119 ? [
120 "/Library/Fonts",
121 path.posix.join(home, "Library/Fonts"),
122 ]
123 : [
124 "/usr/share/fonts",
125 "/usr/local/share/fonts",
126 path.posix.join(home, ".local/share/fonts"),
127 ];
128 for (const base of bases) {
129 const found = fs.readDirRecOptionalSync(base)
130 .find((file) => path.basename(file) === source);
131 if (found) {
132 return path.join(base, found);
133 }
134 }
135 errs.push(
136 new Error(
137 `Font file ${source} not found in the following directories: ${
138 bases.join(", ")
139 }`,
140 ),
141 );
142 }
143 }
144
145 throw new AggregateError(errs, `Failed to fetch the ${name} font`);
146}
147
148function fontRangesToString(ranges: sg.FontRange[]): string {
149 const segments: Array<{ start: number; end: number }> = [];
150
151 ranges.forEach((range) => {
152 if (typeof range === "string") {
153 for (let i = 0; i < range.length; i++) {
154 const cp = range.codePointAt(i) || 0;
155 segments.push({ start: cp, end: cp });
156 }
157 } else if (typeof range === "number") {
158 segments.push({ start: range, end: range });
159 } else {
160 segments.push({ start: range.start, end: range.end });
161 }
162 });
163
164 segments.sort((a, b) => a.start - b.start);
165
166 const merged: Array<{ start: number; end: number }> = [];
167 for (const seg of segments) {
168 const last = merged[merged.length - 1];
169 if (last && seg.start <= last.end + 1) {
170 last.end = Math.max(last.end, seg.end);
171 } else {
172 merged.push({ ...seg });
173 }
174 }
175
176 return merged.map(({ start, end }) =>
177 start === end
178 ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}`
179 : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${
180 end.toString(16).toUpperCase().padStart(4, "0")
181 }`
182 ).join(",");
183}
184
185import * as fs from "#sitegen/fs";
186import * as os from "node:os";
187import * as path from "node:path";
188import * as sg from "#sitegen";
189import * as subprocess from "#sitegen/subprocess";
190import * as incr from "./incremental.ts";
191import * as crypto from "node:crypto";
192import * as async from "#sitegen/async";
framework/generate.ts+20-8
...@@ -22,6 +22,9 @@ export async function generate() {...@@ -22,6 +22,9 @@ export async function generate() {
2222
23 // TODO: loadMarkoCache23 // TODO: loadMarkoCache
2424
25 // -- start font work --
26 const builtFonts = fonts.buildFonts(siteConfig.fonts);
27
25 // -- perform build-time rendering --28 // -- perform build-time rendering --
26 const builtPages = pages.map((item) => incr.work(preparePage, item));29 const builtPages = pages.map((item) => incr.work(preparePage, item));
27 const builtViews = views.map((item) => incr.work(prepareView, item));30 const builtViews = views.map((item) => incr.work(prepareView, item));
...@@ -70,7 +73,12 @@ export async function generate() {...@@ -70,7 +73,12 @@ export async function generate() {
70 assembleAndWritePage(page, styleMap, scriptMap)73 assembleAndWritePage(page, styleMap, scriptMap)
71 );74 );
7275
73 await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);76 await Promise.all([
77 builtBackend,
78 builtStaticFiles,
79 ...pAssemblePages,
80 builtFonts,
81 ]);
74}82}
7583
76export async function readManifest(io: Io) {84export async function readManifest(io: Io) {
...@@ -80,6 +88,7 @@ export async function readManifest(io: Io) {...@@ -80,6 +88,7 @@ export async function readManifest(io: Io) {
80 root: toRel(section.root),88 root: toRel(section.root),
81 })),89 })),
82 backends: cfg.backends.map(toRel),90 backends: cfg.backends.map(toRel),
91 fonts: cfg.fonts,
83 };92 };
84}93}
8594
...@@ -324,14 +333,17 @@ export async function assembleAndWritePage(...@@ -324,14 +333,17 @@ export async function assembleAndWritePage(
324333
325export type PageOrView = PreparedPage | PreparedView;334export type PageOrView = PreparedPage | PreparedView;
326335
336import * as path from "node:path";
337
338import * as fs from "#sitegen/fs";
339import * as meta from "#sitegen/meta";
340import * as render from "#engine/render";
327import * as sg from "#sitegen";341import * as sg from "#sitegen";
328import * as incr from "./incremental.ts";342import type { FileItem } from "#sitegen";
329import { Io } from "./incremental.ts";343
330import * as bundle from "./bundle.ts";344import * as bundle from "./bundle.ts";
331import * as css from "./css.ts";345import * as css from "./css.ts";
332import * as render from "#engine/render";346import * as fonts from "./font.ts";
333import * as hot from "./hot.ts";347import * as hot from "./hot.ts";
334import * as fs from "#sitegen/fs";348import * as incr from "./incremental.ts";
335import type { FileItem } from "#sitegen";349import { Io } from "./incremental.ts";
336import * as path from "node:path";
337import * as meta from "#sitegen/meta";
framework/lib/assets.ts+67-8
...@@ -2,6 +2,7 @@...@@ -2,6 +2,7 @@
2// This module implements decoding and serving of the asset blobs,2// This module implements decoding and serving of the asset blobs,
3// but also implements patching of dynamic assets. The `Manifest`3// but also implements patching of dynamic assets. The `Manifest`
4// is generated by `incremental.ts`4// is generated by `incremental.ts`
5const debug = console.scoped("assets");
5const root = import.meta.dirname;6const root = import.meta.dirname;
6let current: Loaded | null = null;7let current: Loaded | null = null;
78
...@@ -36,6 +37,7 @@ interface Loaded {...@@ -36,6 +37,7 @@ interface Loaded {
36export interface DynamicEntry extends AssetBase {37export interface DynamicEntry extends AssetBase {
37 buffer: Buffer;38 buffer: Buffer;
38}39}
40type CacheMode = "auto" | "long-term" | "temporary" | "no-cache";
3941
40export async function reload() {42export async function reload() {
41 const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));43 const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
...@@ -59,7 +61,7 @@ export async function reload() {...@@ -59,7 +61,7 @@ export async function reload() {
59export async function middleware(c: Context, next: Next) {61export async function middleware(c: Context, next: Next) {
60 if (!current) current = await reload();62 if (!current) current = await reload();
61 const asset = current.map[c.req.path];63 const asset = current.map[c.req.path];
62 if (asset) return assetInner(c, asset, 200);64 if (asset) return assetInner(c, c.req.path, asset, 200);
63 return next();65 return next();
64}66}
6567
...@@ -68,16 +70,27 @@ export async function notFound(c: Context) {...@@ -68,16 +70,27 @@ export async function notFound(c: Context) {
68 let pathname = c.req.path;70 let pathname = c.req.path;
69 do {71 do {
70 const asset = current.map[pathname + "/404"];72 const asset = current.map[pathname + "/404"];
71 if (asset) return assetInner(c, asset, 404);73 if (asset) return assetInner(c, pathname + "/404", asset, 404);
72 pathname = pathname.slice(0, pathname.lastIndexOf("/"));74 pathname = pathname.slice(0, pathname.lastIndexOf("/"));
73 } while (pathname);75 } while (pathname);
74 const asset = current.map["/404"];76 const asset = current.map["/404"];
75 if (asset) return assetInner(c, asset, 404);77 if (asset) return assetInner(c, "/404", asset, 404);
76 return c.text("the 'Not Found' page was not found", 404);78 return c.text("the 'Not Found' page was not found", 404);
77}79}
7880
79export async function serveAsset(c: Context, id: Key, status: StatusCode) {81export async function serveAsset(
80 return assetInner(c, (current ?? (await reload())).map[id], status);82 c: Context,
83 id: Key,
84 status: StatusCode = 200,
85 cache: CacheMode = 'auto',
86) {
87 return assetInner(
88 c,
89 id,
90 (current ?? (await reload())).map[id],
91 status,
92 cache,
93 );
81}94}
8295
83/** @deprecated */96/** @deprecated */
...@@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) {...@@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
89 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;102 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
90}103}
91104
92function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {105function assetInner(
106 c: Context,
107 key: string,
108 asset: Manifest[Key],
109 status: StatusCode,
110 cache: CacheMode = "auto",
111) {
93 ASSERT(current);112 ASSERT(current);
94 if (asset.type === 0) {113 if (asset.type === 0) {
95 return respondWithBufferAndViews(c, current.static, asset, status);114 return respondWithBufferAndViews(
115 c,
116 key,
117 current.static,
118 asset,
119 status,
120 cache,
121 );
96 } else {122 } else {
97 const entry = UNWRAP(current.dynamic.get(asset.id));123 const entry = UNWRAP(current.dynamic.get(asset.id));
98 return respondWithBufferAndViews(c, entry.buffer, entry, status);124 return respondWithBufferAndViews(
125 c,
126 key,
127 entry.buffer,
128 entry,
129 status,
130 cache,
131 );
99 }132 }
100}133}
101134
102function respondWithBufferAndViews(135function respondWithBufferAndViews(
103 c: Context,136 c: Context,
137 key: string,
104 buffer: Buffer,138 buffer: Buffer,
105 asset: AssetBase,139 asset: AssetBase,
106 status: StatusCode,140 status: StatusCode,
141 cache: CacheMode,
107) {142) {
108 const ifNoneMatch = c.req.header("If-None-Match");143 const ifNoneMatch = c.req.header("If-None-Match");
109 if (ifNoneMatch) {144 if (ifNoneMatch) {
...@@ -136,6 +171,29 @@ function respondWithBufferAndViews(...@@ -136,6 +171,29 @@ function respondWithBufferAndViews(
136 } else {171 } else {
137 body = buffer.subarray(...asset.raw);172 body = buffer.subarray(...asset.raw);
138 }173 }
174 debug(
175 `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`,
176 );
177 if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) {
178 if (cache === "auto") {
179 if (status < 200 || status >= 300) {
180 cache = 'no-cache';
181 } else if (headers['content-type']?.includes('text/html')) {
182 cache = 'temporary';
183 } else {
184 cache = 'long-term';
185 }
186 }
187 if (cache === "no-cache") {
188 headers["Cache-Control"] = "no-store";
189 } else if (cache === "long-term") {
190 headers["Cache-Control"] =
191 "public, max-age=7200, stale-while-revalidate=7200, immutable";
192 } else if (cache === "temporary") {
193 headers["Cache-Control"] =
194 "public, max-age=7200, stale-while-revalidate=7200";
195 }
196 }
139 return (c.res = new Response(body, { headers, status }));197 return (c.res = new Response(body, { headers, status }));
140}198}
141199
...@@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";...@@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
232import * as crypto from "node:crypto";290import * as crypto from "node:crypto";
233import * as zlib from "node:zlib";291import * as zlib from "node:zlib";
234import * as util from "node:util";292import * as util from "node:util";
293import * as console from "@paperclover/console";
framework/lib/error.ts created+47
...@@ -0,0 +1,47 @@
1declare global {
2 interface Error {
3 /* Extra fields on errors are extremely helpful for debugging. */
4 [metadata: string]: unknown;
5 }
6}
7
8/** Retrieve an error message from any value */
9export function message(error: unknown): string {
10 const message = (error as { message: unknown })?.message ?? error;
11 try {
12 return typeof message === "string" ? message : JSON.stringify(message);
13 } catch {}
14 try {
15 return String(message);
16 } catch {}
17 return `Could not stringify error message ${typeof message}`;
18}
19
20/** Retrieve an error-like object from any value. Useful to check fields. */
21export function obj(error: unknown): { message: string } & Partial<Error> {
22 if (error instanceof Error) return error;
23 return {
24 message: message(error),
25 ...(typeof error === "object" && error),
26 };
27}
28
29/* Node.js error codes are strings */
30export function code(error: unknown): NodeErrorCode | null {
31 const code = (error as { code: unknown })?.code;
32 return typeof code === "string" ? code : null;
33}
34
35/** Attach extra fields and throw */
36export function rethrowWithMetadata(
37 err: unknown,
38 meta: Record<string, unknown>,
39): never {
40 const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/
41 Object.assign(error, meta);
42 throw error;
43}
44
45export type NodeErrorCode =
46 | keyof typeof import("node:os").constants.errno
47 | (string & {});
framework/lib/meta.ts+5-2
...@@ -4,6 +4,9 @@ export interface Meta {...@@ -4,6 +4,9 @@ export interface Meta {
4 openGraph?: OpenGraph;4 openGraph?: OpenGraph;
5 alternates?: Alternates;5 alternates?: Alternates;
6}6}
7export interface Template extends Omit<Meta, "title"> {
8 titleTemplate?: (title: string) => string,
9}
7export interface OpenGraph {10export interface OpenGraph {
8 title?: string;11 title?: string;
9 description?: string | undefined;12 description?: string | undefined;
...@@ -19,6 +22,6 @@ export interface AlternateType {...@@ -19,6 +22,6 @@ export interface AlternateType {
19 title: string;22 title: string;
20}23}
21export function renderMeta({ title }: Meta): string {24export function renderMeta({ title }: Meta): string {
22 return `<title>${esc(title)}</title>`;25 return `<title>${esc(title)}</title><link rel="icon" type="image/x-icon" href="/favicon.ico">`;
23}26}
24import { escapeHtml as esc } from "#engine/render";27import { escapeHtmlContent as esc } from "#engine/render";
framework/lib/mime.txt+100-99
...@@ -1,99 +1,100 @@...@@ -1,99 +1,100 @@
1# media types 1# media types
2# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types 2# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types
3.aac audio/x-aac 3.aac audio/x-aac
4.aif audio/x-aiff 4.aif audio/x-aiff
5.aifc audio/x-aiff 5.aifc audio/x-aiff
6.aiff audio/x-aiff 6.aiff audio/x-aiff
7.asm text/x-asm 7.asm text/x-asm
8.avi video/x-msvideo 8.avi video/x-msvideo
9.bat application/x-msdownload 9.bat application/x-msdownload
10.c text/x-c 10.c text/x-c
11.chat text/x-clover-chatlog 11.chat text/x-clover-chatlog
12.class application/java-vm 12.class application/java-vm
13.cmd application/x-msdownload 13.cmd application/x-msdownload
14.com application/x-msdownload 14.com application/x-msdownload
15.conf text/plain 15.conf text/plain
16.cpp text/x-c 16.cpp text/x-c
17.css text/css 17.css text/css
18.csv text/csv 18.csv text/csv
19.cxx text/x-c 19.cxx text/x-c
20.def text/plain 20.def text/plain
21.diff text/plain 21.diff text/plain
22.dll application/x-msdownload 22.dll application/x-msdownload
23.dmg application/octet-stream 23.dmg application/octet-stream
24.doc application/msword 24.doc application/msword
25.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document 25.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document
26.epub application/epub+zip 26.epub application/epub+zip
27.exe application/x-msdownload 27.exe application/x-msdownload
28.flv video/x-flv 28.flv video/x-flv
29.fbx application/fbx 29.fbx application/fbx
30.gz application/x-gzip 30.gz application/x-gzip
31.h text/x-c 31.h text/x-c
32.h264 video/h264 32.h264 video/h264
33.hh text/x-c 33.hh text/x-c
34.htm text/html;charset=utf-8 34.htm text/html;charset=utf-8
35.html text/html;charset=utf-8 35.html text/html;charset=utf-8
36.ico image/x-icon 36.ico image/x-icon
37.ics text/calendar 37.ics text/calendar
38.in text/plain 38.in text/plain
39.jar application/java-archive 39.jar application/java-archive
40.java text/x-java-source 40.java text/x-java-source
41.jpeg image/jpeg 41.jpeg image/jpeg
42.jpg image/jpeg 42.jpg image/jpeg
43.jpgv video/jpeg 43.jpgv video/jpeg
44.jxl image/jxl 44.jxl image/jxl
45.js application/javascript 45.js application/javascript
46.json application/json 46.json application/json
47.latex application/x-latex 47.latex application/x-latex
48.list text/plain 48.list text/plain
49.log text/plain 49.log text/plain
50.m4a audio/mp4 50.m4a audio/mp4
51.man text/troff 51.man text/troff
52.mid audio/midi 52.mid audio/midi
53.midi audio/midi 53.midi audio/midi
54.mov video/quicktime 54.mov video/quicktime
55.mp3 audio/mpeg 55.mp3 audio/mpeg
56.mp4 video/mp4 56.mp4 video/mp4
57.msh model/mesh 57.msh model/mesh
58.msi application/x-msdownload 58.msi application/x-msdownload
59.obj application/octet-stream 59.obj application/octet-stream
60.ogg audio/ogg 60.ogg audio/ogg
61.otf application/x-font-otf 61.otf application/x-font-otf
62.pdf application/pdf 62.pdf application/pdf
63.png image/png 63.png image/png
64.ppt application/vnd.ms-powerpoint 64.ppt application/vnd.ms-powerpoint
65.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation 65.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation
66.psd image/vnd.adobe.photoshop 66.psd image/vnd.adobe.photoshop
67.py text/x-python 67.py text/x-python
68.rar application/x-rar-compressed 68.rar application/x-rar-compressed
69.rss application/rss+xml 69.rss application/rss+xml
70.rtf application/rtf 70.rtf application/rtf
71.rtx text/richtext 71.rtx text/richtext
72.s text/x-asm 72.s text/x-asm
73.pem application/x-pem-file" 73.pem application/x-pem-file"
74.ser application/java-serialized-object 74.ser application/java-serialized-object
75.sh application/x-sh 75.sh application/x-sh
76.sig application/pgp-signature 76.sig application/pgp-signature
77.silo model/mesh 77.silo model/mesh
78.svg image/svg+xml 78.svg image/svg+xml
79.t text/troff 79.t text/troff
80.tar application/x-tar 80.tar application/x-tar
81.text text/plain 81.text text/plain
82.tgz application/x-gzip 82.tgz application/x-gzip
83.tif image/tiff 83.tif image/tiff
84.tiff image/tiff 84.tiff image/tiff
85.torrent application/x-bittorrent 85.torrent application/x-bittorrent
86.ttc application/x-font-ttf 86.ttc application/x-font-ttf
87.ttf application/x-font-ttf 87.ttf application/x-font-ttf
88.txt text/plain 88.txt text/plain
89.urls text/uri-list 89.urls text/uri-list
90.v text/x-v 90.v text/x-v
91.wav audio/x-wav 91.wav audio/x-wav
92.wmv video/x-ms-wmv 92.wmv video/x-ms-wmv
93.xls application/vnd.ms-excel 93.woff2 font/woff2
94.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet 94.xls application/vnd.ms-excel
95.xml application/xml 95.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
96.xps application/vnd.ms-xpsdocument 96.xml application/xml
97 97.xps application/vnd.ms-xpsdocument
98# special cased based on file name 98
99rss.xml application/rss+xml 99# special cased based on file name
100rss.xml application/rss+xml
framework/lib/sitegen.ts+18
...@@ -31,6 +31,24 @@ export interface FileItem {...@@ -31,6 +31,24 @@ export interface FileItem {
31export interface Section {31export interface Section {
32 root: string;32 root: string;
33}33}
34export interface Font {
35 name: string;
36 /**
37 * Specify either font name, file path, or URL to fetch it.
38 * Private fonts do not have a URL and will fail to build if missing.
39 */
40 sources: string[];
41 subsets: Array<{
42 vars?: Record<string, FontVariableAxis>;
43 layoutFeatures?: string[];
44 unicodes: FontRange | FontRange[];
45 /** Include [ext] to autofill 'woff2' */
46 asset: string;
47 }>;
48}
49export type FontVars = Record<string, FontVariableAxis>;
50export type FontVariableAxis = number | [min: number, max: number];
51export type FontRange = string | number | { start: number, end: number };
3452
35export const userData = render.userData<SitegenRender>(() => {53export const userData = render.userData<SitegenRender>(() => {
36 throw new Error("This function can only be used in a page (static or view)");54 throw new Error("This function can only be used in a page (static or view)");
framework/lib/sqlite.ts+5-4
...@@ -11,9 +11,9 @@ export function getDb(file: string) {...@@ -11,9 +11,9 @@ export function getDb(file: string) {
11 let db = map.get(file);11 let db = map.get(file);
12 if (db) return db;12 if (db) return db;
13 const fileWithExt = file.includes(".") ? file : file + ".sqlite";13 const fileWithExt = file.includes(".") ? file : file + ".sqlite";
14 db = new WrappedDatabase(14 const dir = process.env.CLOVER_DB ?? ".clover";
15 path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),15 fs.mkdirSync(dir);
16 );16 db = new WrappedDatabase(path.join(dir, fileWithExt));
17 map.set(file, db);17 map.set(file, db);
18 return db;18 return db;
19}19}
...@@ -21,7 +21,7 @@ export function getDb(file: string) {...@@ -21,7 +21,7 @@ export function getDb(file: string) {
21export class WrappedDatabase {21export class WrappedDatabase {
22 file: string;22 file: string;
23 node: DatabaseSync;23 node: DatabaseSync;
24 stmts: Stmt[];24 stmts: Stmt[] = [];
25 stmtTableMigrate: WeakRef<StatementSync> | null = null;25 stmtTableMigrate: WeakRef<StatementSync> | null = null;
2626
27 constructor(file: string) {27 constructor(file: string) {
...@@ -153,3 +153,4 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {...@@ -153,3 +153,4 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
153153
154import { DatabaseSync, StatementSync } from "node:sqlite";154import { DatabaseSync, StatementSync } from "node:sqlite";
155import * as path from "node:path";155import * as path from "node:path";
156import * as fs from "#sitegen/fs";
framework/lib/subprocess.ts created+15
...@@ -0,0 +1,15 @@
1const execFileRaw = util.promisify(child_process.execFile);
2export const exec: typeof execFileRaw = ((
3 ...args: Parameters<typeof execFileRaw>
4) =>
5 execFileRaw(...args).catch((e: any) => {
6 if (e?.message?.startsWith?.("Command failed")) {
7 if (e.code > 2 ** 31) e.code |= 0;
8 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
9 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
10 }
11 throw e;
12 })) as any;
13
14import * as util from 'node:util';
15import * as child_process from 'node:child_process';
framework/watch.ts+1
...@@ -25,6 +25,7 @@ function serve() {...@@ -25,6 +25,7 @@ function serve() {
25 "--development",25 "--development",
26 ], {26 ], {
27 stdio: "inherit",27 stdio: "inherit",
28 execArgv: ['--no-warnings'],
28 });29 });
29 subprocess.on("close", onSubprocessClose);30 subprocess.on("close", onSubprocessClose);
30}31}
readme.md+37-17
...@@ -11,23 +11,21 @@ that assist building websites. these tools power <https://paperclover.net>....@@ -11,23 +11,21 @@ that assist building websites. these tools power <https://paperclover.net>.
11 - Different languages can be used at the same time. Supports `async function`11 - Different languages can be used at the same time. Supports `async function`
12 components, `<Suspense />`, and custom extensions.12 components, `<Suspense />`, and custom extensions.
13- **Incremental static site generator and build system.**13- **Incremental static site generator and build system.**
14 - Build entire production site at start, incremental updates when pages14 - Build both development and production sites on startup start. Incremental
15 change; Build system state survives coding sessions.15 generator rebuilds changed pages; Build system state survives coding sessions.
16 - The only difference in development and production mode is hidden source-maps16 - Multiple backend support.
17 and stripped `console.debug` calls. The site you see locally is the same
18 site you see deployed.
19 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs17 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
20 checks when the files change. For example, changing a component re-tests18 checks when the files change. For example, changing a component re-tests
21 only pages that use that component and re-lints only the changed file.19 only pages that use that component and re-lints only the changed file.
22- **Integrated libraries for building complex, content heavy web sites.**20- **Integrated libraries for building complex, content heavy web sites.**
23 - Static asset serving with ETag and build-time compression.21 - Static asset serving with ETag and build-time compression.
24 - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)22 - Dynamicly rendered pages with static client. (`import "#sitegen/view"`)
25 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)23 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
26 - TODO: Meta and Open Graph generation. (`export const meta`)24 - TODO: Meta and Open Graph generation. (`export const meta`)
27 - TODO: Font subsetting tools to reduce bytes downloaded by fonts.25 - Font subsetting to reduce page bandwidth.
28- **Built on the battle-tested Node.js runtime.**26- **Built on the stable and battle-tested Node.js runtime.**
2927
30None of these tools are complex or revolutionary. Rather, this project is the28none of these tools are complex or revolutionary. rather, this project is the
31sum of many years of experience on managing content heavy websites, and an29sum of many years of experience on managing content heavy websites, and an
32example on how other over-complicate other frameworks.30example on how other over-complicate other frameworks.
3331
...@@ -35,9 +33,11 @@ example on how other over-complicate other frameworks....@@ -35,9 +33,11 @@ example on how other over-complicate other frameworks.
3533
36Included is `src`, which contains `paperclover.net`. Website highlights:34Included is `src`, which contains `paperclover.net`. Website highlights:
3735
36- TODO: flashy homepage.
38- [Question/Answer board, custom markdown parser and components][q+a].37- [Question/Answer board, custom markdown parser and components][q+a].
39- [File viewer with fast ui/ux + optimized media streaming][file].38- [File viewer with fast ui/ux + optimized media streaming][file].
40- [Personal, friends-only blog with password protection][friends].39- [Personal, friends-only blog with password protection][friends].
40- TODO: digital garden styled blog.
4141
42[q+a]: https://paperclover.net/q+a42[q+a]: https://paperclover.net/q+a
43[file]: https://paperclover.net/file43[file]: https://paperclover.net/file
...@@ -51,15 +51,21 @@ minimum system requirements:...@@ -51,15 +51,21 @@ minimum system requirements:
51- random access memory.51- random access memory.
52- windows 7 or later, macos, or other operating system.52- windows 7 or later, macos, or other operating system.
5353
54required software:
55
56- node.js v24
57- python v3
58
54my development machine, for example, is Dell Inspiron 7348 with Core i759my development machine, for example, is Dell Inspiron 7348 with Core i7
5560
56 npm install61 npm install
5762
58 # production generation63 # build site using 'run.js' to enable runtime plugins
59 node run generate64 node run generate
60 node .clover/o/backend65 # the built site runs in regular node.js
66 node --enable-source-maps .clover/o/backend
6167
62 # "development" watch mode68 # watch-rebuild mode
63 node run watch69 node run watch
6470
65for unix systems, the provided `flake.nix` can be used with `nix develop` to71for unix systems, the provided `flake.nix` can be used with `nix develop` to
...@@ -67,7 +73,7 @@ open a shell with all needed system dependencies....@@ -67,7 +73,7 @@ open a shell with all needed system dependencies.
6773
68## Deployment74## Deployment
6975
70There are two primary server components to be deployed: the web server and the76there are two primary server components to be deployed: the web server and the
71sourth of truth server. The latter is a singleton that runs on Clover's NAS,77sourth of truth server. The latter is a singleton that runs on Clover's NAS,
72which holds the full contents of the file storage. The web server pulls data78which holds the full contents of the file storage. The web server pulls data
73from the source of truth and renders web pages, and can be duplicated to79from the source of truth and renders web pages, and can be duplicated to
...@@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose:...@@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose:
79 backend:85 backend:
80 container_name: backend86 container_name: backend
81 build:87 build:
82 # this uses loopback to hit the self-hosted git server88 # this uses loopback to hit the self-hosted git server,
89 # docker will cache the image to not re-fetch on reboot.
83 context: http://127.0.0.1:3000/clo/sitegen.git90 context: http://127.0.0.1:3000/clo/sitegen.git
84 dockerfile: src/source-of-truth.dockerfile91 dockerfile: src/source-of-truth.dockerfile
85 environment:92 environment:
...@@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose:...@@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose:
96 - /mnt/storage1/clover/Documents/Config/paperclover:/data103 - /mnt/storage1/clover/Documents/Config/paperclover:/data
97 - /mnt/storage1/clover/Published:/published104 - /mnt/storage1/clover/Published:/published
98105
99Due to caching, one may need to manually purge images via106Due to caching, images may need to be purged via `docker image rm {image} -f`
100`docker image rm ix-clover-backend -f` when an update is desired107when an update is desired. Some docker GUIs support force pulls, some are buggy.
101108
102TODO: deployment instructions for a web node109The web server performs rendering. A `Dockerfile` for it is present in
110`src/web.dockerfile` but it is currently unused. Deployments are done by
111building the project locally, and then using `rsync` to copy files.
112
113 node run generate
114 rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \
115 --delete-after --progress --human-readable
116 ssh "$REMOTE_USER" /bin/bash << EOF
117 set -e
118 cd ~/paperclover
119 npm ci
120 pm2 restart site
121 echo "-> https://paperclover.net"
122 EOF
103123
104## Contributions124## Contributions
105125
run.js+22-13
...@@ -3,7 +3,6 @@...@@ -3,7 +3,6 @@
3import * as util from "node:util";3import * as util from "node:util";
4import * as zlib from "node:zlib";4import * as zlib from "node:zlib";
5import * as url from "node:url";5import * as url from "node:url";
6import * as module from "node:module";
7import process from "node:process";6import process from "node:process";
87
9// Disable experimental warnings (Type Stripping, etc)8// Disable experimental warnings (Type Stripping, etc)
...@@ -28,8 +27,8 @@ try {...@@ -28,8 +27,8 @@ try {
28 const brand = process.versions.bun27 const brand = process.versions.bun
29 ? `bun ${process.versions.bun}`28 ? `bun ${process.versions.bun}`
30 : process.versions.deno29 : process.versions.deno
31 ? `deno ${process.versions.deno}`30 ? `deno ${process.versions.deno}`
32 : null;31 : null;
33 const brandText = brand ? ` (${brand})` : "";32 const brandText = brand ? ` (${brand})` : "";
34 globalThis.console.error(33 globalThis.console.error(
35 `sitegen depends on a node.js v24. your runtime is missing features.\n` +34 `sitegen depends on a node.js v24. your runtime is missing features.\n` +
...@@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {...@@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
75 else if (mod.default?.fetch) {74 else if (mod.default?.fetch) {
76 const protocol = "http";75 const protocol = "http";
77 const { serve } = hot.load("@hono/node-server");76 const { serve } = hot.load("@hono/node-server");
78 serve({77 serve(
79 fetch: mod.default.fetch,78 {
80 }, ({ address, port }) => {79 fetch: mod.default.fetch,
81 if (address === "::") address = "::1";80 },
82 console.info(url.format({81 ({ address, port }) => {
83 protocol,82 if (address === "::") address = "::1";
84 hostname: address,83 console.info(
85 port,84 url.format({
86 }));85 protocol,
87 });86 hostname: address,
87 port,
88 }),
89 );
90 },
91 );
92 } else {
93 console.warn(
94 hot.load("path").relative('.', found),
95 'does not export a "main" function',
96 );
88 }97 }
89 } catch (e) {98 } catch (e) {
90 console.error(util.inspect(e));99 console.error(util.inspect(e));
src/admin.ts+5-1
...@@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year...@@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year
3let lastKnownToken: string | null = null;3let lastKnownToken: string | null = null;
4function compareToken(token: string) {4function compareToken(token: string) {
5 if (token === lastKnownToken) return true;5 if (token === lastKnownToken) return true;
6 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();6 try {
7 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
8 } catch {
9 return false;
10 }
7 return token === lastKnownToken;11 return token === lastKnownToken;
8}12}
913
src/backend.ts+14
...@@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app);...@@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app);
15// Asset middleware has least precedence15// Asset middleware has least precedence
16app.use(assets.middleware);16app.use(assets.middleware);
1717
18// Asset Aliases
19app.get(
20 "/apple-touch-icon-precomposed.png",
21 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
22);
23app.get(
24 "/apple-touch-icon-57x57.png",
25 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
26);
27app.get(
28 "/apple-touch-icon-57x57-precomposed.png",
29 (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
30);
31
18// Handlers32// Handlers
19app.notFound(assets.notFound);33app.notFound(assets.notFound);
2034
src/file-viewer/bin/scan3.ts+3-15
...@@ -10,7 +10,6 @@...@@ -10,7 +10,6 @@
10// This is the third iteration of the scanner, hence its name "scan3";10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.12// quality cannot be written with AI.
13const root = path.resolve("/Volumes/clover/Published");
14const workDir = path.resolve(".clover/derived");13const workDir = path.resolve(".clover/derived");
15const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);14const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
1615
...@@ -404,18 +403,6 @@ interface Process {...@@ -404,18 +403,6 @@ interface Process {
404 undo?(mediaFile: MediaFile): Promise<void>;403 undo?(mediaFile: MediaFile): Promise<void>;
405}404}
406405
407const execFileRaw = util.promisify(child_process.execFile);
408const execFile: typeof execFileRaw = ((
409 ...args: Parameters<typeof execFileRaw>
410) =>
411 execFileRaw(...args).catch((e: any) => {
412 if (e?.message?.startsWith?.("Command failed")) {
413 if (e.code > 2 ** 31) e.code |= 0;
414 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
415 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
416 }
417 throw e;
418 })) as any;
419const ffprobeBin = testProgram("ffprobe", "--help");406const ffprobeBin = testProgram("ffprobe", "--help");
420const ffmpegBin = testProgram("ffmpeg", "--help");407const ffmpegBin = testProgram("ffmpeg", "--help");
421408
...@@ -426,7 +413,7 @@ const procDuration: Process = {...@@ -426,7 +413,7 @@ const procDuration: Process = {
426 enable: ffprobeBin !== null,413 enable: ffprobeBin !== null,
427 include: rules.extsDuration,414 include: rules.extsDuration,
428 async run({ absPath, mediaFile }) {415 async run({ absPath, mediaFile }) {
429 const { stdout } = await execFile(ffprobeBin!, [416 const { stdout } = await subprocess.exec(ffprobeBin!, [
430 "-v",417 "-v",
431 "error",418 "error",
432 "-show_entries",419 "-show_entries",
...@@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress";...@@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress";
793import { Spinner } from "@paperclover/console/Spinner";780import { Spinner } from "@paperclover/console/Spinner";
794import * as async from "#sitegen/async";781import * as async from "#sitegen/async";
795import * as fs from "#sitegen/fs";782import * as fs from "#sitegen/fs";
783import * as subprocess from "#sitegen/subprocess";
796784
797import * as path from "node:path";785import * as path from "node:path";
798import * as zlib from "node:zlib";786import * as zlib from "node:zlib";
799import * as child_process from "node:child_process";787import * as child_process from "node:child_process";
800import * as util from "node:util";
801import * as crypto from "node:crypto";788import * as crypto from "node:crypto";
802import * as stream from "node:stream";789import * as stream from "node:stream";
803790
...@@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts";...@@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts";
814import * as ffmpeg from "@/file-viewer/ffmpeg.ts";801import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
815import * as rsync from "@/file-viewer/rsync.ts";802import * as rsync from "@/file-viewer/rsync.ts";
816import * as transcodeRules from "@/file-viewer/transcode-rules.ts";803import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
804import { rawFileRoot as root } from "../paths.ts";
src/file-viewer/cache.ts+4-4
...@@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts";...@@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts";
12declare const Deno: any;12declare const Deno: any;
1313
14const sourceOfTruth = "https://nas.paperclover.net:43250";14const sourceOfTruth = "https://nas.paperclover.net:43250";
15const caCert = fs.readFileSync("src/file-viewer/cert.pem");15// const caCert = fs.readFileSync("src/file-viewer/cert.pem");
1616
17const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/");17const diskCacheRoot = path.join(import.meta.dirname, ".filecache/");
18const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB18const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB
19const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB19const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB
20const loadInProgress = new Map<20const loadInProgress = new Map<
...@@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined"...@@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined"
338 // Bun has two non-standard fetch extensions338 // Bun has two non-standard fetch extensions
339 decompress: false,339 decompress: false,
340 tls: {340 tls: {
341 ca: caCert,341 // ca: caCert,
342 },342 },
343 }343 }
344 // TODO: https://github.com/denoland/deno/issues/12291344 // TODO: https://github.com/denoland/deno/issues/12291
...@@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined"...@@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined"
351 // }351 // }
352 // Node.js supports node:http352 // Node.js supports node:http
353 : new Agent({353 : new Agent({
354 ca: caCert,354 // ca: caCert,
355 });355 });
356356
357function fetchFileNode(pathname: string): Promise<ReadableStream> {357function fetchFileNode(pathname: string): Promise<ReadableStream> {
src/file-viewer/paths.ts created+12
...@@ -0,0 +1,12 @@
1export const nasRoot = process.platform === "win32"
2 ? "\\\\zenith\\clover"
3 : process.platform === "darwin"
4 ? "/Volumes/clover"
5 : "/media/clover";
6
7export const rawFileRoot = process.env.CLOVER_FILE_RAW ??
8 path.join(nasRoot, "Published");
9export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
10 path.join(nasRoot, "Documents/Config/paperclover/derived");
11
12import * as path from "node:path";
src/site.ts+93-7
...@@ -1,11 +1,11 @@...@@ -1,11 +1,11 @@
1// This file defines the different "Sections" of the website. The sections act
2// as separate codebases, but hosted as one. This allows me to have
3// sub-projects like the file viewer in 'file', or the question answer system
4// in 'q+a'. Each section can define configuration, pages, backend routes, and
5// contain other files.
6const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);1const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
72
8export const siteSections: Section[] = [3// Different sections of the website are split into their own folders. Acting as
4// as separate codebases, they are hosted as one. This allows me to have
5// sub-projects like the file viewer in 'file/', and the question answer system
6// in 'q+a', but maintain clear boundaries. Each section can define
7// configuration, pages, backend routes, and contain other files.
8export const siteSections: sg.Section[] = [
9 { root: join(".") },9 { root: join(".") },
10 { root: join("q+a/") },10 { root: join("q+a/") },
11 { root: join("file-viewer/") },11 { root: join("file-viewer/") },
...@@ -14,10 +14,96 @@ export const siteSections: Section[] = [...@@ -14,10 +14,96 @@ export const siteSections: Section[] = [
14 // { root: join("fiction/"), pageBase: "/fiction" },14 // { root: join("fiction/"), pageBase: "/fiction" },
15];15];
1616
17// All backends are bundled. The backend named "backend" is run by "node run watch"
17export const backends: string[] = [18export const backends: string[] = [
18 join("backend.ts"),19 join("backend.ts"),
19 join("source-of-truth.ts"),20 join("source-of-truth.ts"),
20];21];
2122
23// Font subsets reduce bandwidth and protect against proprietary font theft.
24const fontRoot = path.join(nasRoot, 'Documents/Font');
25const ascii = { start: 0x20, end: 0x7E };
26const nonAscii: sg.FontRange[] = [
27 { start: 0xC0, end: 0xFF },
28 { start: 0x2190, end: 0x2193 },
29 { start: 0xA0, end: 0xA8 },
30 { start: 0xAA, end: 0xBF },
31 { start: 0x2194, end: 0x2199 },
32 { start: 0x100, end: 0x17F },
33 0xA9,
34 0x2018,
35 0x2019,
36 0x201C,
37 0x201D,
38 0x2022,
39];
40const recursiveVars: sg.FontVars = {
41 WGHT: [400, 750],
42 SLNT: [-15, 0],
43 CASL: 0.25,
44 CRSV: 0.5,
45 MONO: 0,
46};
47const recursiveVarsMono = { ...recursiveVars, MONO: 1 };
48const recursiveVarsQuestion: sg.FontVars = {
49 ...recursiveVars,
50 MONO: [0, 1],
51 WGHT: [400, 1000],
52};
53const layoutFeatures = ["numr", "dnom", "frac"];
54export const fonts: sg.Font[] = [
55 {
56 name: "Recursive",
57 sources: [
58 "Recursive_VF_1.085.ttf",
59 path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
60 "https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf",
61 ],
62 subsets: [
63 {
64 asset: "/recultramin.[ext]",
65 layoutFeatures,
66 vars: recursiveVars,
67 unicodes: ascii,
68 },
69 {
70 asset: "/recmono.[ext]",
71 vars: recursiveVarsMono,
72 unicodes: ascii,
73 },
74 {
75 asset: "/recqa.[ext]",
76 vars: recursiveVarsQuestion,
77 unicodes: ascii,
78 },
79 {
80 asset: "/recexotic.[ext]",
81 vars: recursiveVarsQuestion,
82 unicodes: nonAscii,
83 },
84 ],
85 },
86 {
87 name: "AT Name Sans Display Hairline",
88 sources: [
89 "ATNameSansDisplay-Hairline.woff2",
90 path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
91 ],
92 subsets: [
93 {
94 asset: "/cydn_header.[ext]",
95 layoutFeatures,
96 unicodes: "cotlyedon",
97 },
98 ],
99 },
100];
101
102export async function main() {
103 await font.buildFonts(fonts);
104}
105
22import * as path from "node:path";106import * as path from "node:path";
23import type { Section } from "#sitegen";107import * as font from "../framework/font.ts";
108import type * as sg from "#sitegen";
109import { nasRoot } from "./file-viewer/paths.ts";
src/source-of-truth.ts+1-11
...@@ -22,17 +22,6 @@ export default app;...@@ -22,17 +22,6 @@ export default app;
2222
23const token = UNWRAP(process.env.CLOVER_SOT_KEY);23const token = UNWRAP(process.env.CLOVER_SOT_KEY);
2424
25const nasRoot = process.platform === "win32"
26 ? "\\\\zenith\\clover"
27 : process.platform === "darwin"
28 ? "/Volumes/clover"
29 : "/media/clover";
30
31const rawFileRoot = process.env.CLOVER_FILE_RAW ??
32 path.join(nasRoot, "Published");
33const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
34 path.join(nasRoot, "Documents/Config/paperclover/derived");
35
36if (!fs.existsSync(rawFileRoot)) {25if (!fs.existsSync(rawFileRoot)) {
37 throw new Error(`${rawFileRoot} does not exist`);26 throw new Error(`${rawFileRoot} does not exist`);
38}27}
...@@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs";...@@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs";
125import * as util from "node:util";114import * as util from "node:util";
126import * as stream from "node:stream";115import * as stream from "node:stream";
127import * as mime from "#sitegen/mime";116import * as mime from "#sitegen/mime";
117import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts";
src/static/apple-touch-icon.png created
Binary files /dev/null and b/src/static/apple-touch-icon.png differ
src/static/favicon.ico created
Binary files /dev/null and b/src/static/favicon.ico differ
src/web.dockerfile created+19
...@@ -0,0 +1,19 @@
1from node:24 as builder
2
3run apt install -y git
4
5workdir /tmp/builder
6copy package*.json ./
7run npm ci
8copy . ./
9run node run generate
10run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/
11
12from node:24 as runtime
13
14workdir /app
15copy --from=builder /app/ ./
16
17env PORT=80
18
19cmd ["node", "--enable-source-maps", "/app/backend.js"]