diff --git a/flake.nix b/flake.nix index e2f36c1077022b81fff46184b81c173b4580489e..4b73e17bc068a0c005e38a0f3e277b2d57aebf5b 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,12 @@ with inputs.nixpkgs.legacyPackages.${system}; { devShells.default = pkgs.mkShell { buildInputs = [ + # clover sitegen v3 pkgs.nodejs_24 # runtime pkgs.deno # formatter + pkgs.python3 # for font subsetting + + # paperclover.net (pkgs.ffmpeg.override { withOpus = true; withSvtav1 = true; diff --git a/framework/backend/entry-node.ts b/framework/backend/entry-node.ts index 4f4a24902e94c56f5d6c4111b8192a75a31028c6..35292723ec4e6e68e8c92158e9980aefc6067a2e 100644 --- a/framework/backend/entry-node.ts +++ b/framework/backend/entry-node.ts @@ -9,7 +9,7 @@ const server = serve({ fetch: app.default.fetch, port: Number(process.env.PORT ?? 3000), }, ({ address, port }) => { - if (address === "::") address = "::1"; + if (address === "::") address = "localhost"; console.info(url.format({ protocol, hostname: address, diff --git a/framework/bundle.ts b/framework/bundle.ts index 3a4f3a1cc1b3f71fccb7e06cbcb192a4c2886dea..01c9e02d4bd43177bfb83dade79bab18965ab34e 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -94,7 +94,6 @@ export async function bundleClientJavaScript( const chunk = route.startsWith("/js/c."); if (!chunk) { const key = hot.getScriptId(toAbs(UNWRAP(entryPoint))); - console.log(route, key); route = "/js/" + key.replace(/\.client\.tsx?/, ".js"); scripts[key] = text; } diff --git a/framework/font.ts b/framework/font.ts new file mode 100644 index 0000000000000000000000000000000000000000..7645a05b8f54b0bbb4018d728a3c65a19cc726f3 --- /dev/null +++ b/framework/font.ts @@ -0,0 +1,192 @@ +export async function buildFonts(fonts: sg.Font[]) { + if (fonts.length === 0) return; + + const temp = path.resolve(".clover/font"); + const venv = path.join(temp, "venv"); + const bin = path.join(venv, "bin"); + + if (!fs.existsSync(venv)) { + await subprocess.exec("python", ["-m", "venv", venv]); + } + if (!fs.existsSync(path.join(bin, "pyftsubset"))) { + await subprocess.exec(path.join(bin, "pip"), [ + "install", + "fonttools==4.38.0", + "brotli==1.0.7", + ]); + } + + const instances = new async.OnceMap(); + const subsets = new async.OnceMap(); + async function makeInstance(input: string, vars: sg.FontVars) { + const args = Object.entries(vars).map((x) => { + const lower = x[0].toLowerCase(); + const k = lower === "wght" || lower === "slnt" ? lower : x[0]; + const v = Array.isArray(x[1]) ? x[1].join(":") : x[1]; + return `${k}=${v}`; + }).sort(); + const hash = crypto + .createHash("sha256") + .update(`${input}${args.join(" ")}`) + .digest("hex") + .slice(0, 16) + .toLowerCase(); + const outfile = path.join(temp, `${hash}${path.extname(input)}`); + await instances.get(outfile, async () => { + await fs.rm(outfile, { force: true }); + await subprocess.exec(path.join(bin, "fonttools"), [ + "varLib.instancer", + input, + ...args, + `--output=${outfile}`, + ]); + ASSERT(fs.existsSync(outfile)); + }); + return outfile; + } + + await Promise.all(fonts.map((font) => + incr.work(async (io, font) => { + const baseFile = await fetchFont(font.name, font.sources); + await Promise.all(font.subsets.map(async (subset) => { + let file = baseFile; + if (subset.vars) { + file = await makeInstance(baseFile, subset.vars); + } + const unicodes = fontRangesToString( + Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], + ); + const hash = crypto + .createHash("sha256") + .update(`${file}${unicodes}`) + .digest("hex") + .slice(0, 16) + .toLowerCase(); + const woff = path.join(temp, hash + ".woff2"); + await subprocess.exec(path.join(bin, "pyftsubset"), [ + file, + "--flavor=woff2", + `--output-file=${woff}`, + ...subset.layoutFeatures + ? [`--layout-features=${subset.layoutFeatures.join(",")}`] + : [], + `--unicodes=${unicodes}`, + ]); + await io.writeAsset({ + pathname: subset.asset.replace("[ext]", "woff2"), + buffer: await fs.readFile(woff), + }); + })); + }, font) + )); +} + +export async function fetchFont(name: string, sources: string[]) { + const errs = []; + + for (const source of sources) { + const cacheName = path.join("", name + path.extname(source)); + if (fs.existsSync(cacheName)) return cacheName; + + if (source.startsWith("https://")) { + const response = await fetch(source); + if (response.ok) { + await fs.writeMkdir( + cacheName, + Buffer.from(await response.arrayBuffer()), + ); + } else { + errs.push( + new Error( + `Fetching from ${source} failed: ${response.status} ${response.statusText}`, + ), + ); + continue; + } + } + + if (path.isAbsolute(source)) { + if (fs.existsSync(source)) return source; + errs.push(new Error(`Font not available at absolute path ${source}`)); + } else if (!source.includes("/") && !source.includes("\\")) { + const home = os.homedir(); + const bases = process.platform === "win32" + ? [ + "\\Windows\\Fonts", + path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"), + ] + : process.platform === "darwin" + ? [ + "/Library/Fonts", + path.posix.join(home, "Library/Fonts"), + ] + : [ + "/usr/share/fonts", + "/usr/local/share/fonts", + path.posix.join(home, ".local/share/fonts"), + ]; + for (const base of bases) { + const found = fs.readDirRecOptionalSync(base) + .find((file) => path.basename(file) === source); + if (found) { + return path.join(base, found); + } + } + errs.push( + new Error( + `Font file ${source} not found in the following directories: ${ + bases.join(", ") + }`, + ), + ); + } + } + + throw new AggregateError(errs, `Failed to fetch the ${name} font`); +} + +function fontRangesToString(ranges: sg.FontRange[]): string { + const segments: Array<{ start: number; end: number }> = []; + + ranges.forEach((range) => { + if (typeof range === "string") { + for (let i = 0; i < range.length; i++) { + const cp = range.codePointAt(i) || 0; + segments.push({ start: cp, end: cp }); + } + } else if (typeof range === "number") { + segments.push({ start: range, end: range }); + } else { + segments.push({ start: range.start, end: range.end }); + } + }); + + segments.sort((a, b) => a.start - b.start); + + const merged: Array<{ start: number; end: number }> = []; + for (const seg of segments) { + const last = merged[merged.length - 1]; + if (last && seg.start <= last.end + 1) { + last.end = Math.max(last.end, seg.end); + } else { + merged.push({ ...seg }); + } + } + + return merged.map(({ start, end }) => + start === end + ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}` + : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${ + end.toString(16).toUpperCase().padStart(4, "0") + }` + ).join(","); +} + +import * as fs from "#sitegen/fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import * as sg from "#sitegen"; +import * as subprocess from "#sitegen/subprocess"; +import * as incr from "./incremental.ts"; +import * as crypto from "node:crypto"; +import * as async from "#sitegen/async"; diff --git a/framework/generate.ts b/framework/generate.ts index 31ca14cd512e41ad5df7516e1209d55ef7779a1e..90fb65a5d509ee5b3bfe5577403422c9b3f15af4 100644 --- a/framework/generate.ts +++ b/framework/generate.ts @@ -22,6 +22,9 @@ export async function generate() { // TODO: loadMarkoCache + // -- start font work -- + const builtFonts = fonts.buildFonts(siteConfig.fonts); + // -- perform build-time rendering -- const builtPages = pages.map((item) => incr.work(preparePage, item)); const builtViews = views.map((item) => incr.work(prepareView, item)); @@ -70,7 +73,12 @@ export async function generate() { assembleAndWritePage(page, styleMap, scriptMap) ); - await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]); + await Promise.all([ + builtBackend, + builtStaticFiles, + ...pAssemblePages, + builtFonts, + ]); } export async function readManifest(io: Io) { @@ -80,6 +88,7 @@ export async function readManifest(io: Io) { root: toRel(section.root), })), backends: cfg.backends.map(toRel), + fonts: cfg.fonts, }; } @@ -324,14 +333,17 @@ export async function assembleAndWritePage( export type PageOrView = PreparedPage | PreparedView; +import * as path from "node:path"; + +import * as fs from "#sitegen/fs"; +import * as meta from "#sitegen/meta"; +import * as render from "#engine/render"; import * as sg from "#sitegen"; -import * as incr from "./incremental.ts"; -import { Io } from "./incremental.ts"; +import type { FileItem } from "#sitegen"; + import * as bundle from "./bundle.ts"; import * as css from "./css.ts"; -import * as render from "#engine/render"; +import * as fonts from "./font.ts"; import * as hot from "./hot.ts"; -import * as fs from "#sitegen/fs"; -import type { FileItem } from "#sitegen"; -import * as path from "node:path"; -import * as meta from "#sitegen/meta"; +import * as incr from "./incremental.ts"; +import { Io } from "./incremental.ts"; diff --git a/framework/lib/assets.ts b/framework/lib/assets.ts index ac77d492d9a610c34f44373f7480c9a7a18a13ca..fa5338df254446950959c8253577b71edabd6224 100644 --- a/framework/lib/assets.ts +++ b/framework/lib/assets.ts @@ -2,6 +2,7 @@ // This module implements decoding and serving of the asset blobs, // but also implements patching of dynamic assets. The `Manifest` // is generated by `incremental.ts` +const debug = console.scoped("assets"); const root = import.meta.dirname; let current: Loaded | null = null; @@ -36,6 +37,7 @@ interface Loaded { export interface DynamicEntry extends AssetBase { buffer: Buffer; } +type CacheMode = "auto" | "long-term" | "temporary" | "no-cache"; export async function reload() { const map = await fs.readJson(path.join(root, "asset.json")); @@ -59,7 +61,7 @@ export async function reload() { export async function middleware(c: Context, next: Next) { if (!current) current = await reload(); const asset = current.map[c.req.path]; - if (asset) return assetInner(c, asset, 200); + if (asset) return assetInner(c, c.req.path, asset, 200); return next(); } @@ -68,16 +70,27 @@ export async function notFound(c: Context) { let pathname = c.req.path; do { const asset = current.map[pathname + "/404"]; - if (asset) return assetInner(c, asset, 404); + if (asset) return assetInner(c, pathname + "/404", asset, 404); pathname = pathname.slice(0, pathname.lastIndexOf("/")); } while (pathname); const asset = current.map["/404"]; - if (asset) return assetInner(c, asset, 404); + if (asset) return assetInner(c, "/404", asset, 404); return c.text("the 'Not Found' page was not found", 404); } -export async function serveAsset(c: Context, id: Key, status: StatusCode) { - return assetInner(c, (current ?? (await reload())).map[id], status); +export async function serveAsset( + c: Context, + id: Key, + status: StatusCode = 200, + cache: CacheMode = 'auto', +) { + return assetInner( + c, + id, + (current ?? (await reload())).map[id], + status, + cache, + ); } /** @deprecated */ @@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) { return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; } -function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) { +function assetInner( + c: Context, + key: string, + asset: Manifest[Key], + status: StatusCode, + cache: CacheMode = "auto", +) { ASSERT(current); if (asset.type === 0) { - return respondWithBufferAndViews(c, current.static, asset, status); + return respondWithBufferAndViews( + c, + key, + current.static, + asset, + status, + cache, + ); } else { const entry = UNWRAP(current.dynamic.get(asset.id)); - return respondWithBufferAndViews(c, entry.buffer, entry, status); + return respondWithBufferAndViews( + c, + key, + entry.buffer, + entry, + status, + cache, + ); } } function respondWithBufferAndViews( c: Context, + key: string, buffer: Buffer, asset: AssetBase, status: StatusCode, + cache: CacheMode, ) { const ifNoneMatch = c.req.header("If-None-Match"); if (ifNoneMatch) { @@ -136,6 +171,29 @@ function respondWithBufferAndViews( } else { body = buffer.subarray(...asset.raw); } + debug( + `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`, + ); + if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) { + if (cache === "auto") { + if (status < 200 || status >= 300) { + cache = 'no-cache'; + } else if (headers['content-type']?.includes('text/html')) { + cache = 'temporary'; + } else { + cache = 'long-term'; + } + } + if (cache === "no-cache") { + headers["Cache-Control"] = "no-store"; + } else if (cache === "long-term") { + headers["Cache-Control"] = + "public, max-age=7200, stale-while-revalidate=7200, immutable"; + } else if (cache === "temporary") { + headers["Cache-Control"] = + "public, max-age=7200, stale-while-revalidate=7200"; + } + } return (c.res = new Response(body, { headers, status })); } @@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts"; import * as crypto from "node:crypto"; import * as zlib from "node:zlib"; import * as util from "node:util"; +import * as console from "@paperclover/console"; diff --git a/framework/lib/error.ts b/framework/lib/error.ts new file mode 100644 index 0000000000000000000000000000000000000000..816249161d61c9a1f9e32454ef38a718c2f966f0 --- /dev/null +++ b/framework/lib/error.ts @@ -0,0 +1,47 @@ +declare global { + interface Error { + /* Extra fields on errors are extremely helpful for debugging. */ + [metadata: string]: unknown; + } +} + +/** Retrieve an error message from any value */ +export function message(error: unknown): string { + const message = (error as { message: unknown })?.message ?? error; + try { + return typeof message === "string" ? message : JSON.stringify(message); + } catch {} + try { + return String(message); + } catch {} + return `Could not stringify error message ${typeof message}`; +} + +/** Retrieve an error-like object from any value. Useful to check fields. */ +export function obj(error: unknown): { message: string } & Partial { + if (error instanceof Error) return error; + return { + message: message(error), + ...(typeof error === "object" && error), + }; +} + +/* Node.js error codes are strings */ +export function code(error: unknown): NodeErrorCode | null { + const code = (error as { code: unknown })?.code; + return typeof code === "string" ? code : null; +} + +/** Attach extra fields and throw */ +export function rethrowWithMetadata( + err: unknown, + meta: Record, +): never { + const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/ + Object.assign(error, meta); + throw error; +} + +export type NodeErrorCode = + | keyof typeof import("node:os").constants.errno + | (string & {}); diff --git a/framework/lib/meta.ts b/framework/lib/meta.ts index 511b2a9f08d05cd8d0305431d35d88bf221196b7..0e2c272899676863e8fd6daf3fa8bdf8fb9e15c0 100644 --- a/framework/lib/meta.ts +++ b/framework/lib/meta.ts @@ -4,6 +4,9 @@ export interface Meta { openGraph?: OpenGraph; alternates?: Alternates; } +export interface Template extends Omit { + titleTemplate?: (title: string) => string, +} export interface OpenGraph { title?: string; description?: string | undefined; @@ -19,6 +22,6 @@ export interface AlternateType { title: string; } export function renderMeta({ title }: Meta): string { - return `${esc(title)}`; + return `${esc(title)}`; } -import { escapeHtml as esc } from "#engine/render"; +import { escapeHtmlContent as esc } from "#engine/render"; diff --git a/framework/lib/mime.txt b/framework/lib/mime.txt index 7cf59936e371a7a55e468ab37eb0725fbf92e11e..ad6220bb80616c843b28316e8f814a89ca8d8d89 100644 --- a/framework/lib/mime.txt +++ b/framework/lib/mime.txt @@ -1,99 +1,100 @@ -# media types -# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types -.aac audio/x-aac -.aif audio/x-aiff -.aifc audio/x-aiff -.aiff audio/x-aiff -.asm text/x-asm -.avi video/x-msvideo -.bat application/x-msdownload -.c text/x-c -.chat text/x-clover-chatlog -.class application/java-vm -.cmd application/x-msdownload -.com application/x-msdownload -.conf text/plain -.cpp text/x-c -.css text/css -.csv text/csv -.cxx text/x-c -.def text/plain -.diff text/plain -.dll application/x-msdownload -.dmg application/octet-stream -.doc application/msword -.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document -.epub application/epub+zip -.exe application/x-msdownload -.flv video/x-flv -.fbx application/fbx -.gz application/x-gzip -.h text/x-c -.h264 video/h264 -.hh text/x-c -.htm text/html;charset=utf-8 -.html text/html;charset=utf-8 -.ico image/x-icon -.ics text/calendar -.in text/plain -.jar application/java-archive -.java text/x-java-source -.jpeg image/jpeg -.jpg image/jpeg -.jpgv video/jpeg -.jxl image/jxl -.js application/javascript -.json application/json -.latex application/x-latex -.list text/plain -.log text/plain -.m4a audio/mp4 -.man text/troff -.mid audio/midi -.midi audio/midi -.mov video/quicktime -.mp3 audio/mpeg -.mp4 video/mp4 -.msh model/mesh -.msi application/x-msdownload -.obj application/octet-stream -.ogg audio/ogg -.otf application/x-font-otf -.pdf application/pdf -.png image/png -.ppt application/vnd.ms-powerpoint -.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation -.psd image/vnd.adobe.photoshop -.py text/x-python -.rar application/x-rar-compressed -.rss application/rss+xml -.rtf application/rtf -.rtx text/richtext -.s text/x-asm -.pem application/x-pem-file" -.ser application/java-serialized-object -.sh application/x-sh -.sig application/pgp-signature -.silo model/mesh -.svg image/svg+xml -.t text/troff -.tar application/x-tar -.text text/plain -.tgz application/x-gzip -.tif image/tiff -.tiff image/tiff -.torrent application/x-bittorrent -.ttc application/x-font-ttf -.ttf application/x-font-ttf -.txt text/plain -.urls text/uri-list -.v text/x-v -.wav audio/x-wav -.wmv video/x-ms-wmv -.xls application/vnd.ms-excel -.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -.xml application/xml -.xps application/vnd.ms-xpsdocument - -# special cased based on file name -rss.xml application/rss+xml +# media types +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types +.aac audio/x-aac +.aif audio/x-aiff +.aifc audio/x-aiff +.aiff audio/x-aiff +.asm text/x-asm +.avi video/x-msvideo +.bat application/x-msdownload +.c text/x-c +.chat text/x-clover-chatlog +.class application/java-vm +.cmd application/x-msdownload +.com application/x-msdownload +.conf text/plain +.cpp text/x-c +.css text/css +.csv text/csv +.cxx text/x-c +.def text/plain +.diff text/plain +.dll application/x-msdownload +.dmg application/octet-stream +.doc application/msword +.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document +.epub application/epub+zip +.exe application/x-msdownload +.flv video/x-flv +.fbx application/fbx +.gz application/x-gzip +.h text/x-c +.h264 video/h264 +.hh text/x-c +.htm text/html;charset=utf-8 +.html text/html;charset=utf-8 +.ico image/x-icon +.ics text/calendar +.in text/plain +.jar application/java-archive +.java text/x-java-source +.jpeg image/jpeg +.jpg image/jpeg +.jpgv video/jpeg +.jxl image/jxl +.js application/javascript +.json application/json +.latex application/x-latex +.list text/plain +.log text/plain +.m4a audio/mp4 +.man text/troff +.mid audio/midi +.midi audio/midi +.mov video/quicktime +.mp3 audio/mpeg +.mp4 video/mp4 +.msh model/mesh +.msi application/x-msdownload +.obj application/octet-stream +.ogg audio/ogg +.otf application/x-font-otf +.pdf application/pdf +.png image/png +.ppt application/vnd.ms-powerpoint +.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation +.psd image/vnd.adobe.photoshop +.py text/x-python +.rar application/x-rar-compressed +.rss application/rss+xml +.rtf application/rtf +.rtx text/richtext +.s text/x-asm +.pem application/x-pem-file" +.ser application/java-serialized-object +.sh application/x-sh +.sig application/pgp-signature +.silo model/mesh +.svg image/svg+xml +.t text/troff +.tar application/x-tar +.text text/plain +.tgz application/x-gzip +.tif image/tiff +.tiff image/tiff +.torrent application/x-bittorrent +.ttc application/x-font-ttf +.ttf application/x-font-ttf +.txt text/plain +.urls text/uri-list +.v text/x-v +.wav audio/x-wav +.wmv video/x-ms-wmv +.woff2 font/woff2 +.xls application/vnd.ms-excel +.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +.xml application/xml +.xps application/vnd.ms-xpsdocument + +# special cased based on file name +rss.xml application/rss+xml diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index af99dca9498891b5936809cf4c1581897fb9cddd..e51e5fc19f0d39d1bbf5e82383ef66df640e11c0 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -31,6 +31,24 @@ export interface FileItem { export interface Section { root: string; } +export interface Font { + name: string; + /** + * Specify either font name, file path, or URL to fetch it. + * Private fonts do not have a URL and will fail to build if missing. + */ + sources: string[]; + subsets: Array<{ + vars?: Record; + layoutFeatures?: string[]; + unicodes: FontRange | FontRange[]; + /** Include [ext] to autofill 'woff2' */ + asset: string; + }>; +} +export type FontVars = Record; +export type FontVariableAxis = number | [min: number, max: number]; +export type FontRange = string | number | { start: number, end: number }; export const userData = render.userData(() => { throw new Error("This function can only be used in a page (static or view)"); diff --git a/framework/lib/sqlite.ts b/framework/lib/sqlite.ts index 477ab9e0ba6a5efe39f2a53c73852e9b2f44f5d3..9b49b2fe3d739c1c4c19819a260fb94452281c4f 100644 --- a/framework/lib/sqlite.ts +++ b/framework/lib/sqlite.ts @@ -11,9 +11,9 @@ export function getDb(file: string) { let db = map.get(file); if (db) return db; const fileWithExt = file.includes(".") ? file : file + ".sqlite"; - db = new WrappedDatabase( - path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt), - ); + const dir = process.env.CLOVER_DB ?? ".clover"; + fs.mkdirSync(dir); + db = new WrappedDatabase(path.join(dir, fileWithExt)); map.set(file, db); return db; } @@ -21,7 +21,7 @@ export function getDb(file: string) { export class WrappedDatabase { file: string; node: DatabaseSync; - stmts: Stmt[]; + stmts: Stmt[] = []; stmtTableMigrate: WeakRef | null = null; constructor(file: string) { @@ -153,3 +153,4 @@ export class Stmt { import { DatabaseSync, StatementSync } from "node:sqlite"; import * as path from "node:path"; +import * as fs from "#sitegen/fs"; diff --git a/framework/lib/subprocess.ts b/framework/lib/subprocess.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d183de6923d8b0fe2dced3266ff2352c30c28fa --- /dev/null +++ b/framework/lib/subprocess.ts @@ -0,0 +1,15 @@ +const execFileRaw = util.promisify(child_process.execFile); +export const exec: typeof execFileRaw = (( + ...args: Parameters +) => + execFileRaw(...args).catch((e: any) => { + if (e?.message?.startsWith?.("Command failed")) { + if (e.code > 2 ** 31) e.code |= 0; + const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; + e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; + } + throw e; + })) as any; + +import * as util from 'node:util'; +import * as child_process from 'node:child_process'; diff --git a/framework/watch.ts b/framework/watch.ts index 9fe21571a5a84e17d88550047f0a0ccbe1872b24..453dbceb8fc6f73277b95bfa4e6b30913e22f76c 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -25,6 +25,7 @@ function serve() { "--development", ], { stdio: "inherit", + execArgv: ['--no-warnings'], }); subprocess.on("close", onSubprocessClose); } diff --git a/readme.md b/readme.md index cecf111e0470714a96e42ea06ddbbc4236f06cab..830119073ada4e43856d6721ab0167aa31e05ced 100644 --- a/readme.md +++ b/readme.md @@ -11,23 +11,21 @@ that assist building websites. these tools power . - Different languages can be used at the same time. Supports `async function` components, ``, and custom extensions. - **Incremental static site generator and build system.** - - Build entire production site at start, incremental updates when pages - change; Build system state survives coding sessions. - - The only difference in development and production mode is hidden source-maps - and stripped `console.debug` calls. The site you see locally is the same - site you see deployed. + - Build both development and production sites on startup start. Incremental + generator rebuilds changed pages; Build system state survives coding sessions. + - Multiple backend support. - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs checks when the files change. For example, changing a component re-tests only pages that use that component and re-lints only the changed file. - **Integrated libraries for building complex, content heavy web sites.** - Static asset serving with ETag and build-time compression. - - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`) + - Dynamicly rendered pages with static client. (`import "#sitegen/view"`) - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`) - TODO: Meta and Open Graph generation. (`export const meta`) - - TODO: Font subsetting tools to reduce bytes downloaded by fonts. -- **Built on the battle-tested Node.js runtime.** + - Font subsetting to reduce page bandwidth. +- **Built on the stable and battle-tested Node.js runtime.** -None of these tools are complex or revolutionary. Rather, this project is the +none of these tools are complex or revolutionary. rather, this project is the sum of many years of experience on managing content heavy websites, and an example on how other over-complicate other frameworks. @@ -35,9 +33,11 @@ example on how other over-complicate other frameworks. Included is `src`, which contains `paperclover.net`. Website highlights: +- TODO: flashy homepage. - [Question/Answer board, custom markdown parser and components][q+a]. - [File viewer with fast ui/ux + optimized media streaming][file]. - [Personal, friends-only blog with password protection][friends]. +- TODO: digital garden styled blog. [q+a]: https://paperclover.net/q+a [file]: https://paperclover.net/file @@ -51,15 +51,21 @@ minimum system requirements: - random access memory. - windows 7 or later, macos, or other operating system. +required software: + +- node.js v24 +- python v3 + my development machine, for example, is Dell Inspiron 7348 with Core i7 npm install - # production generation + # build site using 'run.js' to enable runtime plugins node run generate - node .clover/o/backend + # the built site runs in regular node.js + node --enable-source-maps .clover/o/backend - # "development" watch mode + # watch-rebuild mode node run watch for unix systems, the provided `flake.nix` can be used with `nix develop` to @@ -67,7 +73,7 @@ open a shell with all needed system dependencies. ## Deployment -There are two primary server components to be deployed: the web server and the +there are two primary server components to be deployed: the web server and the sourth of truth server. The latter is a singleton that runs on Clover's NAS, which holds the full contents of the file storage. The web server pulls data from the source of truth and renders web pages, and can be duplicated to @@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose: backend: container_name: backend build: - # this uses loopback to hit the self-hosted git server + # this uses loopback to hit the self-hosted git server, + # docker will cache the image to not re-fetch on reboot. context: http://127.0.0.1:3000/clo/sitegen.git dockerfile: src/source-of-truth.dockerfile environment: @@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose: - /mnt/storage1/clover/Documents/Config/paperclover:/data - /mnt/storage1/clover/Published:/published -Due to caching, one may need to manually purge images via -`docker image rm ix-clover-backend -f` when an update is desired +Due to caching, images may need to be purged via `docker image rm {image} -f` +when an update is desired. Some docker GUIs support force pulls, some are buggy. -TODO: deployment instructions for a web node +The web server performs rendering. A `Dockerfile` for it is present in +`src/web.dockerfile` but it is currently unused. Deployments are done by +building the project locally, and then using `rsync` to copy files. + + node run generate + rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \ + --delete-after --progress --human-readable + ssh "$REMOTE_USER" /bin/bash << EOF + set -e + cd ~/paperclover + npm ci + pm2 restart site + echo "-> https://paperclover.net" + EOF ## Contributions diff --git a/run.js b/run.js index ea0c1ddd54dc96ac852f85ae276f85c73f5d1e58..8e46ce1a0fefe0ace88def876df7c7569549f877 100644 --- a/run.js +++ b/run.js @@ -3,7 +3,6 @@ import * as util from "node:util"; import * as zlib from "node:zlib"; import * as url from "node:url"; -import * as module from "node:module"; import process from "node:process"; // Disable experimental warnings (Type Stripping, etc) @@ -28,8 +27,8 @@ try { const brand = process.versions.bun ? `bun ${process.versions.bun}` : process.versions.deno - ? `deno ${process.versions.deno}` - : null; + ? `deno ${process.versions.deno}` + : null; const brandText = brand ? ` (${brand})` : ""; globalThis.console.error( `sitegen depends on a node.js v24. your runtime is missing features.\n` + @@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) { else if (mod.default?.fetch) { const protocol = "http"; const { serve } = hot.load("@hono/node-server"); - serve({ - fetch: mod.default.fetch, - }, ({ address, port }) => { - if (address === "::") address = "::1"; - console.info(url.format({ - protocol, - hostname: address, - port, - })); - }); + serve( + { + fetch: mod.default.fetch, + }, + ({ address, port }) => { + if (address === "::") address = "::1"; + console.info( + url.format({ + protocol, + hostname: address, + port, + }), + ); + }, + ); + } else { + console.warn( + hot.load("path").relative('.', found), + 'does not export a "main" function', + ); } } catch (e) { console.error(util.inspect(e)); diff --git a/src/admin.ts b/src/admin.ts index dd90fdaf34e8cd10b8b2340ef0175d6127542350..f9ee787d56ab687bc47130c5489cfd0e5219ec0f 100644 --- a/src/admin.ts +++ b/src/admin.ts @@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year let lastKnownToken: string | null = null; function compareToken(token: string) { if (token === lastKnownToken) return true; - lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + try { + lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + } catch { + return false; + } return token === lastKnownToken; } diff --git a/src/backend.ts b/src/backend.ts index 48b5a11eb0d51d1d93b71bab10e8760fa04ce47c..74236a90bd6db9940aa6e4e93ed272e509ac84ca 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app); // Asset middleware has least precedence app.use(assets.middleware); +// Asset Aliases +app.get( + "/apple-touch-icon-precomposed.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); +app.get( + "/apple-touch-icon-57x57.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); +app.get( + "/apple-touch-icon-57x57-precomposed.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); + // Handlers app.notFound(assets.notFound); diff --git a/src/file-viewer/bin/scan3.ts b/src/file-viewer/bin/scan3.ts index 261100e569e2941289271e458caea386fe826b4d..be408d00c9d276dff544294074c434dbe96a0429 100644 --- a/src/file-viewer/bin/scan3.ts +++ b/src/file-viewer/bin/scan3.ts @@ -10,7 +10,6 @@ // This is the third iteration of the scanner, hence its name "scan3"; // Remember that any software you want to be maintainable and high // quality cannot be written with AI. -const root = path.resolve("/Volumes/clover/Published"); const workDir = path.resolve(".clover/derived"); const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY); @@ -404,18 +403,6 @@ interface Process { undo?(mediaFile: MediaFile): Promise; } -const execFileRaw = util.promisify(child_process.execFile); -const execFile: typeof execFileRaw = (( - ...args: Parameters -) => - execFileRaw(...args).catch((e: any) => { - if (e?.message?.startsWith?.("Command failed")) { - if (e.code > 2 ** 31) e.code |= 0; - const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; - e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; - } - throw e; - })) as any; const ffprobeBin = testProgram("ffprobe", "--help"); const ffmpegBin = testProgram("ffmpeg", "--help"); @@ -426,7 +413,7 @@ const procDuration: Process = { enable: ffprobeBin !== null, include: rules.extsDuration, async run({ absPath, mediaFile }) { - const { stdout } = await execFile(ffprobeBin!, [ + const { stdout } = await subprocess.exec(ffprobeBin!, [ "-v", "error", "-show_entries", @@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress"; import { Spinner } from "@paperclover/console/Spinner"; import * as async from "#sitegen/async"; import * as fs from "#sitegen/fs"; +import * as subprocess from "#sitegen/subprocess"; import * as path from "node:path"; import * as zlib from "node:zlib"; import * as child_process from "node:child_process"; -import * as util from "node:util"; import * as crypto from "node:crypto"; import * as stream from "node:stream"; @@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts"; import * as ffmpeg from "@/file-viewer/ffmpeg.ts"; import * as rsync from "@/file-viewer/rsync.ts"; import * as transcodeRules from "@/file-viewer/transcode-rules.ts"; +import { rawFileRoot as root } from "../paths.ts"; diff --git a/src/file-viewer/cache.ts b/src/file-viewer/cache.ts index fd85eeb2c07d3f38f9cdb6f7a506fb92e8a7bcb4..74121d1472b7d065cf6a4ee5d9a44ec40096e72e 100644 --- a/src/file-viewer/cache.ts +++ b/src/file-viewer/cache.ts @@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts"; declare const Deno: any; const sourceOfTruth = "https://nas.paperclover.net:43250"; -const caCert = fs.readFileSync("src/file-viewer/cert.pem"); +// const caCert = fs.readFileSync("src/file-viewer/cert.pem"); -const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/"); +const diskCacheRoot = path.join(import.meta.dirname, ".filecache/"); const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB const loadInProgress = new Map< @@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined" // Bun has two non-standard fetch extensions decompress: false, tls: { - ca: caCert, + // ca: caCert, }, } // TODO: https://github.com/denoland/deno/issues/12291 @@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined" // } // Node.js supports node:http : new Agent({ - ca: caCert, + // ca: caCert, }); function fetchFileNode(pathname: string): Promise { diff --git a/src/file-viewer/paths.ts b/src/file-viewer/paths.ts new file mode 100644 index 0000000000000000000000000000000000000000..8d974d985ec497cfa7e2e2655d9f278ad240959f --- /dev/null +++ b/src/file-viewer/paths.ts @@ -0,0 +1,12 @@ +export const nasRoot = process.platform === "win32" + ? "\\\\zenith\\clover" + : process.platform === "darwin" + ? "/Volumes/clover" + : "/media/clover"; + +export const rawFileRoot = process.env.CLOVER_FILE_RAW ?? + path.join(nasRoot, "Published"); +export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ?? + path.join(nasRoot, "Documents/Config/paperclover/derived"); + +import * as path from "node:path"; diff --git a/src/site.ts b/src/site.ts index 3c47ebcb44ea4c7c14ca7a392e347f7c8a46da88..214e8f6a93368669df1dfd12d377f1ccb57d5b3c 100644 --- a/src/site.ts +++ b/src/site.ts @@ -1,11 +1,11 @@ -// This file defines the different "Sections" of the website. The sections act -// as separate codebases, but hosted as one. This allows me to have -// sub-projects like the file viewer in 'file', or the question answer system -// in 'q+a'. Each section can define configuration, pages, backend routes, and -// contain other files. const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); -export const siteSections: Section[] = [ +// Different sections of the website are split into their own folders. Acting as +// as separate codebases, they are hosted as one. This allows me to have +// sub-projects like the file viewer in 'file/', and the question answer system +// in 'q+a', but maintain clear boundaries. Each section can define +// configuration, pages, backend routes, and contain other files. +export const siteSections: sg.Section[] = [ { root: join(".") }, { root: join("q+a/") }, { root: join("file-viewer/") }, @@ -14,10 +14,96 @@ export const siteSections: Section[] = [ // { root: join("fiction/"), pageBase: "/fiction" }, ]; +// All backends are bundled. The backend named "backend" is run by "node run watch" export const backends: string[] = [ join("backend.ts"), join("source-of-truth.ts"), ]; +// Font subsets reduce bandwidth and protect against proprietary font theft. +const fontRoot = path.join(nasRoot, 'Documents/Font'); +const ascii = { start: 0x20, end: 0x7E }; +const nonAscii: sg.FontRange[] = [ + { start: 0xC0, end: 0xFF }, + { start: 0x2190, end: 0x2193 }, + { start: 0xA0, end: 0xA8 }, + { start: 0xAA, end: 0xBF }, + { start: 0x2194, end: 0x2199 }, + { start: 0x100, end: 0x17F }, + 0xA9, + 0x2018, + 0x2019, + 0x201C, + 0x201D, + 0x2022, +]; +const recursiveVars: sg.FontVars = { + WGHT: [400, 750], + SLNT: [-15, 0], + CASL: 0.25, + CRSV: 0.5, + MONO: 0, +}; +const recursiveVarsMono = { ...recursiveVars, MONO: 1 }; +const recursiveVarsQuestion: sg.FontVars = { + ...recursiveVars, + MONO: [0, 1], + WGHT: [400, 1000], +}; +const layoutFeatures = ["numr", "dnom", "frac"]; +export const fonts: sg.Font[] = [ + { + name: "Recursive", + sources: [ + "Recursive_VF_1.085.ttf", + path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"), + "https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf", + ], + subsets: [ + { + asset: "/recultramin.[ext]", + layoutFeatures, + vars: recursiveVars, + unicodes: ascii, + }, + { + asset: "/recmono.[ext]", + vars: recursiveVarsMono, + unicodes: ascii, + }, + { + asset: "/recqa.[ext]", + vars: recursiveVarsQuestion, + unicodes: ascii, + }, + { + asset: "/recexotic.[ext]", + vars: recursiveVarsQuestion, + unicodes: nonAscii, + }, + ], + }, + { + name: "AT Name Sans Display Hairline", + sources: [ + "ATNameSansDisplay-Hairline.woff2", + path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"), + ], + subsets: [ + { + asset: "/cydn_header.[ext]", + layoutFeatures, + unicodes: "cotlyedon", + }, + ], + }, +]; + +export async function main() { + await font.buildFonts(fonts); +} + import * as path from "node:path"; -import type { Section } from "#sitegen"; +import * as font from "../framework/font.ts"; +import type * as sg from "#sitegen"; +import { nasRoot } from "./file-viewer/paths.ts"; diff --git a/src/source-of-truth.ts b/src/source-of-truth.ts index af2826f441a1a4a3becd720022decd49226675e0..1189ec06de52a128cb21bff4d9b9b8fa07bbc469 100644 --- a/src/source-of-truth.ts +++ b/src/source-of-truth.ts @@ -22,17 +22,6 @@ export default app; const token = UNWRAP(process.env.CLOVER_SOT_KEY); -const nasRoot = process.platform === "win32" - ? "\\\\zenith\\clover" - : process.platform === "darwin" - ? "/Volumes/clover" - : "/media/clover"; - -const rawFileRoot = process.env.CLOVER_FILE_RAW ?? - path.join(nasRoot, "Published"); -const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ?? - path.join(nasRoot, "Documents/Config/paperclover/derived"); - if (!fs.existsSync(rawFileRoot)) { throw new Error(`${rawFileRoot} does not exist`); } @@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs"; import * as util from "node:util"; import * as stream from "node:stream"; import * as mime from "#sitegen/mime"; +import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts"; diff --git a/src/static/apple-touch-icon.png b/src/static/apple-touch-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..5f24ff6a41358bfc31ee9909f3205d7413223523 Binary files /dev/null and b/src/static/apple-touch-icon.png differ diff --git a/src/static/favicon.ico b/src/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..738099c56829dee9a6ad2ef5f0a6e7ee92110d49 Binary files /dev/null and b/src/static/favicon.ico differ diff --git a/src/web.dockerfile b/src/web.dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..03c3a6cf0b6aa453142c74efc49d47f8412949f1 --- /dev/null +++ b/src/web.dockerfile @@ -0,0 +1,19 @@ +from node:24 as builder + +run apt install -y git + +workdir /tmp/builder +copy package*.json ./ +run npm ci +copy . ./ +run node run generate +run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/ + +from node:24 as runtime + +workdir /app +copy --from=builder /app/ ./ + +env PORT=80 + +cmd ["node", "--enable-source-maps", "/app/backend.js"]