From b01c9ea08ff53136b25ad5c35f26fcfa1917aaf0 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Fri, 27 Mar 2026 02:54:41 -0700 Subject: [PATCH] feat: wasm-based font subsetting drops the python system that was a holdover from my old manual font subsetting workflows. now it is all js+wasm. over time i will replace every other native dependency with a WebAssembly alternative. resolves #101 --- framework/font.ts | 351 +++++++++++++++++++++++++++++---------- framework/hot.ts | 1 + framework/incremental.ts | 2 +- framework/lib/fs.ts | 4 +- package-lock.json | 29 +++- package.json | 4 +- readme.md | 1 - 7 files changed, 298 insertions(+), 94 deletions(-) diff --git a/framework/font.ts b/framework/font.ts index f3141aa993c151449b1d12389df0db85c4f1f003..3d25c2bfa8a8b1cb79f9bc0efcc836a6e085d594 100644 --- a/framework/font.ts +++ b/framework/font.ts @@ -1,28 +1,6 @@ export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { if (fonts.length === 0) return; using node = p.start("build fonts"); - - const temp = path.resolve(".clover/font"); - const venv = path.join(temp, "venv"); - const bin = path.join(venv, process.platform === "win32" ? "Scripts" : "bin"); - - if (!fs.existsSync(venv)) { - using _ = node.start("setup python env"); - await subprocess.exec("python", ["-m", "venv", venv]); - } - if (!fs.existsSync(path.join(bin, "pyftsubset"))) { - using _ = node.start("install pyftsubset"); - await subprocess.exec(path.join(bin, "pip"), [ - "install", - "fonttools==4.38.0", - "brotli==1.0.7", - ]); - } - - const instances = new async.OnceMap( - (_: string, x: () => Promise) => x(), - (x) => x[0], - ); function getArgs(vars: sg.FontVars) { return Object.entries(vars).sort((x) => x[0] === "wght" || x[0] === "slnt" ? 1 : 0 @@ -43,58 +21,35 @@ export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { .toLowerCase(); } - async function makeInstance(input: string, vars: sg.FontVars) { - const args = getArgs(vars); - const hash = getHash(input, args); - const outfile = path.join(temp, `${hash}${path.extname(input)}`); - await instances.getOrRun(outfile, async () => { - await fs.rm(outfile, { force: true }); - await subprocess.exec(path.join(bin, "fonttools"), [ - "varLib.instancer", - input, - ...args, - `--output=${outfile}`, - ]); - ASSERT(fs.existsSync(outfile)); - }); - return outfile; + function getVariationAxes(vars: sg.FontVars) { + return Object.fromEntries(Object.entries(vars).map(([key, value]) => [ + normalizeAxisTag(key), + Array.isArray(value) ? { min: value[0], max: value[1] } : value, + ])) as VariationAxes; } await Promise.all(fonts.map((font) => incr.work(node, async (io, font) => { using node = io.start(font.name); const baseFile = await fetchFont(font.name, font.sources); + const baseBuffer = await fs.readFile(baseFile); await Promise.all(font.subsets.map(async (subset) => { const args = getArgs(subset.vars ?? {}); - const unicodes = fontRangesToString( + const subsetText = fontRangesToText( Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], ); - const hash = crypto - .createHash("sha256") - .update(`${baseFile}${args.join()}${unicodes}`) - .digest("hex") - .slice(0, 16) - .toLowerCase(); - const woff = path.join(temp, hash + ".woff2"); - if (!fs.existsSync(woff)) { - using _ = node.start(args.join(" ")); - let file = baseFile; - if (subset.vars) { - file = await makeInstance(baseFile, subset.vars); - } - await subprocess.exec(path.join(bin, "pyftsubset"), [ - file, - "--flavor=woff2", - `--output-file=${woff}`, - ...subset.layoutFeatures - ? [`--layout-features=${subset.layoutFeatures.join(",")}`] - : [], - `--unicodes=${unicodes}`, - ]); - } + using _ = node.start(args.join(" ")); + const woff = await subsetFontBuffer(baseBuffer, subsetText, { + targetFormat: "woff2", + variationAxes: subset.vars + ? getVariationAxes(subset.vars) + : undefined, + sourceName: baseFile, + assetName: subset.asset.replace("[ext]", "woff2"), + }); await io.writeAsset({ pathname: subset.asset.replace("[ext]", "woff2"), - buffer: await fs.readFile(woff), + buffer: woff, }); })); }, font) @@ -114,7 +69,7 @@ export async function fetchFont(name: string, sources: string[]) { if (response.ok) { await fs.writeMkdir( cacheName, - Buffer.from(await response.arrayBuffer()), + new Uint8Array(await response.arrayBuffer()), ); } else { errs.push( @@ -170,51 +125,271 @@ export async function fetchFont(name: string, sources: string[]) { throw new AggregateError(errs, `Failed to fetch the ${name} font`); } -function fontRangesToString(ranges: sg.FontRange[]): string { - const segments: Array<{ start: number; end: number }> = []; - +function fontRangesToText(ranges: sg.FontRange[]): string { + const chars = new Set(); ranges.forEach((range) => { if (typeof range === "string") { - for (let i = 0; i < range.length; i++) { - const cp = range.codePointAt(i) || 0; - segments.push({ start: cp, end: cp }); + for (const char of range) { + chars.add(char.codePointAt(0) || 0); } } else if (typeof range === "number") { - segments.push({ start: range, end: range }); + chars.add(range); } else { - segments.push({ start: range.start, end: range.end }); + for (let cp = range.start; cp <= range.end; cp++) chars.add(cp); } }); + return Array.from(chars) + .sort((a, b) => a - b) + .map((cp) => String.fromCodePoint(cp)) + .join(""); +} - segments.sort((a, b) => a.start - b.start); +function normalizeAxisTag(tag: string) { + const lower = tag.toLowerCase(); + return lower === "wght" || lower === "slnt" ? lower : tag; +} - const merged: Array<{ start: number; end: number }> = []; - for (const seg of segments) { - const last = merged[merged.length - 1]; - if (last && seg.start <= last.end + 1) { - last.end = Math.max(last.end, seg.end); - } else { - merged.push({ ...seg }); - } +function detectFontFormat(buffer: Uint8Array): "sfnt" | "woff2" { + const signature = String.fromCharCode(...buffer.subarray(0, 4)); + if (signature === "wOF2") return "woff2"; + if ( + signature === "true" || + signature === "OTTO" || + signature === "\x00\x01\x00\x00" + ) { + return "sfnt"; } + throw new Error(`Unsupported font format signature ${JSON.stringify(signature)}`); +} - return merged.map(({ start, end }) => - start === end - ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}` - : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${ - end.toString(16).toUpperCase().padStart(4, "0") +async function convertToSfnt(buffer: Uint8Array) { + const format = detectFontFormat(buffer); + if (format === "sfnt") return buffer; + return new Uint8Array(await wawoff2.decompress(buffer)); +} + +async function convertFromSfnt( + buffer: Uint8Array, + targetFormat: "sfnt" | "woff2", +) { + if (targetFormat === "sfnt") return buffer; + return new Uint8Array(await wawoff2.compress(buffer)); +} + +function getSubsetContext(opt: { + sourceName?: string; + assetName?: string; + variationAxes?: VariationAxes; + targetFormat: "sfnt" | "woff2"; + text: string; +}) { + const fields = [ + opt.sourceName ? `source=${opt.sourceName}` : "", + opt.assetName ? `asset=${opt.assetName}` : "", + `target=${opt.targetFormat}`, + `textLength=${opt.text.length}`, + opt.variationAxes + ? `axes=${ + Object.entries(opt.variationAxes).map(([key, value]) => + typeof value === "number" + ? `${key}=${value}` + : `${key}=${value.min}:${value.max}${ + value.default == null ? "" : `:${value.default}` + }` + ).join(",") }` - ).join(","); + : "", + ].filter(Boolean); + return fields.join(" "); +} + +async function subsetFontBuffer( + originalFont: Uint8Array, + text: string, + opt: { + targetFormat: "sfnt" | "woff2"; + variationAxes?: VariationAxes; + sourceName?: string; + assetName?: string; + }, +) { + return await subsetFontQueue(async () => { + const ctx = getSubsetContext({ ...opt, text }); + if (typeof text !== "string") { + throw new Error(`Subset text must be a string. ${ctx}`); + } + + const { harfbuzzJsWasm, getHeapU8 } = await loadAndInitializeHarfbuzz(); + const sfntFont = await convertToSfnt(originalFont); + + const input = harfbuzzJsWasm.hb_subset_input_create_or_fail(); + if (input === 0) { + throw new Error(`Failed to create HarfBuzz subset input. ${ctx}`); + } + + const fontBuffer = harfbuzzJsWasm.malloc(sfntFont.byteLength); + getHeapU8().set(new Uint8Array(sfntFont), fontBuffer); + + const blob = harfbuzzJsWasm.hb_blob_create( + fontBuffer, + sfntFont.byteLength, + 2, + 0, + 0, + ); + const face = harfbuzzJsWasm.hb_face_create(blob, 0); + harfbuzzJsWasm.hb_blob_destroy(blob); + + try { + const layoutFeatures = harfbuzzJsWasm.hb_subset_input_set(input, 6); + harfbuzzJsWasm.hb_set_clear(layoutFeatures); + harfbuzzJsWasm.hb_set_invert(layoutFeatures); + + const inputUnicodes = harfbuzzJsWasm.hb_subset_input_unicode_set(input); + for (const char of text) { + harfbuzzJsWasm.hb_set_add(inputUnicodes, char.codePointAt(0) || 0); + } + + if (opt.variationAxes) { + for (const [axisName, value] of Object.entries(opt.variationAxes)) { + if (typeof value === "number") { + if ( + !harfbuzzJsWasm.hb_subset_input_pin_axis_location( + input, + face, + hbTag(axisName), + value, + ) + ) { + throw new Error( + `Failed to pin variation axis ${axisName}=${value}. ${ctx}`, + ); + } + } else if ( + !harfbuzzJsWasm.hb_subset_input_set_axis_range( + input, + face, + hbTag(axisName), + value.min, + value.max, + value.default ?? Number.NaN, + ) + ) { + throw new Error( + `Failed to set variation axis range ${axisName}=[${value.min}, ${value.max}]${ + value.default == null ? "" : ` default=${value.default}` + }. ${ctx}`, + ); + } + } + } + + const subset = harfbuzzJsWasm.hb_subset_or_fail(face, input); + if (subset === 0) { + throw new Error(`HarfBuzz failed to subset font. ${ctx}`); + } + + try { + const result = harfbuzzJsWasm.hb_face_reference_blob(subset); + try { + const offset = harfbuzzJsWasm.hb_blob_get_data(result, 0); + const subsetByteLength = harfbuzzJsWasm.hb_blob_get_length(result); + if (subsetByteLength === 0) { + throw new Error(`Subset font was empty. ${ctx}`); + } + const subsetFont = Uint8Array.from( + getHeapU8().subarray(offset, offset + subsetByteLength), + ); + return await convertFromSfnt(subsetFont, opt.targetFormat); + } finally { + harfbuzzJsWasm.hb_blob_destroy(result); + } + } finally { + harfbuzzJsWasm.hb_face_destroy(subset); + } + } finally { + harfbuzzJsWasm.hb_subset_input_destroy(input); + harfbuzzJsWasm.hb_face_destroy(face); + harfbuzzJsWasm.free(fontBuffer); + } + }); +} + +type VariationAxisRange = { min: number; max: number; default?: number }; +type VariationAxes = Record; + +interface HarfbuzzSubset { + memory: WebAssembly.Memory; + malloc(size: number): number; + free(ptr: number): void; + hb_blob_create( + data: number, + length: number, + mode: number, + userData: number, + destroy: number, + ): number; + hb_blob_destroy(blob: number): void; + hb_blob_get_data(blob: number, length: number): number; + hb_blob_get_length(blob: number): number; + hb_face_create(blob: number, index: number): number; + hb_face_destroy(face: number): void; + hb_face_reference_blob(face: number): number; + hb_set_add(set: number, value: number): void; + hb_set_clear(set: number): void; + hb_set_invert(set: number): void; + hb_subset_input_create_or_fail(): number; + hb_subset_input_destroy(input: number): void; + hb_subset_input_get_flags(input: number): number; + hb_subset_input_set(input: number, setType: number): number; + hb_subset_input_set_flags(input: number, flags: number): void; + hb_subset_input_unicode_set(input: number): number; + hb_subset_input_pin_axis_location( + input: number, + face: number, + axisTag: number, + value: number, + ): number; + hb_subset_input_set_axis_range( + input: number, + face: number, + axisTag: number, + min: number, + max: number, + defaultValue: number, + ): number; + hb_subset_or_fail(face: number, input: number): number; +} + +const loadAndInitializeHarfbuzz = async.once(async () => { + const wasm = await readFile(new URL(import.meta.resolve("harfbuzzjs/hb-subset.wasm"))); + const { instance } = await WebAssembly.instantiate(wasm); + const harfbuzzJsWasm = instance.exports as unknown as HarfbuzzSubset; + return { + harfbuzzJsWasm, + getHeapU8: () => new Uint8Array(harfbuzzJsWasm.memory.buffer), + }; +}); + +let subsetFontLock = Promise.resolve(); +function subsetFontQueue(fn: () => Promise) { + const run = subsetFontLock.then(fn, fn); + subsetFontLock = run.then(() => undefined, () => undefined); + return run; +} + +function hbTag(str: string) { + return str.split("").reduce((a, ch) => (a << 8) + ch.charCodeAt(0), 0); } import * as fs from "#sitegen/fs"; import * as os from "node:os"; import * as path from "node:path"; import * as sg from "#sitegen"; -import * as subprocess from "@clo/lib/subprocess"; import * as incr from "./incremental.ts"; import * as crypto from "node:crypto"; import * as async from "@clo/lib/async"; import * as error from "@clo/lib/error"; import * as progress from "@clo/lib/progress"; -import { ASSERT } from "@clo/lib/assert"; +import { readFile } from "node:fs/promises"; +import * as wawoff2 from "wawoff2"; diff --git a/framework/hot.ts b/framework/hot.ts index a48951a21e8bb1c615bf3b9586224f2e950553c1..e9289532531933515ca2e38c4a4c47a8e3dd7ce9 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -123,6 +123,7 @@ export function loadEsbuildCode( import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())}; import.meta.dirname = ${JSON.stringify(path.dirname(filepath))}; import.meta.filename = ${JSON.stringify(filepath)}; + import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r)); ` .trim() .replace(/[\n\s]/g, "") + src; diff --git a/framework/incremental.ts b/framework/incremental.ts index 1c58fd49a44d093db2129c52663fa6a09ad9ead3..70fca3423a01fb11c130071feee649c790a96085 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -578,7 +578,7 @@ export class Io { } async writeAsset(asset: { pathname: string; - buffer: string | Buffer; + buffer: string | Uint8Array; regenerative?: boolean; headers?: HeadersInit; }) { diff --git a/framework/lib/fs.ts b/framework/lib/fs.ts index ade3909256493490c8f0d6c3f4bedf0fb4a7bcc6..c8ffd30f15d3e5005a151b1fa4a61d53041520e7 100644 --- a/framework/lib/fs.ts +++ b/framework/lib/fs.ts @@ -31,14 +31,14 @@ export type WriteFileAsyncOptions = Parameters[2]; export async function writeMkdir( file: string, - contents: Buffer | string, + contents: Uint8Array | string, options?: WriteFileAsyncOptions, ) { await mkdir(path.dirname(file)); return writeFile(file, contents, options); } -export function writeMkdirSync(file: string, contents: Buffer | string) { +export function writeMkdirSync(file: string, contents: Uint8Array | string) { mkdirSync(path.dirname(file)); return writeFileSync(file, contents); } diff --git a/package-lock.json b/package-lock.json index 2727c61bb9ac9a81ea533a2b97ce2d52d8b5fc36..f6dcf2dd0d148c14dfdc6f75dc10d2685af3f9a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "dashjs": "5.0.3", "devalue": "5.6.4", "esbuild": "0.25.5", + "harfbuzzjs": "0.10.1", "hls.js": "1.6.5", "hono": "4.12.8", "marko": "5.38.33", @@ -23,7 +24,8 @@ "unique-names-generator": "4.7.1", "vitest": "^4.0.18", "vscode-oniguruma": "2.0.1", - "vscode-textmate": "9.2.0" + "vscode-textmate": "9.2.0", + "wawoff2": "2.0.1" } }, "lib": { @@ -1085,6 +1087,12 @@ "integrity": "sha512-+F3InkcH2XOGK7Jf/ZQis4cwZ4wbfmpBKo5J+SAA9bglT1gVd0e9hroHWarU076pJrAfs8JKuRPwDqwPBOKHnw==", "license": "MIT" }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -1384,6 +1392,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/harfbuzzjs": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/harfbuzzjs/-/harfbuzzjs-0.10.1.tgz", + "integrity": "sha512-Snspnku2xVnKvoXkdLeOgDWFmsjv8bhCkiA5o+c12I5NGO+aXsLNYmIjaEfIU//SSzoZ63xaOAb3A7eT/Ju+Sw==", + "license": "MIT" + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -2868,6 +2882,19 @@ "integrity": "sha512-krhkqzJdUxAZv2Cx0Gz6dN1r7TTrG9RDewkDHBbJQIqbNTCdB5ZUHVh7VkA4DgrKW4ZXPPUQKCwmI/3btDse9A==", "license": "MIT" }, + "node_modules/wawoff2": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/wawoff2/-/wawoff2-2.0.1.tgz", + "integrity": "sha512-r0CEmvpH63r4T15ebFqeOjGqU4+EgTx4I510NtK35EMciSdcTxCw3Byy3JnBonz7iyIFZ0AbVo0bbFpEVuhCYA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "woff2_compress.js": "bin/woff2_compress.js", + "woff2_decompress.js": "bin/woff2_decompress.js" + } + }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", diff --git a/package.json b/package.json index 4cfacaf0cba601acbccdf6169ce2ed2930fab93f..e2ecb8def63d579643a8bd6ab2ad272f7ba6fef3 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dashjs": "5.0.3", "devalue": "5.6.4", "esbuild": "0.25.5", + "harfbuzzjs": "0.10.1", "hls.js": "1.6.5", "hono": "4.12.8", "marko": "5.38.33", @@ -19,7 +20,8 @@ "unique-names-generator": "4.7.1", "vitest": "^4.0.18", "vscode-oniguruma": "2.0.1", - "vscode-textmate": "9.2.0" + "vscode-textmate": "9.2.0", + "wawoff2": "2.0.1" }, "imports": { "#src/*": "./src/*", diff --git a/readme.md b/readme.md index 117df684b36cb2afcc23f75de6c55b12004d6a28..e5763f218927a2cabe4a2361dbd0d2824ccc4fda 100644 --- a/readme.md +++ b/readme.md @@ -59,7 +59,6 @@ minimum system requirements: required software: - node.js v24 -- python v3 my development machine, for example, is Dell Inspiron 7348 with Core i7 -- 2.54.0