authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-03-27 02:54:41-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-03-27 03:28:08-07:00
logb01c9ea08ff53136b25ad5c35f26fcfa1917aaf0
treeab73d15ef669b0132079f91fe637f442403d80c6
parentc7ff451dfa7c20ba6d9b616eabf57d73eddf813e
signature Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

feat: wasm-based font subsetting

drops the python system that was a holdover from my old manual font subsetting workflows. now it is all js+wasm. over time i will replace every other native dependency with a WebAssembly alternative. resolves #101

7 files changed, 298 insertions(+), 94 deletions(-)

framework/font.ts+263-88
......@@ -1,28 +1,6 @@
11export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) {
22 if (fonts.length === 0) return;
33 using node = p.start("build fonts");
4
5 const temp = path.resolve(".clover/font");
6 const venv = path.join(temp, "venv");
7 const bin = path.join(venv, process.platform === "win32" ? "Scripts" : "bin");
8
9 if (!fs.existsSync(venv)) {
10 using _ = node.start("setup python env");
11 await subprocess.exec("python", ["-m", "venv", venv]);
12 }
13 if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
14 using _ = node.start("install pyftsubset");
15 await subprocess.exec(path.join(bin, "pip"), [
16 "install",
17 "fonttools==4.38.0",
18 "brotli==1.0.7",
19 ]);
20 }
21
22 const instances = new async.OnceMap(
23 (_: string, x: () => Promise<void>) => x(),
24 (x) => x[0],
25 );
264 function getArgs(vars: sg.FontVars) {
275 return Object.entries(vars).sort((x) =>
286 x[0] === "wght" || x[0] === "slnt" ? 1 : 0
......@@ -43,58 +21,35 @@ export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) {
4321 .toLowerCase();
4422 }
4523
46 async function makeInstance(input: string, vars: sg.FontVars) {
47 const args = getArgs(vars);
48 const hash = getHash(input, args);
49 const outfile = path.join(temp, `${hash}${path.extname(input)}`);
50 await instances.getOrRun(outfile, async () => {
51 await fs.rm(outfile, { force: true });
52 await subprocess.exec(path.join(bin, "fonttools"), [
53 "varLib.instancer",
54 input,
55 ...args,
56 `--output=${outfile}`,
57 ]);
58 ASSERT(fs.existsSync(outfile));
59 });
60 return outfile;
24 function getVariationAxes(vars: sg.FontVars) {
25 return Object.fromEntries(Object.entries(vars).map(([key, value]) => [
26 normalizeAxisTag(key),
27 Array.isArray(value) ? { min: value[0], max: value[1] } : value,
28 ])) as VariationAxes;
6129 }
6230
6331 await Promise.all(fonts.map((font) =>
6432 incr.work(node, async (io, font) => {
6533 using node = io.start(font.name);
6634 const baseFile = await fetchFont(font.name, font.sources);
35 const baseBuffer = await fs.readFile(baseFile);
6736 await Promise.all(font.subsets.map(async (subset) => {
6837 const args = getArgs(subset.vars ?? {});
69 const unicodes = fontRangesToString(
38 const subsetText = fontRangesToText(
7039 Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
7140 );
72 const hash = crypto
73 .createHash("sha256")
74 .update(`${baseFile}${args.join()}${unicodes}`)
75 .digest("hex")
76 .slice(0, 16)
77 .toLowerCase();
78 const woff = path.join(temp, hash + ".woff2");
79 if (!fs.existsSync(woff)) {
80 using _ = node.start(args.join(" "));
81 let file = baseFile;
82 if (subset.vars) {
83 file = await makeInstance(baseFile, subset.vars);
84 }
85 await subprocess.exec(path.join(bin, "pyftsubset"), [
86 file,
87 "--flavor=woff2",
88 `--output-file=${woff}`,
89 ...subset.layoutFeatures
90 ? [`--layout-features=${subset.layoutFeatures.join(",")}`]
91 : [],
92 `--unicodes=${unicodes}`,
93 ]);
94 }
41 using _ = node.start(args.join(" "));
42 const woff = await subsetFontBuffer(baseBuffer, subsetText, {
43 targetFormat: "woff2",
44 variationAxes: subset.vars
45 ? getVariationAxes(subset.vars)
46 : undefined,
47 sourceName: baseFile,
48 assetName: subset.asset.replace("[ext]", "woff2"),
49 });
9550 await io.writeAsset({
9651 pathname: subset.asset.replace("[ext]", "woff2"),
97 buffer: await fs.readFile(woff),
52 buffer: woff,
9853 });
9954 }));
10055 }, font)
......@@ -114,7 +69,7 @@ export async function fetchFont(name: string, sources: string[]) {
11469 if (response.ok) {
11570 await fs.writeMkdir(
11671 cacheName,
117 Buffer.from(await response.arrayBuffer()),
72 new Uint8Array(await response.arrayBuffer()),
11873 );
11974 } else {
12075 errs.push(
......@@ -170,51 +125,271 @@ export async function fetchFont(name: string, sources: string[]) {
170125 throw new AggregateError(errs, `Failed to fetch the ${name} font`);
171126}
172127
173function fontRangesToString(ranges: sg.FontRange[]): string {
174 const segments: Array<{ start: number; end: number }> = [];
175
128function fontRangesToText(ranges: sg.FontRange[]): string {
129 const chars = new Set<number>();
176130 ranges.forEach((range) => {
177131 if (typeof range === "string") {
178 for (let i = 0; i < range.length; i++) {
179 const cp = range.codePointAt(i) || 0;
180 segments.push({ start: cp, end: cp });
132 for (const char of range) {
133 chars.add(char.codePointAt(0) || 0);
181134 }
182135 } else if (typeof range === "number") {
183 segments.push({ start: range, end: range });
136 chars.add(range);
184137 } else {
185 segments.push({ start: range.start, end: range.end });
138 for (let cp = range.start; cp <= range.end; cp++) chars.add(cp);
186139 }
187140 });
141 return Array.from(chars)
142 .sort((a, b) => a - b)
143 .map((cp) => String.fromCodePoint(cp))
144 .join("");
145}
188146
189 segments.sort((a, b) => a.start - b.start);
147function normalizeAxisTag(tag: string) {
148 const lower = tag.toLowerCase();
149 return lower === "wght" || lower === "slnt" ? lower : tag;
150}
190151
191 const merged: Array<{ start: number; end: number }> = [];
192 for (const seg of segments) {
193 const last = merged[merged.length - 1];
194 if (last && seg.start <= last.end + 1) {
195 last.end = Math.max(last.end, seg.end);
196 } else {
197 merged.push({ ...seg });
198 }
152function detectFontFormat(buffer: Uint8Array): "sfnt" | "woff2" {
153 const signature = String.fromCharCode(...buffer.subarray(0, 4));
154 if (signature === "wOF2") return "woff2";
155 if (
156 signature === "true" ||
157 signature === "OTTO" ||
158 signature === "\x00\x01\x00\x00"
159 ) {
160 return "sfnt";
199161 }
162 throw new Error(`Unsupported font format signature ${JSON.stringify(signature)}`);
163}
164
165async function convertToSfnt(buffer: Uint8Array) {
166 const format = detectFontFormat(buffer);
167 if (format === "sfnt") return buffer;
168 return new Uint8Array(await wawoff2.decompress(buffer));
169}
200170
201 return merged.map(({ start, end }) =>
202 start === end
203 ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}`
204 : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${
205 end.toString(16).toUpperCase().padStart(4, "0")
171async function convertFromSfnt(
172 buffer: Uint8Array,
173 targetFormat: "sfnt" | "woff2",
174) {
175 if (targetFormat === "sfnt") return buffer;
176 return new Uint8Array(await wawoff2.compress(buffer));
177}
178
179function getSubsetContext(opt: {
180 sourceName?: string;
181 assetName?: string;
182 variationAxes?: VariationAxes;
183 targetFormat: "sfnt" | "woff2";
184 text: string;
185}) {
186 const fields = [
187 opt.sourceName ? `source=${opt.sourceName}` : "",
188 opt.assetName ? `asset=${opt.assetName}` : "",
189 `target=${opt.targetFormat}`,
190 `textLength=${opt.text.length}`,
191 opt.variationAxes
192 ? `axes=${
193 Object.entries(opt.variationAxes).map(([key, value]) =>
194 typeof value === "number"
195 ? `${key}=${value}`
196 : `${key}=${value.min}:${value.max}${
197 value.default == null ? "" : `:${value.default}`
198 }`
199 ).join(",")
206200 }`
207 ).join(",");
201 : "",
202 ].filter(Boolean);
203 return fields.join(" ");
204}
205
206async function subsetFontBuffer(
207 originalFont: Uint8Array,
208 text: string,
209 opt: {
210 targetFormat: "sfnt" | "woff2";
211 variationAxes?: VariationAxes;
212 sourceName?: string;
213 assetName?: string;
214 },
215) {
216 return await subsetFontQueue(async () => {
217 const ctx = getSubsetContext({ ...opt, text });
218 if (typeof text !== "string") {
219 throw new Error(`Subset text must be a string. ${ctx}`);
220 }
221
222 const { harfbuzzJsWasm, getHeapU8 } = await loadAndInitializeHarfbuzz();
223 const sfntFont = await convertToSfnt(originalFont);
224
225 const input = harfbuzzJsWasm.hb_subset_input_create_or_fail();
226 if (input === 0) {
227 throw new Error(`Failed to create HarfBuzz subset input. ${ctx}`);
228 }
229
230 const fontBuffer = harfbuzzJsWasm.malloc(sfntFont.byteLength);
231 getHeapU8().set(new Uint8Array(sfntFont), fontBuffer);
232
233 const blob = harfbuzzJsWasm.hb_blob_create(
234 fontBuffer,
235 sfntFont.byteLength,
236 2,
237 0,
238 0,
239 );
240 const face = harfbuzzJsWasm.hb_face_create(blob, 0);
241 harfbuzzJsWasm.hb_blob_destroy(blob);
242
243 try {
244 const layoutFeatures = harfbuzzJsWasm.hb_subset_input_set(input, 6);
245 harfbuzzJsWasm.hb_set_clear(layoutFeatures);
246 harfbuzzJsWasm.hb_set_invert(layoutFeatures);
247
248 const inputUnicodes = harfbuzzJsWasm.hb_subset_input_unicode_set(input);
249 for (const char of text) {
250 harfbuzzJsWasm.hb_set_add(inputUnicodes, char.codePointAt(0) || 0);
251 }
252
253 if (opt.variationAxes) {
254 for (const [axisName, value] of Object.entries(opt.variationAxes)) {
255 if (typeof value === "number") {
256 if (
257 !harfbuzzJsWasm.hb_subset_input_pin_axis_location(
258 input,
259 face,
260 hbTag(axisName),
261 value,
262 )
263 ) {
264 throw new Error(
265 `Failed to pin variation axis ${axisName}=${value}. ${ctx}`,
266 );
267 }
268 } else if (
269 !harfbuzzJsWasm.hb_subset_input_set_axis_range(
270 input,
271 face,
272 hbTag(axisName),
273 value.min,
274 value.max,
275 value.default ?? Number.NaN,
276 )
277 ) {
278 throw new Error(
279 `Failed to set variation axis range ${axisName}=[${value.min}, ${value.max}]${
280 value.default == null ? "" : ` default=${value.default}`
281 }. ${ctx}`,
282 );
283 }
284 }
285 }
286
287 const subset = harfbuzzJsWasm.hb_subset_or_fail(face, input);
288 if (subset === 0) {
289 throw new Error(`HarfBuzz failed to subset font. ${ctx}`);
290 }
291
292 try {
293 const result = harfbuzzJsWasm.hb_face_reference_blob(subset);
294 try {
295 const offset = harfbuzzJsWasm.hb_blob_get_data(result, 0);
296 const subsetByteLength = harfbuzzJsWasm.hb_blob_get_length(result);
297 if (subsetByteLength === 0) {
298 throw new Error(`Subset font was empty. ${ctx}`);
299 }
300 const subsetFont = Uint8Array.from(
301 getHeapU8().subarray(offset, offset + subsetByteLength),
302 );
303 return await convertFromSfnt(subsetFont, opt.targetFormat);
304 } finally {
305 harfbuzzJsWasm.hb_blob_destroy(result);
306 }
307 } finally {
308 harfbuzzJsWasm.hb_face_destroy(subset);
309 }
310 } finally {
311 harfbuzzJsWasm.hb_subset_input_destroy(input);
312 harfbuzzJsWasm.hb_face_destroy(face);
313 harfbuzzJsWasm.free(fontBuffer);
314 }
315 });
316}
317
318type VariationAxisRange = { min: number; max: number; default?: number };
319type VariationAxes = Record<string, number | VariationAxisRange>;
320
321interface HarfbuzzSubset {
322 memory: WebAssembly.Memory;
323 malloc(size: number): number;
324 free(ptr: number): void;
325 hb_blob_create(
326 data: number,
327 length: number,
328 mode: number,
329 userData: number,
330 destroy: number,
331 ): number;
332 hb_blob_destroy(blob: number): void;
333 hb_blob_get_data(blob: number, length: number): number;
334 hb_blob_get_length(blob: number): number;
335 hb_face_create(blob: number, index: number): number;
336 hb_face_destroy(face: number): void;
337 hb_face_reference_blob(face: number): number;
338 hb_set_add(set: number, value: number): void;
339 hb_set_clear(set: number): void;
340 hb_set_invert(set: number): void;
341 hb_subset_input_create_or_fail(): number;
342 hb_subset_input_destroy(input: number): void;
343 hb_subset_input_get_flags(input: number): number;
344 hb_subset_input_set(input: number, setType: number): number;
345 hb_subset_input_set_flags(input: number, flags: number): void;
346 hb_subset_input_unicode_set(input: number): number;
347 hb_subset_input_pin_axis_location(
348 input: number,
349 face: number,
350 axisTag: number,
351 value: number,
352 ): number;
353 hb_subset_input_set_axis_range(
354 input: number,
355 face: number,
356 axisTag: number,
357 min: number,
358 max: number,
359 defaultValue: number,
360 ): number;
361 hb_subset_or_fail(face: number, input: number): number;
362}
363
364const loadAndInitializeHarfbuzz = async.once(async () => {
365 const wasm = await readFile(new URL(import.meta.resolve("harfbuzzjs/hb-subset.wasm")));
366 const { instance } = await WebAssembly.instantiate(wasm);
367 const harfbuzzJsWasm = instance.exports as unknown as HarfbuzzSubset;
368 return {
369 harfbuzzJsWasm,
370 getHeapU8: () => new Uint8Array(harfbuzzJsWasm.memory.buffer),
371 };
372});
373
374let subsetFontLock = Promise.resolve();
375function subsetFontQueue<T>(fn: () => Promise<T>) {
376 const run = subsetFontLock.then(fn, fn);
377 subsetFontLock = run.then(() => undefined, () => undefined);
378 return run;
379}
380
381function hbTag(str: string) {
382 return str.split("").reduce((a, ch) => (a << 8) + ch.charCodeAt(0), 0);
208383}
209384
210385import * as fs from "#sitegen/fs";
211386import * as os from "node:os";
212387import * as path from "node:path";
213388import * as sg from "#sitegen";
214import * as subprocess from "@clo/lib/subprocess";
215389import * as incr from "./incremental.ts";
216390import * as crypto from "node:crypto";
217391import * as async from "@clo/lib/async";
218392import * as error from "@clo/lib/error";
219393import * as progress from "@clo/lib/progress";
220import { ASSERT } from "@clo/lib/assert";
394import { readFile } from "node:fs/promises";
395import * as wawoff2 from "wawoff2";
framework/hot.ts+1
......@@ -123,6 +123,7 @@ export function loadEsbuildCode(
123123 import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
124124 import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
125125 import.meta.filename = ${JSON.stringify(filepath)};
126 import.meta.resolve = (r) => require('url').pathToFileURL(require.resolve(r));
126127 `
127128 .trim()
128129 .replace(/[\n\s]/g, "") + src;
framework/incremental.ts+1-1
......@@ -578,7 +578,7 @@ export class Io {
578578 }
579579 async writeAsset(asset: {
580580 pathname: string;
581 buffer: string | Buffer;
581 buffer: string | Uint8Array;
582582 regenerative?: boolean;
583583 headers?: HeadersInit;
584584 }) {
framework/lib/fs.ts+2-2
......@@ -31,14 +31,14 @@ export type WriteFileAsyncOptions = Parameters<typeof writeFile>[2];
3131
3232export async function writeMkdir(
3333 file: string,
34 contents: Buffer | string,
34 contents: Uint8Array | string,
3535 options?: WriteFileAsyncOptions,
3636) {
3737 await mkdir(path.dirname(file));
3838 return writeFile(file, contents, options);
3939}
4040
41export function writeMkdirSync(file: string, contents: Buffer | string) {
41export function writeMkdirSync(file: string, contents: Uint8Array | string) {
4242 mkdirSync(path.dirname(file));
4343 return writeFileSync(file, contents);
4444}
package-lock.json+28-1
......@@ -13,6 +13,7 @@
1313 "dashjs": "5.0.3",
1414 "devalue": "5.6.4",
1515 "esbuild": "0.25.5",
16 "harfbuzzjs": "0.10.1",
1617 "hls.js": "1.6.5",
1718 "hono": "4.12.8",
1819 "marko": "5.38.33",
......@@ -23,7 +24,8 @@
2324 "unique-names-generator": "4.7.1",
2425 "vitest": "^4.0.18",
2526 "vscode-oniguruma": "2.0.1",
26 "vscode-textmate": "9.2.0"
27 "vscode-textmate": "9.2.0",
28 "wawoff2": "2.0.1"
2729 }
2830 },
2931 "lib": {
......@@ -1085,6 +1087,12 @@
10851087 "integrity": "sha512-+F3InkcH2XOGK7Jf/ZQis4cwZ4wbfmpBKo5J+SAA9bglT1gVd0e9hroHWarU076pJrAfs8JKuRPwDqwPBOKHnw==",
10861088 "license": "MIT"
10871089 },
1090 "node_modules/argparse": {
1091 "version": "2.0.1",
1092 "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
1093 "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
1094 "license": "Python-2.0"
1095 },
10881096 "node_modules/assertion-error": {
10891097 "version": "1.1.0",
10901098 "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
......@@ -1384,6 +1392,12 @@
13841392 "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
13851393 }
13861394 },
1395 "node_modules/harfbuzzjs": {
1396 "version": "0.10.1",
1397 "resolved": "https://registry.npmjs.org/harfbuzzjs/-/harfbuzzjs-0.10.1.tgz",
1398 "integrity": "sha512-Snspnku2xVnKvoXkdLeOgDWFmsjv8bhCkiA5o+c12I5NGO+aXsLNYmIjaEfIU//SSzoZ63xaOAb3A7eT/Ju+Sw==",
1399 "license": "MIT"
1400 },
13871401 "node_modules/he": {
13881402 "version": "1.2.0",
13891403 "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
......@@ -2868,6 +2882,19 @@
28682882 "integrity": "sha512-krhkqzJdUxAZv2Cx0Gz6dN1r7TTrG9RDewkDHBbJQIqbNTCdB5ZUHVh7VkA4DgrKW4ZXPPUQKCwmI/3btDse9A==",
28692883 "license": "MIT"
28702884 },
2885 "node_modules/wawoff2": {
2886 "version": "2.0.1",
2887 "resolved": "https://registry.npmjs.org/wawoff2/-/wawoff2-2.0.1.tgz",
2888 "integrity": "sha512-r0CEmvpH63r4T15ebFqeOjGqU4+EgTx4I510NtK35EMciSdcTxCw3Byy3JnBonz7iyIFZ0AbVo0bbFpEVuhCYA==",
2889 "license": "MIT",
2890 "dependencies": {
2891 "argparse": "^2.0.1"
2892 },
2893 "bin": {
2894 "woff2_compress.js": "bin/woff2_compress.js",
2895 "woff2_decompress.js": "bin/woff2_decompress.js"
2896 }
2897 },
28712898 "node_modules/why-is-node-running": {
28722899 "version": "2.3.0",
28732900 "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz",
package.json+3-1
......@@ -9,6 +9,7 @@
99 "dashjs": "5.0.3",
1010 "devalue": "5.6.4",
1111 "esbuild": "0.25.5",
12 "harfbuzzjs": "0.10.1",
1213 "hls.js": "1.6.5",
1314 "hono": "4.12.8",
1415 "marko": "5.38.33",
......@@ -19,7 +20,8 @@
1920 "unique-names-generator": "4.7.1",
2021 "vitest": "^4.0.18",
2122 "vscode-oniguruma": "2.0.1",
22 "vscode-textmate": "9.2.0"
23 "vscode-textmate": "9.2.0",
24 "wawoff2": "2.0.1"
2325 },
2426 "imports": {
2527 "#src/*": "./src/*",
readme.md-1
......@@ -59,7 +59,6 @@ minimum system requirements:
5959required software:
6060
6161- node.js v24
62- python v3
6362
6463my development machine, for example, is Dell Inspiron 7348 with Core i7
6564