| ... | @@ -1,28 +1,6 @@ | ... | @@ -1,28 +1,6 @@ |
| 1 | export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { | 1 | export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { |
| 2 | if (fonts.length === 0) return; | 2 | if (fonts.length === 0) return; |
| 3 | using node = p.start("build fonts"); | 3 | using node = p.start("build fonts"); |
| 4 | | | |
| 5 | const temp = path.resolve(".clover/font"); | | |
| 6 | const venv = path.join(temp, "venv"); | | |
| 7 | const bin = path.join(venv, process.platform === "win32" ? "Scripts" : "bin"); | | |
| 8 | | | |
| 9 | if (!fs.existsSync(venv)) { | | |
| 10 | using _ = node.start("setup python env"); | | |
| 11 | await subprocess.exec("python", ["-m", "venv", venv]); | | |
| 12 | } | | |
| 13 | if (!fs.existsSync(path.join(bin, "pyftsubset"))) { | | |
| 14 | using _ = node.start("install pyftsubset"); | | |
| 15 | await subprocess.exec(path.join(bin, "pip"), [ | | |
| 16 | "install", | | |
| 17 | "fonttools==4.38.0", | | |
| 18 | "brotli==1.0.7", | | |
| 19 | ]); | | |
| 20 | } | | |
| 21 | | | |
| 22 | const instances = new async.OnceMap( | | |
| 23 | (_: string, x: () => Promise<void>) => x(), | | |
| 24 | (x) => x[0], | | |
| 25 | ); | | |
| 26 | function getArgs(vars: sg.FontVars) { | 4 | function getArgs(vars: sg.FontVars) { |
| 27 | return Object.entries(vars).sort((x) => | 5 | return Object.entries(vars).sort((x) => |
| 28 | x[0] === "wght" || x[0] === "slnt" ? 1 : 0 | 6 | x[0] === "wght" || x[0] === "slnt" ? 1 : 0 |
| ... | @@ -43,58 +21,35 @@ export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { | ... | @@ -43,58 +21,35 @@ export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { |
| 43 | .toLowerCase(); | 21 | .toLowerCase(); |
| 44 | } | 22 | } |
| 45 | | 23 | |
| 46 | async function makeInstance(input: string, vars: sg.FontVars) { | 24 | function getVariationAxes(vars: sg.FontVars) { |
| 47 | const args = getArgs(vars); | 25 | return Object.fromEntries(Object.entries(vars).map(([key, value]) => [ |
| 48 | const hash = getHash(input, args); | 26 | normalizeAxisTag(key), |
| 49 | const outfile = path.join(temp, `${hash}${path.extname(input)}`); | 27 | Array.isArray(value) ? { min: value[0], max: value[1] } : value, |
| 50 | await instances.getOrRun(outfile, async () => { | 28 | ])) as VariationAxes; |
| 51 | await fs.rm(outfile, { force: true }); | | |
| 52 | await subprocess.exec(path.join(bin, "fonttools"), [ | | |
| 53 | "varLib.instancer", | | |
| 54 | input, | | |
| 55 | ...args, | | |
| 56 | `--output=${outfile}`, | | |
| 57 | ]); | | |
| 58 | ASSERT(fs.existsSync(outfile)); | | |
| 59 | }); | | |
| 60 | return outfile; | | |
| 61 | } | 29 | } |
| 62 | | 30 | |
| 63 | await Promise.all(fonts.map((font) => | 31 | await Promise.all(fonts.map((font) => |
| 64 | incr.work(node, async (io, font) => { | 32 | incr.work(node, async (io, font) => { |
| 65 | using node = io.start(font.name); | 33 | using node = io.start(font.name); |
| 66 | const baseFile = await fetchFont(font.name, font.sources); | 34 | const baseFile = await fetchFont(font.name, font.sources); |
| | 35 | const baseBuffer = await fs.readFile(baseFile); |
| 67 | await Promise.all(font.subsets.map(async (subset) => { | 36 | await Promise.all(font.subsets.map(async (subset) => { |
| 68 | const args = getArgs(subset.vars ?? {}); | 37 | const args = getArgs(subset.vars ?? {}); |
| 69 | const unicodes = fontRangesToString( | 38 | const subsetText = fontRangesToText( |
| 70 | Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], | 39 | Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], |
| 71 | ); | 40 | ); |
| 72 | const hash = crypto | 41 | using _ = node.start(args.join(" ")); |
| 73 | .createHash("sha256") | 42 | const woff = await subsetFontBuffer(baseBuffer, subsetText, { |
| 74 | .update(`${baseFile}${args.join()}${unicodes}`) | 43 | targetFormat: "woff2", |
| 75 | .digest("hex") | 44 | variationAxes: subset.vars |
| 76 | .slice(0, 16) | 45 | ? getVariationAxes(subset.vars) |
| 77 | .toLowerCase(); | 46 | : undefined, |
| 78 | const woff = path.join(temp, hash + ".woff2"); | 47 | sourceName: baseFile, |
| 79 | if (!fs.existsSync(woff)) { | 48 | assetName: subset.asset.replace("[ext]", "woff2"), |
| 80 | using _ = node.start(args.join(" ")); | 49 | }); |
| 81 | let file = baseFile; | | |
| 82 | if (subset.vars) { | | |
| 83 | file = await makeInstance(baseFile, subset.vars); | | |
| 84 | } | | |
| 85 | await subprocess.exec(path.join(bin, "pyftsubset"), [ | | |
| 86 | file, | | |
| 87 | "--flavor=woff2", | | |
| 88 | `--output-file=${woff}`, | | |
| 89 | ...subset.layoutFeatures | | |
| 90 | ? [`--layout-features=${subset.layoutFeatures.join(",")}`] | | |
| 91 | : [], | | |
| 92 | `--unicodes=${unicodes}`, | | |
| 93 | ]); | | |
| 94 | } | | |
| 95 | await io.writeAsset({ | 50 | await io.writeAsset({ |
| 96 | pathname: subset.asset.replace("[ext]", "woff2"), | 51 | pathname: subset.asset.replace("[ext]", "woff2"), |
| 97 | buffer: await fs.readFile(woff), | 52 | buffer: woff, |
| 98 | }); | 53 | }); |
| 99 | })); | 54 | })); |
| 100 | }, font) | 55 | }, font) |
| ... | @@ -114,7 +69,7 @@ export async function fetchFont(name: string, sources: string[]) { | ... | @@ -114,7 +69,7 @@ export async function fetchFont(name: string, sources: string[]) { |
| 114 | if (response.ok) { | 69 | if (response.ok) { |
| 115 | await fs.writeMkdir( | 70 | await fs.writeMkdir( |
| 116 | cacheName, | 71 | cacheName, |
| 117 | Buffer.from(await response.arrayBuffer()), | 72 | new Uint8Array(await response.arrayBuffer()), |
| 118 | ); | 73 | ); |
| 119 | } else { | 74 | } else { |
| 120 | errs.push( | 75 | errs.push( |
| ... | @@ -170,51 +125,271 @@ export async function fetchFont(name: string, sources: string[]) { | ... | @@ -170,51 +125,271 @@ export async function fetchFont(name: string, sources: string[]) { |
| 170 | throw new AggregateError(errs, `Failed to fetch the ${name} font`); | 125 | throw new AggregateError(errs, `Failed to fetch the ${name} font`); |
| 171 | } | 126 | } |
| 172 | | 127 | |
| 173 | function fontRangesToString(ranges: sg.FontRange[]): string { | 128 | function fontRangesToText(ranges: sg.FontRange[]): string { |
| 174 | const segments: Array<{ start: number; end: number }> = []; | 129 | const chars = new Set<number>(); |
| 175 | | | |
| 176 | ranges.forEach((range) => { | 130 | ranges.forEach((range) => { |
| 177 | if (typeof range === "string") { | 131 | if (typeof range === "string") { |
| 178 | for (let i = 0; i < range.length; i++) { | 132 | for (const char of range) { |
| 179 | const cp = range.codePointAt(i) || 0; | 133 | chars.add(char.codePointAt(0) || 0); |
| 180 | segments.push({ start: cp, end: cp }); | | |
| 181 | } | 134 | } |
| 182 | } else if (typeof range === "number") { | 135 | } else if (typeof range === "number") { |
| 183 | segments.push({ start: range, end: range }); | 136 | chars.add(range); |
| 184 | } else { | 137 | } else { |
| 185 | segments.push({ start: range.start, end: range.end }); | 138 | for (let cp = range.start; cp <= range.end; cp++) chars.add(cp); |
| 186 | } | 139 | } |
| 187 | }); | 140 | }); |
| | 141 | return Array.from(chars) |
| | 142 | .sort((a, b) => a - b) |
| | 143 | .map((cp) => String.fromCodePoint(cp)) |
| | 144 | .join(""); |
| | 145 | } |
| 188 | | 146 | |
| 189 | segments.sort((a, b) => a.start - b.start); | 147 | function normalizeAxisTag(tag: string) { |
| | 148 | const lower = tag.toLowerCase(); |
| | 149 | return lower === "wght" || lower === "slnt" ? lower : tag; |
| | 150 | } |
| 190 | | 151 | |
| 191 | const merged: Array<{ start: number; end: number }> = []; | 152 | function detectFontFormat(buffer: Uint8Array): "sfnt" | "woff2" { |
| 192 | for (const seg of segments) { | 153 | const signature = String.fromCharCode(...buffer.subarray(0, 4)); |
| 193 | const last = merged[merged.length - 1]; | 154 | if (signature === "wOF2") return "woff2"; |
| 194 | if (last && seg.start <= last.end + 1) { | 155 | if ( |
| 195 | last.end = Math.max(last.end, seg.end); | 156 | signature === "true" || |
| 196 | } else { | 157 | signature === "OTTO" || |
| 197 | merged.push({ ...seg }); | 158 | signature === "\x00\x01\x00\x00" |
| 198 | } | 159 | ) { |
| | 160 | return "sfnt"; |
| 199 | } | 161 | } |
| | 162 | throw new Error(`Unsupported font format signature ${JSON.stringify(signature)}`); |
| | 163 | } |
| | 164 | |
| | 165 | async function convertToSfnt(buffer: Uint8Array) { |
| | 166 | const format = detectFontFormat(buffer); |
| | 167 | if (format === "sfnt") return buffer; |
| | 168 | return new Uint8Array(await wawoff2.decompress(buffer)); |
| | 169 | } |
| 200 | | 170 | |
| 201 | return merged.map(({ start, end }) => | 171 | async function convertFromSfnt( |
| 202 | start === end | 172 | buffer: Uint8Array, |
| 203 | ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}` | 173 | targetFormat: "sfnt" | "woff2", |
| 204 | : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${ | 174 | ) { |
| 205 | end.toString(16).toUpperCase().padStart(4, "0") | 175 | if (targetFormat === "sfnt") return buffer; |
| | 176 | return new Uint8Array(await wawoff2.compress(buffer)); |
| | 177 | } |
| | 178 | |
| | 179 | function getSubsetContext(opt: { |
| | 180 | sourceName?: string; |
| | 181 | assetName?: string; |
| | 182 | variationAxes?: VariationAxes; |
| | 183 | targetFormat: "sfnt" | "woff2"; |
| | 184 | text: string; |
| | 185 | }) { |
| | 186 | const fields = [ |
| | 187 | opt.sourceName ? `source=${opt.sourceName}` : "", |
| | 188 | opt.assetName ? `asset=${opt.assetName}` : "", |
| | 189 | `target=${opt.targetFormat}`, |
| | 190 | `textLength=${opt.text.length}`, |
| | 191 | opt.variationAxes |
| | 192 | ? `axes=${ |
| | 193 | Object.entries(opt.variationAxes).map(([key, value]) => |
| | 194 | typeof value === "number" |
| | 195 | ? `${key}=${value}` |
| | 196 | : `${key}=${value.min}:${value.max}${ |
| | 197 | value.default == null ? "" : `:${value.default}` |
| | 198 | }` |
| | 199 | ).join(",") |
| 206 | }` | 200 | }` |
| 207 | ).join(","); | 201 | : "", |
| | 202 | ].filter(Boolean); |
| | 203 | return fields.join(" "); |
| | 204 | } |
| | 205 | |
| | 206 | async function subsetFontBuffer( |
| | 207 | originalFont: Uint8Array, |
| | 208 | text: string, |
| | 209 | opt: { |
| | 210 | targetFormat: "sfnt" | "woff2"; |
| | 211 | variationAxes?: VariationAxes; |
| | 212 | sourceName?: string; |
| | 213 | assetName?: string; |
| | 214 | }, |
| | 215 | ) { |
| | 216 | return await subsetFontQueue(async () => { |
| | 217 | const ctx = getSubsetContext({ ...opt, text }); |
| | 218 | if (typeof text !== "string") { |
| | 219 | throw new Error(`Subset text must be a string. ${ctx}`); |
| | 220 | } |
| | 221 | |
| | 222 | const { harfbuzzJsWasm, getHeapU8 } = await loadAndInitializeHarfbuzz(); |
| | 223 | const sfntFont = await convertToSfnt(originalFont); |
| | 224 | |
| | 225 | const input = harfbuzzJsWasm.hb_subset_input_create_or_fail(); |
| | 226 | if (input === 0) { |
| | 227 | throw new Error(`Failed to create HarfBuzz subset input. ${ctx}`); |
| | 228 | } |
| | 229 | |
| | 230 | const fontBuffer = harfbuzzJsWasm.malloc(sfntFont.byteLength); |
| | 231 | getHeapU8().set(new Uint8Array(sfntFont), fontBuffer); |
| | 232 | |
| | 233 | const blob = harfbuzzJsWasm.hb_blob_create( |
| | 234 | fontBuffer, |
| | 235 | sfntFont.byteLength, |
| | 236 | 2, |
| | 237 | 0, |
| | 238 | 0, |
| | 239 | ); |
| | 240 | const face = harfbuzzJsWasm.hb_face_create(blob, 0); |
| | 241 | harfbuzzJsWasm.hb_blob_destroy(blob); |
| | 242 | |
| | 243 | try { |
| | 244 | const layoutFeatures = harfbuzzJsWasm.hb_subset_input_set(input, 6); |
| | 245 | harfbuzzJsWasm.hb_set_clear(layoutFeatures); |
| | 246 | harfbuzzJsWasm.hb_set_invert(layoutFeatures); |
| | 247 | |
| | 248 | const inputUnicodes = harfbuzzJsWasm.hb_subset_input_unicode_set(input); |
| | 249 | for (const char of text) { |
| | 250 | harfbuzzJsWasm.hb_set_add(inputUnicodes, char.codePointAt(0) || 0); |
| | 251 | } |
| | 252 | |
| | 253 | if (opt.variationAxes) { |
| | 254 | for (const [axisName, value] of Object.entries(opt.variationAxes)) { |
| | 255 | if (typeof value === "number") { |
| | 256 | if ( |
| | 257 | !harfbuzzJsWasm.hb_subset_input_pin_axis_location( |
| | 258 | input, |
| | 259 | face, |
| | 260 | hbTag(axisName), |
| | 261 | value, |
| | 262 | ) |
| | 263 | ) { |
| | 264 | throw new Error( |
| | 265 | `Failed to pin variation axis ${axisName}=${value}. ${ctx}`, |
| | 266 | ); |
| | 267 | } |
| | 268 | } else if ( |
| | 269 | !harfbuzzJsWasm.hb_subset_input_set_axis_range( |
| | 270 | input, |
| | 271 | face, |
| | 272 | hbTag(axisName), |
| | 273 | value.min, |
| | 274 | value.max, |
| | 275 | value.default ?? Number.NaN, |
| | 276 | ) |
| | 277 | ) { |
| | 278 | throw new Error( |
| | 279 | `Failed to set variation axis range ${axisName}=[${value.min}, ${value.max}]${ |
| | 280 | value.default == null ? "" : ` default=${value.default}` |
| | 281 | }. ${ctx}`, |
| | 282 | ); |
| | 283 | } |
| | 284 | } |
| | 285 | } |
| | 286 | |
| | 287 | const subset = harfbuzzJsWasm.hb_subset_or_fail(face, input); |
| | 288 | if (subset === 0) { |
| | 289 | throw new Error(`HarfBuzz failed to subset font. ${ctx}`); |
| | 290 | } |
| | 291 | |
| | 292 | try { |
| | 293 | const result = harfbuzzJsWasm.hb_face_reference_blob(subset); |
| | 294 | try { |
| | 295 | const offset = harfbuzzJsWasm.hb_blob_get_data(result, 0); |
| | 296 | const subsetByteLength = harfbuzzJsWasm.hb_blob_get_length(result); |
| | 297 | if (subsetByteLength === 0) { |
| | 298 | throw new Error(`Subset font was empty. ${ctx}`); |
| | 299 | } |
| | 300 | const subsetFont = Uint8Array.from( |
| | 301 | getHeapU8().subarray(offset, offset + subsetByteLength), |
| | 302 | ); |
| | 303 | return await convertFromSfnt(subsetFont, opt.targetFormat); |
| | 304 | } finally { |
| | 305 | harfbuzzJsWasm.hb_blob_destroy(result); |
| | 306 | } |
| | 307 | } finally { |
| | 308 | harfbuzzJsWasm.hb_face_destroy(subset); |
| | 309 | } |
| | 310 | } finally { |
| | 311 | harfbuzzJsWasm.hb_subset_input_destroy(input); |
| | 312 | harfbuzzJsWasm.hb_face_destroy(face); |
| | 313 | harfbuzzJsWasm.free(fontBuffer); |
| | 314 | } |
| | 315 | }); |
| | 316 | } |
| | 317 | |
| | 318 | type VariationAxisRange = { min: number; max: number; default?: number }; |
| | 319 | type VariationAxes = Record<string, number | VariationAxisRange>; |
| | 320 | |
| | 321 | interface HarfbuzzSubset { |
| | 322 | memory: WebAssembly.Memory; |
| | 323 | malloc(size: number): number; |
| | 324 | free(ptr: number): void; |
| | 325 | hb_blob_create( |
| | 326 | data: number, |
| | 327 | length: number, |
| | 328 | mode: number, |
| | 329 | userData: number, |
| | 330 | destroy: number, |
| | 331 | ): number; |
| | 332 | hb_blob_destroy(blob: number): void; |
| | 333 | hb_blob_get_data(blob: number, length: number): number; |
| | 334 | hb_blob_get_length(blob: number): number; |
| | 335 | hb_face_create(blob: number, index: number): number; |
| | 336 | hb_face_destroy(face: number): void; |
| | 337 | hb_face_reference_blob(face: number): number; |
| | 338 | hb_set_add(set: number, value: number): void; |
| | 339 | hb_set_clear(set: number): void; |
| | 340 | hb_set_invert(set: number): void; |
| | 341 | hb_subset_input_create_or_fail(): number; |
| | 342 | hb_subset_input_destroy(input: number): void; |
| | 343 | hb_subset_input_get_flags(input: number): number; |
| | 344 | hb_subset_input_set(input: number, setType: number): number; |
| | 345 | hb_subset_input_set_flags(input: number, flags: number): void; |
| | 346 | hb_subset_input_unicode_set(input: number): number; |
| | 347 | hb_subset_input_pin_axis_location( |
| | 348 | input: number, |
| | 349 | face: number, |
| | 350 | axisTag: number, |
| | 351 | value: number, |
| | 352 | ): number; |
| | 353 | hb_subset_input_set_axis_range( |
| | 354 | input: number, |
| | 355 | face: number, |
| | 356 | axisTag: number, |
| | 357 | min: number, |
| | 358 | max: number, |
| | 359 | defaultValue: number, |
| | 360 | ): number; |
| | 361 | hb_subset_or_fail(face: number, input: number): number; |
| | 362 | } |
| | 363 | |
| | 364 | const loadAndInitializeHarfbuzz = async.once(async () => { |
| | 365 | const wasm = await readFile(new URL(import.meta.resolve("harfbuzzjs/hb-subset.wasm"))); |
| | 366 | const { instance } = await WebAssembly.instantiate(wasm); |
| | 367 | const harfbuzzJsWasm = instance.exports as unknown as HarfbuzzSubset; |
| | 368 | return { |
| | 369 | harfbuzzJsWasm, |
| | 370 | getHeapU8: () => new Uint8Array(harfbuzzJsWasm.memory.buffer), |
| | 371 | }; |
| | 372 | }); |
| | 373 | |
| | 374 | let subsetFontLock = Promise.resolve(); |
| | 375 | function subsetFontQueue<T>(fn: () => Promise<T>) { |
| | 376 | const run = subsetFontLock.then(fn, fn); |
| | 377 | subsetFontLock = run.then(() => undefined, () => undefined); |
| | 378 | return run; |
| | 379 | } |
| | 380 | |
| | 381 | function hbTag(str: string) { |
| | 382 | return str.split("").reduce((a, ch) => (a << 8) + ch.charCodeAt(0), 0); |
| 208 | } | 383 | } |
| 209 | | 384 | |
| 210 | import * as fs from "#sitegen/fs"; | 385 | import * as fs from "#sitegen/fs"; |
| 211 | import * as os from "node:os"; | 386 | import * as os from "node:os"; |
| 212 | import * as path from "node:path"; | 387 | import * as path from "node:path"; |
| 213 | import * as sg from "#sitegen"; | 388 | import * as sg from "#sitegen"; |
| 214 | import * as subprocess from "@clo/lib/subprocess"; | | |
| 215 | import * as incr from "./incremental.ts"; | 389 | import * as incr from "./incremental.ts"; |
| 216 | import * as crypto from "node:crypto"; | 390 | import * as crypto from "node:crypto"; |
| 217 | import * as async from "@clo/lib/async"; | 391 | import * as async from "@clo/lib/async"; |
| 218 | import * as error from "@clo/lib/error"; | 392 | import * as error from "@clo/lib/error"; |
| 219 | import * as progress from "@clo/lib/progress"; | 393 | import * as progress from "@clo/lib/progress"; |
| 220 | import { ASSERT } from "@clo/lib/assert"; | 394 | import { readFile } from "node:fs/promises"; |
| | 395 | import * as wawoff2 from "wawoff2"; |
clover caruso <git@paperclover.net> 2026-03-27 02:54:41-07:00