authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-07-03 01:22:59-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:45-07:00
logcd82bee436de52183b6310615e651ff0561ebe36
treec572dcc7b90c8a9e650055e34dbd365f8a1a1e0e
parent56a1174bf97856be619b2f7526505cc90d6a7161
signature Commit is signed but in an unrecognized format.

move discovered ffmpeg presets

8 files changed, 404 insertions(+), 115 deletions(-)

framework/incremental.ts+4-8
......@@ -318,14 +318,10 @@ export class Incremental {
318318
319319 async compressImpl({ algo, buffer, hash }: CompressJob) {
320320 let out;
321 switch (algo) {
322 case "zstd":
323 out = await zstd(buffer);
324 break;
325 case "gzip":
326 out = await gzip(buffer, { level: 9 });
327 break;
328 }
321 if (algo === "zstd") out = await zstd(buffer);
322 else if (algo === "gzip") out = await gzip(buffer, { level: 9 });
323 else algo satisfies never;
324
329325 let entry = this.compress.get(hash);
330326 if (!entry) {
331327 this.compress.set(
package-lock.json+9-9
......@@ -1,5 +1,5 @@
11{
2 "name": "paperclover.net",
2 "name": "sitegen",
33 "lockfileVersion": 3,
44 "requires": true,
55 "packages": {
......@@ -22,7 +22,7 @@
2222 "vscode-textmate": "^9.2.0"
2323 },
2424 "devDependencies": {
25 "@types/node": "^22.15.29",
25 "@types/node": "^24.0.10",
2626 "typescript": "^5.8.3"
2727 }
2828 },
......@@ -1568,13 +1568,13 @@
15681568 "license": "MIT"
15691569 },
15701570 "node_modules/@types/node": {
1571 "version": "22.15.30",
1572 "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.30.tgz",
1573 "integrity": "sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==",
1571 "version": "24.0.10",
1572 "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.10.tgz",
1573 "integrity": "sha512-ENHwaH+JIRTDIEEbDK6QSQntAYGtbvdDXnMXnZaZ6k13Du1dPMmprkEHIL7ok2Wl2aZevetwTAb5S+7yIF+enA==",
15741574 "devOptional": true,
15751575 "license": "MIT",
15761576 "dependencies": {
1577 "undici-types": "~6.21.0"
1577 "undici-types": "~7.8.0"
15781578 }
15791579 },
15801580 "node_modules/@types/unist": {
......@@ -4509,9 +4509,9 @@
45094509 }
45104510 },
45114511 "node_modules/undici-types": {
4512 "version": "6.21.0",
4513 "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
4514 "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
4512 "version": "7.8.0",
4513 "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",
4514 "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==",
45154515 "devOptional": true,
45164516 "license": "MIT"
45174517 },
package.json+1-1
......@@ -18,7 +18,7 @@
1818 "vscode-textmate": "^9.2.0"
1919 },
2020 "devDependencies": {
21 "@types/node": "^22.15.29",
21 "@types/node": "^24.0.10",
2222 "typescript": "^5.8.3"
2323 },
2424 "imports": {
readme.md+1-1
......@@ -25,7 +25,7 @@ that assist building websites. these tools power https://paperclover.net.
2525 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
2626 - TODO: Meta and Open Graph generation. (`export const meta`)
2727 - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
28- **Built on the battle-tested Node.js runtime.** Partial support for Deno and Bun.
28- **Built on the battle-tested Node.js runtime.**
2929
3030[1]: https://next.markojs.com
3131
run.js+18
......@@ -1,8 +1,26 @@
11// This file allows using Node.js in combination with
22// all available plugins. Usage: "node run <script>"
33import * as util from "node:util";
4import * as zlib from "node:zlib";
45import process from "node:process";
56
7if (!zlib.zstdCompress) {
8 const brand = process.versions.bun
9 ? `bun ${process.versions.bun}`
10 : process.versions.deno
11 ? `deno ${process.versions.deno}`
12 : null;
13
14 globalThis.console.error(
15 `sitegen depends on a node.js-compatibile runtime that supports zstd compression\n` +
16 `this is node.js version ${process.version}${
17 brand ? ` (${brand})` : ""
18 }\n\n` +
19 `get node.js --> https://nodejs.org/en/download/current`,
20 );
21 process.exit(1);
22}
23
624// Disable experimental warnings (Type Stripping, etc)
725const { emit: originalEmit } = process;
826const warnings = ["ExperimentalWarning"];
src/file-viewer/bin/scan3.ts+56-96
......@@ -1,3 +1,15 @@
1// The file scanner incrementally updates an sqlite database with file
2// stats. Additionally, it runs "processors" on files, which precompute
3// expensive data such as running `ffprobe` on all media to get the
4// duration.
5//
6// Processors are also used to derive compressed and optimized assets,
7// which is how automatic JXL / AV1 encoding is done. Derived files are
8// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9//
10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.
113const root = path.resolve("C:/media");
214const workDir = path.resolve(".clover/file-assets");
315
......@@ -258,8 +270,8 @@ const execFile: typeof execFileRaw = ((
258270 }
259271 throw e;
260272 })) as any;
261const ffprobe = testProgram("ffprobe", "--help");
262const ffmpeg = testProgram("ffmpeg", "--help");
273const ffprobeBin = testProgram("ffprobe", "--help");
274const ffmpegBin = testProgram("ffmpeg", "--help");
263275
264276const ffmpegOptions = [
265277 "-hide_banner",
......@@ -267,14 +279,12 @@ const ffmpegOptions = [
267279 "warning",
268280];
269281
270const imageSizes = [64, 128, 256, 512, 1024, 2048];
271
272282const procDuration: Process = {
273283 name: "calculate duration",
274 enable: ffprobe !== null,
284 enable: ffprobeBin !== null,
275285 include: rules.extsDuration,
276286 async run({ absPath, mediaFile }) {
277 const { stdout } = await execFile(ffprobe!, [
287 const { stdout } = await execFile(ffprobeBin!, [
278288 "-v",
279289 "error",
280290 "-show_entries",
......@@ -295,7 +305,7 @@ const procDuration: Process = {
295305// NOTE: Never re-order the processors. Add new ones at the end.
296306const procDimensions: Process = {
297307 name: "calculate dimensions",
298 enable: ffprobe != null,
308 enable: ffprobeBin != null,
299309 include: rules.extsDimensions,
300310 async run({ absPath, mediaFile }) {
301311 const ext = path.extname(absPath);
......@@ -370,40 +380,19 @@ const procHighlightCode: Process = {
370380 },
371381};
372382
373const imageSubsets = [
374 {
375 ext: ".webp",
376 // deno-fmt-disable-line
377 args: [
378 "-lossless",
379 "0",
380 "-compression_level",
381 "6",
382 "-quality",
383 "95",
384 "-method",
385 "6",
386 ],
387 },
388 {
389 ext: ".jxl",
390 args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
391 },
392];
393
394383const procImageSubsets: Process = {
395384 name: "encode image subsets",
396385 include: rules.extsImage,
397386 depends: ["calculate dimensions"],
398 async run({ absPath, mediaFile, stat, spin }) {
387 async run({ absPath, mediaFile, spin }) {
399388 const { width, height } = UNWRAP(mediaFile.parseDimensions());
400 const targetSizes = imageSizes.filter((w) => w < width);
389 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
401390 const baseStatus = spin.text;
402391
403392 using stack = new DisposableStack();
404393 for (const size of targetSizes) {
405394 const { w, h } = resizeDimensions(width, height, size);
406 for (const { ext, args } of imageSubsets) {
395 for (const { ext, args } of transcodeRules.imagePresets) {
407396 spin.text = baseStatus +
408397 ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
409398
......@@ -413,7 +402,7 @@ const procImageSubsets: Process = {
413402 async (out) => {
414403 await fs.mkdir(path.dirname(out));
415404 await fs.rm(out, { force: true });
416 await execFile(ffmpeg!, [
405 await execFile(ffmpegBin!, [
417406 ...ffmpegOptions,
418407 "-i",
419408 absPath,
......@@ -433,81 +422,51 @@ const procImageSubsets: Process = {
433422 },
434423 async undo(mediaFile) {
435424 const { width } = UNWRAP(mediaFile.parseDimensions());
436 const targetSizes = imageSizes.filter((w) => w < width);
425 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
437426 for (const size of targetSizes) {
438 for (const { ext } of imageSubsets) {
427 for (const { ext } of transcodeRules.imagePresets) {
439428 unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
440429 }
441430 }
442431 },
443432};
444433
445type VideoEncodePreset = {
446 id: string;
447 codec: "av1";
448 mbit: number;
449 crf?: number;
450 maxHeight?: number;
451 audioKbit?: number;
452} | {
453 id: string;
454 codec: "vp9";
455 mbit: number;
456 maxHeight?: number;
457 audioKbit?: number;
458};
459// const av1Pass1 = [
460// '-c:v', 'libsvtav1 -b:v 2M -pass 1 -svtav1-params "rc=1:tbr=2000000" -preset 8 -f null NUL
461// ];
462// const av1BaseArgs = [
463// ["-c:v", "libsvtav1"],
464// ["-preset", "3"],
465// ["-c:a", "libopus"],
466// ["-b:a", "192k"],
467// ].flat();
468// const webmBaseArgs = [
469// "-c:v",
470// "libsvtav1",
471// ["-vf",],
472// ];
473const presets: VideoEncodePreset[] = [
474 { id: "av1-max", codec: "av1", mbit: 5 },
475 { id: "av1-hq", codec: "av1", mbit: 2, maxHeight: 900 },
476 { id: "av1-md", codec: "av1", mbit: 0.5, maxHeight: 720 },
477 { id: "av1-lq", codec: "av1", mbit: 0.2, maxHeight: 640 },
478];
479const procVideos = presets.map((preset) => ({
434const procVideos = transcodeRules.videoFormats.map((preset) => ({
480435 name: "encode image subsets",
481436 include: rules.extsImage,
437 enable: ffmpegBin != null,
482438 async run({ absPath, mediaFile, stat, spin }) {
483 await produceAsset(`${mediaFile}/${preset.id}`, (base) => {
484 const root = path.dirname(base);
485 const pass1 = [ffmpeg!, ...ffmpegOptions, "-i", absPath];
486 if (preset.codec === "av1") {
487 pass1.push("-c:v", "libstvav1");
488 pass1.push("-b:v", preset.mbit + "M");
489 if (preset.crf) pass1.push("-crf", preset.crf.toString());
490 pass1.push(
491 "-svtav1-params",
492 ["rc=1", `tbr=${preset.mbit * 1_000_000}`].join(":"),
493 );
494 } else if (preset.codec === "vp9") {
495 pass1.push("-c:v", "libvpx-vp9");
496 } else preset satisfies never;
497
498 if (preset.maxHeight != null) {
499 pass1.push("-vf", `scale=-2:min(${preset.maxHeight}\\,ih)`);
439 await produceAsset(`${mediaFile}/${preset.id}`, async (base) => {
440 base = path.dirname(base);
441 const args = transcodeRules.getVideoArgs(
442 preset,
443 base,
444 ["-i", absPath],
445 );
446 try {
447 const quality = {
448 u: "ultra-high",
449 h: "high",
450 m: "medium",
451 l: "low",
452 d: "data-saving",
453 }[preset.id[1]] ?? preset.id;
454 await ffmpeg.spawn({
455 ffmpeg: ffmpegBin!,
456 title: `${mediaFile.path.slice(1)} (${preset.codec} ${quality})`,
457 args,
458 });
459 return await collectFiles();
460 } catch (err) {
461 for (const file of await collectFiles()) {
462 // TODO: delete assets off disk
463 }
464 throw err;
500465 }
501466
502 const logfile = path.join(preset.id + ".log");
503 pass1.push("-passlogfile", logfile);
504 const pass2 = pass1.slice();
505
506 pass1.push("-f", "null");
507 pass1.push(os.devNull);
508
509 pass2.push("-c:a", "libopus");
510 pass2.push("-b:a", (preset.audioKbit ?? 192) + "k");
467 async function collectFiles(): Promise<string[]> {
468 throw new Error("!");
469 }
511470 });
512471 },
513472} satisfies Process));
......@@ -645,10 +604,11 @@ import * as path from "node:path";
645604import * as child_process from "node:child_process";
646605import * as util from "node:util";
647606import * as crypto from "node:crypto";
648import * as os from "node:os";
649607
650608import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
651609import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
652610import { formatDate } from "@/file-viewer/format.ts";
653611import * as rules from "@/file-viewer/rules.ts";
654612import * as highlight from "@/file-viewer/highlight.ts";
613import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
614import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
src/file-viewer/ffmpeg.ts created+161
......@@ -0,0 +1,161 @@
1// Utilities for spawning ffmpeg and consuming its output as a `Progress`
2// A headless parser is available with `Parse`
3
4export type Line =
5 | { kind: "ignore" }
6 | { kind: "log"; level: "info" | "warn" | "error"; message: string }
7 | {
8 kind: "progress";
9 frame: number;
10 totalFrames: number;
11 speed: string | null;
12 fps: number | null;
13 rest: Record<string, string>;
14 };
15
16export const defaultExtraOptions = [
17 "-hide_banner",
18 "-stats",
19];
20
21export interface SpawnOptions {
22 args: string[];
23 title: string;
24 ffmpeg?: string;
25 progress?: Progress;
26}
27
28export async function spawn(options: SpawnOptions) {
29 const { ffmpeg = "ffmpeg", args, title } = options;
30 const proc = child_process.spawn(ffmpeg, args, {
31 stdio: ["ignore", "inherit", "pipe"],
32 env: { ...process.env, SVT_LOG: "2" },
33 });
34 const parser = new Parse();
35 const bar = options.progress ?? new Progress({ text: title });
36 let running = true;
37 const splitter = readline.createInterface({ input: proc.stderr });
38 splitter.on("line", (line) => {
39 const result = parser.onLine(line);
40 if (result.kind === "ignore") {
41 return;
42 } else if (result.kind === "log") {
43 console[result.level](result.message);
44 } else if (result.kind === "progress") {
45 if (!running) return;
46 const { frame, totalFrames, fps, speed } = result;
47 bar.value = frame;
48 bar.total = totalFrames;
49 const extras = [
50 `${fps} fps`,
51 speed,
52 parser.hlsFile,
53 ].filter(Boolean).join(", ");
54 bar.text = `${title} ${frame}/${totalFrames} ${
55 extras.length > 0 ? `(${extras})` : ""
56 }`;
57 } else result satisfies never;
58 });
59 const [code, signal] = await events.once(proc, "close");
60 if (code !== 0) {
61 const fmt = code ? `code ${code}` : `signal ${signal}`;
62 const e: any = new Error(`ffmpeg failed with ${fmt}`);
63 e.args = [ffmpeg, ...args].join(" ");
64 e.code = code;
65 e.signal = signal;
66 return e;
67 }
68}
69
70export class Parse {
71 parsingStart = true;
72 inIndentedIgnore: null | "out" | "inp" | "other" = null;
73 durationTime = 0;
74 targetFps: number | null = null;
75
76 hlsFile: string | null = null;
77 durationFrames = 0;
78
79 onLine(line: string): Line {
80 line = line.trimEnd();
81 if (/^frame=/.test(line)) {
82 if (this.parsingStart) {
83 this.parsingStart = false;
84 this.durationFrames = Math.ceil(
85 (this.targetFps ?? 25) * this.durationTime,
86 );
87 }
88 const parts = Object.fromEntries(
89 [...line.matchAll(/\b([a-z0-9]+)=\s*([^ ]+)(?= |$)/ig)].map((
90 [, k, v],
91 ) => [k, v]),
92 );
93 const { frame, fps, speed, ...rest } = parts;
94 return {
95 kind: "progress",
96 frame: Number(frame),
97 totalFrames: this.durationFrames,
98 fps: Number(fps),
99 speed,
100 rest,
101 };
102 }
103
104 if (this.parsingStart) {
105 if (this.inIndentedIgnore) {
106 if (line.startsWith(" ") || line.startsWith("\t")) {
107 line = line.trimStart();
108 if (this.inIndentedIgnore === "inp") {
109 const match = line.match(/^Duration: (\d+):(\d+):(\d+\.\d+)/);
110 if (match) {
111 const [h, m, s] = match.slice(1).map((x) => Number(x));
112 this.durationTime = Math.max(
113 this.durationTime,
114 h * 60 * 60 + m * 60 + s,
115 );
116 }
117 if (!this.targetFps) {
118 const match = line.match(/^Stream.*, (\d+) fps/);
119 if (match) this.targetFps = Number(match[1]);
120 }
121 }
122 return { kind: "ignore" };
123 }
124 this.inIndentedIgnore = null;
125 }
126 if (line === "Press [q] to stop, [?] for help") {
127 return { kind: "ignore" };
128 }
129 if (line === "Stream mapping:") {
130 this.inIndentedIgnore = "other";
131 return { kind: "ignore" };
132 }
133 if (line.startsWith("Output #") || line.startsWith("Input #")) {
134 this.inIndentedIgnore = line.slice(0, 3).toLowerCase() as "inp" | "out";
135 return { kind: "ignore" };
136 }
137 }
138
139 const hlsMatch = line.match(/^\[hls @ .*Opening '(.+)' for writing/);
140 if (hlsMatch) {
141 if (!hlsMatch[1].endsWith(".tmp")) {
142 this.hlsFile = path.basename(hlsMatch[1]);
143 }
144 return { kind: "ignore" };
145 }
146
147 let level: Extract<Line, { kind: "log" }>["level"] = "info";
148 if (line.toLowerCase().includes("err")) level = "error";
149 else if (line.toLowerCase().includes("warn")) level = "warn";
150
151 return { kind: "log", level, message: line };
152 }
153}
154
155import * as child_process from "node:child_process";
156import * as readline from "node:readline";
157import * as process from "node:process";
158import events from "node:events";
159import * as path from "node:path";
160import { Progress } from "@paperclover/console/Progress";
161
src/file-viewer/transcode-rules.ts created+154
......@@ -0,0 +1,154 @@
1type VideoEncodePreset = {
2 id: string;
3 codec: "av1";
4 preset: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8;
5 mbitMax?: number;
6 crf: number;
7 maxHeight?: number;
8 audioKbit?: number;
9 depth?: 8 | 10;
10} | {
11 id: string;
12 codec: "vp9";
13 crf: number;
14 mbitMax?: number;
15 maxHeight?: number;
16 audioKbit?: number;
17};
18
19export const videoFormats = [
20 {
21 id: "au", // AV1 Ultra-High
22 codec: "av1",
23 preset: 1,
24 crf: 28,
25 depth: 10,
26 },
27 {
28 id: "vu", // VP9 Ultra-High
29 codec: "vp9",
30 crf: 30,
31 },
32 {
33 id: "ah", // AV1 High
34 preset: 2,
35 codec: "av1",
36 mbitMax: 5,
37 crf: 35,
38 depth: 10,
39 },
40 {
41 id: "am", // AV1 Medium
42 preset: 2,
43 codec: "av1",
44 mbitMax: 2,
45 crf: 40,
46 maxHeight: 900,
47 },
48 {
49 id: "vm", // VP9 Medium
50 codec: "vp9",
51 crf: 35,
52 mbitMax: 4,
53 maxHeight: 1080,
54 },
55 {
56 id: "al", // AV1 Low
57 preset: 2,
58 codec: "av1",
59 mbitMax: 1.25,
60 crf: 40,
61 maxHeight: 600,
62 },
63 {
64 id: "vl", // VP9 Low
65 codec: "vp9",
66 crf: 45,
67 mbitMax: 2,
68 maxHeight: 600,
69 },
70 {
71 id: "ad", // AV1 Data-saving
72 codec: "av1",
73 preset: 1,
74 mbitMax: 0.5,
75 crf: 10,
76 maxHeight: 360,
77 },
78 {
79 id: "vl", // VP9 Low
80 codec: "vp9",
81 crf: 10,
82 mbitMax: 0.75,
83 maxHeight: 360,
84 },
85] satisfies VideoEncodePreset[] as VideoEncodePreset[];
86
87export const imageSizes = [64, 128, 256, 512, 1024, 2048];
88export const imagePresets = [
89 {
90 ext: ".webp",
91 args: [
92 "-lossless",
93 "0",
94 "-compression_level",
95 "6",
96 "-quality",
97 "95",
98 "-method",
99 "6",
100 ],
101 },
102 // TODO: avif
103 {
104 ext: ".jxl",
105 args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
106 },
107];
108
109export function getVideoArgs(preset: VideoEncodePreset, outbase: string, input: string[]) {
110 const cmd = [...input];
111
112 if (preset.codec === "av1") {
113 cmd.push("-c:v", "libsvtav1");
114 cmd.push(
115 "-svtav1-params",
116 [
117 `preset=${preset.preset}`,
118 "keyint=2s",
119 preset.depth && `input-depth=${preset.depth}`, // EncoderBitDepth
120 `crf=${preset.crf}`, // ConstantRateFactor
121 preset.mbitMax && `mbr=${preset.mbitMax}m`, // MaxBitRate
122 "tune=1", // Tune, PSNR
123 "enable-overlays=1", // EnableOverlays
124 "fast-decode=1", // FastDecode
125 "scm=2", // ScreenContentMode, adaptive
126 ].filter(Boolean).join(":"),
127 );
128 } else if (preset.codec === "vp9") {
129 // Not much research has gone into this, since it is only going to be used on old Safari browsers.
130 cmd.push("-c:v", "libvpx-vp9");
131 cmd.push("-crf", String(preset.crf));
132 cmd.push("-b:v", preset.mbitMax ? `${preset.mbitMax * 1000}k` : "0");
133 } else preset satisfies never;
134
135 if (preset.maxHeight != null) {
136 cmd.push("-vf", `scale=-2:min(${preset.maxHeight}\\,ih)`);
137 }
138
139 cmd.push("-c:a", "libopus");
140 cmd.push("-b:a", (preset.audioKbit ?? 192) + "k");
141
142 cmd.push("-y");
143 cmd.push("-f", "hls");
144 cmd.push("-hls_list_size", "0");
145 cmd.push("-hls_segment_type", "fmp4");
146 cmd.push("-hls_time", "2");
147 cmd.push("-hls_allow_cache", "1");
148 cmd.push("-hls_fmp4_init_filename", preset.id + ".mp4");
149 cmd.push(path.join(outbase, preset.id + ".m3u8"));
150
151 return cmd;
152}
153
154import * as path from "node:path";