diff --git a/lib/subprocess/ffmpeg.ts b/lib/subprocess/ffmpeg.ts index d9e526ad1e5ba3ff63aa31cbde1a09fd7077a479..c3ba2d8c233925807efa2860f8b3db049e89136f 100644 --- a/lib/subprocess/ffmpeg.ts +++ b/lib/subprocess/ffmpeg.ts @@ -21,7 +21,7 @@ export interface SpawnOptions { * * ```ts * await ffmpeg.spawn({ - * cmd: ["-i", "hello.mov", "-c:v", "libsvtav1", "hello.mp4"], + * args: ["-i", "hello.mov", "-c:v", "libsvtav1", "hello.mp4"], * progress: progress.start("encode hello.mov"), * }); * ``` diff --git a/src/blog/backend.ts b/src/blog/backend.ts index da6a631ba2e4e6e798dcb169c152716a3564dfad..ad1a55a8eb36200dd981d636f640b587e62c3f41 100644 --- a/src/blog/backend.ts +++ b/src/blog/backend.ts @@ -1,5 +1,9 @@ export const app = new Hono(); +app.get("/blog/webdev/progress-and-api-design", async (c) => { + return assets.serveAsset(c, "/blog/webdev/progress-and-api-design/en", 200); +}); + app.get("/blog/webdev/one-year-next-app-router", async (c) => { return assets.serveAsset(c, "/blog/webdev/one-year-next-app-router/en", 200); }); diff --git a/src/blog/drafts/incrementality.mdx b/src/blog/drafts/incrementality.mdx new file mode 100644 index 0000000000000000000000000000000000000000..de340bcc856ad7009dbdd562eefe261de3947323 --- /dev/null +++ b/src/blog/drafts/incrementality.mdx @@ -0,0 +1,128 @@ +# Incremental Design is why Bun's Dev-Server is Fast + +> While I wrote from scratch and maintained on Bun's hot-reloading development +> server until May 2025, I am not affiliated with Bun or Anthropic. I will be +> discussing the revision of [`DevServer.zig`] with my latest commit. + +[`DevServer.zig`]: https://github.com/oven-sh/bun/blob/67f0c3e016aa479738469adac2b79a1862b88122/src/bake/DevServer.zig + +Time and time again, I've learned that with a long running, interactive +process, the best way to improve it's speed is not hyper-optimizing a routine +or throwing more threads at something. It's finding ways to reuse work that had +already been performed. Sure, adding threads and reducing the time of a +computation helps, but imagine cutting re-compilation time of a hundred +thousand line codebase in half, versus making it always rebuild in under 16 +milliseconds. + +That was the goal with the design of Bun's `DevServer.zig`, the subsystem that +powered bundling, serving, and rebuilding in the `bun ./index.html` command. +And when it got highlighted in the 1.3 release, people noted. My favorite +reaction is during Theo Browne's video on the release with his editor +configured to save on every keypress. The changes arrived within a frame, two, +or sometimes even the same frame the changes appear in the editor window. + +https://youtu.be/dSIgEJSi0rY?t=1664 + +This post will go over general strategies and examples of applying incremental +and resumable design to software, and then we'll also take a deep dive into how +Bun exactly bundles its code. + +TOC + +## General Incremental Design + +## `sitegen`'s Incremental Builder + +## How Bun's DevServer Works + +one rule: every file must be compilable in isolation of every other file. with +this achieved, a thread pool can build each file isolated its own thread, but +more importantly it means that changing a file will guarantee that all +unchanged files are used as is. + +the devserver was implemented mostly as a single file, one spanning 8.5k lines +before i left. large files are honestly slept on; they are extremely easy to +navigate with CMD+F (or `/` in Vim) (and a shame that the code got split up +into many files for the sake of LLMs) + +### the output format + +The output format is tightly coupled with the design. Let's take a look at it: + +```ts +((modules, config) => { + // *insert HMR runtime* +})({ + // Bundled Modules + "index.html": [ [ "app.ts", 0 ], [], [], () => {}, false], + + "app.ts": [ [ "name.ts", 1, "name" ], [], [], (hmr) => { + var [import_name] = hmr.imports; + hmr.updateImport = [(module) => import_name = module]; + + console.info(`Hello, ${import_name.name}`); + }, false], + + "name.ts": [ [], [ "name" ], [], (hmr) => { + hmr.exports = { + name: "clover" + }; + }, false], +}, { + main: "index.html", + bun: "1.3.0", + generation: "7e327d2c", + version: "f621f7ac5ecff08a", + console: false +}) +//# sourceMappingURL=/_bun/client/000000007e327d2c.js.map +``` + +The foundation of this model is a callback to how Webpack bundles looked, at +least a decade ago when I was picking apart webpack bundles trying to +understand what these new magic tools were actually doing. + +- webpack inspiration +- compact representation +- concatenatable +- esm is not incremental +- async modules +- missing imports +- star imports + +### random fun facts + + +``` + /// The conversion logic is completely different for format .internal_bake_dev + /// For CommonJS, all statements are copied `inside_wrapper_suffix` and this returns. + /// + /// For ESM, this function populates all three lists: + /// 1. outside_wrapper_prefix: all import statements, unmodified. + /// 2. inside_wrapper_prefix: a var decl line and a call to `module.retrieve` + /// 3. inside_wrapper_suffix: all non-import statements + /// + /// The imports are rewritten at print time to fit the packed array format + /// that the HMR runtime can decode. This encoding is low on JS objects and + /// indentation. + /// + /// 1 ┃ "module/esm": [ [ + /// ┃ 'module_1', 1, "add", + /// ┃ 'module_2', 2, "mul", "div", + /// ┃ 'module_3', 0, // bare or import star + /// ], [ "default" ], [], (hmr) => { + /// 2 ┃ var [module_1, module_2, module_3] = hmr.imports; + /// ┃ hmr.onUpdate = [ + /// ┃ (module) => (module_1 = module), + /// ┃ (module) => (module_2 = module), + /// ┃ (module) => (module_3 = module), + /// ┃ ]; + /// + /// 3 ┃ console.log("my module", module_1.add(1, module_2.mul(2, 3)); + /// ┃ module.exports = { + /// ┃ default: module_3.something(module_2.div), + /// ┃ }; + /// }, false ], + /// ----- "is the module async?" + fn convertStmtsForChunkForDevServer( + ``` diff --git a/src/blog/pages/webdev/progress-and-api-design/demos/sample-scan.ts b/src/blog/pages/webdev/progress-and-api-design/demos/sample-scan.ts new file mode 100644 index 0000000000000000000000000000000000000000..b38da7beed7f38ce15100023e4fca738b3f2b772 --- /dev/null +++ b/src/blog/pages/webdev/progress-and-api-design/demos/sample-scan.ts @@ -0,0 +1,68 @@ +import * as progress from "lib/progress.ts"; +// these other libraries will not be focused on that much +import * as queue from "lib/queue.ts"; +import * as ffmpeg from "lib/subprocess/ffmpeg.ts"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +export async function mediaScanner( + rootDir: string, + // accept `Ref` into any function you'd like to trace + progress: progress.Ref, +) { + // Automatically called `rootNode.end()` with the `using` syntax. + using rootNode = progress.start(`Scan ${rootDir}`, { total: 1 }); + + // With a `progress.Node`, changing the status is done with setters + rootNode.text = "Scanning..."; // to change the status text + rootNode.value = 0; // to change the number of completed items + rootNode.total = 1; // to change the number of total items + + const completion = Promise.withResolvers(); + // `queue.wrap` returns a function that limits concurrency to + // the number of cpu cores available (also usable as a priority queue) + const recurse = queue.wrap(async (file: string) => { + // Progress Nodes can have children with `.start()` + // Since it is not given a `total`, this won't have a bar. + using fileNode = rootNode.start(file); + + await new Promise((resolve) => setTimeout(resolve, 100)); // demo + + const stat = await fs.stat(file); + if (stat.isDirectory()) { + const children = await fs.readdir(file); + + // Mutation schedules a re-render, batched reasonably. + rootNode.total += children.length; + + for (const child of children) { + recurse(path.join(file, child)); + } + return; + } + + if (file.endsWith(".mov")) { + // in my library, there is also a wrapper for spawning `ffmpeg` with + // automatic progress tracking. we'll track this under the file node. + await ffmpeg.spawn({ + args: ["-i", file, "-c:v", "libx264", file.replace(/\.mov/, ".mp4")], + progress: fileNode.start("encode h.264 mp4"), + }); + // (the ffmpeg helper calls `.end()` for us. + } + + rootNode.value += 1; // increment the progress by one + + if (rootNode.value === rootNode.total) { + completion.resolve(); // all done! + } + }); + + recurse(rootDir); + + await completion.promise; +} + +export async function main() { + await mediaScanner("C:\\media", progress); +} diff --git a/src/blog/pages/webdev/progress-and-api-design/en.mdo b/src/blog/pages/webdev/progress-and-api-design/en.mdo new file mode 100644 index 0000000000000000000000000000000000000000..fb1434b1f7f1a970b3cd39056f5ccc0cd34d1bbd --- /dev/null +++ b/src/blog/pages/webdev/progress-and-api-design/en.mdo @@ -0,0 +1,346 @@ +--- +layout: "#src/blog/tags/blog-layout.marko" +meta: + title: Clover's Progress API, Modular Abstractions, and "Good" API Design + description: yo we meow these + keywords: ["webdev", "software design"] + authors: ["clover caruso"] + embed: + thumbnail: /open-graph/next-js.png + canonical: /blog/webdev/progress-and-api-design +date: "Mar 10th, 2026" +--- + +-- A small demo here + +Great Libraries begin as small helpers for a specialized use case, then +extracted into their own projects because they are deemed useful. When a small +component of one project becomes its own piece of software, it's critical to +design the abstraction to be simple, understandable and modular. + +In this post, we'll take a look thru two of my recent library projects: +[`@clo/lib/progress.ts`][progress] and [`@clo/react-mutation`]. Both of these +libraries are written with great care to their API surface. They are also great +examples since they are mostly written from scratch (Progress depends on +`node:process`, React Mutation depends on React) and are easy to analyze. + +> While this post is going to be focused on web development with TypeScript, +> the overall ideas translate to any language or tool. For example, a carefully +> designed `interface` could be represented in C as a pointer table, or in Rust +> with a `dyn` trait, obviously with proper consideration to the problem. + + + +## Why is a *Progress Bar* Library Exciting? + +Everything in Clover Progress is modular. For an introduction into how it's +actually used, we'll start with instrumenting a little video encoding workflow. + +```tsx diff ++import * as progress from "lib/progress.ts"; + + // these other libraries will not be focused on that much + import * as ffmpeg from "lib/subprocess/ffmpeg.ts"; + import * as queue from "lib/queue.ts"; + import * as fs from "node:fs/promises"; + import * as path from "node:path"; + + export async function mediaScanner( + rootDir: string, ++ // accept `Ref` into any function you'd like to trace ++ progress: progress.Ref, + ) { ++ // Automatically called `rootNode.end()` with the `using` syntax. ++ using rootNode = progress.start(`Scan ${rootDir}`, { total: 1 }); ++ ++ // With a `progress.Node`, changing the status is done with setters ++ rootNode.text = "Scanning..."; // to change the status text ++ rootNode.value = 0; // to change the number of completed items ++ rootNode.total = 1; // to change the number of total items + +- let active = 1; + const completion = Promise.withResolvers(); + + // `queue.wrap` returns a function that limits concurrency to + // the number of cpu cores available (also usable as a priority queue) + const recurse = queue.wrap(async (file: string) => { ++ // Progress Nodes can have children with `.start()` ++ // Since it is not given a `total`, this won't have a bar. ++ using fileNode = rootNode.start(file); + + await new Promise((resolve) => setTimeout(resolve, 100)); // demo + + const stat = await fs.stat(file); + if (stat.isDirectory()) { + const children = await fs.readdir(file); + ++ // Mutation schedules a re-render, batched reasonably. ++ rootNode.total += children.length; +- active += children.length; + + for (const child of children) { + recurse(path.join(file, child)); + } + return; + } + + if (file.endsWith(".mov")) { ++ // in my library, there is also a wrapper for spawning `ffmpeg` with ++ // automatic progress tracking. we'll track this under the file node. + await ffmpeg.spawn({ + args: ["-i", file, "-c:v", "libx264", file.replace(".mov", ".mp4"), "-y"], ++ progress: fileNode.start("encode h.264 mp4"), + }); ++ // (the ffmpeg helper calls `.end()` for us. + } + ++ rootNode.value += 1; // increment the progress by one + ++ // You can use the value and total as regular variables. ++ if (rootNode.value === rootNode.total) { +- if (--active === 0) { + completion.resolve(); // all done! + } + }); + + recurse(rootDir); + + await completion.promise; + } +``` + +To run it, an existing progress node could be passed, or the global `progress` +module also satisfies this interface (`Ref` is simply just anything with the +`start` function). + +```ts +import * as progress from "@clo/lib/progress.ts"; + +await mediaScanner("/Users/clo/media", progress); + +// or compose as a part of a larger program +export function doTheMediaScanningPart(ref: progress.Ref) { + await mediaScanner("/Users/clo/media", ref); + + using _ = ref.start("upload resulting files"); + // ... +} +``` + +The above example is a very, very abridged version of the [`file-scan.ts`] +script that powers my website's [file viewer]. But with just this code, we've +shown a real world use case made better with this simple progress tracing. Watch +as the logs for each `ffmpeg` process are displayed under their respective node, +and then when the process finishes each encoding's logs are grouped together. +This has made it much easier for me to debug errors when they happen, especially +on long encoding sessions when I was first writing the script. + +[`file-scan.ts`]: https://git.paperclover.net/clo/sitegen/src/branch/master/src/file-viewer/bin/file-scan.ts +[file viewer]: /file + + + +I've still been exploring more uses of Clover Progress, but here are some more +little demos. + +-- TODO: make all of these visual demos with asciinema / real demo + +- DB Migrations + - At my work, we used an [early version] of Clover Progress to visualize the + long-running migrations. Top level `console.log`s don't interfere with the + Terminal UI, making it easy to add this to an existing codebase. The + estimation algorithm was added upstream after we kept copy-pasting it + between migrations. +- Developer CLIs + - The presentation of `value`/`total` can be customized, for example + `units: "bytes"`. Progress trees are just amazing for visualizing + parallel or multi-threaded work. +- HTTP Server + - In development, separate trees can be used to show different requests, or + also identify slow routes by visually spotting them in the log. This example + can be tested with the simple HTTP server provided by `@clo/lib/http`. +- Server${'<->'}Browser IPC + - A headless progress instance can be created with `new progress.Root`, and + that root can be serialized into a `ReadableStream` to be decoded and + rendered in a browser. +- AI Sub-agents + - Thinking traces and complex tool calls from concurrent agents are hard to + visualize. This demo isn't really concrete yet, but I'm looking to + optionally integrate Clover Progress into a friend's [AI SDK project]. + +[early version]: https://jsr.io/@clo/console + +## The Most Important Aspect of Library Design + +Before I dive into concepts , I want to share the most important thing about +library design: **you MUST drive library decisions from +real-world testing**, otherwise you have no idea what actually works or not. An +idea may seem great on paper, but with its hidden flaws only revealled after +it's done. + +As one creates more and more libraries, this becomes less of a concern. I'm +able to somewhat correctly predict how an API will feel to use (insert joke +aboout Next.js 16), so I can get pretty far without testing. But even then, the +feedback provided from testing will always shine light on the gaps, especially +when *other* people test it. + +With my library demo out of the way, let's take a look at some fun patterns I've +found: + +## Trivial Interfaces + +`progress` mostly revolves around one interface, `progress.Ref`, a reference +point for reporting progress. Functions that can report live progress take it as +a parameter. + +```ts +import * as progress from "@clo/lib/progress.ts"; +import { delay } from "@clo/lib/async.ts"; + +// (imported as progress.Ref) +interface Ref { + /** + * creates a new trackable unit of work as a child of this one. + * when given an estimate, a progress bar is rendered. + */ + start(text: string, opts?: StartOptions): Node; +} + +async function doInterestingWork(p: progress.Ref) { + const node = p.start("doing some interesting work"); + + const subtask1 = node.start("subtask", { total: 10 }); + const subtask2 = node.start("subtask", { total: 30 }); + + for (let i = 0; i < 30; i += 1) { + subtask1.value += 1; + if (i % 3 === 0) subtask2.value += 1; + await delay(100); + } + + subtask1.end(); + subtask2.end(); + + node.end(); +} +``` + +> To make `progress.Node` easier to use, it aliases `end` to +> `[Symbol.dispose]`, which can be used with the recently stabilized +> [`using` syntax][using]. I'll be doing that for the rest of this post. + +By taking in the `Ref` parameter (that only declares `start` instead of a full +`Node`), it makes this function modular to however the caller wants to report +progress. There are four primary ways to create a valid Ref, and they all flow +extremely naturally when instrumenting code. + +- `progress.Node` implements `start` to create sub-nodes. I could pass + `subtask1` to another function to report its progress within the sub-task. +- The progress module exports `start`, which means the module namespace import + satisfies the interface, eg `doInterestingWork(progress)`. In Node.js, tree + shaking isn't worth worrying about, but in the browser you could also pass + `progress.global` or `{ start: progress.start }` if you really wanted to be sure. +- A headless progress tree created via `new progress.Root()`, which is explained + in the next section. +- `progress.nullNode` returns a no-op `Node` where the `start` function returns + itself, all setters are no-ops. Part of the design of `Node` is that most of the + fields are optional, meaning a no-op `Node` implementation can just ignore the + existence of all of most of its fields. + +## Headless Design + +A pattern I love for many reasons is the *headless design* pattern, where +something connected to global state is built up with. For Clover Progress, it +is possible to construct a `Root` node that does not render to a screen. +Instead, it takes in host APIs and acts as an event emitter. + +```ts +const root = new progress.Root({ + delay, // optionally pass a different timer function + now, // optionally pass a different "now" function +}); +root.on("change", (activeItems: progress.ReadOnlyNode[]) => { + console.log(activeItems); // update the screen +}); + +// root has a start function, so this works with no edits +// to the actual codebase. +mediaScanner("C:\\media", root); +``` + +There's actually a second layer to this. The code used to implement TUI +"widgets", the items in the terminal that persist after console logs, is a more +advanced version of this. In addition to taking timing APIs, it also takes a +handle to the terminal output. By providing a system environment, a few +functions are returned. The most important, `startWidget`, is what the Clover +Progress TUI is made up on. With this abstraction boundary, the progress code +does not need to worry about redrawing the screen, but instead just formatting +the ANSI output. + +```ts +// @clo/lib/log.ts +export function headlessWidgetHost(env: HeadlessWidgetEnv): HeadlessWidgetHost; + +/** {@linkcode widgetHost}'s input takes terminal i/o as well as timing APIs */ +export interface HeadlessWidgetEnv { + /** recieves ANSI escape sequences for interactive data */ + writeInteractive(text: string): void; + /** recieves log content (from `writeLine`) */ + writeOutput(text: string): void; + /** monotonic milliseconds */ + now(): ReturnType; + /** after resolving, `now()` should have increased by the delay time */ + delay: typeof async.delay; + /** called often. */ + getSize(): { columns: number; rows: number }; +} + +/** an implementation of an ANSI-based widget host */ +export interface HeadlessWidgetHost { + /** see the top-level {@linkcode writeLine} function */ + writeLine(text: string): void; + /** see the top-level {@linkcode getDrawLock} function */ + getDrawLock(): ts.Dispose; + /** see the top-level {@linkcode startWidget} function */ + startWidget(widget: Widget): ts.Dispose; + /** stop all widgets and remove all timers. */ + cancel(): void; + /** generic delay function */ + delay?: typeof async.delay; + /** generic now function */ + now?: typeof performance.now; +} +``` + +What I love about this is it means my code doesn't have any dependencies, +except for an *optional* dependency on `node:process` if you use the global +progress node. + +### Testing + +-- theyre just beautiful holy shit + +### Alternate Platforms + +-- talk about browser progress more + +### Serialization + +-- talk about encodeByteStream +-- this use case is still being proven + +## Good Abstractions Should be Hard to Design + +(or, why i don't want to release a billion libraries) + +### Overcooked Design + +-- It's very easy to get carried away, especially in javascript +-- Avoid Unreadable Generics +-- Please do not change the language (cough cough Next) + +### Okay so I might have actually overcooked it + +... diff --git a/src/blog/tags/table-of-contents.css b/src/blog/tags/table-of-contents.css index 440f2d71d0a7eaf3a0dc0d914239e790854ccca6..01e4868828bbc6d36728eb1eceb81616639d89fe 100644 --- a/src/blog/tags/table-of-contents.css +++ b/src/blog/tags/table-of-contents.css @@ -2,6 +2,7 @@ background-color: #0003; border-radius: 8px; padding: 1rem; + margin: 1rem 0; h2 { text-decoration: none;