From 27b17df57b1d672f6fee971407b0e4d9916bbf11 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 27 Sep 2025 18:13:11 -0700 Subject: [PATCH] feat(framework): use progress in sitegen / revive sitegen fixes #30 fixes #32 --- framework/bundle.ts | 22 +++++-- framework/css.ts | 1 + framework/font.ts | 55 +++++++++++------ framework/generate.ts | 94 +++++++++++++++++++++--------- framework/incremental.ts | 66 ++++++++++++--------- lib/log.ts | 5 +- lib/log/progress.ts | 12 ++-- lib/packaging.ts | 7 +-- lib/string.ts | 8 +++ readme.md | 16 +++-- run.js | 2 - src/file-viewer/backend.tsx | 2 +- src/file-viewer/bin/scan3.ts | 7 ++- src/file-viewer/ffmpeg.ts | 5 +- src/file-viewer/transcode-rules.ts | 17 ++++-- 15 files changed, 213 insertions(+), 106 deletions(-) diff --git a/framework/bundle.ts b/framework/bundle.ts index 4bcb159db5479c5889308463700115b49cf0f52b..cc05cef76ad33ac39a515d4e0fe4f92e9713b7ac 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -41,6 +41,8 @@ export async function bundleClientJavaScript( markoViaBuildCache(), ]; + using _ = io.start(`bundle ${entryPoints.length} files`); + const bundle = await esbuild .build({ assetNames: "/asset/[hash]", @@ -108,6 +110,7 @@ export async function bundleClientJavaScript( export type ServerPlatform = "node" | "passthru"; export interface ServerSideOptions { + progress: progress.Ref; entries: string[]; viewItems: sg.FileItem[]; viewRefs: incr.Ref[]; @@ -117,6 +120,7 @@ export interface ServerSideOptions { metaTemplate: meta.Template; } export async function bundleServerJavaScript({ + progress, viewItems, viewRefs, styleMap, @@ -143,6 +147,7 @@ export async function bundleServerJavaScript({ } const wViewSource = incr.work( + progress, async ( _, { viewItems, regenKeys, regenTtls }: { @@ -188,6 +193,7 @@ export async function bundleServerJavaScript({ ); await incr.work( + progress, async (io, { regenKeys, viewItems }) => { io.writeFile( "../ts/view.d.ts", @@ -212,8 +218,11 @@ export async function bundleServerJavaScript({ { regenKeys: Object.keys(regenKeys), viewItems }, ); - const wBundles = entries.map((entry) => - incr.work(async (io, entry) => { + const wBundles = entries.map((entry) => { + const node = progress.start(incr.toRel(entry), { passive: true }); + return incr.work(node, async (io, entry) => { + using _ = io.start("esbuild"); + const pkg = await io.readJson<{ dependencies: Record; }>("package.json"); @@ -313,12 +322,14 @@ export async function bundleServerJavaScript({ } } return fileWithMagicWord; - }, entry) - ); + }, entry); + }); const wProcessed = wBundles.map(async (wBundle) => { if (!(await wBundle)) return; - await incr.work(async (io) => { + await incr.work(wBundle.progress, async (io) => { + using _ = io.start("postprocess"); + // Only the reachable resources need to be read and inserted into the bundle. // This is what Map is for const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle)); @@ -380,3 +391,4 @@ import * as sg from "#sitegen"; import type { PageOrView } from "./generate.ts"; import type * as view from "#sitegen/view"; import * as meta from "#sitegen/meta"; +import * as progress from "lib/log/progress.ts"; diff --git a/framework/css.ts b/framework/css.ts index f52e0a7d23077d78199b4b1f283ac251a7c73437..efe5ba70af10e9b49c6b7be78249661c2459731d 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -62,6 +62,7 @@ export async function bundleCssFiles( dev: boolean; }, ) { + using _ = io.start(styleKey(cssImports, theme)); cssImports = await Promise.all( cssImports.map((file) => io.trackFile("src/" + file)), ); diff --git a/framework/font.ts b/framework/font.ts index 24d4bb98e0305bc322e05b56c7db734648f248c2..2dbb5ed3f99f01d69d3d6a0fa127830b59f8142d 100644 --- a/framework/font.ts +++ b/framework/font.ts @@ -1,14 +1,17 @@ -export async function buildFonts(fonts: sg.Font[]) { +export async function buildFonts(p: progress.Ref, fonts: sg.Font[]) { if (fonts.length === 0) return; + using node = p.start("build fonts"); const temp = path.resolve(".clover/font"); const venv = path.join(temp, "venv"); const bin = path.join(venv, "bin"); if (!fs.existsSync(venv)) { + using _ = node.start("setup python env"); await subprocess.exec("python", ["-m", "venv", venv]); } if (!fs.existsSync(path.join(bin, "pyftsubset"))) { + using _ = node.start("install pyftsubset"); await subprocess.exec(path.join(bin, "pip"), [ "install", "fonttools==4.38.0", @@ -17,19 +20,27 @@ export async function buildFonts(fonts: sg.Font[]) { } const instances = new async.OnceMap(); - async function makeInstance(input: string, vars: sg.FontVars) { - const args = Object.entries(vars).map((x) => { + function getArgs(vars: sg.FontVars) { + return Object.entries(vars).map((x) => { const lower = x[0].toLowerCase(); const k = lower === "wght" || lower === "slnt" ? lower : x[0]; const v = Array.isArray(x[1]) ? x[1].join(":") : x[1]; return `${k}=${v}`; }).sort(); - const hash = crypto + } + + function getHash(input: string, args: string[]) { + return crypto .createHash("sha256") .update(`${input}${args.join(" ")}`) .digest("hex") .slice(0, 16) .toLowerCase(); + } + + async function makeInstance(input: string, vars: sg.FontVars) { + const args = getArgs(vars); + const hash = getHash(input, args); const outfile = path.join(temp, `${hash}${path.extname(input)}`); await instances.get(outfile, async () => { await fs.rm(outfile, { force: true }); @@ -45,32 +56,37 @@ export async function buildFonts(fonts: sg.Font[]) { } await Promise.all(fonts.map((font) => - incr.work(async (io, font) => { + incr.work(node, async (io, font) => { + using node = io.start(font.name); const baseFile = await fetchFont(font.name, font.sources); await Promise.all(font.subsets.map(async (subset) => { - let file = baseFile; - if (subset.vars) { - file = await makeInstance(baseFile, subset.vars); - } + const args = getArgs(subset.vars ?? {}); const unicodes = fontRangesToString( Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], ); const hash = crypto .createHash("sha256") - .update(`${file}${unicodes}`) + .update(`${baseFile}${args.join()}${unicodes}`) .digest("hex") .slice(0, 16) .toLowerCase(); const woff = path.join(temp, hash + ".woff2"); - await subprocess.exec(path.join(bin, "pyftsubset"), [ - file, - "--flavor=woff2", - `--output-file=${woff}`, - ...subset.layoutFeatures - ? [`--layout-features=${subset.layoutFeatures.join(",")}`] - : [], - `--unicodes=${unicodes}`, - ]); + if (!fs.existsSync(woff)) { + using _ = node.start(hash); + let file = baseFile; + if (subset.vars) { + file = await makeInstance(baseFile, subset.vars); + } + await subprocess.exec(path.join(bin, "pyftsubset"), [ + file, + "--flavor=woff2", + `--output-file=${woff}`, + ...subset.layoutFeatures + ? [`--layout-features=${subset.layoutFeatures.join(",")}`] + : [], + `--unicodes=${unicodes}`, + ]); + } await io.writeAsset({ pathname: subset.asset.replace("[ext]", "woff2"), buffer: await fs.readFile(woff), @@ -195,3 +211,4 @@ import * as incr from "./incremental.ts"; import * as crypto from "node:crypto"; import * as async from "#sitegen/async"; import * as error from "#sitegen/error"; +import * as progress from "../lib/log/progress.ts"; diff --git a/framework/generate.ts b/framework/generate.ts index cc84babad77393de76a428f820b6be3634f9a1c2..7e558e72efd51ea09aad279ea1cb1e20d56ccbb8 100644 --- a/framework/generate.ts +++ b/framework/generate.ts @@ -11,44 +11,70 @@ export async function main() { await incr.compile(generate); } -export async function generate() { +export async function generate(p: progress.Ref) { // -- read config and discover files -- - const siteConfig = await incr.work(readManifest); + const siteConfig = await incr.work(p, readManifest); const metaTemplate: meta.Template = { ...siteConfig.meta, base: new URL(siteConfig.meta.base), }; + // -- start font work -- const { staticFiles, scripts, views, pages, - } = await discoverAllFiles(siteConfig); + } = await discoverAllFiles(p, siteConfig); - // TODO: make sure that `static` and `pages` does not overlap + const aliasingInputs = Array.from( + Map.groupBy([...staticFiles, ...pages], (x) => x.id), + ).filter((x) => x[1].length > 1); + if ( + aliasingInputs.length > 0 + ) { + throw new Error( + `source files alias:\n` + + aliasingInputs.map(([id, files]) => + ` - ${id}\n` + files.map((x) => " - " + x.file).join("\n") + ).join("\n"), + ); + } + + const builtFonts = fonts.buildFonts(p, siteConfig.fonts); // TODO: loadMarkoCache - // -- start font work -- - const builtFonts = fonts.buildFonts(siteConfig.fonts); + const pageNode = p.start("generate static pages", {}); + const viewNode = p.start("compile dynamic views", {}); + const staticNode = p.start("compress static asset", {}); // -- perform build-time rendering -- const builtPages = pages.map((item) => - incr.work(preparePage, { item, metaTemplate }) + incr.work( + pageNode.start(item.file, { passive: true }), + preparePage, + { item, metaTemplate }, + ) ); - const builtViews = views.map((item) => incr.work(prepareView, item)); + const builtViews = views.map((item) => + incr.work( + viewNode.start(item.file, { passive: true }), + prepareView, + item, + ) + ); + Promise.allSettled(builtViews).then(() => viewNode.end()); const builtStaticFiles = Promise.all( staticFiles.map((item) => - incr.work( - async (io, { id, file }) => - void (await io.writeAsset({ - pathname: id, - buffer: await io.readFile(file), - })), - item, - ) + incr.work(staticNode, async (io, { id, file }) => { + using _ = io.start(file); + return void (await io.writeAsset({ + pathname: id, + buffer: await io.readFile(file), + })); + }, item) ), - ); + ).then(() => staticNode.end()); const routes = await Promise.all([...builtViews, ...builtPages]); const viewsAndDynPages: incr.Ref[] = [ @@ -57,15 +83,16 @@ export async function generate() { ]; // -- page resources -- - const scriptMap = incr.work(bundle.bundleClientJavaScript, { + const scriptMap = incr.work(p, bundle.bundleClientJavaScript, { clientRefs: routes.flatMap((x) => x.clientRefs), extraPublicScripts: scripts.map((entry) => entry.file), dev: false, }); - const styleMap = prepareInlineCss(routes); + const styleMap = prepareInlineCss(p, routes); // -- backend -- const builtBackend = bundle.bundleServerJavaScript({ + progress: p, entries: siteConfig.backends, platform: "node", styleMap, @@ -79,19 +106,20 @@ export async function generate() { }); // -- assemble page assets -- - const pAssemblePages = builtPages.map((page) => - assembleAndWritePage(page, styleMap, scriptMap) - ); + const pAssemblePages = Promise.all( + builtPages.map((page) => assembleAndWritePage(page, styleMap, scriptMap)), + ).then(() => pageNode.end()); await Promise.all([ builtBackend, builtStaticFiles, - ...pAssemblePages, + pAssemblePages, builtFonts, ]); } export async function readManifest(io: Io) { + using _ = io.start("read site manifest"); const cfg = await io.import("src/site.ts"); return { siteSections: cfg.siteSections.map((section) => ({ @@ -107,12 +135,14 @@ export async function readManifest(io: Io) { } export async function discoverAllFiles( + p: progress.Ref, siteConfig: Awaited>, ) { + using node = p.start("discover files"); return ( await Promise.all( siteConfig.siteSections.map(({ root: sectionRoot }) => - incr.work(scanSiteSection, toAbs(sectionRoot)) + incr.work(node, scanSiteSection, toAbs(sectionRoot)) ), ) ).reduce((acc, next) => ({ @@ -124,6 +154,7 @@ export async function discoverAllFiles( } export async function scanSiteSection(io: Io, sectionRoot: string) { + using _ = io.start(toRel(sectionRoot)); // Static files are compressed and served as-is. // - "{section}/static/*.png" let staticFiles: FileItem[] = []; @@ -274,7 +305,7 @@ export async function preparePage( cssImports, theme: theme ?? null, styleKey, - clientRefs: Array.from(addon[sg.userData.key].scripts), + clientRefs: Array.from(UNWRAP(addon[sg.userData.key]).scripts), } as const; } @@ -303,16 +334,19 @@ export async function prepareView(io: Io, item: sg.FileItem) { export type PreparedView = Awaited>; export function prepareInlineCss( + p: progress.Ref, items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>, ) { + const node = p.start("bundle css"); const map = new Map>(); for (const { styleKey, cssImports, theme } of items) { if (map.has(styleKey)) continue; map.set( styleKey, - incr.work(css.bundleCssFiles, { cssImports, theme, dev: false }), + incr.work(node, css.bundleCssFiles, { cssImports, theme, dev: false }), ); } + Promise.allSettled([...map.values()]).then(() => node.end()); return map; } @@ -323,12 +357,18 @@ export async function assembleAndWritePage( scriptWork: incr.Ref>, ) { const page = await pageWork; + ASSERT("passive" in pageWork.progress); + const pageNode = pageWork.progress; return incr.work( + pageWork.progress, async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => { const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey))); const scriptIds = clientRefs.map(hot.getScriptId); const scriptMap = await io.readWork(scriptWork); + + pageNode.passive = false; + const scripts = scriptIds .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`)) .map((x) => `{${x}}`) @@ -348,6 +388,7 @@ export async function assembleAndWritePage( }, regenerative: !!regenerate, }); + pageNode.passive = true; }, page, ); @@ -362,6 +403,7 @@ import * as meta from "#sitegen/meta"; import * as render from "#engine/render"; import * as sg from "#sitegen"; import type { FileItem } from "#sitegen"; +import * as progress from "../lib/log/progress.ts"; import * as bundle from "./bundle.ts"; import * as css from "./css.ts"; diff --git a/framework/incremental.ts b/framework/incremental.ts index 0c81b886a58087099ec1fcae94c7e84311d55e2c..e5228710c317b0aa98ff40578f91ba9ee10116ba 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -18,6 +18,8 @@ export interface Ref { /** This method is compatible with `await` syntax */ then(resolve: (value: T) => void, reject: (error: unknown) => void): void; get value(): T | null; + /** The progress node passed into the input */ + progress: progress.Node | progress.Ref; } type Job = (io: Io, input: I) => Promise; @@ -29,9 +31,13 @@ type Job = (io: Io, input: I) => Promise; * * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed. */ -export function work(job: Job): Ref; -export function work(job: Job, input: I): Ref; -export function work(job: Job, input: I = null as I): Ref { +export function work(p: progress.Ref, job: Job): Ref; +export function work(p: progress.Ref, job: Job, input: I): Ref; +export function work( + p: progress.Ref, + job: Job, + input?: I, +): Ref { const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1])); const keySource = [source, util.inspect(input)].join(":"); const key = crypto.createHash("sha1").update(keySource).digest("base64url"); @@ -45,15 +51,21 @@ export function work(job: Job, input: I = null as I): Ref { const prev = works.get(key) as Work | null; if (prev) { - return { key, then: (done) => done(prev.value), value: prev.value }; + return { + key, + then: (done) => done(prev.value), + value: prev.value, + progress: p, + }; } async function perform() { - const io = new Io(key); + const io = new Io(key, p); jobs += 1; newKeys += 1; + await timers.setTimeout(200); try { - const value = await job(io, input); + const value = await job(io, input as I); validateSerializable(value, ""); const { reads, writes } = io; works.set(key, { value, affects: [], reads, writes }); @@ -82,27 +94,24 @@ export function work(job: Job, input: I = null as I): Ref { get value() { return (works.get(this.key)?.value as O) ?? null; }, + progress: p, }; } -export async function compile(compiler: () => Promise) { +export async function compile( + compiler: (node: progress.Ref) => Promise, + p: progress.Ref = progress, +) { ASSERT(!running, `Cannot run twice at the same time.`); + const node = p.start("sitegen!"); try { running = true; ASSERT(jobs === 0); const start = performance.now(); - const timerSpinner = new Spinner({ - text: () => - `sitegen! [${((performance.now() - start) / 1000).toFixed(1)}s]`, - fps: 10, - }); - using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() }; - const value = await compiler(); + const value = await compiler(node); ASSERT(jobs === 0); - timerSpinner.text = "incremental flush"; - await flush(start); - timerSpinner.stop(); + await flush(node, start); return { value, watchFiles: new Set(files.keys()), @@ -112,6 +121,7 @@ export async function compile(compiler: () => Promise) { newAssets: !Array.from(seenWrites).some((x) => x.startsWith("a:")), }; } finally { + node.end(); running = false; newKeys = 0; seenWrites.clear(); @@ -119,7 +129,8 @@ export async function compile(compiler: () => Promise) { } } -export async function flush(start: number) { +export async function flush(progress: progress.Node, start: number) { + using _ = progress.start("incremental flush"); // Trim const detachedFiles = new Set(); const referencedAssets = new Set(); @@ -163,11 +174,11 @@ export async function flush(start: number) { const serialized = msgpackr.pack(serialize()); await fs.writeMkdir(".clover/incr.state", serialized); const time = (performance.now() - start).toFixed(0); - console.success(`sitegen! in ${time} ms`); - console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`); - console.writeLine(` - ${assets.size} static assets`); - console.writeLine( - ` - dist: ${formatSize(dist)},` + + progress.log.info( + `sitegen! in ${time} ms\n` + + ` - ${works.size} keys (${works.size - newKeys} cached)\n` + + ` - ${assets.size} static assets\n` + + ` - dist: ${formatSize(dist)},` + ` incremental: ${formatSize(serialized.byteLength)}`, ); } @@ -453,10 +464,14 @@ export function getAssetManifest() { * - Outputs written with Io are deleted when abandoned. */ export class Io { - constructor(public key: string) {} + constructor(public key: string, private node: progress.Ref) {} reads: Reads = { files: new Set(), works: new Set() }; writes: Writes = { files: new Set(), assets: new Map() }; + start(text: string, options?: progress.StartOptions) { + return this.node.start(text, options); + } + #trackFs(file: string) { const resolved = toAbs(file); const key = toRel(resolved); @@ -720,8 +735,7 @@ import * as util from "node:util"; import * as crypto from "node:crypto"; import * as zlib from "node:zlib"; -import * as console from "@paperclover/console"; -import { Spinner } from "@paperclover/console/Spinner"; +import * as progress from "../lib/log/progress.ts"; import { formatSize } from "@/file-viewer/format.ts"; import * as msgpackr from "msgpackr"; diff --git a/lib/log.ts b/lib/log.ts index 4351d7a58c6bfb31f0381d096b3a53513b3c1916..47b079870e79311c6b14077b7d1a89f3aadb8bc6 100644 --- a/lib/log.ts +++ b/lib/log.ts @@ -66,10 +66,7 @@ export interface Widget { * may get called more often than the specified `fps`. * supports color codes but not ansi cursor movements. */ - format( - now: ReturnType, - focus: boolean, - ): string | null; + format(now: ReturnType): string | null; /** 'null' to never update (use 'onChange'). defaults to 12 fps */ fps?: number | null; /** Subscribe to manual widget updates. Call `rerender` when needed. */ diff --git a/lib/log/progress.ts b/lib/log/progress.ts index 242da8e9bce01c4dcc7cf05e632bc47a5fe151e7..ede2e98b211a812b05af6dad962e9e9511f9bec2 100644 --- a/lib/log/progress.ts +++ b/lib/log/progress.ts @@ -233,7 +233,7 @@ function newNode( function formatProgress(now: number, list: Internal[]): string { let out = ""; for (const top of list) { - if (top.passive && top.children.length === 0) continue; + if (top.passive && !hasChildren(top)) continue; out += renderMainLine(top, now, 0) + "\n"; out += renderChildren(top, now, []); } @@ -283,15 +283,19 @@ function renderMainLine(state: Internal, now: number, depth: number) { (state.value > 0 ? `[${state.value}] ` : "") + state.text; } +function hasChildren(states: Internal): boolean { + return states.children.some((x) => !x.passive || hasChildren(x)); +} + function renderChildren(state: Internal, now: number, depth: boolean[]) { - let maxHeight = 25; // TODO: + let maxHeight = 50; // TODO: let truncated = 0; let out = ""; - const { children } = state; + let { children } = state; + children = children.filter((x) => !x.passive || hasChildren(x)); if (children.length === 0) return ""; for (let i = 0, { length } = children; i < length; i += 1) { const child = UNWRAP(children[i]); - if (child.passive && child.children.length === 0) continue; let item = ""; const left = depth.map((x) => x ? box.line : " ").join(""); item += left + (i === length - 1 && !truncated ? box.langle : box.tee); diff --git a/lib/packaging.ts b/lib/packaging.ts index 6b642282f1876724551d79e6af83d7aee92ebbb6..9e0bc826f1ea953feaeba00f148714872b33d8c3 100644 --- a/lib/packaging.ts +++ b/lib/packaging.ts @@ -33,12 +33,11 @@ export async function transpileForNpm( packageJson: Record, tsToJsdoc?: TsToJsdoc, ) { + const libName = await Promise.resolve("ts-to-jsdoc"); try { - // @ts-expect-error optional dependency - tsToJsdoc ??= await import("ts-to-jsdoc") as TsToJsdoc; + tsToJsdoc ??= await import(libName) as TsToJsdoc; } catch { - // @ts-expect-error unsupported import specifier - tsToJsdoc ??= await import("npm:ts-to-jsdoc@2.4.0") as TsToJsdoc; + tsToJsdoc ??= await import("npm:" + libName + "@2.4.0") as TsToJsdoc; } const { transpileFile } = tsToJsdoc; diff --git a/lib/string.ts b/lib/string.ts index 8905daa419dd1a00b4c69c6481a3a9053666eb4a..be749150f1aad50652067d4c553f00fe7411d901 100644 --- a/lib/string.ts +++ b/lib/string.ts @@ -155,4 +155,12 @@ function segmentWidth(segment: string) { ); } +export function escapeShellArgument(s: string): string { + if (s.length === 0) return "''"; + if (/^[a-zA-Z0-9_\-.\/@,+=%:]+$/.test(s) && s[0] !== ":") return s; + if (!s.includes("'")) return `'${s}'`; + if (!/["$`\\]/.test(s)) return `"${s}"`; + return `'${s.replace(/'/g, "'\\''")}'`; +} + import { UNWRAP } from "lib/assert.ts"; diff --git a/readme.md b/readme.md index 8a23afc637f80d96feb0b45d1084020e6a367a03..ab8687cb566dd35ddad250f3fcac538e9a6a5733 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,9 @@ -# clover sitegen framework +# clover sitegen framework and typescript library this repository contains clover's "sitegen" framework, which is a set of tools -that assist building websites. these tools power . +that assist building websites. additionally, this is the home of her typescript +library `lib`, containing many high quality standalone sub-projects. all these +tools power . - **HTML ("Server Side Rendering") engine written from scratch.** (~500 lines) - A more practical JSX runtime (`class` instead of `className`, built-in @@ -18,12 +20,14 @@ that assist building websites. these tools power . - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs checks when the files change. For example, changing a component re-tests only pages that use that component and re-lints only the changed file. -- **Integrated libraries for building complex, content heavy web sites.** - Static asset serving with automatic ETag and build-time compression. - Dynamic pages. Static pages can be regenerated (and cached) at runtime. - - Databases with a minimal SQLite wrapper. - - Meta Tags and Open Graph generation. - Font subsetting to reduce page bandwidth. +- **General purpose typescript libraries.** (See [`./lib`](./lib/readme.md)) + - [Logging](./lib/log.ts) and [progress](./lib/log/progress.ts) indicators. + - [Priority Queue](./lib/queue.ts) with automatic reschedulability. + - [Meta Tags](./framework/lib/meta.ts) and Open Graph generation. + - Databases with a minimal SQLite wrapper. - **Built on the stable and battle-tested Node.js runtime.** none of these tools are complex or revolutionary. rather, this project is the @@ -32,7 +36,7 @@ example on how other over-complicate other frameworks. [Marko]: https://next.markojs.com -Included is `src`, which contains `paperclover.net`. Website highlights: +Included is `src`, which contains `paperclover.net`'s source code. Highlights: - TODO: flashy homepage. - [Question/Answer board, custom markdown parser and components][q+a]. diff --git a/run.js b/run.js index a421ea4cb2b01e5ac07c18fca9b9f9ebd24cc4e8..a78f57f33964cf781b0df3bf81f75d7d33e3537c 100644 --- a/run.js +++ b/run.js @@ -53,8 +53,6 @@ process.on("uncaughtException", (error) => { process.exit(1); }); -hot.load("node:fs").readFileSync(undefined); - // Load with hooks if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) { if (process.argv.length == 2) { diff --git a/src/file-viewer/backend.tsx b/src/file-viewer/backend.tsx index 9f85f18c2771c100b7bd863978c7f07ba7550b7d..42af23d8114adc839e93b0a877a1f25e99ffd88a 100644 --- a/src/file-viewer/backend.tsx +++ b/src/file-viewer/backend.tsx @@ -126,7 +126,7 @@ app.get("/file/*", async (c, next) => { rangeHeader = null; } - const found = await cache.read(filePath + const found = await cache.read(filePath); // foundFile = await cache.read(filePath, encoding, { // start: rangeHeader ? rangeHeader[0] : 0, // end: rangeHeader ? rangeHeader[1] : Infinity, diff --git a/src/file-viewer/bin/scan3.ts b/src/file-viewer/bin/scan3.ts index 7aeb411c18e2bb7d039c7f0649bd1b4e9f0b4341..4f1c0f9f5b872d38560053734218fe905a1328f8 100644 --- a/src/file-viewer/bin/scan3.ts +++ b/src/file-viewer/bin/scan3.ts @@ -22,6 +22,7 @@ export async function main() { const walkQueue = new queue.PriorityQueue(10); const dirsNode = progress.start("Walk Tree", { estimate: 1 }); + dirsNode.passive = true; const fileNode = progress.start("Process File", { estimate: 0 }); fileNode.sortChildren = (a, b) => { const ac = a.children.length > 0 ? 1 : 0; @@ -260,7 +261,11 @@ export async function main() { } // Add the root & recursively iterate! - scanDirectory(Path.resolve(root)); + const rootPath = Path.resolve(root); + if (!rootPath.ifExistsSync()) { + throw new Error(`file store ${rootPath} is not mounted`); + } + scanDirectory(rootPath); // await qList.done(); // await qMeta.done(); // await qProcess.done(); diff --git a/src/file-viewer/ffmpeg.ts b/src/file-viewer/ffmpeg.ts index 5d53fd05db2b6f9927bf09ab38a5e061e3a6fd56..fe639b72b4969cb0c827e3dc2f37cc07b4e44732 100644 --- a/src/file-viewer/ffmpeg.ts +++ b/src/file-viewer/ffmpeg.ts @@ -61,7 +61,7 @@ export async function spawn(options: SpawnOptions) { if (code !== 0) { const fmt = code ? `code ${code}` : `signal ${signal}`; const e: any = new Error(`ffmpeg failed with ${fmt}`); - e.args = [ffmpeg, ...args].join(" "); + e.args = [ffmpeg, ...args].map(string.escapeShellArgument).join(" "); e.code = code; e.signal = signal; throw e; @@ -155,7 +155,7 @@ export class Parse { } } - const hlsMatch = line.match(/^\[hls @ .*Opening '(.+)' for writing/); + const hlsMatch = line.match(/.*Opening '(.+)' for writing/); if (hlsMatch) { if (!hlsMatch[1].endsWith(".tmp")) { this.hlsFile = path.basename(hlsMatch[1]); @@ -179,3 +179,4 @@ import events from "node:events"; import * as path from "node:path"; import * as progress from "../../lib/log/progress.ts"; import type { Path } from "#sitegen/path"; +import * as string from "lib/string.ts"; diff --git a/src/file-viewer/transcode-rules.ts b/src/file-viewer/transcode-rules.ts index 0692406f76ab271519128f1a7b39822678623e99..292472b5d8959e9fc5f822c212c28dacfc17bfcc 100644 --- a/src/file-viewer/transcode-rules.ts +++ b/src/file-viewer/transcode-rules.ts @@ -220,12 +220,17 @@ export function getMpegDashArgs( cmd.push("-use_template", "1"); // build adaptation sets string - const videoIndices = videos.map((_, i) => i).join(","); - const audioIndices = audios.map((_, i) => videos.length + i).join(","); - cmd.push( - "-adaptation_sets", - `id=0,streams=${videoIndices} id=1,streams=${audioIndices}`, - ); + const videoAdaptationSets = videos.map((_, i) => `id=${i},streams=${i}`) + .join(" "); + let adaptationSets = videoAdaptationSets; + + if (audios.length > 0) { + const audioId = videos.length; + const audioIndices = audios.map((_, i) => videos.length + i).join(","); + adaptationSets += ` id=${audioId},streams=${audioIndices}`; + } + + cmd.push("-adaptation_sets", adaptationSets); cmd.push("-y"); cmd.push(outDir.join("dash.mpd").toString()); -- 2.54.0