diff --git a/framework/backend/entry-node.ts b/framework/backend/entry-node.ts index 0a540c7d3b4640e8257c2d84b2e3883809673002..75f80bf1df5abff3fb0cb728f85d6403bde23f22 100644 --- a/framework/backend/entry-node.ts +++ b/framework/backend/entry-node.ts @@ -1,5 +1,3 @@ -import "#debug"; - const app = require(globalThis.CLOVER_SERVER_ENTRY ?? process.argv[2]); const protocol = "http"; diff --git a/framework/backend/entry-passthru.ts b/framework/backend/entry-passthru.ts deleted file mode 100644 index 74468a052e8b77f669a03b149ef985f3f5da78cd..0000000000000000000000000000000000000000 --- a/framework/backend/entry-passthru.ts +++ /dev/null @@ -1,3 +0,0 @@ -export default app; - -import app from "#backend"; diff --git a/framework/bundle.ts b/framework/bundle.ts index 5c4bcfa82dd8b6b06d9f64874ee582179d95a96e..3a95c5ac440b1f88d031de7a96b1d3bf94abe898 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -65,7 +65,7 @@ export async function bundleClientJavaScript( format: "esm", jsx: "automatic", jsxDev: dev, - jsxImportSource: "#engine", + jsxImportSource: "#jsx", logLevel: "silent", metafile: true, minify: !dev, @@ -75,7 +75,6 @@ export async function bundleClientJavaScript( external: ["/js/scripts/vendor/dash.js", "/js/scripts/vendor/hls.js"], define: { ASSERT: "console.assert", - MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText), }, }) .catch((err: any) => { @@ -296,10 +295,9 @@ export async function bundleServerJavaScript({ write: false, metafile: true, jsx: "automatic", - jsxImportSource: "#engine", + jsxImportSource: "#jsx", jsxDev: false, define: { - MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText), "globalThis.CLOVER_SERVER_ENTRY": JSON.stringify(entry), }, external: Object.keys(pkg.dependencies).filter( @@ -398,10 +396,10 @@ import { } from "./esbuild-support.ts"; import { Io, toAbs } from "./incremental.ts"; import * as fs from "#sitegen/fs"; -import * as mime from "#sitegen/mime"; import * as incr from "./incremental.ts"; import * as sg from "#sitegen"; import type { PageOrView } from "./generate.ts"; import type * as view from "#sitegen/view"; -import * as meta from "#sitegen/meta"; +import * as meta from "lib/render/meta.ts"; import * as progress from "lib/log/progress.ts"; +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/framework/css.ts b/framework/css.ts index efe5ba70af10e9b49c6b7be78249661c2459731d..32a8c0babecad6cc0a713568966c0cc7e0c21959 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -107,7 +107,7 @@ export async function bundleCssFiles( throw new AggregateError(warnings, "CSS Build Failed"); } if (outputFiles.length > 1) throw new Error("Too many output files"); - return outputFiles[0].text; + return UNWRAP(outputFiles[0]).text; } import * as esbuild from "esbuild"; @@ -116,3 +116,4 @@ import * as hot from "./hot.ts"; import * as path from "node:path"; import { virtualFiles } from "./esbuild-support.ts"; import type { Io } from "./incremental.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/framework/debug.safe.ts b/framework/debug.safe.ts deleted file mode 100644 index 6807d2951d56acf3559b7aa2dc6878fef39bec23..0000000000000000000000000000000000000000 --- a/framework/debug.safe.ts +++ /dev/null @@ -1,18 +0,0 @@ -process.on("uncaughtException", (error) => { - log.error(error); - process.exit(1); -}); -import * as log from "lib/log.ts"; - -globalThis.UNWRAP = (t, ...args) => { - if (t == null) { - throw new Error( - args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")", - ); - } - return t; -}; -globalThis.ASSERT = assert.ok; - -import * as util from "node:util"; -import * as assert from "node:assert"; diff --git a/framework/engine/jsx-runtime.ts b/framework/engine/jsx-runtime.ts deleted file mode 100644 index f21a6b77e0ff9a875c33a6031aff2a53d03968c2..0000000000000000000000000000000000000000 --- a/framework/engine/jsx-runtime.ts +++ /dev/null @@ -1,54 +0,0 @@ -export const Fragment = ({ children }: { children: render.Node[] }) => children; - -export function jsx( - type: string | render.Component, - props: Record, -): render.Element { - if (typeof type !== "function" && typeof type !== "string") { - throw new Error("Invalid component type: " + render.inspect(type)); - } - return [render.kElement, type, props]; -} - -export function jsxDEV( - type: string | render.Component, - props: Record, - // Unused with the clover engine - _key: string, - // Unused with the clover engine - _isStaticChildren: boolean, - source: render.SrcLoc, -): render.Element { - const { fileName, lineNumber, columnNumber } = source; - - // Assert the component type is valid to render. - if (typeof type !== "function" && typeof type !== "string") { - throw new Error( - `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` + - render.inspect(type) + - ". Clover SSR element must be a function or string", - ); - } - - // Construct an `ssr.Element` - return [render.kElement, type, props, "", source]; -} - -// jsxs -export { jsx as jsxs }; - -declare global { - namespace JSX { - interface IntrinsicElements { - [name: string]: Record; - } - interface ElementChildrenAttribute { - children: Node; - } - type Element = render.Element; - type ElementType = keyof IntrinsicElements | render.Component; - type ElementClass = ReturnType; - } -} - -import * as render from "#engine/render"; diff --git a/framework/engine/marko-runtime.ts b/framework/engine/marko-runtime.ts index beb038ad467c32635fc7dcaf2bb756ceee8a077c..46458f0db5072063c6e39c0d9b83932f94409b7b 100644 --- a/framework/engine/marko-runtime.ts +++ b/framework/engine/marko-runtime.ts @@ -2,11 +2,6 @@ // To use, replace the "marko/html" import with this file. export * from "#marko/html"; -interface BodyContentObject { - [x: PropertyKey]: unknown; - content: ServerRenderer; -} - export const _template: typeof marko._template = ( templateId, renderer, @@ -52,7 +47,7 @@ export const _dynamic_tag: typeof marko._dynamic_tag = ( } const r = render.current ?? (marko.$global().clover as render.State); if (!r) throw new Error("No Clover Render Active"); - const subRender = render.init(r.async !== -1, r.addon); + const subRender = render.init(r.async !== -1, r.context); const resolved = render.resolveNode( subRender, render.element( @@ -133,7 +128,6 @@ interface Async { isAsync: boolean; } -import * as render from "#engine/render"; -import type { ServerRenderer } from "marko/html/template"; import type { Template } from "marko/common/types"; +import * as render from "lib/render.ts"; import * as marko from "#marko/html"; diff --git a/framework/engine/render.test.tsx b/framework/engine/render.test.tsx deleted file mode 100644 index f22be74279dd889c4bb224a64380c1c42a15d816..0000000000000000000000000000000000000000 --- a/framework/engine/render.test.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import { test } from "node:test"; -import * as render from "#engine/render"; - -test("sanity", (t) => t.assert.equal(render.sync("gm <3").text, "gm <3")); -test("simple tree", (t) => - t.assert.equal( - render.sync( -
-

hello world

-

haha

- {1}|{0}|{true}|{false}|{null}|{undefined}| -
, - ).text, - '

hello world

haha

1|0|||||
', - )); -test("unescaped/escaped html", (t) => - t.assert.equal( - render.sync( -
- {render.raw("")} - {"\"&'`<>"} -
, - ).text, - "
"&'`<>
", - )); -test("clsx built-in", (t) => - t.assert.equal( - render.sync( - <> - - - - - - - , - ).text, - '', - )); diff --git a/framework/engine/render.ts b/framework/engine/render.ts deleted file mode 100644 index 8fa81556fd16cdd0a329bdfe4a46ebc6a12504d4..0000000000000000000000000000000000000000 --- a/framework/engine/render.ts +++ /dev/null @@ -1,355 +0,0 @@ -// Clover's Rendering Engine is the backbone of her website generator. It -// converts objects and components (functions returning 'Node') into HTML. The -// engine is simple and self-contained, with integrations for JSX and Marko -// (which can interop with each-other) are provided next to this file. -// -// Add-ons to the rendering engine can provide opaque data, And retrieve it -// within component calls with 'getAddonData'. For example, 'sitegen' uses this -// to track needed client scripts without introducing patches to the engine. -// -// This module is not production ready -export let current: State | null = null; -export function setCurrent(r: State | null) { - current = r ?? null; -} - -/* Convert a UI description into a string synchronously. */ -export function sync(node: Node, addon: A = {} as A) { - const state = init(false, addon); - const resolved = resolveNode(state, node); - return { text: stringifyNode(resolved), addon }; -} - -/* Convert a UI description into a string asynchronously. */ -export function async(node: Node, addon: A = {} as A) { - const state = init(true, addon); - const resolved = resolveNode(state, node); - if (state.async === 0) { - return Promise.resolve({ text: stringifyNode(resolved), addon }); - } - const { resolve, reject, promise } = Promise.withResolvers>(); - state.asyncDone = () => { - const rejections = state.rejections; - if (!rejections) return resolve({ text: stringifyNode(resolved), addon }); - if (rejections.length === 1) return reject(rejections[0]); - return reject(new AggregateError(rejections)); - }; - return promise; -} - -export type Addons = Record; -export interface Result { - text: string; - addon: A; -} - -export function userData(def: () => T) { - const k: unique symbol = Symbol(); - return { - key: k, - get: () => ((UNWRAP(current).addon[k] as T) ??= def() as T), - set: (value: T) => void (UNWRAP(current).addon[k] = value), - }; -} - -/** Inline HTML into a render without escaping it */ -export function raw(node: ResolvedNode): Raw { - return [kRaw, node]; -} -export function element(type: Element[1], props: Element[2] = {}): Element { - return [kElement, type, props]; -} -export function resolvedElement( - type: ResolvedElement[1], - props: ResolvedElement[2], - children: ResolvedElement[3], -): ResolvedElement { - return [kElement, type, props, children]; -} - -export interface State { - /** - * Set to '-1' if rendering synchronously - * Number of async promises the render is waiting on. - */ - async: number | -1; - asyncDone: null | (() => void); - /** When components reject, those are logged here */ - rejections: unknown[] | null; - /** Add-ons to the rendering engine store state here */ - addon: Addons; -} - -export const kElement = Symbol("Element"); -export const kRaw = Symbol("Raw"); - -/** Node represents a webpage that can be 'rendered' into HTML. */ -export type Node = - | number - | string // Escape HTML - | Node[] // Concat - | Element // Stringify - | Raw // Insert - | Promise // Await - // Ignore - | (undefined | null | boolean); -export type Element = [ - tag: typeof kElement, - type: string | Component, - props: Record, - _?: "", // children - source?: SrcLoc, -]; -export type Raw = [tag: typeof kRaw, html: ResolvedNode]; -/** Components must return a value; 'undefined' is prohibited here - * to avoid functions that are missing a return statement. */ -export type Component = (props: Record) => Exclude; -/** Emitted by JSX runtime */ -export interface SrcLoc { - fileName: string; - lineNumber: number; - columnNumber: number; -} - -/** - * Resolution narrows the type 'Node' into 'ResolvedNode'. Async trees are - * marked in 'State'. This operation performs everything besides the final - * string concatenation. This function is agnostic across async/sync modes. - */ -export function resolveNode(r: State, node: unknown): ResolvedNode { - if (!node && node !== 0) return ""; // falsy, non numeric - if (typeof node !== "object") { - if (node === true) return ""; // booleans are ignored - if (typeof node === "string") return escapeHtmlContent(node); - if (typeof node === "number") return String(node); // no escaping ever - if (typeof node === "symbol" && node.toString() === kElement.toString()) { - throw new Error(`There are two instances of Clover SSR loaded!`); - } - throw new Error(`Cannot render ${inspect(node)} to HTML`); - } - if (node instanceof Promise) { - if (r.async === -1) { - throw new Error(`Asynchronous rendering is not supported here.`); - } - const placeholder: InsertionPoint = [null]; - r.async += 1; - node - .then((result) => void (placeholder[0] = resolveNode(r, result))) - // Intentionally catching errors in `resolveNode` - .catch((e) => (r.rejections ??= []).push(e)) - .finally(() => { - if (--r.async == 0) { - if (r.asyncDone == null) throw new Error("r.asyncDone == null"); - r.asyncDone(); - r.asyncDone = null; - } - }); - // This lie is checked with an assertion in `renderNode` - return placeholder as [ResolvedNode]; - } - if (!Array.isArray(node)) { - throw new Error(`Invalid node type: ${inspect(node)}`); - } - const type = node[0]; - if (type === kElement) { - const { 1: tag, 2: props } = node; - if (typeof tag === "function") { - current = r; - try { - return resolveNode(r, tag(props)); - } catch (e) { - const { 4: src } = node; - if (e && typeof e === "object") (e as { src?: string }).src ??= src; - throw e; - } finally { - current = null; - } - } - if (typeof tag !== "string") throw new Error("Unexpected " + inspect(type)); - const children = props?.children; - if (children) return [kElement, tag, props, resolveNode(r, children)]; - return node; - } - if (type === kRaw) return node[1]; - return node.map((elem) => resolveNode(r, elem)); -} - -export type ResolvedNode = - | ResolvedNode[] // Concat - | ResolvedElement // Render - | string; // Raw HTML -export type ResolvedElement = [ - tag: typeof kElement, - type: string, - props: Record, - children: ResolvedNode, -]; -/** - * Async rendering is done by creating an array of one item, - * which is already a valid 'Node', but the element is written - * once the data is available. The 'Render' contains a count - * of how many async jobs are left. - */ -export type InsertionPoint = [null | ResolvedNode]; - -/** - * Convert 'ResolvedNode' into HTML text. This operation happens after all - * async work is settled. The HTML is emitted as concisely as possible. - */ -export function stringifyNode(node: ResolvedNode): string { - if (typeof node === "string") return node; - ASSERT(node, "Unresolved Render Node"); - const type = node[0]; - if (type === kElement) { - return stringifyElement(node as ResolvedElement); - } - node = node as ResolvedNode[]; // TS cannot infer. - let out = type ? stringifyNode(type) : ""; - let len = node.length; - for (let i = 1; i < len; i++) { - const elem = node[i]; - if (elem) out += stringifyNode(elem); - } - return out; -} -function stringifyElement(element: ResolvedElement) { - const { 1: tag, 2: props, 3: children } = element; - let out = "<" + tag + " "; - for (const prop in props) { - const value = props[prop]; - if (!value || typeof value === "function") continue; - let attr; - switch (prop) { - default: - attr = `${prop}=${quoteIfNeeded(escapeAttribute(String(value)))}`; - break; - case "className": // Legacy React Compat - case "class": { - const str = clsx(value as ClsxInput); - if (!str) continue; - attr = `class=${quoteIfNeeded(escapeAttribute(str))}`; - break; - } - case "style": { - const str = typeof value === "string" - ? value - : Object.entries(value).map((kv) => kv.join(":")).join(";"); - if (!str) continue; - attr = `style=${quoteIfNeeded(escapeAttribute(str))}`; - break; - } - case "htmlFor": - throw new Error("Do not use the `htmlFor` attribute. Use `for`"); - case "dangerouslySetInnerHTML": - throw new Error( - "Do not use the `dangerouslySetInnerHTML` attribute. Use `render.raw()`", - ); - // Do not process these - case "children": - case "ref": - case "key": - continue; - } - if (!out.endsWith('"')) out += " "; - out += attr; - } - out = out.trimEnd() + ">"; - if (children) out += stringifyNode(children); - if ( - tag !== "br" && - tag !== "img" && - tag !== "input" && - tag !== "meta" && - tag !== "link" && - tag !== "hr" - ) { - out += ``; - } - return out; -} -export function stringifyStyleAttribute(style: Record) { - let out = ``; - for (const styleName in style) { - if (out) out += ";"; - out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${ - escapeAttribute( - String(style[styleName]), - ) - }`; - } - return "style=" + quoteIfNeeded(out); -} -export function quoteIfNeeded(text: string) { - if (text.match(/["/> ]/)) return '"' + text + '"'; - return text; -} - -// -- utility functions -- - -export function init(allowAsync: boolean, addon: Addons): State { - return { - async: allowAsync ? 0 : -1, - rejections: null, - asyncDone: null, - addon, - }; -} - -export type ClsxInput = string | Record | ClsxInput[]; -export function clsx(mix: ClsxInput) { - var k, - y, - str = ""; - if (typeof mix === "string") { - return mix; - } else if (typeof mix === "object") { - if (Array.isArray(mix)) { - for (k = 0; k < mix.length; k++) { - if (mix[k] && (y = clsx(mix[k]))) { - str && (str += " "); - str += y; - } - } - } else { - for (k in mix) { - if (mix[k]) { - str && (str += " "); - str += k; - } - } - } - } - return str; -} - -export const escapeHtmlContent = (unsafeText: string) => - String(unsafeText) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """); -// TODO: combine into one function which decides if an attribute needs quotes -// and escapes it correctly depending on the context. -const escapeAttribute = (unsafeText: string) => - String(unsafeText) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """); -/** @deprecated */ -export const escapeHtml = (unsafeText: string) => - String(unsafeText) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """) - .replace(/'/g, "'") - .replace(/`/g, "`"); - -export function inspect(object: unknown) { - try { - return require("node:util").inspect(object); - } catch { - return typeof object; - } -} diff --git a/framework/engine/suspense.test.tsx b/framework/engine/suspense.test.tsx deleted file mode 100644 index 1750c5e7f514b8587023b689eef860ccb0b1f97b..0000000000000000000000000000000000000000 --- a/framework/engine/suspense.test.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import { test } from "node:test"; -import { renderStreaming, Suspense } from "./suspense.ts"; - -test("sanity", async (t) => { - let resolve: () => void = null!; - - // @ts-expect-error - async function AsyncComponent() { - await new Promise((done) => resolve = done); - return ; - } - - const example = ( -
-

app shell

- - - -
(c) 2025
-
- ); - - const iterator = renderStreaming(example); - const assertContinue = (actual: unknown, value: unknown) => - t.assert.deepEqual(actual, { done: false, value }); - - assertContinue( - await iterator.next(), - "", - ); - t.assert.ok(resolve !== null), resolve(); - assertContinue( - await iterator.next(), - "", - ); - t.assert.deepEqual( - await iterator.next(), - { done: true, value: {} }, - ); -}); diff --git a/framework/engine/suspense.ts b/framework/engine/suspense.ts deleted file mode 100644 index 7ed7ccefccca7e107f05609f09e26aeacac4f266..0000000000000000000000000000000000000000 --- a/framework/engine/suspense.ts +++ /dev/null @@ -1,102 +0,0 @@ -// This file implements out-of-order HTML streaming, mimicking the React -// Suspense API. To use, place Suspense around an expensive async component -// and render the page with 'renderStreaming'. -// -// Implementation of this article: -// https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/ -// -// I would link to an article from Next.js or React, but their examples -// are too verbose and not informative to what they actually do. -const userData = render.userData(() => null); - -interface SuspenseProps { - children: render.Node; - fallback?: render.Node; -} - -interface State { - nested: boolean; - nextId: number; - completed: number; - pushChunk(name: string, node: render.ResolvedNode): void; -} - -export function Suspense({ children, fallback }: SuspenseProps): render.Node { - const state = userData.get(); - if (!state) return children; - if (state.nested) throw new Error(" cannot be nested"); - const parent = UNWRAP(render.current); - const r = render.initRender(true, { [userData.key]: { nested: true } }); - const resolved = render.resolveNode(r, children); - if (r.async == 0) return render.raw(resolved); - const name = "suspended_" + ++state.nextId; - state.nested = true; - const ip: [render.ResolvedNode] = [ - render.resolvedElement( - "slot", - { name }, - fallback ? render.resolveNode(parent, fallback) : "", - ), - ]; - state.nested = false; - r.asyncDone = () => { - const rejections = r.rejections; - if (rejections && rejections.length > 0) throw new Error("TODO"); - state.pushChunk?.(name, ip[0] = resolved); - }; - return render.raw(ip); -} - -// TODO: add a User-Agent parameter, which is used to determine if a -// fallback path must be used. -// - Before ~2024 needs to use a JS implementation. -// - IE should probably bail out entirely. -export async function* renderStreaming< - T extends render.Addons = Record, ->(node: render.Node, addon: T = {} as T) { - const { - text: begin, - addon: { [userData.key]: state, ...addonOutput }, - } = await render.async(node, { - ...addon, - [userData.key]: { - nested: false, - nextId: 0, - completed: 0, - pushChunk: () => {}, - } satisfies State as State, - }); - if (state.nextId === 0) { - yield begin; - return addonOutput as unknown as T; - } - let resolve: (() => void) | null = null; - let chunks: string[] = []; - state.pushChunk = (slot, node) => { - while (node.length === 1 && Array.isArray(node)) node = node[0]; - if (node[0] === render.kElement) { - (node as render.ResolvedElement)[2].slot = slot; - } else { - node = [ - render.kElement, - "clover-suspense", - { - style: "display:contents", - slot, - }, - node, - ]; - } - chunks.push(render.stringifyNode(node)); - resolve?.(); - }; - yield ``; - do { - await new Promise((done) => (resolve = done)); - yield* chunks; - chunks = []; - } while (state.nextId < state.completed); - return addonOutput as unknown as T; -} - -import * as render from "#engine/render"; diff --git a/framework/font.ts b/framework/font.ts index 2dbb5ed3f99f01d69d3d6a0fa127830b59f8142d..e886a96b9ada4301e54a1fdf78fd223b72759b12 100644 --- a/framework/font.ts +++ b/framework/font.ts @@ -206,9 +206,9 @@ import * as fs from "#sitegen/fs"; import * as os from "node:os"; import * as path from "node:path"; import * as sg from "#sitegen"; -import * as subprocess from "#sitegen/subprocess"; +import * as subprocess from "lib/subprocess.ts"; import * as incr from "./incremental.ts"; import * as crypto from "node:crypto"; -import * as async from "#sitegen/async"; -import * as error from "#sitegen/error"; +import * as async from "lib/async.ts"; +import * as error from "lib/error.ts"; import * as progress from "../lib/log/progress.ts"; diff --git a/framework/generate.ts b/framework/generate.ts index 66d530a2a7bf93a4893de64dd65a6cb6de663958..4fa203076575b1aee1bf40ab0631de5ad420f6db 100644 --- a/framework/generate.ts +++ b/framework/generate.ts @@ -280,11 +280,11 @@ export async function preparePage( if (layout?.default) { page = render.element(layout.default, { children: page }); } - const bodyPromise = render.async(page, { - [sg.userData.key]: sg.initRender(), - }); - const [{ text, addon }, renderedMeta] = await Promise.all([ + const sitegen = sg.context.init(); + const bodyPromise = render.async(page, [sitegen]); + + const [{ text }, renderedMeta] = await Promise.all([ bodyPromise, renderedMetaPromise, ]); @@ -305,7 +305,7 @@ export async function preparePage( cssImports, theme: theme ?? null, styleKey, - clientRefs: Array.from(UNWRAP(addon[sg.userData.key]).scripts), + clientRefs: Array.from(sitegen.scripts), } as const; } @@ -397,8 +397,8 @@ export type PageOrView = PreparedPage | PreparedView; import * as path from "node:path"; import * as fs from "#sitegen/fs"; -import * as meta from "#sitegen/meta"; -import * as render from "#engine/render"; +import * as meta from "lib/render/meta.ts"; +import * as render from "lib/render.ts"; import * as sg from "#sitegen"; import type { FileItem } from "#sitegen"; import * as progress from "../lib/log/progress.ts"; @@ -409,3 +409,4 @@ import * as fonts from "./font.ts"; import * as hot from "./hot.ts"; import * as incr from "./incremental.ts"; import { Io } from "./incremental.ts"; +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/framework/global.d.ts b/framework/global.d.ts index 46244b99b9cfefc5d48016ce89e9a302ef1d2d06..5a6fa147977ec0e33b8ff7c25e99158b9b1fc0ac 100644 --- a/framework/global.d.ts +++ b/framework/global.d.ts @@ -1,5 +1,2 @@ -declare function UNWRAP(value: T | null | undefined, ...log: unknown[]): T; -declare function ASSERT(value: unknown, message?: string): asserts value; - type Timer = ReturnType; type Awaitable = T | Promise; diff --git a/framework/hot.ts b/framework/hot.ts index 18fb3f8dfebfda9ba512c3296d918c2a93515eb0..324b86ff3d226041a25697f36e3a02cd9b5e3743 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -149,7 +149,7 @@ export function loadEsbuildCode( format: "cjs", target: "esnext", jsx: "automatic", - jsxImportSource: "#engine", + jsxImportSource: "#jsx", jsxDev: true, sourcefile: filepath, sourcemap: "inline", @@ -174,7 +174,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) { function loadMdx(module: NodeJS.Module, filepath: string) { const input = fs.readFileSync(filepath); - const out = mdx.compileSync(input, { jsxImportSource: "#engine" }).value; + const out = mdx.compileSync(input, { jsxImportSource: "#jsx" }).value; const src = typeof out === "string" ? out : Buffer.from(out).toString("utf8"); return loadEsbuildCode(module, filepath, src); } @@ -241,7 +241,10 @@ export function resolveFrom(src: string, dest: string) { try { return createRequire(src).resolve(dest); } catch (err: any) { - if (err.code === "MODULE_NOT_FOUND" && err.requireStack.length <= 1) { + if ( + err.code === "MODULE_NOT_FOUND" && err.requireStack && + err.requireStack.length <= 1 + ) { err.message = err.message.split("\n")[0] + " from '" + src + "'"; } throw err; @@ -262,7 +265,7 @@ export function resolveClientRefs( // This match finds a call to 'import ... from "#sitegen"' const importMatch = code.match(importRegExp); if (!importMatch) return { code, refs: [] }; - const items = importMatch[1]; + const items = UNWRAP(importMatch[1]); let identifier = ""; if (items.startsWith("{")) { const clauseMatch = items.match(getSitegenAddScriptRegExp); @@ -322,3 +325,4 @@ import { createRequire } from "node:module"; import * as mdx from "@mdx-js/mdx"; import * as self from "./hot.ts"; import { Buffer } from "node:buffer"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/framework/incremental.ts b/framework/incremental.ts index 58fbba1fd402b254b3f068138b51a648f3437551..086640bde51f8b593dfc3f5621acbcc64d0c7706 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -495,11 +495,13 @@ export class Io { } return resolved; } - async readFile(file: string) { - return fs.readFile(await this.trackFile(file)); + async readFile(file: string): Promise; + async readFile(file: string, encoding: "utf-8"): Promise; + async readFile(file: string, encoding?: "utf-8") { + return fs.readFile(await this.trackFile(file), encoding); } async readJson(file: string) { - return JSON.parse(await this.readFile(file)) as T; + return JSON.parse(await this.readFile(file, "utf-8")) as T; } async readDir(dir: string) { const { key, resolved } = this.#trackFs(dir); @@ -585,7 +587,7 @@ export class Io { const headers = new Headers(asset.headers ?? {}); const hash = crypto.createHash("sha1").update(buffer).digest("hex"); if (!headers.has("Content-Type")) { - headers.set("Content-Type", mime.contentTypeFor(asset.pathname)); + headers.set("Content-Type", mime.for(asset.pathname)); } headers.set("etag", JSON.stringify(hash)); this.writes.assets.set(asset.pathname, { @@ -726,7 +728,7 @@ const zstdCompress = util.promisify(zlib.zstdCompress); import * as hot from "./hot.ts"; import * as fs from "#sitegen/fs"; -import * as mime from "#sitegen/mime"; +import * as mime from "lib/mime.ts"; import * as manifest from "#sitegen/assets"; import * as path from "node:path"; @@ -739,3 +741,4 @@ import * as stack from "lib/log/stack.ts"; import { formatSize } from "@/file-viewer/format.ts"; import * as msgpackr from "msgpackr"; +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/framework/lib/assets.ts b/framework/lib/assets.ts index d093d0af90cf209c53ca504e4fd84dfaac4df773..94e658ae2a9b75bb2db8238b92acfc26120b05d0 100644 --- a/framework/lib/assets.ts +++ b/framework/lib/assets.ts @@ -286,3 +286,4 @@ import * as crypto from "node:crypto"; import * as zlib from "node:zlib"; import * as util from "node:util"; import * as log from "lib/log.ts"; +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/framework/lib/async.ts b/framework/lib/async.ts deleted file mode 100644 index 9a8c4e4e030938251a02511add1ca449e8c51276..0000000000000000000000000000000000000000 --- a/framework/lib/async.ts +++ /dev/null @@ -1,218 +0,0 @@ -const five_minutes = 5 * 60 * 1000; - -/** @deprecated use OnceMap2 */ -export class OnceMap { - private ongoing = new Map>(); - - get(key: string, compute: () => Promise) { - if (this.ongoing.has(key)) { - return this.ongoing.get(key)!; - } - - const result = compute(); - this.ongoing.set(key, result); - return result; - } -} - -interface ARCEValue { - value: T; - [Symbol.dispose]: () => void; -} - -export function RefCountedExpirable( - init: () => Promise, - deinit: (value: T) => void, - expire: number = five_minutes, -): () => Promise> { - let refs = 0; - let item: ARCEValue | null = null; - let loading: Promise> | null = null; - let timer: ReturnType | null = null; - - function deref() { - ASSERT(item !== null); - if (--refs !== 0) return; - ASSERT(timer === null); - timer = setTimeout(() => { - ASSERT(refs === 0); - ASSERT(loading === null); - ASSERT(item !== null); - deinit(item.value); - item = null; - timer = null; - }, expire); - } - - return async function () { - if (timer !== null) { - clearTimeout(timer); - timer = null; - } - if (item !== null) { - refs++; - return item; - } - if (loading !== null) { - refs++; - return loading; - } - const p = Promise.withResolvers>(); - loading = p.promise; - try { - const value = await init(); - item = { value, [Symbol.dispose]: deref }; - refs++; - p.resolve(item); - return item; - } catch (e) { - p.reject(e); - throw e; - } finally { - loading = null; - } - }; -} - -export function once(fn: () => Promise): () => Promise { - let result: T | Promise | null = null; - return async () => { - if (result) return result; - try { - result = await (result = fn()); - } catch (err) { - result = null; - throw err; - } - return result; - }; -} - -export class Watch { - #value: T; - #observers = new Set>(); - constructor(value: T) { - this.#value = value; - } - - on(cb: WatchCallback) { - this.#observers.add(cb); - return () => void this.#observers.delete(cb); - } - once(cb: WatchCallback) { - const once = (n: T, p: T) => (cb(n, p), this.#observers.delete(once)); - this.#observers.add(once); - } - - next(): Promise { - return new Promise((resolve) => this.once(resolve)); - } - - until(condition: (v: T) => boolean): Promise { - if (condition(this.#value)) return Promise.resolve(this.#value); - return new Promise((resolve) => { - function check(next: T) { - if (condition(next)) resolve(next), release(); - } - const release = this.on(check); - }); - } - - get value() { - return this.#value; - } - set value(next: T) { - const prev = this.#value; - if (prev !== next) { - this.#value = next; - this.#observers.forEach((cb) => cb(next, prev)); - } - } -} -type WatchCallback = (next: T, prev: T) => void; - -/** - * When two requests with the same args come in at the same time, the response - * from them is shared. After returning, the data is no longer stored. - */ -export class DedupeConcurrent { - pending = new Map>(); - constructor( - public fn: (...args: Args) => Promise, - public keyFn: (args: Args) => unknown = JSON.stringify, - ) {} - - isPending(...args: Args) { - return this.pending.has(this.keyFn(args)); - } - - /** Unbound to allow easily exporting this callback */ - get = (...args: Args): Promise => { - const k = this.keyFn(args); - let promise = this.pending.get(k); - if (promise) return promise; - promise = this.fn(...args); - this.pending.set(k, promise); - void promise.finally(() => void this.pending.delete(k)); - return promise; - }; -} - -/** - * When two requests with the same args come, the result of the first is memoized. - */ -export class OnceMap2 { - cache = new Map>(); - constructor( - public fn: (...args: Args) => Promise, - public keyFn: (args: Args) => unknown = JSON.stringify, - ) {} - - has(...args: Args) { - return this.cache.has(this.keyFn(args)); - } - - /** Unbound to allow easily exporting this callback */ - getOrRun = (...args: Args): Promise => { - const k = this.keyFn(args); - let promise = this.cache.get(k); - if (promise) return promise; - promise = this.fn(...args); - this.cache.set(k, promise); - return promise; - }; -} - -export class PromiseAggregator { - failures: unknown[] = []; - promises = new Set>(); - - push(p: Promise) { - p = p.then( - () => { - this.promises.delete(p); - }, - (err) => { - this.failures.push(err); - }, - ); - this.promises.add(p); - } - - async all() { - while (this.promises.size > 0) { - const wait = [...this.promises]; - this.promises.clear(); - await Promise.all(wait); - } - if (this.failures.length > 0) { - const agg = new AggregateError(this.failures); - this.failures = []; - throw agg; - } - } - - [Symbol.asyncDispose]() { - return this.all(); - } -} diff --git a/framework/lib/error.ts b/framework/lib/error.ts deleted file mode 100644 index 816249161d61c9a1f9e32454ef38a718c2f966f0..0000000000000000000000000000000000000000 --- a/framework/lib/error.ts +++ /dev/null @@ -1,47 +0,0 @@ -declare global { - interface Error { - /* Extra fields on errors are extremely helpful for debugging. */ - [metadata: string]: unknown; - } -} - -/** Retrieve an error message from any value */ -export function message(error: unknown): string { - const message = (error as { message: unknown })?.message ?? error; - try { - return typeof message === "string" ? message : JSON.stringify(message); - } catch {} - try { - return String(message); - } catch {} - return `Could not stringify error message ${typeof message}`; -} - -/** Retrieve an error-like object from any value. Useful to check fields. */ -export function obj(error: unknown): { message: string } & Partial { - if (error instanceof Error) return error; - return { - message: message(error), - ...(typeof error === "object" && error), - }; -} - -/* Node.js error codes are strings */ -export function code(error: unknown): NodeErrorCode | null { - const code = (error as { code: unknown })?.code; - return typeof code === "string" ? code : null; -} - -/** Attach extra fields and throw */ -export function rethrowWithMetadata( - err: unknown, - meta: Record, -): never { - const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/ - Object.assign(error, meta); - throw error; -} - -export type NodeErrorCode = - | keyof typeof import("node:os").constants.errno - | (string & {}); diff --git a/framework/lib/lru.test.ts b/framework/lib/lru.test.ts deleted file mode 100644 index 0667d7bac1a957ae29a4a431fe61a15e4069a204..0000000000000000000000000000000000000000 --- a/framework/lib/lru.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { test } from "node:test"; -import { Lru } from "./lru.ts"; - -test("lru basic", (t) => { - const lru = new Lru({ capacity: 3 }); - lru.set("a", 1); - lru.set("b", 2); - lru.set("c", 3); - t.assert.deepEqual(lru.serialize(), [ - ["c", 3, 1], - ["b", 2, 1], - ["a", 1, 1], - ]); - lru.set("d", 4); - t.assert.deepEqual(lru.serialize(), [ - ["d", 4, 1], - ["c", 3, 1], - ["b", 2, 1], - ]); - lru.set("c", 5); - lru.get("b"); - lru.set("e", 6); - t.assert.deepEqual(lru.serialize(), [ - ["e", 6, 1], - ["b", 2, 1], - ["c", 5, 1], - ]); - lru.capacity = 10; - lru.set("a", 0); - lru.set("z", 1); - lru.set("x", 1); - lru.set("w", 1); - t.assert.deepEqual(lru.serialize(), [ - ["w", 1, 1], - ["x", 1, 1], - ["z", 1, 1], - ["a", 0, 1], - ["e", 6, 1], - ["b", 2, 1], - ["c", 5, 1], - ]); - lru.delete("a"); - t.assert.deepEqual(lru.serialize(), [ - ["w", 1, 1], - ["x", 1, 1], - ["z", 1, 1], - ["e", 6, 1], - ["b", 2, 1], - ["c", 5, 1], - ]); - lru.capacity = 2; - t.assert.deepEqual(lru.serialize(), [ - ["w", 1, 1], - ["x", 1, 1], - ]); - lru.ensureUnusedCapacity(1); - t.assert.deepEqual(lru.serialize(), [ - ["w", 1, 1], - ]); - lru.ensureUnusedCapacity(1.2); - t.assert.deepEqual(lru.serialize(), []); - lru.ensureUnusedCapacity(2); -}); -test("lru sized elements", (t) => { - const lru = new Lru({ capacity: 10, sizeFn: (v) => v }); - lru.set("a", 3); - lru.set("b", 3); - lru.set("c", 3); - t.assert.deepEqual(lru.serialize(), [ - ["c", 3, 3], - ["b", 3, 3], - ["a", 3, 3], - ]); - // hitting exact capacity - lru.set("d", 1); - lru.set("d", 1); - t.assert.deepEqual(lru.serialize(), [ - ["d", 1, 1], - ["c", 3, 3], - ["b", 3, 3], - ["a", 3, 3], - ]); - lru.set("e", 1); - t.assert.deepEqual(lru.serialize(), [ - ["e", 1, 1], - ["d", 1, 1], - ["c", 3, 3], - ["b", 3, 3], - ]); - lru.clear(); - lru.set("a", 3); - lru.set("b", 3); - lru.set("c", 3); - lru.set("d", 1); - lru.get("a"); - lru.get("b"); - lru.get("c"); - lru.set("e", 1); - t.assert.deepEqual(lru.serialize(), [ - ["e", 1, 1], - ["c", 3, 3], - ["b", 3, 3], - ["a", 3, 3], - ]); -}); -test("lru throw on exceeding capacity", (t) => { - const lru = new Lru({ capacity: 1, sizeFn: (v) => v }); - lru.set("e", 1); - lru.set("e", 1); - t.assert.throws(() => lru.set("e", 2), RangeError); - t.assert.throws(() => lru.set("d", 2), RangeError); - lru.set("d", 1); -}); -test("locked items cannot be evicted", (t) => { - const lru = new Lru({ capacity: 3, sizeFn: (v) => v }); - lru.set("e", 1); - lru.set("d", 1); - lru.set("f", 1); - - const unlock = lru.lock("e"); - lru.ensureUnusedCapacity(2); - t.assert.deepEqual(lru.serialize(), [ - ["e", 1, 1], - ]); - - unlock(); - - lru.ensureUnusedCapacity(3); - - t.assert.deepEqual(lru.serialize(), []); -}); -test("throws on evicting a locked item", (t) => { - const lru = new Lru({ capacity: 3, sizeFn: (v) => v }); - lru.set("e", 1); - void lru.lock("e"); - t.assert.throws(() => lru.ensureUnusedCapacity(3), RangeError); -}); diff --git a/framework/lib/lru.ts b/framework/lib/lru.ts deleted file mode 100644 index f58d2e9ed78af6f1608e052258ff3b998bfba55d..0000000000000000000000000000000000000000 --- a/framework/lib/lru.ts +++ /dev/null @@ -1,259 +0,0 @@ -interface Options { - capacity: number; - /** - * Decide how many units of size each entry takes. - * For example, accessing `.byteLength` of a Buffer. - */ - sizeFn?: (value: V, key: K) => number; - /** Called when `set` or `ensureUnusedCapacity` removes items. */ - evict?: (value: V, key: K) => void; - /** Called when anything removes items. */ - delete?: (value: V, key: K) => void; -} - -/** Trivial Least-recently-used cache data structure. */ -export class Lru extends Map { - #order = new Map>(); - /** The most recently used */ - #head: K | none = none; - /** The least recently used */ - #tail: K | none = none; - #used: number = 0; - #locked: number = 0; - #capacity: number; - #sizeFn: (value: V, key: K) => number; - #deleteFn: null | ((value: V, key: K) => void); - #evictFn: null | ((value: V, key: K) => void); - - constructor(opts: Options) { - super(); - const { capacity, sizeFn, delete: deleteFn, evict } = opts; - this.#capacity = capacity; - this.#sizeFn = sizeFn ?? (() => 1); - this.#deleteFn = deleteFn ?? null; - this.#evictFn = evict ?? null; - } - - override clear() { - super.clear(); - this.#order.clear(); - this.#head = none; - this.#tail = none; - this.#used = 0; - } - override delete(key: K) { - const entry = this.#evict(key); - if (!entry) return false; - this.#used -= entry[0]; - ASSERT(this.#order.delete(key)); - ASSERT(super.delete(key)); - return true; - } - override get(key: K): V | undefined { - if (this.#head !== key) { - const entry = this.#evict(key); - if (!entry) return undefined; - entry[1] = none, entry[2] = this.#head; - UNWRAP(this.#order.get(this.#head as K))[1] = key; - this.#head = key; - } - return super.get(key) as V; - } - override set(key: K, value: V) { - const size = this.#sizeFn(value, key); - { - const uc = this.unlockedCapacity; - if (size > uc) { - const label = uc === this.#capacity ? " " : " unlocked "; - const msg = `Object of size ${size} exceeds${label}capacity ${uc}`; - throw new RangeError(msg); - } - } - - const entry = this.#evict(key); - if (entry) { - // relocate - this.#used -= entry[0]; - const head = this.#head; - ASSERT(head !== key); - entry[0] = size, entry[1] = none, entry[2] = this.#head; - if (head !== none) UNWRAP(this.#order.get(this.#head as K))[1] = key; - if (this.#tail === none) this.#tail = key; - } else { - // insertion - const { size: mapSize } = this; - const order: Order = [size, none, mapSize > 0 ? this.#head : none, 0]; - this.#order.set(key, order); - if (mapSize === 0) this.#tail = key; - else if (this.#head !== none) { - UNWRAP(this.#order.get(this.#head as K))[1] = key; - } - } - - this.#head = key; - this.#used += size; - try { - this.ensureUnusedCapacity(0); - } catch (err) { - console.error(err); - ASSERT(false, "capacity is already verified"); - } - super.set(key, value); - return this; - } - - /** Prevent an item from being automatically evicted */ - lock(key: K): Lock { - const order = this.#order.get(key); - if (!order) throw new Error(`Key ${key} not in Lru`); - const count = order[3] += 1; - if (count === 1) this.#locked += order[0]; - const unlock = () => { - const count = order![3] -= 1; - if (count === 0) this.#locked -= order[0]; - }; - unlock[Symbol.dispose] = unlock; - return unlock; - } - - /** Evict items until there are `unused` capacity slots. */ - ensureUnusedCapacity(unused: number) { - const uc = this.unlockedCapacity; - if (unused > uc) { - const label = uc === this.#capacity ? "capacity" : "unlocked capacity"; - const msg = `Can't ensure ${unused} unused capacity with ${label} ${uc}`; - throw new RangeError(msg); - } - const needed = this.#used - this.#capacity + unused; - if (needed <= 0) return; - let remain = needed; - const evictFn = this.#evictFn; - const deleteFn = this.#deleteFn; - const order = this.#order; - let key = this.#tail; - loop: { - let hadLocks = false; - while (key !== none) { - const entry = UNWRAP(order.get(key)); - if (entry[3] > 0) { - key = entry[1]; - hadLocks = true; - continue; - } - ASSERT(this.#evict(key) === entry); - if (evictFn) evictFn(super.get(key) as V, key); - if (deleteFn) deleteFn(super.get(key) as V, key); - ASSERT(this.#order.delete(key)); - ASSERT(super.delete(key)); - remain -= entry[0]; - if (remain <= 0) break loop; - key = entry[1]; - } - ASSERT(hadLocks); - const msg = `Too many locked objects, cannot free ${unused} capacity`; - throw new RangeError(msg); - } - this.#used -= needed - remain; - } - - /** Unlink an entry without removing it. */ - #evict(key: K) { - const entry = this.#order.get(key); - if (!entry) return null; - const [, before, after] = entry; - if (before !== none) { - UNWRAP(this.#order.get(before as K))[2] = after; - } else this.#head = after; - if (after !== none) { - UNWRAP(this.#order.get(after as K))[1] = before; - } else this.#tail = before; - return entry; - } - - get capacity() { - return this.#capacity; - } - set capacity(capacity: number) { - this.#capacity = capacity; - this.ensureUnusedCapacity(0); - } - get remaining() { - return this.#capacity - this.#used; - } - get unlockedCapacity() { - return this.#capacity - this.#locked; - } - get used() { - return this.#used; - } - - serialize(): Serialized; - serialize(o: SerializeOptions): Serialized; - serialize( - opts: SerializeOptions = {}, - ): Serialized { - const { map = (k, v) => [k as unknown as OK, v as unknown as OV] } = opts; - const result: Serialized = []; - const order = this.#order; - let head = this.#head; - while (head !== none) { - const entry = UNWRAP(order.get(head), head); - const kv = map(head, super.get(head) as V); - head = entry[2]; - if (!kv) continue; - const [key, value] = kv; - result.push([key, value, entry[0]]); - } - return result; - } - revive( - entries: Serialized, - opts: SerializeOptions = {}, - ) { - const { map = (k, v) => [k as unknown as K, v as unknown as V] } = opts; - const order = this.#order; - let prev: Order | null = null; - let pk!: K; - for (let i = entries.length - 1; i > 0; i -= 1) { - const [ik, iv, size] = UNWRAP(entries[i]); - const kv = map(ik, iv); - if (!kv) continue; - const [k, v] = kv; - super.set(k, v); - const current: Order = [size, none, prev ? pk : none, 0]; - if (prev) prev[1] = k; - order.set(k, prev = current); - pk = k; - } - } -} - -/** The first item is the most recently used. */ -export type Serialized = Array<[K, V, number]>; - -interface SerializeOptions { - /** null = omit in serialization/serialization */ - map?: (key: IK, value: IV) => [OK, OV] | null; -} - -type Order = [ - /** Number of units computed at insertion time. */ - size: number, - /** Towards most recently used */ - before: K | none, - /** Towards least recently used */ - after: K | none, - /** Locking prevents automatic eviction */ - locks: number, -]; - -interface Lock extends Disposable { - (): void; -} - -export declare namespace Lru { - export type { Lock, Serialized, SerializeOptions }; -} - -const none: unique symbol = Symbol(); -type none = typeof none; diff --git a/framework/lib/meta.ts b/framework/lib/meta.ts deleted file mode 100644 index da20a51429068b2b88472738ac8ab00fed3aec6e..0000000000000000000000000000000000000000 --- a/framework/lib/meta.ts +++ /dev/null @@ -1,256 +0,0 @@ -export { renderMetadata as render }; -export interface Meta { - /** Required for all pages. `{content}` */ - title: string; - /** Recommended for all pages. `` */ - description?: string | null; - /** Automatically added for static renders from the 'pages' folders. */ - canonical?: string | null; - /** Add ``. Object keys are interpretted as - * mime types if they contain a slash, otherwise seen as an alternative language. */ - alternates?: Alternate[] | Record; - - /** Automatically generate both OpenGraph and Twitter meta tags */ - embed?: AutoEmbed | null; - /** Add a robots tag for `noindex` and `nofollow` */ - denyRobots?: boolean | null; - /** Add 'og:*' meta tags */ - openGraph?: OpenGraph | null; - /** Add 'twitter:*' meta tags */ - twitter?: Twitter | null; - /** * Refer to an oEmbed file. See https://oembed.com - * TODO: support passing the oEmbed file directly here. */ - oEmbed?: string; - /** - * '#sitegen/meta' intentionally excludes a lot of exotic tags. - * Add these manually using JSX syntax: - * - * extra: [ - * , - * ], - */ - extra?: render.Node; - - /** Adds `` */ - authors?: string[]; - /** Credit sitegen by setting this to true aka "clover sitegen 3; paperclover.net/sitegen" */ - generator?: string | true | null; - /** Adds `` */ - keywords?: string[]; - /** URL to a manifest; https://developer.mozilla.org/en-US/docs/Web/Manifest */ - manifest?: string | null; - /** Adds `` */ - publisher?: string | null; - /** https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/meta/name/referrer */ - referrer?: Referrer | null; - /** Adds `` */ - themeColor?: string | { dark: string; light: string } | null; - /** Defaults to `width=device-width, initial-scale=1.0` for mobile compatibility. */ - viewport?: string; -} -export type Alternate = { type: string; url: string } | { - lang: string; - url: string; -}; -export interface AutoEmbed { - /* Defaults to the page title. */ - title?: string | null; - /* Defaults to the page description. */ - description?: string | null; - /* Provide to add an embed image. */ - thumbnail?: string | null; - /** @default "banner", which applies twitter:card = "summary_large_image" */ - thumbnailSize?: "banner" | "icon"; - /* Ignored if not passed */ - siteTitle?: string | null; -} -/** See https://ogp.me for extra rules. */ -export interface OpenGraph { - /** The title of your object as it should appear within the graph */ - title?: string; - /** A one to two sentence description of your object. */ - description?: string | null; - /** The type of your object, e.g., "video.movie". Depending on the type you specify, other properties may also be required */ - type?: string; - /** An image URL which should represent your object within the graph */ - image?: OpenGraphField; - /** The canonical URL of your object that will be used as its permanent ID in the graph, e.g., "https://www.imdb.com/title/tt0117500/" */ - url?: string; - /** A URL to an audio file to accompany this object */ - audio?: OpenGraphField; - /** The word that appears before this object's title in a sentence. An enum of (a, an, the, "", auto). If auto is chosen, the consumer of your data should choose between "a" or "an". Default is "" (blank) */ - determiner?: string; - /** The locale these tags are marked up in. Of the format language_TERRITORY. Default is en_US */ - locale?: string; - /** An array of other locales this page is available in */ - "locale:alternate"?: string[]; - /** If your object is part of a larger web site, the name which should be displayed for the overall site. e.g., "IMDb" */ - site_name?: string; - /** A URL to a video file that complements this object */ - video?: OpenGraphField; - [field: string]: OpenGraphField; -} -/** - * When passing an array, the property is duplicated. - * When passing an object, the fields are emitted as namespaced with ':'. - */ -type OpenGraphField = - | string - | { [field: string]: OpenGraphField } - | Array - | (null | undefined); -/** Twitter uses various OpenGraph fields if these are not specified. */ -export interface Twitter { - card: string; - title?: string; - description?: string | null; - url?: string; - image?: string; - player?: string; - /** Same logic as Open Graph */ - [field: string]: OpenGraphField; -} -export interface Template - extends Omit { - base: URL; - titleTemplate?: (title: string) => string; -} -export interface Alternates { - canonical: string; - types: { [mime: string]: AlternateType }; -} -export interface AlternateType { - url: string; - title: string; -} -export type Referrer = - | "no-referrer" - | "origin" - | "no-referrer-when-downgrade" - | "origin-when-cross-origin" - | "same-origin" - | "strict-origin" - | "strict-origin-when-cross-origin"; - -/* Convert a metadata definition into text. */ -function renderMetadata(meta: Meta, template: Template): string { - const { titleTemplate, base } = template; - const resolve = (str: string) => new URL(str, base).href; - - const title = titleTemplate ? titleTemplate(meta.title) : meta.title; - const description = meta.description ?? null; - const canonical = meta.canonical ? resolve(meta.canonical) : null; - const denyRobots = Boolean(meta.denyRobots || template.denyRobots); - const authors = meta.authors ?? template.authors ?? null; - let generator = meta.generator ?? template.generator ?? null; - const keywords = meta.keywords ?? template.keywords ?? null; - const manifest = meta.manifest ?? template.manifest ?? null; - const publisher = meta.publisher ?? template.publisher ?? null; - const referrer = meta.referrer ?? template.referrer ?? null; - const themeColor = meta.themeColor ?? template.themeColor ?? null; - const viewport = meta.viewport ?? template.viewport ?? - "width=device-width, initial-scale=1.0"; - - const embed = meta.embed ?? template.embed ?? null; - let openGraph = meta.openGraph ?? template.openGraph ?? null; - let twitter = meta.twitter ?? template.twitter ?? null; - if (embed) { - const { thumbnail, thumbnailSize, siteTitle } = embed; - openGraph = { - type: "website", - title: embed.title ?? title, - description: embed.description ?? description, - ...openGraph, - }; - twitter = { - card: (thumbnailSize ?? (thumbnail ? "banner" : "icon")) === "banner" - ? "summary_large_image" - : "summary", - ...twitter, - }; - if (thumbnail) { - openGraph.image = embed.thumbnail; - } - if (siteTitle) { - openGraph.site_name = siteTitle; - } - if (canonical) { - openGraph.url = canonical; - } - } - - let out = `${esc(title)}`; - if (description) { - out += ``; - } - for (const author of authors ?? []) { - out += ``; - } - if (keywords) { - out += ``; - } - if (generator) { - if (generator === true) { - generator = "clover sitegen 3; paperclover.net/sitegen"; - } - out += ``; - } - if (publisher) { - out += ``; - } - if (referrer) { - out += ``; - } - if (themeColor) { - if (typeof themeColor === "string") { - out += ``; - } else { - out += '`; - out += '`; - } - } - if (denyRobots) { - out += ``; - } - if (viewport) { - out += ``; - } - if (canonical) { - out += ``; - } - if (manifest) { - out += ``; - } - - if (openGraph) out += renderOpenGraph("og:", openGraph); - if (twitter) out += renderOpenGraph("twitter:", twitter); - - if (meta.extra) out += render.sync(meta.extra); - if (template.extra) out += render.sync(template.extra); - - return out; -} - -function renderOpenGraph(prefix: string, value: OpenGraphField): string { - if (!value) return ""; - if (typeof value === "string") { - return ``; - } - if (Array.isArray(value)) { - return value - .map((item) => renderOpenGraph(prefix, item)) - .join(""); - } - return Object.entries(value) - .map(([key, item]) => renderOpenGraph(`${prefix}:${key}`, item)) - .join(""); -} - -function attr(value: string) { - return render.quoteIfNeeded(esc(value)); -} - -import { escapeHtmlContent as esc } from "#engine/render"; -import * as render from "#engine/render"; diff --git a/framework/lib/mime.ts b/framework/lib/mime.ts deleted file mode 100644 index 3913ebbab69054ed12b26c2d03992c8edfe332d9..0000000000000000000000000000000000000000 --- a/framework/lib/mime.ts +++ /dev/null @@ -1,32 +0,0 @@ -declare const MIME_INLINE_DATA: never; -const entries = typeof MIME_INLINE_DATA !== "undefined" - ? MIME_INLINE_DATA - : fs.readFileSync(path.join(import.meta.dirname, "mime.txt"), "utf8") - .split("\n") - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith("#")) - .map((line) => line.split(/\s+/, 2) as [string, string]); -export const rawEntriesText = entries; -const extensions = new Map(entries.filter((x) => x[0].startsWith("."))); -const fullNames = new Map(entries.filter((x) => !x[0].startsWith("."))); - -/** - * Accepts: - * - Full file path or basename - * - Extension (with or without dot) - */ -export function contentTypeFor(file: string) { - const slash = file.indexOf("/"); - if (slash !== -1) file = file.slice(slash + 1); - const dot = file.lastIndexOf("."); - if (dot === -1) file = "." + file; - else if (dot > 0) { - let entry = fullNames.get(file); - if (entry) return entry; - file = file.slice(dot); - } - return extensions.get(file) ?? "application/octet-stream"; -} - -import * as fs from "#sitegen/fs"; -import * as path from "node:path"; diff --git a/framework/lib/mime.txt b/framework/lib/mime.txt deleted file mode 100644 index 06af2f96260e0f3add34e7d81ba7dab195b0c310..0000000000000000000000000000000000000000 --- a/framework/lib/mime.txt +++ /dev/null @@ -1,100 +0,0 @@ -# media types -# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types -.aac audio/x-aac -.aif audio/x-aiff -.aifc audio/x-aiff -.aiff audio/x-aiff -.asm text/x-asm -.avi video/x-msvideo -.bat application/x-msdownload -.c text/x-c -.chat text/x-clover-chatlog -.class application/java-vm -.cmd application/x-msdownload -.com application/x-msdownload -.conf text/plain -.cpp text/x-c -.css text/css -.csv text/csv -.cxx text/x-c -.def text/plain -.diff text/plain -.dll application/x-msdownload -.dmg application/octet-stream -.doc application/msword -.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document -.epub application/epub+zip -.exe application/x-msdownload -.flv video/x-flv -.fbx application/fbx -.gz application/x-gzip -.h text/x-c -.h264 video/h264 -.hh text/x-c -.htm text/html;charset=utf-8 -.html text/html;charset=utf-8 -.ico image/x-icon -.ics text/calendar -.in text/plain -.jar application/java-archive -.java text/x-java-source -.jpeg image/jpeg -.jpg image/jpeg -.jpgv video/jpeg -.jxl image/jxl -.js application/javascript;charset=utf-8 -.json application/json -.latex application/x-latex -.list text/plain -.log text/plain -.m4a audio/mp4 -.man text/troff -.mid audio/midi -.midi audio/midi -.mov video/quicktime -.mp3 audio/mpeg -.mp4 video/mp4 -.msh model/mesh -.msi application/x-msdownload -.obj application/octet-stream -.ogg audio/ogg -.otf application/x-font-otf -.pdf application/pdf -.png image/png -.ppt application/vnd.ms-powerpoint -.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation -.psd image/vnd.adobe.photoshop -.py text/x-python -.rar application/x-rar-compressed -.rss application/rss+xml -.rtf application/rtf -.rtx text/richtext -.s text/x-asm -.pem application/x-pem-file" -.ser application/java-serialized-object -.sh application/x-sh -.sig application/pgp-signature -.silo model/mesh -.svg image/svg+xml -.t text/troff -.tar application/x-tar -.text text/plain -.tgz application/x-gzip -.tif image/tiff -.tiff image/tiff -.torrent application/x-bittorrent -.ttc application/x-font-ttf -.ttf application/x-font-ttf -.txt text/plain -.urls text/uri-list -.v text/x-v -.wav audio/x-wav -.wmv video/x-ms-wmv -.woff2 font/woff2 -.xls application/vnd.ms-excel -.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -.xml application/xml -.xps application/vnd.ms-xpsdocument - -# special cased based on file name -rss.xml application/rss+xml diff --git a/framework/lib/path.ts b/framework/lib/path.ts index 70971262e0f8a838838096df27ada11af79d0ff2..846d5580d6063f66cd29f4925c0d3e0a8ff710d4 100644 --- a/framework/lib/path.ts +++ b/framework/lib/path.ts @@ -207,6 +207,6 @@ export class Path { import * as path from "node:path"; import * as fs from "./fs.ts"; import * as util from "node:util"; -import * as error from "./error.ts"; +import * as error from "lib/error.ts"; import type { RmOptions } from "node:fs"; -import type { Serialized } from "./lru.ts"; +import { ASSERT } from "lib/assert.ts"; diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index a25a464f5f82c8f3ac787e80f1abe5891d7a08b5..f2413829534ccc8c9e60b1a3afc6e210800fa8b3 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -50,19 +50,13 @@ export type FontVars = Record; export type FontVariableAxis = number | [min: number, max: number]; export type FontRange = string | number | { start: number; end: number }; -export const userData = render.userData(() => initRender()); - -export interface SitegenRender { - scripts: Set; -} - -export function initRender(): SitegenRender { - return { scripts: new Set() }; -} +export const context = new render.Context(() => ({ + scripts: new Set(), +})); /** Add a client-side script to the page. */ export function addScript(id: ScriptId | { value: ScriptId }) { - userData.get().scripts.add(typeof id === "string" ? id : id.value); + context.get().scripts.add(typeof id === "string" ? id : id.value); } export function wrapDocument({ @@ -91,6 +85,7 @@ export function joinScripts(scriptSources: string[]) { "].map(e=>e())"; } -import * as render from "#engine/render"; -import type * as meta from "./meta.ts"; +import * as render from "lib/render.ts"; +import type * as meta from "lib/render/meta.ts"; import type * as css from "../css.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/framework/lib/subprocess.ts b/framework/lib/subprocess.ts deleted file mode 100644 index 959e235d59fb3db67b863b75d7867458f6dd7920..0000000000000000000000000000000000000000 --- a/framework/lib/subprocess.ts +++ /dev/null @@ -1,82 +0,0 @@ -// This file is expecting a full rewrite to abstract the Node API away -// entirely, exposing web streams I/O. - -const execFileRaw = util.promisify(child_process.execFile); -export const exec: typeof execFileRaw = (( - ...args: Parameters -) => - execFileRaw(...args).catch((e: any) => { - if (e?.message?.startsWith?.("Command failed")) { - if (e.code > 2 ** 31) e.code |= 0; - const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; - e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; - } - throw e; - })) as any; - -// const activePids = new Set(); - -export interface SpawnOptions extends child_process.SpawnOptions { - cmd: string[]; -} -export function spawn({ cmd, ...rest }: SpawnOptions) { - // rest.detached ??= true; - const process = child_process.spawn(cmd[0]!, cmd.slice(1), rest); - // const pid = rest.detached && process.pid ? -process.pid : process.pid; - // if (pid) { - // activePids.add(pid); - // process.on("exit", () => void activePids.delete(pid)); - // } - return process; -} - -export interface ForkOptions extends child_process.ForkOptions { - cmd: string[]; -} - -export function fork({ cmd, ...rest }: ForkOptions) { - // rest.detached ??= true; - const process = child_process.fork(cmd[0]!, cmd.slice(1), rest); - // const pid = rest.detached && process.pid ? -process.pid : process.pid; - // if (pid) { - // activePids.add(pid); - // process.on("exit", () => void activePids.delete(pid)); - // } - return process; -} - -export async function spawnAndWait(opts: SpawnOptions) { - opts.stdio ??= ["ignore", "inherit", "inherit"]; - const process = spawn(opts); - const output = Promise.all([ - process.stdout - ? Array.fromAsync(process.stdout).then(Buffer.concat) - : null, - process.stderr - ? Array.fromAsync(process.stderr).then(Buffer.concat) - : null, - ]); - let [code, signal] = (await events.EventEmitter.once(process, "exit")) as [ - number | null, - string | null, - ]; - if (code !== 0) { - if (code !== null && code > 2 ** 31) code |= 0; - const exit = signal ? `signal ${signal}` : `code ${code}`; - throw new Error(`${opts.cmd[0]} failed with ${exit}`); - } - const [stdout, stderr] = await output; - return { stdout, stderr }; -} - -// process.on("exit", () => { -// for (const pid of activePids) { -// console.info(`Kill ${Math.abs(pid)}`); -// process.kill(pid, 9); -// } -// }); - -// import * as process from "node:process"; -import * as events from "node:events"; -import * as util from "node:util"; -import * as child_process from "node:child_process"; diff --git a/framework/lib/view.ts b/framework/lib/view.ts index 708295686a663e5376dcbcd491de7a15672a8172..ae08181439fa9b102a65a8984c79365086f0b5df 100644 --- a/framework/lib/view.ts +++ b/framework/lib/view.ts @@ -73,10 +73,8 @@ export async function renderToString( // -- html -- let page: render.Element = render.element(component, props); if (layout) page = render.element(layout, { children: page }); - const { - text: body, - addon: { [sg.userData.key]: sitegen }, - } = await render.async(page, { [sg.userData.key]: sg.initRender() }); + const sitegen = sg.context.init(); + const { text: body } = await render.async(page, [sitegen]); // -- join document and send -- return sg.wrapDocument({ @@ -85,7 +83,7 @@ export async function renderToString( inlineCss, scripts: sg.joinScripts( Array.from( - sitegen!.scripts, + sitegen.scripts, (id) => UNWRAP(codegen.scripts[id], `Missing script ${id}`), ), ), @@ -108,12 +106,13 @@ export function regenerate(tag: RegenKey) { } } -import * as meta from "./meta.ts"; +import * as meta from "lib/render/meta.ts"; import type * as hono from "#hono"; -import * as render from "#engine/render"; +import * as render from "lib/render.ts"; import * as sg from "./sitegen.ts"; import * as asset from "./assets.ts"; import type { RegenKey, RegisteredViews as ViewMap, } from "../../.clover/ts/view.d.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/framework/marko.ts b/framework/marko.ts index bb82832a1847799649bebca76cdfc0bf2adfa285..0b3b829d5e638adac2e101b7cb506b7b491532ad 100644 --- a/framework/marko.ts +++ b/framework/marko.ts @@ -35,7 +35,7 @@ export function loadMarko(module: NodeJS.Module, filepath: string) { // workaround an esbuild bug? (compiled.map as unknown as { sourceRoot: string }).sourceRoot = "/"; compiled.map.sources[0] = filepath; - src = compiled.code.replace("marko/debug/html", "#engine/marko-runtime") + + src = compiled.code.replace("marko/debug/html", "#marko/runtime") + "\n//# sourceMappingURL=data:application/json;charset=utf-8;base64," + Buffer.from(JSON.stringify(compiled.map), "utf-8").toString("base64"); cache = { src, scannedClientRefs: Array.from(scannedClientRefs) }; diff --git a/framework/watch.ts b/framework/watch.ts index 740b15afd6747b4ac88bcca2a14356c6383601c6..4e8e2e5a353a3babe7dbe02ab557adea87999333 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -170,4 +170,4 @@ import * as generate from "./generate.ts"; import * as incr from "./incremental.ts"; import * as path from "node:path"; import * as util from "node:util"; -import * as subprocess from "#sitegen/subprocess"; +import * as subprocess from "lib/subprocess.ts"; diff --git a/lib/Lru.test.ts b/lib/Lru.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..b80fc4d4f6dcc7d5f7131e1851c545b73d96f09d --- /dev/null +++ b/lib/Lru.test.ts @@ -0,0 +1,138 @@ +test("lru basic", () => { + const lru = new Lru({ capacity: 3 }); + lru.set("a", 1); + lru.set("b", 2); + lru.set("c", 3); + assert.deepEqual(lru.serialize(), [ + ["c", 3, 1], + ["b", 2, 1], + ["a", 1, 1], + ]); + lru.set("d", 4); // deletes `a` + assert.deepEqual(lru.serialize(), [ + ["d", 4, 1], + ["c", 3, 1], + ["b", 2, 1], + ]); + lru.set("c", 5); + lru.get("b"); + lru.set("e", 6); + assert.deepEqual(lru.serialize(), [ + ["e", 6, 1], + ["b", 2, 1], + ["c", 5, 1], + ]); + lru.capacity = 10; + lru.set("a", 0); + lru.set("z", 1); + lru.set("x", 1); + lru.set("w", 1); + assert.deepEqual(lru.serialize(), [ + ["w", 1, 1], + ["x", 1, 1], + ["z", 1, 1], + ["a", 0, 1], + ["e", 6, 1], + ["b", 2, 1], + ["c", 5, 1], + ]); + lru.delete("a"); + assert.deepEqual(lru.serialize(), [ + ["w", 1, 1], + ["x", 1, 1], + ["z", 1, 1], + ["e", 6, 1], + ["b", 2, 1], + ["c", 5, 1], + ]); + lru.capacity = 2; + assert.deepEqual(lru.serialize(), [ + ["w", 1, 1], + ["x", 1, 1], + ]); + lru.ensureUnusedCapacity(1); + assert.deepEqual(lru.serialize(), [ + ["w", 1, 1], + ]); + lru.ensureUnusedCapacity(1.2); + assert.deepEqual(lru.serialize(), []); + lru.ensureUnusedCapacity(2); +}); +test("lru sized elements", () => { + const lru = new Lru({ capacity: 10, sizeFn: (v) => v }); + lru.set("a", 3); + lru.set("b", 3); + lru.set("c", 3); + assert.deepEqual(lru.serialize(), [ + ["c", 3, 3], + ["b", 3, 3], + ["a", 3, 3], + ]); + // hitting exact capacity + lru.set("d", 1); + lru.set("d", 1); + assert.deepEqual(lru.serialize(), [ + ["d", 1, 1], + ["c", 3, 3], + ["b", 3, 3], + ["a", 3, 3], + ]); + lru.set("e", 1); + assert.deepEqual(lru.serialize(), [ + ["e", 1, 1], + ["d", 1, 1], + ["c", 3, 3], + ["b", 3, 3], + ]); + lru.clear(); + lru.set("a", 3); + lru.set("b", 3); + lru.set("c", 3); + lru.set("d", 1); + lru.get("a"); + lru.get("b"); + lru.get("c"); + lru.set("e", 1); + assert.deepEqual(lru.serialize(), [ + ["e", 1, 1], + ["c", 3, 3], + ["b", 3, 3], + ["a", 3, 3], + ]); +}); +test("lru throw on exceeding capacity", () => { + const lru = new Lru({ capacity: 1, sizeFn: (v) => v }); + lru.set("e", 1); + lru.set("e", 1); + assert.throws(() => lru.set("e", 2), RangeError); + assert.throws(() => lru.set("d", 2), RangeError); + lru.set("d", 1); +}); +test("locked items cannot be evicted", () => { + const lru = new Lru({ capacity: 3, sizeFn: (v) => v }); + lru.set("e", 1); + lru.set("d", 1); + lru.set("f", 1); + + const unlock = lru.lock("e"); + lru.ensureUnusedCapacity(2); + assert.deepEqual(lru.serialize(), [ + ["e", 1, 1], + ]); + + unlock(); + + lru.ensureUnusedCapacity(3); + + assert.deepEqual(lru.serialize(), []); +}); +test("throws on evicting a locked item", () => { + const lru = new Lru({ capacity: 3, sizeFn: (v) => v }); + lru.set("e", 1); + void lru.lock("e"); + assert.throws(() => lru.ensureUnusedCapacity(3), RangeError); +}); + +import assert from "node:assert"; +import { test } from "node:test"; +import { Lru } from "lib/Lru.ts"; diff --git a/lib/Lru.ts b/lib/Lru.ts new file mode 100644 index 0000000000000000000000000000000000000000..7716386975db3b21f75d2bab3b041095a34c10c3 --- /dev/null +++ b/lib/Lru.ts @@ -0,0 +1,273 @@ +/** + * This module is intended to be imported via the main class. + * + * import { Lru } from '@clo/lib/Lru'; + * // Extra types are in a `declare namespace` + * const opts: Lru.Options = { capacity: 2 }; + * const lru = new Lru(opts); + * + * @module + */ + +interface Options { + capacity: number; + /** + * Decide how many units of size each entry takes. + * For example, accessing `.byteLength` of a Buffer. + */ + sizeFn?: (value: V, key: K) => number; + /** Called when `set` or `ensureUnusedCapacity` removes items. */ + evict?: (value: V, key: K) => void; + /** Called when anything removes items. */ + delete?: (value: V, key: K) => void; +} + +/** + * A Least-recently-used cache has a set capacity, removing the oldest item + * when trying to insert an item that would exceed such capacity. This + * implementation supports variable sized items, eviction callbacks, entry + * locking, and JSON-compatible serialization. + * + * https://en.wikipedia.org/wiki/Cache_replacement_policies#LRU + */ +export class Lru extends Map { + #order = new Map>(); + /** The most recently used */ + #head: K | none = none; + /** The least recently used */ + #tail: K | none = none; + #used: number = 0; + #locked: number = 0; + #capacity: number; + #sizeFn: (value: V, key: K) => number; + #deleteFn: null | ((value: V, key: K) => void); + #evictFn: null | ((value: V, key: K) => void); + + constructor(opts: Options) { + super(); + const { capacity, sizeFn, delete: deleteFn, evict } = opts; + this.#capacity = capacity; + this.#sizeFn = sizeFn ?? (() => 1); + this.#deleteFn = deleteFn ?? null; + this.#evictFn = evict ?? null; + } + + override clear() { + super.clear(); + this.#order.clear(); + this.#head = none; + this.#tail = none; + this.#used = 0; + } + override delete(key: K): boolean { + const entry = this.#evict(key); + if (!entry) return false; + this.#used -= entry[0]; + ASSERT(this.#order.delete(key)); + ASSERT(super.delete(key)); + return true; + } + override get(key: K): V | undefined { + if (this.#head !== key) { + const entry = this.#evict(key); + if (!entry) return undefined; + entry[1] = none, entry[2] = this.#head; + UNWRAP(this.#order.get(this.#head as K))[1] = key; + this.#head = key; + } + return super.get(key) as V; + } + override set(key: K, value: V): this { + const size = this.#sizeFn(value, key); + { + const uc = this.unlockedCapacity; + if (size > uc) { + const label = uc === this.#capacity ? " " : " unlocked "; + const msg = `Object of size ${size} exceeds${label}capacity ${uc}`; + throw new RangeError(msg); + } + } + + const entry = this.#evict(key); + if (entry) { + // relocate + this.#used -= entry[0]; + const head = this.#head; + ASSERT(head !== key); + entry[0] = size, entry[1] = none, entry[2] = this.#head; + if (head !== none) UNWRAP(this.#order.get(this.#head as K))[1] = key; + if (this.#tail === none) this.#tail = key; + } else { + // insertion + const { size: mapSize } = this; + const order: Order = [size, none, mapSize > 0 ? this.#head : none, 0]; + this.#order.set(key, order); + if (mapSize === 0) this.#tail = key; + else if (this.#head !== none) { + UNWRAP(this.#order.get(this.#head as K))[1] = key; + } + } + + this.#head = key; + this.#used += size; + try { + this.ensureUnusedCapacity(0); + } catch { + ASSERT(false, "capacity is already verified"); + } + super.set(key, value); + return this; + } + + /** Prevent an item from being automatically evicted */ + lock(key: K): ts.Dispose { + const order = this.#order.get(key); + if (!order) throw new Error(`Key ${key} not in Lru`); + const count = order[3] += 1; + if (count === 1) this.#locked += order[0]; + return ts.defer(() => { + const count = order![3] -= 1; + if (count === 0) this.#locked -= order[0]; + }); + } + + /** Evict items until there are `unused` capacity slots. */ + ensureUnusedCapacity(unused: number) { + const uc = this.unlockedCapacity; + if (unused > uc) { + const label = uc === this.#capacity ? "capacity" : "unlocked capacity"; + const msg = `Can't ensure ${unused} unused capacity with ${label} ${uc}`; + throw new RangeError(msg); + } + const needed = this.#used - this.#capacity + unused; + if (needed <= 0) return; + let remain = needed; + const evictFn = this.#evictFn; + const deleteFn = this.#deleteFn; + const order = this.#order; + let key = this.#tail; + loop: { + let hadLocks = false; + while (key !== none) { + const entry = UNWRAP(order.get(key)); + if (entry[3] > 0) { + key = entry[1]; + hadLocks = true; + continue; + } + ASSERT(this.#evict(key) === entry); + if (evictFn) evictFn(super.get(key) as V, key); + if (deleteFn) deleteFn(super.get(key) as V, key); + ASSERT(this.#order.delete(key)); + ASSERT(super.delete(key)); + remain -= entry[0]; + if (remain <= 0) break loop; + key = entry[1]; + } + ASSERT(hadLocks); + const msg = `Too many locked objects, cannot free ${unused} capacity`; + throw new RangeError(msg); + } + this.#used -= needed - remain; + } + + /** Unlink an entry without removing it. */ + #evict(key: K) { + const entry = this.#order.get(key); + if (!entry) return null; + const [, before, after] = entry; + if (before !== none) { + UNWRAP(this.#order.get(before as K))[2] = after; + } else this.#head = after; + if (after !== none) { + UNWRAP(this.#order.get(after as K))[1] = before; + } else this.#tail = before; + return entry; + } + + get capacity(): number { + return this.#capacity; + } + set capacity(capacity: number) { + this.#capacity = capacity; + this.ensureUnusedCapacity(0); + } + get remaining(): number { + return this.#capacity - this.#used; + } + get unlockedCapacity(): number { + return this.#capacity - this.#locked; + } + get used(): number { + return this.#used; + } + + serialize(): Serialized; + serialize(o: SerializeOptions): Serialized; + serialize( + opts: SerializeOptions = {}, + ): Serialized { + const { map = (k, v) => [k as unknown as OK, v as unknown as OV] } = opts; + const result: Serialized = []; + const order = this.#order; + let head = this.#head; + while (head !== none) { + const entry = UNWRAP(order.get(head), head); + const kv = map(head, super.get(head) as V); + head = entry[2]; + if (!kv) continue; + const [key, value] = kv; + result.push([key, value, entry[0]]); + } + return result; + } + revive( + entries: Serialized, + opts: SerializeOptions = {}, + ) { + const { map = (k, v) => [k as unknown as K, v as unknown as V] } = opts; + const order = this.#order; + let prev: Order | null = null; + let pk!: K; + for (let i = entries.length - 1; i > 0; i -= 1) { + const [ik, iv, size] = UNWRAP(entries[i]); + const kv = map(ik, iv); + if (!kv) continue; + const [k, v] = kv; + super.set(k, v); + const current: Order = [size, none, prev ? pk : none, 0]; + if (prev) prev[1] = k; + order.set(k, prev = current); + pk = k; + } + } +} + +/** The first item is the most recently used. */ +type Serialized = Array<[K, V, number]>; + +interface SerializeOptions { + /** null = omit in serialization/serialization */ + map?: (key: IK, value: IV) => [OK, OV] | null; +} + +type Order = [ + /** Number of units computed at insertion time. */ + size: number, + /** Towards most recently used */ + before: K | none, + /** Towards least recently used */ + after: K | none, + /** Locking prevents automatic eviction */ + locks: number, +]; + +export declare namespace Lru { + export type { Serialized, SerializeOptions }; +} + +const none: unique symbol = Symbol(); +type none = typeof none; + +import { ASSERT, UNWRAP } from "lib/assert.ts"; +import * as ts from "lib/ts.ts"; diff --git a/lib/assert.ts b/lib/assert.ts index 34a4c72b01b779553822de0959cd891c3a2c95a5..3ec06206cc3aefde3e46901669e2cbef1984fa9f 100644 --- a/lib/assert.ts +++ b/lib/assert.ts @@ -1,13 +1,17 @@ -// This module is intended to be imported as +/** + * assertions and type narrowing helpers. + * intended to be imported by symbol. + * @module + */ -/** Assert that `condition` is truthy */ -export function ASSERT(condition: boolean, msg?: Message): asserts condition { +/** assert that `condition` is truthy. */ +export function ASSERT(condition: unknown, msg?: Message): asserts condition { if (!condition) { throw new Error(msg ? messageToString(msg) : "Assertion Error"); } } -/** Assert that `nullable` is non-null, returning a non-nullable T */ +/** assert that `nullable` is non-null, returning a non-nullable T. */ export function UNWRAP(nullable: T | null | undefined, msg?: Message): T { if (nullable == null) { throw new Error( @@ -17,7 +21,7 @@ export function UNWRAP(nullable: T | null | undefined, msg?: Message): T { return nullable; } -/** Assert that `number` is not NaN */ +/** assert that `number` is not NaN. */ export function FINITE(number: number, msg?: Message): number { if (Number.isNaN(number)) { throw new Error(`Unexpected NaN${msg ? `: ${messageToString(msg)}` : ""}`); @@ -25,7 +29,7 @@ export function FINITE(number: number, msg?: Message): number { return number; } -/** Assert that a `default` case in a switch is never reached. */ +/** assert that a `default` case in a switch is never reached. */ export function unreachableCase(value: never): never { throw new Error(`Unexpected value in switch: ${inspect(value as unknown)}`); } diff --git a/lib/async.ts b/lib/async.ts new file mode 100644 index 0000000000000000000000000000000000000000..41f0e82ab7dc2e65183c1e69edd79760ec155453 --- /dev/null +++ b/lib/async.ts @@ -0,0 +1,207 @@ +const five_minutes = 5 * 60 * 1000; + +/*** @deprecated */ +interface ARCEValue { + value: T; + [Symbol.dispose]: () => void; +} + +/*** @deprecated */ +export function RefCountedExpirable( + init: () => Promise, + deinit: (value: T) => void, + expire: number = five_minutes, +): () => Promise> { + let refs = 0; + let item: ARCEValue | null = null; + let loading: Promise> | null = null; + let timer: ReturnType | null = null; + + function deref() { + ASSERT(item !== null); + if (--refs !== 0) return; + ASSERT(timer === null); + timer = setTimeout(() => { + ASSERT(refs === 0); + ASSERT(loading === null); + ASSERT(item !== null); + deinit(item.value); + item = null; + timer = null; + }, expire); + } + + return async function (): Promise> { + if (timer !== null) { + clearTimeout(timer); + timer = null; + } + if (item !== null) { + refs++; + return item; + } + if (loading !== null) { + refs++; + return loading; + } + const p = Promise.withResolvers>(); + loading = p.promise; + try { + const value = await init(); + item = { value, [Symbol.dispose]: deref }; + refs++; + p.resolve(item); + return item; + } catch (e) { + p.reject(e); + throw e; + } finally { + loading = null; + } + }; +} + +export function once(fn: () => Promise): () => Promise { + let result: T | Promise | null = null; + return async () => { + if (result) return result; + try { + result = await (result = fn()); + } catch (err) { + result = null; + throw err; + } + return result; + }; +} + +export class Watch { + #value: T; + #observers = new Set>(); + constructor(value: T) { + this.#value = value; + } + + on(cb: WatchCallback): () => void { + this.#observers.add(cb); + return () => void this.#observers.delete(cb); + } + once(cb: WatchCallback): void { + const once = (n: T, p: T) => (cb(n, p), this.#observers.delete(once)); + this.#observers.add(once); + } + + next(): Promise { + return new Promise((resolve) => this.once(resolve)); + } + + until(condition: (v: T) => boolean): Promise { + if (condition(this.#value)) return Promise.resolve(this.#value); + return new Promise((resolve) => { + function check(next: T) { + if (condition(next)) resolve(next), release(); + } + const release = this.on(check); + }); + } + + get value(): T { + return this.#value; + } + set value(next: T) { + const prev = this.#value; + if (prev !== next) { + this.#value = next; + this.#observers.forEach((cb) => cb(next, prev)); + } + } +} +type WatchCallback = (next: T, prev: T) => void; + +/** + * when two requests with the same args come in at the same time, the response + * from them is shared. after returning, the data is no longer stored. + */ +export class DedupeConcurrent { + pending: Map> = new Map(); + constructor( + public fn: (...args: Args) => Promise, + public keyFn: (args: Args) => unknown = JSON.stringify, + ) {} + + isPending(...args: Args): boolean { + return this.pending.has(this.keyFn(args)); + } + + /** Unbound to allow easily exporting this callback */ + get = (...args: Args): Promise => { + const k = this.keyFn(args); + let promise = this.pending.get(k); + if (promise) return promise; + promise = this.fn(...args); + this.pending.set(k, promise); + void promise.finally(() => void this.pending.delete(k)); + return promise; + }; +} + +/** + * when two requests with the same args come, the result of the first is memoized. + */ +export class OnceMap { + cache: Map> = new Map(); + constructor( + public fn: (...args: Args) => Promise, + public keyFn: (args: Args) => unknown = JSON.stringify, + ) {} + + has(...args: Args): boolean { + return this.cache.has(this.keyFn(args)); + } + + /** Unbound to allow easily exporting this callback */ + getOrRun = (...args: Args): Promise => { + const k = this.keyFn(args); + let promise = this.cache.get(k); + if (promise) return promise; + promise = this.fn(...args); + this.cache.set(k, promise); + return promise; + }; +} + +export class PromiseAggregator { + failures: unknown[] = []; + promises: Set> = new Set(); + + push(p: Promise) { + p = p.then( + () => { + this.promises.delete(p); + }, + (err) => { + this.failures.push(err); + }, + ); + this.promises.add(p); + } + + async all() { + while (this.promises.size > 0) { + const wait = [...this.promises]; + this.promises.clear(); + await Promise.all(wait); + } + if (this.failures.length > 0) { + const agg = new AggregateError(this.failures); + this.failures = []; + throw agg; + } + } + + [Symbol.asyncDispose](): Promise { + return this.all(); + } +} + +import { ASSERT } from "lib/assert.ts"; diff --git a/lib/deno.json b/lib/deno.json index 7d582025b455d4b832c764c75c99b39f7294a246..a8f612f26c6d11dca12a5692a56d5d52bceeb5f5 100644 --- a/lib/deno.json +++ b/lib/deno.json @@ -1,24 +1,40 @@ { "name": "@clo/lib", - "description": "", - "version": "1.0.0", + "description": "high quality typescript libraries", + "version": "2.0.0", "license": "MIT", "exports": { "./assert.ts": "./assert.ts", + "./async.ts": "./async.ts", + "./error.ts": "./error.ts", "./log.ts": "./log.ts", - "./log/ansi.ts": "./log/ansi.ts", "./log/headless.ts": "./log/headless.ts", "./log/progress.ts": "./log/progress.ts", "./log/stack.ts": "./log/stack.ts", - "./packaging.ts": "./packaging.ts", + "./Lru.ts": "./Lru.ts", + "./mime.ts": "./mime.ts", + "./node.ts": "./node.ts", "./queue.ts": "./queue.ts", + "./render.ts": "./render.ts", + "./render/jsx-runtime.ts": "./render/jsx-runtime.ts", + "./render/meta.ts": "./render/meta.ts", "./string.ts": "./string.ts", + "./string/ansi.ts": "./string/ansi.ts", + "./subprocess.ts": "./subprocess.ts", + "./subprocess/ffmpeg.ts": "./subprocess/ffmpeg.ts", "./ts.ts": "./ts.ts" }, + "compilerOptions": { + "jsx": "react-jsx", + "jsxImportSource": "lib/render" + }, "imports": { - "lib/": "./" + "lib/": "./", + "lib/render/jsx-runtime": "./render/jsx-runtime.ts", + "lib/render/jsx-dev-runtime": "./render/jsx-runtime.ts" }, "exclude": [ - "readme.npm.md" + "readme.npm.md", + "deno.lock" ] } \ No newline at end of file diff --git a/lib/deno.lock b/lib/deno.lock index 82ef046ffa510d16beae371abe1aa574e77739a3..d6e835a6e402f1e620683474327bf64f48bd88f0 100644 --- a/lib/deno.lock +++ b/lib/deno.lock @@ -1,171 +1,17 @@ { "version": "5", "specifiers": { - "npm:@types/node@*": "24.2.0", - "npm:ts-to-jsdoc@2.4.0": "2.4.0" + "npm:@types/node@*": "22.15.15" }, "npm": { - "@nodelib/fs.scandir@2.1.5": { - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dependencies": [ - "@nodelib/fs.stat", - "run-parallel" - ] - }, - "@nodelib/fs.stat@2.0.5": { - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==" - }, - "@nodelib/fs.walk@1.2.8": { - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dependencies": [ - "@nodelib/fs.scandir", - "fastq" - ] - }, - "@ts-morph/common@0.24.0": { - "integrity": "sha512-c1xMmNHWpNselmpIqursHeOHHBTIsJLbB+NuovbTTRCNiTLEr/U9dbJ8qy0jd/O2x5pc3seWuOUN5R2IoOTp8A==", - "dependencies": [ - "fast-glob", - "minimatch", - "mkdirp", - "path-browserify" - ] - }, - "@types/node@24.2.0": { - "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", + "@types/node@22.15.15": { + "integrity": "sha512-R5muMcZob3/Jjchn5LcO8jdKwSCbzqmPB6ruBxMcf9kbxtniZHP327s6C37iOfuw8mbKK3cAQa7sEl7afLrQ8A==", "dependencies": [ "undici-types" ] }, - "arg@5.0.2": { - "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" - }, - "balanced-match@1.0.2": { - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "brace-expansion@2.0.2": { - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "dependencies": [ - "balanced-match" - ] - }, - "braces@3.0.3": { - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dependencies": [ - "fill-range" - ] - }, - "code-block-writer@13.0.3": { - "integrity": "sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==" - }, - "fast-glob@3.3.3": { - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dependencies": [ - "@nodelib/fs.stat", - "@nodelib/fs.walk", - "glob-parent", - "merge2", - "micromatch" - ] - }, - "fastq@1.19.1": { - "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", - "dependencies": [ - "reusify" - ] - }, - "fill-range@7.1.1": { - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dependencies": [ - "to-regex-range" - ] - }, - "glob-parent@5.1.2": { - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dependencies": [ - "is-glob" - ] - }, - "is-extglob@2.1.1": { - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" - }, - "is-glob@4.0.3": { - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dependencies": [ - "is-extglob" - ] - }, - "is-number@7.0.0": { - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" - }, - "merge2@1.4.1": { - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" - }, - "micromatch@4.0.8": { - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dependencies": [ - "braces", - "picomatch" - ] - }, - "minimatch@9.0.5": { - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dependencies": [ - "brace-expansion" - ] - }, - "mkdirp@3.0.1": { - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": true - }, - "path-browserify@1.0.1": { - "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" - }, - "picomatch@2.3.1": { - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" - }, - "queue-microtask@1.2.3": { - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" - }, - "reusify@1.1.0": { - "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==" - }, - "run-parallel@1.2.0": { - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dependencies": [ - "queue-microtask" - ] - }, - "to-regex-range@5.0.1": { - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dependencies": [ - "is-number" - ] - }, - "ts-morph@23.0.0": { - "integrity": "sha512-FcvFx7a9E8TUe6T3ShihXJLiJOiqyafzFKUO4aqIHDUCIvADdGNShcbc2W5PMr3LerXRv7mafvFZ9lRENxJmug==", - "dependencies": [ - "@ts-morph/common", - "code-block-writer" - ] - }, - "ts-to-jsdoc@2.4.0": { - "integrity": "sha512-RiI9ipxkLjXESp5+YjXIQJ5GgtjwWTZU4uDahxsIdr1HKLFaC/nQOtgkGRJ+AFxqHrrhwrXtPh093r0nXQwKmw==", - "dependencies": [ - "arg", - "ts-morph" - ], - "optionalDependencies": [ - "typescript" - ], - "bin": true - }, - "typescript@5.9.2": { - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", - "bin": true - }, - "undici-types@7.10.0": { - "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==" + "undici-types@6.21.0": { + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" } } } diff --git a/lib/error.ts b/lib/error.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c2817b0c8a10ac3331d3207cd251d33ef14eba6 --- /dev/null +++ b/lib/error.ts @@ -0,0 +1,50 @@ +/** + * Helper functions for dealing with the `unknown` type, mainly in `catch` + * blocks or promise rejection callbacks. + * + * @module + */ + +/** extra fields on errors are extremely helpful for debugging. */ +export interface Error extends globalThis.Error { + [metadata: string]: unknown; +} + +/** retrieve an error message from any value */ +export function message(error: unknown): string { + const message = (error as { message: unknown })?.message ?? error; + try { + return typeof message === "string" ? message : JSON.stringify(message); + } catch {} + try { + return String(message); + } catch {} + return `Could not stringify error message ${typeof message}`; +} + +/** retrieve an error-like object from any value. Useful to check fields. */ +export function obj(error: unknown): { message: string } & Partial { + if (error instanceof Error) return error as Error; + return { + message: message(error), + ...(typeof error === "object" && error), + }; +} + +/** node.js error codes are string values on the `.code` property. */ +export function code(error: unknown): node.ErrorCode | null { + const code = (error as { code: unknown })?.code; + return typeof code === "string" ? code : null; +} + +/** attach extra fields and rethrow a value. */ +export function rethrowWithMetadata( + err: unknown, + meta: Record, +): never { + const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/ + Object.assign(error, meta); + throw error; +} + +import type * as node from "lib/node.ts"; diff --git a/lib/log.test.ts b/lib/log.test.ts index 91d8ba08516dc88eec94465366fb46454cda5080..35aeeb1821d3747e76ad2d3d85bbeb5c6ef78328 100644 --- a/lib/log.test.ts +++ b/lib/log.test.ts @@ -5,6 +5,64 @@ import * as log from "lib/log.ts"; log satisfies log.Scope; // the namespace import is a valid Scope +test("writeLine buffers", () => { + using host = new TestWidgetHost(); + host.writeLine("hello world"); + host.writeLine("and so on"); + host.expectFrame(0, { + stdout: "hello world\nand so on\n", + }); + host.writeLine("more logs"); + host.expectFrame(0, { + stdout: "more logs\n", + }); +}); + +test("widget with interweaving logs", () => { + const host = new TestWidgetHost(); + + const _ = host.startWidget({ + format: (now) => `line one ${now}\nline two`, + }); + + host.expectFrame(0, { + stderr: sync([ + ansi.startOfLine, + "line one 0\nline two\n", + ]), + }); + + host.writeLine("info: line of log"); + host.expectFrame(0, { + out: sync([ + ansi.startOfLine, + ansi.cursorUp(2), + ansi.clearFullLine, + "info: line of log\n", + "line one 0\nline two\n", + ]), + }); + + host.writeLine("warn: line of log"); + host.writeLine("error: line of log"); + host.expectFrame(0, { + out: sync([ + ansi.startOfLine, + ansi.cursorUp(1), + ansi.clearFullLine, + ansi.cursorUp(1), + ansi.clearFullLine, + "warn: line of log\n", + "error: line of log\n", + "line one 0\nline two\n", + ]), + }); +}); + +function sync(text: string[]) { + return ansi.syncStart + text.join("") + ansi.syncEnd; +} + class TestWidgetHost implements headless.WidgetHost { columns = 80; rows = 33; @@ -103,62 +161,4 @@ class TestWidgetHost implements headless.WidgetHost { } } -test("writeLine buffers", () => { - using host = new TestWidgetHost(); - host.writeLine("hello world"); - host.writeLine("and so on"); - host.expectFrame(0, { - stdout: "hello world\nand so on\n", - }); - host.writeLine("more logs"); - host.expectFrame(0, { - stdout: "more logs\n", - }); -}); - -test("widget with interweaving logs", () => { - const host = new TestWidgetHost(); - - const _ = host.startWidget({ - format: (now) => `line one ${now}\nline two`, - }); - - host.expectFrame(0, { - stderr: sync([ - ansi.startOfLine, - "line one 0\nline two\n", - ]), - }); - - host.writeLine("info: line of log"); - host.expectFrame(0, { - out: sync([ - ansi.startOfLine, - ansi.cursorUp(2), - ansi.clearFullLine, - "info: line of log\n", - "line one 0\nline two\n", - ]), - }); - - host.writeLine("warn: line of log"); - host.writeLine("error: line of log"); - host.expectFrame(0, { - out: sync([ - ansi.startOfLine, - ansi.cursorUp(1), - ansi.clearFullLine, - ansi.cursorUp(1), - ansi.clearFullLine, - "warn: line of log\n", - "error: line of log\n", - "line one 0\nline two\n", - ]), - }); -}); - -function sync(text: string[]) { - return ansi.syncStart + text.join("") + ansi.syncEnd; -} - -import * as ansi from "lib/log/ansi.ts"; +import * as ansi from "lib/string/ansi.ts"; diff --git a/lib/log.ts b/lib/log.ts index 9067b4a701d7cd2c6d5136f7482b811c86b6fe9c..0fdcdd8b1286557a6e686c08427bd87604702031 100644 --- a/lib/log.ts +++ b/lib/log.ts @@ -1,7 +1,9 @@ /** * By using `lib/log.ts`, your application gets easy scoped logging as well as - * integration with terminal widgets such as `lib/log/progress.ts`. The pattern - * for using this module is to shadow the global `console` with a local scope: + * integration with terminal widgets such as `lib/log/progress.ts`. + * + * The pattern for using this module is to shadow the global `console` with a + * local scope: * * import * as log from "@clo/lib/log"; * const console = log.scoped("http"); @@ -36,6 +38,7 @@ export interface Scope { /** * emit a debugging message. this method is in place for compatibility with * the `console` API. prefer calling `.log` directly. + * @internal */ debug(...args: unknown[]): void; @@ -48,33 +51,38 @@ export function scoped(name: string): Scope { return globalLog.scoped(name); } -// These functions implement `Scope`. If a function takes in `Scope`, the +// these functions implement `Scope`. If a function takes in `Scope`, the // namespace import of this file can be passed to it. -/** informational */ +/** emit an informational message on the global log scope */ export function info(...args: unknown[]) { globalLog.info(...args); } -/** advisory */ +/** emit an advisory message on the global log scope */ export function warn(...args: unknown[]) { globalLog.warn(...args); } -/** failure */ +/** emit an failure message on the global log scope */ export function error(...args: unknown[]) { globalLog.error(...args); } -/** debugging */ +/** emit an debug message on the global log scope */ export function log(...args: unknown[]) { globalLog.log(...args); } -/** debugging */ +/** + * emit an debug message on the global log scope. this method is in place for + * compatibility with the `console` API. prefer calling `.log` directly. + * @internal + */ export function debug(...args: unknown[]) { globalLog.debug(...args); } /** - * no built-in prefix or formatting. ensures the text does not interweave. - * data will be flushed in the next frame or when drawing is unlocked. + * no built-in prefix or formatting. ensures the text does not interweave. data + * will be flushed in the next frame or when drawing is + * {@link getDrawLock|unlocked}. */ export function writeLine(text: string) { globalWidgetHost.writeLine(text); @@ -115,7 +123,7 @@ export interface Widget { } const globalWidgetHost = /* @__PURE__ */ ((): headless.WidgetHost => { - const { process } = globalThis; + const { process } = node; if (!process || !process.stderr.isTTY) { return { // TODO: allow passing a level here @@ -145,9 +153,9 @@ const globalWidgetHost = /* @__PURE__ */ ((): headless.WidgetHost => { })(); const globalLog = /* @__PURE__ */ headless.logger({ log: globalWidgetHost, - colors: process.stderr.isTTY, + colors: node.process?.stderr.isTTY ?? false, }); -import process from "node:process"; +import * as node from "lib/node.ts"; import * as headless from "lib/log/headless.ts"; import * as ts from "lib/ts.ts"; diff --git a/lib/log/ansi.ts b/lib/log/ansi.ts deleted file mode 100644 index f991b3f8bf9a7dc16fd00a63b9b8a2123154644c..0000000000000000000000000000000000000000 --- a/lib/log/ansi.ts +++ /dev/null @@ -1,131 +0,0 @@ -export const reset: string = `\x1b[0m`; - -export const bold: string = `\x1b[1m`; -export const dim: string = `\x1b[2m`; -export const resetWeight: string = `\x1b[22m`; - -export const fgReset: string = `\x1b[39m`; -export const fgBlack: string = `\x1b[30m`; -export const fgRed: string = `\x1b[31m`; -export const fgGreen: string = `\x1b[32m`; -export const fgYellow: string = `\x1b[33m`; -export const fgBlue: string = `\x1b[34m`; -export const fgMagenta: string = `\x1b[35m`; -export const fgCyan: string = `\x1b[36m`; -export const fgWhite: string = `\x1b[37m`; -export const fgBrightBlack: string = `\x1b[90m`; -export const fgBrightRed: string = `\x1b[91m`; -export const fgBrightGreen: string = `\x1b[92m`; -export const fgBrightYellow: string = `\x1b[93m`; -export const fgBrightBlue: string = `\x1b[94m`; -export const fgBrightMagenta: string = `\x1b[95m`; -export const fgBrightCyan: string = `\x1b[96m`; -export const fgBrightWhite: string = `\x1b[97m`; -export function fg256(n: number): string { - return `\x1b[38;5;${n}m`; -} -export function fgRgb(r: number, g: number, b: number): string { - return `\x1b[38;2;${r};${g};${b}m`; -} - -export const bgReset: string = `\x1b[49m`; -export const bgBlack: string = `\x1b[40m`; -export const bgRed: string = `\x1b[41m`; -export const bgGreen: string = `\x1b[42m`; -export const bgYellow: string = `\x1b[43m`; -export const bgBlue: string = `\x1b[44m`; -export const bgMagenta: string = `\x1b[45m`; -export const bgCyan: string = `\x1b[46m`; -export const bgWhite: string = `\x1b[47m`; -export const bgBrightBlack: string = `\x1b[100m`; -export const bgBrightRed: string = `\x1b[101m`; -export const bgBrightGreen: string = `\x1b[102m`; -export const bgBrightYellow: string = `\x1b[103m`; -export const bgBrightBlue: string = `\x1b[104m`; -export const bgBrightMagenta: string = `\x1b[105m`; -export const bgBrightCyan: string = `\x1b[106m`; -export const bgBrightWhite: string = `\x1b[107m`; -export function bg256(n: number): string { - return `\x1b[48;5;${n}m`; -} -export function bgRgb(r: number, g: number, b: number): string { - return `\x1b[48;2;${r};${g};${b}m`; -} - -// cursor management -export function cursorUp(n: number): string { - return n ? `\x1b[${n}A` : ""; -} -export const startOfLine: string = "\r"; -export const startOfNextLine: string = "\n"; - -// erasure -export const clearToEndOfLine: string = "\x1b[K"; -export const clearToStartOfLine: string = "\x1b[1K"; -export const clearFullLine: string = "\x1b[2K"; - -// Synchronized Output -// https://gist.github.com/christianparpart/d8a62cc1ab659194337d73e399004036 -export const syncStart: string = "\x1b[?2026h"; -export const syncEnd: string = "\x1b[?2026l"; - -export function style(code: string, text: string): string { - code = mergeStyles(code); - const fg = /[\[;][39]/.test(code); - const bg = /[\[;](?:4|10)/.test(code); - const bold = /[\[;](?:1)[;m]/.test(code); - const dim = /[\[;](?:2)[;m]/.test(code); - return code + text + - mergeStyles( - (fg ? fgReset : "") + - (bg ? bgReset : "") + - (bold ? resetWeight : "") + - (dim ? resetWeight : ""), - ); -} - -/** - * Given a string of multiple style ansi codes, combines all of them. - * Must be used on a string of only ansi style codes. - */ -export function mergeStyles(code: string): string { - let styles = []; - while (code.length) { - if (!code.startsWith("\x1b[")) { - throw new Error("invalid style string: " + JSON.stringify(code)); - } - const end = code.indexOf("m"); - if (end === -1) { - throw new Error("invalid style string: " + JSON.stringify(code)); - } - styles.push(code.slice(2, end)); - code = code.slice(end + 1); - } - return `\x1b[${styles.join(";")}m`; -} - -export function debugAnsi(text: string): string { - // deno-fmt-ignore - const verbaitim: (keyof typeof self)[] = [ - "bgReset", "bgBlack", "bgRed", "bgGreen", "bgYellow", "bgBlue", "bgMagenta", - "bgCyan", "bgWhite", "bgBrightBlack", "bgBrightRed", "bgBrightGreen", - "bgBrightYellow", "bgBrightBlue", "bgBrightMagenta", "bgBrightCyan", - "bgBrightWhite", "fgReset", "fgBlack", "fgRed", "fgGreen", "fgYellow", - "fgBlue", "fgMagenta", "fgCyan", "fgWhite", "fgBrightBlack", "fgBrightRed", - "fgBrightGreen", "fgBrightYellow", "fgBrightBlue", "fgBrightMagenta", - "fgBrightCyan", "fgBrightWhite", "clearToEndOfLine", "clearToStartOfLine", - "clearFullLine", "syncStart", "syncEnd", "bold", "dim", "resetWeight" - ]; - for (const key of verbaitim) { - text = text.replaceAll( - String(self[key]), - style(key.includes("cursor") ? fgBlue : fgCyan, `\${${key}}`), - ); - } - text = text.replace(/\x1b\[(\d+)A/g, style(fgYellow, "$${cursorUp($1)}")); - text = text.replace(/\n/g, style(fgRed, "\\n")); - text = text.replace(/\r/g, style(fgMagenta, "\\r")); - return text; -} - -import * as self from "lib/log/ansi.ts"; diff --git a/lib/log/headless.ts b/lib/log/headless.ts index 1d168453cec2eb77311c1b81febed22226b3dc51..0d16bcc89523a5e34e0072b955904bdcfde733f2 100644 --- a/lib/log/headless.ts +++ b/lib/log/headless.ts @@ -1,37 +1,47 @@ -// This file is under heavy construction. -// - I am happy with the overall API, likely no changes. -// - There are certainly a lot of bugs. -// -// Rendering for 'lib/term' is done by specifying all system APIs up front in -// an interface and creating an instance of the widget host. The main file -// fufills this interface with `node:process` and `globalThis`. -// -// Some notes on widget rendering: -// - Never flush immediately (exception for `getDrawLock`), always do it next tick. -// - Maximum of one `wait` call at once. When the expected time suddenly -// shrinks, the timer is rescheduled. -// - When redrawing widget lines, three tricks are done to reduce flickering: -// 1. Tell the terminal not to flicker (ansi.syncStart/syncEnd) -// 2. A simple prefix-based diffing algorithm for skipping unchanged text -// 3. Avoid clearing a line before redrawing it. -// Points 2 and 3 are used for terminals that are slow or have -// +/** + * terminal widget rendering and logging is done by specifying all system APIs up front in an + * interface, creating an instance of the widget host. `lib/log.ts` fufills + * this interface with `node:process` and `globalThis`. + * + * this file is under heavy construction. + * - I am happy with the overall API, likely no changes. + * - There are certainly a lot of bugs. + * + * some notes on widget rendering: + * - Never flush immediately (exception for + * {@linkcode WidgetHost["getDrawLock"]|getDrawLock}), always do it next tick. + * - Maximum of one `wait` call at once. When the expected time suddenly + * shrinks, the timer is rescheduled. + * - When redrawing widget lines, three tricks are done to reduce flickering: + * 1. Tell the terminal not to flicker (ansi.syncStart/syncEnd) + * 2. A simple prefix-based diffing algorithm for skipping unchanged text + * 3. Avoid clearing a line before redrawing it. + * Points 2 and 3 are used for terminals that are slow or do not support sync. + * + * @module + */ +/** {@linkcode widgetHost}'s input takes terminal i/o as well as timing APIs */ export interface WidgetHostEnv { + /** Recieves ANSI escape sequences for interactive data. */ writeInteractive(text: string): void; + /** Recieves log content (from `writeLine`). */ writeOutput(text: string): void; + /** Monotonic. */ now(): ReturnType; - /** 0ms indicates "one frame" */ + /** 0ms indicates "one frame". */ wait(ms: number, cb: () => void): () => void; + /** Called often. */ getSize(): { columns: number; rows: number }; // onInput?(write: (bytes: Uint8Array | string) => void): () => void; } -/** Not intended to be implemented manually. See `widgetHost` */ +/** Returned by {@linkcode widgetHost} */ export interface WidgetHost { /** - * no built-in prefix or formatting. ensures the text does not interweave. - * data will be flushed in the next frame or when drawing is unlocked. + * no built-in prefix or formatting. ensures the text does not interweave + * with widgets. data will be flushed in the next frame or when drawing is + * unlocked. */ writeLine(text: string): void; /** @@ -55,7 +65,7 @@ interface WidgetState { unsub: (() => void) | null; } -/** Implement a widget host by providing an environment. */ +/** Implement a widget host by providing an environment implementation. */ export function widgetHost(env: WidgetHostEnv): WidgetHost { const { writeOutput, writeInteractive, now, wait, getSize } = env; type Cancel = ReturnType; @@ -97,7 +107,7 @@ export function widgetHost(env: WidgetHostEnv): WidgetHost { if (rowsLeft === 1) break; const lines = outText.split("\n").slice(0, rowsLeft); newWidgetLines.push( - ...lines.map((line) => string.trimToWidth(line, columns - 1)), + ...lines.map((line) => ansi.trimToWidth(line, columns - 1)), ); next = Math.min(next, UNWRAP(internals[w]).frameTime); @@ -157,8 +167,8 @@ export function widgetHost(env: WidgetHostEnv): WidgetHost { : newLine) + // clear rest of line if needed (oldLines[i] && - string.widthInTerminal(oldLines[i]) > - string.widthInTerminal(newLine) + ansi.widthInTerminal(oldLines[i]) > + ansi.widthInTerminal(newLine) ? ansi.clearToEndOfLine : "") + "\n" @@ -185,8 +195,8 @@ export function widgetHost(env: WidgetHostEnv): WidgetHost { : newLine) + // clear rest of line if needed (lines[i] && - string.widthInTerminal(lines[i]) > - string.widthInTerminal(newLine) + ansi.widthInTerminal(lines[i]) > + ansi.widthInTerminal(newLine) ? ansi.clearToEndOfLine : "") + "\n" @@ -264,6 +274,7 @@ export function widgetHost(env: WidgetHostEnv): WidgetHost { }; } +/** Input interface for {@linkcode logger} */ export interface LoggerOptions { /** Provide either a line writer or a log writer. */ log: { @@ -274,6 +285,7 @@ export interface LoggerOptions { colors: boolean; } +/** Implement a logging scope by providing an environment interface. */ export function logger(env: LoggerOptions): log.Scope { const { colors, log } = env; const levels = colors @@ -314,6 +326,7 @@ export function logger(env: LoggerOptions): log.Scope { } function createSubScope(this: log.Scope, name: string) { + // @ts-ignore const { name: parent } = this; return scoped(parent ? parent + "/" + name : name); } @@ -323,7 +336,7 @@ export function logger(env: LoggerOptions): log.Scope { type Level = 0 | 1 | 2; -import * as ansi from "lib/log/ansi.ts"; +import * as ansi from "lib/string/ansi.ts"; import * as string from "lib/string.ts"; import * as ts from "lib/ts.ts"; import * as stack from "lib/log/stack.ts"; diff --git a/lib/log/progress.ts b/lib/log/progress.ts index 07a5bd43695d9837338d619dd5d7865075dbd38a..85c60bd81d79b0e58560d7c45cc7d8e1768b825b 100644 --- a/lib/log/progress.ts +++ b/lib/log/progress.ts @@ -5,12 +5,10 @@ // core, but still leaving the top level `start` function. // - There are an overwhelming amount of bugs // -// A progress tree based on my past uses with clover console v3 and the Zig -// Progress API. Pass `Progress` to functions which should track progress, -// calling `.start()` to create child items. Or, multiple top-level progress -// bars can be created with the singleton's `start()` function. -import * as self from "lib/log/progress.ts"; -self satisfies Ref; // prove you can use the namespace import as a `Progress` +// a progress tree based on my past uses with clover console v3 and the Zig +// `Progress` API. Pass `Ref` to functions which should track progress, calling +// `.start()` to create child items. Or, multiple top-level progress bars can +// be created with the singleton's `start()` function. /** * creates a new trackable unit of work as a child of this one. @@ -28,7 +26,7 @@ export interface Ref { start(text: string, opts?: StartOptions): Node; } -export interface Node extends Ref { +export interface Node extends Ref, Disposable { /** reactive */ text: string; /** reactive */ @@ -62,8 +60,6 @@ export interface Node extends Ref { /** end this Node or RootNode. ending collapses children */ end(): void; - - [Symbol.dispose](): void; } export interface StartOptions { @@ -415,6 +411,6 @@ const global = /** @__PURE__ */ new Headless(log); import * as process from "node:process"; import * as log from "lib/log.ts"; import * as headless from "lib/log/headless.ts"; -import * as ansi from "lib/log/ansi.ts"; +import * as ansi from "lib/string/ansi.ts"; import * as string from "lib/string.ts"; -import { UNWRAP } from "lib/assert.ts"; +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/lib/log/stack.ts b/lib/log/stack.ts index dc9ab5b42cfcd7e266ce63d4bdd5e6e6b5ed4a89..bf2e4f6ac867be55cc978d3ece4a498a18fa0d9c 100644 --- a/lib/log/stack.ts +++ b/lib/log/stack.ts @@ -2,7 +2,8 @@ * the default stack trace formatters in node.js, deno, and bun are all * extremely low quality. they don't highlight the most important information * and spend too much space on showing source code. this module provides tools - * to `parse` and `format` errors. see `format`'s comment for design decisions. + * to {@linkcode parse} and {@linkcode format} errors. see {@linkcode format}'s + * comment for design decisions. * * to get a taste of the formatter: * ```ts @@ -39,7 +40,7 @@ export interface Frame { /** * parse a error stack trace from either v8 (node, deno, chrome), * jsc (safari, bun), or spidermonkey (firefox). - * + * * this function is derived from clover's work on bun's DevServer, which she * derived from stacktracejs. this file is more refined than those two: * https://github.com/oven-sh/bun/blob/b5f31a6ee2f52ea67eabeb61f6e6e71215d55b26/src/bake/client/stack-trace.ts @@ -52,7 +53,9 @@ export interface Frame { export function parse(error: Error | string): Frame[] | null { const stack = (error as Error)?.stack ?? error; if (typeof stack !== "string") return null; - return stack.match(regexV8Stack) ? parseV8OrIe(stack) : parseJscOrSpidermonkey(stack); + return stack.match(regexV8Stack) + ? parseV8OrIe(stack) + : parseJscOrSpidermonkey(stack); } /** @@ -65,7 +68,7 @@ export function capture(trimStart = 2): Frame[] { /** * convert an error into a human readable and debuggable string. - * + * * this printer puts the error metadata at the top. each stack frame gets two * lines: the absolute path or url, and then one line of source code. having * five lines of context on just the top frame is useless, since five lines is @@ -238,8 +241,8 @@ export function formatFrame(frame: Frame, colors: boolean): string { const existCache = new Map(); function getPackageRoot(absPath: string) { - const fs = nodeBuiltin("node:fs"); - const path = nodeBuiltin("node:path"); + const fs = node.builtin("fs"); + const path = node.builtin("path"); const process = globalThis.process; if (!fs || !path || !process) return null; @@ -302,14 +305,12 @@ function getPackageRoot(absPath: string) { const codeCache = new Map(); function getSourceCode(file: string) { - const fs = nodeBuiltin("node:fs"); - const path = nodeBuiltin("node:path"); + const fs = node.builtin("fs"); + const path = node.builtin("path"); if (!fs || !path) return null; if (file.startsWith("node:")) { // node.js has a secret binding for all the internal source codes - type Process = { binding(name: "natives"): Record }; - return (globalThis.process as unknown as Process) - ?.binding?.("natives")[file.slice(5)]?.split("\n") ?? null; + return node.binding("natives")?.[file.slice(5)]?.split("\n") ?? null; } else { // bun and deno do not offer a way to get source code of internal modules, // probably for good reason. @@ -341,16 +342,16 @@ function parseV8OrIe(stack: string): Frame[] { // capture and preserve the parenthesized location "(/foo/my bar.js:12:87)" in // case it has spaces in it, as the string is split on \s+ later on - let loc = sanitizedLine.match(/ (\(.+\)$)/); + const loc = sanitizedLine.match(/ (\(.+\)$)/); // remove the parenthesized location from the line, if it was matched sanitizedLine = loc ? sanitizedLine.replace(loc[0], "") : sanitizedLine; // if a location was matched, pass it to extractLocation() otherwise pass all sanitizedLine // because this line doesn't have function name - let locationParts = extractLocation(loc?.[1] ?? sanitizedLine); - let functionName = (loc && sanitizedLine) || undefined; - let fileName = ["eval", ""].indexOf(locationParts[0]) > -1 + const locationParts = extractLocation(loc?.[1] ?? sanitizedLine); + const functionName = (loc && sanitizedLine) || undefined; + const fileName = ["eval", ""].indexOf(locationParts[0]) > -1 ? undefined : locationParts[0]; @@ -377,11 +378,10 @@ function parseJscOrSpidermonkey(stack: string): Frame[] { // Safari eval frames only have function names and nothing else fn = source.endsWith("@") ? source.slice(0, -1) : source; } else { - var functionNameRegex = /((.*".+"[^@]*)?[^@]*)(?:@)/; - var matches = source.match(functionNameRegex); - var functionName = matches && matches[1] ? matches[1] : undefined; - var locationParts = extractLocation( - source.replace(functionNameRegex, ""), + const matches = source.match(regexFunctionName); + const functionName = matches && matches[1] ? matches[1] : undefined; + const locationParts = extractLocation( + source.replace(regexFunctionName, ""), ); fn = functionName!; file = locationParts[0]; @@ -408,14 +408,9 @@ function extractLocation(urlLike: string) { return [parts[1], parts[2] || undefined, parts[3] || undefined]; } -// TODO: extract -function nodeBuiltin(name: "node:path"): typeof import("node:path") | null; -function nodeBuiltin(name: "node:fs"): typeof import("node:fs") | null; -function nodeBuiltin(name: string) { - return globalThis.process?.getBuiltinModule(name) ?? null; -} - +const regexFunctionName = /((.*".+"[^@]*)?[^@]*)(?:@)/; const regexV8Stack = /^\s*at .*(\S+:\d+|\(native\))/m; const regexLocation = /(.+?)(?::(\d+))?(?::(\d+))?$/; -import * as ansi from "lib/log/ansi.ts"; +import * as ansi from "lib/string/ansi.ts"; +import * as node from "lib/node.ts"; diff --git a/lib/mime.test.ts b/lib/mime.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e130a08fb98e94eed36d102ec0272abbb317a5c --- /dev/null +++ b/lib/mime.test.ts @@ -0,0 +1,28 @@ +// # `mime.for` +// Retrieves the 'Content-Type' header to use given a file extension. +test("for", () => { + // accepts extension as string + assert.equal(mime.for("txt"), "text/plain"); + // accepts + assert.equal(mime.for(".xml"), "application/xml"); + // accepts file path + assert.equal(mime.for("filename.xml"), "application/xml"); + assert.equal(mime.for("a.directory/.xml"), "application/xml"); + // accepts absolute file path + assert.equal(mime.for("/path/to/filename.xml"), "application/xml"); + assert.equal(mime.for("C:\\path\\to\\filename.xml"), "application/xml"); + + // rss is special-cased + assert.equal(mime.for("rss.xml"), "application/rss+xml"); + + // supports modern innovations + assert.equal(mime.for(".avif"), "image/avif"); + assert.equal(mime.for(".jxl"), "image/jxl"); + + // fallback + assert.equal(mime.for(".unknown"), "application/octet-stream"); +}); + +import { test } from "node:test"; +import assert from "node:assert"; +import * as mime from "lib/mime.ts"; diff --git a/lib/mime.ts b/lib/mime.ts new file mode 100644 index 0000000000000000000000000000000000000000..9bfa1809eba7798d8c6f4bd76addb38ebb2f9dc9 --- /dev/null +++ b/lib/mime.ts @@ -0,0 +1,127 @@ +/** + * retrieves the 'Content-Type' header to use given a file extension. + * accepts: + * - full file path or basename + * - extension (with or without dot) + */ +function contentTypeFor(file: string): string { + const slash = file.indexOf("/"); + if (slash !== -1) file = file.slice(slash + 1); + const dot = file.lastIndexOf("."); + if (dot === -1) file = "." + file; + else if (dot > 0) { + let entry = fullNames.get(file); + if (entry) return entry; + file = file.slice(dot); + } + return extensions.get(file) ?? "application/octet-stream"; +} +export { contentTypeFor as for }; + +// special-cased file names +const fullNames = new Map([ + ["rss.xml", "application/rss+xml"], +]); + +// this string literal, when minified, is shorter than key-value pairs. +const extensions = new Map( + ( // deno-fmt-ignore + '.aac' + '#' + 'audio/x-aac' + '\n' + + '.aif' + '#' + 'audio/x-aiff' + '\n' + + '.aifc' + '#' + 'audio/x-aiff' + '\n' + + '.aiff' + '#' + 'audio/x-aiff' + '\n' + + '.asm' + '#' + 'text/x-asm' + '\n' + + '.avi' + '#' + 'video/x-msvideo' + '\n' + + '.avif' + '#' + 'image/avif' + '\n' + + '.bat' + '#' + 'application/x-msdownload' + '\n' + + '.c' + '#' + 'text/x-c' + '\n' + + '.chat' + '#' + 'text/x-clover-chatlog' + '\n' + + '.class' + '#' + 'application/java-vm' + '\n' + + '.cmd' + '#' + 'application/x-msdownload' + '\n' + + '.com' + '#' + 'application/x-msdownload' + '\n' + + '.conf' + '#' + 'text/plain' + '\n' + + '.cpp' + '#' + 'text/x-c' + '\n' + + '.css' + '#' + 'text/css' + '\n' + + '.csv' + '#' + 'text/csv' + '\n' + + '.cxx' + '#' + 'text/x-c' + '\n' + + '.def' + '#' + 'text/plain' + '\n' + + '.diff' + '#' + 'text/plain' + '\n' + + '.dll' + '#' + 'application/x-msdownload' + '\n' + + '.dmg' + '#' + 'application/octet-stream' + '\n' + + '.doc' + '#' + 'application/msword' + '\n' + + '.docx' + '#' + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + '\n' + + '.epub' + '#' + 'application/epub+zip' + '\n' + + '.exe' + '#' + 'application/x-msdownload' + '\n' + + '.flv' + '#' + 'video/x-flv' + '\n' + + '.fbx' + '#' + 'application/fbx' + '\n' + + '.gz' + '#' + 'application/x-gzip' + '\n' + + '.h' + '#' + 'text/x-c' + '\n' + + '.h264' + '#' + 'video/h264' + '\n' + + '.hh' + '#' + 'text/x-c' + '\n' + + '.htm' + '#' + 'text/html;charset=utf-8' + '\n' + + '.html' + '#' + 'text/html;charset=utf-8' + '\n' + + '.ico' + '#' + 'image/x-icon' + '\n' + + '.ics' + '#' + 'text/calendar' + '\n' + + '.in' + '#' + 'text/plain' + '\n' + + '.jar' + '#' + 'application/java-archive' + '\n' + + '.java' + '#' + 'text/x-java-source' + '\n' + + '.jpeg' + '#' + 'image/jpeg' + '\n' + + '.jpg' + '#' + 'image/jpeg' + '\n' + + '.jpgv' + '#' + 'video/jpeg' + '\n' + + '.jxl' + '#' + 'image/jxl' + '\n' + + '.js' + '#' + 'application/javascript;charset=utf-8' + '\n' + + '.json' + '#' + 'application/json' + '\n' + + '.latex' + '#' + 'application/x-latex' + '\n' + + '.list' + '#' + 'text/plain' + '\n' + + '.log' + '#' + 'text/plain' + '\n' + + '.m4a' + '#' + 'audio/mp4' + '\n' + + '.man' + '#' + 'text/troff' + '\n' + + '.mid' + '#' + 'audio/midi' + '\n' + + '.midi' + '#' + 'audio/midi' + '\n' + + '.mov' + '#' + 'video/quicktime' + '\n' + + '.mp3' + '#' + 'audio/mpeg' + '\n' + + '.mp4' + '#' + 'video/mp4' + '\n' + + '.msh' + '#' + 'model/mesh' + '\n' + + '.msi' + '#' + 'application/x-msdownload' + '\n' + + '.obj' + '#' + 'application/octet-stream' + '\n' + + '.ogg' + '#' + 'audio/ogg' + '\n' + + '.otf' + '#' + 'application/x-font-otf' + '\n' + + '.pdf' + '#' + 'application/pdf' + '\n' + + '.png' + '#' + 'image/png' + '\n' + + '.ppt' + '#' + 'application/vnd.ms-powerpoint' + '\n' + + '.pptx' + '#' + 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + '\n' + + '.psd' + '#' + 'image/vnd.adobe.photoshop' + '\n' + + '.py' + '#' + 'text/x-python' + '\n' + + '.rar' + '#' + 'application/x-rar-compressed' + '\n' + + '.rss' + '#' + 'application/rss+xml' + '\n' + + '.rtf' + '#' + 'application/rtf' + '\n' + + '.rtx' + '#' + 'text/richtext' + '\n' + + '.s' + '#' + 'text/x-asm' + '\n' + + '.pem' + '#' + 'application/x-pem-file"' + '\n' + + '.ser' + '#' + 'application/java-serialized-object' + '\n' + + '.sh' + '#' + 'application/x-sh' + '\n' + + '.sig' + '#' + 'application/pgp-signature' + '\n' + + '.silo' + '#' + 'model/mesh' + '\n' + + '.svg' + '#' + 'image/svg+xml' + '\n' + + '.t' + '#' + 'text/troff' + '\n' + + '.tar' + '#' + 'application/x-tar' + '\n' + + '.text' + '#' + 'text/plain' + '\n' + + '.tgz' + '#' + 'application/x-gzip' + '\n' + + '.tif' + '#' + 'image/tiff' + '\n' + + '.tiff' + '#' + 'image/tiff' + '\n' + + '.torrent'+ '#' + 'application/x-bittorrent' + '\n' + + '.ttc' + '#' + 'application/x-font-ttf' + '\n' + + '.ttf' + '#' + 'application/x-font-ttf' + '\n' + + '.txt' + '#' + 'text/plain' + '\n' + + '.urls' + '#' + 'text/uri-list' + '\n' + + '.v' + '#' + 'text/x-v' + '\n' + + '.wav' + '#' + 'audio/x-wav' + '\n' + + '.wmv' + '#' + 'video/x-ms-wmv' + '\n' + + '.woff2' + '#' + 'font/woff2' + '\n' + + '.xls' + '#' + 'application/vnd.ms-excel' + '\n' + + '.xlsx' + '#' + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + '\n' + + '.xml' + '#' + 'application/xml' + '\n' + + '.xps' + '#' + 'application/vnd.ms-xpsdocument' + '\n' + + + '' + ).slice(0, -1).split("\n").map((line) => line.split("#") as [string, string]), +); diff --git a/lib/node.ts b/lib/node.ts new file mode 100644 index 0000000000000000000000000000000000000000..87d34f0de6e7f776f6366a42ecebdaed287a5541 --- /dev/null +++ b/lib/node.ts @@ -0,0 +1,73 @@ +/** + * functions to load Node.js apis via `globalThis.process`. trivially bundlable + * for the browser. does not depend on `@types/node` and does not intend to + * define types for the entire api. Instead, this is used for other library + * modules like `lib/log.ts` to bind to the system. + * @module + */ + +export type ErrorCode = + | keyof typeof import("node:os").constants.errno + | (string & {}); + +export const process: Process | undefined = + (globalThis as typeof globalThis & { process?: Process }).process; + +interface Process { + getBuiltinModule(name: K): Builtins[K] | null; + binding(name: K): Bindings[K] | null; + addListener(event: string, callback: () => void): this; + + stdout: Tty; + stderr: Tty; +} + +interface Tty { + isTTY: boolean; + columns: number; + rows: number; + addListener(event: string, callback: () => void): this; + write(text: string | Uint8Array): void; +} + +/** + * Subset of Node.js builtin types needed to not depend on the problematic + * "@types/node" package. + */ +interface Builtins { + "fs": { + existsSync(file: string): boolean; + readFileSync(file: string, encoding: "utf-8"): string; + readFileSync(file: string): Uint8Array; + }; + "path": { + isAbsolute(file: string): boolean; + dirname(file: string): string; + basename(file: string): string; + join(...parts: string[]): string; + sep: string; + }; +} +/** + * Subset of Node.js binding types + */ +interface Bindings { + /** + * key value mapping of internal module id to its source code. + * + * { "fs": "// Copyright Joyent, Inc. and other Node contributors.\n... + */ + "natives": Partial>; +} + +export function builtin(name: K): Builtins[K] | null { + return process?.getBuiltinModule(name) ?? null; +} + +export function binding(name: K): Bindings[K] | null { + try { + return process?.binding(name) ?? null; + } catch { + return null; + } +} diff --git a/lib/packaging.ts b/lib/packaging.ts deleted file mode 100644 index 9e0bc826f1ea953feaeba00f148714872b33d8c3..0000000000000000000000000000000000000000 --- a/lib/packaging.ts +++ /dev/null @@ -1,130 +0,0 @@ -// Tools that `lib` uses to package itself for NPM. -// This depends on an installation of `ts-to-jsdoc` available as a resolvable node module. - -export async function updateJsrPackageExportsToAll( - absPath: string, - p: progress.Ref, -) { - using _ = p.start("update deno.json"); - const manifestPath = path.join(absPath, "deno.json"); - const deno = JSON.parse( - await fs.readFile(manifestPath, "utf-8"), - ); - const newExports: Record = {}; - for (const subPath of await fs.readdir(absPath, { recursive: true })) { - if (!subPath.match(/\.tsx?$/)) continue; - if (subPath.match(/\.(test|sample)\.tsx?$/)) continue; - if (subPath.startsWith("dist")) continue; - newExports["./" + subPath] = "./" + subPath; - } - deno.exports = Object.fromEntries( - Object.entries(newExports).sort(([a], [b]) => a.localeCompare(b)), - ) as typeof deno.exports; - await fs.writeFile(manifestPath, JSON.stringify(deno, null, 2)); -} - -interface TsToJsdoc { - transpileFile: (opts: { code: string }) => string; -} - -export async function transpileForNpm( - absPath: string, - p: progress.Ref, - packageJson: Record, - tsToJsdoc?: TsToJsdoc, -) { - const libName = await Promise.resolve("ts-to-jsdoc"); - try { - tsToJsdoc ??= await import(libName) as TsToJsdoc; - } catch { - tsToJsdoc ??= await import("npm:" + libName + "@2.4.0") as TsToJsdoc; - } - const { transpileFile } = tsToJsdoc; - - using node = p.start("transpile into jsdoc for npm"); - const manifestPath = path.join(absPath, "deno.json"); - const deno = JSON.parse( - await fs.readFile(manifestPath, "utf-8"), - ); - const distPath = path.join(absPath, "dist-npm"); - await fs.rm(distPath, { recursive: true, force: true }); - await fs.mkdir(distPath, { recursive: true }); - - const files: string[] = []; - const exports: Record = {}; - for (const subPath of await fs.readdir(absPath, { recursive: true })) { - if (!subPath.match(/\.tsx?$/)) continue; - if (subPath.startsWith("dist")) continue; - files.push(subPath); - } - node.estimate = Object.keys(exports).length; - for (const subPath of files) { - using _ = node.start(subPath); - const targetSubPath = subPath.replace(/\.tsx?$/, ".js"); - if (!subPath.match(/\.(test|sample)\.tsx?$/)) { - exports["./" + subPath] = "./" + targetSubPath; - } - - let code = transpileFile({ - code: await fs.readFile(path.join(absPath, subPath), "utf-8"), - }); - code = code.replace( - / from "lib\/([^"]+)"/g, - (_, src) => { - return ' from "./' + path.relative( - path.dirname(subPath), - src.replace(/\.tsx?$/, ".js"), - ) + '"'; - }, - ); - if (packageJson.homepage) { - code = - "// This file is transpiled from TypeScript from paper clover's sitegen\n" + - "// " + packageJson.homepage + "/" + subPath + "\n" + - code; - } - const out = path.join(distPath, targetSubPath); - await fs.mkdir(path.dirname(out), { recursive: true }); - await fs.writeFile(out, code); - node.value += 1; - } - await fs.cp( - path.join(absPath, "readme.npm.md"), - path.join(distPath, "readme.md"), - ).catch((e) => { - if (e.code !== "ENOENT") throw e; - }); - await fs.cp( - path.join(absPath, "readme.changes.md"), - path.join(distPath, "readme.changes.md"), - ).catch((e) => { - if (e.code !== "ENOENT") throw e; - }); - const { name, description, ...rest } = packageJson; - await fs.writeFile( - path.join(distPath, "package.json"), - JSON.stringify({ - name, - description, - version: deno.version, - type: "module", - ...rest, - }), - ); -} - -if (import.meta.main) { - updateJsrPackageExportsToAll(import.meta.dirname!, progress) - .then(() => - transpileForNpm(import.meta.dirname!, progress, { - name: "@paperclover/lib", - description: "high quality typescript libraries", - homepage: - "https://git.paperclover.net/clo/sitegen/src/branch/master/lib", - }) - ); -} - -import * as progress from "lib/log/progress.ts"; -import * as path from "node:path"; -import * as fs from "node:fs/promises"; diff --git a/lib/queue.ts b/lib/queue.ts index c3ec818699c2fd8b0bafb0c240152cd97fa81725..aedbeeca36c6147062242d43e4b01523d883643e 100644 --- a/lib/queue.ts +++ b/lib/queue.ts @@ -1,5 +1,8 @@ -// Implements a priority queue. Jobs can be automatically cancelled via -// AbortSignal rescheduled if higher priority tasks come in. +/** + * implements a priority queue. jobs are be automatically cancelled via + * `AbortSignal` and rescheduled when higher priority tasks come in. + * @module + */ /** schedule `item.run` to be called at some point on the global queue */ export function run(item: Job): Promise { @@ -28,10 +31,15 @@ export interface Job { } export class PriorityQueue { + /** Read only */ concurrency: number; + /** Read only */ coresRemain: number; + /** Read only */ jobs: Array = []; + /** Read only */ active: Array = []; + /** Can be assigned to react to queue changes. */ onChange: (() => void) | null = null; constructor(concurrency: number) { @@ -165,6 +173,7 @@ function insertSorted(arr: T[], item: T) { arr.push(item); } +declare var navigator: { hardwareConcurrency: number }; const global = new PriorityQueue(navigator.hardwareConcurrency); import { UNWRAP } from "lib/assert.ts"; diff --git a/lib/readme.changes.md b/lib/readme.changes.md index 5680cecee0eef32ffc7770a934b9910621aa708f..ab60a01103611a0691b1ddaa1895f750edeaf7b2 100644 --- a/lib/readme.changes.md +++ b/lib/readme.changes.md @@ -2,18 +2,33 @@ ## v2 -- top level readme now explains general usage patterns. +### breaking + - rework `log.Scope`, remove self call signature so that the `log` module satisfies `log.Scope`, while adding `log` and `debug` for explicit debugging logs. documentation prescribes doing `const console = log.scope("module");`. -- add `stack.capture()` +- remove `packaging.ts` +- move `log/ansi.ts` into `string/ansi.ts` +- move ansi related `string.ts` functions into `string/ansi.ts` + +### features + +- top level readme now explains general usage patterns. more docs overall. +- new library `render` +- new library `async.ts` +- new library `Lru.ts` +- new library `mime.ts` +- new library `render.ts`, `render/jsx-runtime.ts`, `render/meta.ts` +- new library `node.ts` +- new library `subprocess.ts`, `subprocess/ffmpeg.ts` - add `string.escapeRegExp()` +- add `stack.capture()` ## v1 -- new library `assert` -- new library `log` -- new library `packaging` -- new library `queue` -- new library `string` -- new library `ts` +- new library `assert.ts` +- new library `log.ts` +- new library `packaging.ts` +- new library `queue.ts` +- new library `string.ts` +- new library `ts.ts` diff --git a/lib/readme.md b/lib/readme.md index 66e7b24d2573bed22676e14df6520b506faf71ce..53639743a37e9c312fdf9bf30df0d0167bfeb61e 100644 --- a/lib/readme.md +++ b/lib/readme.md @@ -12,23 +12,26 @@ projects and at work. all top-level namespaces stand alone as their own projects, and generally do not depend on each other. there are no external dependencies; code is easy to audit. - - - - - - - `assert.ts` +- `async.ts` +- `error.ts` - interop with the `unknown` type in error handlers - `log.ts` - logging, terminal i/o, interactive widgets -- `log/ansi.ts` - cursor and color constants - `log/headless.ts` - use `log` and `log/progress` without a tty - `log/progress.ts` - progress bar, spinner, and tree - `log/stack.ts` - stack trace parse and formatter -- `packaging.ts` - workflows used for `lib`'s publications +- `Lru.ts` - least recently used cache +- `mime.ts` - trimmed database of ext -> mime types +- `node.ts` - the Node.js javascript runtime - `queue.ts` - priority queue with rescheduling +- `render.ts` - html generation with extensions. +- `render/jsx-runtime.ts` - integration with JSX syntax +- `render/meta.ts` - html meta tag generation - `string.ts` +- `string/ansi.ts` - cursor and color constants +- `subprocess.ts` - rebuilt api for `node:child_process` +- `subprocess/ffmpeg.ts` - spawn and parse progress output - `ts.ts` typescript types and general purpose code there is currently no hosted documentation site. please refer to the source @@ -52,7 +55,9 @@ library separately: ```ts import * as log from "@clo/lib/log.ts"; -import * as progress from "@clo/lib/log/progress.ts"; +// since the namespace cannot be a constructor, capitalized +// filenames indicate the single export to use. +import { Lru } from "@clo/lib/Lru.ts"; ``` these patterns developed initially when using a text editor without a language diff --git a/lib/readme.npm.md b/lib/readme.npm.md index ac6951c39461f1c0220f93372ccf5d2e52b4e343..a2890434830c4a077ee1f4510223ce070bb9ce31 100644 --- a/lib/readme.npm.md +++ b/lib/readme.npm.md @@ -1,6 +1,6 @@ ```sh # alias install "@paperclover/lib" as "@clo/lib" -npm install lib@npm:@paperclover/lib +npm install @clo/lib@npm:@paperclover/lib ``` the distribution on npm is transpiled with JSDoc annotations instead of diff --git a/lib/render.ts b/lib/render.ts new file mode 100644 index 0000000000000000000000000000000000000000..e26b182987ef60159fee6875230cfbb457687572 --- /dev/null +++ b/lib/render.ts @@ -0,0 +1,466 @@ +/** + * Clover's rendering engine is the backbone of her website generator. It + * converts objects and components (functions returning {@linkcode Node}) into + * HTML. the engine is simple, self-contained, and easily extendable. + * + * extensions to the rendering engine can made via {@linkcode Context}. for + * example, 'sitegen' uses a context to track needed client scripts without + * introducing patches to {@linkcode State}. + * + * the easiest way to start is with the JSX adapter `./jsx-runtime.ts` and the + * synchronous rendering function {@linkcode sync|render.sync}: + * + * ```tsx + * import * as render from "@clo/lib/render.ts"; + * console.info(render.sync(

good morning

)); + * // jsx is equivilent to: + * console.info(render.sync(render.element("h1", { children: "good morning" }))); + * ``` + * + * NOTE: This module is not production ready, but it is in use at https://paperclover.net + * + * @module + */ + +/* convert a UI description into a string synchronously. + * optional `context` argument pre-populates render context. */ +export function sync(node: Node, contexts?: ContextValue[]): Result { + const state = init(false, contexts ?? []); + const resolved = resolveNode(state, node); + return { text: stringifyNode(resolved), context: state.context }; +} + +/* convert a UI description into a string asynchronously. + * optional `context` argument pre-populates render context. */ +export function async(node: Node, contexts?: ContextValue[]): Promise { + const state = init(true, contexts ?? []); + const resolved = resolveNode(state, node); + const { context } = state; + if (!state.async) { + return Promise.resolve({ text: stringifyNode(resolved), context }); + } + const { resolve, reject, promise } = Promise.withResolvers(); + state.asyncDone = () => { + const { rejections } = state; + if (!rejections) return resolve({ text: stringifyNode(resolved), context }); + if (rejections.length === 1) return reject(rejections[0]); + return reject(new AggregateError(rejections)); + }; + return promise; +} + +interface Result { + /** HTML/XML/RSS Text*/ + text: string; + /** For use with {@linkcode Context}. */ + context: Map; +} + +/** + * mechanism for extending the rendering engine by providing the component tree + * with a shared context. + * + * ```tsx + * const exampleContext = new render.Context(() => ({ count: 0 })); + * + * function Write({ add }) { + * const example: { count: number } = exampleContext.get() + * example.count += add; + * return null; + * } + * function Read() { + * const { count } = exampleContext.get() + * return `Value is: ${count}`; + * } + * + * render.sync([ + * + * + * + * ]); // -> "Value is: 42" + * ``` + */ +export class Context { + #default: (() => T) | null; + + constructor(init?: () => T) { + this.#default = init ?? null; + } + + /** + * Allow pre-populating a render's context + * + * ```tsx + * const state = exampleContext.init({ count: 5 }); + * void render.sync(, [state]); + * state.count; // 10 + * ``` + */ + init(value: T | void): ContextValue { + value ??= UNWRAP(this.#default)(); + (value as ContextValue)[kContext] = this; + return value as ContextValue; + } + + /** + * retrieve the context value from + * - passing `undefined`, retrieves from current render + * - passing a `Result`, typesafe access + * + * due to the implementation using simple global state, this function will + * not work within `async` components. + */ + get(result: Result): T | null; + get(): T; + get(result?: Result): T | null { + if (result) return (result.context.get(this) ?? null) as T | null; + const { context } = UNWRAP(current, "Not in an active render."); + let value = (context.get(this) ?? null) as T | null; + if (!value) { + const init = this.#default; + context.set(this, value = UNWRAP(init, "No init function")()); + } + return value; + } +} + +export type ContextValue = T & { + [kContext]: Context; +}; + +/** the global state */ +export let current: State | null = null; + +/** set the global state */ +export function setCurrent(r: State | null) { + current = r ?? null; +} + +/** inline HTML into a render without escaping it */ +export function raw(node: ResolvedNode): Raw { + return [kRaw, node]; +} + +/** constructs {@linkcode Element} */ +export function element( + type: Element[1], + attrs: Element[2] = {}, + source?: SrcLoc, +): Element { + return source ? [kElement, type, attrs, "", source] : [kElement, type, attrs]; +} + +/** constructs {@linkcode ResolvedElement} */ +export function resolvedElement( + type: ResolvedElement[1], + attrs: ResolvedElement[2], + children: ResolvedElement[3], +): ResolvedElement { + return [kElement, type, attrs, children]; +} + +/** internal global state while rendering */ +export interface State { + /** + * set to '-1' if rendering synchronously + * number of async promises the render is waiting on. + */ + async: number | -1; + asyncDone: null | (() => void); + /** when components reject, those are collected here */ + rejections: unknown[] | null; + context: Map; +} + +/** `Node` represents a webpage that can be "rendered' into HTML. */ +export type Node = + | number + | string // Escape HTML + | Node[] // Concat + | Element // Stringify + | Raw // Insert + | Promise // Await + // Ignore + | (undefined | null | boolean); +/** `Element` represents a component call or HTML element. */ +export type Element = [ + tag: typeof kElement, + type: string | Component, + attrs: Record, + _?: "", // contents + source?: SrcLoc, +]; +export type Raw = [tag: typeof kRaw, html: ResolvedNode]; +/** + * components must return a value; 'undefined' is prohibited here + * to avoid functions that are missing a return statement. + */ +export type Component = (attrs: Record) => Exclude; +/** emitted by JSX runtime */ +export interface SrcLoc { + fileName: string; + lineNumber: number; + columnNumber: number; +} + +/** + * resolution narrows the type {@linkcode Node} into {@linkcode ResolvedNode}. + * Async trees are marked in {@linkcode State}. This operation performs + * everything besides the final string concatenation. This function is agnostic + * across {@linkcode async}/{@linkcode sync} modes. + */ +export function resolveNode(r: State, node: unknown): ResolvedNode { + if (!node && node !== 0) return ""; // falsy, non numeric + if (typeof node !== "object") { + if (node === true) return ""; // booleans are ignored + if (typeof node === "string") return escapeHtmlContent(node); + if (typeof node === "number") return String(node); // no escaping ever + if (typeof node === "symbol" && node.toString() === kElement.toString()) { + throw new Error(`There are two instances of Clover SSR loaded!`); + } + throw new Error(`Cannot render ${inspect(node)} to HTML`); + } + if (node instanceof Promise) { + if (r.async === -1) { + throw new Error(`Asynchronous rendering is not supported here.`); + } + const placeholder: InsertionPoint = [null]; + r.async += 1; + node + .then((result) => void (placeholder[0] = resolveNode(r, result))) + // Intentionally catching errors in `resolveNode` + .catch((e) => (r.rejections ??= []).push(e)) + .finally(() => { + if (--r.async == 0) { + if (r.asyncDone == null) throw new Error("r.asyncDone == null"); + r.asyncDone(); + r.asyncDone = null; + } + }); + // This lie is checked with an assertion in `renderNode` + return placeholder as [ResolvedNode]; + } + if (!Array.isArray(node)) { + throw new Error(`Invalid node type: ${inspect(node)}`); + } + const type = node[0]; + if (type === kElement) { + const { 1: tag, 2: attrs } = node; + if (typeof tag === "function") { + current = r; + try { + return resolveNode(r, tag(attrs)); + } catch (e) { + const { 4: src } = node; + if (e && typeof e === "object") (e as { src?: string }).src ??= src; + throw e; + } finally { + current = null; + } + } + if (typeof tag !== "string") throw new Error("Unexpected " + inspect(type)); + const children = attrs?.children; + if (children) return [kElement, tag, attrs, resolveNode(r, children)]; + return node; + } + if (type === kRaw) return node[1]; + return node.map((elem) => resolveNode(r, elem)); +} + +export type ResolvedNode = + | ResolvedNode[] // Concat + | ResolvedElement // Render + | string; // Raw HTML +export type ResolvedElement = [ + tag: typeof kElement, + type: string, + attrs: Record, + children: ResolvedNode, +]; +/** + * async rendering is done by creating an array of one item, which is already a + * valid {@linkcode Node}, but the element is written once the data is + * available. {@linkcode State} contains a count of how many async jobs are + * left. + */ +export type InsertionPoint = [null | ResolvedNode]; + +/** + * convert {@linkcode ResolvedNode} into HTML text. This operation happens + * after all async work is settled. markup is emitted as concisely as possible. + */ +export function stringifyNode(node: ResolvedNode): string { + ASSERT(node != null, "Unresolved Render Node"); + if (typeof node === "string") return node; + const type = node[0]; + if (type === kElement) { + return stringifyElement(node as ResolvedElement); + } + node = node as ResolvedNode[]; // TS cannot infer. + let out = type ? stringifyNode(type) : ""; + const len = node.length; + for (let i = 1; i < len; i++) { + const elem = node[i]; + if (elem) out += stringifyNode(elem); + } + return out; +} + +function stringifyElement(element: ResolvedElement) { + const { 1: tag, 2: attrs, 3: children } = element; + let out = "<" + tag; + for (const prop in attrs) { + const value = attrs[prop]; + if (!value || typeof value === "function") continue; + let attr; + switch (prop) { + default: + attr = `${prop}=${quoteIfNeeded(escapeAttribute(String(value)))}`; + break; + case "className": // Legacy React Compat + case "class": { + const str = clsx(value as ClassName); + if (!str) continue; + attr = `class=${quoteIfNeeded(escapeAttribute(str))}`; + break; + } + case "style": { + attr = typeof value === "string" + ? `style=${quoteIfNeeded(escapeAttribute(value))}` + : stringifyStyleAttribute(value as Record); + break; + } + case "htmlFor": + throw new Error("Do not use the `htmlFor` attribute. Use `for`"); + case "dangerouslySetInnerHTML": + throw new Error( + "Do not use the `dangerouslySetInnerHTML` attribute. Use `render.raw()`", + ); + // Do not process these + case "children": + case "ref": + case "key": + continue; + } + if (!out.endsWith('"')) out += " "; + out += attr; + } + out = out.trimEnd() + ">"; + if (children) out += stringifyNode(children); + if ( + tag !== "br" && + tag !== "img" && + tag !== "input" && + tag !== "meta" && + tag !== "link" && + tag !== "hr" + ) { + out += ``; + } + return out; +} + +function stringifyStyleAttribute(style: Record): string { + let out = ``; + for (const styleName in style) { + if (out) out += ";"; + out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${ + escapeAttribute( + String(style[styleName]), + ) + }`; + } + return "style=" + quoteIfNeeded(out); +} + +export function quoteIfNeeded(text: string): string { + if (text.match(/["/> ]/)) return '"' + text + '"'; + return text; +} + +// -- utility functions -- + +/** Construct {@link State} */ +export function init( + allowAsync: boolean, + context: ContextValue[] | Map, +): State { + return { + async: allowAsync ? 0 : -1, + rejections: null, + asyncDone: null, + context: Array.isArray(context) + ? new Map(context.map((value) => [value[kContext], value])) + : context, + }; +} + +export type ClassName = string | Record | ClassName[]; +export function clsx(mix: ClassName): string { + let k, y, str = ""; + if (typeof mix === "string") { + return mix; + } else if (typeof mix === "object") { + if (Array.isArray(mix)) { + for (k = 0; k < mix.length; k++) { + if (mix[k] && (y = clsx(mix[k]))) { + str && (str += " "); + str += y; + } + } + } else { + for (k in mix) { + if (mix[k]) { + str && (str += " "); + str += k; + } + } + } + } + return str; +} + +// TODO: combine into one function which decides if an attribute needs quotes +// and escapes it correctly depending on the context. + +/** @deprecated */ +export const escapeHtmlContent = (unsafeText: string): string => + String(unsafeText) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +/** @deprecated */ +const escapeAttribute = (unsafeText: string): string => + String(unsafeText) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +/** @deprecated */ +export const escapeHtml = (unsafeText: string): string => + String(unsafeText) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'") + .replace(/`/g, "`"); + +/** @deprecated */ +export function inspect(object: unknown): string { + try { + return require("node:util").inspect(object); + } catch { + return typeof object; + } +} + +/** descriminator for {@linkcode Element} */ +const kElement = Symbol("Element"); +/** descriminator for {@linkcode Raw} */ +const kRaw = Symbol("Raw"); +/** secret key to facilitate {@linkcode init} with a `ContextValue[]` */ +const kContext: unique symbol = Symbol(); + +import { ASSERT, UNWRAP } from "lib/assert.ts"; diff --git a/lib/render/jsx-runtime.ts b/lib/render/jsx-runtime.ts new file mode 100644 index 0000000000000000000000000000000000000000..b0df60f1bef0fdb47440f0944b374d1e04c35b60 --- /dev/null +++ b/lib/render/jsx-runtime.ts @@ -0,0 +1,79 @@ +/** @module + * provides JSX integration with `lib/render.ts`. + * + * because the jsr does not allow default global augmentations, type + * installation must be done manually by pasting this in your project: + * + * declare global { + * namespace JSX { + * export type IntrinsicElements = jsx.IntrinsicElements; + * export type ElementChildrenAttribute = jsx.ElementChildrenAttribute; + * export type Element = jsx.Element; + * export type ElementType = jsx.ElementType; + * export type ElementClass = jsx.ElementClass; + * } + * } + * import * as jsx from "@clo/lib/render/jsx-runtime.ts"; + */ + +/** this element returns just its contents, and is used by `<>` fragment syntax. */ +export function Fragment( + { children }: { children: render.Node[] }, +): render.Node[] { + return children; +} + +/** construct a JSX element in production mode. */ +export function jsx( + type: string | render.Component, + attrs: Record, +): render.Element { + if (typeof type !== "function" && typeof type !== "string") { + throw new Error("Invalid component type: " + render.inspect(type)); + } + return render.element(type, attrs); +} + +/** construct a JSX element in development mode. */ +export function jsxDEV( + type: string | render.Component, + attrs: Record, + // Unused with the clover engine + _key: string, + // Unused with the clover engine + _isStaticChildren: boolean, + source: render.SrcLoc, +): render.Element { + ASSERT( + source != null, + `Build tool does not emit correct JSX debugging information.`, + ); + const { fileName, lineNumber, columnNumber } = source; + + // Assert the component type is valid to render. + if (typeof type !== "function" && typeof type !== "string") { + throw new Error( + `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` + + render.inspect(type) + + ". Clover SSR element must be a function or string.", + ); + } + + return render.element(type, attrs, source); +} + +// Clover SSR engine does not yet distinguish between static and non-static JSX. +export { jsx as jsxs }; + +export interface IntrinsicElements { + [name: string]: Record; +} +export interface ElementChildrenAttribute { + children: render.Node; +} +export type Element = render.Element; +export type ElementType = keyof IntrinsicElements | render.Component; +export type ElementClass = ReturnType; + +import * as render from "lib/render.ts"; +import { ASSERT } from "lib/assert.ts"; diff --git a/lib/render/meta.test.tsx b/lib/render/meta.test.tsx new file mode 100644 index 0000000000000000000000000000000000000000..8ac85e3d9b02cc2b486c4d0878686d1a63540e75 --- /dev/null +++ b/lib/render/meta.test.tsx @@ -0,0 +1,49 @@ +// every metadata render requires a base URL +const base: meta.Template = { + base: new URL("https://paperclover.net"), +}; + +test("default", () => + assert.equal( + meta.render({ title: "my website <3" }, base), + "my website <3" + + '', + )); +test("some other meta tags", () => + assert.equal( + meta.render( + { + title: "hello", + description: "my website is a place to be", + canonical: "/", + }, + base, + ), + "hello" + + '' + + '' + + '', + )); +test("automatic embed with '{}'", () => + assert.equal( + meta.render( + { + title: "main title", + description: "main description", + embed: {}, + }, + base, + ), + "main title" + + '' + + '' + + "" + + '' + + '' + + "" + + "", + )); + +import { test } from "node:test"; +import assert from "node:assert"; +import * as meta from "lib/render/meta.ts"; diff --git a/lib/render/meta.ts b/lib/render/meta.ts new file mode 100644 index 0000000000000000000000000000000000000000..10a962f2638ac91029f36715d6918b5422ebf77b --- /dev/null +++ b/lib/render/meta.ts @@ -0,0 +1,255 @@ +export { renderMetadata as render }; +export interface Meta { + /** Required for all pages. `{content}` */ + title: string; + /** Recommended for all pages. `` */ + description?: string | null; + /** Automatically added for static renders from the 'pages' folders. */ + canonical?: string | null; + /** Add ``. Object keys are interpretted as + * mime types if they contain a slash, otherwise seen as an alternative language. */ + alternates?: Alternate[] | Record; + + /** Automatically generate both OpenGraph and Twitter meta tags */ + embed?: AutoEmbed | null; + /** Add a robots tag for `noindex` and `nofollow` */ + denyRobots?: boolean | null; + /** Add 'og:*' meta tags */ + openGraph?: OpenGraph | null; + /** Add 'twitter:*' meta tags */ + twitter?: Twitter | null; + /** Refer to an oEmbed file. See https://oembed.com */ + oEmbed?: string; + /** + * '#sitegen/meta' intentionally excludes a lot of exotic tags. + * Add these manually using JSX syntax: + * + * extra: [ + * , + * ], + */ + extra?: render.Node; + + /** Adds `` */ + authors?: string[]; + /** Credit sitegen by setting this to true aka "clover sitegen 3; paperclover.net/sitegen" */ + generator?: string | true | null; + /** Adds `` */ + keywords?: string[]; + /** URL to a manifest; https://developer.mozilla.org/en-US/docs/Web/Manifest */ + manifest?: string | null; + /** Adds `` */ + publisher?: string | null; + /** https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/meta/name/referrer */ + referrer?: Referrer | null; + /** Adds `` */ + themeColor?: string | { dark: string; light: string } | null; + /** Defaults to `width=device-width, initial-scale=1.0` for mobile compatibility. */ + viewport?: string; +} +export type Alternate = { type: string; url: string } | { + lang: string; + url: string; +}; +export interface AutoEmbed { + /* Defaults to the page title. */ + title?: string | null; + /* Defaults to the page description. */ + description?: string | null; + /* Provide to add an embed image. */ + thumbnail?: string | null; + /** @default "banner", which applies twitter:card = "summary_large_image" */ + thumbnailSize?: "banner" | "icon"; + /* Ignored if not passed */ + siteTitle?: string | null; +} +/** See https://ogp.me for extra rules. */ +export interface OpenGraph { + /** The title of your object as it should appear within the graph */ + title?: string; + /** A one to two sentence description of your object. */ + description?: string | null; + /** The type of your object, e.g., "video.movie". Depending on the type you specify, other properties may also be required */ + type?: string; + /** An image URL which should represent your object within the graph */ + image?: OpenGraphField; + /** The canonical URL of your object that will be used as its permanent ID in the graph, e.g., "https://www.imdb.com/title/tt0117500/" */ + url?: string; + /** A URL to an audio file to accompany this object */ + audio?: OpenGraphField; + /** The word that appears before this object's title in a sentence. An enum of (a, an, the, "", auto). If auto is chosen, the consumer of your data should choose between "a" or "an". Default is "" (blank) */ + determiner?: string; + /** The locale these tags are marked up in. Of the format language_TERRITORY. Default is en_US */ + locale?: string; + /** An array of other locales this page is available in */ + "locale:alternate"?: string[]; + /** If your object is part of a larger web site, the name which should be displayed for the overall site. e.g., "IMDb" */ + site_name?: string; + /** A URL to a video file that complements this object */ + video?: OpenGraphField; + [field: string]: OpenGraphField; +} +/** + * When passing an array, the property is duplicated. + * When passing an object, the fields are emitted as namespaced with ':'. + */ +type OpenGraphField = + | string + | { [field: string]: OpenGraphField } + | Array + | (null | undefined); +/** Twitter uses various OpenGraph fields if these are not specified. */ +export interface Twitter { + card: string; + title?: string; + description?: string | null; + url?: string; + image?: string; + player?: string; + /** Same logic as Open Graph */ + [field: string]: OpenGraphField; +} +export interface Template + extends Omit { + base: URL; + titleTemplate?: (title: string) => string; +} +export interface Alternates { + canonical: string; + types: { [mime: string]: AlternateType }; +} +export interface AlternateType { + url: string; + title: string; +} +export type Referrer = + | "no-referrer" + | "origin" + | "no-referrer-when-downgrade" + | "origin-when-cross-origin" + | "same-origin" + | "strict-origin" + | "strict-origin-when-cross-origin"; + +/* Convert a metadata definition into text. */ +function renderMetadata(meta: Meta, template: Template): string { + const { titleTemplate, base } = template; + const resolve = (str: string) => new URL(str, base).href; + + const title = titleTemplate ? titleTemplate(meta.title) : meta.title; + const description = meta.description ?? null; + const canonical = meta.canonical ? resolve(meta.canonical) : null; + const denyRobots = Boolean(meta.denyRobots || template.denyRobots); + const authors = meta.authors ?? template.authors ?? null; + let generator = meta.generator ?? template.generator ?? null; + const keywords = meta.keywords ?? template.keywords ?? null; + const manifest = meta.manifest ?? template.manifest ?? null; + const publisher = meta.publisher ?? template.publisher ?? null; + const referrer = meta.referrer ?? template.referrer ?? null; + const themeColor = meta.themeColor ?? template.themeColor ?? null; + const viewport = meta.viewport ?? template.viewport ?? + "width=device-width, initial-scale=1.0"; + + const embed = meta.embed ?? template.embed ?? null; + let openGraph = meta.openGraph ?? template.openGraph ?? null; + let twitter = meta.twitter ?? template.twitter ?? null; + if (embed) { + const { thumbnail, thumbnailSize, siteTitle } = embed; + openGraph = { + type: "website", + title: embed.title ?? title, + description: embed.description ?? description, + ...openGraph, + }; + twitter = { + card: (thumbnailSize ?? (thumbnail ? "banner" : "icon")) === "banner" + ? "summary_large_image" + : "summary", + ...twitter, + }; + if (thumbnail) { + openGraph.image = embed.thumbnail; + } + if (siteTitle) { + openGraph.site_name = siteTitle; + } + if (canonical) { + openGraph.url = canonical; + } + } + + let out = `${esc(title)}`; + if (description) { + out += ``; + } + for (const author of authors ?? []) { + out += ``; + } + if (keywords) { + out += ``; + } + if (generator) { + if (generator === true) { + generator = "clover sitegen 3; paperclover.net/sitegen"; + } + out += ``; + } + if (publisher) { + out += ``; + } + if (referrer) { + out += ``; + } + if (themeColor) { + if (typeof themeColor === "string") { + out += ``; + } else { + out += '`; + out += '`; + } + } + if (denyRobots) { + out += ``; + } + if (viewport) { + out += ``; + } + if (canonical) { + out += ``; + } + if (manifest) { + out += ``; + } + + if (openGraph) out += renderOpenGraph("og", openGraph); + if (twitter) out += renderOpenGraph("twitter", twitter); + + if (meta.extra) out += render.sync(meta.extra); + if (template.extra) out += render.sync(template.extra); + + return out; +} + +function renderOpenGraph(prefix: string, value: OpenGraphField): string { + if (!value) return ""; + if (typeof value === "string") { + return ``; + } + if (Array.isArray(value)) { + return value + .map((item) => renderOpenGraph(prefix, item)) + .join(""); + } + return Object.entries(value) + .map(([key, item]) => renderOpenGraph(`${prefix}:${key}`, item)) + .join(""); +} + +function attr(value: string) { + return render.quoteIfNeeded(esc(value)); +} + +import { escapeHtmlContent as esc } from "lib/render.ts"; +import * as render from "lib/render.ts"; diff --git a/lib/string.test.ts b/lib/string.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..f6184a3fff465ef1000171c7a4c13c4adc42a088 --- /dev/null +++ b/lib/string.test.ts @@ -0,0 +1,35 @@ +test("countNewlines", () => { + assert.equal(string.countNewlines("line 1\nline 2\nline 3"), 2); + assert.equal(string.countNewlines(""), 0); +}); +test("escapeRegExp", () => { + assert.equal( + string.escapeRegExp("delete * from users where id = $id"), + "delete \\* from users where id = \\$id", + ); +}); +test("escapeShellArgument", () => { + // escapeShellArgument leaves it alone when the input is trivial + assert.equal(string.escapeShellArgument("test"), "test"); + assert.equal(string.escapeShellArgument("abc/file.@.txt"), "abc/file.@.txt"); + + // it adds quotes when there are spaces and globs + assert.equal( + string.escapeShellArgument("hello world"), + "'hello world'", + ); + + // handles other cases + assert.equal( + string.escapeShellArgument( + 'delete * from "users" where "username" = \'clover\'', + ), + "'delete * from \"users\" where \"username\" = '\\''clover'\\'", + ); + assert.equal(string.escapeShellArgument("'string"), '"\'string"'); + assert.equal(string.escapeShellArgument("'str\"ing"), "\\''str\"ing'"); +}); + +import { test } from "node:test"; +import assert from "node:assert"; +import * as string from "lib/string.ts"; diff --git a/lib/string.ts b/lib/string.ts index aa6543f42fb3030341e61aaf858cdfd883fe0ec9..969aefea9f98a372a7020d4913183e544ab9f916 100644 --- a/lib/string.ts +++ b/lib/string.ts @@ -1,8 +1,3 @@ -/** Escape a value for use in a regular expression. */ -export function escapeRegExp(unsafeText: string) { - return unsafeText.replace(/[\$\\]/g, "\\$&"); -} - // TODO: this function is not benchmarked export function countNewlines(str: string): number { let count = 0; @@ -12,160 +7,28 @@ export function countNewlines(str: string): number { return count; } -export function widthInTerminal(str: string): number { - let width = 0; - if (str.includes("\x1b")) str = str.replace(ansiEscapes, ""); - for (let i = 0, { length } = str; i < length; i += 1) { - const code = str.charCodeAt(i); - if (code < 32) continue; - if (code < 127) { - width += 1; - continue; - } - for (const { segment } of segmenter.segment(str.slice(i))) { - if (segment.length === 1) { - if (code < 32) continue; - if (code < 127) { - width += 1; - continue; - } - } - width += segmentWidth(segment); - } - return width; - } - return width; +/** Escape a string for use in a regular expression. */ +export function escapeRegExp(unsafeText: string): string { + // TODO: in May 2025, RegExp.escape exists + return unsafeText.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } -export function trimToWidth(str: string, columns: number): string { - let width = 0; - let ansi = false; - for (let i = 0, { length } = str; i < length; i += 1) { - const code = str.charCodeAt(i); - if (code < 32) { - if (code === 0x1b) { - const esc = str.slice().match(ansiEscapes); - if (esc) i += esc[0].length - 1, ansi = true; - } - continue; - } - if (code < 127) { - width += 1; - if (width >= columns) { - const result = str.slice(0, i + 1); - return ansi ? result + "\x1b[0m" : result; - } - continue; - } - // non-ascii path - let out = str.slice(0, i); - str = str.slice(i); - const segments = segmenter.segment(str); - { - const { segment } = UNWRAP(segments.containing(0)); - width += segmentWidth(segment); - if (width > columns) return ansi ? out + "\x1b[0m" : out; - out += segment; - i = segment.length; - } - for ({ length } = str; i < length; i += 1) { - const code = str.charCodeAt(i); - if (code === 0x1b) { - const esc = str.slice(i).match(ansiEscapes); - if (esc) { - i += esc[0].length - 1, ansi = true; - out += esc[0]; - continue; - } - } - const { segment } = UNWRAP(segments.containing(i)); - width += segmentWidth(segment); - if (width > columns) return ansi ? out + "\x1b[0m" : out; - out += segment; - } - return out; - } - return str; -} - -// derived from sindresorhus's wonderful `string-width` -// https://github.com/sindresorhus/string-width/blob/42e7b697393a9b9ff732f9bd3ab1db87c3c208b2/index.js -// https://github.com/sindresorhus/get-east-asian-width/blob/6aee3824b5e2ade910a38ed93cf9d89b7b5a8e5b/index.js -// MIT license -const segmenter = /* @__PURE__ */ new Intl.Segmenter(); -const ansiEscapes = - /[\u001B\u009B][[\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\d\/\#&.:=?%@~_]+)*|[a-zA-Z\d]+(?:;[-a-zA-Z\d\/\#&.:=?%@~_]*)*)?(?:\u0007|\u001B\u005C|\u009C))|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g; -const zeroWidthClusterRegex = - /^(?:\p{Default_Ignorable_Code_Point}|\p{Control}|\p{Mark}|\p{Surrogate})+$/v; -const leadingNonPrintingRegex = - /^[\p{Default_Ignorable_Code_Point}\p{Control}\p{Format}\p{Mark}\p{Surrogate}]+/v; -const rgiEmojiRegex = /^\p{RGI_Emoji}$/v; -function segmentWidth(segment: string) { - if (zeroWidthClusterRegex.test(segment)) return 0; - if (rgiEmojiRegex.test(segment)) return 2; - - // deno-fmt-ignore - const list = [ - 0x3000, 0x231A, 0x231B, 0x2329, 0x232A, 0x23F0, 0x23F3, 0x25FD, 0x25FE, 0x2614, 0x2615, - 0x267F, 0x2693, 0x26A1, 0x26AA, 0x26AB, 0x26BD, 0x26BE, 0x26C4, 0x26C5, 0x26CE, 0x26D4, - 0x26EA, 0x26F2, 0x26F3, 0x26F5, 0x26FA, 0x26FD, 0x2705, 0x270A, 0x270B, 0x2728, 0x274C, - 0x274E, 0x2757, 0x27B0, 0x27BF, 0x2B1B, 0x2B1C, 0x2B50, 0x2B55, 0x1AFFD, 0x1AFFE, 0x1B132, - 0x1B155, 0x1F004, 0x1F0CF, 0x1F18E, 0x1F250, 0x1F251, 0x1F3F4, 0x1F440, 0x1F57A, 0x1F595, - 0x1F596, 0x1F5A4, 0x1F6CC, 0x1F6EB, 0x1F6EC, 0x1F7F0, 0x1FAC8, - ]; - const x = segment.replace(leadingNonPrintingRegex, "").codePointAt(0) ?? 0; - return 1 + +( - // full - x >= 0xFF01 && x <= 0xFF60 || x >= 0xFFE0 && x <= 0xFFE6 || - // wide - x >= 0x1100 && x <= 0x115F || x >= 0x23E9 && x <= 0x23EC || - x >= 0x2630 && x <= 0x2637 || x >= 0x2648 && x <= 0x2653 || - x >= 0x268A && x <= 0x268F || x >= 0x2753 && x <= 0x2755 || - x >= 0x2795 && x <= 0x2797 || x >= 0x2E80 && x <= 0x2E99 || - x >= 0x2E9B && x <= 0x2EF3 || x >= 0x2F00 && x <= 0x2FD5 || - x >= 0x2FF0 && x <= 0x2FFF || x >= 0x3001 && x <= 0x303E || - x >= 0x3041 && x <= 0x3096 || x >= 0x3099 && x <= 0x30FF || - x >= 0x3105 && x <= 0x312F || x >= 0x3131 && x <= 0x318E || - x >= 0x3190 && x <= 0x31E5 || x >= 0x31EF && x <= 0x321E || - x >= 0x3220 && x <= 0x3247 || x >= 0x3250 && x <= 0xA48C || - x >= 0xA490 && x <= 0xA4C6 || x >= 0xA960 && x <= 0xA97C || - x >= 0xAC00 && x <= 0xD7A3 || x >= 0xF900 && x <= 0xFAFF || - x >= 0xFE10 && x <= 0xFE19 || x >= 0xFE30 && x <= 0xFE52 || - x >= 0xFE54 && x <= 0xFE66 || x >= 0xFE68 && x <= 0xFE6B || - x >= 0x16FE0 && x <= 0x16FE4 || x >= 0x16FF0 && x <= 0x16FF6 || - x >= 0x17000 && x <= 0x18CD5 || x >= 0x18CFF && x <= 0x18D1E || - x >= 0x18D80 && x <= 0x18DF2 || x >= 0x1AFF0 && x <= 0x1AFF3 || - x >= 0x1AFF5 && x <= 0x1AFFB || x >= 0x1B150 && x <= 0x1B152 || - x >= 0x1B164 && x <= 0x1B167 || x >= 0x1B170 && x <= 0x1B2FB || - x >= 0x1D300 && x <= 0x1D356 || x >= 0x1D360 && x <= 0x1D376 || - x >= 0x1F191 && x <= 0x1F19A || x >= 0x1F200 && x <= 0x1F202 || - x >= 0x1F210 && x <= 0x1F23B || x >= 0x1F240 && x <= 0x1F248 || - x >= 0x1F260 && x <= 0x1F265 || x >= 0x1F300 && x <= 0x1F320 || - x >= 0x1F32D && x <= 0x1F335 || x >= 0x1F337 && x <= 0x1F37C || - x >= 0x1F37E && x <= 0x1F393 || x >= 0x1F3A0 && x <= 0x1F3CA || - x >= 0x1F3CF && x <= 0x1F3D3 || x >= 0x1F3E0 && x <= 0x1F3F0 || - x >= 0x1F3F8 && x <= 0x1F43E || x >= 0x1F442 && x <= 0x1F4FC || - x >= 0x1F4FF && x <= 0x1F53D || x >= 0x1F54B && x <= 0x1F54E || - x >= 0x1F550 && x <= 0x1F567 || x >= 0x1F5FB && x <= 0x1F64F || - x >= 0x1F680 && x <= 0x1F6C5 || x >= 0x1F6D0 && x <= 0x1F6D2 || - x >= 0x1F6D5 && x <= 0x1F6D8 || x >= 0x1F6DC && x <= 0x1F6DF || - x >= 0x1F6F4 && x <= 0x1F6FC || x >= 0x1F7E0 && x <= 0x1F7EB || - x >= 0x1F90C && x <= 0x1F93A || x >= 0x1F93C && x <= 0x1F945 || - x >= 0x1F947 && x <= 0x1F9FF || x >= 0x1FA70 && x <= 0x1FA7C || - x >= 0x1FA80 && x <= 0x1FA8A || x >= 0x1FA8E && x <= 0x1FAC6 || - x >= 0x1FACD && x <= 0x1FADC || x >= 0x1FADF && x <= 0x1FAEA || - x >= 0x1FAEF && x <= 0x1FAF8 || x >= 0x20000 && x <= 0x2FFFD || - x >= 0x30000 && x <= 0x3FFFD || x >= 0x1B000 && x <= 0x1B122 || - list.includes(x) - ); +/** Escape a string for use in a regular expression. */ +export function escapeRegExpReplacement(unsafeText: string): string { + return unsafeText.replace(/\$/g, "$$$$"); } +/** + * Escape a string for use in a POSIX shell. Prefers easier to read output vs + * simpler implementation. + */ export function escapeShellArgument(s: string): string { if (s.length === 0) return "''"; - if (/^[a-zA-Z0-9_\-.\/@,+=%:]+$/.test(s) && s[0] !== ":") return s; + if (/^[a-zA-Z0-9_\-./@,+=%:]+$/.test(s) && s[0] !== ":") return s; if (!s.includes("'")) return `'${s}'`; if (!/["$`\\]/.test(s)) return `"${s}"`; - return `'${s.replace(/'/g, "'\\''")}'`; + let complex = `'${s.replace(/'/g, "'\\''")}'`; + if (s.endsWith("'")) complex = complex.slice(0, -2); + if (s.startsWith("'")) complex = complex.slice(2); + return complex; } - -import { UNWRAP } from "lib/assert.ts"; diff --git a/lib/string/ansi.test.ts b/lib/string/ansi.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..e48423bf4675a6002c0004624cd8d544a8e0eb09 --- /dev/null +++ b/lib/string/ansi.test.ts @@ -0,0 +1,24 @@ +test("widthInTerminal", () => { + assert.equal( + ansi.widthInTerminal("snowmobile"), + 10, + ); + assert.equal( + ansi.widthInTerminal("🏳️‍⚧️ " + ansi.style(ansi.bold, "rights")), + 9, + ); +}); +test("trimToWidth", () => { + assert.equal( + ansi.trimToWidth("snowmobile", 6), + "snowmo", + ); + assert.equal( + ansi.trimToWidth("🏳️‍⚧️ " + ansi.style(ansi.bold, "rights"), 5), + "🏳️‍⚧️ " + ansi.bold + "ri" + ansi.reset, + ); +}); + +import { test } from "node:test"; +import assert from "node:assert"; +import * as ansi from "lib/string/ansi.ts"; diff --git a/lib/string/ansi.ts b/lib/string/ansi.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a54b671115034de86891b2fe6384f57ba2253c1 --- /dev/null +++ b/lib/string/ansi.ts @@ -0,0 +1,382 @@ +/** + * ANSI escape sequences can be used to style text in a tty or perform other + * interactions. This file contains many constants for escape codes, as well as + * functions for ones that take an argument, such as `cursorUp`. + * + * Note that this file only produces escape sequences, and does not yet feature + * detection or fallback code. + * + * @module + */ + +/** resets all ansi styles */ +export const reset: string = `\x1b[0m`; + +/** + * puts the text in bold. can be combined with `dim` and colors. + * reset with `resetWeight`. + */ +export const bold: string = `\x1b[1m`; +/** + * puts the text a more faded color, dimming it. can be combined with `bold` to + * produce bold-dim text. dim can be combined with any color to make a dimmed + * version of that color. reset with `resetWeight`. + */ +export const dim: string = `\x1b[2m`; +/** resets `bold` and `dim` */ +export const resetWeight: string = `\x1b[22m`; + +/** resets foreground color */ +export const fgReset: string = `\x1b[39m`; +/** sets the foreground color to black. on light mode terminals, this color may + * be visible. */ +export const fgBlack: string = `\x1b[30m`; +/** sets the foreground color to red */ +export const fgRed: string = `\x1b[31m`; +/** sets the foreground color to green */ +export const fgGreen: string = `\x1b[32m`; +/** sets the foreground color to yellow */ +export const fgYellow: string = `\x1b[33m`; +/** sets the foreground color to blue */ +export const fgBlue: string = `\x1b[34m`; +/** sets the foreground color to magenta */ +export const fgMagenta: string = `\x1b[35m`; +/** sets the foreground color to cyan */ +export const fgCyan: string = `\x1b[36m`; +/** + * sets the foreground color to (dark) white / grey / gray. this color is + * brighter than `fgBrightBlack`. consider `dim` for greyed out text and + * `fgReset` to set the default foreground color. + */ +export const fgWhite: string = `\x1b[37m`; +/** + * sets the foreground color to bright black / grey / gray. consider `dim` for + * greyed out text. + */ +export const fgBrightBlack: string = `\x1b[90m`; +/** sets the foreground color to bright red */ +export const fgBrightRed: string = `\x1b[91m`; +/** sets the foreground color to bright green */ +export const fgBrightGreen: string = `\x1b[92m`; +/** sets the foreground color to bright yellow */ +export const fgBrightYellow: string = `\x1b[93m`; +/** sets the foreground color to bright blue */ +export const fgBrightBlue: string = `\x1b[94m`; +/** sets the foreground color to bright magenta */ +export const fgBrightMagenta: string = `\x1b[95m`; +/** sets the foreground color to bright cyan */ +export const fgBrightCyan: string = `\x1b[96m`; +/** + * sets the foreground color to bright white. consider `dim` for greyed out + * text and `fgReset` to set the default foreground color. + */ +export const fgBrightWhite: string = `\x1b[97m`; +/** sets the foreground color to a color in the 256 color palette. */ +export function fg256(n: number): string { + return `\x1b[38;5;${n}m`; +} +/** sets the foreground color to an rgb color. */ +export function fgRgb(r: number, g: number, b: number): string { + return `\x1b[38;2;${r};${g};${b}m`; +} + +/** resets the cell background color. */ +export const bgReset: string = `\x1b[49m`; +/** sets the cell background to black. consider `bgReset` to clear the background. */ +export const bgBlack: string = `\x1b[40m`; +/** sets the cell background to red. */ +export const bgRed: string = `\x1b[41m`; +/** sets the cell background to green. */ +export const bgGreen: string = `\x1b[42m`; +/** sets the cell background to yellow. */ +export const bgYellow: string = `\x1b[43m`; +/** sets the cell background to blue. */ +export const bgBlue: string = `\x1b[44m`; +/** sets the cell background to magenta. */ +export const bgMagenta: string = `\x1b[45m`; +/** sets the cell background to cyan. */ +export const bgCyan: string = `\x1b[46m`; +/** + * sets the cell background to (dark) white / grey / gray. this color is + * brighter than `bgBrightBlack`. + */ +export const bgWhite: string = `\x1b[47m`; +/** + * sets the cell background to bright black. consider `bgReset` to clear the + * background. + */ +export const bgBrightBlack: string = `\x1b[100m`; +/** sets the cell background to bright red. */ +export const bgBrightRed: string = `\x1b[101m`; +/** sets the cell background to bright green. */ +export const bgBrightGreen: string = `\x1b[102m`; +/** sets the cell background to bright yellow. */ +export const bgBrightYellow: string = `\x1b[103m`; +/** sets the cell background to bright blue. */ +export const bgBrightBlue: string = `\x1b[104m`; +/** sets the cell background to bright magenta. */ +export const bgBrightMagenta: string = `\x1b[105m`; +/** sets the cell background to bright cyan. */ +export const bgBrightCyan: string = `\x1b[106m`; +/** sets the cell background to bright white. */ +export const bgBrightWhite: string = `\x1b[107m`; +/** sets the background color to a color in the 256 color palette. */ +export function bg256(n: number): string { + return `\x1b[48;5;${n}m`; +} +/** sets the background color to an rgb color. */ +export function bgRgb(r: number, g: number, b: number): string { + return `\x1b[48;2;${r};${g};${b}m`; +} + +// cursor management + +/** move the cursor up `n` units. */ +export function cursorUp(n: number): string { + return n ? `\x1b[${n}A` : ""; +} +/** move the cursor to the start of it's current line. */ +export const startOfLine: string = "\r"; +/** move the cursor to the start of the next line. */ +export const startOfNextLine: string = "\n"; + +// erasure +/** clear from and including the current cell right to the end of the line. */ +export const clearToEndOfLine: string = "\x1b[K"; +/** clear from and including the current cell left to the start of the line. */ +export const clearToStartOfLine: string = "\x1b[1K"; +/** clear the entire line that the cursor is on without moving it. */ +export const clearFullLine: string = "\x1b[2K"; + +/** + * Begin Synchronized Output + * https://gist.github.com/christianparpart/d8a62cc1ab659194337d73e399004036 + */ +export const syncStart: string = "\x1b[?2026h"; +/** End Synchronized Output */ +export const syncEnd: string = "\x1b[?2026l"; + +/** + * Apply ANSI styles to text. `code` is a concatenation of zero or more styles, + * which will be cleared as required. + */ +export function style(code: string, text: string): string { + code = mergeStyles(code); + if (!code) return text; + const fg = /[\[;][39]/.test(code); + const bg = /[\[;](?:4|10)/.test(code); + const bold = /[\[;](?:1)[;m]/.test(code); + const dim = /[\[;](?:2)[;m]/.test(code); + return code + text + + mergeStyles( + (fg ? fgReset : "") + + (bg ? bgReset : "") + + (bold ? resetWeight : "") + + (dim ? resetWeight : ""), + ); +} + +/** + * Given a string of multiple style ansi codes, combines all of them. + * Must be used on a string of only ansi style codes. + */ +export function mergeStyles(code: string): string { + let styles = []; + while (code.length) { + if (!code.startsWith("\x1b[")) { + throw new Error("invalid style string: " + JSON.stringify(code)); + } + const end = code.indexOf("m"); + if (end === -1) { + throw new Error("invalid style string: " + JSON.stringify(code)); + } + styles.push(code.slice(2, end)); + code = code.slice(end + 1); + } + return `\x1b[${styles.join(";")}m`; +} + +/** + * Convert a string with ansi text into a string where all the codes have been + * turned into their original names. + */ +export function debugAnsi(text: string): string { + // deno-fmt-ignore + const verbaitim: (keyof typeof self)[] = [ + "bgReset", "bgBlack", "bgRed", "bgGreen", "bgYellow", "bgBlue", "bgMagenta", + "bgCyan", "bgWhite", "bgBrightBlack", "bgBrightRed", "bgBrightGreen", + "bgBrightYellow", "bgBrightBlue", "bgBrightMagenta", "bgBrightCyan", + "bgBrightWhite", "fgReset", "fgBlack", "fgRed", "fgGreen", "fgYellow", + "fgBlue", "fgMagenta", "fgCyan", "fgWhite", "fgBrightBlack", "fgBrightRed", + "fgBrightGreen", "fgBrightYellow", "fgBrightBlue", "fgBrightMagenta", + "fgBrightCyan", "fgBrightWhite", "clearToEndOfLine", "clearToStartOfLine", + "clearFullLine", "syncStart", "syncEnd", "bold", "dim", "resetWeight" + ]; + for (const key of verbaitim) { + text = text.replaceAll( + String(self[key]), + style(key.includes("cursor") ? fgBlue : fgCyan, `\${${key}}`), + ); + } + text = text.replace(/\x1b\[(\d+)A/g, style(fgYellow, "$${cursorUp($1)}")); + text = text.replace(/\n/g, style(fgRed, "\\n")); + text = text.replace(/\r/g, style(fgMagenta, "\\r")); + return text; +} + +/** Measure the visible length of a string, ignoring ansi codes. */ +export function widthInTerminal(str: string): number { + let width = 0; + if (str.includes("\x1b")) str = str.replace(ansiEscapes, ""); + for (let i = 0, { length } = str; i < length; i += 1) { + const code = str.charCodeAt(i); + if (code < 32) continue; + if (code < 127) { + width += 1; + continue; + } + for (const { segment } of segmenter.segment(str.slice(i))) { + if (segment.length === 1) { + if (code < 32) continue; + if (code < 127) { + width += 1; + continue; + } + } + width += segmentWidth(segment); + } + return width; + } + return width; +} + +/** + * Like `str.slice(0, columns)`, but respecting width in a terminal. + * preserves ansi escape sequences. + */ +// TODO: rename to trimForTerminal +export function trimToWidth(str: string, columns: number): string { + let width = 0; + let ansi = false; + for (let i = 0, { length } = str; i < length; i += 1) { + const code = str.charCodeAt(i); + if (code < 32) { + if (code === 0x1b) { + const esc = str.slice().match(ansiEscapes); + if (esc) i += esc[0].length - 1, ansi = true; + } + continue; + } + if (code < 127) { + width += 1; + if (width >= columns) { + const result = str.slice(0, i + 1); + return ansi ? result + "\x1b[0m" : result; + } + continue; + } + // non-ascii path + let out = str.slice(0, i); + str = str.slice(i); + const segments = segmenter.segment(str); + { + const { segment } = UNWRAP(segments.containing(0)); + width += segmentWidth(segment); + if (width > columns) return ansi ? out + "\x1b[0m" : out; + out += segment; + i = segment.length; + } + for ({ length } = str; i < length; i += 1) { + const code = str.charCodeAt(i); + if (code === 0x1b) { + const esc = str.slice(i).match(ansiEscapes); + if (esc) { + i += esc[0].length - 1, ansi = true; + out += esc[0]; + continue; + } + } + const { segment } = UNWRAP(segments.containing(i)); + width += segmentWidth(segment); + if (width > columns) return ansi ? out + "\x1b[0m" : out; + out += segment; + } + return out; + } + return str; +} + +// derived from sindresorhus's wonderful `string-width` +// https://github.com/sindresorhus/string-width/blob/42e7b697393a9b9ff732f9bd3ab1db87c3c208b2/index.js +// https://github.com/sindresorhus/get-east-asian-width/blob/6aee3824b5e2ade910a38ed93cf9d89b7b5a8e5b/index.js +// MIT license +const segmenter = /* @__PURE__ */ new Intl.Segmenter(); +const ansiEscapes = + /[\u001B\u009B][[\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\d\/\#&.:=?%@~_]+)*|[a-zA-Z\d]+(?:;[-a-zA-Z\d\/\#&.:=?%@~_]*)*)?(?:\u0007|\u001B\u005C|\u009C))|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g; +const zeroWidthClusterRegex = + /^(?:\p{Default_Ignorable_Code_Point}|\p{Control}|\p{Mark}|\p{Surrogate})+$/v; +const leadingNonPrintingRegex = + /^[\p{Default_Ignorable_Code_Point}\p{Control}\p{Format}\p{Mark}\p{Surrogate}]+/v; +const rgiEmojiRegex = /^\p{RGI_Emoji}$/v; +function segmentWidth(segment: string) { + if (zeroWidthClusterRegex.test(segment)) return 0; + if (rgiEmojiRegex.test(segment)) return 2; + + // deno-fmt-ignore + const list = [ + 0x3000, 0x231A, 0x231B, 0x2329, 0x232A, 0x23F0, 0x23F3, 0x25FD, 0x25FE, 0x2614, 0x2615, + 0x267F, 0x2693, 0x26A1, 0x26AA, 0x26AB, 0x26BD, 0x26BE, 0x26C4, 0x26C5, 0x26CE, 0x26D4, + 0x26EA, 0x26F2, 0x26F3, 0x26F5, 0x26FA, 0x26FD, 0x2705, 0x270A, 0x270B, 0x2728, 0x274C, + 0x274E, 0x2757, 0x27B0, 0x27BF, 0x2B1B, 0x2B1C, 0x2B50, 0x2B55, 0x1AFFD, 0x1AFFE, 0x1B132, + 0x1B155, 0x1F004, 0x1F0CF, 0x1F18E, 0x1F250, 0x1F251, 0x1F3F4, 0x1F440, 0x1F57A, 0x1F595, + 0x1F596, 0x1F5A4, 0x1F6CC, 0x1F6EB, 0x1F6EC, 0x1F7F0, 0x1FAC8, + ]; + const x = segment.replace(leadingNonPrintingRegex, "").codePointAt(0) ?? 0; + return 1 + +( + // full + x >= 0xFF01 && x <= 0xFF60 || x >= 0xFFE0 && x <= 0xFFE6 || + // wide + x >= 0x1100 && x <= 0x115F || x >= 0x23E9 && x <= 0x23EC || + x >= 0x2630 && x <= 0x2637 || x >= 0x2648 && x <= 0x2653 || + x >= 0x268A && x <= 0x268F || x >= 0x2753 && x <= 0x2755 || + x >= 0x2795 && x <= 0x2797 || x >= 0x2E80 && x <= 0x2E99 || + x >= 0x2E9B && x <= 0x2EF3 || x >= 0x2F00 && x <= 0x2FD5 || + x >= 0x2FF0 && x <= 0x2FFF || x >= 0x3001 && x <= 0x303E || + x >= 0x3041 && x <= 0x3096 || x >= 0x3099 && x <= 0x30FF || + x >= 0x3105 && x <= 0x312F || x >= 0x3131 && x <= 0x318E || + x >= 0x3190 && x <= 0x31E5 || x >= 0x31EF && x <= 0x321E || + x >= 0x3220 && x <= 0x3247 || x >= 0x3250 && x <= 0xA48C || + x >= 0xA490 && x <= 0xA4C6 || x >= 0xA960 && x <= 0xA97C || + x >= 0xAC00 && x <= 0xD7A3 || x >= 0xF900 && x <= 0xFAFF || + x >= 0xFE10 && x <= 0xFE19 || x >= 0xFE30 && x <= 0xFE52 || + x >= 0xFE54 && x <= 0xFE66 || x >= 0xFE68 && x <= 0xFE6B || + x >= 0x16FE0 && x <= 0x16FE4 || x >= 0x16FF0 && x <= 0x16FF6 || + x >= 0x17000 && x <= 0x18CD5 || x >= 0x18CFF && x <= 0x18D1E || + x >= 0x18D80 && x <= 0x18DF2 || x >= 0x1AFF0 && x <= 0x1AFF3 || + x >= 0x1AFF5 && x <= 0x1AFFB || x >= 0x1B150 && x <= 0x1B152 || + x >= 0x1B164 && x <= 0x1B167 || x >= 0x1B170 && x <= 0x1B2FB || + x >= 0x1D300 && x <= 0x1D356 || x >= 0x1D360 && x <= 0x1D376 || + x >= 0x1F191 && x <= 0x1F19A || x >= 0x1F200 && x <= 0x1F202 || + x >= 0x1F210 && x <= 0x1F23B || x >= 0x1F240 && x <= 0x1F248 || + x >= 0x1F260 && x <= 0x1F265 || x >= 0x1F300 && x <= 0x1F320 || + x >= 0x1F32D && x <= 0x1F335 || x >= 0x1F337 && x <= 0x1F37C || + x >= 0x1F37E && x <= 0x1F393 || x >= 0x1F3A0 && x <= 0x1F3CA || + x >= 0x1F3CF && x <= 0x1F3D3 || x >= 0x1F3E0 && x <= 0x1F3F0 || + x >= 0x1F3F8 && x <= 0x1F43E || x >= 0x1F442 && x <= 0x1F4FC || + x >= 0x1F4FF && x <= 0x1F53D || x >= 0x1F54B && x <= 0x1F54E || + x >= 0x1F550 && x <= 0x1F567 || x >= 0x1F5FB && x <= 0x1F64F || + x >= 0x1F680 && x <= 0x1F6C5 || x >= 0x1F6D0 && x <= 0x1F6D2 || + x >= 0x1F6D5 && x <= 0x1F6D8 || x >= 0x1F6DC && x <= 0x1F6DF || + x >= 0x1F6F4 && x <= 0x1F6FC || x >= 0x1F7E0 && x <= 0x1F7EB || + x >= 0x1F90C && x <= 0x1F93A || x >= 0x1F93C && x <= 0x1F945 || + x >= 0x1F947 && x <= 0x1F9FF || x >= 0x1FA70 && x <= 0x1FA7C || + x >= 0x1FA80 && x <= 0x1FA8A || x >= 0x1FA8E && x <= 0x1FAC6 || + x >= 0x1FACD && x <= 0x1FADC || x >= 0x1FADF && x <= 0x1FAEA || + x >= 0x1FAEF && x <= 0x1FAF8 || x >= 0x20000 && x <= 0x2FFFD || + x >= 0x30000 && x <= 0x3FFFD || x >= 0x1B000 && x <= 0x1B122 || + list.includes(x) + ); +} + +import * as self from "lib/string/ansi.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/lib/subprocess.ts b/lib/subprocess.ts new file mode 100644 index 0000000000000000000000000000000000000000..d1e605e609e9b1e52da9dc4533575b74696f3c87 --- /dev/null +++ b/lib/subprocess.ts @@ -0,0 +1,93 @@ +// This file is expecting a full rewrite to abstract the Node API away +// entirely, exposing web streams I/O. + +const execFileRaw: typeof child_process.execFile.__promisify__ = util.promisify( + child_process.execFile, +); +/*** @deprecated */ +export const exec: typeof execFileRaw = (( + ...args: Parameters +) => + execFileRaw(...args).catch((e: any) => { + if (e?.message?.startsWith?.("Command failed")) { + if (e.code > 2 ** 31) e.code |= 0; + const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; + e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; + } + throw e; + })) as any; + +// const activePids = new Set(); + +/*** @deprecated */ +export interface SpawnOptions extends child_process.SpawnOptions { + cmd: string[]; +} +/*** @deprecated */ +export function spawn( + { cmd, ...rest }: SpawnOptions, +): child_process.ChildProcess { + // rest.detached ??= true; + const process = child_process.spawn(cmd[0]!, cmd.slice(1), rest); + // const pid = rest.detached && process.pid ? -process.pid : process.pid; + // if (pid) { + // activePids.add(pid); + // process.on("exit", () => void activePids.delete(pid)); + // } + return process; +} + +/*** @deprecated */ +export interface ForkOptions extends child_process.ForkOptions { + cmd: string[]; +} + +/*** @deprecated */ +export function fork( + { cmd, ...rest }: ForkOptions, +): child_process.ChildProcess { + // rest.detached ??= true; + const process = child_process.fork(cmd[0]!, cmd.slice(1), rest); + // const pid = rest.detached && process.pid ? -process.pid : process.pid; + // if (pid) { + // activePids.add(pid); + // process.on("exit", () => void activePids.delete(pid)); + // } + return process; +} + +/*** @deprecated */ +export async function spawnAndWait( + opts: SpawnOptions, +): Promise<{ stdout: Buffer | null; stderr: Buffer | null }> { + opts.stdio ??= ["ignore", "inherit", "inherit"]; + const process = spawn(opts); + const output = Promise.all([ + process.stdout ? Array.fromAsync(process.stdout).then(Buffer.concat) : null, + process.stderr ? Array.fromAsync(process.stderr).then(Buffer.concat) : null, + ]); + let [code, signal] = (await events.EventEmitter.once(process, "exit")) as [ + number | null, + string | null, + ]; + if (code !== 0) { + if (code !== null && code > 2 ** 31) code |= 0; + const exit = signal ? `signal ${signal}` : `code ${code}`; + throw new Error(`${opts.cmd[0]} failed with ${exit}`); + } + const [stdout, stderr] = await output; + return { stdout, stderr }; +} + +// process.on("exit", () => { +// for (const pid of activePids) { +// console.info(`Kill ${Math.abs(pid)}`); +// process.kill(pid, 9); +// } +// }); + +// import * as process from "node:process"; +import * as events from "node:events"; +import * as util from "node:util"; +import * as child_process from "node:child_process"; +import { Buffer } from "node:buffer"; diff --git a/lib/subprocess/ffmpeg.ts b/lib/subprocess/ffmpeg.ts new file mode 100644 index 0000000000000000000000000000000000000000..ee09a4f2560e07e24d1964d142c2501330088809 --- /dev/null +++ b/lib/subprocess/ffmpeg.ts @@ -0,0 +1,198 @@ +export interface SpawnOptions { + args: string[]; + ffmpeg?: string; + progress: progress.Node; + cwd?: string; // TODO: Path +} + +/** + * Spawn an instance of `ffmpeg` and bind it's live status to the given + * `progress.Node`. Logs are routed through the progress node, and dismissed on + * exit. `lib/log/progress.ts` will buffer and colocate the logs when running + * parallel jobs. + * + * await ffmpeg.spawn({ + * cmd: ["-i", "hello.mov", "-c:v", "libsvtav1", "hello.mp4"], + * progress: progress.start("encode hello.mov"), + * }); + */ +export async function spawn(options: SpawnOptions) { + const { ffmpeg = "ffmpeg", args, cwd } = options; + using node = options.progress; + const title = node.text; + const proc = child_process.spawn(ffmpeg, [...defaultExtraOptions, ...args], { + stdio: ["ignore", "inherit", "pipe"], + env: { ...process.env, SVT_LOG: "2" }, + cwd: cwd?.toString(), + }); + const parser = new Parse(); + let running = true; + const splitter = readline.createInterface({ input: proc.stderr }); + splitter.on("line", (line) => { + const result = parser.onLine(line); + if (result.kind === "ignore") { + return; + } else if (result.kind === "log") { + node.log[result.level](result.message); + } else if (result.kind === "progress") { + if (!running) return; + const { frame, totalFrames, fps, speed } = result; + node.value = frame; + node.estimate = totalFrames; + const extras = [ + `${fps} fps`, + speed, + parser.hlsFile, + ].filter(Boolean).join(", "); + node.text = `${title}${extras.length > 0 ? ` (${extras})` : ""}`; + } else result satisfies never; + }); + const [code, signal] = await events.once(proc, "close"); + running = false; + if (code !== 0) { + const fmt = code ? `code ${code}` : `signal ${signal}`; + const e: any = new Error(`ffmpeg failed with ${fmt}`); + e.args = [ffmpeg, ...args].map(string.escapeShellArgument).join(" "); + e.code = code; + e.signal = signal; + throw e; + } +} + +export interface Streams { + audio: boolean; + video: boolean; +} + +export async function probeStreams(path: string): Promise { + const { stdout } = await subprocess.spawnAndWait({ + cmd: [ + "ffprobe", + "-v", + "error", + "-show_entries", + "stream=codec_type", + path.toString(), + "-of", + "default=noprint_wrappers=1:nokey=1", + ], + stdio: "pipe", + }); + + const lines = new TextDecoder().decode(stdout!).trim().split("\n"); + return { + audio: lines.includes("audio"), + video: lines.includes("video"), + }; +} + +export type Line = + | { kind: "ignore" } + | { kind: "log"; level: "info" | "warn" | "error"; message: string } + | { + kind: "progress"; + frame: number; + totalFrames: number; + speed: string | null; + fps: number | null; + rest: Record; + }; + +export const defaultExtraOptions = [ + "-hide_banner", + "-stats", +]; + +export class Parse { + parsingStart = true; + inIndentedIgnore: null | "out" | "inp" | "other" = null; + durationTime = 0; + targetFps: number | null = null; + + hlsFile: string | null = null; + durationFrames = 0; + + onLine(line: string): Line { + line = line.trimEnd(); + if (/^frame=/.test(line)) { + if (this.parsingStart) { + this.parsingStart = false; + this.durationFrames = Math.ceil( + (this.targetFps ?? 25) * this.durationTime, + ); + } + const parts = Object.fromEntries( + [...line.matchAll(/\b([a-z0-9]+)=\s*([^ ]+)(?= |$)/ig)].map(( + [, k, v], + ) => [k, v]), + ); + const { frame, fps, speed, ...rest } = parts; + return { + kind: "progress", + frame: Number(frame), + totalFrames: this.durationFrames, + fps: Number(fps), + speed, + rest, + }; + } + + if (this.parsingStart) { + if (this.inIndentedIgnore) { + if (line.startsWith(" ") || line.startsWith("\t")) { + line = line.trimStart(); + if (this.inIndentedIgnore === "inp") { + const match = line.match(/^Duration: (\d+):(\d+):(\d+\.\d+)/); + if (match) { + const [h, m, s] = match.slice(1).map((x) => Number(x)); + this.durationTime = Math.max( + this.durationTime, + h * 60 * 60 + m * 60 + s, + ); + } + if (!this.targetFps) { + const match = line.match(/^Stream.*, (\d+) fps/); + if (match) this.targetFps = Number(match[1]); + } + } + return { kind: "ignore" }; + } + this.inIndentedIgnore = null; + } + if (line === "Press [q] to stop, [?] for help") { + return { kind: "ignore" }; + } + if (line === "Stream mapping:") { + this.inIndentedIgnore = "other"; + return { kind: "ignore" }; + } + if (line.startsWith("Output #") || line.startsWith("Input #")) { + this.inIndentedIgnore = line.slice(0, 3).toLowerCase() as "inp" | "out"; + return { kind: "ignore" }; + } + } + + const hlsMatch = line.match(/.*Opening '(.+)' for writing/); + if (hlsMatch) { + if (!hlsMatch[1].endsWith(".tmp")) { + this.hlsFile = path.basename(hlsMatch[1]); + } + return { kind: "ignore" }; + } + + let level: Extract["level"] = "info"; + if (line.toLowerCase().includes("err")) level = "error"; + else if (line.toLowerCase().includes("warn")) level = "warn"; + + return { kind: "log", level, message: line }; + } +} + +import * as child_process from "node:child_process"; +import * as subprocess from "lib/subprocess.ts"; +import * as readline from "node:readline"; +import * as process from "node:process"; +import events from "node:events"; +import * as path from "node:path"; +import type * as progress from "lib/log/progress.ts"; +import * as string from "lib/string.ts"; diff --git a/lib/ts.ts b/lib/ts.ts index 9ced87c49f3dfd67775cd75df971710359c7744e..3a4caa240ca373643e5d1ce5dfdf8816c918a68e 100644 --- a/lib/ts.ts +++ b/lib/ts.ts @@ -1,9 +1,19 @@ export type Timer = ReturnType; export type Interval = ReturnType; -export interface Dispose extends Disposable { +export interface VoidFunction { (): void; } + +/** Convenience type. Construct with `defer` */ +export interface Dispose extends Disposable, VoidFunction {} + +/** + * using _ = ts.defer(() => { ... }); + * + * Additionally, this can be used to construct the convenience type + * `ts.Dispose` + */ export function defer(fn: () => void): Dispose { const f = fn as Dispose; f[Symbol.dispose] = f; diff --git a/package.json b/package.json index 3ba0610088552a158e83917d8a7617985c1fc718..eb14ab45b74b926f15ff1134dea9f5c07244ceea 100644 --- a/package.json +++ b/package.json @@ -25,12 +25,11 @@ "typescript": "5.8.3" }, "imports": { - "#backend": "./src/backend.ts", - "#debug": "./framework/debug.safe.ts", "#sitegen": "./framework/lib/sitegen.ts", "#sitegen/*": "./framework/lib/*.ts", - "#engine/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts", - "#engine/*": "./framework/engine/*.ts", + "#jsx/jsx-runtime": "./lib/render/jsx-runtime.ts", + "#jsx/jsx-dev-runtime": "./lib/render/jsx-runtime.ts", + "#marko/runtime": "./framework/engine/marko-runtime.ts", "#marko/html": { "types": "marko/html", "production": "marko/production", diff --git a/run.js b/run.js index c891dc5606619eb95ea60ea6abf9dec74d9e605f..adc7d9ec7d3a3873b8ded5a3dadac630b65fae5a 100644 --- a/run.js +++ b/run.js @@ -4,6 +4,7 @@ import * as zlib from "node:zlib"; import * as url from "node:url"; import * as fs from "node:fs"; import * as os from "node:os"; +import assert from "node:assert"; import process from "node:process"; // Disable experimental warnings (Type Stripping, etc) @@ -16,11 +17,8 @@ process.emit = function (event, error) { }; try { - // fails in a runtime without ts support - await import("#debug"); - // required APIs - ASSERT(zlib.zstdCompress); + assert(zlib.zstdCompress); // seems to fail in bun (await import("source-map-support")).install({ hookRequire: true }); diff --git a/src/deploy.ts b/src/deploy.ts index e7ee235a69c35c75826fa938a3dd3d1573edf679..15bb02b59370263c504d57ed235d320ecfbb7b98 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -46,6 +46,6 @@ export async function main() { import * as incr from "../framework/incremental.ts"; import { generate } from "../framework/generate.ts"; import * as progress from "lib/log/progress.ts"; -import * as subprocess from "#sitegen/subprocess"; +import * as subprocess from "lib/subprocess.ts"; import * as rsync from "./file-viewer/rsync.ts"; import { Path } from "#sitegen/path"; diff --git a/src/file-viewer/backend.tsx b/src/file-viewer/backend.tsx index 99a4532a3044d24b9261fffc78831172e74d88ec..1fd159b8990d635f64e710900fe20ccb9dedf5d9 100644 --- a/src/file-viewer/backend.tsx +++ b/src/file-viewer/backend.tsx @@ -93,7 +93,7 @@ app.get("/file/*", async (c, next) => { const headers = new Headers({ Vary: "Accept-Encoding, Accept", - "Content-Type": contentTypeFor(derivedKey || file.path), + "Content-Type": mime.for(derivedKey || file.path), "Content-Length": size.toString(), ETag: etag, "Last-Modified": date.toUTCString(), @@ -269,7 +269,7 @@ function getPartialPage(c: Context, rawFilePath: string) { return c.json({ error: "File not found" }, 404); } - const r = render.init(false, {}); + const r = render.init(false, []); let root = render.resolveNode( r, ; - }; - -export const defaultExtraOptions = [ - "-hide_banner", - "-stats", -]; - -export interface SpawnOptions { - args: string[]; - ffmpeg?: string; - progress: progress.Node; - cwd: Path; -} - -export async function spawn(options: SpawnOptions) { - const { ffmpeg = "ffmpeg", args, cwd } = options; - using node = options.progress; - const title = node.text; - const proc = child_process.spawn(ffmpeg, [...defaultExtraOptions, ...args], { - stdio: ["ignore", "inherit", "pipe"], - env: { ...process.env, SVT_LOG: "2" }, - cwd: cwd.toString(), - }); - const parser = new Parse(); - let running = true; - const splitter = readline.createInterface({ input: proc.stderr }); - splitter.on("line", (line) => { - const result = parser.onLine(line); - if (result.kind === "ignore") { - return; - } else if (result.kind === "log") { - node.log[result.level](result.message); - } else if (result.kind === "progress") { - if (!running) return; - const { frame, totalFrames, fps, speed } = result; - node.value = frame; - node.estimate = totalFrames; - const extras = [ - `${fps} fps`, - speed, - parser.hlsFile, - ].filter(Boolean).join(", "); - node.text = `${title}${extras.length > 0 ? ` (${extras})` : ""}`; - } else result satisfies never; - }); - const [code, signal] = await events.once(proc, "close"); - running = false; - if (code !== 0) { - const fmt = code ? `code ${code}` : `signal ${signal}`; - const e: any = new Error(`ffmpeg failed with ${fmt}`); - e.args = [ffmpeg, ...args].map(string.escapeShellArgument).join(" "); - e.code = code; - e.signal = signal; - throw e; - } -} - -export async function probeStreams(path: Path) { - const { stdout } = await subprocess.spawnAndWait({ - cmd: [ - "ffprobe", - "-v", - "error", - "-show_entries", - "stream=codec_type", - path.toString(), - "-of", - "default=noprint_wrappers=1:nokey=1", - ], - stdio: "pipe", - }); - - const lines = new TextDecoder().decode(stdout!).trim().split("\n"); - return { - audio: lines.includes("audio"), - video: lines.includes("video"), - }; -} - -export class Parse { - parsingStart = true; - inIndentedIgnore: null | "out" | "inp" | "other" = null; - durationTime = 0; - targetFps: number | null = null; - - hlsFile: string | null = null; - durationFrames = 0; - - onLine(line: string): Line { - line = line.trimEnd(); - if (/^frame=/.test(line)) { - if (this.parsingStart) { - this.parsingStart = false; - this.durationFrames = Math.ceil( - (this.targetFps ?? 25) * this.durationTime, - ); - } - const parts = Object.fromEntries( - [...line.matchAll(/\b([a-z0-9]+)=\s*([^ ]+)(?= |$)/ig)].map(( - [, k, v], - ) => [k, v]), - ); - const { frame, fps, speed, ...rest } = parts; - return { - kind: "progress", - frame: Number(frame), - totalFrames: this.durationFrames, - fps: Number(fps), - speed, - rest, - }; - } - - if (this.parsingStart) { - if (this.inIndentedIgnore) { - if (line.startsWith(" ") || line.startsWith("\t")) { - line = line.trimStart(); - if (this.inIndentedIgnore === "inp") { - const match = line.match(/^Duration: (\d+):(\d+):(\d+\.\d+)/); - if (match) { - const [h, m, s] = match.slice(1).map((x) => Number(x)); - this.durationTime = Math.max( - this.durationTime, - h * 60 * 60 + m * 60 + s, - ); - } - if (!this.targetFps) { - const match = line.match(/^Stream.*, (\d+) fps/); - if (match) this.targetFps = Number(match[1]); - } - } - return { kind: "ignore" }; - } - this.inIndentedIgnore = null; - } - if (line === "Press [q] to stop, [?] for help") { - return { kind: "ignore" }; - } - if (line === "Stream mapping:") { - this.inIndentedIgnore = "other"; - return { kind: "ignore" }; - } - if (line.startsWith("Output #") || line.startsWith("Input #")) { - this.inIndentedIgnore = line.slice(0, 3).toLowerCase() as "inp" | "out"; - return { kind: "ignore" }; - } - } - - const hlsMatch = line.match(/.*Opening '(.+)' for writing/); - if (hlsMatch) { - if (!hlsMatch[1].endsWith(".tmp")) { - this.hlsFile = path.basename(hlsMatch[1]); - } - return { kind: "ignore" }; - } - - let level: Extract["level"] = "info"; - if (line.toLowerCase().includes("err")) level = "error"; - else if (line.toLowerCase().includes("warn")) level = "warn"; - - return { kind: "log", level, message: line }; - } -} - -import * as child_process from "node:child_process"; -import * as subprocess from "#sitegen/subprocess"; -import * as readline from "node:readline"; -import * as process from "node:process"; -import events from "node:events"; -import * as path from "node:path"; -import * as progress from "../../lib/log/progress.ts"; -import type { Path } from "#sitegen/path"; -import * as string from "lib/string.ts"; diff --git a/src/file-viewer/format.ts b/src/file-viewer/format.ts index 0e218aecac05343cfe1b547a42317303238e7cdd..63b50857837c689d5a082c79ed4d894c64cbad49 100644 --- a/src/file-viewer/format.ts +++ b/src/file-viewer/format.ts @@ -288,4 +288,4 @@ export function formatDate(dateTime: Date) { } import type { MediaFile } from "@/file-viewer/models/MediaFile.ts"; -import { escapeHtml } from "#engine/render"; +import { escapeHtml } from "lib/render.ts"; diff --git a/src/file-viewer/highlight.ts b/src/file-viewer/highlight.ts index 13ab7c78101fa4dd0fc2487b73ed0323fa39db8f..f565051e368febca3ba819694e38723e33ca35c4 100644 --- a/src/file-viewer/highlight.ts +++ b/src/file-viewer/highlight.ts @@ -192,9 +192,9 @@ export async function main() { } } -import * as async from "#sitegen/async"; +import * as async from "lib/async.ts"; import * as fs from "#sitegen/fs"; import * as path from "node:path"; import * as oniguruma from "vscode-oniguruma"; import * as textmate from "vscode-textmate"; -import * as render from "#engine/render"; +import * as render from "lib/render.ts"; diff --git a/src/file-viewer/models/MediaFile.ts b/src/file-viewer/models/MediaFile.ts index f99c10f99611f14ac6d55f2ad3d082039860d247..f9e75d4a64925e4306602ba7162bedbeb5c1f2d7 100644 --- a/src/file-viewer/models/MediaFile.ts +++ b/src/file-viewer/models/MediaFile.ts @@ -461,3 +461,4 @@ const getDirectoriesToReindexQuery = db.prepare(` import { getDb } from "#sitegen/sqlite"; import * as path from "node:path/posix"; import { FilePermissions } from "./FilePermissions.ts"; +import { ASSERT } from "lib/assert.ts"; diff --git a/src/file-viewer/transcode-rules.ts b/src/file-viewer/transcode-rules.ts index f6108b6812d93f006cf9639b1eef58f615c4918b..802c917115c680ff6788b8d8d5035d1e0e08cb5c 100644 --- a/src/file-viewer/transcode-rules.ts +++ b/src/file-viewer/transcode-rules.ts @@ -313,4 +313,4 @@ export function getH264HlsArgs( } import type { Path } from "#sitegen/path"; -import * as ffmpeg from "./ffmpeg.ts"; +import * as ffmpeg from "lib/subprocess/ffmpeg.ts"; diff --git a/src/file-viewer/views/clofi.tsx b/src/file-viewer/views/clofi.tsx index daaffa39af40620945309eb8b35dc0b4af9c6f84..0871f8dcce6751cce75fce31f64579d9ff9783d3 100644 --- a/src/file-viewer/views/clofi.tsx +++ b/src/file-viewer/views/clofi.tsx @@ -89,7 +89,7 @@ export default function MediaList({ ); @@ -163,7 +163,7 @@ export function MediaPanel({ className="custom mobile-back" /> ); - const canvases: Record = { + const canvases: Record void> = { "/2017": () => addScript("../scripts/canvas_2017.client.ts"), "/2018": () => addScript("../scripts/canvas_2018.client.ts"), "/2019": () => addScript("../scripts/canvas_2019.client.ts"), @@ -923,8 +923,9 @@ import { highlightLinksInTextView, } from "@/file-viewer/format.ts"; import { ForEveryone } from "@/file-viewer/cotyledon.tsx"; -import * as mime from "#sitegen/mime"; +import * as mime from "lib/mime.ts"; import * as sort from "../sort.ts"; import CloverVideo from "@/tags/clover-video.marko"; import Image from "@/tags/image.tsx"; -import * as render from "#engine/render"; +import * as render from "lib/render.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/src/package-lib.ts b/src/package-lib.ts new file mode 100644 index 0000000000000000000000000000000000000000..c2cc080675f5dd313cb7f303709e9848dd9d4181 --- /dev/null +++ b/src/package-lib.ts @@ -0,0 +1,226 @@ +export async function updateJsrPackageExportsToAll( + absPath: string, + p: progress.Ref, +) { + using _ = p.start("update deno.json"); + const manifestPath = path.join(absPath, "deno.json"); + const deno = JSON.parse( + await fs.readFile(manifestPath, "utf-8"), + ); + const newExports: Record = {}; + for (const subPath of await fs.readdir(absPath, { recursive: true })) { + if (!subPath.match(/\.tsx?$/)) continue; + if (subPath.match(/\.(test|sample)\.tsx?$/)) continue; + if (subPath.startsWith("dist")) continue; + newExports["./" + subPath] = "./" + subPath; + } + deno.exports = Object.fromEntries( + Object.entries(newExports).sort(([a], [b]) => a.localeCompare(b)), + ) as typeof deno.exports; + await fs.writeFile(manifestPath, JSON.stringify(deno, null, 2)); +} + +export async function transpileForNpm( + absPath: string, + p: progress.Ref, + packageJson: Record, +) { + using node = p.start("transpile into js for npm"); + const manifestPath = path.join(absPath, "deno.json"); + const deno = JSON.parse( + await fs.readFile(manifestPath, "utf-8"), + ); + const distPath = path.join(absPath, "dist-npm"); + await fs.rm(distPath, { recursive: true, force: true }); + await fs.mkdir(distPath, { recursive: true }); + + const files: string[] = []; + const exports: Record = {}; + for (const subPath of await fs.readdir(absPath, { recursive: true })) { + if (!subPath.match(/\.tsx?$/)) continue; + if (subPath.startsWith("dist")) continue; + files.push(subPath); + } + const tsFiles: string[] = []; + for (const subPath of files) { + using _ = node.start(subPath); + const targetSubPath = subPath.replace(/\.tsx?$/, ".js"); + if (!subPath.match(/\.(test|sample)\.tsx?$/)) { + exports["./" + subPath] = "./" + targetSubPath; + } + const out = path.join(distPath, subPath); + await fs.mkdir(path.dirname(out), { recursive: true }); + tsFiles.push(out); + + let code = await fs.readFile(path.join(absPath, subPath), "utf-8"); + code = code.replace( + / from "lib\/([^"]+)"/g, + (_, src) => { + return ' from "./' + path.relative( + path.dirname(subPath), + src.replace(/\.tsx?$/, ".js"), + ) + '"'; + }, + ); + await fs.writeFile(out, code); + } + await subprocess.spawnAndWait({ + cmd: [ + "npx", + "tsc", + "--outdir", + distPath, + "--sourceMap", + "--declarationMap", + "--declaration", + "--module", + "nodenext", + "--lib", + "esnext", + "--skipLibCheck", + "--jsx", + "react-jsx", + "--jsxImportSource", + "./lib/dist-npm/render", + "--target", + "esnext", + "--isolatedModules", + "--verbatimModuleSyntax", + "--allowSyntheticDefaultImports", + "--moduleResolution", + "nodenext", + ...tsFiles, + ], + }); + await fs.writeFile( + path.join(distPath, "readme.md"), + (await fs.readFile(path.join(absPath, "readme.md"), "utf-8")).replace( + "\n", + await fs.readFile(path.join(absPath, "readme.npm.md"), "utf-8"), + ), + ).catch((e) => { + if (e.code !== "ENOENT") throw e; + }); + await fs.cp( + path.join(absPath, "readme.changes.md"), + path.join(distPath, "readme.changes.md"), + ).catch((e) => { + if (e.code !== "ENOENT") throw e; + }); + const { name, description, ...rest } = packageJson; + await fs.writeFile( + path.join(distPath, "package.json"), + JSON.stringify( + { + name, + description, + version: deno.version, + type: "module", + exports, + ...rest, + }, + null, + 2, + ), + ); +} + +// TODO: failed experiment +// +// export async function transpileForNpm( +// absPath: string, +// p: progress.Ref, +// packageJson: Record, +// ) { +// const { transpileFile } = await import("ts-to-jsdoc") as TsToJsdoc; +// +// using node = p.start("transpile into jsdoc for npm"); +// const manifestPath = path.join(absPath, "deno.json"); +// const deno = JSON.parse( +// await fs.readFile(manifestPath, "utf-8"), +// ); +// const distPath = path.join(absPath, "dist-npm"); +// await fs.rm(distPath, { recursive: true, force: true }); +// await fs.mkdir(distPath, { recursive: true }); +// +// const files: string[] = []; +// const exports: Record = {}; +// for (const subPath of await fs.readdir(absPath, { recursive: true })) { +// if (!subPath.match(/\.tsx?$/)) continue; +// if (subPath.startsWith("dist")) continue; +// files.push(subPath); +// } +// node.estimate = Object.keys(exports).length; +// for (const subPath of files) { +// using _ = node.start(subPath); +// const targetSubPath = subPath.replace(/\.tsx?$/, ".js"); +// if (!subPath.match(/\.(test|sample)\.tsx?$/)) { +// exports["./" + subPath] = "./" + targetSubPath; +// } +// +// let code = transpileFile({ +// code: await fs.readFile(path.join(absPath, subPath), "utf-8"), +// }); +// code = code.replace( +// / from "lib\/([^"]+)"/g, +// (_, src) => { +// return ' from "./' + path.relative( +// path.dirname(subPath), +// src.replace(/\.tsx?$/, ".js"), +// ) + '"'; +// }, +// ); +// if (packageJson.homepage) { +// code = +// "// This file is transpiled from TypeScript from paper clover's sitegen\n" + +// "// " + packageJson.homepage + "/" + subPath + "\n" + +// code; +// } +// const out = path.join(distPath, targetSubPath); +// await fs.mkdir(path.dirname(out), { recursive: true }); +// await fs.writeFile(out, code); +// node.value += 1; +// } +// await fs.cp( +// path.join(absPath, "readme.npm.md"), +// path.join(distPath, "readme.md"), +// ).catch((e) => { +// if (e.code !== "ENOENT") throw e; +// }); +// await fs.cp( +// path.join(absPath, "readme.changes.md"), +// path.join(distPath, "readme.changes.md"), +// ).catch((e) => { +// if (e.code !== "ENOENT") throw e; +// }); +// const { name, description, ...rest } = packageJson; +// await fs.writeFile( +// path.join(distPath, "package.json"), +// JSON.stringify({ +// name, +// description, +// version: deno.version, +// type: "module", +// ...rest, +// }), +// ); +// } +// +export function main() { + const main = path.resolve("lib"); + updateJsrPackageExportsToAll(main, progress) + .then(() => + transpileForNpm(main, progress, { + name: "@paperclover/lib", + description: "high quality typescript libraries", + homepage: + "https://git.paperclover.net/clo/sitegen/src/branch/master/lib", + }) + ); +} + +import * as progress from "lib/log/progress.ts"; +import * as path from "node:path"; +import * as fs from "node:fs/promises"; +import * as subprocess from "lib/subprocess.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/src/pages/index.marko b/src/pages/index.marko index becf253be35e58443558cc9ceedb0f12b89b788c..1a8c9e7fdbdd51e425f2732a575ea4d3905fb3f7 100644 --- a/src/pages/index.marko +++ b/src/pages/index.marko @@ -20,7 +20,7 @@ export const meta = {

questions and answers

file browser

mailing list

-

donate

+

donate

rss feed

paper clover

diff --git a/src/q+a/clover-markdown.tsx b/src/q+a/clover-markdown.tsx index 6de9f1e090d8f051ef7225738652d1942e349357..e577f33636569c0631b8c2e7551b77d93c622262 100644 --- a/src/q+a/clover-markdown.tsx +++ b/src/q+a/clover-markdown.tsx @@ -268,4 +268,4 @@ import { parseCaptureInline, parseInline, } from "./simple-markdown.ts"; -import * as render from "#engine/render"; +import * as render from "lib/render.ts"; diff --git a/src/q+a/image.tsx b/src/q+a/image.tsx index 9b9f8798d0a5d302f0f93c927c58be82f0d4d1ae..44a183183ba462b312d9e6fc0c80253857c1374e 100644 --- a/src/q+a/image.tsx +++ b/src/q+a/image.tsx @@ -78,5 +78,5 @@ import * as fs from "#sitegen/fs"; import * as path from "node:path"; import * as puppeteer from "puppeteer"; import { Question } from "@/q+a/models/Question.ts"; -import { RefCountedExpirable } from "#sitegen/async"; +import { RefCountedExpirable } from "lib/async.ts"; import * as view from "#sitegen/view"; diff --git a/src/q+a/scripts/editor.client.tsx b/src/q+a/scripts/editor.client.tsx index 3b4faac09ade27b937bb99c640f10cbd3f5f857a..c82d576b051cc10a5fa8e2e3df07c37297ac7f75 100644 --- a/src/q+a/scripts/editor.client.tsx +++ b/src/q+a/scripts/editor.client.tsx @@ -1,6 +1,6 @@ import { EditorState } from "@codemirror/state"; import { basicSetup, EditorView } from "codemirror"; -import * as render from "#engine/render"; +import * as render from "lib/render.ts"; import type { ScriptPayload } from "@/q+a/views/editor.marko"; import QuestionRender from "@/q+a/tags/question.marko"; diff --git a/src/site.ts b/src/site.ts index d8cfc0f8d2d686e1ed04bd0e2d5e4dc4fc838a8f..a558d6c405917d76334f2d66de7bc16eddd731b3 100644 --- a/src/site.ts +++ b/src/site.ts @@ -113,6 +113,6 @@ export async function main() { import * as path from "node:path"; import * as font from "../framework/font.ts"; -import type { Template as MetaTemplate } from "#sitegen/meta"; +import type { Template as MetaTemplate } from "lib/render/meta.ts"; import type * as sg from "#sitegen"; import { nasRoot } from "./file-viewer/paths.ts"; diff --git a/src/source-of-truth.ts b/src/source-of-truth.ts index b5520c192ec40d212113fa52ec904fb0e0d0ffa5..5f9e3c92d0c4a2fdef482998a335121637bc65b6 100644 --- a/src/source-of-truth.ts +++ b/src/source-of-truth.ts @@ -87,7 +87,7 @@ app.get("/file/*", async (c) => { read: fsCallbacks.read, }, }); - c.header("Content-Type", mime.contentTypeFor(fullPath)); + c.header("Content-Type", mime.for(fullPath)); return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream); }); @@ -111,5 +111,6 @@ import * as fs from "#sitegen/fs"; import * as fsCallbacks from "node:fs"; import * as util from "node:util"; import * as stream from "node:stream"; -import * as mime from "#sitegen/mime"; +import * as mime from "lib/mime.ts"; import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts"; +import { ASSERT } from "lib/assert.ts"; diff --git a/src/tags/clover-video.marko b/src/tags/clover-video.marko index 41051ce86eacbaf024cf5b030ec71186e8ed9389..2d2a4c145ad5abb1ed338871645db5910d05c7e8 100644 --- a/src/tags/clover-video.marko +++ b/src/tags/clover-video.marko @@ -57,4 +57,4 @@ export interface Input { client import "./clover-video.client.ts"; import { MediaFile } from "@/file-viewer/models/MediaFile.ts"; - +import { UNWRAP, ASSERT } from "lib/assert.ts"; diff --git a/src/tags/image.tsx b/src/tags/image.tsx index 0106c239c07757d9c44cd9ea49feec48b62df293..c7eea907b1c8b64b98e82c28444f5a20a1bb6149 100644 --- a/src/tags/image.tsx +++ b/src/tags/image.tsx @@ -5,7 +5,7 @@ interface Props { objectFit?: string; style?: string; - className?: string; + class?: string; } export default function Image( @@ -53,3 +53,4 @@ export default function Image( import * as transcodeRules from "@/file-viewer/transcode-rules.ts"; import { MediaFile } from "@/file-viewer/models/MediaFile.ts"; import { escapeUri } from "@/file-viewer/format.ts"; +import { UNWRAP } from "lib/assert.ts"; diff --git a/suspense.test.tsx b/suspense.test.tsx new file mode 100644 index 0000000000000000000000000000000000000000..1750c5e7f514b8587023b689eef860ccb0b1f97b --- /dev/null +++ b/suspense.test.tsx @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import { renderStreaming, Suspense } from "./suspense.ts"; + +test("sanity", async (t) => { + let resolve: () => void = null!; + + // @ts-expect-error + async function AsyncComponent() { + await new Promise((done) => resolve = done); + return ; + } + + const example = ( +
+

app shell

+ + + +
(c) 2025
+
+ ); + + const iterator = renderStreaming(example); + const assertContinue = (actual: unknown, value: unknown) => + t.assert.deepEqual(actual, { done: false, value }); + + assertContinue( + await iterator.next(), + "", + ); + t.assert.ok(resolve !== null), resolve(); + assertContinue( + await iterator.next(), + "", + ); + t.assert.deepEqual( + await iterator.next(), + { done: true, value: {} }, + ); +}); diff --git a/suspense.ts b/suspense.ts new file mode 100644 index 0000000000000000000000000000000000000000..ab6b52a941cbc199a36ec3640ec2ff3a22b4900c --- /dev/null +++ b/suspense.ts @@ -0,0 +1,116 @@ +// TODO: revive the suspense renderer +/** + * This file implements out-of-order HTML streaming, mimicking the React + * Suspense API. To use, place Suspense around an expensive async component + * and render the page with 'renderStreaming'. + * + * Implementation of this article: + * https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/ + * + * I would link to an article from Next.js or React, but their examples + * are too verbose and not informative to what they actually do. + * @module + */ +const userData = render.userData(() => null); + +export interface Input { + children: render.Node; + fallback?: render.Node; +} + +/** + * When used in combination with `Suspense.renderStreaming`, this component + * displays `fallback` until the async `children` `Node` has resolved.. + * + * return
+ *

page shell

+ * + * + * + *
; + */ +export function Suspense({ children, fallback }: Input): render.Node { + const state = userData.get(); + if (!state) return children; + if (state.nested) throw new Error(" cannot be nested"); + const parent = UNWRAP(render.current); + const r = render.init(true, { [userData.key]: { nested: true } }); + const resolved = render.resolveNode(r, children); + if (r.async == 0) return render.raw(resolved); + const name = "suspended_" + ++state.nextId; + state.nested = true; + const ip: [render.ResolvedNode] = [ + render.resolvedElement( + "slot", + { name }, + fallback ? render.resolveNode(parent, fallback) : "", + ), + ]; + state.nested = false; + r.asyncDone = () => { + const rejections = r.rejections; + if (rejections && rejections.length > 0) throw new Error("TODO"); + state.pushChunk?.(name, ip[0] = resolved); + }; + return render.raw(ip); +} + +// TODO: add a User-Agent parameter, which is used to determine if a +// fallback path must be used. +// - Before ~2024 needs to use a JS implementation. +// - IE should probably bail out entirely. +export async function* renderStreaming< + T extends render.Addons = Record, +>(node: render.Node, addon: T = {} as T) { + const { + text: begin, + addon: { [userData.key]: state, ...addonOutput }, + } = await render.async(node, { + ...addon, + [userData.key]: { + nested: false, + nextId: 0, + completed: 0, + pushChunk: () => {}, + } satisfies State as State, + }); + if (state.nextId === 0) { + yield begin; + return addonOutput as unknown as T; + } + let resolve: (() => void) | null = null; + let chunks: string[] = []; + state.pushChunk = (slot, node) => { + while (node.length === 1 && Array.isArray(node)) node = UNWRAP(node[0]); + if (node[0] === render.kElement) { + (node as render.ResolvedElement)[2].slot = slot; + } else { + node = render.resolvedElement("clover-suspense", { + style: "display:contents", + slot, + }, node); + } + chunks.push(render.stringifyNode(node)); + resolve?.(); + }; + yield ``; + do { + await new Promise((done) => (resolve = done)); + yield* chunks; + chunks = []; + } while (state!.nextId < state!.completed); + return addonOutput as unknown as T; +} + +interface State { + nested: boolean; + nextId: number; + completed: number; + pushChunk(name: string, node: render.ResolvedNode): void; +} + +export declare namespace Suspense { + export type { Input }; +} + +import * as render from "lib/render.ts"; diff --git a/tsconfig.json b/tsconfig.json index ef99ba0bb8e224d768f2172acd64da26c7540688..e7217bd66f71ea30ed7db9188d4c7e9ef7ce3842 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "baseUrl": ".", "incremental": true, "jsx": "react-jsxdev", - "jsxImportSource": "#engine", + "jsxImportSource": "#jsx", "lib": ["esnext", "dom", "dom.iterable"], "module": "nodenext", "noEmit": true,