authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-07-07 20:58:02-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:45-07:00
log84fe2100063fd3f75c749270095c416ef5c69804
tree3938afec0b730a6fece5af88eafcdcf578290745
parent43c83a513c406ddfbf3d46079672a0576f5d7cd3
signature Commit is signed but in an unrecognized format.

format

48 files changed, 5217 insertions(+), 5177 deletions(-)

flake.nix+3
......@@ -16,7 +16,10 @@
1616 pkgs.nodejs_24 # runtime
1717 pkgs.deno # formatter
1818 (pkgs.ffmpeg.override {
19 withOpus = true;
1920 withSvtav1 = true;
21 withJxl = true;
22 withWebp = true;
2023 })
2124 ];
2225 };
framework/backend/entry-node.ts+35-35
......@@ -1,35 +1,35 @@
1import "@paperclover/console/inject";
2import "#debug";
3
4const protocol = "http";
5
6const server = serve({
7 fetch: app.fetch,
8}, ({ address, port }) => {
9 if (address === "::") address = "::1";
10 console.info(url.format({
11 protocol,
12 hostname: address,
13 port,
14 }));
15});
16
17process.on("SIGINT", () => {
18 server.close();
19 process.exit(0);
20});
21
22process.on("SIGTERM", () => {
23 server.close((err) => {
24 if (err) {
25 console.error(err);
26 process.exit(1);
27 }
28 process.exit(0);
29 });
30});
31
32import app from "#backend";
33import url from "node:url";
34import { serve } from "@hono/node-server";
35import process from "node:process";
1import "@paperclover/console/inject";
2import "#debug";
3
4const protocol = "http";
5
6const server = serve({
7 fetch: app.fetch,
8}, ({ address, port }) => {
9 if (address === "::") address = "::1";
10 console.info(url.format({
11 protocol,
12 hostname: address,
13 port,
14 }));
15});
16
17process.on("SIGINT", () => {
18 server.close();
19 process.exit(0);
20});
21
22process.on("SIGTERM", () => {
23 server.close((err) => {
24 if (err) {
25 console.error(err);
26 process.exit(1);
27 }
28 process.exit(0);
29 });
30});
31
32import app from "#backend";
33import url from "node:url";
34import { serve } from "@hono/node-server";
35import process from "node:process";
framework/backend/entry-passthru.ts+4-4
......@@ -1,4 +1,4 @@
1import "@paperclover/console/inject";
2export default app;
3
4import app from "#backend";
1import "@paperclover/console/inject";
2export default app;
3
4import app from "#backend";
framework/debug.safe.ts+17-17
......@@ -1,17 +1,17 @@
1globalThis.UNWRAP = (t, ...args) => {
2 if (t == null) {
3 throw new Error(
4 args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")",
5 );
6 }
7 return t;
8};
9globalThis.ASSERT = (t, ...args) => {
10 if (!t) {
11 throw new Error(
12 args.length > 0 ? util.format(...args) : "Assertion Failed",
13 );
14 }
15};
16
17import * as util from "node:util";
1globalThis.UNWRAP = (t, ...args) => {
2 if (t == null) {
3 throw new Error(
4 args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")",
5 );
6 }
7 return t;
8};
9globalThis.ASSERT = (t, ...args) => {
10 if (!t) {
11 throw new Error(
12 args.length > 0 ? util.format(...args) : "Assertion Failed",
13 );
14 }
15};
16
17import * as util from "node:util";
framework/definitions.d.ts+4-4
......@@ -1,4 +1,4 @@
1declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
2declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
3
4type Timer = ReturnType<typeof setTimeout>;
1declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
2declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
3
4type Timer = ReturnType<typeof setTimeout>;
framework/engine/jsx-runtime.ts+54-54
......@@ -1,54 +1,54 @@
1export const Fragment = ({ children }: { children: engine.Node[] }) => children;
2
3export function jsx(
4 type: string | engine.Component,
5 props: Record<string, unknown>,
6): engine.Element {
7 if (typeof type !== "function" && typeof type !== "string") {
8 throw new Error("Invalid component type: " + engine.inspect(type));
9 }
10 return [engine.kElement, type, props];
11}
12
13export function jsxDEV(
14 type: string | engine.Component,
15 props: Record<string, unknown>,
16 // Unused with the clover engine
17 _key: string,
18 // Unused with the clover engine
19 _isStaticChildren: boolean,
20 source: engine.SrcLoc,
21): engine.Element {
22 const { fileName, lineNumber, columnNumber } = source;
23
24 // Assert the component type is valid to render.
25 if (typeof type !== "function" && typeof type !== "string") {
26 throw new Error(
27 `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` +
28 engine.inspect(type) +
29 ". Clover SSR element must be a function or string",
30 );
31 }
32
33 // Construct an `ssr.Element`
34 return [engine.kElement, type, props, "", source];
35}
36
37// jsxs
38export { jsx as jsxs };
39
40declare global {
41 namespace JSX {
42 interface IntrinsicElements {
43 [name: string]: Record<string, unknown>;
44 }
45 interface ElementChildrenAttribute {
46 children: Node;
47 }
48 type Element = engine.Element;
49 type ElementType = keyof IntrinsicElements | engine.Component;
50 type ElementClass = ReturnType<engine.Component>;
51 }
52}
53
54import * as engine from "./ssr.ts";
1export const Fragment = ({ children }: { children: engine.Node[] }) => children;
2
3export function jsx(
4 type: string | engine.Component,
5 props: Record<string, unknown>,
6): engine.Element {
7 if (typeof type !== "function" && typeof type !== "string") {
8 throw new Error("Invalid component type: " + engine.inspect(type));
9 }
10 return [engine.kElement, type, props];
11}
12
13export function jsxDEV(
14 type: string | engine.Component,
15 props: Record<string, unknown>,
16 // Unused with the clover engine
17 _key: string,
18 // Unused with the clover engine
19 _isStaticChildren: boolean,
20 source: engine.SrcLoc,
21): engine.Element {
22 const { fileName, lineNumber, columnNumber } = source;
23
24 // Assert the component type is valid to render.
25 if (typeof type !== "function" && typeof type !== "string") {
26 throw new Error(
27 `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` +
28 engine.inspect(type) +
29 ". Clover SSR element must be a function or string",
30 );
31 }
32
33 // Construct an `ssr.Element`
34 return [engine.kElement, type, props, "", source];
35}
36
37// jsxs
38export { jsx as jsxs };
39
40declare global {
41 namespace JSX {
42 interface IntrinsicElements {
43 [name: string]: Record<string, unknown>;
44 }
45 interface ElementChildrenAttribute {
46 children: Node;
47 }
48 type Element = engine.Element;
49 type ElementType = keyof IntrinsicElements | engine.Component;
50 type ElementClass = ReturnType<engine.Component>;
51 }
52}
53
54import * as engine from "./ssr.ts";
framework/engine/marko-runtime.ts+147-147
......@@ -1,147 +1,147 @@
1// This file is used to integrate Marko into the Clover Engine and Sitegen
2// To use, replace the "marko/html" import with this file.
3export * from "#marko/html";
4
5interface BodyContentObject {
6 [x: PropertyKey]: unknown;
7 content: ServerRenderer;
8}
9
10export const createTemplate = (
11 templateId: string,
12 renderer: ServerRenderer,
13) => {
14 const { render } = marko.createTemplate(templateId, renderer);
15 function wrap(props: Record<string, unknown>, n: number) {
16 // Marko Custom Tags
17 const cloverAsyncMarker = { isAsync: false };
18 let r: engine.Render | undefined = undefined;
19 try {
20 r = engine.getCurrentRender();
21 } catch {}
22 // Support using Marko outside of Clover SSR
23 if (r) {
24 engine.setCurrentRender(null);
25 const markoResult = render.call(renderer, {
26 ...props,
27 $global: { clover: r, cloverAsyncMarker },
28 });
29 if (cloverAsyncMarker.isAsync) {
30 return markoResult.then(engine.html);
31 }
32 const rr = markoResult.toString();
33 return engine.html(rr);
34 } else {
35 return renderer(props, n);
36 }
37 }
38 wrap.render = render;
39 wrap.unwrapped = renderer;
40 return wrap;
41};
42
43export const dynamicTag = (
44 scopeId: number,
45 accessor: Accessor,
46 tag: unknown | string | ServerRenderer | BodyContentObject,
47 inputOrArgs: unknown,
48 content?: (() => void) | 0,
49 inputIsArgs?: 1,
50 serializeReason?: 1 | 0,
51) => {
52 if (typeof tag === "function") {
53 clover: {
54 const unwrapped = (tag as any).unwrapped;
55 if (unwrapped) {
56 tag = unwrapped;
57 break clover;
58 }
59 let r: engine.Render;
60 try {
61 r = engine.getCurrentRender();
62 if (!r) throw 0;
63 } catch {
64 r = marko.$global().clover as engine.Render;
65 }
66 if (!r) throw new Error("No Clover Render Active");
67 const subRender = engine.initRender(r.async !== -1, r.addon);
68 const resolved = engine.resolveNode(subRender, [
69 engine.kElement,
70 tag,
71 inputOrArgs,
72 ]);
73
74 if (subRender.async > 0) {
75 const marker = marko.$global().cloverAsyncMarker as Async;
76 marker.isAsync = true;
77
78 // Wait for async work to finish
79 const { resolve, reject, promise } = Promise.withResolvers<string>();
80 subRender.asyncDone = () => {
81 const rejections = subRender.rejections;
82 if (!rejections) return resolve(engine.renderNode(resolved));
83 (r.rejections ??= []).push(...rejections);
84 return reject(new Error("Render had errors"));
85 };
86 marko.fork(
87 scopeId,
88 accessor,
89 promise,
90 (string: string) => marko.write(string),
91 0,
92 );
93 } else {
94 marko.write(engine.renderNode(resolved));
95 }
96 return;
97 }
98 }
99 return marko.dynamicTag(
100 scopeId,
101 accessor,
102 tag,
103 inputOrArgs,
104 content,
105 inputIsArgs,
106 serializeReason,
107 );
108};
109
110export function fork(
111 scopeId: number,
112 accessor: Accessor,
113 promise: Promise<unknown>,
114 callback: (data: unknown) => void,
115 serializeMarker?: 0 | 1,
116) {
117 const marker = marko.$global().cloverAsyncMarker as Async;
118 marker.isAsync = true;
119 marko.fork(scopeId, accessor, promise, callback, serializeMarker);
120}
121
122export function escapeXML(input: unknown) {
123 // The rationale of this check is that the default toString method
124 // creating `[object Object]` is universally useless to any end user.
125 if (
126 input == null ||
127 (typeof input === "object" && input &&
128 // only block this if it's the default `toString`
129 input.toString === Object.prototype.toString)
130 ) {
131 throw new Error(
132 `Unexpected value in template placeholder: '` +
133 engine.inspect(input) + "'. " +
134 `To emit a literal '${input}', use \${String(value)}`,
135 );
136 }
137 return marko.escapeXML(input);
138}
139
140interface Async {
141 isAsync: boolean;
142}
143
144import * as engine from "./ssr.ts";
145import type { ServerRenderer } from "marko/html/template";
146import { type Accessor } from "marko/common/types";
147import * as marko from "#marko/html";
1// This file is used to integrate Marko into the Clover Engine and Sitegen
2// To use, replace the "marko/html" import with this file.
3export * from "#marko/html";
4
5interface BodyContentObject {
6 [x: PropertyKey]: unknown;
7 content: ServerRenderer;
8}
9
10export const createTemplate = (
11 templateId: string,
12 renderer: ServerRenderer,
13) => {
14 const { render } = marko.createTemplate(templateId, renderer);
15 function wrap(props: Record<string, unknown>, n: number) {
16 // Marko Custom Tags
17 const cloverAsyncMarker = { isAsync: false };
18 let r: engine.Render | undefined = undefined;
19 try {
20 r = engine.getCurrentRender();
21 } catch {}
22 // Support using Marko outside of Clover SSR
23 if (r) {
24 engine.setCurrentRender(null);
25 const markoResult = render.call(renderer, {
26 ...props,
27 $global: { clover: r, cloverAsyncMarker },
28 });
29 if (cloverAsyncMarker.isAsync) {
30 return markoResult.then(engine.html);
31 }
32 const rr = markoResult.toString();
33 return engine.html(rr);
34 } else {
35 return renderer(props, n);
36 }
37 }
38 wrap.render = render;
39 wrap.unwrapped = renderer;
40 return wrap;
41};
42
43export const dynamicTag = (
44 scopeId: number,
45 accessor: Accessor,
46 tag: unknown | string | ServerRenderer | BodyContentObject,
47 inputOrArgs: unknown,
48 content?: (() => void) | 0,
49 inputIsArgs?: 1,
50 serializeReason?: 1 | 0,
51) => {
52 if (typeof tag === "function") {
53 clover: {
54 const unwrapped = (tag as any).unwrapped;
55 if (unwrapped) {
56 tag = unwrapped;
57 break clover;
58 }
59 let r: engine.Render;
60 try {
61 r = engine.getCurrentRender();
62 if (!r) throw 0;
63 } catch {
64 r = marko.$global().clover as engine.Render;
65 }
66 if (!r) throw new Error("No Clover Render Active");
67 const subRender = engine.initRender(r.async !== -1, r.addon);
68 const resolved = engine.resolveNode(subRender, [
69 engine.kElement,
70 tag,
71 inputOrArgs,
72 ]);
73
74 if (subRender.async > 0) {
75 const marker = marko.$global().cloverAsyncMarker as Async;
76 marker.isAsync = true;
77
78 // Wait for async work to finish
79 const { resolve, reject, promise } = Promise.withResolvers<string>();
80 subRender.asyncDone = () => {
81 const rejections = subRender.rejections;
82 if (!rejections) return resolve(engine.renderNode(resolved));
83 (r.rejections ??= []).push(...rejections);
84 return reject(new Error("Render had errors"));
85 };
86 marko.fork(
87 scopeId,
88 accessor,
89 promise,
90 (string: string) => marko.write(string),
91 0,
92 );
93 } else {
94 marko.write(engine.renderNode(resolved));
95 }
96 return;
97 }
98 }
99 return marko.dynamicTag(
100 scopeId,
101 accessor,
102 tag,
103 inputOrArgs,
104 content,
105 inputIsArgs,
106 serializeReason,
107 );
108};
109
110export function fork(
111 scopeId: number,
112 accessor: Accessor,
113 promise: Promise<unknown>,
114 callback: (data: unknown) => void,
115 serializeMarker?: 0 | 1,
116) {
117 const marker = marko.$global().cloverAsyncMarker as Async;
118 marker.isAsync = true;
119 marko.fork(scopeId, accessor, promise, callback, serializeMarker);
120}
121
122export function escapeXML(input: unknown) {
123 // The rationale of this check is that the default toString method
124 // creating `[object Object]` is universally useless to any end user.
125 if (
126 input == null ||
127 (typeof input === "object" && input &&
128 // only block this if it's the default `toString`
129 input.toString === Object.prototype.toString)
130 ) {
131 throw new Error(
132 `Unexpected value in template placeholder: '` +
133 engine.inspect(input) + "'. " +
134 `To emit a literal '${input}', use \${String(value)}`,
135 );
136 }
137 return marko.escapeXML(input);
138}
139
140interface Async {
141 isAsync: boolean;
142}
143
144import * as engine from "./ssr.ts";
145import type { ServerRenderer } from "marko/html/template";
146import { type Accessor } from "marko/common/types";
147import * as marko from "#marko/html";
framework/engine/ssr.test.tsx+41-41
......@@ -1,41 +1,41 @@
1import { test } from "node:test";
2import * as engine from "./ssr.ts";
3
4test("sanity", (t) => t.assert.equal(engine.ssrSync("gm <3").text, "gm &lt;3"));
5test("simple tree", (t) =>
6 t.assert.equal(
7 engine.ssrSync(
8 <main class={["a", "b"]}>
9 <h1 style="background-color:red">hello world</h1>
10 <p>haha</p>
11 {1}|
12 {0}|
13 {true}|
14 {false}|
15 {null}|
16 {undefined}|
17 </main>,
18 ).text,
19 '<main class="a b"><h1 style=background-color:red>hello world</h1><p>haha</p>1|0|||||</main>',
20 ));
21test("unescaped/escaped html", (t) =>
22 t.assert.equal(
23 engine.ssrSync(<div>{engine.html("<fuck>")}{"\"&'`<>"}</div>).text,
24 "<div><fuck>&quot;&amp;&#x27;&#x60;&lt;&gt;</div>",
25 ));
26test("clsx built-in", (t) =>
27 t.assert.equal(
28 engine.ssrSync(
29 <>
30 <a class="a" />
31 <b class={null} />
32 <c class={undefined} />
33 <d class={["a", "b", null]} />
34 <e class={{ a: true, b: false }} />
35 <e
36 class={[null, "x", { z: true }, [{ m: true }, null, { v: false }]]}
37 />
38 </>,
39 ).text,
40 '<a class=a></a><b></b><c></c><d class="a b"></d><e class=a></e><e class="x z m"></e>',
41 ));
1import { test } from "node:test";
2import * as engine from "./ssr.ts";
3
4test("sanity", (t) => t.assert.equal(engine.ssrSync("gm <3").text, "gm &lt;3"));
5test("simple tree", (t) =>
6 t.assert.equal(
7 engine.ssrSync(
8 <main class={["a", "b"]}>
9 <h1 style="background-color:red">hello world</h1>
10 <p>haha</p>
11 {1}|
12 {0}|
13 {true}|
14 {false}|
15 {null}|
16 {undefined}|
17 </main>,
18 ).text,
19 '<main class="a b"><h1 style=background-color:red>hello world</h1><p>haha</p>1|0|||||</main>',
20 ));
21test("unescaped/escaped html", (t) =>
22 t.assert.equal(
23 engine.ssrSync(<div>{engine.html("<fuck>")}{"\"&'`<>"}</div>).text,
24 "<div><fuck>&quot;&amp;&#x27;&#x60;&lt;&gt;</div>",
25 ));
26test("clsx built-in", (t) =>
27 t.assert.equal(
28 engine.ssrSync(
29 <>
30 <a class="a" />
31 <b class={null} />
32 <c class={undefined} />
33 <d class={["a", "b", null]} />
34 <e class={{ a: true, b: false }} />
35 <e
36 class={[null, "x", { z: true }, [{ m: true }, null, { v: false }]]}
37 />
38 </>,
39 ).text,
40 '<a class=a></a><b></b><c></c><d class="a b"></d><e class=a></e><e class="x z m"></e>',
41 ));
framework/engine/ssr.ts+311-311
......@@ -1,311 +1,311 @@
1// Clover's Rendering Engine is the backbone of her website generator. It
2// converts objects and components (functions returning 'Node') into HTML. The
3// engine is simple and self-contained, with integrations for JSX and Marko
4// (which can interop with each-other) are provided next to this file.
5//
6// Add-ons to the rendering engine can provide opaque data, And retrieve it
7// within component calls with 'getAddonData'. For example, 'sitegen' uses this
8// to track needed client scripts without introducing patches to the engine.
9export type Addons = Record<string | symbol, unknown>;
10
11export function ssrSync(node: Node): Result;
12export function ssrSync<A extends Addons>(node: Node, addon: A): Result<A>;
13export function ssrSync(node: Node, addon: Addons = {}) {
14 const r = initRender(false, addon);
15 const resolved = resolveNode(r, node);
16 return { text: renderNode(resolved), addon };
17}
18
19export function ssrAsync(node: Node): Promise<Result>;
20export function ssrAsync<A extends Addons>(
21 node: Node,
22 addon: A,
23): Promise<Result<A>>;
24export function ssrAsync(node: Node, addon: Addons = {}) {
25 const r = initRender(true, addon);
26 const resolved = resolveNode(r, node);
27 if (r.async === 0) {
28 return Promise.resolve({ text: renderNode(resolved), addon });
29 }
30 const { resolve, reject, promise } = Promise.withResolvers<Result>();
31 r.asyncDone = () => {
32 const rejections = r.rejections;
33 if (!rejections) return resolve({ text: renderNode(resolved), addon });
34 if (rejections.length === 1) return reject(rejections[0]);
35 return reject(new AggregateError(rejections));
36 };
37 return promise;
38}
39
40/** Inline HTML into a render without escaping it */
41export function html(rawText: ResolvedNode): DirectHtml {
42 return [kDirectHtml, rawText];
43}
44
45interface Result<A extends Addons = Addons> {
46 text: string;
47 addon: A;
48}
49
50export interface Render {
51 /**
52 * Set to '-1' if rendering synchronously
53 * Number of async promises the render is waiting on.
54 */
55 async: number | -1;
56 asyncDone: null | (() => void);
57 /** When components reject, those are logged here */
58 rejections: unknown[] | null;
59 /** Add-ons to the rendering engine store state here */
60 addon: Addons;
61}
62
63export const kElement = Symbol("Element");
64export const kDirectHtml = Symbol("DirectHtml");
65
66/** Node represents a webpage that can be 'rendered' into HTML. */
67export type Node =
68 | number
69 | string // Escape HTML
70 | Node[] // Concat
71 | Element // Render
72 | DirectHtml // Insert
73 | Promise<Node> // Await
74 // Ignore
75 | undefined
76 | null
77 | boolean;
78export type Element = [
79 tag: typeof kElement,
80 type: string | Component,
81 props: Record<string, unknown>,
82 _?: "",
83 source?: SrcLoc,
84];
85export type DirectHtml = [tag: typeof kDirectHtml, html: ResolvedNode];
86/**
87 * Components must return a value; 'undefined' is prohibited here
88 * to avoid functions that are missing a return statement.
89 */
90export type Component = (
91 props: Record<any, any>,
92) => Exclude<Node, undefined>;
93/** Emitted by JSX runtime */
94export interface SrcLoc {
95 fileName: string;
96 lineNumber: number;
97 columnNumber: number;
98}
99
100/**
101 * Resolution narrows the type 'Node' into 'ResolvedNode'. Async trees are
102 * marked in the 'Render'. This operation performs everything besides the final
103 * string concatenation. This function is agnostic across async/sync modes.
104 */
105export function resolveNode(r: Render, node: unknown): ResolvedNode {
106 if (!node && node !== 0) return ""; // falsy, non numeric
107 if (typeof node !== "object") {
108 if (node === true) return ""; // booleans are ignored
109 if (typeof node === "string") return escapeHtml(node);
110 if (typeof node === "number") return String(node); // no escaping ever
111 throw new Error(`Cannot render ${inspect(node)} to HTML`);
112 }
113 if (node instanceof Promise) {
114 if (r.async === -1) {
115 throw new Error(`Asynchronous rendering is not supported here.`);
116 }
117 const placeholder: InsertionPoint = [null];
118 r.async += 1;
119 node
120 .then((result) => void (placeholder[0] = resolveNode(r, result)))
121 // Intentionally catching errors in `resolveNode`
122 .catch((e) => (r.rejections ??= []).push(e))
123 .finally(() => {
124 if (--r.async == 0) {
125 if (r.asyncDone == null) throw new Error("r.asyncDone == null");
126 r.asyncDone();
127 r.asyncDone = null;
128 }
129 });
130 // This lie is checked with an assertion in `renderNode`
131 return placeholder as [ResolvedNode];
132 }
133 if (!Array.isArray(node)) {
134 throw new Error(`Invalid node type: ${inspect(node)}`);
135 }
136 const type = node[0];
137 if (type === kElement) {
138 const { 1: tag, 2: props } = node;
139 if (typeof tag === "function") {
140 currentRender = r;
141 try {
142 return resolveNode(r, tag(props));
143 } catch (e) {
144 const { 4: src } = node;
145 if (e && typeof e === "object") {
146 }
147 } finally {
148 currentRender = null;
149 }
150 }
151 if (typeof tag !== "string") throw new Error("Unexpected " + typeof type);
152 const children = props?.children;
153 if (children) return [kElement, tag, props, resolveNode(r, children)];
154 return node;
155 }
156 if (type === kDirectHtml) return node[1];
157 return node.map((elem) => resolveNode(r, elem));
158}
159
160export type ResolvedNode =
161 | ResolvedNode[] // Concat
162 | ResolvedElement // Render
163 | string; // Direct HTML
164export type ResolvedElement = [
165 tag: typeof kElement,
166 type: string,
167 props: Record<string, unknown>,
168 children: ResolvedNode,
169];
170/**
171 * Async rendering is done by creating an array of one item,
172 * which is already a valid 'Node', but the element is written
173 * once the data is available. The 'Render' contains a count
174 * of how many async jobs are left.
175 */
176export type InsertionPoint = [null | ResolvedNode];
177
178/**
179 * Convert 'ResolvedNode' into HTML text. This operation happens after all
180 * async work is settled. The HTML is emitted as concisely as possible.
181 */
182export function renderNode(node: ResolvedNode): string {
183 if (typeof node === "string") return node;
184 ASSERT(node, "Unresolved Render Node");
185 const type = node[0];
186 if (type === kElement) {
187 return renderElement(node as ResolvedElement);
188 }
189 node = node as ResolvedNode[]; // TS cannot infer.
190 let out = type ? renderNode(type) : "";
191 let len = node.length;
192 for (let i = 1; i < len; i++) {
193 const elem = node[i];
194 if (elem) out += renderNode(elem);
195 }
196 return out;
197}
198function renderElement(element: ResolvedElement) {
199 const { 1: tag, 2: props, 3: children } = element;
200 let out = "<" + tag;
201 let needSpace = true;
202 for (const prop in props) {
203 const value = props[prop];
204 if (!value || typeof value === "function") continue;
205 let attr;
206 switch (prop) {
207 default:
208 attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`;
209 break;
210 case "className":
211 // Legacy React Compat
212 case "class":
213 attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`;
214 break;
215 case "htmlFor":
216 throw new Error("Do not use the `htmlFor` attribute. Use `for`");
217 // Do not process these
218 case "children":
219 case "ref":
220 case "dangerouslySetInnerHTML":
221 case "key":
222 continue;
223 }
224 if (needSpace) out += " ", needSpace = !attr.endsWith('"');
225 out += attr;
226 }
227 out += ">";
228 if (children) out += renderNode(children);
229 if (
230 tag !== "br" && tag !== "img" && tag !== "input" && tag !== "meta" &&
231 tag !== "link" && tag !== "hr"
232 ) {
233 out += `</${tag}>`;
234 }
235 return out;
236}
237export function renderStyleAttribute(style: Record<string, string>) {
238 let out = ``;
239 for (const styleName in style) {
240 if (out) out += ";";
241 out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${
242 escapeHtml(String(style[styleName]))
243 }`;
244 }
245 return "style=" + quoteIfNeeded(out);
246}
247export function quoteIfNeeded(text: string) {
248 if (text.includes(" ")) return '"' + text + '"';
249 return text;
250}
251
252// -- utility functions --
253
254export function initRender(allowAsync: boolean, addon: Addons): Render {
255 return {
256 async: allowAsync ? 0 : -1,
257 rejections: null,
258 asyncDone: null,
259 addon,
260 };
261}
262
263let currentRender: Render | null = null;
264export function getCurrentRender() {
265 if (!currentRender) throw new Error("No Render Active");
266 return currentRender;
267}
268export function setCurrentRender(r?: Render | null) {
269 currentRender = r ?? null;
270}
271export function getUserData<T>(namespace: PropertyKey, def: () => T): T {
272 return (getCurrentRender().addon[namespace] ??= def()) as T;
273}
274
275export function inspect(object: unknown) {
276 try {
277 return require("node:util").inspect(object);
278 } catch {
279 return typeof object;
280 }
281}
282
283export type ClsxInput = string | Record<string, boolean | null> | ClsxInput[];
284export function clsx(mix: ClsxInput) {
285 var k, y, str = "";
286 if (typeof mix === "string") {
287 return mix;
288 } else if (typeof mix === "object") {
289 if (Array.isArray(mix)) {
290 for (k = 0; k < mix.length; k++) {
291 if (mix[k] && (y = clsx(mix[k]))) {
292 str && (str += " ");
293 str += y;
294 }
295 }
296 } else {
297 for (k in mix) {
298 if (mix[k]) {
299 str && (str += " ");
300 str += k;
301 }
302 }
303 }
304 }
305 return str;
306}
307
308export const escapeHtml = (unsafeText: string) =>
309 String(unsafeText)
310 .replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
311 .replace(/"/g, "&quot;").replace(/'/g, "&#x27;").replace(/`/g, "&#x60;");
1// Clover's Rendering Engine is the backbone of her website generator. It
2// converts objects and components (functions returning 'Node') into HTML. The
3// engine is simple and self-contained, with integrations for JSX and Marko
4// (which can interop with each-other) are provided next to this file.
5//
6// Add-ons to the rendering engine can provide opaque data, And retrieve it
7// within component calls with 'getAddonData'. For example, 'sitegen' uses this
8// to track needed client scripts without introducing patches to the engine.
9export type Addons = Record<string | symbol, unknown>;
10
11export function ssrSync(node: Node): Result;
12export function ssrSync<A extends Addons>(node: Node, addon: A): Result<A>;
13export function ssrSync(node: Node, addon: Addons = {}) {
14 const r = initRender(false, addon);
15 const resolved = resolveNode(r, node);
16 return { text: renderNode(resolved), addon };
17}
18
19export function ssrAsync(node: Node): Promise<Result>;
20export function ssrAsync<A extends Addons>(
21 node: Node,
22 addon: A,
23): Promise<Result<A>>;
24export function ssrAsync(node: Node, addon: Addons = {}) {
25 const r = initRender(true, addon);
26 const resolved = resolveNode(r, node);
27 if (r.async === 0) {
28 return Promise.resolve({ text: renderNode(resolved), addon });
29 }
30 const { resolve, reject, promise } = Promise.withResolvers<Result>();
31 r.asyncDone = () => {
32 const rejections = r.rejections;
33 if (!rejections) return resolve({ text: renderNode(resolved), addon });
34 if (rejections.length === 1) return reject(rejections[0]);
35 return reject(new AggregateError(rejections));
36 };
37 return promise;
38}
39
40/** Inline HTML into a render without escaping it */
41export function html(rawText: ResolvedNode): DirectHtml {
42 return [kDirectHtml, rawText];
43}
44
45interface Result<A extends Addons = Addons> {
46 text: string;
47 addon: A;
48}
49
50export interface Render {
51 /**
52 * Set to '-1' if rendering synchronously
53 * Number of async promises the render is waiting on.
54 */
55 async: number | -1;
56 asyncDone: null | (() => void);
57 /** When components reject, those are logged here */
58 rejections: unknown[] | null;
59 /** Add-ons to the rendering engine store state here */
60 addon: Addons;
61}
62
63export const kElement = Symbol("Element");
64export const kDirectHtml = Symbol("DirectHtml");
65
66/** Node represents a webpage that can be 'rendered' into HTML. */
67export type Node =
68 | number
69 | string // Escape HTML
70 | Node[] // Concat
71 | Element // Render
72 | DirectHtml // Insert
73 | Promise<Node> // Await
74 // Ignore
75 | undefined
76 | null
77 | boolean;
78export type Element = [
79 tag: typeof kElement,
80 type: string | Component,
81 props: Record<string, unknown>,
82 _?: "",
83 source?: SrcLoc,
84];
85export type DirectHtml = [tag: typeof kDirectHtml, html: ResolvedNode];
86/**
87 * Components must return a value; 'undefined' is prohibited here
88 * to avoid functions that are missing a return statement.
89 */
90export type Component = (
91 props: Record<any, any>,
92) => Exclude<Node, undefined>;
93/** Emitted by JSX runtime */
94export interface SrcLoc {
95 fileName: string;
96 lineNumber: number;
97 columnNumber: number;
98}
99
100/**
101 * Resolution narrows the type 'Node' into 'ResolvedNode'. Async trees are
102 * marked in the 'Render'. This operation performs everything besides the final
103 * string concatenation. This function is agnostic across async/sync modes.
104 */
105export function resolveNode(r: Render, node: unknown): ResolvedNode {
106 if (!node && node !== 0) return ""; // falsy, non numeric
107 if (typeof node !== "object") {
108 if (node === true) return ""; // booleans are ignored
109 if (typeof node === "string") return escapeHtml(node);
110 if (typeof node === "number") return String(node); // no escaping ever
111 throw new Error(`Cannot render ${inspect(node)} to HTML`);
112 }
113 if (node instanceof Promise) {
114 if (r.async === -1) {
115 throw new Error(`Asynchronous rendering is not supported here.`);
116 }
117 const placeholder: InsertionPoint = [null];
118 r.async += 1;
119 node
120 .then((result) => void (placeholder[0] = resolveNode(r, result)))
121 // Intentionally catching errors in `resolveNode`
122 .catch((e) => (r.rejections ??= []).push(e))
123 .finally(() => {
124 if (--r.async == 0) {
125 if (r.asyncDone == null) throw new Error("r.asyncDone == null");
126 r.asyncDone();
127 r.asyncDone = null;
128 }
129 });
130 // This lie is checked with an assertion in `renderNode`
131 return placeholder as [ResolvedNode];
132 }
133 if (!Array.isArray(node)) {
134 throw new Error(`Invalid node type: ${inspect(node)}`);
135 }
136 const type = node[0];
137 if (type === kElement) {
138 const { 1: tag, 2: props } = node;
139 if (typeof tag === "function") {
140 currentRender = r;
141 try {
142 return resolveNode(r, tag(props));
143 } catch (e) {
144 const { 4: src } = node;
145 if (e && typeof e === "object") {
146 }
147 } finally {
148 currentRender = null;
149 }
150 }
151 if (typeof tag !== "string") throw new Error("Unexpected " + typeof type);
152 const children = props?.children;
153 if (children) return [kElement, tag, props, resolveNode(r, children)];
154 return node;
155 }
156 if (type === kDirectHtml) return node[1];
157 return node.map((elem) => resolveNode(r, elem));
158}
159
160export type ResolvedNode =
161 | ResolvedNode[] // Concat
162 | ResolvedElement // Render
163 | string; // Direct HTML
164export type ResolvedElement = [
165 tag: typeof kElement,
166 type: string,
167 props: Record<string, unknown>,
168 children: ResolvedNode,
169];
170/**
171 * Async rendering is done by creating an array of one item,
172 * which is already a valid 'Node', but the element is written
173 * once the data is available. The 'Render' contains a count
174 * of how many async jobs are left.
175 */
176export type InsertionPoint = [null | ResolvedNode];
177
178/**
179 * Convert 'ResolvedNode' into HTML text. This operation happens after all
180 * async work is settled. The HTML is emitted as concisely as possible.
181 */
182export function renderNode(node: ResolvedNode): string {
183 if (typeof node === "string") return node;
184 ASSERT(node, "Unresolved Render Node");
185 const type = node[0];
186 if (type === kElement) {
187 return renderElement(node as ResolvedElement);
188 }
189 node = node as ResolvedNode[]; // TS cannot infer.
190 let out = type ? renderNode(type) : "";
191 let len = node.length;
192 for (let i = 1; i < len; i++) {
193 const elem = node[i];
194 if (elem) out += renderNode(elem);
195 }
196 return out;
197}
198function renderElement(element: ResolvedElement) {
199 const { 1: tag, 2: props, 3: children } = element;
200 let out = "<" + tag;
201 let needSpace = true;
202 for (const prop in props) {
203 const value = props[prop];
204 if (!value || typeof value === "function") continue;
205 let attr;
206 switch (prop) {
207 default:
208 attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`;
209 break;
210 case "className":
211 // Legacy React Compat
212 case "class":
213 attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`;
214 break;
215 case "htmlFor":
216 throw new Error("Do not use the `htmlFor` attribute. Use `for`");
217 // Do not process these
218 case "children":
219 case "ref":
220 case "dangerouslySetInnerHTML":
221 case "key":
222 continue;
223 }
224 if (needSpace) out += " ", needSpace = !attr.endsWith('"');
225 out += attr;
226 }
227 out += ">";
228 if (children) out += renderNode(children);
229 if (
230 tag !== "br" && tag !== "img" && tag !== "input" && tag !== "meta" &&
231 tag !== "link" && tag !== "hr"
232 ) {
233 out += `</${tag}>`;
234 }
235 return out;
236}
237export function renderStyleAttribute(style: Record<string, string>) {
238 let out = ``;
239 for (const styleName in style) {
240 if (out) out += ";";
241 out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${
242 escapeHtml(String(style[styleName]))
243 }`;
244 }
245 return "style=" + quoteIfNeeded(out);
246}
247export function quoteIfNeeded(text: string) {
248 if (text.includes(" ")) return '"' + text + '"';
249 return text;
250}
251
252// -- utility functions --
253
254export function initRender(allowAsync: boolean, addon: Addons): Render {
255 return {
256 async: allowAsync ? 0 : -1,
257 rejections: null,
258 asyncDone: null,
259 addon,
260 };
261}
262
263let currentRender: Render | null = null;
264export function getCurrentRender() {
265 if (!currentRender) throw new Error("No Render Active");
266 return currentRender;
267}
268export function setCurrentRender(r?: Render | null) {
269 currentRender = r ?? null;
270}
271export function getUserData<T>(namespace: PropertyKey, def: () => T): T {
272 return (getCurrentRender().addon[namespace] ??= def()) as T;
273}
274
275export function inspect(object: unknown) {
276 try {
277 return require("node:util").inspect(object);
278 } catch {
279 return typeof object;
280 }
281}
282
283export type ClsxInput = string | Record<string, boolean | null> | ClsxInput[];
284export function clsx(mix: ClsxInput) {
285 var k, y, str = "";
286 if (typeof mix === "string") {
287 return mix;
288 } else if (typeof mix === "object") {
289 if (Array.isArray(mix)) {
290 for (k = 0; k < mix.length; k++) {
291 if (mix[k] && (y = clsx(mix[k]))) {
292 str && (str += " ");
293 str += y;
294 }
295 }
296 } else {
297 for (k in mix) {
298 if (mix[k]) {
299 str && (str += " ");
300 str += k;
301 }
302 }
303 }
304 }
305 return str;
306}
307
308export const escapeHtml = (unsafeText: string) =>
309 String(unsafeText)
310 .replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
311 .replace(/"/g, "&quot;").replace(/'/g, "&#x27;").replace(/`/g, "&#x60;");
framework/engine/suspense.test.tsx+40-40
......@@ -1,40 +1,40 @@
1import { test } from "node:test";
2import { renderStreaming, Suspense } from "./suspense.ts";
3
4test("sanity", async (t) => {
5 let resolve: () => void = null!;
6
7 // @ts-expect-error
8 async function AsyncComponent() {
9 await new Promise<void>((done) => resolve = done);
10 return <button>wow!</button>;
11 }
12
13 const example = (
14 <main>
15 <h1>app shell</h1>
16 <Suspense fallback="loading...">
17 <AsyncComponent />
18 </Suspense>
19 <footer>(c) 2025</footer>
20 </main>
21 );
22
23 const iterator = renderStreaming(example);
24 const assertContinue = (actual: unknown, value: unknown) =>
25 t.assert.deepEqual(actual, { done: false, value });
26
27 assertContinue(
28 await iterator.next(),
29 "<template shadowrootmode=open><main><h1>app shell</h1><slot name=suspended_1>loading...</slot><footer>(c) 2025</footer></main></template>",
30 );
31 t.assert.ok(resolve !== null), resolve();
32 assertContinue(
33 await iterator.next(),
34 "<button slot=suspended_1>wow!</button>",
35 );
36 t.assert.deepEqual(
37 await iterator.next(),
38 { done: true, value: {} },
39 );
40});
1import { test } from "node:test";
2import { renderStreaming, Suspense } from "./suspense.ts";
3
4test("sanity", async (t) => {
5 let resolve: () => void = null!;
6
7 // @ts-expect-error
8 async function AsyncComponent() {
9 await new Promise<void>((done) => resolve = done);
10 return <button>wow!</button>;
11 }
12
13 const example = (
14 <main>
15 <h1>app shell</h1>
16 <Suspense fallback="loading...">
17 <AsyncComponent />
18 </Suspense>
19 <footer>(c) 2025</footer>
20 </main>
21 );
22
23 const iterator = renderStreaming(example);
24 const assertContinue = (actual: unknown, value: unknown) =>
25 t.assert.deepEqual(actual, { done: false, value });
26
27 assertContinue(
28 await iterator.next(),
29 "<template shadowrootmode=open><main><h1>app shell</h1><slot name=suspended_1>loading...</slot><footer>(c) 2025</footer></main></template>",
30 );
31 t.assert.ok(resolve !== null), resolve();
32 assertContinue(
33 await iterator.next(),
34 "<button slot=suspended_1>wow!</button>",
35 );
36 t.assert.deepEqual(
37 await iterator.next(),
38 { done: true, value: {} },
39 );
40});
framework/engine/suspense.ts+102-102
......@@ -1,102 +1,102 @@
1// This file implements out-of-order HTML streaming, mimicking the React
2// Suspense API. To use, place Suspense around an expensive async component
3// and render the page with 'renderStreaming'.
4//
5// Implementation of this article:
6// https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/
7//
8// I would link to an article from Next.js or React, but their examples
9// are too verbose and not informative to what they actually do.
10const kState = Symbol("SuspenseState");
11
12interface SuspenseProps {
13 children: ssr.Node;
14 fallback?: ssr.Node;
15}
16
17interface State {
18 nested: boolean;
19 nextId: number;
20 completed: number;
21 pushChunk(name: string, node: ssr.ResolvedNode): void;
22}
23
24export function Suspense({ children, fallback }: SuspenseProps): ssr.Node {
25 const state = ssr.getUserData<State>(kState, () => {
26 throw new Error("Can only use <Suspense> with 'renderStreaming'");
27 });
28 if (state.nested) throw new Error("<Suspense> cannot be nested");
29 const parent = ssr.getCurrentRender()!;
30 const r = ssr.initRender(true, { [kState]: { nested: true } });
31 const resolved = ssr.resolveNode(r, children);
32 if (r.async == 0) return ssr.html(resolved);
33 const name = "suspended_" + (++state.nextId);
34 state.nested = true;
35 const ip: [ssr.ResolvedNode] = [
36 [
37 ssr.kElement,
38 "slot",
39 { name },
40 fallback ? ssr.resolveNode(parent, fallback) : "",
41 ],
42 ];
43 state.nested = false;
44 r.asyncDone = () => {
45 const rejections = r.rejections;
46 if (rejections && rejections.length > 0) throw new Error("TODO");
47 state.pushChunk?.(name, ip[0] = resolved);
48 };
49 return ssr.html(ip);
50}
51
52// TODO: add a User-Agent parameter, which is used to determine if a
53// fallback path must be used.
54// - Before ~2024 needs to use a JS implementation.
55// - IE should probably bail out entirely.
56export async function* renderStreaming<
57 T extends ssr.Addons = Record<never, unknown>,
58>(
59 node: ssr.Node,
60 addon: T = {} as T,
61) {
62 const {
63 text: begin,
64 addon: { [kState]: state, ...addonOutput },
65 } = await ssr.ssrAsync(node, {
66 ...addon,
67 [kState]: {
68 nested: false,
69 nextId: 0,
70 completed: 0,
71 pushChunk: () => {},
72 } satisfies State as State,
73 });
74 if (state.nextId === 0) {
75 yield begin;
76 return addonOutput as unknown as T;
77 }
78 let resolve: (() => void) | null = null;
79 let chunks: string[] = [];
80 state.pushChunk = (slot, node) => {
81 while (node.length === 1 && Array.isArray(node)) node = node[0];
82 if (node[0] === ssr.kElement) {
83 (node as ssr.ResolvedElement)[2].slot = slot;
84 } else {
85 node = [ssr.kElement, "clover-suspense", {
86 style: "display:contents",
87 slot,
88 }, node];
89 }
90 chunks.push(ssr.renderNode(node));
91 resolve?.();
92 };
93 yield `<template shadowrootmode=open>${begin}</template>`;
94 do {
95 await new Promise<void>((done) => resolve = done);
96 yield* chunks;
97 chunks = [];
98 } while (state.nextId < state.completed);
99 return addonOutput as unknown as T;
100}
101
102import * as ssr from "./ssr.ts";
1// This file implements out-of-order HTML streaming, mimicking the React
2// Suspense API. To use, place Suspense around an expensive async component
3// and render the page with 'renderStreaming'.
4//
5// Implementation of this article:
6// https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/
7//
8// I would link to an article from Next.js or React, but their examples
9// are too verbose and not informative to what they actually do.
10const kState = Symbol("SuspenseState");
11
12interface SuspenseProps {
13 children: ssr.Node;
14 fallback?: ssr.Node;
15}
16
17interface State {
18 nested: boolean;
19 nextId: number;
20 completed: number;
21 pushChunk(name: string, node: ssr.ResolvedNode): void;
22}
23
24export function Suspense({ children, fallback }: SuspenseProps): ssr.Node {
25 const state = ssr.getUserData<State>(kState, () => {
26 throw new Error("Can only use <Suspense> with 'renderStreaming'");
27 });
28 if (state.nested) throw new Error("<Suspense> cannot be nested");
29 const parent = ssr.getCurrentRender()!;
30 const r = ssr.initRender(true, { [kState]: { nested: true } });
31 const resolved = ssr.resolveNode(r, children);
32 if (r.async == 0) return ssr.html(resolved);
33 const name = "suspended_" + (++state.nextId);
34 state.nested = true;
35 const ip: [ssr.ResolvedNode] = [
36 [
37 ssr.kElement,
38 "slot",
39 { name },
40 fallback ? ssr.resolveNode(parent, fallback) : "",
41 ],
42 ];
43 state.nested = false;
44 r.asyncDone = () => {
45 const rejections = r.rejections;
46 if (rejections && rejections.length > 0) throw new Error("TODO");
47 state.pushChunk?.(name, ip[0] = resolved);
48 };
49 return ssr.html(ip);
50}
51
52// TODO: add a User-Agent parameter, which is used to determine if a
53// fallback path must be used.
54// - Before ~2024 needs to use a JS implementation.
55// - IE should probably bail out entirely.
56export async function* renderStreaming<
57 T extends ssr.Addons = Record<never, unknown>,
58>(
59 node: ssr.Node,
60 addon: T = {} as T,
61) {
62 const {
63 text: begin,
64 addon: { [kState]: state, ...addonOutput },
65 } = await ssr.ssrAsync(node, {
66 ...addon,
67 [kState]: {
68 nested: false,
69 nextId: 0,
70 completed: 0,
71 pushChunk: () => {},
72 } satisfies State as State,
73 });
74 if (state.nextId === 0) {
75 yield begin;
76 return addonOutput as unknown as T;
77 }
78 let resolve: (() => void) | null = null;
79 let chunks: string[] = [];
80 state.pushChunk = (slot, node) => {
81 while (node.length === 1 && Array.isArray(node)) node = node[0];
82 if (node[0] === ssr.kElement) {
83 (node as ssr.ResolvedElement)[2].slot = slot;
84 } else {
85 node = [ssr.kElement, "clover-suspense", {
86 style: "display:contents",
87 slot,
88 }, node];
89 }
90 chunks.push(ssr.renderNode(node));
91 resolve?.();
92 };
93 yield `<template shadowrootmode=open>${begin}</template>`;
94 do {
95 await new Promise<void>((done) => resolve = done);
96 yield* chunks;
97 chunks = [];
98 } while (state.nextId < state.completed);
99 return addonOutput as unknown as T;
100}
101
102import * as ssr from "./ssr.ts";
framework/esbuild-support.ts+79-79
......@@ -1,79 +1,79 @@
1export function virtualFiles(
2 map: Record<string, string | esbuild.OnLoadResult>,
3) {
4 return {
5 name: "clover vfs",
6 setup(b) {
7 b.onResolve(
8 {
9 filter: new RegExp(
10 `^(?:${
11 Object.keys(map).map((file) => string.escapeRegExp(file)).join(
12 "|",
13 )
14 })\$`,
15 ),
16 },
17 ({ path }) => ({ path, namespace: "vfs" }),
18 );
19 b.onLoad(
20 { filter: /./, namespace: "vfs" },
21 ({ path }) => {
22 const entry = map[path];
23 return ({
24 resolveDir: ".",
25 loader: "ts",
26 ...typeof entry === "string" ? { contents: entry } : entry,
27 });
28 },
29 );
30 },
31 } satisfies esbuild.Plugin;
32}
33
34export function banFiles(
35 files: string[],
36) {
37 return {
38 name: "clover vfs",
39 setup(b) {
40 b.onResolve(
41 {
42 filter: new RegExp(
43 `^(?:${
44 files.map((file) => string.escapeRegExp(file)).join("|")
45 })\$`,
46 ),
47 },
48 ({ path, importer }) => {
49 throw new Error(
50 `Loading ${path} (from ${importer}) is banned!`,
51 );
52 },
53 );
54 },
55 } satisfies esbuild.Plugin;
56}
57
58export function projectRelativeResolution(root = process.cwd() + "/src") {
59 return {
60 name: "project relative resolution ('@/' prefix)",
61 setup(b) {
62 b.onResolve({ filter: /^@\// }, ({ path: id }) => {
63 return {
64 path: path.resolve(root, id.slice(2)),
65 };
66 });
67 b.onResolve({ filter: /^#/ }, ({ path: id, importer }) => {
68 return {
69 path: hot.resolveFrom(importer, id),
70 };
71 });
72 },
73 } satisfies esbuild.Plugin;
74}
75
76import * as esbuild from "esbuild";
77import * as string from "#sitegen/string";
78import * as path from "node:path";
79import * as hot from "./hot.ts";
1export function virtualFiles(
2 map: Record<string, string | esbuild.OnLoadResult>,
3) {
4 return {
5 name: "clover vfs",
6 setup(b) {
7 b.onResolve(
8 {
9 filter: new RegExp(
10 `^(?:${
11 Object.keys(map).map((file) => string.escapeRegExp(file)).join(
12 "|",
13 )
14 })\$`,
15 ),
16 },
17 ({ path }) => ({ path, namespace: "vfs" }),
18 );
19 b.onLoad(
20 { filter: /./, namespace: "vfs" },
21 ({ path }) => {
22 const entry = map[path];
23 return ({
24 resolveDir: ".",
25 loader: "ts",
26 ...typeof entry === "string" ? { contents: entry } : entry,
27 });
28 },
29 );
30 },
31 } satisfies esbuild.Plugin;
32}
33
34export function banFiles(
35 files: string[],
36) {
37 return {
38 name: "clover vfs",
39 setup(b) {
40 b.onResolve(
41 {
42 filter: new RegExp(
43 `^(?:${
44 files.map((file) => string.escapeRegExp(file)).join("|")
45 })\$`,
46 ),
47 },
48 ({ path, importer }) => {
49 throw new Error(
50 `Loading ${path} (from ${importer}) is banned!`,
51 );
52 },
53 );
54 },
55 } satisfies esbuild.Plugin;
56}
57
58export function projectRelativeResolution(root = process.cwd() + "/src") {
59 return {
60 name: "project relative resolution ('@/' prefix)",
61 setup(b) {
62 b.onResolve({ filter: /^@\// }, ({ path: id }) => {
63 return {
64 path: path.resolve(root, id.slice(2)),
65 };
66 });
67 b.onResolve({ filter: /^#/ }, ({ path: id, importer }) => {
68 return {
69 path: hot.resolveFrom(importer, id),
70 };
71 });
72 },
73 } satisfies esbuild.Plugin;
74}
75
76import * as esbuild from "esbuild";
77import * as string from "#sitegen/string";
78import * as path from "node:path";
79import * as hot from "./hot.ts";
framework/hot.ts+3-1
......@@ -96,7 +96,9 @@ Module._resolveFilename = (...args) => {
9696 try {
9797 return require.resolve(replacedPath, { paths: [projectSrc] });
9898 } catch (err: any) {
99 if (err.code === "MODULE_NOT_FOUND" && (err?.requireStack?.length ?? 0) <= 1) {
99 if (
100 err.code === "MODULE_NOT_FOUND" && (err?.requireStack?.length ?? 0) <= 1
101 ) {
100102 err.message.replace(replacedPath, args[0]);
101103 }
102104 }
framework/lib/async.ts+299-297
......@@ -1,297 +1,299 @@
1const five_minutes = 5 * 60 * 1000;
2
3interface QueueOptions<T, R> {
4 name: string;
5 fn: (item: T, spin: Spinner) => Promise<R>;
6 getItemText?: (item: T) => string;
7 maxJobs?: number;
8 passive?: boolean;
9}
10
11// Process multiple items in parallel, queue up as many.
12export class Queue<T, R> {
13 #name: string;
14 #fn: (item: T, spin: Spinner) => Promise<R>;
15 #maxJobs: number;
16 #getItemText: (item: T) => string;
17 #passive: boolean;
18
19 #active: Spinner[] = [];
20 #queue: Array<[T] | [T, (result: R) => void, (err: unknown) => void]> = [];
21
22 #cachedProgress: Progress<{ active: Spinner[] }> | null = null;
23 #done: number = 0;
24 #total: number = 0;
25 #onComplete: (() => void) | null = null;
26 #estimate: number | null = null;
27 #errors: unknown[] = [];
28
29 constructor(options: QueueOptions<T, R>) {
30 this.#name = options.name;
31 this.#fn = options.fn;
32 this.#maxJobs = options.maxJobs ?? 5;
33 this.#getItemText = options.getItemText ?? defaultGetItemText;
34 this.#passive = options.passive ?? false;
35 }
36
37 cancel() {
38 const bar = this.#cachedProgress;
39 bar?.stop();
40 this.#queue = [];
41 }
42
43 get bar() {
44 const cached = this.#cachedProgress;
45 if (!cached) {
46 const bar = this.#cachedProgress = new Progress({
47 spinner: null,
48 text: ({ active }) => {
49 const now = performance.now();
50 let text = `[${this.#done}/${this.#total}] ${this.#name}`;
51 let n = 0;
52 for (const item of active) {
53 let itemText = "- " + item.format(now);
54 text += `\n` +
55 itemText.slice(0, Math.max(0, process.stdout.columns - 1));
56 if (n > 10) {
57 text += `\n ... + ${active.length - n} more`;
58 break;
59 }
60 n++;
61 }
62 return text;
63 },
64 props: {
65 active: [] as Spinner[],
66 },
67 });
68 bar.value = 0;
69 return bar;
70 }
71 return cached;
72 }
73
74 addReturn(args: T) {
75 this.#total += 1;
76 this.updateTotal();
77 if (this.#active.length >= this.#maxJobs) {
78 const { promise, resolve, reject } = Promise.withResolvers<R>();
79 this.#queue.push([args, resolve, reject]);
80 return promise;
81 }
82 return this.#run(args);
83 }
84
85 add(args: T) {
86 return this.addReturn(args).then(() => {}, () => {});
87 }
88
89 addMany(items: T[]) {
90 this.#total += items.length;
91 this.updateTotal();
92
93 const runNowCount = this.#maxJobs - this.#active.length;
94 const runNow = items.slice(0, runNowCount);
95 const runLater = items.slice(runNowCount);
96 this.#queue.push(...runLater.reverse().map<[T]>((x) => [x]));
97 runNow.map((item) => this.#run(item).catch(() => {}));
98 }
99
100 async #run(args: T): Promise<R> {
101 const bar = this.bar;
102 const itemText = this.#getItemText(args);
103 const spinner = new Spinner(itemText);
104 spinner.stop();
105 (spinner as any).redraw = () => (bar as any).redraw();
106 const active = this.#active;
107 try {
108 active.unshift(spinner);
109 bar.props = { active };
110 console.log(this.#name + ": " + itemText);
111 const result = await this.#fn(args, spinner);
112 this.#done++;
113 return result;
114 } catch (err) {
115 if (err && typeof err === "object") {
116 (err as any).job = itemText;
117 }
118 this.#errors.push(err);
119 throw err;
120 } finally {
121 active.splice(active.indexOf(spinner), 1);
122 bar.props = { active };
123 bar.value = this.#done;
124
125 // Process next item
126 const next = this.#queue.shift();
127 if (next) {
128 const args = next[0];
129 this.#run(args)
130 .then((result) => next[1]?.(result))
131 .catch((err) => next[2]?.(err));
132 } else if (this.#active.length === 0) {
133 if (this.#passive) {
134 this.bar.stop();
135 this.#cachedProgress = null;
136 }
137 this.#onComplete?.();
138 }
139 }
140 }
141
142 updateTotal() {
143 const bar = this.bar;
144 bar.total = Math.max(this.#total, this.#estimate ?? 0);
145 }
146
147 set estimate(e: number) {
148 this.#estimate = e;
149 if (this.#cachedProgress) {
150 this.updateTotal();
151 }
152 }
153
154 async done(o?: { method: "success" | "stop" }) {
155 if (this.#active.length === 0) {
156 this.#end(o);
157 return;
158 }
159
160 const { promise, resolve } = Promise.withResolvers<void>();
161 this.#onComplete = resolve;
162 await promise;
163 this.#end(o);
164 }
165
166 #end(
167 { method = this.#passive ? "stop" : "success" }: {
168 method?: "success" | "stop";
169 } = {},
170 ) {
171 const bar = this.#cachedProgress;
172 if (this.#errors.length > 0) {
173 if (bar) bar.stop();
174 throw new AggregateError(
175 this.#errors,
176 this.#errors.length + " jobs failed in '" + this.#name + "'",
177 );
178 }
179
180 if (bar) bar[method]();
181 }
182
183 get active(): boolean {
184 return this.#active.length !== 0;
185 }
186
187 [Symbol.dispose]() {
188 if (this.active) {
189 this.cancel();
190 }
191 }
192}
193
194const cwd = process.cwd();
195function defaultGetItemText(item: unknown) {
196 let itemText = "";
197 if (typeof item === "string") {
198 itemText = item;
199 } else if (typeof item === "object" && item !== null) {
200 const { path, label, id } = item as any;
201 itemText = label ?? path ?? id ?? JSON.stringify(item);
202 } else {
203 itemText = JSON.stringify(item);
204 }
205
206 if (itemText.startsWith(cwd)) {
207 itemText = path.relative(cwd, itemText);
208 }
209 return itemText;
210}
211
212export class OnceMap<T> {
213 private ongoing = new Map<string, Promise<T>>();
214
215 get(key: string, compute: () => Promise<T>) {
216 if (this.ongoing.has(key)) {
217 return this.ongoing.get(key)!;
218 }
219
220 const result = compute();
221 this.ongoing.set(key, result);
222 return result;
223 }
224}
225
226interface ARCEValue<T> {
227 value: T;
228 [Symbol.dispose]: () => void;
229}
230
231export function RefCountedExpirable<T>(
232 init: () => Promise<T>,
233 deinit: (value: T) => void,
234 expire: number = five_minutes,
235): () => Promise<ARCEValue<T>> {
236 let refs = 0;
237 let item: ARCEValue<T> | null = null;
238 let loading: Promise<ARCEValue<T>> | null = null;
239 let timer: ReturnType<typeof setTimeout> | null = null;
240
241 function deref() {
242 ASSERT(item !== null);
243 if (--refs !== 0) return;
244 ASSERT(timer === null);
245 timer = setTimeout(() => {
246 ASSERT(refs === 0);
247 ASSERT(loading === null);
248 ASSERT(item !== null);
249 deinit(item.value);
250 item = null;
251 timer = null;
252 }, expire);
253 }
254
255 return async function () {
256 if (timer !== null) {
257 clearTimeout(timer);
258 timer = null;
259 }
260 if (item !== null) {
261 refs++;
262 return item;
263 }
264 if (loading !== null) {
265 refs++;
266 return loading;
267 }
268 const p = Promise.withResolvers<ARCEValue<T>>();
269 loading = p.promise;
270 try {
271 const value = await init();
272 item = { value, [Symbol.dispose]: deref };
273 refs++;
274 p.resolve(item);
275 return item;
276 } catch (e) {
277 p.reject(e);
278 throw e;
279 } finally {
280 loading = null;
281 }
282 };
283}
284
285export function once<T>(fn: () => Promise<T>): () => Promise<T> {
286 let result: T | Promise<T> | null = null;
287 return async () => {
288 if (result) return result;
289 result = await fn();
290 return result;
291 };
292}
293
294import { Progress } from "@paperclover/console/Progress";
295import { Spinner } from "@paperclover/console/Spinner";
296import * as path from "node:path";
297import process from "node:process";
1const five_minutes = 5 * 60 * 1000;
2
3interface QueueOptions<T, R> {
4 name: string;
5 fn: (item: T, spin: Spinner) => Promise<R>;
6 getItemText?: (item: T) => string;
7 maxJobs?: number;
8 passive?: boolean;
9}
10
11// Process multiple items in parallel, queue up as many.
12export class Queue<T, R> {
13 #name: string;
14 #fn: (item: T, spin: Spinner) => Promise<R>;
15 #maxJobs: number;
16 #getItemText: (item: T) => string;
17 #passive: boolean;
18
19 #active: Spinner[] = [];
20 #queue: Array<[T] | [T, (result: R) => void, (err: unknown) => void]> = [];
21
22 #cachedProgress: Progress<{ active: Spinner[] }> | null = null;
23 #done: number = 0;
24 #total: number = 0;
25 #onComplete: (() => void) | null = null;
26 #estimate: number | null = null;
27 #errors: unknown[] = [];
28
29 constructor(options: QueueOptions<T, R>) {
30 this.#name = options.name;
31 this.#fn = options.fn;
32 this.#maxJobs = options.maxJobs ?? 5;
33 this.#getItemText = options.getItemText ?? defaultGetItemText;
34 this.#passive = options.passive ?? false;
35 }
36
37 cancel() {
38 const bar = this.#cachedProgress;
39 bar?.stop();
40 this.#queue = [];
41 }
42
43 get bar() {
44 const cached = this.#cachedProgress;
45 if (!cached) {
46 const bar = this.#cachedProgress = new Progress({
47 spinner: null,
48 text: ({ active }) => {
49 const now = performance.now();
50 let text = `[${this.#done}/${this.#total}] ${this.#name}`;
51 let n = 0;
52 for (const item of active) {
53 let itemText = "- " + item.format(now);
54 text += `\n` +
55 itemText.slice(0, Math.max(0, process.stdout.columns - 1));
56 if (n > 10) {
57 text += `\n ... + ${active.length - n} more`;
58 break;
59 }
60 n++;
61 }
62 return text;
63 },
64 props: {
65 active: [] as Spinner[],
66 },
67 });
68 bar.value = 0;
69 return bar;
70 }
71 return cached;
72 }
73
74 addReturn(args: T) {
75 this.#total += 1;
76 this.updateTotal();
77 if (this.#active.length >= this.#maxJobs) {
78 const { promise, resolve, reject } = Promise.withResolvers<R>();
79 this.#queue.push([args, resolve, reject]);
80 return promise;
81 }
82 return this.#run(args);
83 }
84
85 add(args: T) {
86 return this.addReturn(args).then(() => {}, () => {});
87 }
88
89 addMany(items: T[]) {
90 this.#total += items.length;
91 this.updateTotal();
92
93 const runNowCount = this.#maxJobs - this.#active.length;
94 const runNow = items.slice(0, runNowCount);
95 const runLater = items.slice(runNowCount);
96 this.#queue.push(...runLater.reverse().map<[T]>((x) => [x]));
97 runNow.map((item) => this.#run(item).catch(() => {}));
98 }
99
100 async #run(args: T): Promise<R> {
101 const bar = this.bar;
102 const itemText = this.#getItemText(args);
103 const spinner = new Spinner(itemText);
104 spinner.stop();
105 (spinner as any).redraw = () => (bar as any).redraw();
106 const active = this.#active;
107 try {
108 active.unshift(spinner);
109 bar.props = { active };
110 // console.log(this.#name + ": " + itemText);
111 const result = await this.#fn(args, spinner);
112 this.#done++;
113 return result;
114 } catch (err) {
115 if (err && typeof err === "object") {
116 (err as any).job = itemText;
117 }
118 this.#errors.push(err);
119 console.error(util.inspect(err, false, Infinity, true));
120 throw err;
121 } finally {
122 active.splice(active.indexOf(spinner), 1);
123 bar.props = { active };
124 bar.value = this.#done;
125
126 // Process next item
127 const next = this.#queue.shift();
128 if (next) {
129 const args = next[0];
130 this.#run(args)
131 .then((result) => next[1]?.(result))
132 .catch((err) => next[2]?.(err));
133 } else if (this.#active.length === 0) {
134 if (this.#passive) {
135 this.bar.stop();
136 this.#cachedProgress = null;
137 }
138 this.#onComplete?.();
139 }
140 }
141 }
142
143 updateTotal() {
144 const bar = this.bar;
145 bar.total = Math.max(this.#total, this.#estimate ?? 0);
146 }
147
148 set estimate(e: number) {
149 this.#estimate = e;
150 if (this.#cachedProgress) {
151 this.updateTotal();
152 }
153 }
154
155 async done(o?: { method: "success" | "stop" }) {
156 if (this.#active.length === 0) {
157 this.#end(o);
158 return;
159 }
160
161 const { promise, resolve } = Promise.withResolvers<void>();
162 this.#onComplete = resolve;
163 await promise;
164 this.#end(o);
165 }
166
167 #end(
168 { method = this.#passive ? "stop" : "success" }: {
169 method?: "success" | "stop";
170 } = {},
171 ) {
172 const bar = this.#cachedProgress;
173 if (this.#errors.length > 0) {
174 if (bar) bar.stop();
175 throw new AggregateError(
176 this.#errors,
177 this.#errors.length + " jobs failed in '" + this.#name + "'",
178 );
179 }
180
181 if (bar) bar[method]();
182 }
183
184 get active(): boolean {
185 return this.#active.length !== 0;
186 }
187
188 [Symbol.dispose]() {
189 if (this.active) {
190 this.cancel();
191 }
192 }
193}
194
195const cwd = process.cwd();
196function defaultGetItemText(item: unknown) {
197 let itemText = "";
198 if (typeof item === "string") {
199 itemText = item;
200 } else if (typeof item === "object" && item !== null) {
201 const { path, label, id } = item as any;
202 itemText = label ?? path ?? id ?? JSON.stringify(item);
203 } else {
204 itemText = JSON.stringify(item);
205 }
206
207 if (itemText.startsWith(cwd)) {
208 itemText = path.relative(cwd, itemText);
209 }
210 return itemText;
211}
212
213export class OnceMap<T> {
214 private ongoing = new Map<string, Promise<T>>();
215
216 get(key: string, compute: () => Promise<T>) {
217 if (this.ongoing.has(key)) {
218 return this.ongoing.get(key)!;
219 }
220
221 const result = compute();
222 this.ongoing.set(key, result);
223 return result;
224 }
225}
226
227interface ARCEValue<T> {
228 value: T;
229 [Symbol.dispose]: () => void;
230}
231
232export function RefCountedExpirable<T>(
233 init: () => Promise<T>,
234 deinit: (value: T) => void,
235 expire: number = five_minutes,
236): () => Promise<ARCEValue<T>> {
237 let refs = 0;
238 let item: ARCEValue<T> | null = null;
239 let loading: Promise<ARCEValue<T>> | null = null;
240 let timer: ReturnType<typeof setTimeout> | null = null;
241
242 function deref() {
243 ASSERT(item !== null);
244 if (--refs !== 0) return;
245 ASSERT(timer === null);
246 timer = setTimeout(() => {
247 ASSERT(refs === 0);
248 ASSERT(loading === null);
249 ASSERT(item !== null);
250 deinit(item.value);
251 item = null;
252 timer = null;
253 }, expire);
254 }
255
256 return async function () {
257 if (timer !== null) {
258 clearTimeout(timer);
259 timer = null;
260 }
261 if (item !== null) {
262 refs++;
263 return item;
264 }
265 if (loading !== null) {
266 refs++;
267 return loading;
268 }
269 const p = Promise.withResolvers<ARCEValue<T>>();
270 loading = p.promise;
271 try {
272 const value = await init();
273 item = { value, [Symbol.dispose]: deref };
274 refs++;
275 p.resolve(item);
276 return item;
277 } catch (e) {
278 p.reject(e);
279 throw e;
280 } finally {
281 loading = null;
282 }
283 };
284}
285
286export function once<T>(fn: () => Promise<T>): () => Promise<T> {
287 let result: T | Promise<T> | null = null;
288 return async () => {
289 if (result) return result;
290 result = await fn();
291 return result;
292 };
293}
294
295import { Progress } from "@paperclover/console/Progress";
296import { Spinner } from "@paperclover/console/Spinner";
297import * as path from "node:path";
298import process from "node:process";
299import * as util from "node:util";
framework/lib/meta.ts+24-24
......@@ -1,24 +1,24 @@
1export interface Meta {
2 title: string;
3 description?: string | undefined;
4 openGraph?: OpenGraph;
5 alternates?: Alternates;
6}
7export interface OpenGraph {
8 title?: string;
9 description?: string | undefined;
10 type: string;
11 url: string;
12}
13export interface Alternates {
14 canonical: string;
15 types: { [mime: string]: AlternateType };
16}
17export interface AlternateType {
18 url: string;
19 title: string;
20}
21export function renderMeta({ title }: Meta): string {
22 return `<title>${esc(title)}</title>`;
23}
24import { escapeHtml as esc } from "../engine/ssr.ts";
1export interface Meta {
2 title: string;
3 description?: string | undefined;
4 openGraph?: OpenGraph;
5 alternates?: Alternates;
6}
7export interface OpenGraph {
8 title?: string;
9 description?: string | undefined;
10 type: string;
11 url: string;
12}
13export interface Alternates {
14 canonical: string;
15 types: { [mime: string]: AlternateType };
16}
17export interface AlternateType {
18 url: string;
19 title: string;
20}
21export function renderMeta({ title }: Meta): string {
22 return `<title>${esc(title)}</title>`;
23}
24import { escapeHtml as esc } from "../engine/ssr.ts";
framework/lib/string.ts+3-3
......@@ -1,3 +1,3 @@
1export function escapeRegExp(source: string) {
2 return source.replace(/[\$\\]/g, "\\$&");
3}
1export function escapeRegExp(source: string) {
2 return source.replace(/[\$\\]/g, "\\$&");
3}
framework/lib/view.ts+100-100
......@@ -1,100 +1,100 @@
1// This import is generated by code 'bundle.ts'
2export interface View {
3 component: engine.Component;
4 meta:
5 | meta.Meta
6 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
7 layout?: engine.Component;
8 inlineCss: string;
9 scripts: Record<string, string>;
10}
11
12let views: Record<string, View> = null!;
13let scripts: Record<string, string> = null!;
14
15// An older version of the Clover Engine supported streaming suspense
16// boundaries, but those were never used. Pages will wait until they
17// are fully rendered before sending.
18export async function renderView(
19 context: hono.Context,
20 id: string,
21 props: Record<string, unknown>,
22) {
23 return context.html(await renderViewToString(id, { context, ...props }));
24}
25
26export async function renderViewToString(
27 id: string,
28 props: Record<string, unknown>,
29) {
30 views ?? ({ views, scripts } = require("$views"));
31 // The view contains pre-bundled CSS and scripts, but keeps the scripts
32 // separate for run-time dynamic scripts. For example, the file viewer
33 // includes the canvas for the current page, but only the current page.
34 const {
35 component,
36 inlineCss,
37 layout,
38 meta: metadata,
39 }: View = UNWRAP(views[id], `Missing view ${id}`);
40
41 // -- metadata --
42 const renderedMetaPromise = Promise.resolve(
43 typeof metadata === "function" ? metadata(props) : metadata,
44 ).then((m) => meta.renderMeta(m));
45
46 // -- html --
47 let page: engine.Element = [engine.kElement, component, props];
48 if (layout) page = [engine.kElement, layout, { children: page }];
49 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
50 sitegen: sg.initRender(),
51 });
52
53 // -- join document and send --
54 return wrapDocument({
55 body,
56 head: await renderedMetaPromise,
57 inlineCss,
58 scripts: joinScripts(
59 Array.from(
60 sitegen.scripts,
61 (id) => UNWRAP(scripts[id], `Missing script ${id}`),
62 ),
63 ),
64 });
65}
66
67export function provideViewData(v: typeof views, s: typeof scripts) {
68 views = v;
69 scripts = s;
70}
71
72export function joinScripts(scriptSources: string[]) {
73 const { length } = scriptSources;
74 if (length === 0) return "";
75 if (length === 1) return scriptSources[0];
76 return scriptSources.map((source) => `{${source}}`).join(";");
77}
78
79export function wrapDocument({
80 body,
81 head,
82 inlineCss,
83 scripts,
84}: {
85 head: string;
86 body: string;
87 inlineCss: string;
88 scripts: string;
89}) {
90 return `<!doctype html><html lang=en><head>${head}${
91 inlineCss ? `<style>${inlineCss}</style>` : ""
92 }</head><body>${body}${
93 scripts ? `<script>${scripts}</script>` : ""
94 }</body></html>`;
95}
96
97import * as meta from "./meta.ts";
98import type * as hono from "#hono";
99import * as engine from "../engine/ssr.ts";
100import * as sg from "./sitegen.ts";
1// This import is generated by code 'bundle.ts'
2export interface View {
3 component: engine.Component;
4 meta:
5 | meta.Meta
6 | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
7 layout?: engine.Component;
8 inlineCss: string;
9 scripts: Record<string, string>;
10}
11
12let views: Record<string, View> = null!;
13let scripts: Record<string, string> = null!;
14
15// An older version of the Clover Engine supported streaming suspense
16// boundaries, but those were never used. Pages will wait until they
17// are fully rendered before sending.
18export async function renderView(
19 context: hono.Context,
20 id: string,
21 props: Record<string, unknown>,
22) {
23 return context.html(await renderViewToString(id, { context, ...props }));
24}
25
26export async function renderViewToString(
27 id: string,
28 props: Record<string, unknown>,
29) {
30 views ?? ({ views, scripts } = require("$views"));
31 // The view contains pre-bundled CSS and scripts, but keeps the scripts
32 // separate for run-time dynamic scripts. For example, the file viewer
33 // includes the canvas for the current page, but only the current page.
34 const {
35 component,
36 inlineCss,
37 layout,
38 meta: metadata,
39 }: View = UNWRAP(views[id], `Missing view ${id}`);
40
41 // -- metadata --
42 const renderedMetaPromise = Promise.resolve(
43 typeof metadata === "function" ? metadata(props) : metadata,
44 ).then((m) => meta.renderMeta(m));
45
46 // -- html --
47 let page: engine.Element = [engine.kElement, component, props];
48 if (layout) page = [engine.kElement, layout, { children: page }];
49 const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
50 sitegen: sg.initRender(),
51 });
52
53 // -- join document and send --
54 return wrapDocument({
55 body,
56 head: await renderedMetaPromise,
57 inlineCss,
58 scripts: joinScripts(
59 Array.from(
60 sitegen.scripts,
61 (id) => UNWRAP(scripts[id], `Missing script ${id}`),
62 ),
63 ),
64 });
65}
66
67export function provideViewData(v: typeof views, s: typeof scripts) {
68 views = v;
69 scripts = s;
70}
71
72export function joinScripts(scriptSources: string[]) {
73 const { length } = scriptSources;
74 if (length === 0) return "";
75 if (length === 1) return scriptSources[0];
76 return scriptSources.map((source) => `{${source}}`).join(";");
77}
78
79export function wrapDocument({
80 body,
81 head,
82 inlineCss,
83 scripts,
84}: {
85 head: string;
86 body: string;
87 inlineCss: string;
88 scripts: string;
89}) {
90 return `<!doctype html><html lang=en><head>${head}${
91 inlineCss ? `<style>${inlineCss}</style>` : ""
92 }</head><body>${body}${
93 scripts ? `<script>${scripts}</script>` : ""
94 }</body></html>`;
95}
96
97import * as meta from "./meta.ts";
98import type * as hono from "#hono";
99import * as engine from "../engine/ssr.ts";
100import * as sg from "./sitegen.ts";
framework/watch.ts+198-198
......@@ -1,198 +1,198 @@
1// File watcher and live reloading site generator
2
3const debounceMilliseconds = 25;
4
5export async function main() {
6 let subprocess: child_process.ChildProcess | null = null;
7
8 // Catch up state by running a main build.
9 const { incr } = await generate.main();
10 // ...and watch the files that cause invals.
11 const watch = new Watch(rebuild);
12 watch.add(...incr.invals.keys());
13 statusLine();
14 // ... and then serve it!
15 serve();
16
17 function serve() {
18 if (subprocess) {
19 subprocess.removeListener("close", onSubprocessClose);
20 subprocess.kill();
21 }
22 subprocess = child_process.fork(".clover/out/server.js", [
23 "--development",
24 ], {
25 stdio: "inherit",
26 });
27 subprocess.on("close", onSubprocessClose);
28 }
29
30 function onSubprocessClose(code: number | null, signal: string | null) {
31 subprocess = null;
32 const status = code != null ? `code ${code}` : `signal ${signal}`;
33 console.error(`Backend process exited with ${status}`);
34 }
35
36 process.on("beforeExit", () => {
37 subprocess?.removeListener("close", onSubprocessClose);
38 });
39
40 function rebuild(files: string[]) {
41 files = files.map((file) => path.relative(hot.projectRoot, file));
42 const changed: string[] = [];
43 for (const file of files) {
44 let mtimeMs: number | null = null;
45 try {
46 mtimeMs = fs.statSync(file).mtimeMs;
47 } catch (err: any) {
48 if (err?.code !== "ENOENT") throw err;
49 }
50 if (incr.updateStat(file, mtimeMs)) changed.push(file);
51 }
52 if (changed.length === 0) {
53 console.warn("Files were modified but the 'modify' time did not change.");
54 return;
55 }
56 withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
57 text: "Rebuilding",
58 successText: generate.successText,
59 failureText: () => "sitegen FAIL",
60 }, async (spinner) => {
61 console.info("---");
62 console.info(
63 "Updated" +
64 (changed.length === 1
65 ? " " + changed[0]
66 : changed.map((file) => "\n- " + file)),
67 );
68 const result = await generate.sitegen(spinner, incr);
69 incr.toDisk(); // Allows picking up this state again
70 for (const file of watch.files) {
71 const relative = path.relative(hot.projectRoot, file);
72 if (!incr.invals.has(relative)) watch.remove(file);
73 }
74 return result;
75 }).then((result) => {
76 // Restart the server if it was changed or not running.
77 if (
78 !subprocess ||
79 result.inserted.some(({ kind }) => kind === "backendReplace")
80 ) {
81 serve();
82 } else if (
83 subprocess &&
84 result.inserted.some(({ kind }) => kind === "asset")
85 ) {
86 subprocess.send({ type: "clover.assets.reload" });
87 }
88 return result;
89 }).catch((err) => {
90 console.error(util.inspect(err));
91 }).finally(statusLine);
92 }
93
94 function statusLine() {
95 console.info(
96 `Watching ${incr.invals.size} files \x1b[36m[last change: ${
97 new Date().toLocaleTimeString()
98 }]\x1b[39m`,
99 );
100 }
101}
102
103class Watch {
104 files = new Set<string>();
105 stale = new Set<string>();
106 onChange: (files: string[]) => void;
107 watchers: fs.FSWatcher[] = [];
108 /** Has a trailing slash */
109 roots: string[] = [];
110 debounce: ReturnType<typeof setTimeout> | null = null;
111
112 constructor(onChange: Watch["onChange"]) {
113 this.onChange = onChange;
114 }
115
116 add(...files: string[]) {
117 const { roots, watchers } = this;
118 let newRoots: string[] = [];
119 for (let file of files) {
120 file = path.resolve(file);
121 if (this.files.has(file)) continue;
122 this.files.add(file);
123 // Find an existing watcher
124 if (roots.some((root) => file.startsWith(root))) continue;
125 if (newRoots.some((root) => file.startsWith(root))) continue;
126 newRoots.push(path.dirname(file) + path.sep);
127 }
128 if (newRoots.length === 0) return;
129 // Filter out directories that are already specified
130 newRoots = newRoots
131 .sort((a, b) => a.length - b.length)
132 .filter((dir, i, a) => {
133 for (let j = 0; j < i; j++) if (dir.startsWith(a[j])) return false;
134 return true;
135 });
136 // Append Watches
137 let i = roots.length;
138 for (const root of newRoots) {
139 this.watchers.push(fs.watch(
140 root,
141 { recursive: true, encoding: "utf-8" },
142 this.#handleEvent.bind(this, root),
143 ));
144 this.roots.push(root);
145 }
146 // If any new roots shadow over and old one, delete it!
147 while (i > 0) {
148 i -= 1;
149 const root = roots[i];
150 if (newRoots.some((newRoot) => root.startsWith(newRoot))) {
151 watchers.splice(i, 1)[0].close();
152 roots.splice(i, 1);
153 }
154 }
155 }
156
157 remove(...files: string[]) {
158 for (const file of files) this.files.delete(path.resolve(file));
159 // Find watches that are covering no files
160 const { roots, watchers } = this;
161 const existingFiles = Array.from(this.files);
162 let i = roots.length;
163 while (i > 0) {
164 i -= 1;
165 const root = roots[i];
166 if (!existingFiles.some((file) => file.startsWith(root))) {
167 watchers.splice(i, 1)[0].close();
168 roots.splice(i, 1);
169 }
170 }
171 }
172
173 stop() {
174 for (const w of this.watchers) w.close();
175 }
176
177 #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) {
178 if (!subPath) return;
179 const file = path.join(root, subPath);
180 if (!this.files.has(file)) return;
181 this.stale.add(file);
182 const { debounce } = this;
183 if (debounce !== null) clearTimeout(debounce);
184 this.debounce = setTimeout(() => {
185 this.debounce = null;
186 this.onChange(Array.from(this.stale));
187 this.stale.clear();
188 }, debounceMilliseconds);
189 }
190}
191
192import * as fs from "node:fs";
193import { withSpinner } from "@paperclover/console/Spinner";
194import * as generate from "./generate.ts";
195import * as path from "node:path";
196import * as util from "node:util";
197import * as hot from "./hot.ts";
198import * as child_process from "node:child_process";
1// File watcher and live reloading site generator
2
3const debounceMilliseconds = 25;
4
5export async function main() {
6 let subprocess: child_process.ChildProcess | null = null;
7
8 // Catch up state by running a main build.
9 const { incr } = await generate.main();
10 // ...and watch the files that cause invals.
11 const watch = new Watch(rebuild);
12 watch.add(...incr.invals.keys());
13 statusLine();
14 // ... and then serve it!
15 serve();
16
17 function serve() {
18 if (subprocess) {
19 subprocess.removeListener("close", onSubprocessClose);
20 subprocess.kill();
21 }
22 subprocess = child_process.fork(".clover/out/server.js", [
23 "--development",
24 ], {
25 stdio: "inherit",
26 });
27 subprocess.on("close", onSubprocessClose);
28 }
29
30 function onSubprocessClose(code: number | null, signal: string | null) {
31 subprocess = null;
32 const status = code != null ? `code ${code}` : `signal ${signal}`;
33 console.error(`Backend process exited with ${status}`);
34 }
35
36 process.on("beforeExit", () => {
37 subprocess?.removeListener("close", onSubprocessClose);
38 });
39
40 function rebuild(files: string[]) {
41 files = files.map((file) => path.relative(hot.projectRoot, file));
42 const changed: string[] = [];
43 for (const file of files) {
44 let mtimeMs: number | null = null;
45 try {
46 mtimeMs = fs.statSync(file).mtimeMs;
47 } catch (err: any) {
48 if (err?.code !== "ENOENT") throw err;
49 }
50 if (incr.updateStat(file, mtimeMs)) changed.push(file);
51 }
52 if (changed.length === 0) {
53 console.warn("Files were modified but the 'modify' time did not change.");
54 return;
55 }
56 withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
57 text: "Rebuilding",
58 successText: generate.successText,
59 failureText: () => "sitegen FAIL",
60 }, async (spinner) => {
61 console.info("---");
62 console.info(
63 "Updated" +
64 (changed.length === 1
65 ? " " + changed[0]
66 : changed.map((file) => "\n- " + file)),
67 );
68 const result = await generate.sitegen(spinner, incr);
69 incr.toDisk(); // Allows picking up this state again
70 for (const file of watch.files) {
71 const relative = path.relative(hot.projectRoot, file);
72 if (!incr.invals.has(relative)) watch.remove(file);
73 }
74 return result;
75 }).then((result) => {
76 // Restart the server if it was changed or not running.
77 if (
78 !subprocess ||
79 result.inserted.some(({ kind }) => kind === "backendReplace")
80 ) {
81 serve();
82 } else if (
83 subprocess &&
84 result.inserted.some(({ kind }) => kind === "asset")
85 ) {
86 subprocess.send({ type: "clover.assets.reload" });
87 }
88 return result;
89 }).catch((err) => {
90 console.error(util.inspect(err));
91 }).finally(statusLine);
92 }
93
94 function statusLine() {
95 console.info(
96 `Watching ${incr.invals.size} files \x1b[36m[last change: ${
97 new Date().toLocaleTimeString()
98 }]\x1b[39m`,
99 );
100 }
101}
102
103class Watch {
104 files = new Set<string>();
105 stale = new Set<string>();
106 onChange: (files: string[]) => void;
107 watchers: fs.FSWatcher[] = [];
108 /** Has a trailing slash */
109 roots: string[] = [];
110 debounce: ReturnType<typeof setTimeout> | null = null;
111
112 constructor(onChange: Watch["onChange"]) {
113 this.onChange = onChange;
114 }
115
116 add(...files: string[]) {
117 const { roots, watchers } = this;
118 let newRoots: string[] = [];
119 for (let file of files) {
120 file = path.resolve(file);
121 if (this.files.has(file)) continue;
122 this.files.add(file);
123 // Find an existing watcher
124 if (roots.some((root) => file.startsWith(root))) continue;
125 if (newRoots.some((root) => file.startsWith(root))) continue;
126 newRoots.push(path.dirname(file) + path.sep);
127 }
128 if (newRoots.length === 0) return;
129 // Filter out directories that are already specified
130 newRoots = newRoots
131 .sort((a, b) => a.length - b.length)
132 .filter((dir, i, a) => {
133 for (let j = 0; j < i; j++) if (dir.startsWith(a[j])) return false;
134 return true;
135 });
136 // Append Watches
137 let i = roots.length;
138 for (const root of newRoots) {
139 this.watchers.push(fs.watch(
140 root,
141 { recursive: true, encoding: "utf-8" },
142 this.#handleEvent.bind(this, root),
143 ));
144 this.roots.push(root);
145 }
146 // If any new roots shadow over and old one, delete it!
147 while (i > 0) {
148 i -= 1;
149 const root = roots[i];
150 if (newRoots.some((newRoot) => root.startsWith(newRoot))) {
151 watchers.splice(i, 1)[0].close();
152 roots.splice(i, 1);
153 }
154 }
155 }
156
157 remove(...files: string[]) {
158 for (const file of files) this.files.delete(path.resolve(file));
159 // Find watches that are covering no files
160 const { roots, watchers } = this;
161 const existingFiles = Array.from(this.files);
162 let i = roots.length;
163 while (i > 0) {
164 i -= 1;
165 const root = roots[i];
166 if (!existingFiles.some((file) => file.startsWith(root))) {
167 watchers.splice(i, 1)[0].close();
168 roots.splice(i, 1);
169 }
170 }
171 }
172
173 stop() {
174 for (const w of this.watchers) w.close();
175 }
176
177 #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) {
178 if (!subPath) return;
179 const file = path.join(root, subPath);
180 if (!this.files.has(file)) return;
181 this.stale.add(file);
182 const { debounce } = this;
183 if (debounce !== null) clearTimeout(debounce);
184 this.debounce = setTimeout(() => {
185 this.debounce = null;
186 this.onChange(Array.from(this.stale));
187 this.stale.clear();
188 }, debounceMilliseconds);
189 }
190}
191
192import * as fs from "node:fs";
193import { withSpinner } from "@paperclover/console/Spinner";
194import * as generate from "./generate.ts";
195import * as path from "node:path";
196import * as util from "node:util";
197import * as hot from "./hot.ts";
198import * as child_process from "node:child_process";
readme.md+20-20
......@@ -4,27 +4,27 @@ this repository contains clover's "sitegen" framework, which is a set of tools
44that assist building websites. these tools power https://paperclover.net.
55
66- **HTML "Server Side Rendering") engine written from scratch.** (~500 lines)
7 - A more practical JSX runtime (`class` instead of `className`, built-in
8 `clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc).
9 - Integration with [Marko][1] for concisely written components.
10 - TODO: MDX-like compiler for content-heavy pages like blogs.
11 - Different languages can be used at the same time. Supports
12 `async function` components, `<Suspense />`, and custom extensions.
7 - A more practical JSX runtime (`class` instead of `className`, built-in
8 `clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc).
9 - Integration with [Marko][1] for concisely written components.
10 - TODO: MDX-like compiler for content-heavy pages like blogs.
11 - Different languages can be used at the same time. Supports `async function`
12 components, `<Suspense />`, and custom extensions.
1313- **Incremental static site generator and build system.**
14 - Build entire production site at start, incremental updates when pages
15 change; Build system state survives coding sessions.
16 - The only difference in development and production mode is hidden
17 source-maps and stripped `console.debug` calls. The site you
18 see locally is the same site you see deployed.
19 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
20 checks when the files change. For example, changing a component re-tests
21 only pages that use that component and re-lints only the changed file.
14 - Build entire production site at start, incremental updates when pages
15 change; Build system state survives coding sessions.
16 - The only difference in development and production mode is hidden source-maps
17 and stripped `console.debug` calls. The site you see locally is the same
18 site you see deployed.
19 - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
20 checks when the files change. For example, changing a component re-tests
21 only pages that use that component and re-lints only the changed file.
2222- **Integrated libraries for building complex, content heavy web sites.**
23 - Static asset serving with ETag and build-time compression.
24 - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)
25 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
26 - TODO: Meta and Open Graph generation. (`export const meta`)
27 - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
23 - Static asset serving with ETag and build-time compression.
24 - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)
25 - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
26 - TODO: Meta and Open Graph generation. (`export const meta`)
27 - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
2828- **Built on the battle-tested Node.js runtime.**
2929
3030[1]: https://next.markojs.com
......@@ -42,6 +42,7 @@ Included is `src`, which contains `paperclover.net`. Website highlights:
4242## Development
4343
4444minimum system requirements:
45
4546- a cpu with at least 1 core.
4647- random access memory.
4748- windows 7 or later, macos, or other operating system.
......@@ -73,4 +74,3 @@ open a shell with all needed system dependencies.
7374## Contributions
7475
7576No contributions to `src` accepted, only `framework`.
76
run.js+1-1
......@@ -12,7 +12,7 @@ if (!zlib.zstdCompress) {
1212 : null;
1313
1414 globalThis.console.error(
15 `sitegen depends on a node.js-compatibile runtime that supports zstd compression\n` +
15 `sitegen depends on a node.js-compatibile runtime\n` +
1616 `this is node.js version ${process.version}${
1717 brand ? ` (${brand})` : ""
1818 }\n\n` +
src/admin.ts+75-75
......@@ -1,75 +1,75 @@
1const cookieAge = 60 * 60 * 24 * 365; // 1 year
2
3let lastKnownToken: string | null = null;
4function compareToken(token: string) {
5 if (token === lastKnownToken) return true;
6 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
7 return token === lastKnownToken;
8}
9
10export async function middleware(c: Context, next: Next) {
11 if (c.req.path.startsWith("/admin")) {
12 return adminInner(c, next);
13 }
14 return next();
15}
16
17export function adminInner(c: Context, next: Next) {
18 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
19
20 if (c.req.path === "/admin/login") {
21 const key = c.req.query("key");
22 if (key) {
23 if (compareToken(key)) {
24 return c.body(null, 303, {
25 "Location": "/admin",
26 "Set-Cookie":
27 `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
28 });
29 }
30 return serveAsset(c, "/admin/login/fail", 403);
31 }
32 if (token && compareToken(token)) {
33 return c.redirect("/admin", 303);
34 }
35 if (c.req.method === "POST") {
36 return serveAsset(c, "/admin/login/fail", 403);
37 } else {
38 return serveAsset(c, "/admin/login", 200);
39 }
40 }
41
42 if (c.req.path === "/admin/logout") {
43 return c.body(null, 303, {
44 "Location": "/admin/login",
45 "Set-Cookie":
46 `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`,
47 });
48 }
49
50 if (token && compareToken(token)) {
51 return next();
52 }
53
54 return c.redirect("/admin/login", 303);
55}
56
57export function hasAdminToken(c: Context) {
58 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
59 return token && compareToken(token);
60}
61
62export async function main() {
63 const key = crypto.randomUUID();
64 await fs.writeMkdir(".clover/admin-token.txt", key);
65 const start = ({
66 win32: "start",
67 darwin: "open",
68 } as Record<string, string>)[process.platform] ?? "xdg-open";
69 child_process.exec(`${start} http://[::1]:3000/admin/login?key=${key}`);
70}
71
72import * as fs from "#sitegen/fs";
73import type { Context, Next } from "hono";
74import { serveAsset } from "#sitegen/assets";
75import * as child_process from "node:child_process";
1const cookieAge = 60 * 60 * 24 * 365; // 1 year
2
3let lastKnownToken: string | null = null;
4function compareToken(token: string) {
5 if (token === lastKnownToken) return true;
6 lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
7 return token === lastKnownToken;
8}
9
10export async function middleware(c: Context, next: Next) {
11 if (c.req.path.startsWith("/admin")) {
12 return adminInner(c, next);
13 }
14 return next();
15}
16
17export function adminInner(c: Context, next: Next) {
18 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
19
20 if (c.req.path === "/admin/login") {
21 const key = c.req.query("key");
22 if (key) {
23 if (compareToken(key)) {
24 return c.body(null, 303, {
25 "Location": "/admin",
26 "Set-Cookie":
27 `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
28 });
29 }
30 return serveAsset(c, "/admin/login/fail", 403);
31 }
32 if (token && compareToken(token)) {
33 return c.redirect("/admin", 303);
34 }
35 if (c.req.method === "POST") {
36 return serveAsset(c, "/admin/login/fail", 403);
37 } else {
38 return serveAsset(c, "/admin/login", 200);
39 }
40 }
41
42 if (c.req.path === "/admin/logout") {
43 return c.body(null, 303, {
44 "Location": "/admin/login",
45 "Set-Cookie":
46 `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`,
47 });
48 }
49
50 if (token && compareToken(token)) {
51 return next();
52 }
53
54 return c.redirect("/admin/login", 303);
55}
56
57export function hasAdminToken(c: Context) {
58 const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
59 return token && compareToken(token);
60}
61
62export async function main() {
63 const key = crypto.randomUUID();
64 await fs.writeMkdir(".clover/admin-token.txt", key);
65 const start = ({
66 win32: "start",
67 darwin: "open",
68 } as Record<string, string>)[process.platform] ?? "xdg-open";
69 child_process.exec(`${start} http://[::1]:3000/admin/login?key=${key}`);
70}
71
72import * as fs from "#sitegen/fs";
73import type { Context, Next } from "hono";
74import { serveAsset } from "#sitegen/assets";
75import * as child_process from "node:child_process";
src/backend.ts+53-53
......@@ -1,53 +1,53 @@
1// This is the main file for the backend
2const app = new Hono();
3const logHttp = scoped("http", { color: "magenta" });
4
5// Middleware
6app.use(trimTrailingSlash());
7app.use(removeDuplicateSlashes);
8app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4))));
9app.use(admin.middleware);
10
11// Backends
12app.route("", require("./q+a/backend.ts").app);
13app.route("", require("./file-viewer/backend.tsx").app);
14
15// Asset middleware has least precedence
16app.use(assets.middleware);
17
18// Handlers
19app.notFound(assets.notFound);
20if (process.argv.includes("--development")) {
21 app.onError((err, c) => {
22 if (err instanceof HTTPException) {
23 // Get the custom response
24 return err.getResponse();
25 }
26
27 return c.text(util.inspect(err), 500);
28 });
29}
30
31export default app;
32
33async function removeDuplicateSlashes(c: Context, next: Next) {
34 const path = c.req.path;
35 if (/\/\/+/.test(path)) {
36 const normalizedPath = path.replace(/\/\/+/g, "/");
37 const query = c.req.query();
38 const queryString = Object.keys(query).length > 0
39 ? "?" + new URLSearchParams(query).toString()
40 : "";
41 return c.redirect(normalizedPath + queryString, 301);
42 }
43 await next();
44}
45
46import { type Context, Hono, type Next } from "#hono";
47import { HTTPException } from "hono/http-exception";
48import { logger } from "hono/logger";
49import { trimTrailingSlash } from "hono/trailing-slash";
50import * as assets from "#sitegen/assets";
51import * as admin from "./admin.ts";
52import { scoped } from "@paperclover/console";
53import * as util from "node:util";
1// This is the main file for the backend
2const app = new Hono();
3const logHttp = scoped("http", { color: "magenta" });
4
5// Middleware
6app.use(trimTrailingSlash());
7app.use(removeDuplicateSlashes);
8app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4))));
9app.use(admin.middleware);
10
11// Backends
12app.route("", require("./q+a/backend.ts").app);
13app.route("", require("./file-viewer/backend.tsx").app);
14
15// Asset middleware has least precedence
16app.use(assets.middleware);
17
18// Handlers
19app.notFound(assets.notFound);
20if (process.argv.includes("--development")) {
21 app.onError((err, c) => {
22 if (err instanceof HTTPException) {
23 // Get the custom response
24 return err.getResponse();
25 }
26
27 return c.text(util.inspect(err), 500);
28 });
29}
30
31export default app;
32
33async function removeDuplicateSlashes(c: Context, next: Next) {
34 const path = c.req.path;
35 if (/\/\/+/.test(path)) {
36 const normalizedPath = path.replace(/\/\/+/g, "/");
37 const query = c.req.query();
38 const queryString = Object.keys(query).length > 0
39 ? "?" + new URLSearchParams(query).toString()
40 : "";
41 return c.redirect(normalizedPath + queryString, 301);
42 }
43 await next();
44}
45
46import { type Context, Hono, type Next } from "#hono";
47import { HTTPException } from "hono/http-exception";
48import { logger } from "hono/logger";
49import { trimTrailingSlash } from "hono/trailing-slash";
50import * as assets from "#sitegen/assets";
51import * as admin from "./admin.ts";
52import { scoped } from "@paperclover/console";
53import * as util from "node:util";
src/file-viewer/bin/list.ts+8-8
......@@ -1,8 +1,8 @@
1export function main() {
2 const meows = MediaFile.db.prepare(`
3 select * from media_files;
4 `).as(MediaFile).array();
5 console.log(meows);
6}
7
8import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
1export function main() {
2 const meows = MediaFile.db.prepare(`
3 select * from media_files;
4 `).as(MediaFile).array();
5 console.log(meows);
6}
7
8import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
src/file-viewer/bin/scan3.ts+655-649
......@@ -1,649 +1,655 @@
1// The file scanner incrementally updates an sqlite database with file
2// stats. Additionally, it runs "processors" on files, which precompute
3// expensive data such as running `ffprobe` on all media to get the
4// duration.
5//
6// Processors are also used to derive compressed and optimized assets,
7// which is how automatic JXL / AV1 encoding is done. Derived files are
8// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9//
10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.
13const root = path.resolve("/Volumes/clover/Published");
14const workDir = path.resolve(".clover/file-assets");
15
16export async function main() {
17 const start = performance.now();
18 const timerSpinner = new Spinner({
19 text: () =>
20 `paper clover's scan3 [${
21 ((performance.now() - start) / 1000).toFixed(1)
22 }s]`,
23 fps: 10,
24 });
25 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
26
27 // Read a directory or file stat and queue up changed files.
28 using qList = new async.Queue({
29 name: "Discover Tree",
30 async fn(absPath: string, spin) {
31 const stat = await fs.stat(absPath);
32
33 const publicPath = toPublicPath(absPath);
34 const mediaFile = MediaFile.getByPath(publicPath);
35
36 if (stat.isDirectory()) {
37 const items = await fs.readdir(absPath);
38 qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
39
40 if (mediaFile) {
41 const deleted = mediaFile.getChildren()
42 .filter((child) => !items.includes(child.basename))
43 .flatMap((child) =>
44 child.kind === MediaFileKind.directory
45 ? child.getRecursiveFileChildren()
46 : child
47 );
48
49 qMeta.addMany(deleted.map((mediaFile) => ({
50 absPath: path.join(root, mediaFile.path),
51 publicPath: mediaFile.path,
52 stat: null,
53 mediaFile,
54 })));
55 }
56
57 return;
58 }
59
60 // All processes must be performed again if there is no file.
61 if (
62 !mediaFile ||
63 stat.size !== mediaFile.size ||
64 stat.mtime.getTime() !== mediaFile.date.getTime()
65 ) {
66 qMeta.add({ absPath, publicPath, stat, mediaFile });
67 return;
68 }
69
70 // If the scanners changed, it may mean more processes should be run.
71 queueProcessors({ absPath, stat, mediaFile });
72 },
73 maxJobs: 24,
74 });
75 using qMeta = new async.Queue({
76 name: "Update Metadata",
77 async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) {
78 if (!stat) {
79 // File was deleted.
80 await runUndoProcessors(UNWRAP(mediaFile));
81 return;
82 }
83 // TODO: run scrubLocationMetadata first
84
85 const hash = await new Promise<string>((resolve, reject) => {
86 const reader = fs.createReadStream(absPath);
87 reader.on("error", reject);
88
89 const hasher = crypto.createHash("sha1").setEncoding("hex");
90 hasher.on("error", reject);
91 hasher.on("readable", () => resolve(hasher.read()));
92
93 reader.pipe(hasher);
94 });
95 let date = stat.mtime;
96 if (
97 mediaFile &&
98 mediaFile.date.getTime() < stat.mtime.getTime() &&
99 (Date.now() - stat.mtime.getTime()) < monthMilliseconds
100 ) {
101 date = mediaFile.date;
102 console.warn(
103 `M-time on ${publicPath} was likely corrupted. ${
104 formatDate(mediaFile.date)
105 } -> ${formatDate(stat.mtime)}`,
106 );
107 }
108 mediaFile = MediaFile.createFile({
109 path: publicPath,
110 date,
111 hash,
112 size: stat.size,
113 duration: mediaFile?.duration ?? 0,
114 dimensions: mediaFile?.dimensions ?? "",
115 contents: mediaFile?.contents ?? "",
116 });
117 await queueProcessors({ absPath, stat, mediaFile });
118 },
119 getItemText: (job) =>
120 job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"),
121 maxJobs: 10,
122 });
123 using qProcess = new async.Queue({
124 name: "Process Contents",
125 async fn(
126 { absPath, stat, mediaFile, processor, index, after }: ProcessJob,
127 spin,
128 ) {
129 await processor.run({ absPath, stat, mediaFile, spin });
130 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
131 for (const dependantJob of after) {
132 ASSERT(dependantJob.needs > 0);
133 dependantJob.needs -= 1;
134 if (dependantJob.needs == 0) qProcess.add(dependantJob);
135 }
136 },
137 getItemText: ({ mediaFile, processor }) =>
138 `${mediaFile.path.slice(1)} - ${processor.name}`,
139 maxJobs: 4,
140 });
141
142 function decodeProcessors(input: string) {
143 return input
144 .split(";")
145 .filter(Boolean)
146 .map(([a, b, c]) => ({
147 id: a,
148 hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0),
149 }));
150 }
151
152 async function queueProcessors(
153 { absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
154 ) {
155 const ext = mediaFile.extension.toLowerCase();
156 let possible = processors.filter((p) => p.include.has(ext));
157 if (possible.length === 0) return;
158
159 const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
160 ASSERT(hash <= 0xFFFF);
161 let processed = mediaFile.processed;
162
163 // If the hash has changed, migrate the bitfield over.
164 // This also runs when the processor hash is in it's initial 0 state.
165 const order = decodeProcessors(mediaFile.processors);
166 if ((processed & 0xFFFF) !== hash) {
167 const previous = order.filter((_, i) =>
168 (processed & (1 << (16 + i))) !== 0
169 );
170 processed = hash;
171 for (const { id, hash } of previous) {
172 const p = processors.find((p) => p.id === id);
173 if (!p) continue;
174 const index = possible.indexOf(p);
175 if (index !== -1 && p.hash === hash) {
176 processed |= 1 << (16 + index);
177 } else {
178 if (p.undo) await p.undo(mediaFile);
179 }
180 }
181 mediaFile.setProcessors(
182 processed,
183 possible.map((p) =>
184 p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF)
185 ).join(";"),
186 );
187 } else {
188 possible = order.map(({ id }) =>
189 UNWRAP(possible.find((p) => p.id === id))
190 );
191 }
192
193 // Queue needed processors.
194 const jobs: ProcessJob[] = [];
195 for (let i = 0, { length } = possible; i < length; i += 1) {
196 if ((processed & (1 << (16 + i))) === 0) {
197 const job: ProcessJob = {
198 absPath,
199 stat,
200 mediaFile,
201 processor: possible[i],
202 index: i,
203 after: [],
204 needs: possible[i].depends.length,
205 };
206 jobs.push(job);
207 if (job.needs === 0) qProcess.add(job);
208 }
209 }
210 for (const job of jobs) {
211 for (const dependId of job.processor.depends) {
212 const dependJob = jobs.find((j) => j.processor.id === dependId);
213 if (dependJob) {
214 dependJob.after.push(job);
215 } else {
216 ASSERT(job.needs > 0);
217 job.needs -= 1;
218 if (job.needs === 0) qProcess.add(job);
219 }
220 }
221 }
222 }
223
224 async function runUndoProcessors(mediaFile: MediaFile) {
225 const { processed } = mediaFile;
226 const previous = decodeProcessors(mediaFile.processors)
227 .filter((_, i) => (processed & (1 << (16 + i))) !== 0);
228 for (const { id } of previous) {
229 const p = processors.find((p) => p.id === id);
230 if (!p) continue;
231 if (p.undo) {
232 await p.undo(mediaFile);
233 }
234 }
235 mediaFile.delete();
236 }
237
238 // Add the root & recursively iterate!
239 qList.add(root);
240 await qList.done();
241 await qMeta.done();
242 await qProcess.done();
243
244 console.info(
245 "Updated file viewer index in " +
246 ((performance.now() - start) / 1000).toFixed(1) + "s",
247 );
248}
249
250interface Process {
251 name: string;
252 enable?: boolean;
253 include: Set<string>;
254 depends?: string[];
255 /* Perform an action. */
256 run(args: ProcessFileArgs): Promise<void>;
257 /* Should detect if `run` was never even run before before undoing state */
258 undo?(mediaFile: MediaFile): Promise<void>;
259}
260
261const execFileRaw = util.promisify(child_process.execFile);
262const execFile: typeof execFileRaw = ((
263 ...args: Parameters<typeof execFileRaw>
264) =>
265 execFileRaw(...args).catch((e: any) => {
266 if (e?.message?.startsWith?.("Command failed")) {
267 if (e.code > (2 ** 31)) e.code |= 0;
268 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
269 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
270 }
271 throw e;
272 })) as any;
273const ffprobeBin = testProgram("ffprobe", "--help");
274const ffmpegBin = testProgram("ffmpeg", "--help");
275
276const ffmpegOptions = [
277 "-hide_banner",
278 "-loglevel",
279 "warning",
280];
281
282const procDuration: Process = {
283 name: "calculate duration",
284 enable: ffprobeBin !== null,
285 include: rules.extsDuration,
286 async run({ absPath, mediaFile }) {
287 const { stdout } = await execFile(ffprobeBin!, [
288 "-v",
289 "error",
290 "-show_entries",
291 "format=duration",
292 "-of",
293 "default=noprint_wrappers=1:nokey=1",
294 absPath,
295 ]);
296
297 const duration = parseFloat(stdout.trim());
298 if (Number.isNaN(duration)) {
299 throw new Error("Could not extract duration from " + stdout);
300 }
301 mediaFile.setDuration(Math.ceil(duration));
302 },
303};
304
305// NOTE: Never re-order the processors. Add new ones at the end.
306const procDimensions: Process = {
307 name: "calculate dimensions",
308 enable: ffprobeBin != null,
309 include: rules.extsDimensions,
310 async run({ absPath, mediaFile }) {
311 const ext = path.extname(absPath);
312
313 let dimensions;
314
315 if (ext === ".svg") {
316 // Parse out of text data
317 const content = await fs.readFile(absPath, "utf8");
318 const widthMatch = content.match(/width="(\d+)"/);
319 const heightMatch = content.match(/height="(\d+)"/);
320
321 if (widthMatch && heightMatch) {
322 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
323 }
324 } else {
325 // Use ffprobe to observe streams
326 const { stdout } = await execFile("ffprobe", [
327 "-v",
328 "error",
329 "-select_streams",
330 "v:0",
331 "-show_entries",
332 "stream=width,height",
333 "-of",
334 "csv=s=x:p=0",
335 absPath,
336 ]);
337 if (stdout.includes("x")) {
338 dimensions = stdout.trim();
339 }
340 }
341
342 mediaFile.setDimensions(dimensions ?? "");
343 },
344};
345
346const procLoadTextContents: Process = {
347 name: "load text content",
348 include: rules.extsReadContents,
349 async run({ absPath, mediaFile, stat }) {
350 if (stat.size > 1_000_000) return;
351 const text = await fs.readFile(absPath, "utf-8");
352 mediaFile.setContents(text);
353 },
354};
355
356const procHighlightCode: Process = {
357 name: "highlight source code",
358 include: new Set(rules.extsCode.keys()),
359 async run({ absPath, mediaFile, stat }) {
360 const language = UNWRAP(
361 rules.extsCode.get(path.extname(absPath).toLowerCase()),
362 );
363 // An issue is that .ts is an overloaded extension, shared between
364 // 'transport stream' and 'typescript'.
365 //
366 // Filter used here is:
367 // - more than 1mb
368 // - invalid UTF-8
369 if (stat.size > 1_000_000) return;
370 let code;
371 const buf = await fs.readFile(absPath);
372 try {
373 code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
374 } catch (error) {
375 mediaFile.setContents("");
376 return;
377 }
378 const content = await highlight.highlightCode(code, language);
379 mediaFile.setContents(content);
380 },
381};
382
383const procImageSubsets: Process = {
384 name: "encode image subsets",
385 include: rules.extsImage,
386 depends: ["calculate dimensions"],
387 async run({ absPath, mediaFile, spin }) {
388 const { width, height } = UNWRAP(mediaFile.parseDimensions());
389 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
390 const baseStatus = spin.text;
391
392 using stack = new DisposableStack();
393 for (const size of targetSizes) {
394 const { w, h } = resizeDimensions(width, height, size);
395 for (const { ext, args } of transcodeRules.imagePresets) {
396 spin.text = baseStatus +
397 ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
398
399 stack.use(
400 await produceAsset(
401 `${mediaFile.hash}/${size}${ext}`,
402 async (out) => {
403 await fs.mkdir(path.dirname(out));
404 await fs.rm(out, { force: true });
405 await execFile(ffmpegBin!, [
406 ...ffmpegOptions,
407 "-i",
408 absPath,
409 "-vf",
410 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
411 ...args,
412 out,
413 ]);
414 return [out];
415 },
416 ),
417 );
418 }
419 }
420
421 stack.move();
422 },
423 async undo(mediaFile) {
424 const { width } = UNWRAP(mediaFile.parseDimensions());
425 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
426 for (const size of targetSizes) {
427 for (const { ext } of transcodeRules.imagePresets) {
428 unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
429 }
430 }
431 },
432};
433const qualityMap: Record<string, string> = {
434 u: "ultra-high",
435 h: "high",
436 m: "medium",
437 l: "low",
438 d: "data-saving",
439};
440const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
441 name: `encode ${preset.codec} ${UNWRAP(qualityMap[preset.id[1]])}`,
442 include: rules.extsVideo,
443 enable: ffmpegBin != null,
444 async run({ absPath, mediaFile, spin }) {
445 if ((mediaFile.duration ?? 0) < 10) return;
446 await produceAsset(`${mediaFile.hash}/${preset.id}`, async (base) => {
447 base = path.dirname(base);
448 await fs.mkdir(base);
449
450 let inputArgs = ["-i", absPath];
451 try {
452 const config = await fs.readJson<any>(
453 path.join(
454 path.dirname(absPath),
455 path.basename(absPath, path.extname(absPath)) + ".json",
456 ),
457 );
458 if (config.encoder && typeof config.encoder.videoSrc === "string") {
459 const { videoSrc, audioSrc, rate } = config.encoder;
460 inputArgs = [
461 ...rate ? ["-r", String(rate)] : [],
462 "-i",
463 videoSrc,
464 ...audioSrc ? ["-i", audioSrc] : [],
465 ];
466 }
467 } catch (err: any) {
468 if (err?.code !== "ENOENT") throw err;
469 }
470
471 const args = transcodeRules.getVideoArgs(
472 preset,
473 base,
474 inputArgs,
475 );
476 try {
477 const fakeProgress = new Progress({ text: spin.text, spinner: null });
478 fakeProgress.stop();
479 spin.format = (now: number) => fakeProgress.format(now);
480 // @ts-expect-error
481 fakeProgress.redraw = () => spin.redraw();
482
483 await ffmpeg.spawn({
484 ffmpeg: ffmpegBin!,
485 title: fakeProgress.text,
486 progress: fakeProgress,
487 args,
488 });
489 return await collectFiles();
490 } catch (err) {
491 for (const file of await collectFiles()) {
492 try {
493 fs.rm(file);
494 } catch {}
495 }
496 throw err;
497 }
498
499 async function collectFiles(): Promise<string[]> {
500 return (await fs.readdir(base))
501 .filter((basename) => basename.startsWith(preset.id))
502 .map((basename) => path.join(base, basename));
503 }
504 });
505 },
506}));
507
508const processors = [
509 procDimensions,
510 procDuration,
511 procLoadTextContents,
512 procHighlightCode,
513 procImageSubsets,
514 ...procVideos,
515]
516 .map((process, id, all) => {
517 const strIndex = (id: number) =>
518 String.fromCharCode("a".charCodeAt(0) + id);
519 return {
520 ...process as Process,
521 id: strIndex(id),
522 // Create a unique key.
523 hash: new Uint16Array(
524 crypto.createHash("sha1")
525 .update(process.run.toString())
526 .digest().buffer,
527 ).reduce((a, b) => a ^ b),
528 depends: (process.depends ?? []).map((depend) => {
529 const index = all.findIndex((p) => p.name === depend);
530 if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
531 if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
532 return strIndex(index);
533 }),
534 };
535 });
536
537function resizeDimensions(w: number, h: number, desiredWidth: number) {
538 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
539 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
540}
541
542async function produceAsset(
543 key: string,
544 builder: (prefix: string) => Promise<string[]>,
545) {
546 const asset = AssetRef.putOrIncrement(key);
547 try {
548 if (asset.refs === 1) {
549 const paths = await builder(path.join(workDir, key));
550 asset.addFiles(
551 paths.map((file) =>
552 path.relative(workDir, file)
553 .replaceAll("\\", "/")
554 ),
555 );
556 }
557 return {
558 [Symbol.dispose]: () => asset.unref(),
559 };
560 } catch (err: any) {
561 if (err && typeof err === "object") err.assetKey = key;
562 asset.unref();
563 throw err;
564 }
565}
566
567async function unproduceAsset(key: string) {
568 const ref = AssetRef.get(key);
569 if (ref) {
570 ref.unref();
571 console.log(`unref ${key}`);
572 // TODO: remove associated files from target
573 }
574}
575
576interface UpdateMetadataJob {
577 absPath: string;
578 publicPath: string;
579 stat: fs.Stats | null;
580 mediaFile: MediaFile | null;
581}
582
583interface ProcessFileArgs {
584 absPath: string;
585 stat: fs.Stats;
586 mediaFile: MediaFile;
587 spin: Spinner;
588}
589
590interface ProcessJob {
591 absPath: string;
592 stat: fs.Stats;
593 mediaFile: MediaFile;
594 processor: typeof processors[0];
595 index: number;
596 after: ProcessJob[];
597 needs: number;
598}
599
600export function skipBasename(basename: string): boolean {
601 // dot files must be incrementally tracked
602 if (basename === ".dirsort") return true;
603 if (basename === ".friends") return true;
604
605 return (
606 basename.startsWith(".") ||
607 basename.startsWith("._") ||
608 basename.startsWith(".tmp") ||
609 basename === ".DS_Store" ||
610 basename.toLowerCase() === "thumbs.db" ||
611 basename.toLowerCase() === "desktop.ini"
612 );
613}
614
615export function toPublicPath(absPath: string) {
616 ASSERT(path.isAbsolute(absPath));
617 if (absPath === root) return "/";
618 return "/" + path.relative(root, absPath).replaceAll("\\", "/");
619}
620
621export function testProgram(name: string, helpArgument: string) {
622 try {
623 child_process.spawnSync(name, [helpArgument]);
624 return name;
625 } catch (err) {
626 console.warn(`Missing or corrupt executable '${name}'`);
627 }
628 return null;
629}
630
631const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
632
633import { Progress } from "@paperclover/console/Progress";
634import { Spinner } from "@paperclover/console/Spinner";
635import * as async from "#sitegen/async";
636import * as fs from "#sitegen/fs";
637
638import * as path from "node:path";
639import * as child_process from "node:child_process";
640import * as util from "node:util";
641import * as crypto from "node:crypto";
642
643import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
644import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
645import { formatDate } from "@/file-viewer/format.ts";
646import * as rules from "@/file-viewer/rules.ts";
647import * as highlight from "@/file-viewer/highlight.ts";
648import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
649import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
1// The file scanner incrementally updates an sqlite database with file
2// stats. Additionally, it runs "processors" on files, which precompute
3// expensive data such as running `ffprobe` on all media to get the
4// duration.
5//
6// Processors are also used to derive compressed and optimized assets,
7// which is how automatic JXL / AV1 encoding is done. Derived files are
8// uploaded to the clover NAS to be pulled by VPS instances for hosting.
9//
10// This is the third iteration of the scanner, hence its name "scan3";
11// Remember that any software you want to be maintainable and high
12// quality cannot be written with AI.
13const root = path.resolve("/Volumes/clover/Published");
14const workDir = path.resolve(".clover/file-assets");
15
16export async function main() {
17 const start = performance.now();
18 const timerSpinner = new Spinner({
19 text: () =>
20 `paper clover's scan3 [${
21 ((performance.now() - start) / 1000).toFixed(1)
22 }s]`,
23 fps: 10,
24 });
25 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
26
27 // Read a directory or file stat and queue up changed files.
28 using qList = new async.Queue({
29 name: "Discover Tree",
30 async fn(absPath: string, spin) {
31 const stat = await fs.stat(absPath);
32
33 const publicPath = toPublicPath(absPath);
34 const mediaFile = MediaFile.getByPath(publicPath);
35
36 if (stat.isDirectory()) {
37 const items = await fs.readdir(absPath);
38 qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
39
40 if (mediaFile) {
41 const deleted = mediaFile.getChildren()
42 .filter((child) => !items.includes(child.basename))
43 .flatMap((child) =>
44 child.kind === MediaFileKind.directory
45 ? child.getRecursiveFileChildren()
46 : child
47 );
48
49 qMeta.addMany(deleted.map((mediaFile) => ({
50 absPath: path.join(root, mediaFile.path),
51 publicPath: mediaFile.path,
52 stat: null,
53 mediaFile,
54 })));
55 }
56
57 return;
58 }
59
60 // All processes must be performed again if there is no file.
61 if (
62 !mediaFile ||
63 stat.size !== mediaFile.size ||
64 stat.mtime.getTime() !== mediaFile.date.getTime()
65 ) {
66 qMeta.add({ absPath, publicPath, stat, mediaFile });
67 return;
68 }
69
70 // If the scanners changed, it may mean more processes should be run.
71 queueProcessors({ absPath, stat, mediaFile });
72 },
73 maxJobs: 24,
74 });
75 using qMeta = new async.Queue({
76 name: "Update Metadata",
77 async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) {
78 if (!stat) {
79 // File was deleted.
80 await runUndoProcessors(UNWRAP(mediaFile));
81 return;
82 }
83 // TODO: run scrubLocationMetadata first
84
85 const hash = await new Promise<string>((resolve, reject) => {
86 const reader = fs.createReadStream(absPath);
87 reader.on("error", reject);
88
89 const hasher = crypto.createHash("sha1").setEncoding("hex");
90 hasher.on("error", reject);
91 hasher.on("readable", () => resolve(hasher.read()));
92
93 reader.pipe(hasher);
94 });
95 let date = stat.mtime;
96 if (
97 mediaFile &&
98 mediaFile.date.getTime() < stat.mtime.getTime() &&
99 (Date.now() - stat.mtime.getTime()) < monthMilliseconds
100 ) {
101 date = mediaFile.date;
102 console.warn(
103 `M-time on ${publicPath} was likely corrupted. ${
104 formatDate(mediaFile.date)
105 } -> ${formatDate(stat.mtime)}`,
106 );
107 }
108 mediaFile = MediaFile.createFile({
109 path: publicPath,
110 date,
111 hash,
112 size: stat.size,
113 duration: mediaFile?.duration ?? 0,
114 dimensions: mediaFile?.dimensions ?? "",
115 contents: mediaFile?.contents ?? "",
116 });
117 await queueProcessors({ absPath, stat, mediaFile });
118 },
119 getItemText: (job) =>
120 job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"),
121 maxJobs: 10,
122 });
123 using qProcess = new async.Queue({
124 name: "Process Contents",
125 async fn(
126 { absPath, stat, mediaFile, processor, index, after }: ProcessJob,
127 spin,
128 ) {
129 await processor.run({ absPath, stat, mediaFile, spin });
130 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
131 for (const dependantJob of after) {
132 ASSERT(dependantJob.needs > 0);
133 dependantJob.needs -= 1;
134 if (dependantJob.needs == 0) qProcess.add(dependantJob);
135 }
136 },
137 getItemText: ({ mediaFile, processor }) =>
138 `${mediaFile.path.slice(1)} - ${processor.name}`,
139 maxJobs: 4,
140 });
141
142 function decodeProcessors(input: string) {
143 return input
144 .split(";")
145 .filter(Boolean)
146 .map(([a, b, c]) => ({
147 id: a,
148 hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0),
149 }));
150 }
151
152 async function queueProcessors(
153 { absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
154 ) {
155 const ext = mediaFile.extension.toLowerCase();
156 let possible = processors.filter((p) => p.include.has(ext));
157 if (possible.length === 0) return;
158
159 const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
160 ASSERT(hash <= 0xFFFF);
161 let processed = mediaFile.processed;
162
163 // If the hash has changed, migrate the bitfield over.
164 // This also runs when the processor hash is in it's initial 0 state.
165 const order = decodeProcessors(mediaFile.processors);
166 if ((processed & 0xFFFF) !== hash) {
167 const previous = order.filter((_, i) =>
168 (processed & (1 << (16 + i))) !== 0
169 );
170 processed = hash;
171 for (const { id, hash } of previous) {
172 const p = processors.find((p) => p.id === id);
173 if (!p) continue;
174 const index = possible.indexOf(p);
175 if (index !== -1 && p.hash === hash) {
176 processed |= 1 << (16 + index);
177 } else {
178 if (p.undo) await p.undo(mediaFile);
179 }
180 }
181 mediaFile.setProcessors(
182 processed,
183 possible.map((p) =>
184 p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF)
185 ).join(";"),
186 );
187 } else {
188 possible = order.map(({ id }) =>
189 UNWRAP(possible.find((p) => p.id === id))
190 );
191 }
192
193 // Queue needed processors.
194 const jobs: ProcessJob[] = [];
195 for (let i = 0, { length } = possible; i < length; i += 1) {
196 if ((processed & (1 << (16 + i))) === 0) {
197 const job: ProcessJob = {
198 absPath,
199 stat,
200 mediaFile,
201 processor: possible[i],
202 index: i,
203 after: [],
204 needs: possible[i].depends.length,
205 };
206 jobs.push(job);
207 if (job.needs === 0) qProcess.add(job);
208 }
209 }
210 for (const job of jobs) {
211 for (const dependId of job.processor.depends) {
212 const dependJob = jobs.find((j) => j.processor.id === dependId);
213 if (dependJob) {
214 dependJob.after.push(job);
215 } else {
216 ASSERT(job.needs > 0);
217 job.needs -= 1;
218 if (job.needs === 0) qProcess.add(job);
219 }
220 }
221 }
222 }
223
224 async function runUndoProcessors(mediaFile: MediaFile) {
225 const { processed } = mediaFile;
226 const previous = decodeProcessors(mediaFile.processors)
227 .filter((_, i) => (processed & (1 << (16 + i))) !== 0);
228 for (const { id } of previous) {
229 const p = processors.find((p) => p.id === id);
230 if (!p) continue;
231 if (p.undo) {
232 await p.undo(mediaFile);
233 }
234 }
235 mediaFile.delete();
236 }
237
238 // Add the root & recursively iterate!
239 qList.add(root);
240 await qList.done();
241 await qMeta.done();
242 await qProcess.done();
243
244 console.info(
245 "Updated file viewer index in " +
246 ((performance.now() - start) / 1000).toFixed(1) + "s",
247 );
248}
249
250interface Process {
251 name: string;
252 enable?: boolean;
253 include: Set<string>;
254 depends?: string[];
255 version?: number;
256 /* Perform an action. */
257 run(args: ProcessFileArgs): Promise<void>;
258 /* Should detect if `run` was never even run before before undoing state */
259 undo?(mediaFile: MediaFile): Promise<void>;
260}
261
262const execFileRaw = util.promisify(child_process.execFile);
263const execFile: typeof execFileRaw = ((
264 ...args: Parameters<typeof execFileRaw>
265) =>
266 execFileRaw(...args).catch((e: any) => {
267 if (e?.message?.startsWith?.("Command failed")) {
268 if (e.code > (2 ** 31)) e.code |= 0;
269 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
270 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
271 }
272 throw e;
273 })) as any;
274const ffprobeBin = testProgram("ffprobe", "--help");
275const ffmpegBin = testProgram("ffmpeg", "--help");
276
277const ffmpegOptions = [
278 "-hide_banner",
279 "-loglevel",
280 "warning",
281];
282
283const procDuration: Process = {
284 name: "calculate duration",
285 enable: ffprobeBin !== null,
286 include: rules.extsDuration,
287 async run({ absPath, mediaFile }) {
288 const { stdout } = await execFile(ffprobeBin!, [
289 "-v",
290 "error",
291 "-show_entries",
292 "format=duration",
293 "-of",
294 "default=noprint_wrappers=1:nokey=1",
295 absPath,
296 ]);
297
298 const duration = parseFloat(stdout.trim());
299 if (Number.isNaN(duration)) {
300 throw new Error("Could not extract duration from " + stdout);
301 }
302 mediaFile.setDuration(Math.ceil(duration));
303 },
304};
305
306// NOTE: Never re-order the processors. Add new ones at the end.
307const procDimensions: Process = {
308 name: "calculate dimensions",
309 enable: ffprobeBin != null,
310 include: rules.extsDimensions,
311 async run({ absPath, mediaFile }) {
312 const ext = path.extname(absPath);
313
314 let dimensions;
315
316 if (ext === ".svg") {
317 // Parse out of text data
318 const content = await fs.readFile(absPath, "utf8");
319 const widthMatch = content.match(/width="(\d+)"/);
320 const heightMatch = content.match(/height="(\d+)"/);
321
322 if (widthMatch && heightMatch) {
323 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
324 }
325 } else {
326 // Use ffprobe to observe streams
327 const { stdout } = await execFile("ffprobe", [
328 "-v",
329 "error",
330 "-select_streams",
331 "v:0",
332 "-show_entries",
333 "stream=width,height",
334 "-of",
335 "csv=s=x:p=0",
336 absPath,
337 ]);
338 if (stdout.includes("x")) {
339 dimensions = stdout.trim();
340 }
341 }
342
343 mediaFile.setDimensions(dimensions ?? "");
344 },
345};
346
347const procLoadTextContents: Process = {
348 name: "load text content",
349 include: rules.extsReadContents,
350 async run({ absPath, mediaFile, stat }) {
351 if (stat.size > 1_000_000) return;
352 const text = await fs.readFile(absPath, "utf-8");
353 mediaFile.setContents(text);
354 },
355};
356
357const procHighlightCode: Process = {
358 name: "highlight source code",
359 include: new Set(rules.extsCode.keys()),
360 async run({ absPath, mediaFile, stat }) {
361 const language = UNWRAP(
362 rules.extsCode.get(path.extname(absPath).toLowerCase()),
363 );
364 // An issue is that .ts is an overloaded extension, shared between
365 // 'transport stream' and 'typescript'.
366 //
367 // Filter used here is:
368 // - more than 1mb
369 // - invalid UTF-8
370 if (stat.size > 1_000_000) return;
371 let code;
372 const buf = await fs.readFile(absPath);
373 try {
374 code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
375 } catch (error) {
376 mediaFile.setContents("");
377 return;
378 }
379 const content = await highlight.highlightCode(code, language);
380 mediaFile.setContents(content);
381 },
382};
383
384const procImageSubsets: Process = {
385 name: "encode image subsets",
386 include: rules.extsImage,
387 depends: ["calculate dimensions"],
388 version: 2,
389 async run({ absPath, mediaFile, spin }) {
390 const { width, height } = UNWRAP(mediaFile.parseDimensions());
391 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
392 const baseStatus = spin.text;
393
394 using stack = new DisposableStack();
395 for (const size of targetSizes) {
396 const { w, h } = resizeDimensions(width, height, size);
397 for (const { ext, args } of transcodeRules.imagePresets) {
398 spin.text = baseStatus +
399 ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
400
401 stack.use(
402 await produceAsset(
403 `${mediaFile.hash}/${size}${ext}`,
404 async (out) => {
405 await fs.mkdir(path.dirname(out));
406 await fs.rm(out, { force: true });
407 await execFile(ffmpegBin!, [
408 ...ffmpegOptions,
409 "-i",
410 absPath,
411 "-vf",
412 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
413 ...args,
414 out,
415 ]);
416 return [out];
417 },
418 ),
419 );
420 }
421 }
422
423 stack.move();
424 },
425 async undo(mediaFile) {
426 const { width } = UNWRAP(mediaFile.parseDimensions());
427 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
428 for (const size of targetSizes) {
429 for (const { ext } of transcodeRules.imagePresets) {
430 unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
431 }
432 }
433 },
434};
435const qualityMap: Record<string, string> = {
436 u: "ultra-high",
437 h: "high",
438 m: "medium",
439 l: "low",
440 d: "data-saving",
441};
442const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
443 name: `encode ${preset.codec} ${UNWRAP(qualityMap[preset.id[1]])}`,
444 include: rules.extsVideo,
445 enable: ffmpegBin != null,
446 async run({ absPath, mediaFile, spin }) {
447 if ((mediaFile.duration ?? 0) < 10) return;
448 await produceAsset(`${mediaFile.hash}/${preset.id}`, async (base) => {
449 base = path.dirname(base);
450 await fs.mkdir(base);
451
452 let inputArgs = ["-i", absPath];
453 try {
454 const config = await fs.readJson<any>(
455 path.join(
456 path.dirname(absPath),
457 path.basename(absPath, path.extname(absPath)) + ".json",
458 ),
459 );
460 if (config.encoder && typeof config.encoder.videoSrc === "string") {
461 const { videoSrc, audioSrc, rate } = config.encoder;
462 inputArgs = [
463 ...rate ? ["-r", String(rate)] : [],
464 "-i",
465 videoSrc,
466 ...audioSrc ? ["-i", audioSrc] : [],
467 ];
468 }
469 } catch (err: any) {
470 if (err?.code !== "ENOENT") throw err;
471 }
472
473 const args = transcodeRules.getVideoArgs(
474 preset,
475 base,
476 inputArgs,
477 );
478 try {
479 const fakeProgress = new Progress({ text: spin.text, spinner: null });
480 fakeProgress.stop();
481 spin.format = (now: number) => fakeProgress.format(now);
482 // @ts-expect-error
483 fakeProgress.redraw = () => spin.redraw();
484
485 await ffmpeg.spawn({
486 ffmpeg: ffmpegBin!,
487 title: fakeProgress.text,
488 progress: fakeProgress,
489 args,
490 cwd: base,
491 });
492 return await collectFiles();
493 } catch (err) {
494 for (const file of await collectFiles()) {
495 try {
496 fs.rm(file);
497 } catch {}
498 }
499 throw err;
500 }
501
502 async function collectFiles(): Promise<string[]> {
503 return (await fs.readdir(base))
504 .filter((basename) => basename.startsWith(preset.id))
505 .map((basename) => path.join(base, basename));
506 }
507 });
508 },
509}));
510
511const processors = [
512 procDimensions,
513 procDuration,
514 procLoadTextContents,
515 procHighlightCode,
516 procImageSubsets,
517 ...procVideos,
518]
519 .map((process, id, all) => {
520 const strIndex = (id: number) =>
521 String.fromCharCode("a".charCodeAt(0) + id);
522 return {
523 ...process as Process,
524 id: strIndex(id),
525 // Create a unique key.
526 hash: new Uint16Array(
527 crypto.createHash("sha1")
528 .update(
529 process.run.toString() +
530 (process.version ? String(process.version) : ""),
531 )
532 .digest().buffer,
533 ).reduce((a, b) => a ^ b),
534 depends: (process.depends ?? []).map((depend) => {
535 const index = all.findIndex((p) => p.name === depend);
536 if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
537 if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
538 return strIndex(index);
539 }),
540 };
541 });
542
543function resizeDimensions(w: number, h: number, desiredWidth: number) {
544 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
545 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
546}
547
548async function produceAsset(
549 key: string,
550 builder: (prefix: string) => Promise<string[]>,
551) {
552 const asset = AssetRef.putOrIncrement(key);
553 try {
554 if (asset.refs === 1) {
555 const paths = await builder(path.join(workDir, key));
556 asset.addFiles(
557 paths.map((file) =>
558 path.relative(workDir, file)
559 .replaceAll("\\", "/")
560 ),
561 );
562 }
563 return {
564 [Symbol.dispose]: () => asset.unref(),
565 };
566 } catch (err: any) {
567 if (err && typeof err === "object") err.assetKey = key;
568 asset.unref();
569 throw err;
570 }
571}
572
573async function unproduceAsset(key: string) {
574 const ref = AssetRef.get(key);
575 if (ref) {
576 ref.unref();
577 console.log(`unref ${key}`);
578 // TODO: remove associated files from target
579 }
580}
581
582interface UpdateMetadataJob {
583 absPath: string;
584 publicPath: string;
585 stat: fs.Stats | null;
586 mediaFile: MediaFile | null;
587}
588
589interface ProcessFileArgs {
590 absPath: string;
591 stat: fs.Stats;
592 mediaFile: MediaFile;
593 spin: Spinner;
594}
595
596interface ProcessJob {
597 absPath: string;
598 stat: fs.Stats;
599 mediaFile: MediaFile;
600 processor: typeof processors[0];
601 index: number;
602 after: ProcessJob[];
603 needs: number;
604}
605
606export function skipBasename(basename: string): boolean {
607 // dot files must be incrementally tracked
608 if (basename === ".dirsort") return true;
609 if (basename === ".friends") return true;
610
611 return (
612 basename.startsWith(".") ||
613 basename.startsWith("._") ||
614 basename.startsWith(".tmp") ||
615 basename === ".DS_Store" ||
616 basename.toLowerCase() === "thumbs.db" ||
617 basename.toLowerCase() === "desktop.ini"
618 );
619}
620
621export function toPublicPath(absPath: string) {
622 ASSERT(path.isAbsolute(absPath));
623 if (absPath === root) return "/";
624 return "/" + path.relative(root, absPath).replaceAll("\\", "/");
625}
626
627export function testProgram(name: string, helpArgument: string) {
628 try {
629 child_process.spawnSync(name, [helpArgument]);
630 return name;
631 } catch (err) {
632 console.warn(`Missing or corrupt executable '${name}'`);
633 }
634 return null;
635}
636
637const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
638
639import { Progress } from "@paperclover/console/Progress";
640import { Spinner } from "@paperclover/console/Spinner";
641import * as async from "#sitegen/async";
642import * as fs from "#sitegen/fs";
643
644import * as path from "node:path";
645import * as child_process from "node:child_process";
646import * as util from "node:util";
647import * as crypto from "node:crypto";
648
649import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
650import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
651import { formatDate } from "@/file-viewer/format.ts";
652import * as rules from "@/file-viewer/rules.ts";
653import * as highlight from "@/file-viewer/highlight.ts";
654import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
655import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
src/file-viewer/highlight.ts+1-4
......@@ -135,10 +135,7 @@ function highlightLines({
135135
136136export const getRegistry = async.once(async () => {
137137 const wasmBin = await fs.readFile(
138 path.join(
139 import.meta.dirname,
140 "../node_modules/vscode-oniguruma/release/onig.wasm",
141 ),
138 require.resolve("vscode-oniguruma/release/onig.wasm"),
142139 );
143140 await oniguruma.loadWASM(wasmBin);
144141
src/file-viewer/models/AssetRef.ts+73-73
......@@ -1,73 +1,73 @@
1const db = getDb("cache.sqlite");
2db.table(
3 "asset_refs",
4 /* SQL */ `
5 create table if not exists asset_refs (
6 id integer primary key autoincrement,
7 key text not null UNIQUE,
8 refs integer not null
9 );
10 create table if not exists asset_ref_files (
11 file text not null,
12 id integer not null,
13 foreign key (id) references asset_refs(id)
14 );
15 create index asset_ref_files_id on asset_ref_files(id);
16`,
17);
18
19/**
20 * Uncompressed files are read directly from the media store root. Derivied
21 * assets like compressed files, optimized images, and streamable video are
22 * stored in the `derived` folder. After scanning, the derived assets are
23 * uploaded into the store (storage1/clofi-derived dataset on NAS). Since
24 * multiple files can share the same hash, the number of references is
25 * tracked, and the derived content is only produced once. This means if a
26 * file is deleted, it should only decrement a reference count; deleting it
27 * once all references are removed.
28 */
29export class AssetRef {
30 /** Key which aws referenced */
31 id!: number;
32 key!: string;
33 refs!: number;
34
35 unref() {
36 decrementQuery.run(this.key);
37 deleteUnreferencedQuery.run().changes > 0;
38 }
39
40 addFiles(files: string[]) {
41 for (const file of files) {
42 addFileQuery.run({ id: this.id, file });
43 }
44 }
45
46 static get(key: string) {
47 return getQuery.get(key);
48 }
49
50 static putOrIncrement(key: string) {
51 putOrIncrementQuery.get(key);
52 return UNWRAP(AssetRef.get(key));
53 }
54}
55
56const getQuery = db.prepare<[key: string]>(/* SQL */ `
57 select * from asset_refs where key = ?;
58`).as(AssetRef);
59const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ `
60 insert into asset_refs (key, refs) values (?, 1)
61 on conflict(key) do update set refs = refs + 1;
62`);
63const decrementQuery = db.prepare<[key: string]>(/* SQL */ `
64 update asset_refs set refs = refs - 1 where key = ? and refs > 0;
65`);
66const deleteUnreferencedQuery = db.prepare(/* SQL */ `
67 delete from asset_refs where refs <= 0;
68`);
69const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ `
70 insert into asset_ref_files (id, file) values ($id, $file);
71`);
72
73import { getDb } from "#sitegen/sqlite";
1const db = getDb("cache.sqlite");
2db.table(
3 "asset_refs",
4 /* SQL */ `
5 create table if not exists asset_refs (
6 id integer primary key autoincrement,
7 key text not null UNIQUE,
8 refs integer not null
9 );
10 create table if not exists asset_ref_files (
11 file text not null,
12 id integer not null,
13 foreign key (id) references asset_refs(id) ON DELETE CASCADE
14 );
15 create index asset_ref_files_id on asset_ref_files(id);
16`,
17);
18
19/**
20 * Uncompressed files are read directly from the media store root. Derivied
21 * assets like compressed files, optimized images, and streamable video are
22 * stored in the `derived` folder. After scanning, the derived assets are
23 * uploaded into the store (storage1/clofi-derived dataset on NAS). Since
24 * multiple files can share the same hash, the number of references is
25 * tracked, and the derived content is only produced once. This means if a
26 * file is deleted, it should only decrement a reference count; deleting it
27 * once all references are removed.
28 */
29export class AssetRef {
30 /** Key which aws referenced */
31 id!: number;
32 key!: string;
33 refs!: number;
34
35 unref() {
36 decrementQuery.run(this.key);
37 deleteUnreferencedQuery.run().changes > 0;
38 }
39
40 addFiles(files: string[]) {
41 for (const file of files) {
42 addFileQuery.run({ id: this.id, file });
43 }
44 }
45
46 static get(key: string) {
47 return getQuery.get(key);
48 }
49
50 static putOrIncrement(key: string) {
51 putOrIncrementQuery.get(key);
52 return UNWRAP(AssetRef.get(key));
53 }
54}
55
56const getQuery = db.prepare<[key: string]>(/* SQL */ `
57 select * from asset_refs where key = ?;
58`).as(AssetRef);
59const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ `
60 insert into asset_refs (key, refs) values (?, 1)
61 on conflict(key) do update set refs = refs + 1;
62`);
63const decrementQuery = db.prepare<[key: string]>(/* SQL */ `
64 update asset_refs set refs = refs - 1 where key = ? and refs > 0;
65`);
66const deleteUnreferencedQuery = db.prepare(/* SQL */ `
67 delete from asset_refs where refs <= 0;
68`);
69const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ `
70 insert into asset_ref_files (id, file) values ($id, $file);
71`);
72
73import { getDb } from "#sitegen/sqlite";
src/file-viewer/models/FilePermissions.ts+59-59
......@@ -1,59 +1,59 @@
1const db = getDb("cache.sqlite");
2
3db.table(
4 "permissions",
5 /* SQL */ `
6 CREATE TABLE IF NOT EXISTS permissions (
7 prefix TEXT PRIMARY KEY,
8 allow INTEGER NOT NULL
9 );
10`,
11);
12export class FilePermissions {
13 prefix!: string;
14 /** Currently set to 1 always */
15 allow!: number;
16
17 // -- static ops --
18 static getByPrefix(filePath: string): number {
19 return getByPrefixQuery.get(filePath)?.allow ?? 0;
20 }
21
22 static getExact(filePath: string): number {
23 return getExactQuery.get(filePath)?.allow ?? 0;
24 }
25
26 static setPermissions(prefix: string, allow: number) {
27 if (allow) {
28 insertQuery.run({ prefix, allow });
29 } else {
30 deleteQuery.run(prefix);
31 }
32 }
33}
34
35const getByPrefixQuery = db.prepare<
36 [prefix: string],
37 Pick<FilePermissions, "allow">
38>(/* SQL */ `
39 SELECT allow
40 FROM permissions
41 WHERE ? GLOB prefix || '*'
42 ORDER BY LENGTH(prefix) DESC
43 LIMIT 1;
44`);
45const getExactQuery = db.prepare<
46 [file: string],
47 Pick<FilePermissions, "allow">
48>(/* SQL */ `
49 SELECT allow FROM permissions WHERE ? == prefix
50`);
51
52const insertQuery = db.prepare<[{ prefix: string; allow: number }]>(/* SQL */ `
53 REPLACE INTO permissions(prefix, allow) VALUES($prefix, $allow);
54`);
55const deleteQuery = db.prepare<[file: string]>(/* SQL */ `
56 DELETE FROM permissions WHERE prefix = ?;
57`);
58
59import { getDb } from "#sitegen/sqlite";
1const db = getDb("cache.sqlite");
2
3db.table(
4 "permissions",
5 /* SQL */ `
6 CREATE TABLE IF NOT EXISTS permissions (
7 prefix TEXT PRIMARY KEY,
8 allow INTEGER NOT NULL
9 );
10`,
11);
12export class FilePermissions {
13 prefix!: string;
14 /** Currently set to 1 always */
15 allow!: number;
16
17 // -- static ops --
18 static getByPrefix(filePath: string): number {
19 return getByPrefixQuery.get(filePath)?.allow ?? 0;
20 }
21
22 static getExact(filePath: string): number {
23 return getExactQuery.get(filePath)?.allow ?? 0;
24 }
25
26 static setPermissions(prefix: string, allow: number) {
27 if (allow) {
28 insertQuery.run({ prefix, allow });
29 } else {
30 deleteQuery.run(prefix);
31 }
32 }
33}
34
35const getByPrefixQuery = db.prepare<
36 [prefix: string],
37 Pick<FilePermissions, "allow">
38>(/* SQL */ `
39 SELECT allow
40 FROM permissions
41 WHERE ? GLOB prefix || '*'
42 ORDER BY LENGTH(prefix) DESC
43 LIMIT 1;
44`);
45const getExactQuery = db.prepare<
46 [file: string],
47 Pick<FilePermissions, "allow">
48>(/* SQL */ `
49 SELECT allow FROM permissions WHERE ? == prefix
50`);
51
52const insertQuery = db.prepare<[{ prefix: string; allow: number }]>(/* SQL */ `
53 REPLACE INTO permissions(prefix, allow) VALUES($prefix, $allow);
54`);
55const deleteQuery = db.prepare<[file: string]>(/* SQL */ `
56 DELETE FROM permissions WHERE prefix = ?;
57`);
58
59import { getDb } from "#sitegen/sqlite";
src/file-viewer/models/MediaFile.ts+436-436
......@@ -1,436 +1,436 @@
1const db = getDb("cache.sqlite");
2db.table(
3 "media_files",
4 /* SQL */ `
5 create table media_files (
6 id integer primary key autoincrement,
7 parent_id integer,
8 path text unique,
9 kind integer not null,
10 timestamp integer not null,
11 timestamp_updated integer not null default current_timestamp,
12 hash text not null,
13 size integer not null,
14 duration integer not null default 0,
15 dimensions text not null default "",
16 contents text not null,
17 dirsort text,
18 processed integer not null,
19 processors text not null default "",
20 foreign key (parent_id) references media_files(id)
21 );
22 -- index for quickly looking up files by path
23 create index media_files_path on media_files (path);
24 -- index for quickly looking up children
25 create index media_files_parent_id on media_files (parent_id);
26 -- index for quickly looking up recursive file children
27 create index media_files_file_children on media_files (kind, path);
28 -- index for finding directories that need to be processed
29 create index media_files_directory_processed on media_files (kind, processed);
30`,
31);
32
33export enum MediaFileKind {
34 directory = 0,
35 file = 1,
36}
37export class MediaFile {
38 id!: number;
39 parent_id!: number | null;
40 /**
41 * Has leading slash, does not have `/file` prefix.
42 * @example "/2025/waterfalls/waterfalls.mp3"
43 */
44 path!: string;
45 kind!: MediaFileKind;
46 private timestamp!: number;
47 private timestamp_updated!: number;
48 /** for mp3/mp4 files, measured in seconds */
49 duration?: number;
50 /** for images and videos, the dimensions. Two numbers split by `x` */
51 dimensions?: string;
52 /**
53 * sha1 of
54 * - files: the contents
55 * - directories: the JSON array of strings + the content of `readme.txt`
56 * this is used
57 * - to inform changes in caching mechanisms (etag, page render cache)
58 * - as a filename for compressed files (.clover/compressed/<hash>.{gz,zstd})
59 */
60 hash!: string;
61 /**
62 * Depends on the file kind.
63 *
64 * - For directories, this is the contents of `readme.txt`, if it exists.
65 * - Otherwise, it is an empty string.
66 */
67 contents!: string;
68 /**
69 * For directories, if this is set, it is a JSON-encoded array of the explicit
70 * sorting order. Derived off of `.dirsort` files.
71 */
72 dirsort!: string | null;
73 /** in bytes */
74 size!: number;
75 /**
76 * 0 - not processed
77 * non-zero - processed
78 *
79 * file: a bit-field of the processors.
80 * directory: this is for re-indexing contents
81 */
82 processed!: number;
83 processors!: string;
84
85 // -- instance ops --
86 get date() {
87 return new Date(this.timestamp);
88 }
89 get lastUpdateDate() {
90 return new Date(this.timestamp_updated);
91 }
92 parseDimensions() {
93 const dimensions = this.dimensions;
94 if (!dimensions) return null;
95 const [width, height] = dimensions.split("x").map(Number);
96 return { width, height };
97 }
98 get basename() {
99 return path.basename(this.path);
100 }
101 get basenameWithoutExt() {
102 return path.basename(this.path, path.extname(this.path));
103 }
104 get extension() {
105 return path.extname(this.path);
106 }
107 getChildren() {
108 return MediaFile.getChildren(this.id)
109 .filter((file) => !file.basename.startsWith("."));
110 }
111 getPublicChildren() {
112 const children = MediaFile.getChildren(this.id);
113 if (FilePermissions.getByPrefix(this.path) == 0) {
114 return children.filter(({ path }) => FilePermissions.getExact(path) == 0);
115 }
116 return children;
117 }
118 getParent() {
119 const dirPath = this.path;
120 if (dirPath === "/") return null;
121 const parentPath = path.dirname(dirPath);
122 if (parentPath === dirPath) return null;
123 const result = MediaFile.getByPath(parentPath);
124 if (!result) return null;
125 ASSERT(result.kind === MediaFileKind.directory);
126 return result;
127 }
128 setProcessed(processed: number) {
129 setProcessedQuery.run({ id: this.id, processed });
130 this.processed = processed;
131 }
132 setProcessors(processed: number, processors: string) {
133 setProcessorsQuery.run({ id: this.id, processed, processors });
134 this.processed = processed;
135 this.processors = processors;
136 }
137 setDuration(duration: number) {
138 setDurationQuery.run({ id: this.id, duration });
139 this.duration = duration;
140 }
141 setDimensions(dimensions: string) {
142 setDimensionsQuery.run({ id: this.id, dimensions });
143 this.dimensions = dimensions;
144 }
145 setContents(contents: string) {
146 setContentsQuery.run({ id: this.id, contents });
147 this.contents = contents;
148 }
149 getRecursiveFileChildren() {
150 if (this.kind !== MediaFileKind.directory) return [];
151 return getChildrenFilesRecursiveQuery.array(this.path + "/");
152 }
153 delete() {
154 deleteCascadeQuery.run({ id: this.id });
155 }
156
157 // -- static ops --
158 static getByPath(filePath: string): MediaFile | null {
159 const result = getByPathQuery.get(filePath);
160 if (result) return result;
161 if (filePath === "/") {
162 return Object.assign(new MediaFile(), {
163 id: 0,
164 parent_id: null,
165 path: "/",
166 kind: MediaFileKind.directory,
167 timestamp: 0,
168 timestamp_updated: Date.now(),
169 hash: "0".repeat(40),
170 contents: "the file scanner has not been run yet",
171 dirsort: null,
172 size: 0,
173 processed: 1,
174 });
175 }
176 return null;
177 }
178 static createFile({
179 path: filePath,
180 date,
181 hash,
182 size,
183 duration,
184 dimensions,
185 contents,
186 }: CreateFile) {
187 ASSERT(
188 !filePath.includes("\\") && filePath.startsWith("/"),
189 `Invalid path: ${filePath}`,
190 );
191 return createFileQuery.getNonNull({
192 path: filePath,
193 parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)),
194 timestamp: date.getTime(),
195 timestampUpdated: Date.now(),
196 hash,
197 size,
198 duration,
199 dimensions,
200 contents,
201 });
202 }
203 static getOrPutDirectoryId(filePath: string) {
204 ASSERT(
205 !filePath.includes("\\") && filePath.startsWith("/"),
206 `Invalid path: ${filePath}`,
207 );
208 filePath = path.normalize(filePath);
209 const row = getDirectoryIdQuery.get(filePath);
210 if (row) return row.id;
211 let current = filePath;
212 let parts = [];
213 let parentId: null | number = null;
214 if (filePath === "/") {
215 return createDirectoryQuery.getNonNull({
216 path: filePath,
217 parentId,
218 }).id;
219 }
220 // walk up the path until we find a directory that exists
221 do {
222 parts.unshift(path.basename(current));
223 current = path.dirname(current);
224 parentId = getDirectoryIdQuery.get(current)?.id ?? null;
225 } while (parentId == undefined && current !== "/");
226 if (parentId == undefined) {
227 parentId = createDirectoryQuery.getNonNull({
228 path: current,
229 parentId,
230 }).id;
231 }
232 // walk back down the path, creating directories as needed
233 for (const part of parts) {
234 current = path.join(current, part);
235 ASSERT(parentId != undefined);
236 parentId = createDirectoryQuery.getNonNull({
237 path: current,
238 parentId,
239 }).id;
240 }
241 return parentId;
242 }
243 static markDirectoryProcessed({
244 id,
245 timestamp,
246 contents,
247 size,
248 hash,
249 dirsort,
250 }: MarkDirectoryProcessed) {
251 markDirectoryProcessedQuery.get({
252 id,
253 timestamp: timestamp.getTime(),
254 contents,
255 dirsort: dirsort ? JSON.stringify(dirsort) : "",
256 hash,
257 size,
258 });
259 }
260 static setProcessed(id: number, processed: number) {
261 setProcessedQuery.run({ id, processed });
262 }
263 static createOrUpdateDirectory(dirPath: string) {
264 const id = MediaFile.getOrPutDirectoryId(dirPath);
265 return updateDirectoryQuery.get(id);
266 }
267 static getChildren(id: number) {
268 return getChildrenQuery.array(id);
269 }
270 static db = db;
271}
272
273// Create a `file` entry with a given path, date, file hash, size, and duration
274// If the file already exists, update the date and duration.
275// If the file exists and the hash is different, sets `compress` to 0.
276interface CreateFile {
277 path: string;
278 date: Date;
279 hash: string;
280 size: number;
281 duration: number;
282 dimensions: string;
283 contents: string;
284}
285
286// Set the `processed` flag true and update the metadata for a directory
287export interface MarkDirectoryProcessed {
288 id: number;
289 timestamp: Date;
290 contents: string;
291 size: number;
292 hash: string;
293 dirsort: null | string[];
294}
295
296export interface DirConfig {
297 /** Overridden sorting */
298 sort: string[];
299}
300
301// -- queries --
302
303// Get a directory ID by path, creating it if it doesn't exist
304const createDirectoryQuery = db.prepare<
305 [{ path: string; parentId: number | null }],
306 { id: number }
307>(
308 /* SQL */ `
309 insert into media_files (
310 path, parent_id, kind, timestamp, hash, size,
311 duration, dimensions, contents, dirsort, processed)
312 values (
313 $path, $parentId, ${MediaFileKind.directory}, 0, '', 0,
314 0, '', '', '', 0)
315 returning id;
316`,
317);
318const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ `
319 SELECT id FROM media_files WHERE path = ? AND kind = ${MediaFileKind.directory};
320`);
321const createFileQuery = db.prepare<[{
322 path: string;
323 parentId: number;
324 timestamp: number;
325 timestampUpdated: number;
326 hash: string;
327 size: number;
328 duration: number;
329 dimensions: string;
330 contents: string;
331}], void>(/* SQL */ `
332 insert into media_files (
333 path, parent_id, kind, timestamp, timestamp_updated, hash,
334 size, duration, dimensions, contents, processed)
335 values (
336 $path, $parentId, ${MediaFileKind.file}, $timestamp, $timestampUpdated,
337 $hash, $size, $duration, $dimensions, $contents, 0)
338 on conflict(path) do update set
339 timestamp = excluded.timestamp,
340 timestamp_updated = excluded.timestamp_updated,
341 duration = excluded.duration,
342 size = excluded.size,
343 contents = excluded.contents,
344 processed = case
345 when media_files.hash != excluded.hash then 0
346 else media_files.processed
347 end
348 returning *;
349`).as(MediaFile);
350const setProcessedQuery = db.prepare<[{
351 id: number;
352 processed: number;
353}]>(/* SQL */ `
354 update media_files set processed = $processed where id = $id;
355`);
356const setProcessorsQuery = db.prepare<[{
357 id: number;
358 processed: number;
359 processors: string;
360}]>(/* SQL */ `
361 update media_files set
362 processed = $processed,
363 processors = $processors
364 where id = $id;
365`);
366const setDurationQuery = db.prepare<[{
367 id: number;
368 duration: number;
369}]>(/* SQL */ `
370 update media_files set duration = $duration where id = $id;
371`);
372const setDimensionsQuery = db.prepare<[{
373 id: number;
374 dimensions: string;
375}]>(/* SQL */ `
376 update media_files set dimensions = $dimensions where id = $id;
377`);
378const setContentsQuery = db.prepare<[{
379 id: number;
380 contents: string;
381}]>(/* SQL */ `
382 update media_files set contents = $contents where id = $id;
383`);
384const getByPathQuery = db.prepare<[string]>(/* SQL */ `
385 select * from media_files where path = ?;
386`).as(MediaFile);
387const markDirectoryProcessedQuery = db.prepare<[{
388 timestamp: number;
389 contents: string;
390 dirsort: string;
391 hash: string;
392 size: number;
393 id: number;
394}]>(/* SQL */ `
395 update media_files set
396 processed = 1,
397 timestamp = $timestamp,
398 contents = $contents,
399 dirsort = $dirsort,
400 hash = $hash,
401 size = $size
402 where id = $id;
403`);
404const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ `
405 update media_files set processed = 0 where id = ?;
406`);
407
408const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ `
409 select * from media_files where parent_id = ?;
410`).as(MediaFile);
411const getChildrenFilesRecursiveQuery = db.prepare<[dir: string]>(/* SQL */ `
412 select * from media_files
413 where path like ? || '%'
414 and kind = ${MediaFileKind.file}
415`).as(MediaFile);
416const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
417 with recursive items as (
418 select id, parent_id from media_files where id = $id
419 union all
420 select p.id, p.parent_id
421 from media_files p
422 join items c on p.id = c.parent_id
423 where p.parent_id is not null
424 and not exists (
425 select 1 from media_files child
426 where child.parent_id = p.id
427 and child.id <> c.id
428 )
429 )
430 delete from media_files
431 where id in (select id from items)
432`);
433
434import { getDb } from "#sitegen/sqlite";
435import * as path from "node:path/posix";
436import { FilePermissions } from "./FilePermissions.ts";
1const db = getDb("cache.sqlite");
2db.table(
3 "media_files",
4 /* SQL */ `
5 create table media_files (
6 id integer primary key autoincrement,
7 parent_id integer,
8 path text unique,
9 kind integer not null,
10 timestamp integer not null,
11 timestamp_updated integer not null default current_timestamp,
12 hash text not null,
13 size integer not null,
14 duration integer not null default 0,
15 dimensions text not null default "",
16 contents text not null,
17 dirsort text,
18 processed integer not null,
19 processors text not null default "",
20 foreign key (parent_id) references media_files(id)
21 );
22 -- index for quickly looking up files by path
23 create index media_files_path on media_files (path);
24 -- index for quickly looking up children
25 create index media_files_parent_id on media_files (parent_id);
26 -- index for quickly looking up recursive file children
27 create index media_files_file_children on media_files (kind, path);
28 -- index for finding directories that need to be processed
29 create index media_files_directory_processed on media_files (kind, processed);
30`,
31);
32
33export enum MediaFileKind {
34 directory = 0,
35 file = 1,
36}
37export class MediaFile {
38 id!: number;
39 parent_id!: number | null;
40 /**
41 * Has leading slash, does not have `/file` prefix.
42 * @example "/2025/waterfalls/waterfalls.mp3"
43 */
44 path!: string;
45 kind!: MediaFileKind;
46 private timestamp!: number;
47 private timestamp_updated!: number;
48 /** for mp3/mp4 files, measured in seconds */
49 duration?: number;
50 /** for images and videos, the dimensions. Two numbers split by `x` */
51 dimensions?: string;
52 /**
53 * sha1 of
54 * - files: the contents
55 * - directories: the JSON array of strings + the content of `readme.txt`
56 * this is used
57 * - to inform changes in caching mechanisms (etag, page render cache)
58 * - as a filename for compressed files (.clover/compressed/<hash>.{gz,zstd})
59 */
60 hash!: string;
61 /**
62 * Depends on the file kind.
63 *
64 * - For directories, this is the contents of `readme.txt`, if it exists.
65 * - Otherwise, it is an empty string.
66 */
67 contents!: string;
68 /**
69 * For directories, if this is set, it is a JSON-encoded array of the explicit
70 * sorting order. Derived off of `.dirsort` files.
71 */
72 dirsort!: string | null;
73 /** in bytes */
74 size!: number;
75 /**
76 * 0 - not processed
77 * non-zero - processed
78 *
79 * file: a bit-field of the processors.
80 * directory: this is for re-indexing contents
81 */
82 processed!: number;
83 processors!: string;
84
85 // -- instance ops --
86 get date() {
87 return new Date(this.timestamp);
88 }
89 get lastUpdateDate() {
90 return new Date(this.timestamp_updated);
91 }
92 parseDimensions() {
93 const dimensions = this.dimensions;
94 if (!dimensions) return null;
95 const [width, height] = dimensions.split("x").map(Number);
96 return { width, height };
97 }
98 get basename() {
99 return path.basename(this.path);
100 }
101 get basenameWithoutExt() {
102 return path.basename(this.path, path.extname(this.path));
103 }
104 get extension() {
105 return path.extname(this.path);
106 }
107 getChildren() {
108 return MediaFile.getChildren(this.id)
109 .filter((file) => !file.basename.startsWith("."));
110 }
111 getPublicChildren() {
112 const children = MediaFile.getChildren(this.id);
113 if (FilePermissions.getByPrefix(this.path) == 0) {
114 return children.filter(({ path }) => FilePermissions.getExact(path) == 0);
115 }
116 return children;
117 }
118 getParent() {
119 const dirPath = this.path;
120 if (dirPath === "/") return null;
121 const parentPath = path.dirname(dirPath);
122 if (parentPath === dirPath) return null;
123 const result = MediaFile.getByPath(parentPath);
124 if (!result) return null;
125 ASSERT(result.kind === MediaFileKind.directory);
126 return result;
127 }
128 setProcessed(processed: number) {
129 setProcessedQuery.run({ id: this.id, processed });
130 this.processed = processed;
131 }
132 setProcessors(processed: number, processors: string) {
133 setProcessorsQuery.run({ id: this.id, processed, processors });
134 this.processed = processed;
135 this.processors = processors;
136 }
137 setDuration(duration: number) {
138 setDurationQuery.run({ id: this.id, duration });
139 this.duration = duration;
140 }
141 setDimensions(dimensions: string) {
142 setDimensionsQuery.run({ id: this.id, dimensions });
143 this.dimensions = dimensions;
144 }
145 setContents(contents: string) {
146 setContentsQuery.run({ id: this.id, contents });
147 this.contents = contents;
148 }
149 getRecursiveFileChildren() {
150 if (this.kind !== MediaFileKind.directory) return [];
151 return getChildrenFilesRecursiveQuery.array(this.path + "/");
152 }
153 delete() {
154 deleteCascadeQuery.run({ id: this.id });
155 }
156
157 // -- static ops --
158 static getByPath(filePath: string): MediaFile | null {
159 const result = getByPathQuery.get(filePath);
160 if (result) return result;
161 if (filePath === "/") {
162 return Object.assign(new MediaFile(), {
163 id: 0,
164 parent_id: null,
165 path: "/",
166 kind: MediaFileKind.directory,
167 timestamp: 0,
168 timestamp_updated: Date.now(),
169 hash: "0".repeat(40),
170 contents: "the file scanner has not been run yet",
171 dirsort: null,
172 size: 0,
173 processed: 1,
174 });
175 }
176 return null;
177 }
178 static createFile({
179 path: filePath,
180 date,
181 hash,
182 size,
183 duration,
184 dimensions,
185 contents,
186 }: CreateFile) {
187 ASSERT(
188 !filePath.includes("\\") && filePath.startsWith("/"),
189 `Invalid path: ${filePath}`,
190 );
191 return createFileQuery.getNonNull({
192 path: filePath,
193 parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)),
194 timestamp: date.getTime(),
195 timestampUpdated: Date.now(),
196 hash,
197 size,
198 duration,
199 dimensions,
200 contents,
201 });
202 }
203 static getOrPutDirectoryId(filePath: string) {
204 ASSERT(
205 !filePath.includes("\\") && filePath.startsWith("/"),
206 `Invalid path: ${filePath}`,
207 );
208 filePath = path.normalize(filePath);
209 const row = getDirectoryIdQuery.get(filePath);
210 if (row) return row.id;
211 let current = filePath;
212 let parts = [];
213 let parentId: null | number = null;
214 if (filePath === "/") {
215 return createDirectoryQuery.getNonNull({
216 path: filePath,
217 parentId,
218 }).id;
219 }
220 // walk up the path until we find a directory that exists
221 do {
222 parts.unshift(path.basename(current));
223 current = path.dirname(current);
224 parentId = getDirectoryIdQuery.get(current)?.id ?? null;
225 } while (parentId == undefined && current !== "/");
226 if (parentId == undefined) {
227 parentId = createDirectoryQuery.getNonNull({
228 path: current,
229 parentId,
230 }).id;
231 }
232 // walk back down the path, creating directories as needed
233 for (const part of parts) {
234 current = path.join(current, part);
235 ASSERT(parentId != undefined);
236 parentId = createDirectoryQuery.getNonNull({
237 path: current,
238 parentId,
239 }).id;
240 }
241 return parentId;
242 }
243 static markDirectoryProcessed({
244 id,
245 timestamp,
246 contents,
247 size,
248 hash,
249 dirsort,
250 }: MarkDirectoryProcessed) {
251 markDirectoryProcessedQuery.get({
252 id,
253 timestamp: timestamp.getTime(),
254 contents,
255 dirsort: dirsort ? JSON.stringify(dirsort) : "",
256 hash,
257 size,
258 });
259 }
260 static setProcessed(id: number, processed: number) {
261 setProcessedQuery.run({ id, processed });
262 }
263 static createOrUpdateDirectory(dirPath: string) {
264 const id = MediaFile.getOrPutDirectoryId(dirPath);
265 return updateDirectoryQuery.get(id);
266 }
267 static getChildren(id: number) {
268 return getChildrenQuery.array(id);
269 }
270 static db = db;
271}
272
273// Create a `file` entry with a given path, date, file hash, size, and duration
274// If the file already exists, update the date and duration.
275// If the file exists and the hash is different, sets `compress` to 0.
276interface CreateFile {
277 path: string;
278 date: Date;
279 hash: string;
280 size: number;
281 duration: number;
282 dimensions: string;
283 contents: string;
284}
285
286// Set the `processed` flag true and update the metadata for a directory
287export interface MarkDirectoryProcessed {
288 id: number;
289 timestamp: Date;
290 contents: string;
291 size: number;
292 hash: string;
293 dirsort: null | string[];
294}
295
296export interface DirConfig {
297 /** Overridden sorting */
298 sort: string[];
299}
300
301// -- queries --
302
303// Get a directory ID by path, creating it if it doesn't exist
304const createDirectoryQuery = db.prepare<
305 [{ path: string; parentId: number | null }],
306 { id: number }
307>(
308 /* SQL */ `
309 insert into media_files (
310 path, parent_id, kind, timestamp, hash, size,
311 duration, dimensions, contents, dirsort, processed)
312 values (
313 $path, $parentId, ${MediaFileKind.directory}, 0, '', 0,
314 0, '', '', '', 0)
315 returning id;
316`,
317);
318const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ `
319 SELECT id FROM media_files WHERE path = ? AND kind = ${MediaFileKind.directory};
320`);
321const createFileQuery = db.prepare<[{
322 path: string;
323 parentId: number;
324 timestamp: number;
325 timestampUpdated: number;
326 hash: string;
327 size: number;
328 duration: number;
329 dimensions: string;
330 contents: string;
331}], void>(/* SQL */ `
332 insert into media_files (
333 path, parent_id, kind, timestamp, timestamp_updated, hash,
334 size, duration, dimensions, contents, processed)
335 values (
336 $path, $parentId, ${MediaFileKind.file}, $timestamp, $timestampUpdated,
337 $hash, $size, $duration, $dimensions, $contents, 0)
338 on conflict(path) do update set
339 timestamp = excluded.timestamp,
340 timestamp_updated = excluded.timestamp_updated,
341 duration = excluded.duration,
342 size = excluded.size,
343 contents = excluded.contents,
344 processed = case
345 when media_files.hash != excluded.hash then 0
346 else media_files.processed
347 end
348 returning *;
349`).as(MediaFile);
350const setProcessedQuery = db.prepare<[{
351 id: number;
352 processed: number;
353}]>(/* SQL */ `
354 update media_files set processed = $processed where id = $id;
355`);
356const setProcessorsQuery = db.prepare<[{
357 id: number;
358 processed: number;
359 processors: string;
360}]>(/* SQL */ `
361 update media_files set
362 processed = $processed,
363 processors = $processors
364 where id = $id;
365`);
366const setDurationQuery = db.prepare<[{
367 id: number;
368 duration: number;
369}]>(/* SQL */ `
370 update media_files set duration = $duration where id = $id;
371`);
372const setDimensionsQuery = db.prepare<[{
373 id: number;
374 dimensions: string;
375}]>(/* SQL */ `
376 update media_files set dimensions = $dimensions where id = $id;
377`);
378const setContentsQuery = db.prepare<[{
379 id: number;
380 contents: string;
381}]>(/* SQL */ `
382 update media_files set contents = $contents where id = $id;
383`);
384const getByPathQuery = db.prepare<[string]>(/* SQL */ `
385 select * from media_files where path = ?;
386`).as(MediaFile);
387const markDirectoryProcessedQuery = db.prepare<[{
388 timestamp: number;
389 contents: string;
390 dirsort: string;
391 hash: string;
392 size: number;
393 id: number;
394}]>(/* SQL */ `
395 update media_files set
396 processed = 1,
397 timestamp = $timestamp,
398 contents = $contents,
399 dirsort = $dirsort,
400 hash = $hash,
401 size = $size
402 where id = $id;
403`);
404const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ `
405 update media_files set processed = 0 where id = ?;
406`);
407
408const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ `
409 select * from media_files where parent_id = ?;
410`).as(MediaFile);
411const getChildrenFilesRecursiveQuery = db.prepare<[dir: string]>(/* SQL */ `
412 select * from media_files
413 where path like ? || '%'
414 and kind = ${MediaFileKind.file}
415`).as(MediaFile);
416const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
417 with recursive items as (
418 select id, parent_id from media_files where id = $id
419 union all
420 select p.id, p.parent_id
421 from media_files p
422 join items c on p.id = c.parent_id
423 where p.parent_id is not null
424 and not exists (
425 select 1 from media_files child
426 where child.parent_id = p.id
427 and child.id <> c.id
428 )
429 )
430 delete from media_files
431 where id in (select id from items)
432`);
433
434import { getDb } from "#sitegen/sqlite";
435import * as path from "node:path/posix";
436import { FilePermissions } from "./FilePermissions.ts";
src/file-viewer/pages/file.404.tsx+34-34
......@@ -1,34 +1,34 @@
1import { MediaFile } from "../models/MediaFile.ts";
2import { MediaPanel } from "../views/clofi.tsx";
3import { addScript } from "#sitegen";
4
5export const theme = {
6 bg: "#312652",
7 fg: "#f0f0ff",
8 primary: "#fabe32",
9};
10
11export const meta = { title: "file not found" };
12
13export default function CotyledonPage() {
14 addScript("../scripts/canvas_cotyledon.client.ts");
15 return (
16 <div class="files ctld ctld-sb">
17 <MediaPanel
18 file={MediaFile.getByPath("/")!}
19 isLast={false}
20 activeFilename={null}
21 hasCotyledonCookie={false}
22 />
23 <div class="panel last">
24 <div className="header"></div>
25 <div className="content file-view notfound">
26 <p>this file does not exist ...</p>
27 <p>
28 <a href="/file">return</a>
29 </p>
30 </div>
31 </div>
32 </div>
33 );
34}
1import { MediaFile } from "../models/MediaFile.ts";
2import { MediaPanel } from "../views/clofi.tsx";
3import { addScript } from "#sitegen";
4
5export const theme = {
6 bg: "#312652",
7 fg: "#f0f0ff",
8 primary: "#fabe32",
9};
10
11export const meta = { title: "file not found" };
12
13export default function CotyledonPage() {
14 addScript("../scripts/canvas_cotyledon.client.ts");
15 return (
16 <div class="files ctld ctld-sb">
17 <MediaPanel
18 file={MediaFile.getByPath("/")!}
19 isLast={false}
20 activeFilename={null}
21 hasCotyledonCookie={false}
22 />
23 <div class="panel last">
24 <div className="header"></div>
25 <div className="content file-view notfound">
26 <p>this file does not exist ...</p>
27 <p>
28 <a href="/file">return</a>
29 </p>
30 </div>
31 </div>
32 </div>
33 );
34}
src/file-viewer/rules.ts+147-143
......@@ -1,143 +1,147 @@
1// -- file extension rules --
2
3/** Extensions that must have EXIF/etc data stripped */
4export const extScrubExif = new Set([
5 ".jpg",
6 ".jpeg",
7 ".png",
8 ".mov",
9 ".mp4",
10 ".m4a",
11]);
12/** Extensions that rendered syntax-highlighted code */
13export const extsCode = new Map<string, highlight.Language>(Object.entries({
14 ".json": "json",
15 ".toml": "toml",
16 ".ts": "ts",
17 ".js": "ts",
18 ".tsx": "tsx",
19 ".jsx": "tsx",
20 ".css": "css",
21 ".py": "python",
22 ".lua": "lua",
23 ".sh": "shell",
24 ".bat": "dosbatch",
25 ".ps1": "powershell",
26 ".cmd": "dosbatch",
27 ".yaml": "yaml",
28 ".yml": "yaml",
29 ".zig": "zig",
30 ".astro": "astro",
31 ".mdx": "mdx",
32 ".xml": "xml",
33 ".jsonc": "json",
34 ".php": "php",
35 ".patch": "diff",
36 ".diff": "diff",
37}));
38/** These files show an audio embed. */
39export const extsAudio = new Set([
40 ".mp3",
41 ".flac",
42 ".wav",
43 ".ogg",
44 ".m4a",
45]);
46/** These files show a video embed. */
47export const extsVideo = new Set([
48 ".mp4",
49 ".mkv",
50 ".webm",
51 ".avi",
52 ".mov",
53]);
54/** These files show an image embed */
55export const extsImage = new Set([
56 ".jpg",
57 ".jpeg",
58 ".png",
59 ".gif",
60 ".webp",
61 ".avif",
62 ".heic",
63 ".svg",
64]);
65
66/** These files populate `duration` using `ffprobe` */
67export const extsDuration = new Set([...extsAudio, ...extsVideo]);
68/** These files populate `dimensions` using `ffprobe` */
69export const extsDimensions = new Set([...extsImage, ...extsVideo]);
70
71/** These files read file contents into `contents`, as-is */
72export const extsReadContents = new Set([".txt", ".chat"]);
73
74export const extsArchive = new Set([
75 ".zip",
76 ".rar",
77 ".7z",
78 ".tar",
79 ".gz",
80 ".bz2",
81 ".xz",
82]);
83
84/**
85 * Formats which are already compression formats, meaning a pass
86 * through zstd would offer little to negative benefits
87 */
88export const extsPreCompressed = new Set([
89 ...extsAudio,
90 ...extsVideo,
91 ...extsImage,
92 ...extsArchive,
93 // TODO: are any of these NOT good for compression
94]);
95
96export function fileIcon(
97 file: Pick<MediaFile, "kind" | "basename" | "path">,
98 dirOpen?: boolean,
99) {
100 const { kind, basename } = file;
101 if (kind === MediaFileKind.directory) return dirOpen ? "dir-open" : "dir";
102
103 // -- special cases --
104 if (file.path === "/2024/for everyone") return "snow";
105
106 // -- basename cases --
107 if (basename === "readme.txt") return "readme";
108
109 // -- extension cases --
110 const ext = path.extname(file.basename).toLowerCase();
111 if ([".comp", ".fuse", ".setting"].includes(ext)) return "fusion";
112 if ([".json", ".toml", ".yaml", ".yml"].includes(ext)) return "json";
113 if (ext === ".blend") return "blend";
114 if (ext === ".chat") return "chat";
115 if (ext === ".html") return "webpage";
116 if (ext === ".lnk") return "link";
117 if (ext === ".txt" || ext === ".md") return "text";
118
119 // -- extension categories --
120 if (extsVideo.has(ext)) return "video";
121 if (extsAudio.has(ext)) return "audio";
122 if (extsImage.has(ext)) return "image";
123 if (extsArchive.has(ext)) return "archive";
124 if (extsCode.has(ext)) return "code";
125
126 return "file";
127}
128
129// -- viewer rules --
130const pathToCanvas = new Map<string, string>(Object.entries({
131 "/2017": "2017",
132 "/2018": "2018",
133 "/2019": "2019",
134 "/2020": "2020",
135 "/2021": "2021",
136 "/2022": "2022",
137 "/2023": "2023",
138 "/2024": "2024",
139}));
140
141import type * as highlight from "./highlight.ts";
142import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
143import * as path from "node:path";
1// -- file extension rules --
2
3/** Extensions that must have EXIF/etc data stripped */
4export const extScrubExif = new Set([
5 ".jpg",
6 ".jpeg",
7 ".png",
8 ".mov",
9 ".mp4",
10 ".m4a",
11]);
12/** Extensions that rendered syntax-highlighted code */
13export const extsCode = new Map<string, highlight.Language>(Object.entries({
14 ".json": "json",
15 ".toml": "toml",
16 ".ts": "ts",
17 ".js": "ts",
18 ".tsx": "tsx",
19 ".jsx": "tsx",
20 ".css": "css",
21 ".py": "python",
22 ".lua": "lua",
23 ".sh": "shell",
24 ".bat": "dosbatch",
25 ".ps1": "powershell",
26 ".cmd": "dosbatch",
27 ".yaml": "yaml",
28 ".yml": "yaml",
29 ".zig": "zig",
30 ".astro": "astro",
31 ".mdx": "mdx",
32 ".xml": "xml",
33 ".jsonc": "json",
34 ".php": "php",
35 ".patch": "diff",
36 ".diff": "diff",
37}));
38/** These files show an audio embed. */
39export const extsAudio = new Set([
40 ".mp3",
41 ".flac",
42 ".wav",
43 ".ogg",
44 ".m4a",
45]);
46/** These files show a video embed. */
47export const extsVideo = new Set([
48 ".mp4",
49 ".mkv",
50 ".webm",
51 ".avi",
52 ".mov",
53]);
54/** These files show an image embed */
55export const extsImage = new Set([
56 ".jpg",
57 ".jpeg",
58 ".png",
59 ".webp",
60 ".avif",
61 ".heic",
62]);
63/** These files show an image embed, but aren't optimized */
64export const extsImageLike = new Set([
65 ...extsImage,
66 ".svg",
67 ".gif",
68]);
69
70/** These files populate `duration` using `ffprobe` */
71export const extsDuration = new Set([...extsAudio, ...extsVideo]);
72/** These files populate `dimensions` using `ffprobe` */
73export const extsDimensions = new Set([...extsImage, ...extsVideo]);
74
75/** These files read file contents into `contents`, as-is */
76export const extsReadContents = new Set([".txt", ".chat"]);
77
78export const extsArchive = new Set([
79 ".zip",
80 ".rar",
81 ".7z",
82 ".tar",
83 ".gz",
84 ".bz2",
85 ".xz",
86]);
87
88/**
89 * Formats which are already compression formats, meaning a pass
90 * through zstd would offer little to negative benefits
91 */
92export const extsPreCompressed = new Set([
93 ...extsAudio,
94 ...extsVideo,
95 ...extsImage,
96 ...extsArchive,
97 // TODO: are any of these NOT good for compression
98]);
99
100export function fileIcon(
101 file: Pick<MediaFile, "kind" | "basename" | "path">,
102 dirOpen?: boolean,
103) {
104 const { kind, basename } = file;
105 if (kind === MediaFileKind.directory) return dirOpen ? "dir-open" : "dir";
106
107 // -- special cases --
108 if (file.path === "/2024/for everyone") return "snow";
109
110 // -- basename cases --
111 if (basename === "readme.txt") return "readme";
112
113 // -- extension cases --
114 const ext = path.extname(file.basename).toLowerCase();
115 if ([".comp", ".fuse", ".setting"].includes(ext)) return "fusion";
116 if ([".json", ".toml", ".yaml", ".yml"].includes(ext)) return "json";
117 if (ext === ".blend") return "blend";
118 if (ext === ".chat") return "chat";
119 if (ext === ".html") return "webpage";
120 if (ext === ".lnk") return "link";
121 if (ext === ".txt" || ext === ".md") return "text";
122
123 // -- extension categories --
124 if (extsVideo.has(ext)) return "video";
125 if (extsAudio.has(ext)) return "audio";
126 if (extsImage.has(ext)) return "image";
127 if (extsArchive.has(ext)) return "archive";
128 if (extsCode.has(ext)) return "code";
129
130 return "file";
131}
132
133// -- viewer rules --
134const pathToCanvas = new Map<string, string>(Object.entries({
135 "/2017": "2017",
136 "/2018": "2018",
137 "/2019": "2019",
138 "/2020": "2020",
139 "/2021": "2021",
140 "/2022": "2022",
141 "/2023": "2023",
142 "/2024": "2024",
143}));
144
145import type * as highlight from "./highlight.ts";
146import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
147import * as path from "node:path";
src/file-viewer/sort.ts+58-58
......@@ -1,58 +1,58 @@
1export function splitRootDirFiles(dir: MediaFile, hasCotyledonCookie: boolean) {
2 const children = dir.getPublicChildren();
3 let readme: MediaFile | null = null;
4
5 const groups = {
6 // years 2025 and onwards
7 years: [] as MediaFile[],
8 // named categories
9 categories: [] as MediaFile[],
10 // years 2017 to 2024
11 cotyledon: [] as MediaFile[],
12 };
13 const colorMap: Record<string, string> = {
14 years: "#a2ff91",
15 categories: "#9c91ff",
16 cotyledon: "#ff91ca",
17 };
18 for (const child of children) {
19 const basename = child.basename;
20 if (basename === "readme.txt") {
21 readme = child;
22 continue;
23 }
24
25 const year = basename.match(/^(\d{4})/);
26 if (year) {
27 const n = parseInt(year[1]);
28 if (n >= 2025) {
29 groups.years.push(child);
30 } else {
31 groups.cotyledon.push(child);
32 }
33 } else {
34 groups.categories.push(child);
35 }
36 }
37
38 let sections = [];
39 for (const [key, files] of Object.entries(groups)) {
40 if (key === "cotyledon" && !hasCotyledonCookie) {
41 continue;
42 }
43 if (key === "years" || key === "cotyledon") {
44 files.sort((a, b) => {
45 return b.basename.localeCompare(a.basename);
46 });
47 } else {
48 files.sort((a, b) => {
49 return a.basename.localeCompare(b.basename);
50 });
51 }
52 sections.push({ key, titleColor: colorMap[key], files });
53 }
54
55 return { readme, sections };
56}
57
58import { MediaFile } from "./models/MediaFile.ts";
1export function splitRootDirFiles(dir: MediaFile, hasCotyledonCookie: boolean) {
2 const children = dir.getPublicChildren();
3 let readme: MediaFile | null = null;
4
5 const groups = {
6 // years 2025 and onwards
7 years: [] as MediaFile[],
8 // named categories
9 categories: [] as MediaFile[],
10 // years 2017 to 2024
11 cotyledon: [] as MediaFile[],
12 };
13 const colorMap: Record<string, string> = {
14 years: "#a2ff91",
15 categories: "#9c91ff",
16 cotyledon: "#ff91ca",
17 };
18 for (const child of children) {
19 const basename = child.basename;
20 if (basename === "readme.txt") {
21 readme = child;
22 continue;
23 }
24
25 const year = basename.match(/^(\d{4})/);
26 if (year) {
27 const n = parseInt(year[1]);
28 if (n >= 2025) {
29 groups.years.push(child);
30 } else {
31 groups.cotyledon.push(child);
32 }
33 } else {
34 groups.categories.push(child);
35 }
36 }
37
38 let sections = [];
39 for (const [key, files] of Object.entries(groups)) {
40 if (key === "cotyledon" && !hasCotyledonCookie) {
41 continue;
42 }
43 if (key === "years" || key === "cotyledon") {
44 files.sort((a, b) => {
45 return b.basename.localeCompare(a.basename);
46 });
47 } else {
48 files.sort((a, b) => {
49 return a.basename.localeCompare(b.basename);
50 });
51 }
52 sections.push({ key, titleColor: colorMap[key], files });
53 }
54
55 return { readme, sections };
56}
57
58import { MediaFile } from "./models/MediaFile.ts";
src/file-viewer/transcode-rules.ts+27-3
......@@ -99,14 +99,38 @@ export const imagePresets = [
9999 "6",
100100 ],
101101 },
102 // TODO: avif
102 {
103 ext: ".avif",
104 args: [
105 "-c:v",
106 "libaom-av1",
107 "-crf",
108 "30",
109 "-pix_fmt",
110 "yuv420p10le",
111 ],
112 },
103113 {
104114 ext: ".jxl",
105 args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
115 args: [
116 "-c:v",
117 "libjxl",
118 "-distance",
119 "0.8",
120 "-effort",
121 "9",
122 "-update",
123 "-frames:v",
124 "1",
125 ],
106126 },
107127];
108128
109export function getVideoArgs(preset: VideoEncodePreset, outbase: string, input: string[]) {
129export function getVideoArgs(
130 preset: VideoEncodePreset,
131 outbase: string,
132 input: string[],
133) {
110134 const cmd = [...input];
111135
112136 if (preset.codec === "av1") {
src/file-viewer/views/lofi.css+43-43
......@@ -1,43 +1,43 @@
1body {
2 margin: 0;
3 padding: 0;
4}
5#lofi {
6 padding: 32px;
7}
8h1 {
9 margin-top: 0;
10 font-size: 3em;
11 color: var(--primary);
12 font-family: monospace;
13}
14ul, li {
15 margin: 0;
16 padding: 0;
17 list-style-type: none;
18}
19ul {
20 padding-right: 4em;
21}
22li a {
23 display: block;
24 color: white;
25 line-height: 2em;
26 padding: 0 1em;
27 border-radius: 4px;
28}
29li a:hover {
30 background-color: rgba(255,255,255,0.2);
31 font-weight: bold;
32 text-decoration: none!important;
33}
34.dir a {
35 color: #99eeFF
36}
37.ext {
38 opacity: 0.5;
39}
40.meta {
41 margin-left: 1em;
42 opacity: 0.75;
43}
1body {
2 margin: 0;
3 padding: 0;
4}
5#lofi {
6 padding: 32px;
7}
8h1 {
9 margin-top: 0;
10 font-size: 3em;
11 color: var(--primary);
12 font-family: monospace;
13}
14ul, li {
15 margin: 0;
16 padding: 0;
17 list-style-type: none;
18}
19ul {
20 padding-right: 4em;
21}
22li a {
23 display: block;
24 color: white;
25 line-height: 2em;
26 padding: 0 1em;
27 border-radius: 4px;
28}
29li a:hover {
30 background-color: rgba(255, 255, 255, 0.2);
31 font-weight: bold;
32 text-decoration: none !important;
33}
34.dir a {
35 color: #99eeff;
36}
37.ext {
38 opacity: 0.5;
39}
40.meta {
41 margin-left: 1em;
42 opacity: 0.75;
43}
src/friend-auth.ts+75-75
......@@ -1,75 +1,75 @@
1let friendPassword = "";
2try {
3 friendPassword = require("./friends/hardcoded-password.ts").friendPassword;
4} catch {}
5
6export const app = new Hono();
7
8const cookieAge = 60 * 60 * 24 * 30; // 1 month
9
10function checkFriendsCookie(c: Context) {
11 const cookie = c.req.header("Cookie");
12 if (!cookie) return false;
13 const cookies = cookie.split("; ").map((x) => x.split("="));
14 return cookies.some(
15 (kv) =>
16 kv[0].trim() === "friends_password" &&
17 kv[1].trim() &&
18 kv[1].trim() === friendPassword,
19 );
20}
21
22export function requireFriendAuth(c: Context) {
23 const k = c.req.query("password") || c.req.query("k");
24 if (k) {
25 if (k === friendPassword) {
26 return c.body(null, 303, {
27 Location: "/friends",
28 "Set-Cookie":
29 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
30 });
31 } else {
32 return c.body(null, 303, {
33 Location: "/friends",
34 });
35 }
36 }
37 if (checkFriendsCookie(c)) {
38 return undefined;
39 } else {
40 return serveAsset(c, "/friends/auth", 403);
41 }
42}
43
44app.get("/friends", (c) => {
45 const friendAuthChallenge = requireFriendAuth(c);
46 if (friendAuthChallenge) return friendAuthChallenge;
47 return serveAsset(c, "/friends", 200);
48});
49
50let incorrectMap: Record<string, boolean> = {};
51app.post("/friends", async (c) => {
52 const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??
53 "unknown";
54 if (incorrectMap[ip]) {
55 return serveAsset(c, "/friends/auth/fail", 403);
56 }
57 const data = await c.req.formData();
58 const k = data.get("password");
59 if (k === friendPassword) {
60 return c.body(null, 303, {
61 Location: "/friends",
62 "Set-Cookie":
63 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
64 });
65 }
66 incorrectMap[ip] = true;
67 await setTimeout(2500);
68 incorrectMap[ip] = false;
69 return serveAsset(c, "/friends/auth/fail", 403);
70});
71
72import { type Context, Hono } from "hono";
73import { serveAsset } from "#sitegen/assets";
74import { setTimeout } from "node:timers/promises";
75import { getConnInfo } from "#hono/conninfo";
1let friendPassword = "";
2try {
3 friendPassword = require("./friends/hardcoded-password.ts").friendPassword;
4} catch {}
5
6export const app = new Hono();
7
8const cookieAge = 60 * 60 * 24 * 30; // 1 month
9
10function checkFriendsCookie(c: Context) {
11 const cookie = c.req.header("Cookie");
12 if (!cookie) return false;
13 const cookies = cookie.split("; ").map((x) => x.split("="));
14 return cookies.some(
15 (kv) =>
16 kv[0].trim() === "friends_password" &&
17 kv[1].trim() &&
18 kv[1].trim() === friendPassword,
19 );
20}
21
22export function requireFriendAuth(c: Context) {
23 const k = c.req.query("password") || c.req.query("k");
24 if (k) {
25 if (k === friendPassword) {
26 return c.body(null, 303, {
27 Location: "/friends",
28 "Set-Cookie":
29 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
30 });
31 } else {
32 return c.body(null, 303, {
33 Location: "/friends",
34 });
35 }
36 }
37 if (checkFriendsCookie(c)) {
38 return undefined;
39 } else {
40 return serveAsset(c, "/friends/auth", 403);
41 }
42}
43
44app.get("/friends", (c) => {
45 const friendAuthChallenge = requireFriendAuth(c);
46 if (friendAuthChallenge) return friendAuthChallenge;
47 return serveAsset(c, "/friends", 200);
48});
49
50let incorrectMap: Record<string, boolean> = {};
51app.post("/friends", async (c) => {
52 const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ??
53 "unknown";
54 if (incorrectMap[ip]) {
55 return serveAsset(c, "/friends/auth/fail", 403);
56 }
57 const data = await c.req.formData();
58 const k = data.get("password");
59 if (k === friendPassword) {
60 return c.body(null, 303, {
61 Location: "/friends",
62 "Set-Cookie":
63 `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
64 });
65 }
66 incorrectMap[ip] = true;
67 await setTimeout(2500);
68 incorrectMap[ip] = false;
69 return serveAsset(c, "/friends/auth/fail", 403);
70});
71
72import { type Context, Hono } from "hono";
73import { serveAsset } from "#sitegen/assets";
74import { setTimeout } from "node:timers/promises";
75import { getConnInfo } from "#hono/conninfo";
src/global.css-1
......@@ -118,4 +118,3 @@ code {
118118 font-family: "rmo", monospace;
119119 font-size: inherit;
120120}
121
src/pages/resume.css+47-47
......@@ -1,47 +1,47 @@
1body,html {
2 overflow: hidden;
3}
4h1 {
5 color: #f09;
6 margin-bottom: 0;
7}
8.job {
9 padding: 18px;
10 margin: 1em -18px;
11 border: 1px solid black;
12}
13.job *, footer * {
14 margin: 0;
15 padding: 0;
16}
17.job ul {
18 margin-left: 1em;
19}
20.job li {
21 line-height: 1.5em;
22}
23.job header, footer {
24 display: grid;
25 grid-template-columns: auto max-content;
26 grid-template-rows: 1fr 1fr;
27}
28footer {
29 margin-top: 1.5em;
30}
31footer h2 {
32 font-size: 1em;
33 margin-bottom: 0.5em;
34}
35
36.job header > em, footer > em {
37 margin-top: 2px;
38 font-size: 1.25em;
39}
40
41header h2, header em, footer h2, footer em {
42 display: inline-block;
43}
44 header em, footer em {
45 margin-left: 16px!important;
46 text-align: right;
47}
1body, html {
2 overflow: hidden;
3}
4h1 {
5 color: #f09;
6 margin-bottom: 0;
7}
8.job {
9 padding: 18px;
10 margin: 1em -18px;
11 border: 1px solid black;
12}
13.job *, footer * {
14 margin: 0;
15 padding: 0;
16}
17.job ul {
18 margin-left: 1em;
19}
20.job li {
21 line-height: 1.5em;
22}
23.job header, footer {
24 display: grid;
25 grid-template-columns: auto max-content;
26 grid-template-rows: 1fr 1fr;
27}
28footer {
29 margin-top: 1.5em;
30}
31footer h2 {
32 font-size: 1em;
33 margin-bottom: 0.5em;
34}
35
36.job header > em, footer > em {
37 margin-top: 2px;
38 font-size: 1.25em;
39}
40
41header h2, header em, footer h2, footer em {
42 display: inline-block;
43}
44header em, footer em {
45 margin-left: 16px !important;
46 text-align: right;
47}
src/pages/subscribe.client.ts+97-97
......@@ -1,97 +1,97 @@
1// @ts-nocheck
2// manually obfuscated to make it very difficult to reverse engineer
3// if you want to decode what the email is, visit the page!
4// stops people from automatically scraping the email address
5//
6// Unfortunately this needs a rewrite to support Chrome without
7// hardware acceleration and some Linux stuff. I will probably
8// go with a proof of work alternative.
9requestAnimationFrame(() => {
10 const hash = "SHA";
11 const a = [
12 { parentElement: document.getElementById("subscribe") },
13 function (b) {
14 let c = 0, d = 0;
15 for (let i = 0; i < b.length; i++) {
16 c = (c + b[i] ^ 0xF8) % 8;
17 d = (c * b[i] ^ 0x82) % 193;
18 }
19 a[c + 1]()[c](d, b.buffer);
20 },
21 function () {
22 const i = a[4](a[3]());
23 const b = i.innerText = a.pop();
24 if (a[b.indexOf("@") / 3]) {
25 i.href = "mailto:" + b;
26 }
27 },
28 function () {
29 return a[a.length % 10];
30 },
31 function (x) {
32 return x.parentElement;
33 },
34 function (b, c) {
35 throw new Uint8Array(
36 c,
37 0,
38 64,
39 c.parentElement = this[8].call(b.call(this)).location,
40 );
41 },
42 function (b, c) {
43 this.width = 8;
44 this.height = 16;
45 b.clearColor(0.5, 0.7, 0.9, 1.0);
46 b.clear(16408 ^ this.width ^ this.height);
47 const e = new Uint8Array(4 * this.width * this.height);
48 b.readPixels(0, 0, this.width, this.height, b.RGBA, b.UNSIGNED_BYTE, e);
49 let parent = a[this.width / 2](this);
50 while (parent.tagName !== "BODY") {
51 parent = a[2 * this.height / this.width](parent);
52 }
53 try {
54 let d = [hash, e.length].join("-");
55 const b = a[0.25 * this.height](a.pop()(parent)).subtle.digest(d, e);
56 [, d] = a;
57 b.then(c.bind(a, a[this.width].bind(globalThis))).catch(d);
58 } catch (e) {
59 fetch(e).then(a[5]).catch(a[2]);
60 }
61 },
62 function (b, c) {
63 const d = a.splice(
64 9,
65 1,
66 [
67 a[3]().parentElement.id,
68 c.parentElement.hostname,
69 ].join(String.fromCharCode(b)),
70 );
71 var e = new Error();
72 Object.defineProperty(e, "stack", {
73 get() {
74 a[9] = d;
75 },
76 });
77 a[2].call(console.log(e));
78 },
79 function () {
80 return this;
81 },
82 "[failed to verify your browser]",
83 function (a) {
84 a = a.parentElement.ownerDocument.defaultView;
85 return { parentElement: a.navigator.webdriver || a.crypto };
86 },
87 ];
88 try {
89 const c = document.querySelector("canvas");
90 const g = c.getContext("webgl2") || c.getContext("webgl");
91 a[0].parentElement.innerText = "[...loading...]";
92 g.field || requestAnimationFrame(a[6].bind(c, g, a[5]));
93 } catch {
94 a.pop();
95 fetch(":").then(a[5]).catch(a[2]);
96 }
97});
1// @ts-nocheck
2// manually obfuscated to make it very difficult to reverse engineer
3// if you want to decode what the email is, visit the page!
4// stops people from automatically scraping the email address
5//
6// Unfortunately this needs a rewrite to support Chrome without
7// hardware acceleration and some Linux stuff. I will probably
8// go with a proof of work alternative.
9requestAnimationFrame(() => {
10 const hash = "SHA";
11 const a = [
12 { parentElement: document.getElementById("subscribe") },
13 function (b) {
14 let c = 0, d = 0;
15 for (let i = 0; i < b.length; i++) {
16 c = (c + b[i] ^ 0xF8) % 8;
17 d = (c * b[i] ^ 0x82) % 193;
18 }
19 a[c + 1]()[c](d, b.buffer);
20 },
21 function () {
22 const i = a[4](a[3]());
23 const b = i.innerText = a.pop();
24 if (a[b.indexOf("@") / 3]) {
25 i.href = "mailto:" + b;
26 }
27 },
28 function () {
29 return a[a.length % 10];
30 },
31 function (x) {
32 return x.parentElement;
33 },
34 function (b, c) {
35 throw new Uint8Array(
36 c,
37 0,
38 64,
39 c.parentElement = this[8].call(b.call(this)).location,
40 );
41 },
42 function (b, c) {
43 this.width = 8;
44 this.height = 16;
45 b.clearColor(0.5, 0.7, 0.9, 1.0);
46 b.clear(16408 ^ this.width ^ this.height);
47 const e = new Uint8Array(4 * this.width * this.height);
48 b.readPixels(0, 0, this.width, this.height, b.RGBA, b.UNSIGNED_BYTE, e);
49 let parent = a[this.width / 2](this);
50 while (parent.tagName !== "BODY") {
51 parent = a[2 * this.height / this.width](parent);
52 }
53 try {
54 let d = [hash, e.length].join("-");
55 const b = a[0.25 * this.height](a.pop()(parent)).subtle.digest(d, e);
56 [, d] = a;
57 b.then(c.bind(a, a[this.width].bind(globalThis))).catch(d);
58 } catch (e) {
59 fetch(e).then(a[5]).catch(a[2]);
60 }
61 },
62 function (b, c) {
63 const d = a.splice(
64 9,
65 1,
66 [
67 a[3]().parentElement.id,
68 c.parentElement.hostname,
69 ].join(String.fromCharCode(b)),
70 );
71 var e = new Error();
72 Object.defineProperty(e, "stack", {
73 get() {
74 a[9] = d;
75 },
76 });
77 a[2].call(console.log(e));
78 },
79 function () {
80 return this;
81 },
82 "[failed to verify your browser]",
83 function (a) {
84 a = a.parentElement.ownerDocument.defaultView;
85 return { parentElement: a.navigator.webdriver || a.crypto };
86 },
87 ];
88 try {
89 const c = document.querySelector("canvas");
90 const g = c.getContext("webgl2") || c.getContext("webgl");
91 a[0].parentElement.innerText = "[...loading...]";
92 g.field || requestAnimationFrame(a[6].bind(c, g, a[5]));
93 } catch {
94 a.pop();
95 fetch(":").then(a[5]).catch(a[2]);
96 }
97});
src/q+a/artifacts.ts+89-89
......@@ -1,89 +1,89 @@
1// Artifacts used to be a central system in the old data-driven website.
2// Now, it simply refers to one of these link presets. Every project has
3// one canonical URL, which the questions page can refer to with `@id`.
4type Artifact = [title: string, url: string, type: ArtifactType];
5type ArtifactType = "music" | "game" | "project" | "video";
6export const artifactMap: Record<string, Artifact> = {
7 // 2025
8 "in-the-summer": ["in the summer", "", "music"],
9 waterfalls: ["waterfalls", "/waterfalls", "music"],
10 lolzip: ["lol.zip", "", "project"],
11 "g-is-missing": ["g is missing", "", "music"],
12 "im-18-now": ["i'm 18 now", "", "music"],
13 "programming-comparison": [
14 "thursday programming language comparison",
15 "",
16 "video",
17 ],
18 aaaaaaaaa: ["aaaaaaaaa", "", "music"],
19 "its-snowing": ["it's snowing", "", "video"],
20 // 2023
21 "iphone-15-review": [
22 "iphone 15 review",
23 "/file/2023/iphone%2015%20review/iphone-15-review.mp4",
24 "video",
25 ],
26 // 2022
27 mayday: ["mayday", "/file/2022/mayday/mayday.mp4", "music"],
28 "mystery-of-life": [
29 "mystery of life",
30 "/file/2022/mystery-of-life/mystery-of-life.mp4",
31 "music",
32 ],
33 // 2021
34 "top-10000-bread": [
35 "top 10000 bread",
36 "https://paperclover.net/file/2021/top-10000-bread/output.mp4",
37 "video",
38 ],
39 "phoenix-write-soundtrack": [
40 "Phoenix, WRITE! soundtrack",
41 "/file/2021/phoenix-write/OST",
42 "music",
43 ],
44 "phoenix-write": ["Pheonix, WRITE!", "/file/2021/phoenix-write", "game"],
45 "money-visual-cover": [
46 "money visual cover",
47 "/file/2021/money-visual-cover/money-visual-cover.mp4",
48 "video",
49 ],
50 "i-got-this-thing": [
51 "i got this thing",
52 "/file/2021/i-got-this-thing/i-got-this-thing.mp4",
53 "video",
54 ],
55 // 2020
56 elemental4: ["elemental 4", "/file/2020/elemental4-rewrite", "game"],
57 "elemental-4": ["elemental 4", "/file/2020/elemental4-rewrite", "game"],
58 // 2019
59 "throw-soundtrack": [
60 "throw soundtrack",
61 "/file/2019/throw/soundtrack",
62 "music",
63 ],
64 "elemental-lite": ["elemental lite", "/file/2019/elemental-lite-1.7", "game"],
65 volar: [
66 "volar visual cover",
67 "/file/2019/volar-visual-cover/volar.mp4",
68 "video",
69 ],
70 wpm: [
71 "how to read 500 words per minute",
72 "/file/2019/how-to-read-500-words-per-minute/how-to-read-500-words-per-minute.mp4",
73 "video",
74 ],
75 "dice-roll": [
76 "thursday dice roll",
77 "/file/2019/thursday-dice-roll/thursday-dice-roll.mp4",
78 "video",
79 ],
80 "math-problem": [
81 "thursday math problem",
82 "/file/2019/thursday-math-problem/thursday-math-problem.mp4",
83 "video",
84 ],
85 // 2018
86 // 2017
87 "hatred-island": ["hatred island", "/file/2017/hatred%20island", "game"],
88 "test-video-1": ["test video 1", "/file/2017/test-video1.mp4", "video"],
89};
1// Artifacts used to be a central system in the old data-driven website.
2// Now, it simply refers to one of these link presets. Every project has
3// one canonical URL, which the questions page can refer to with `@id`.
4type Artifact = [title: string, url: string, type: ArtifactType];
5type ArtifactType = "music" | "game" | "project" | "video";
6export const artifactMap: Record<string, Artifact> = {
7 // 2025
8 "in-the-summer": ["in the summer", "", "music"],
9 waterfalls: ["waterfalls", "/waterfalls", "music"],
10 lolzip: ["lol.zip", "", "project"],
11 "g-is-missing": ["g is missing", "", "music"],
12 "im-18-now": ["i'm 18 now", "", "music"],
13 "programming-comparison": [
14 "thursday programming language comparison",
15 "",
16 "video",
17 ],
18 aaaaaaaaa: ["aaaaaaaaa", "", "music"],
19 "its-snowing": ["it's snowing", "", "video"],
20 // 2023
21 "iphone-15-review": [
22 "iphone 15 review",
23 "/file/2023/iphone%2015%20review/iphone-15-review.mp4",
24 "video",
25 ],
26 // 2022
27 mayday: ["mayday", "/file/2022/mayday/mayday.mp4", "music"],
28 "mystery-of-life": [
29 "mystery of life",
30 "/file/2022/mystery-of-life/mystery-of-life.mp4",
31 "music",
32 ],
33 // 2021
34 "top-10000-bread": [
35 "top 10000 bread",
36 "https://paperclover.net/file/2021/top-10000-bread/output.mp4",
37 "video",
38 ],
39 "phoenix-write-soundtrack": [
40 "Phoenix, WRITE! soundtrack",
41 "/file/2021/phoenix-write/OST",
42 "music",
43 ],
44 "phoenix-write": ["Pheonix, WRITE!", "/file/2021/phoenix-write", "game"],
45 "money-visual-cover": [
46 "money visual cover",
47 "/file/2021/money-visual-cover/money-visual-cover.mp4",
48 "video",
49 ],
50 "i-got-this-thing": [
51 "i got this thing",
52 "/file/2021/i-got-this-thing/i-got-this-thing.mp4",
53 "video",
54 ],
55 // 2020
56 elemental4: ["elemental 4", "/file/2020/elemental4-rewrite", "game"],
57 "elemental-4": ["elemental 4", "/file/2020/elemental4-rewrite", "game"],
58 // 2019
59 "throw-soundtrack": [
60 "throw soundtrack",
61 "/file/2019/throw/soundtrack",
62 "music",
63 ],
64 "elemental-lite": ["elemental lite", "/file/2019/elemental-lite-1.7", "game"],
65 volar: [
66 "volar visual cover",
67 "/file/2019/volar-visual-cover/volar.mp4",
68 "video",
69 ],
70 wpm: [
71 "how to read 500 words per minute",
72 "/file/2019/how-to-read-500-words-per-minute/how-to-read-500-words-per-minute.mp4",
73 "video",
74 ],
75 "dice-roll": [
76 "thursday dice roll",
77 "/file/2019/thursday-dice-roll/thursday-dice-roll.mp4",
78 "video",
79 ],
80 "math-problem": [
81 "thursday math problem",
82 "/file/2019/thursday-math-problem/thursday-math-problem.mp4",
83 "video",
84 ],
85 // 2018
86 // 2017
87 "hatred-island": ["hatred island", "/file/2017/hatred%20island", "game"],
88 "test-video-1": ["test video 1", "/file/2017/test-video1.mp4", "video"],
89};
src/q+a/backend.ts+228-228
......@@ -1,228 +1,228 @@
1const PROXYCHECK_API_KEY = process.env.PROXYCHECK_API_KEY;
2
3export const app = new Hono();
4
5// Main page
6app.get("/q+a", async (c) => {
7 if (hasAdminToken(c)) {
8 return serveAsset(c, "/admin/q+a", 200);
9 }
10 return serveAsset(c, "/q+a", 200);
11});
12
13// Submit form
14app.post("/q+a", async (c) => {
15 const form = await c.req.formData();
16 let text = form.get("text");
17 if (typeof text !== "string") {
18 return questionFailure(c, 400, "Bad Request");
19 }
20 text = text.trim();
21 const input = {
22 date: new Date(),
23 prompt: text,
24 sourceName: "unknown",
25 sourceLocation: "unknown",
26 sourceVPN: null,
27 };
28
29 input.date.setMilliseconds(0);
30
31 if (text.length <= 0) {
32 return questionFailure(c, 400, "Content is too short", text);
33 }
34
35 if (text.length > 16000) {
36 return questionFailure(c, 400, "Content is too long", text);
37 }
38
39 // Ban patterns
40 if (
41 text.includes("hams" + "terko" + "mbat" + ".expert") // 2025-02-18. 3 occurrences. All VPN
42 ) {
43 // To prevent known automatic spam-bots from noticing something automatic is
44 // happening, pretend that the question was successfully submitted.
45 return sendSuccess(c, new Date());
46 }
47
48 const ipAddr = c.req.header("cf-connecting-ip");
49 if (ipAddr) {
50 input.sourceName = uniqueNamesGenerator({
51 dictionaries: [adjectives, colors, animals],
52 separator: "-",
53 seed: ipAddr + PROXYCHECK_API_KEY,
54 });
55 }
56
57 const cfIPCountry = c.req.header("cf-ipcountry");
58 if (cfIPCountry) {
59 input.sourceLocation = cfIPCountry;
60 }
61
62 if (ipAddr && PROXYCHECK_API_KEY) {
63 const proxyCheck = await fetch(
64 `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,
65 {
66 method: "POST",
67 body: "ips=" + ipAddr,
68 headers: {
69 "Content-Type": "application/x-www-form-urlencoded",
70 },
71 },
72 ).then((res) => res.json());
73
74 if (ipAddr && proxyCheck[ipAddr]) {
75 if (proxyCheck[ipAddr].proxy === "yes") {
76 input.sourceVPN = proxyCheck[ipAddr].operator?.name ??
77 proxyCheck[ipAddr].organisation ??
78 proxyCheck[ipAddr].provider ?? "unknown";
79 }
80 if (Number(proxyCheck[ipAddr].risk) > 72) {
81 return questionFailure(
82 c,
83 403,
84 "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.",
85 text,
86 );
87 }
88 }
89 }
90
91 const date = Question.create(
92 QuestionType.pending,
93 JSON.stringify(input),
94 input.date,
95 );
96 await sendSuccess(c, date);
97});
98async function sendSuccess(c: Context, date: Date) {
99 if (c.req.header("Accept")?.includes("application/json")) {
100 return c.json({
101 success: true,
102 message: "ok",
103 date: date.getTime(),
104 id: formatQuestionId(date),
105 }, { status: 200 });
106 }
107 c.res = await renderView(c, "q+a/success", {
108 permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,
109 });
110}
111// Question Permalink
112app.get("/q+a/:id", async (c, next) => {
113 // from deadname era, the seconds used to be in the url.
114 // this was removed so that the url can be crafted by hand.
115 let id = c.req.param("id");
116 if (id.length === 12 && /^\d+$/.test(id)) {
117 return c.redirect(`/q+a/${id.slice(0, 10)}`);
118 }
119 let image = false;
120 if (id.endsWith(".png")) {
121 image = true;
122 id = id.slice(0, -4);
123 }
124
125 const timestamp = questionIdToTimestamp(id);
126 if (!timestamp) return next();
127 const question = Question.getByDate(timestamp);
128 if (!question) return next();
129
130 if (image) {
131 return getQuestionImage(question, c.req.method === "HEAD");
132 }
133 return renderView(c, "q+a/permalink", { question });
134});
135
136// Admin
137app.get("/admin/q+a", async (c) => {
138 return serveAsset(c, "/admin/q+a", 200);
139});
140app.get("/admin/q+a/inbox", async (c) => {
141 return renderView(c, "q+a/backend-inbox", {});
142});
143app.delete("/admin/q+a/:id", async (c, next) => {
144 const id = c.req.param("id");
145 const timestamp = questionIdToTimestamp(id);
146 if (!timestamp) return next();
147 const question = Question.getByDate(timestamp);
148 if (!question) return next();
149 const deleteFull = c.req.header("X-Delete-Full") === "true";
150 if (deleteFull) {
151 Question.deleteByQmid(question.qmid);
152 } else {
153 Question.rejectByQmid(question.qmid);
154 }
155 return c.json({ success: true, message: "ok" });
156});
157app.patch("/admin/q+a/:id", async (c, next) => {
158 const id = c.req.param("id");
159 const timestamp = questionIdToTimestamp(id);
160 if (!timestamp) return next();
161 const question = Question.getByDate(timestamp);
162 if (!question) return next();
163 const form = await c.req.raw.json();
164 if (typeof form.text !== "string" || typeof form.type !== "number") {
165 return questionFailure(c, 400, "Bad Request");
166 }
167 Question.updateByQmid(question.qmid, form.text, form.type);
168 return c.json({ success: true, message: "ok" });
169});
170app.get("/admin/q+a/:id", async (c, next) => {
171 const id = c.req.param("id");
172 const timestamp = questionIdToTimestamp(id);
173 if (!timestamp) return next();
174 const question = Question.getByDate(timestamp);
175 if (!question) return next();
176
177 let pendingInfo: null | PendingQuestionData = null;
178 if (question.type === QuestionType.pending) {
179 pendingInfo = JSON.parse(question.text) as PendingQuestionData;
180 question.text = pendingInfo.prompt.trim().split("\n").map((line) =>
181 line.trim().length === 0 ? "" : `q: ${line.trim()}`
182 ).join("\n") + "\n\n";
183 question.type = QuestionType.normal;
184 }
185
186 return renderView(c, "q+a/editor", {
187 pendingInfo,
188 question,
189 });
190});
191
192app.get("/q+a/things/random", async (c) => {
193 c.res = await renderView(c, "q+a/things-random", {});
194});
195
196async function questionFailure(
197 c: Context,
198 status: ContentfulStatusCode,
199 message: string,
200 content?: string,
201) {
202 if (c.req.header("Accept")?.includes("application/json")) {
203 return c.json({ success: false, message, id: null }, { status });
204 }
205 return await renderView(c, "q+a/fail", {
206 error: message,
207 content,
208 });
209}
210
211import { type Context, Hono } from "#hono";
212import type { ContentfulStatusCode } from "hono/utils/http-status";
213import {
214 adjectives,
215 animals,
216 colors,
217 uniqueNamesGenerator,
218} from "unique-names-generator";
219import { hasAdminToken } from "../admin.ts";
220import { serveAsset } from "#sitegen/assets";
221import {
222 PendingQuestion,
223 PendingQuestionData,
224} from "./models/PendingQuestion.ts";
225import { Question, QuestionType } from "./models/Question.ts";
226import { renderView } from "#sitegen/view";
227import { getQuestionImage } from "./image.tsx";
228import { formatQuestionId, questionIdToTimestamp } from "./format.ts";
1const PROXYCHECK_API_KEY = process.env.PROXYCHECK_API_KEY;
2
3export const app = new Hono();
4
5// Main page
6app.get("/q+a", async (c) => {
7 if (hasAdminToken(c)) {
8 return serveAsset(c, "/admin/q+a", 200);
9 }
10 return serveAsset(c, "/q+a", 200);
11});
12
13// Submit form
14app.post("/q+a", async (c) => {
15 const form = await c.req.formData();
16 let text = form.get("text");
17 if (typeof text !== "string") {
18 return questionFailure(c, 400, "Bad Request");
19 }
20 text = text.trim();
21 const input = {
22 date: new Date(),
23 prompt: text,
24 sourceName: "unknown",
25 sourceLocation: "unknown",
26 sourceVPN: null,
27 };
28
29 input.date.setMilliseconds(0);
30
31 if (text.length <= 0) {
32 return questionFailure(c, 400, "Content is too short", text);
33 }
34
35 if (text.length > 16000) {
36 return questionFailure(c, 400, "Content is too long", text);
37 }
38
39 // Ban patterns
40 if (
41 text.includes("hams" + "terko" + "mbat" + ".expert") // 2025-02-18. 3 occurrences. All VPN
42 ) {
43 // To prevent known automatic spam-bots from noticing something automatic is
44 // happening, pretend that the question was successfully submitted.
45 return sendSuccess(c, new Date());
46 }
47
48 const ipAddr = c.req.header("cf-connecting-ip");
49 if (ipAddr) {
50 input.sourceName = uniqueNamesGenerator({
51 dictionaries: [adjectives, colors, animals],
52 separator: "-",
53 seed: ipAddr + PROXYCHECK_API_KEY,
54 });
55 }
56
57 const cfIPCountry = c.req.header("cf-ipcountry");
58 if (cfIPCountry) {
59 input.sourceLocation = cfIPCountry;
60 }
61
62 if (ipAddr && PROXYCHECK_API_KEY) {
63 const proxyCheck = await fetch(
64 `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`,
65 {
66 method: "POST",
67 body: "ips=" + ipAddr,
68 headers: {
69 "Content-Type": "application/x-www-form-urlencoded",
70 },
71 },
72 ).then((res) => res.json());
73
74 if (ipAddr && proxyCheck[ipAddr]) {
75 if (proxyCheck[ipAddr].proxy === "yes") {
76 input.sourceVPN = proxyCheck[ipAddr].operator?.name ??
77 proxyCheck[ipAddr].organisation ??
78 proxyCheck[ipAddr].provider ?? "unknown";
79 }
80 if (Number(proxyCheck[ipAddr].risk) > 72) {
81 return questionFailure(
82 c,
83 403,
84 "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.",
85 text,
86 );
87 }
88 }
89 }
90
91 const date = Question.create(
92 QuestionType.pending,
93 JSON.stringify(input),
94 input.date,
95 );
96 await sendSuccess(c, date);
97});
98async function sendSuccess(c: Context, date: Date) {
99 if (c.req.header("Accept")?.includes("application/json")) {
100 return c.json({
101 success: true,
102 message: "ok",
103 date: date.getTime(),
104 id: formatQuestionId(date),
105 }, { status: 200 });
106 }
107 c.res = await renderView(c, "q+a/success", {
108 permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`,
109 });
110}
111// Question Permalink
112app.get("/q+a/:id", async (c, next) => {
113 // from deadname era, the seconds used to be in the url.
114 // this was removed so that the url can be crafted by hand.
115 let id = c.req.param("id");
116 if (id.length === 12 && /^\d+$/.test(id)) {
117 return c.redirect(`/q+a/${id.slice(0, 10)}`);
118 }
119 let image = false;
120 if (id.endsWith(".png")) {
121 image = true;
122 id = id.slice(0, -4);
123 }
124
125 const timestamp = questionIdToTimestamp(id);
126 if (!timestamp) return next();
127 const question = Question.getByDate(timestamp);
128 if (!question) return next();
129
130 if (image) {
131 return getQuestionImage(question, c.req.method === "HEAD");
132 }
133 return renderView(c, "q+a/permalink", { question });
134});
135
136// Admin
137app.get("/admin/q+a", async (c) => {
138 return serveAsset(c, "/admin/q+a", 200);
139});
140app.get("/admin/q+a/inbox", async (c) => {
141 return renderView(c, "q+a/backend-inbox", {});
142});
143app.delete("/admin/q+a/:id", async (c, next) => {
144 const id = c.req.param("id");
145 const timestamp = questionIdToTimestamp(id);
146 if (!timestamp) return next();
147 const question = Question.getByDate(timestamp);
148 if (!question) return next();
149 const deleteFull = c.req.header("X-Delete-Full") === "true";
150 if (deleteFull) {
151 Question.deleteByQmid(question.qmid);
152 } else {
153 Question.rejectByQmid(question.qmid);
154 }
155 return c.json({ success: true, message: "ok" });
156});
157app.patch("/admin/q+a/:id", async (c, next) => {
158 const id = c.req.param("id");
159 const timestamp = questionIdToTimestamp(id);
160 if (!timestamp) return next();
161 const question = Question.getByDate(timestamp);
162 if (!question) return next();
163 const form = await c.req.raw.json();
164 if (typeof form.text !== "string" || typeof form.type !== "number") {
165 return questionFailure(c, 400, "Bad Request");
166 }
167 Question.updateByQmid(question.qmid, form.text, form.type);
168 return c.json({ success: true, message: "ok" });
169});
170app.get("/admin/q+a/:id", async (c, next) => {
171 const id = c.req.param("id");
172 const timestamp = questionIdToTimestamp(id);
173 if (!timestamp) return next();
174 const question = Question.getByDate(timestamp);
175 if (!question) return next();
176
177 let pendingInfo: null | PendingQuestionData = null;
178 if (question.type === QuestionType.pending) {
179 pendingInfo = JSON.parse(question.text) as PendingQuestionData;
180 question.text = pendingInfo.prompt.trim().split("\n").map((line) =>
181 line.trim().length === 0 ? "" : `q: ${line.trim()}`
182 ).join("\n") + "\n\n";
183 question.type = QuestionType.normal;
184 }
185
186 return renderView(c, "q+a/editor", {
187 pendingInfo,
188 question,
189 });
190});
191
192app.get("/q+a/things/random", async (c) => {
193 c.res = await renderView(c, "q+a/things-random", {});
194});
195
196async function questionFailure(
197 c: Context,
198 status: ContentfulStatusCode,
199 message: string,
200 content?: string,
201) {
202 if (c.req.header("Accept")?.includes("application/json")) {
203 return c.json({ success: false, message, id: null }, { status });
204 }
205 return await renderView(c, "q+a/fail", {
206 error: message,
207 content,
208 });
209}
210
211import { type Context, Hono } from "#hono";
212import type { ContentfulStatusCode } from "hono/utils/http-status";
213import {
214 adjectives,
215 animals,
216 colors,
217 uniqueNamesGenerator,
218} from "unique-names-generator";
219import { hasAdminToken } from "../admin.ts";
220import { serveAsset } from "#sitegen/assets";
221import {
222 PendingQuestion,
223 PendingQuestionData,
224} from "./models/PendingQuestion.ts";
225import { Question, QuestionType } from "./models/Question.ts";
226import { renderView } from "#sitegen/view";
227import { getQuestionImage } from "./image.tsx";
228import { formatQuestionId, questionIdToTimestamp } from "./format.ts";
src/q+a/format.ts+39-39
......@@ -1,39 +1,39 @@
1const dateFormat = new Intl.DateTimeFormat("sv", {
2 timeZone: "EST",
3 year: "numeric",
4 month: "2-digit",
5 hour: "2-digit",
6 day: "2-digit",
7 minute: "2-digit",
8});
9
10// YYYY-MM-DD HH:MM
11export function formatQuestionTimestamp(date: Date) {
12 return dateFormat.format(date);
13}
14
15// YYYY-MM-DDTHH:MM:00Z
16export function formatQuestionISOTimestamp(date: Date) {
17 const str = dateFormat.format(date);
18 return `${str.slice(0, 10)}T${str.slice(11)}-05:00`;
19}
20
21// YYMMDDHHMM
22export function formatQuestionId(date: Date) {
23 return formatQuestionTimestamp(date).replace(/[^\d]/g, "").slice(2, 12);
24}
25
26export function questionIdToTimestamp(id: string) {
27 if (id.length !== 10 || !/^\d+$/.test(id)) {
28 return null;
29 }
30 const date = new Date(
31 `20${id.slice(0, 2)}-${id.slice(2, 4)}-${id.slice(4, 6)} ${
32 id.slice(6, 8)
33 }:${id.slice(8, 10)}:00 EST`,
34 );
35 if (isNaN(date.getTime())) {
36 return null;
37 }
38 return date;
39}
1const dateFormat = new Intl.DateTimeFormat("sv", {
2 timeZone: "EST",
3 year: "numeric",
4 month: "2-digit",
5 hour: "2-digit",
6 day: "2-digit",
7 minute: "2-digit",
8});
9
10// YYYY-MM-DD HH:MM
11export function formatQuestionTimestamp(date: Date) {
12 return dateFormat.format(date);
13}
14
15// YYYY-MM-DDTHH:MM:00Z
16export function formatQuestionISOTimestamp(date: Date) {
17 const str = dateFormat.format(date);
18 return `${str.slice(0, 10)}T${str.slice(11)}-05:00`;
19}
20
21// YYMMDDHHMM
22export function formatQuestionId(date: Date) {
23 return formatQuestionTimestamp(date).replace(/[^\d]/g, "").slice(2, 12);
24}
25
26export function questionIdToTimestamp(id: string) {
27 if (id.length !== 10 || !/^\d+$/.test(id)) {
28 return null;
29 }
30 const date = new Date(
31 `20${id.slice(0, 2)}-${id.slice(2, 4)}-${id.slice(4, 6)} ${
32 id.slice(6, 8)
33 }:${id.slice(8, 10)}:00 EST`,
34 );
35 if (isNaN(date.getTime())) {
36 return null;
37 }
38 return date;
39}
src/q+a/image.tsx+81-81
......@@ -1,81 +1,81 @@
1const width = 768;
2const cacheImageDir = path.resolve(".clover/question_images");
3
4// Cached browser session
5const getBrowser = RefCountedExpirable(
6 () =>
7 puppeteer.launch({
8 args: ["--no-sandbox", "--disable-setuid-sandbox"],
9 }),
10 (b) => b.close(),
11);
12
13export async function renderQuestionImage(question: Question) {
14 const html = await renderViewToString("q+a/image-embed", { question });
15
16 // this browser session will be reused if multiple images are generated
17 // either at the same time or within a 5-minute time span. the dispose
18 // symbol
19 using sharedBrowser = await getBrowser();
20 const b = sharedBrowser.value;
21
22 const p = await b.newPage();
23 await p.setViewport({ width, height: 400 });
24 await p.setContent(html);
25 try {
26 await p.waitForNetworkIdle({ idleTime: 100, timeout: 500 });
27 } catch (e) {}
28 const height = await p.evaluate(() => {
29 const e = document.querySelector("main")!;
30 return e.getBoundingClientRect().height;
31 });
32 const buf = await p.screenshot({
33 path: "screenshot.png",
34 type: "png",
35 captureBeyondViewport: true,
36 clip: { x: 0, width, y: 0, height: height, scale: 1.5 },
37 });
38 await p.close();
39
40 return Buffer.from(buf);
41}
42
43export async function getQuestionImage(
44 question: Question,
45 headOnly: boolean,
46): Promise<Response> {
47 const hash = crypto.createHash("sha1")
48 .update(question.qmid + question.type + question.text)
49 .digest("hex");
50
51 const headers = {
52 "Content-Type": "image/png",
53 "Cache-Control": "public, max-age=31536000",
54 "ETag": `"${hash}"`,
55 "Last-Modified": question.date.toUTCString(),
56 };
57
58 if (headOnly) {
59 return new Response(null, { headers });
60 }
61
62 const cachedFilePath = path.join(cacheImageDir, `/${hash}.png`);
63 let buf: Buffer;
64 try {
65 buf = await fs.readFile(cachedFilePath);
66 } catch (e: any) {
67 if (e.code !== "ENOENT") throw e;
68 buf = await renderQuestionImage(question);
69 fs.writeMkdir(cachedFilePath, buf).catch(() => {});
70 }
71
72 return new Response(buf, { headers });
73}
74
75import * as crypto from "node:crypto";
76import * as fs from "#sitegen/fs";
77import * as path from "node:path";
78import * as puppeteer from "puppeteer";
79import { Question } from "@/q+a/models/Question.ts";
80import { RefCountedExpirable } from "#sitegen/async";
81import { renderViewToString } from "#sitegen/view";
1const width = 768;
2const cacheImageDir = path.resolve(".clover/question_images");
3
4// Cached browser session
5const getBrowser = RefCountedExpirable(
6 () =>
7 puppeteer.launch({
8 args: ["--no-sandbox", "--disable-setuid-sandbox"],
9 }),
10 (b) => b.close(),
11);
12
13export async function renderQuestionImage(question: Question) {
14 const html = await renderViewToString("q+a/image-embed", { question });
15
16 // this browser session will be reused if multiple images are generated
17 // either at the same time or within a 5-minute time span. the dispose
18 // symbol
19 using sharedBrowser = await getBrowser();
20 const b = sharedBrowser.value;
21
22 const p = await b.newPage();
23 await p.setViewport({ width, height: 400 });
24 await p.setContent(html);
25 try {
26 await p.waitForNetworkIdle({ idleTime: 100, timeout: 500 });
27 } catch (e) {}
28 const height = await p.evaluate(() => {
29 const e = document.querySelector("main")!;
30 return e.getBoundingClientRect().height;
31 });
32 const buf = await p.screenshot({
33 path: "screenshot.png",
34 type: "png",
35 captureBeyondViewport: true,
36 clip: { x: 0, width, y: 0, height: height, scale: 1.5 },
37 });
38 await p.close();
39
40 return Buffer.from(buf);
41}
42
43export async function getQuestionImage(
44 question: Question,
45 headOnly: boolean,
46): Promise<Response> {
47 const hash = crypto.createHash("sha1")
48 .update(question.qmid + question.type + question.text)
49 .digest("hex");
50
51 const headers = {
52 "Content-Type": "image/png",
53 "Cache-Control": "public, max-age=31536000",
54 "ETag": `"${hash}"`,
55 "Last-Modified": question.date.toUTCString(),
56 };
57
58 if (headOnly) {
59 return new Response(null, { headers });
60 }
61
62 const cachedFilePath = path.join(cacheImageDir, `/${hash}.png`);
63 let buf: Buffer;
64 try {
65 buf = await fs.readFile(cachedFilePath);
66 } catch (e: any) {
67 if (e.code !== "ENOENT") throw e;
68 buf = await renderQuestionImage(question);
69 fs.writeMkdir(cachedFilePath, buf).catch(() => {});
70 }
71
72 return new Response(buf, { headers });
73}
74
75import * as crypto from "node:crypto";
76import * as fs from "#sitegen/fs";
77import * as path from "node:path";
78import * as puppeteer from "puppeteer";
79import { Question } from "@/q+a/models/Question.ts";
80import { RefCountedExpirable } from "#sitegen/async";
81import { renderViewToString } from "#sitegen/view";
src/q+a/scripts/editor.client.tsx+116-116
......@@ -1,116 +1,116 @@
1import { EditorState } from "@codemirror/state";
2import { basicSetup, EditorView } from "codemirror";
3import { ssrSync } from "#ssr";
4import type { ScriptPayload } from "@/q+a/views/editor.marko";
5import QuestionRender from "@/q+a/tags/question.marko";
6
7declare const payload: ScriptPayload;
8const date = new Date(payload.date);
9
10const main = document.getElementById("edit-grid")! as HTMLDivElement;
11const preview = document.getElementById("preview")! as HTMLDivElement;
12
13function updatePreview(text: string) {
14 preview.innerHTML = ssrSync(
15 <QuestionRender
16 question={{
17 id: payload.id,
18 qmid: payload.qmid,
19 text: text,
20 type: payload.type,
21 date,
22 }}
23 editor
24 />,
25 ).text;
26}
27updatePreview(payload.text);
28
29const startState = EditorState.create({
30 doc: payload.text,
31 extensions: [
32 basicSetup,
33 EditorView.darkTheme.of(true),
34 EditorView.updateListener.of((update) => {
35 if (update.docChanged) {
36 updatePreview(update.state.doc.toString());
37 }
38 }),
39 EditorView.lineWrapping,
40 ],
41 // selection: EditorSelection.create([
42 // EditorSelection.cursor(0),
43 // ], 0),
44});
45
46const view = new EditorView({
47 state: startState,
48 parent: document.getElementById("editor")!,
49});
50view.focus();
51
52(globalThis as any).onCommitQuestion = wrapAction(async () => {
53 const text = view.state.doc.toString();
54 const res = await fetch(`/admin/q+a/${payload.id}`, {
55 method: "PATCH",
56 headers: {
57 Accept: "application/json",
58 "Content-Type": "application/json",
59 },
60 body: JSON.stringify({
61 text,
62 type: payload.type,
63 }),
64 });
65 if (!res.ok) {
66 throw new Error("Failed to update question, status: " + res.status);
67 }
68 if (location.search.includes("return=inbox")) {
69 location.href = "/admin/q+a/inbox";
70 } else {
71 location.href = "/q+a#q" + payload.id;
72 }
73});
74(globalThis as any).onDelete = wrapAction(async () => {
75 if (confirm("Are you sure you want to delete this question?")) {
76 const res = await fetch(`/admin/q+a/${payload.id}`, {
77 method: "DELETE",
78 headers: {
79 Accept: "application/json",
80 },
81 });
82 if (!res.ok) {
83 throw new Error("Failed to delete question, status: " + res.status);
84 }
85 location.href = document.referrer || "/admin/q+a";
86 }
87});
88(globalThis as any).onTypeChange = () => {
89 payload.type = parseInt(
90 (document.getElementById("type") as HTMLSelectElement).value,
91 );
92 updatePreview(view.state.doc.toString());
93};
94
95function wrapAction(cb: () => Promise<void>) {
96 return async () => {
97 main.style.opacity = "0.5";
98 main.style.pointerEvents = "none";
99 const inputs = main.querySelectorAll("button,select,input") as NodeListOf<
100 HTMLButtonElement
101 >;
102 inputs.forEach((b) => {
103 b.disabled = true;
104 });
105 try {
106 await cb();
107 } catch (e: any) {
108 main.style.opacity = "1";
109 main.style.pointerEvents = "auto";
110 inputs.forEach((b) => {
111 b.disabled = false;
112 });
113 alert(e.message);
114 }
115 };
116}
1import { EditorState } from "@codemirror/state";
2import { basicSetup, EditorView } from "codemirror";
3import { ssrSync } from "#ssr";
4import type { ScriptPayload } from "@/q+a/views/editor.marko";
5import QuestionRender from "@/q+a/tags/question.marko";
6
7declare const payload: ScriptPayload;
8const date = new Date(payload.date);
9
10const main = document.getElementById("edit-grid")! as HTMLDivElement;
11const preview = document.getElementById("preview")! as HTMLDivElement;
12
13function updatePreview(text: string) {
14 preview.innerHTML = ssrSync(
15 <QuestionRender
16 question={{
17 id: payload.id,
18 qmid: payload.qmid,
19 text: text,
20 type: payload.type,
21 date,
22 }}
23 editor
24 />,
25 ).text;
26}
27updatePreview(payload.text);
28
29const startState = EditorState.create({
30 doc: payload.text,
31 extensions: [
32 basicSetup,
33 EditorView.darkTheme.of(true),
34 EditorView.updateListener.of((update) => {
35 if (update.docChanged) {
36 updatePreview(update.state.doc.toString());
37 }
38 }),
39 EditorView.lineWrapping,
40 ],
41 // selection: EditorSelection.create([
42 // EditorSelection.cursor(0),
43 // ], 0),
44});
45
46const view = new EditorView({
47 state: startState,
48 parent: document.getElementById("editor")!,
49});
50view.focus();
51
52(globalThis as any).onCommitQuestion = wrapAction(async () => {
53 const text = view.state.doc.toString();
54 const res = await fetch(`/admin/q+a/${payload.id}`, {
55 method: "PATCH",
56 headers: {
57 Accept: "application/json",
58 "Content-Type": "application/json",
59 },
60 body: JSON.stringify({
61 text,
62 type: payload.type,
63 }),
64 });
65 if (!res.ok) {
66 throw new Error("Failed to update question, status: " + res.status);
67 }
68 if (location.search.includes("return=inbox")) {
69 location.href = "/admin/q+a/inbox";
70 } else {
71 location.href = "/q+a#q" + payload.id;
72 }
73});
74(globalThis as any).onDelete = wrapAction(async () => {
75 if (confirm("Are you sure you want to delete this question?")) {
76 const res = await fetch(`/admin/q+a/${payload.id}`, {
77 method: "DELETE",
78 headers: {
79 Accept: "application/json",
80 },
81 });
82 if (!res.ok) {
83 throw new Error("Failed to delete question, status: " + res.status);
84 }
85 location.href = document.referrer || "/admin/q+a";
86 }
87});
88(globalThis as any).onTypeChange = () => {
89 payload.type = parseInt(
90 (document.getElementById("type") as HTMLSelectElement).value,
91 );
92 updatePreview(view.state.doc.toString());
93};
94
95function wrapAction(cb: () => Promise<void>) {
96 return async () => {
97 main.style.opacity = "0.5";
98 main.style.pointerEvents = "none";
99 const inputs = main.querySelectorAll("button,select,input") as NodeListOf<
100 HTMLButtonElement
101 >;
102 inputs.forEach((b) => {
103 b.disabled = true;
104 });
105 try {
106 await cb();
107 } catch (e: any) {
108 main.style.opacity = "1";
109 main.style.pointerEvents = "auto";
110 inputs.forEach((b) => {
111 b.disabled = false;
112 });
113 alert(e.message);
114 }
115 };
116}
src/q+a/simple-markdown.ts+1090-1090
......@@ -1,1090 +1,1090 @@
1// @ts-nocheck
2// # Simple-Markdown Core
3//
4// This is a fork of Khan-academy's Simple-Markdown[1], initially forked in 2022
5// to add Svelte support[2], and used for paper clover's q+a markdown flavor.
6//
7// 1: https://github.com/Khan/perseus/tree/main/packages/simple-markdown/src
8// 2: https://github.com/paperclover/svelte-simple-markdown
9export type Rules = Record<string, ParserRule>;
10export interface ParserRule {
11 name: string;
12 match: MatchFunction;
13 parse: ParseFunction;
14 quality?: QualityFunction;
15}
16
17export class RuleList extends Array<ParserRule> {
18 constructor(input?: ArrayLike<ParserRule>) {
19 super();
20 if (input) {
21 this.push(...Array.from(input));
22 }
23 }
24
25 insertBefore(rule: string, newRule: ParserRule): void {
26 const index = this.findIndex((r) => r.name === rule);
27 if (index === -1) {
28 throw new Error(`Rule ${rule} not found`);
29 }
30 this.splice(index, 0, newRule);
31 }
32
33 insertAfter(rule: string, newRule: ParserRule): void {
34 const index = this.findIndex((r) => r.name === rule);
35 if (index === -1) {
36 throw new Error(`Rule ${rule} not found`);
37 }
38 this.splice(index + 1, 0, newRule);
39 }
40
41 toRuleObject(): Record<string, ParserRule> {
42 const result: Record<string, ParserRule> = {};
43 this.forEach((rule) => {
44 result[rule.name] = rule;
45 });
46 return result;
47 }
48
49 add(rule: ParserRule): void {
50 this.push(rule);
51 }
52
53 get(rule: string): ParserRule | undefined {
54 return this.find((r) => r.name === rule);
55 }
56
57 remove(rule: string): void {
58 const index = this.findIndex((r) => r.name === rule);
59 if (index === -1) {
60 throw new Error(`Rule ${rule} not found`);
61 }
62 this.splice(index, 1);
63 }
64
65 clone() {
66 return new RuleList(this);
67 }
68}
69
70/**
71 * Creates a parser for a given set of rules, with the precedence
72 * specified as a list of rules.
73 *
74 * @param rules
75 * an object containing
76 * rule type -> {match, order, parse} objects
77 * (lower order is higher precedence)
78 * @param [defaultState]
79 *
80 * @returns
81 * The resulting parse function, with the following parameters:
82 * @source: the input source string to be parsed
83 * @state: an optional object to be threaded through parse
84 * calls. Allows clients to add stateful operations to
85 * parsing, such as keeping track of how many levels deep
86 * some nesting is. For an example use-case, see passage-ref
87 * parsing in src/widgets/passage/passage-markdown.jsx
88 */
89export function createParser(
90 ruleListInput: RuleList,
91 defaultState: Partial<ParserState> = {},
92) {
93 let rules = ruleListInput.toRuleObject();
94 let ruleList = Object.keys(rules);
95
96 let latestState: ParserState;
97
98 let nestedParse = function (source: string, state?: ParserState) {
99 let result: ASTNode[] = [];
100 state = state || latestState;
101 latestState = state;
102
103 while (source) {
104 // store the best match, it's rule, and quality:
105 let ruleType = null;
106 let rule = null;
107 let capture = null;
108 let quality = NaN; // loop control variables:
109
110 let i = 0;
111 let currRuleType = ruleList[0];
112
113 let currRule = rules[currRuleType];
114
115 do {
116 let currCapture = currRule.match(source, state);
117
118 if (currCapture) {
119 let currQuality = currRule.quality
120 ? currRule.quality(currCapture, state)
121 : 0;
122
123 // This should always be true the first time because
124 // the initial quality is NaN (that's why there's the
125 // condition negation).
126 if (!(currQuality <= quality)) {
127 ruleType = currRuleType;
128 rule = currRule;
129 capture = currCapture;
130 quality = currQuality;
131 }
132 }
133
134 // Move on to the next item.
135 // Note that this makes `currRule` be the next item
136 i++;
137 currRuleType = ruleList[i];
138 currRule = rules[currRuleType];
139 } while (
140 // keep looping while we're still within the ruleList
141 currRule &&
142 // if we don't have a match yet, continue
143 (!capture ||
144 // or if we have a match, but the next rule is
145 // at the same order, and has a quality measurement
146 // functions, then this rule must have a quality
147 // measurement function (since they are sorted before
148 // those without), and we need to check if there is
149 // a better quality match
150 currRule.quality)
151 );
152
153 if (!rule || !capture || !ruleType) {
154 throw new Error(
155 "Could not find a matching rule for the below " +
156 "content. The rule with highest `order` should " +
157 "always match content provided to it. Check " +
158 "the definition of `match` for '" +
159 ruleList[ruleList.length - 1] +
160 "'. It seems to not match the following source:\n" +
161 source,
162 );
163 }
164
165 if (capture.index) {
166 // If present and non-zero, i.e. a non-^ regexp result:
167 throw new Error(
168 "`match` must return a capture starting at index 0 " +
169 "(the current parse index). Did you forget a ^ at the " +
170 "start of the RegExp?",
171 );
172 }
173
174 let parsed = rule.parse(capture, nestedParse, state);
175
176 // We maintain the same object here so that rules can
177 // store references to the objects they return and
178 // modify them later. (oops sorry! but this adds a lot
179 // of power--see reflinks.)
180
181 // We also let rules override the default type of
182 // their parsed node if they would like to, so that
183 // there can be a single output function for all links,
184 // even if there are several rules to parse them.
185 if (!parsed.type) {
186 parsed.type = ruleType;
187 }
188
189 // Collapse text nodes
190 if (
191 parsed.type === "text" && result[result.length - 1]?.type === "text"
192 ) {
193 result[result.length - 1].content += parsed.content;
194 } else {
195 result.push(parsed as ASTNode);
196 }
197
198 state.prevCapture = capture;
199 source = source.substring(state.prevCapture[0].length);
200 }
201
202 return result;
203 };
204
205 let outerParse = function (
206 source: string,
207 state: ParserState = { inline: false },
208 ) {
209 latestState = populateInitialState(state, defaultState);
210
211 if (!latestState.inline && !latestState.disableAutoBlockNewlines) {
212 source = source + "\n\n";
213 }
214
215 // We store the previous capture so that match functions can
216 // use some limited amount of lookbehind. Lists use this to
217 // ensure they don't match arbitrary '- ' or '* ' in inline
218 // text (see the list rule for more information). This stores
219 // the full regex capture object, if there is one.
220 latestState.prevCapture = undefined;
221 return nestedParse(preprocess(source), latestState);
222 };
223
224 return outerParse;
225}
226
227type Multiple<T> = T | T[];
228type Nullable<T> = T | null | undefined;
229
230export type MatchFunction = (
231 source: string,
232 state: ParserState,
233) => Nullable<RegExpMatchArray>;
234
235export type Parser = (source: string, state?: ParserState) => ASTNode[];
236
237export type ParseFunction = (
238 source: RegExpMatchArray,
239 nestedParse: Parser,
240 state: ParserState,
241) => TypeOptionalASTNode;
242
243export type QualityFunction = (
244 capture: RegExpMatchArray,
245 state: ParserState,
246) => number;
247
248export interface ParserState {
249 inline: boolean;
250 prevCapture?: RegExpMatchArray;
251 [key: string]: any;
252}
253
254export interface ASTNode {
255 type: string;
256 content?: ASTNode[] | string;
257 [key: string]: any;
258}
259
260export type TypeOptionalASTNode = Omit<ASTNode, "type"> & { type?: string };
261
262export interface RefNode {
263 type: string;
264 content?: Multiple<ASTNode>;
265 target?: string;
266 title?: string;
267 alt?: string;
268}
269
270/** Creates a match function for an inline scoped element from a regex */
271export function inlineRegex(regex: RegExp): MatchFunction {
272 return (source, state) => {
273 if (state.inline) {
274 return regex.exec(source);
275 } else {
276 return null;
277 }
278 };
279}
280
281/** Creates a match function for a block scoped element from a regex */
282export function blockRegex(regex: RegExp): MatchFunction {
283 return (source, state) => {
284 if (state.inline) {
285 return null;
286 } else {
287 return regex.exec(source);
288 }
289 };
290}
291
292/** Creates a match function from a regex, ignoring block/inline scope */
293export function anyScopeRegex(regex: RegExp): MatchFunction {
294 return (source) => {
295 return regex.exec(source);
296 };
297}
298
299const UNESCAPE_URL_R = /\\([^0-9A-Za-z\s])/g;
300export function unescapeUrl(rawUrlString: string) {
301 return rawUrlString.replace(UNESCAPE_URL_R, "$1");
302}
303
304/**
305 * Parse some content with the parser `parse`, with state.inline
306 * set to true. Useful for block elements; not generally necessary
307 * to be used by inline elements (where state.inline is already true.
308 */
309export function parseInline(
310 parse: Parser,
311 content: string,
312 state: ParserState,
313) {
314 const isCurrentlyInline = state.inline || false;
315 state.inline = true;
316 const result = parse(content, state);
317 state.inline = isCurrentlyInline;
318 return result;
319}
320
321export function parseBlock(parse: Parser, content: string, state: ParserState) {
322 const isCurrentlyInline = state.inline || false;
323 state.inline = false;
324 const result = parse(content + "\n\n", state);
325 state.inline = isCurrentlyInline;
326 return result;
327}
328
329export function parseCaptureInline(
330 capture: RegExpMatchArray,
331 parse: Parser,
332 state: ParserState,
333) {
334 return {
335 content: parseInline(parse, capture[1], state),
336 };
337}
338
339export function ignoreCapture() {
340 return {};
341}
342
343export function sanitizeUrl(url?: string) {
344 if (url == null) {
345 return null;
346 }
347 try {
348 const prot = new URL(url, "https://localhost").protocol;
349 if (
350 prot.indexOf("javascript:") === 0 ||
351 prot.indexOf("vbscript:") === 0 ||
352 prot.indexOf("data:") === 0
353 ) {
354 return null;
355 }
356 } catch (e) {
357 // invalid URLs should throw a TypeError
358 // see for instance: `new URL("");`
359 return null;
360 }
361 return url;
362}
363
364const CR_NEWLINE_R = /\r\n?/g;
365const TAB_R = /\t/g;
366const FORMFEED_R = /\f/g;
367
368/**
369 * Turn various whitespace into easy-to-process whitespace
370 */
371function preprocess(source: string) {
372 return source.replace(CR_NEWLINE_R, "\n").replace(FORMFEED_R, "").replace(
373 TAB_R,
374 " ",
375 );
376}
377
378function populateInitialState(
379 givenState: Partial<ParserState>,
380 defaultState: Partial<ParserState>,
381) {
382 let state = givenState || {};
383
384 for (let prop in defaultState) {
385 if (Object.prototype.hasOwnProperty.call(defaultState, prop)) {
386 state[prop] = defaultState[prop];
387 }
388 }
389
390 return state as ParserState;
391}
392
393// recognize a `*` `-`, `+`, `1.`, `2.`... list bullet
394const LIST_BULLET = "(?:[*+-]|\\d+\\.)";
395
396// recognize the start of a list item:
397// leading space plus a bullet plus a space (` * `)
398const LIST_ITEM_PREFIX = "( *)(" + LIST_BULLET + ") +";
399const LIST_ITEM_PREFIX_R = new RegExp("^" + LIST_ITEM_PREFIX);
400
401// recognize an individual list item:
402// * hi
403// this is part of the same item
404//
405// as is this, which is a new paragraph in the same item
406//
407// * but this is not part of the same item
408const LIST_ITEM_R = new RegExp(
409 LIST_ITEM_PREFIX + "[^\\n]*(?:\\n" + "(?!\\1" + LIST_BULLET +
410 " )[^\\n]*)*(\n|$)",
411 "gm",
412);
413const BLOCK_END_R = /\n{2,}$/;
414const INLINE_CODE_ESCAPE_BACKTICKS_R = /^ (?= *`)|(` *) $/g;
415
416// recognize the end of a paragraph block inside a list item:
417// two or more newlines at end end of the item
418const LIST_BLOCK_END_R = BLOCK_END_R;
419const LIST_ITEM_END_R = / *\n+$/;
420
421// check whether a list item has paragraphs: if it does,
422// we leave the newlines at the end
423const LIST_R = new RegExp(
424 "^( *)(" +
425 LIST_BULLET +
426 ") " +
427 "[\\s\\S]+?(?:\n{2,}(?! )" +
428 "(?!\\1" +
429 LIST_BULLET +
430 " )\\n*" +
431 // the \\s*$ here is so that we can parse the inside of nested
432 // lists, where our content might end before we receive two `\n`s
433 "|\\s*\n*$)",
434);
435const LIST_LOOKBEHIND_R = /(?:^|\n)( *)$/;
436
437const TABLES = (function () {
438 const TABLE_ROW_SEPARATOR_TRIM = /^ *\| *| *\| *$/g;
439 const TABLE_CELL_END_TRIM = / *$/;
440 const TABLE_RIGHT_ALIGN = /^ *-+: *$/;
441 const TABLE_CENTER_ALIGN = /^ *:-+: *$/;
442 const TABLE_LEFT_ALIGN = /^ *:-+ *$/; // TODO: This needs a real type
443
444 const parseTableAlignCapture = (alignCapture: string) => {
445 if (TABLE_RIGHT_ALIGN.test(alignCapture)) {
446 return "right";
447 } else if (TABLE_CENTER_ALIGN.test(alignCapture)) {
448 return "center";
449 } else if (TABLE_LEFT_ALIGN.test(alignCapture)) {
450 return "left";
451 } else {
452 return null;
453 }
454 };
455
456 const parseTableAlign = (source: string, trimEndSeparators: boolean) => {
457 if (trimEndSeparators) {
458 source = source.replace(TABLE_ROW_SEPARATOR_TRIM, "");
459 }
460
461 const alignText = source.trim().split("|");
462 return alignText.map(parseTableAlignCapture);
463 };
464
465 const parseTableRow = (
466 source: string,
467 parse: Parser,
468 state: ParserState,
469 trimEndSeparators: boolean,
470 ) => {
471 const prevInTable = state.inTable;
472 state.inTable = true;
473 const tableRow = parse(source.trim(), state);
474 state.inTable = prevInTable;
475 const cells: ASTNode[][] = [[]];
476 tableRow.forEach(function (node, i) {
477 if (node.type === "tableSeparator") {
478 // Filter out empty table separators at the start/end:
479 if (!trimEndSeparators || (i !== 0 && i !== tableRow.length - 1)) {
480 // Split the current row:
481 cells.push([]);
482 }
483 } else {
484 if (
485 typeof node.content === "string" &&
486 (tableRow[i + 1] == null || tableRow[i + 1].type === "tableSeparator")
487 ) {
488 node.content = node.content.replace(TABLE_CELL_END_TRIM, "");
489 }
490
491 cells[cells.length - 1].push(node);
492 }
493 });
494 return cells;
495 };
496
497 /**
498 * @param {string} source
499 * @param {SimpleMarkdown.Parser} parse
500 * @param {SimpleMarkdown.State} state
501 * @param {boolean} trimEndSeparators
502 * @returns {SimpleMarkdown.ASTNode[][]}
503 */
504 const parseTableCells = function (
505 source: string,
506 parse: Parser,
507 state: ParserState,
508 trimEndSeparators: boolean,
509 ) {
510 const rowsText = source.trim().split("\n");
511 return rowsText.map(function (rowText) {
512 return parseTableRow(rowText, parse, state, trimEndSeparators);
513 });
514 };
515
516 /**
517 * @param {boolean} trimEndSeparators
518 * @returns {SimpleMarkdown.SingleNodeParseFunction}
519 */
520 const parseTable = function (trimEndSeparators: boolean) {
521 return function (
522 capture: RegExpMatchArray,
523 parse: Parser,
524 state: ParserState,
525 ) {
526 state.inline = true;
527 const header = parseTableRow(capture[1], parse, state, trimEndSeparators);
528 const align = parseTableAlign(capture[2], trimEndSeparators);
529 const cells = parseTableCells(
530 capture[3],
531 parse,
532 state,
533 trimEndSeparators,
534 );
535 state.inline = false;
536 return {
537 type: "table",
538 header: header,
539 align: align,
540 cells: cells,
541 };
542 };
543 };
544
545 return {
546 parseTable: parseTable(true),
547 parseNpTable: parseTable(false),
548 TABLE_REGEX: /^ *(\|.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*/,
549 NPTABLE_REGEX:
550 /^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*/,
551 };
552})();
553
554const LINK_INSIDE = "(?:\\[[^\\]]*\\]|[^\\[\\]]|\\](?=[^\\[]*\\]))*";
555const LINK_HREF_AND_TITLE =
556 "\\s*<?((?:\\([^)]*\\)|[^\\s\\\\]|\\\\.)*?)>?(?:\\s+['\"]([\\s\\S]*?)['\"])?\\s*";
557const AUTOLINK_MAILTO_CHECK_R = /mailto:/i;
558
559function parseRef(
560 capture: RegExpMatchArray,
561 state: ParserState,
562 refNode: RefNode,
563) {
564 const ref = (capture[2] || capture[1]).replace(/\s+/g, " ").toLowerCase();
565
566 // We store information about previously seen defs on
567 // state._defs (_ to deconflict with client-defined
568 // state). If the def for this reflink/refimage has
569 // already been seen, we can use its target/source
570 // and title here:
571 if (state._defs && state._defs[ref]) {
572 const def = state._defs[ref];
573
574 // `refNode` can be a link or an image. Both use
575 // target and title properties.
576 refNode.target = def.target;
577 refNode.title = def.title;
578 }
579
580 // In case we haven't seen our def yet (or if someone
581 // overwrites that def later on), we add this node
582 // to the list of ref nodes for that def. Then, when
583 // we find the def, we can modify this link/image AST
584 // node :).
585 // I'm sorry.
586 state._refs = state._refs || {};
587 state._refs[ref] = state._refs[ref] || [];
588
589 state._refs[ref].push(refNode);
590
591 return refNode;
592}
593
594export const defaultRules = new RuleList();
595{
596 defaultRules.add({
597 name: "heading",
598 match: blockRegex(/^ *(#{1,6})([^\n]+?)#* *(?:\n *)+\n/),
599 parse: function (capture, parse, state) {
600 return {
601 level: capture[1].length,
602 content: parseInline(parse, capture[2].trim(), state),
603 };
604 },
605 });
606
607 defaultRules.add({
608 name: "nptable",
609 match: blockRegex(TABLES.NPTABLE_REGEX),
610 parse: TABLES.parseNpTable,
611 });
612
613 defaultRules.add({
614 name: "lheading",
615 match: blockRegex(/^([^\n]+)\n *(=|-){3,} *(?:\n *)+\n/),
616 parse(capture, parse, state) {
617 return {
618 type: "heading",
619 level: capture[2] === "=" ? 1 : 2,
620 content: parseInline(parse, capture[1], state),
621 };
622 },
623 });
624
625 defaultRules.add({
626 name: "hr",
627 match: blockRegex(/^( *[-*_]){3,} *(?:\n *)+\n/),
628 parse: ignoreCapture,
629 });
630
631 defaultRules.add({
632 name: "codeBlock",
633 match: blockRegex(/^(?: {4}[^\n]+\n*)+(?:\n *)+\n/),
634 parse(capture) {
635 const content = capture[0].replace(/^ {4}/gm, "").replace(/\n+$/, "");
636 return {
637 lang: undefined,
638 content: content,
639 };
640 },
641 });
642
643 defaultRules.add({
644 name: "fence",
645 match: blockRegex(
646 /^ *(`{3,}|~{3,}) *(?:(\S+) *)?\n([\s\S]+?)\n?\1 *(?:\n *)+\n/,
647 ),
648 parse(capture) {
649 return {
650 type: "codeBlock",
651 lang: capture[2] || undefined,
652 content: capture[3],
653 };
654 },
655 });
656
657 defaultRules.add({
658 name: "blockQuote",
659 match: blockRegex(/^( *>[^\n]+(\n[^\n]+)*\n*)+\n{2,}/),
660 parse(capture, parse, state) {
661 const content = capture[0].replace(/^ *> ?/gm, "");
662 return {
663 content: parse(content, state),
664 };
665 },
666 });
667
668 defaultRules.add({
669 name: "list",
670
671 match(source, state) {
672 // We only want to break into a list if we are at the start of a
673 // line. This is to avoid parsing "hi * there" with "* there"
674 // becoming a part of a list.
675 // You might wonder, "but that's inline, so of course it wouldn't
676 // start a list?". You would be correct! Except that some of our
677 // lists can be inline, because they might be inside another list,
678 // in which case we can parse with inline scope, but need to allow
679 // nested lists inside this inline scope.
680 const prevCaptureStr = state.prevCapture == null
681 ? ""
682 : state.prevCapture[0];
683 const isStartOfLineCapture = LIST_LOOKBEHIND_R.exec(prevCaptureStr);
684 const isListBlock = state._list || !state.inline;
685
686 if (isStartOfLineCapture && isListBlock) {
687 source = isStartOfLineCapture[1] + source;
688 return LIST_R.exec(source);
689 } else {
690 return null;
691 }
692 },
693 parse(capture, parse, state) {
694 const bullet = capture[2];
695 const ordered = bullet.length > 1;
696 const start = ordered ? +bullet : undefined;
697 // We know this will match here, because of how the regexes are defined
698 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
699 const items = capture[0].replace(LIST_BLOCK_END_R, "\n").match(
700 LIST_ITEM_R,
701 )!;
702
703 let lastItemWasAParagraph = false;
704 const itemContent = items.map(function (item, i) {
705 // We need to see how far indented this item is:
706 const prefixCapture = LIST_ITEM_PREFIX_R.exec(item);
707 const space = prefixCapture ? prefixCapture[0].length : 0; // And then we construct a regex to "unindent" the subsequent
708 // lines of the items by that amount:
709
710 const spaceRegex = new RegExp("^ {1," + space + "}", "gm"); // Before processing the item, we need a couple things
711
712 const content = item // remove indents on trailing lines:
713 .replace(spaceRegex, "") // remove the bullet:
714 .replace(LIST_ITEM_PREFIX_R, ""); // I'm not sur4 why this is necessary again?
715 // Handling "loose" lists, like:
716 //
717 // * this is wrapped in a paragraph
718 //
719 // * as is this
720 //
721 // * as is this
722
723 const isLastItem = i === items.length - 1;
724 const containsBlocks = content.indexOf("\n\n") !== -1; // Any element in a list is a block if it contains multiple
725 // newlines. The last element in the list can also be a block
726 // if the previous item in the list was a block (this is
727 // because non-last items in the list can end with \n\n, but
728 // the last item can't, so we just "inherit" this property
729 // from our previous element).
730
731 const thisItemIsAParagraph = containsBlocks ||
732 (isLastItem && lastItemWasAParagraph);
733 lastItemWasAParagraph = thisItemIsAParagraph; // backup our state for restoration afterwards. We're going to
734 // want to set state._list to true, and state.inline depending
735 // on our list's looseness.
736
737 const oldStateInline = state.inline;
738 const oldStateList = state._list;
739 state._list = true; // Parse inline if we're in a tight list, or block if we're in
740 // a loose list.
741
742 let adjustedContent;
743
744 if (thisItemIsAParagraph) {
745 state.inline = false;
746 adjustedContent = content.replace(LIST_ITEM_END_R, "\n\n");
747 } else {
748 state.inline = true;
749 adjustedContent = content.replace(LIST_ITEM_END_R, "");
750 }
751
752 const result = parse(adjustedContent, state); // Restore our state before returning
753
754 state.inline = oldStateInline;
755 state._list = oldStateList;
756 return result;
757 });
758
759 return {
760 ordered: ordered,
761 start: start,
762 content: itemContent,
763 };
764 },
765 });
766
767 defaultRules.add({
768 name: "def",
769 // TODO: This will match without a blank line before the next
770 // block element, which is inconsistent with most of the rest of
771 // simple-markdown.
772 match: blockRegex(
773 /^ *\[([^\]]+)\]: *<?([^\s>]*)>?(?: +["(]([^\n]+)[")])? *\n(?: *\n)*/,
774 ),
775 parse(capture, parse, state) {
776 const def = capture[1].replace(/\s+/g, " ").toLowerCase();
777 const target = capture[2];
778 const title = capture[3];
779
780 // Look for previous links/images using this def
781 // If any links/images using this def have already been declared,
782 // they will have added themselves to the state._refs[def] list
783 // (_ to deconflict with client-defined state). We look through
784 // that list of reflinks for this def, and modify those AST nodes
785 // with our newly found information now.
786 // Sorry :(.
787 if (state._refs && state._refs[def]) {
788 // `refNode` can be a link or an image
789 state._refs[def].forEach((refNode: RefNode) => {
790 refNode.target = target;
791 refNode.title = title;
792 });
793 }
794
795 // Add this def to our map of defs for any future links/images
796 // In case we haven't found any or all of the refs referring to
797 // this def yet, we add our def to the table of known defs, so
798 // that future reflinks can modify themselves appropriately with
799 // this information.
800 state._defs = state._defs || {};
801 state._defs[def] = {
802 target: target,
803 title: title,
804 };
805
806 // return the relevant parsed information
807 // for debugging only.
808 return {
809 def: def,
810 target: target,
811 title: title,
812 };
813 },
814 });
815
816 defaultRules.add({
817 name: "table",
818 match: blockRegex(TABLES.TABLE_REGEX),
819 parse: TABLES.parseTable,
820 });
821
822 defaultRules.add({
823 name: "newline",
824 match: blockRegex(/^(?:\n *)*\n/),
825 parse: ignoreCapture,
826 });
827
828 defaultRules.add({
829 name: "paragraph",
830 match: blockRegex(/^((?:[^\n]|\n(?! *\n))+)(?:\n *)+\n/),
831 parse: parseCaptureInline,
832 });
833
834 defaultRules.add({
835 name: "escape",
836 // We don't allow escaping numbers, letters, or spaces here so that
837 // backslashes used in plain text still get rendered. But allowing
838 // escaping anything else provides a very flexible escape mechanism,
839 // regardless of how this grammar is extended.
840 match: inlineRegex(/^\\([^0-9A-Za-z\s])/),
841 parse(capture) {
842 return {
843 type: "text",
844 content: capture[1],
845 };
846 },
847 });
848
849 defaultRules.add({
850 name: "tableSeparator",
851
852 match(source, state) {
853 if (!state.inTable) {
854 return null;
855 }
856
857 return /^ *\| */.exec(source);
858 },
859 parse() {
860 return {
861 type: "tableSeparator",
862 };
863 },
864 });
865
866 defaultRules.add({
867 name: "autolink",
868 match: inlineRegex(/^<([^: >]+:\/[^ >]+)>/),
869 parse(capture) {
870 return {
871 type: "link",
872 content: [
873 {
874 type: "text",
875 content: capture[1],
876 },
877 ],
878 target: capture[1],
879 };
880 },
881 });
882
883 defaultRules.add({
884 name: "mailto",
885 match: inlineRegex(/^<([^ >]+@[^ >]+)>/),
886 parse(capture) {
887 const address = capture[1];
888 let target = capture[1]; // Check for a `mailto:` already existing in the link:
889
890 if (!AUTOLINK_MAILTO_CHECK_R.test(target)) {
891 target = "mailto:" + target;
892 }
893
894 return {
895 type: "link",
896 content: [
897 {
898 type: "text",
899 content: address,
900 },
901 ],
902 target: target,
903 };
904 },
905 });
906
907 defaultRules.add({
908 name: "url",
909 match: inlineRegex(/^(https?:\/\/[^\s<]+[^<.,:;"')\]\s])/),
910 parse(capture) {
911 return {
912 type: "link",
913 content: [
914 {
915 type: "text",
916 content: capture[1],
917 },
918 ],
919 target: capture[1],
920 title: undefined,
921 };
922 },
923 });
924
925 defaultRules.add({
926 name: "link",
927 match: inlineRegex(
928 new RegExp(
929 "^\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)",
930 ),
931 ),
932 parse(capture, parse, state) {
933 const link = {
934 content: parse(capture[1], state),
935 target: unescapeUrl(capture[2]),
936 title: capture[3],
937 };
938 return link;
939 },
940 });
941
942 defaultRules.add({
943 name: "image",
944 match: inlineRegex(
945 new RegExp(
946 "^!\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)",
947 ),
948 ),
949 parse: function (capture) {
950 const image = {
951 alt: capture[1],
952 target: unescapeUrl(capture[2]),
953 title: capture[3],
954 };
955 return image;
956 },
957 });
958
959 defaultRules.add({
960 name: "reflink",
961 match: inlineRegex(
962 new RegExp(
963 // The first [part] of the link
964 "^\\[(" +
965 LINK_INSIDE +
966 ")\\]" + // The [ref] target of the link
967 "\\s*\\[([^\\]]*)\\]",
968 ),
969 ),
970 parse(capture, parse, state) {
971 return parseRef(capture, state, {
972 type: "link",
973 content: parse(capture[1], state),
974 });
975 },
976 });
977
978 defaultRules.add({
979 name: "refimage",
980 match: inlineRegex(
981 new RegExp(
982 // The first [part] of the link
983 "^!\\[(" +
984 LINK_INSIDE +
985 ")\\]" + // The [ref] target of the link
986 "\\s*\\[([^\\]]*)\\]",
987 ),
988 ),
989 parse(capture, parse, state) {
990 return parseRef(capture, state, {
991 type: "image",
992 alt: capture[1],
993 });
994 },
995 });
996
997 defaultRules.add({
998 name: "em",
999 /* same as strong/u */
1000 match: inlineRegex(
1001 new RegExp(
1002 // only match _s surrounding words.
1003 "^\\b_" +
1004 "((?:__|\\\\[\\s\\S]|[^\\\\_])+?)_" +
1005 "\\b" + // Or match *s:
1006 "|" + // Only match *s that are followed by a non-space:
1007 "^\\*(?=\\S)(" + // Match at least one of:
1008 "(?:" + // - `**`: so that bolds inside italics don't close the
1009 // italics
1010 "\\*\\*|" + // - escape sequence: so escaped *s don't close us
1011 "\\\\[\\s\\S]|" + // - whitespace: followed by a non-* (we don't
1012 // want ' *' to close an italics--it might
1013 // start a list)
1014 "\\s+(?:\\\\[\\s\\S]|[^\\s\\*\\\\]|\\*\\*)|" + // - non-whitespace, non-*, non-backslash characters
1015 "[^\\s\\*\\\\]" +
1016 ")+?" + // followed by a non-space, non-* then *
1017 ")\\*(?!\\*)",
1018 ),
1019 ),
1020 quality(capture) {
1021 // precedence by length, `em` wins ties:
1022 return capture[0].length + 0.2;
1023 },
1024 parse(capture, parse, state) {
1025 return {
1026 content: parse(capture[2] || capture[1], state),
1027 };
1028 },
1029 });
1030
1031 defaultRules.add({
1032 name: "strong",
1033 /* same as em */
1034 match: inlineRegex(/^\*\*((?:\\[\s\S]|[^\\])+?)\*\*(?!\*)/),
1035 quality(capture) {
1036 // precedence by length, wins ties vs `u`:
1037 return capture[0].length + 0.1;
1038 },
1039 parse: parseCaptureInline,
1040 });
1041
1042 defaultRules.add({
1043 name: "u",
1044 /* same as em&strong; increment for next rule */
1045 match: inlineRegex(/^__((?:\\[\s\S]|[^\\])+?)__(?!_)/),
1046 quality(capture) {
1047 // precedence by length, loses all ties
1048 return capture[0].length;
1049 },
1050 parse: parseCaptureInline,
1051 });
1052
1053 defaultRules.add({
1054 name: "del",
1055 match: inlineRegex(/^~~(?=\S)((?:\\[\s\S]|~(?!~)|[^\s~\\]|\s(?!~~))+?)~~/),
1056 parse: parseCaptureInline,
1057 });
1058
1059 defaultRules.add({
1060 name: "inlineCode",
1061 match: inlineRegex(/^(`+)([\s\S]*?[^`])\1(?!`)/),
1062 parse(capture) {
1063 return {
1064 content: capture[2].replace(INLINE_CODE_ESCAPE_BACKTICKS_R, "$1"),
1065 };
1066 },
1067 });
1068
1069 defaultRules.add({
1070 name: "br",
1071 match: anyScopeRegex(/^ {2,}\n/),
1072 parse: ignoreCapture,
1073 });
1074
1075 defaultRules.add({
1076 name: "text",
1077 // Here we look for anything followed by non-symbols,
1078 // double newlines, or double-space-newlines
1079 // We break on any symbol characters so that this grammar
1080 // is easy to extend without needing to modify this regex
1081 match: anyScopeRegex(
1082 /^[\s\S]+?(?=[^0-9A-Za-z\s\u00c0-\uffff]|\n\n| {2,}\n|\w+:\S|$)/,
1083 ),
1084 parse: function (capture) {
1085 return {
1086 content: capture[0],
1087 };
1088 },
1089 });
1090}
1// @ts-nocheck
2// # Simple-Markdown Core
3//
4// This is a fork of Khan-academy's Simple-Markdown[1], initially forked in 2022
5// to add Svelte support[2], and used for paper clover's q+a markdown flavor.
6//
7// 1: https://github.com/Khan/perseus/tree/main/packages/simple-markdown/src
8// 2: https://github.com/paperclover/svelte-simple-markdown
9export type Rules = Record<string, ParserRule>;
10export interface ParserRule {
11 name: string;
12 match: MatchFunction;
13 parse: ParseFunction;
14 quality?: QualityFunction;
15}
16
17export class RuleList extends Array<ParserRule> {
18 constructor(input?: ArrayLike<ParserRule>) {
19 super();
20 if (input) {
21 this.push(...Array.from(input));
22 }
23 }
24
25 insertBefore(rule: string, newRule: ParserRule): void {
26 const index = this.findIndex((r) => r.name === rule);
27 if (index === -1) {
28 throw new Error(`Rule ${rule} not found`);
29 }
30 this.splice(index, 0, newRule);
31 }
32
33 insertAfter(rule: string, newRule: ParserRule): void {
34 const index = this.findIndex((r) => r.name === rule);
35 if (index === -1) {
36 throw new Error(`Rule ${rule} not found`);
37 }
38 this.splice(index + 1, 0, newRule);
39 }
40
41 toRuleObject(): Record<string, ParserRule> {
42 const result: Record<string, ParserRule> = {};
43 this.forEach((rule) => {
44 result[rule.name] = rule;
45 });
46 return result;
47 }
48
49 add(rule: ParserRule): void {
50 this.push(rule);
51 }
52
53 get(rule: string): ParserRule | undefined {
54 return this.find((r) => r.name === rule);
55 }
56
57 remove(rule: string): void {
58 const index = this.findIndex((r) => r.name === rule);
59 if (index === -1) {
60 throw new Error(`Rule ${rule} not found`);
61 }
62 this.splice(index, 1);
63 }
64
65 clone() {
66 return new RuleList(this);
67 }
68}
69
70/**
71 * Creates a parser for a given set of rules, with the precedence
72 * specified as a list of rules.
73 *
74 * @param rules
75 * an object containing
76 * rule type -> {match, order, parse} objects
77 * (lower order is higher precedence)
78 * @param [defaultState]
79 *
80 * @returns
81 * The resulting parse function, with the following parameters:
82 * @source: the input source string to be parsed
83 * @state: an optional object to be threaded through parse
84 * calls. Allows clients to add stateful operations to
85 * parsing, such as keeping track of how many levels deep
86 * some nesting is. For an example use-case, see passage-ref
87 * parsing in src/widgets/passage/passage-markdown.jsx
88 */
89export function createParser(
90 ruleListInput: RuleList,
91 defaultState: Partial<ParserState> = {},
92) {
93 let rules = ruleListInput.toRuleObject();
94 let ruleList = Object.keys(rules);
95
96 let latestState: ParserState;
97
98 let nestedParse = function (source: string, state?: ParserState) {
99 let result: ASTNode[] = [];
100 state = state || latestState;
101 latestState = state;
102
103 while (source) {
104 // store the best match, it's rule, and quality:
105 let ruleType = null;
106 let rule = null;
107 let capture = null;
108 let quality = NaN; // loop control variables:
109
110 let i = 0;
111 let currRuleType = ruleList[0];
112
113 let currRule = rules[currRuleType];
114
115 do {
116 let currCapture = currRule.match(source, state);
117
118 if (currCapture) {
119 let currQuality = currRule.quality
120 ? currRule.quality(currCapture, state)
121 : 0;
122
123 // This should always be true the first time because
124 // the initial quality is NaN (that's why there's the
125 // condition negation).
126 if (!(currQuality <= quality)) {
127 ruleType = currRuleType;
128 rule = currRule;
129 capture = currCapture;
130 quality = currQuality;
131 }
132 }
133
134 // Move on to the next item.
135 // Note that this makes `currRule` be the next item
136 i++;
137 currRuleType = ruleList[i];
138 currRule = rules[currRuleType];
139 } while (
140 // keep looping while we're still within the ruleList
141 currRule &&
142 // if we don't have a match yet, continue
143 (!capture ||
144 // or if we have a match, but the next rule is
145 // at the same order, and has a quality measurement
146 // functions, then this rule must have a quality
147 // measurement function (since they are sorted before
148 // those without), and we need to check if there is
149 // a better quality match
150 currRule.quality)
151 );
152
153 if (!rule || !capture || !ruleType) {
154 throw new Error(
155 "Could not find a matching rule for the below " +
156 "content. The rule with highest `order` should " +
157 "always match content provided to it. Check " +
158 "the definition of `match` for '" +
159 ruleList[ruleList.length - 1] +
160 "'. It seems to not match the following source:\n" +
161 source,
162 );
163 }
164
165 if (capture.index) {
166 // If present and non-zero, i.e. a non-^ regexp result:
167 throw new Error(
168 "`match` must return a capture starting at index 0 " +
169 "(the current parse index). Did you forget a ^ at the " +
170 "start of the RegExp?",
171 );
172 }
173
174 let parsed = rule.parse(capture, nestedParse, state);
175
176 // We maintain the same object here so that rules can
177 // store references to the objects they return and
178 // modify them later. (oops sorry! but this adds a lot
179 // of power--see reflinks.)
180
181 // We also let rules override the default type of
182 // their parsed node if they would like to, so that
183 // there can be a single output function for all links,
184 // even if there are several rules to parse them.
185 if (!parsed.type) {
186 parsed.type = ruleType;
187 }
188
189 // Collapse text nodes
190 if (
191 parsed.type === "text" && result[result.length - 1]?.type === "text"
192 ) {
193 result[result.length - 1].content += parsed.content;
194 } else {
195 result.push(parsed as ASTNode);
196 }
197
198 state.prevCapture = capture;
199 source = source.substring(state.prevCapture[0].length);
200 }
201
202 return result;
203 };
204
205 let outerParse = function (
206 source: string,
207 state: ParserState = { inline: false },
208 ) {
209 latestState = populateInitialState(state, defaultState);
210
211 if (!latestState.inline && !latestState.disableAutoBlockNewlines) {
212 source = source + "\n\n";
213 }
214
215 // We store the previous capture so that match functions can
216 // use some limited amount of lookbehind. Lists use this to
217 // ensure they don't match arbitrary '- ' or '* ' in inline
218 // text (see the list rule for more information). This stores
219 // the full regex capture object, if there is one.
220 latestState.prevCapture = undefined;
221 return nestedParse(preprocess(source), latestState);
222 };
223
224 return outerParse;
225}
226
227type Multiple<T> = T | T[];
228type Nullable<T> = T | null | undefined;
229
230export type MatchFunction = (
231 source: string,
232 state: ParserState,
233) => Nullable<RegExpMatchArray>;
234
235export type Parser = (source: string, state?: ParserState) => ASTNode[];
236
237export type ParseFunction = (
238 source: RegExpMatchArray,
239 nestedParse: Parser,
240 state: ParserState,
241) => TypeOptionalASTNode;
242
243export type QualityFunction = (
244 capture: RegExpMatchArray,
245 state: ParserState,
246) => number;
247
248export interface ParserState {
249 inline: boolean;
250 prevCapture?: RegExpMatchArray;
251 [key: string]: any;
252}
253
254export interface ASTNode {
255 type: string;
256 content?: ASTNode[] | string;
257 [key: string]: any;
258}
259
260export type TypeOptionalASTNode = Omit<ASTNode, "type"> & { type?: string };
261
262export interface RefNode {
263 type: string;
264 content?: Multiple<ASTNode>;
265 target?: string;
266 title?: string;
267 alt?: string;
268}
269
270/** Creates a match function for an inline scoped element from a regex */
271export function inlineRegex(regex: RegExp): MatchFunction {
272 return (source, state) => {
273 if (state.inline) {
274 return regex.exec(source);
275 } else {
276 return null;
277 }
278 };
279}
280
281/** Creates a match function for a block scoped element from a regex */
282export function blockRegex(regex: RegExp): MatchFunction {
283 return (source, state) => {
284 if (state.inline) {
285 return null;
286 } else {
287 return regex.exec(source);
288 }
289 };
290}
291
292/** Creates a match function from a regex, ignoring block/inline scope */
293export function anyScopeRegex(regex: RegExp): MatchFunction {
294 return (source) => {
295 return regex.exec(source);
296 };
297}
298
299const UNESCAPE_URL_R = /\\([^0-9A-Za-z\s])/g;
300export function unescapeUrl(rawUrlString: string) {
301 return rawUrlString.replace(UNESCAPE_URL_R, "$1");
302}
303
304/**
305 * Parse some content with the parser `parse`, with state.inline
306 * set to true. Useful for block elements; not generally necessary
307 * to be used by inline elements (where state.inline is already true.
308 */
309export function parseInline(
310 parse: Parser,
311 content: string,
312 state: ParserState,
313) {
314 const isCurrentlyInline = state.inline || false;
315 state.inline = true;
316 const result = parse(content, state);
317 state.inline = isCurrentlyInline;
318 return result;
319}
320
321export function parseBlock(parse: Parser, content: string, state: ParserState) {
322 const isCurrentlyInline = state.inline || false;
323 state.inline = false;
324 const result = parse(content + "\n\n", state);
325 state.inline = isCurrentlyInline;
326 return result;
327}
328
329export function parseCaptureInline(
330 capture: RegExpMatchArray,
331 parse: Parser,
332 state: ParserState,
333) {
334 return {
335 content: parseInline(parse, capture[1], state),
336 };
337}
338
339export function ignoreCapture() {
340 return {};
341}
342
343export function sanitizeUrl(url?: string) {
344 if (url == null) {
345 return null;
346 }
347 try {
348 const prot = new URL(url, "https://localhost").protocol;
349 if (
350 prot.indexOf("javascript:") === 0 ||
351 prot.indexOf("vbscript:") === 0 ||
352 prot.indexOf("data:") === 0
353 ) {
354 return null;
355 }
356 } catch (e) {
357 // invalid URLs should throw a TypeError
358 // see for instance: `new URL("");`
359 return null;
360 }
361 return url;
362}
363
364const CR_NEWLINE_R = /\r\n?/g;
365const TAB_R = /\t/g;
366const FORMFEED_R = /\f/g;
367
368/**
369 * Turn various whitespace into easy-to-process whitespace
370 */
371function preprocess(source: string) {
372 return source.replace(CR_NEWLINE_R, "\n").replace(FORMFEED_R, "").replace(
373 TAB_R,
374 " ",
375 );
376}
377
378function populateInitialState(
379 givenState: Partial<ParserState>,
380 defaultState: Partial<ParserState>,
381) {
382 let state = givenState || {};
383
384 for (let prop in defaultState) {
385 if (Object.prototype.hasOwnProperty.call(defaultState, prop)) {
386 state[prop] = defaultState[prop];
387 }
388 }
389
390 return state as ParserState;
391}
392
393// recognize a `*` `-`, `+`, `1.`, `2.`... list bullet
394const LIST_BULLET = "(?:[*+-]|\\d+\\.)";
395
396// recognize the start of a list item:
397// leading space plus a bullet plus a space (` * `)
398const LIST_ITEM_PREFIX = "( *)(" + LIST_BULLET + ") +";
399const LIST_ITEM_PREFIX_R = new RegExp("^" + LIST_ITEM_PREFIX);
400
401// recognize an individual list item:
402// * hi
403// this is part of the same item
404//
405// as is this, which is a new paragraph in the same item
406//
407// * but this is not part of the same item
408const LIST_ITEM_R = new RegExp(
409 LIST_ITEM_PREFIX + "[^\\n]*(?:\\n" + "(?!\\1" + LIST_BULLET +
410 " )[^\\n]*)*(\n|$)",
411 "gm",
412);
413const BLOCK_END_R = /\n{2,}$/;
414const INLINE_CODE_ESCAPE_BACKTICKS_R = /^ (?= *`)|(` *) $/g;
415
416// recognize the end of a paragraph block inside a list item:
417// two or more newlines at end end of the item
418const LIST_BLOCK_END_R = BLOCK_END_R;
419const LIST_ITEM_END_R = / *\n+$/;
420
421// check whether a list item has paragraphs: if it does,
422// we leave the newlines at the end
423const LIST_R = new RegExp(
424 "^( *)(" +
425 LIST_BULLET +
426 ") " +
427 "[\\s\\S]+?(?:\n{2,}(?! )" +
428 "(?!\\1" +
429 LIST_BULLET +
430 " )\\n*" +
431 // the \\s*$ here is so that we can parse the inside of nested
432 // lists, where our content might end before we receive two `\n`s
433 "|\\s*\n*$)",
434);
435const LIST_LOOKBEHIND_R = /(?:^|\n)( *)$/;
436
437const TABLES = (function () {
438 const TABLE_ROW_SEPARATOR_TRIM = /^ *\| *| *\| *$/g;
439 const TABLE_CELL_END_TRIM = / *$/;
440 const TABLE_RIGHT_ALIGN = /^ *-+: *$/;
441 const TABLE_CENTER_ALIGN = /^ *:-+: *$/;
442 const TABLE_LEFT_ALIGN = /^ *:-+ *$/; // TODO: This needs a real type
443
444 const parseTableAlignCapture = (alignCapture: string) => {
445 if (TABLE_RIGHT_ALIGN.test(alignCapture)) {
446 return "right";
447 } else if (TABLE_CENTER_ALIGN.test(alignCapture)) {
448 return "center";
449 } else if (TABLE_LEFT_ALIGN.test(alignCapture)) {
450 return "left";
451 } else {
452 return null;
453 }
454 };
455
456 const parseTableAlign = (source: string, trimEndSeparators: boolean) => {
457 if (trimEndSeparators) {
458 source = source.replace(TABLE_ROW_SEPARATOR_TRIM, "");
459 }
460
461 const alignText = source.trim().split("|");
462 return alignText.map(parseTableAlignCapture);
463 };
464
465 const parseTableRow = (
466 source: string,
467 parse: Parser,
468 state: ParserState,
469 trimEndSeparators: boolean,
470 ) => {
471 const prevInTable = state.inTable;
472 state.inTable = true;
473 const tableRow = parse(source.trim(), state);
474 state.inTable = prevInTable;
475 const cells: ASTNode[][] = [[]];
476 tableRow.forEach(function (node, i) {
477 if (node.type === "tableSeparator") {
478 // Filter out empty table separators at the start/end:
479 if (!trimEndSeparators || (i !== 0 && i !== tableRow.length - 1)) {
480 // Split the current row:
481 cells.push([]);
482 }
483 } else {
484 if (
485 typeof node.content === "string" &&
486 (tableRow[i + 1] == null || tableRow[i + 1].type === "tableSeparator")
487 ) {
488 node.content = node.content.replace(TABLE_CELL_END_TRIM, "");
489 }
490
491 cells[cells.length - 1].push(node);
492 }
493 });
494 return cells;
495 };
496
497 /**
498 * @param {string} source
499 * @param {SimpleMarkdown.Parser} parse
500 * @param {SimpleMarkdown.State} state
501 * @param {boolean} trimEndSeparators
502 * @returns {SimpleMarkdown.ASTNode[][]}
503 */
504 const parseTableCells = function (
505 source: string,
506 parse: Parser,
507 state: ParserState,
508 trimEndSeparators: boolean,
509 ) {
510 const rowsText = source.trim().split("\n");
511 return rowsText.map(function (rowText) {
512 return parseTableRow(rowText, parse, state, trimEndSeparators);
513 });
514 };
515
516 /**
517 * @param {boolean} trimEndSeparators
518 * @returns {SimpleMarkdown.SingleNodeParseFunction}
519 */
520 const parseTable = function (trimEndSeparators: boolean) {
521 return function (
522 capture: RegExpMatchArray,
523 parse: Parser,
524 state: ParserState,
525 ) {
526 state.inline = true;
527 const header = parseTableRow(capture[1], parse, state, trimEndSeparators);
528 const align = parseTableAlign(capture[2], trimEndSeparators);
529 const cells = parseTableCells(
530 capture[3],
531 parse,
532 state,
533 trimEndSeparators,
534 );
535 state.inline = false;
536 return {
537 type: "table",
538 header: header,
539 align: align,
540 cells: cells,
541 };
542 };
543 };
544
545 return {
546 parseTable: parseTable(true),
547 parseNpTable: parseTable(false),
548 TABLE_REGEX: /^ *(\|.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*/,
549 NPTABLE_REGEX:
550 /^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*/,
551 };
552})();
553
554const LINK_INSIDE = "(?:\\[[^\\]]*\\]|[^\\[\\]]|\\](?=[^\\[]*\\]))*";
555const LINK_HREF_AND_TITLE =
556 "\\s*<?((?:\\([^)]*\\)|[^\\s\\\\]|\\\\.)*?)>?(?:\\s+['\"]([\\s\\S]*?)['\"])?\\s*";
557const AUTOLINK_MAILTO_CHECK_R = /mailto:/i;
558
559function parseRef(
560 capture: RegExpMatchArray,
561 state: ParserState,
562 refNode: RefNode,
563) {
564 const ref = (capture[2] || capture[1]).replace(/\s+/g, " ").toLowerCase();
565
566 // We store information about previously seen defs on
567 // state._defs (_ to deconflict with client-defined
568 // state). If the def for this reflink/refimage has
569 // already been seen, we can use its target/source
570 // and title here:
571 if (state._defs && state._defs[ref]) {
572 const def = state._defs[ref];
573
574 // `refNode` can be a link or an image. Both use
575 // target and title properties.
576 refNode.target = def.target;
577 refNode.title = def.title;
578 }
579
580 // In case we haven't seen our def yet (or if someone
581 // overwrites that def later on), we add this node
582 // to the list of ref nodes for that def. Then, when
583 // we find the def, we can modify this link/image AST
584 // node :).
585 // I'm sorry.
586 state._refs = state._refs || {};
587 state._refs[ref] = state._refs[ref] || [];
588
589 state._refs[ref].push(refNode);
590
591 return refNode;
592}
593
594export const defaultRules = new RuleList();
595{
596 defaultRules.add({
597 name: "heading",
598 match: blockRegex(/^ *(#{1,6})([^\n]+?)#* *(?:\n *)+\n/),
599 parse: function (capture, parse, state) {
600 return {
601 level: capture[1].length,
602 content: parseInline(parse, capture[2].trim(), state),
603 };
604 },
605 });
606
607 defaultRules.add({
608 name: "nptable",
609 match: blockRegex(TABLES.NPTABLE_REGEX),
610 parse: TABLES.parseNpTable,
611 });
612
613 defaultRules.add({
614 name: "lheading",
615 match: blockRegex(/^([^\n]+)\n *(=|-){3,} *(?:\n *)+\n/),
616 parse(capture, parse, state) {
617 return {
618 type: "heading",
619 level: capture[2] === "=" ? 1 : 2,
620 content: parseInline(parse, capture[1], state),
621 };
622 },
623 });
624
625 defaultRules.add({
626 name: "hr",
627 match: blockRegex(/^( *[-*_]){3,} *(?:\n *)+\n/),
628 parse: ignoreCapture,
629 });
630
631 defaultRules.add({
632 name: "codeBlock",
633 match: blockRegex(/^(?: {4}[^\n]+\n*)+(?:\n *)+\n/),
634 parse(capture) {
635 const content = capture[0].replace(/^ {4}/gm, "").replace(/\n+$/, "");
636 return {
637 lang: undefined,
638 content: content,
639 };
640 },
641 });
642
643 defaultRules.add({
644 name: "fence",
645 match: blockRegex(
646 /^ *(`{3,}|~{3,}) *(?:(\S+) *)?\n([\s\S]+?)\n?\1 *(?:\n *)+\n/,
647 ),
648 parse(capture) {
649 return {
650 type: "codeBlock",
651 lang: capture[2] || undefined,
652 content: capture[3],
653 };
654 },
655 });
656
657 defaultRules.add({
658 name: "blockQuote",
659 match: blockRegex(/^( *>[^\n]+(\n[^\n]+)*\n*)+\n{2,}/),
660 parse(capture, parse, state) {
661 const content = capture[0].replace(/^ *> ?/gm, "");
662 return {
663 content: parse(content, state),
664 };
665 },
666 });
667
668 defaultRules.add({
669 name: "list",
670
671 match(source, state) {
672 // We only want to break into a list if we are at the start of a
673 // line. This is to avoid parsing "hi * there" with "* there"
674 // becoming a part of a list.
675 // You might wonder, "but that's inline, so of course it wouldn't
676 // start a list?". You would be correct! Except that some of our
677 // lists can be inline, because they might be inside another list,
678 // in which case we can parse with inline scope, but need to allow
679 // nested lists inside this inline scope.
680 const prevCaptureStr = state.prevCapture == null
681 ? ""
682 : state.prevCapture[0];
683 const isStartOfLineCapture = LIST_LOOKBEHIND_R.exec(prevCaptureStr);
684 const isListBlock = state._list || !state.inline;
685
686 if (isStartOfLineCapture && isListBlock) {
687 source = isStartOfLineCapture[1] + source;
688 return LIST_R.exec(source);
689 } else {
690 return null;
691 }
692 },
693 parse(capture, parse, state) {
694 const bullet = capture[2];
695 const ordered = bullet.length > 1;
696 const start = ordered ? +bullet : undefined;
697 // We know this will match here, because of how the regexes are defined
698 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
699 const items = capture[0].replace(LIST_BLOCK_END_R, "\n").match(
700 LIST_ITEM_R,
701 )!;
702
703 let lastItemWasAParagraph = false;
704 const itemContent = items.map(function (item, i) {
705 // We need to see how far indented this item is:
706 const prefixCapture = LIST_ITEM_PREFIX_R.exec(item);
707 const space = prefixCapture ? prefixCapture[0].length : 0; // And then we construct a regex to "unindent" the subsequent
708 // lines of the items by that amount:
709
710 const spaceRegex = new RegExp("^ {1," + space + "}", "gm"); // Before processing the item, we need a couple things
711
712 const content = item // remove indents on trailing lines:
713 .replace(spaceRegex, "") // remove the bullet:
714 .replace(LIST_ITEM_PREFIX_R, ""); // I'm not sur4 why this is necessary again?
715 // Handling "loose" lists, like:
716 //
717 // * this is wrapped in a paragraph
718 //
719 // * as is this
720 //
721 // * as is this
722
723 const isLastItem = i === items.length - 1;
724 const containsBlocks = content.indexOf("\n\n") !== -1; // Any element in a list is a block if it contains multiple
725 // newlines. The last element in the list can also be a block
726 // if the previous item in the list was a block (this is
727 // because non-last items in the list can end with \n\n, but
728 // the last item can't, so we just "inherit" this property
729 // from our previous element).
730
731 const thisItemIsAParagraph = containsBlocks ||
732 (isLastItem && lastItemWasAParagraph);
733 lastItemWasAParagraph = thisItemIsAParagraph; // backup our state for restoration afterwards. We're going to
734 // want to set state._list to true, and state.inline depending
735 // on our list's looseness.
736
737 const oldStateInline = state.inline;
738 const oldStateList = state._list;
739 state._list = true; // Parse inline if we're in a tight list, or block if we're in
740 // a loose list.
741
742 let adjustedContent;
743
744 if (thisItemIsAParagraph) {
745 state.inline = false;
746 adjustedContent = content.replace(LIST_ITEM_END_R, "\n\n");
747 } else {
748 state.inline = true;
749 adjustedContent = content.replace(LIST_ITEM_END_R, "");
750 }
751
752 const result = parse(adjustedContent, state); // Restore our state before returning
753
754 state.inline = oldStateInline;
755 state._list = oldStateList;
756 return result;
757 });
758
759 return {
760 ordered: ordered,
761 start: start,
762 content: itemContent,
763 };
764 },
765 });
766
767 defaultRules.add({
768 name: "def",
769 // TODO: This will match without a blank line before the next
770 // block element, which is inconsistent with most of the rest of
771 // simple-markdown.
772 match: blockRegex(
773 /^ *\[([^\]]+)\]: *<?([^\s>]*)>?(?: +["(]([^\n]+)[")])? *\n(?: *\n)*/,
774 ),
775 parse(capture, parse, state) {
776 const def = capture[1].replace(/\s+/g, " ").toLowerCase();
777 const target = capture[2];
778 const title = capture[3];
779
780 // Look for previous links/images using this def
781 // If any links/images using this def have already been declared,
782 // they will have added themselves to the state._refs[def] list
783 // (_ to deconflict with client-defined state). We look through
784 // that list of reflinks for this def, and modify those AST nodes
785 // with our newly found information now.
786 // Sorry :(.
787 if (state._refs && state._refs[def]) {
788 // `refNode` can be a link or an image
789 state._refs[def].forEach((refNode: RefNode) => {
790 refNode.target = target;
791 refNode.title = title;
792 });
793 }
794
795 // Add this def to our map of defs for any future links/images
796 // In case we haven't found any or all of the refs referring to
797 // this def yet, we add our def to the table of known defs, so
798 // that future reflinks can modify themselves appropriately with
799 // this information.
800 state._defs = state._defs || {};
801 state._defs[def] = {
802 target: target,
803 title: title,
804 };
805
806 // return the relevant parsed information
807 // for debugging only.
808 return {
809 def: def,
810 target: target,
811 title: title,
812 };
813 },
814 });
815
816 defaultRules.add({
817 name: "table",
818 match: blockRegex(TABLES.TABLE_REGEX),
819 parse: TABLES.parseTable,
820 });
821
822 defaultRules.add({
823 name: "newline",
824 match: blockRegex(/^(?:\n *)*\n/),
825 parse: ignoreCapture,
826 });
827
828 defaultRules.add({
829 name: "paragraph",
830 match: blockRegex(/^((?:[^\n]|\n(?! *\n))+)(?:\n *)+\n/),
831 parse: parseCaptureInline,
832 });
833
834 defaultRules.add({
835 name: "escape",
836 // We don't allow escaping numbers, letters, or spaces here so that
837 // backslashes used in plain text still get rendered. But allowing
838 // escaping anything else provides a very flexible escape mechanism,
839 // regardless of how this grammar is extended.
840 match: inlineRegex(/^\\([^0-9A-Za-z\s])/),
841 parse(capture) {
842 return {
843 type: "text",
844 content: capture[1],
845 };
846 },
847 });
848
849 defaultRules.add({
850 name: "tableSeparator",
851
852 match(source, state) {
853 if (!state.inTable) {
854 return null;
855 }
856
857 return /^ *\| */.exec(source);
858 },
859 parse() {
860 return {
861 type: "tableSeparator",
862 };
863 },
864 });
865
866 defaultRules.add({
867 name: "autolink",
868 match: inlineRegex(/^<([^: >]+:\/[^ >]+)>/),
869 parse(capture) {
870 return {
871 type: "link",
872 content: [
873 {
874 type: "text",
875 content: capture[1],
876 },
877 ],
878 target: capture[1],
879 };
880 },
881 });
882
883 defaultRules.add({
884 name: "mailto",
885 match: inlineRegex(/^<([^ >]+@[^ >]+)>/),
886 parse(capture) {
887 const address = capture[1];
888 let target = capture[1]; // Check for a `mailto:` already existing in the link:
889
890 if (!AUTOLINK_MAILTO_CHECK_R.test(target)) {
891 target = "mailto:" + target;
892 }
893
894 return {
895 type: "link",
896 content: [
897 {
898 type: "text",
899 content: address,
900 },
901 ],
902 target: target,
903 };
904 },
905 });
906
907 defaultRules.add({
908 name: "url",
909 match: inlineRegex(/^(https?:\/\/[^\s<]+[^<.,:;"')\]\s])/),
910 parse(capture) {
911 return {
912 type: "link",
913 content: [
914 {
915 type: "text",
916 content: capture[1],
917 },
918 ],
919 target: capture[1],
920 title: undefined,
921 };
922 },
923 });
924
925 defaultRules.add({
926 name: "link",
927 match: inlineRegex(
928 new RegExp(
929 "^\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)",
930 ),
931 ),
932 parse(capture, parse, state) {
933 const link = {
934 content: parse(capture[1], state),
935 target: unescapeUrl(capture[2]),
936 title: capture[3],
937 };
938 return link;
939 },
940 });
941
942 defaultRules.add({
943 name: "image",
944 match: inlineRegex(
945 new RegExp(
946 "^!\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)",
947 ),
948 ),
949 parse: function (capture) {
950 const image = {
951 alt: capture[1],
952 target: unescapeUrl(capture[2]),
953 title: capture[3],
954 };
955 return image;
956 },
957 });
958
959 defaultRules.add({
960 name: "reflink",
961 match: inlineRegex(
962 new RegExp(
963 // The first [part] of the link
964 "^\\[(" +
965 LINK_INSIDE +
966 ")\\]" + // The [ref] target of the link
967 "\\s*\\[([^\\]]*)\\]",
968 ),
969 ),
970 parse(capture, parse, state) {
971 return parseRef(capture, state, {
972 type: "link",
973 content: parse(capture[1], state),
974 });
975 },
976 });
977
978 defaultRules.add({
979 name: "refimage",
980 match: inlineRegex(
981 new RegExp(
982 // The first [part] of the link
983 "^!\\[(" +
984 LINK_INSIDE +
985 ")\\]" + // The [ref] target of the link
986 "\\s*\\[([^\\]]*)\\]",
987 ),
988 ),
989 parse(capture, parse, state) {
990 return parseRef(capture, state, {
991 type: "image",
992 alt: capture[1],
993 });
994 },
995 });
996
997 defaultRules.add({
998 name: "em",
999 /* same as strong/u */
1000 match: inlineRegex(
1001 new RegExp(
1002 // only match _s surrounding words.
1003 "^\\b_" +
1004 "((?:__|\\\\[\\s\\S]|[^\\\\_])+?)_" +
1005 "\\b" + // Or match *s:
1006 "|" + // Only match *s that are followed by a non-space:
1007 "^\\*(?=\\S)(" + // Match at least one of:
1008 "(?:" + // - `**`: so that bolds inside italics don't close the
1009 // italics
1010 "\\*\\*|" + // - escape sequence: so escaped *s don't close us
1011 "\\\\[\\s\\S]|" + // - whitespace: followed by a non-* (we don't
1012 // want ' *' to close an italics--it might
1013 // start a list)
1014 "\\s+(?:\\\\[\\s\\S]|[^\\s\\*\\\\]|\\*\\*)|" + // - non-whitespace, non-*, non-backslash characters
1015 "[^\\s\\*\\\\]" +
1016 ")+?" + // followed by a non-space, non-* then *
1017 ")\\*(?!\\*)",
1018 ),
1019 ),
1020 quality(capture) {
1021 // precedence by length, `em` wins ties:
1022 return capture[0].length + 0.2;
1023 },
1024 parse(capture, parse, state) {
1025 return {
1026 content: parse(capture[2] || capture[1], state),
1027 };
1028 },
1029 });
1030
1031 defaultRules.add({
1032 name: "strong",
1033 /* same as em */
1034 match: inlineRegex(/^\*\*((?:\\[\s\S]|[^\\])+?)\*\*(?!\*)/),
1035 quality(capture) {
1036 // precedence by length, wins ties vs `u`:
1037 return capture[0].length + 0.1;
1038 },
1039 parse: parseCaptureInline,
1040 });
1041
1042 defaultRules.add({
1043 name: "u",
1044 /* same as em&strong; increment for next rule */
1045 match: inlineRegex(/^__((?:\\[\s\S]|[^\\])+?)__(?!_)/),
1046 quality(capture) {
1047 // precedence by length, loses all ties
1048 return capture[0].length;
1049 },
1050 parse: parseCaptureInline,
1051 });
1052
1053 defaultRules.add({
1054 name: "del",
1055 match: inlineRegex(/^~~(?=\S)((?:\\[\s\S]|~(?!~)|[^\s~\\]|\s(?!~~))+?)~~/),
1056 parse: parseCaptureInline,
1057 });
1058
1059 defaultRules.add({
1060 name: "inlineCode",
1061 match: inlineRegex(/^(`+)([\s\S]*?[^`])\1(?!`)/),
1062 parse(capture) {
1063 return {
1064 content: capture[2].replace(INLINE_CODE_ESCAPE_BACKTICKS_R, "$1"),
1065 };
1066 },
1067 });
1068
1069 defaultRules.add({
1070 name: "br",
1071 match: anyScopeRegex(/^ {2,}\n/),
1072 parse: ignoreCapture,
1073 });
1074
1075 defaultRules.add({
1076 name: "text",
1077 // Here we look for anything followed by non-symbols,
1078 // double newlines, or double-space-newlines
1079 // We break on any symbol characters so that this grammar
1080 // is easy to extend without needing to modify this regex
1081 match: anyScopeRegex(
1082 /^[\s\S]+?(?=[^0-9A-Za-z\s\u00c0-\uffff]|\n\n| {2,}\n|\w+:\S|$)/,
1083 ),
1084 parse: function (capture) {
1085 return {
1086 content: capture[0],
1087 };
1088 },
1089 });
1090}
src/q+a/views/backend-inbox.client.ts+74-74
......@@ -1,74 +1,74 @@
1// @ts-ignore
2globalThis.onReply = (id: string) => {
3 location.href = `/admin/q+a/${id}?return=inbox`;
4};
5// @ts-ignore
6globalThis.onDelete = async (id: string) => {
7 const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement;
8 if (!div) return alert("Question not found");
9
10 // Pending State
11 div.style.opacity = "0.5";
12 div.style.pointerEvents = "none";
13 div?.querySelectorAll("button").forEach((b) => {
14 b.disabled = true;
15 });
16
17 try {
18 const resp = await fetch(`/admin/q+a/${id}`, {
19 method: "DELETE",
20 headers: {
21 Accept: "application/json",
22 },
23 });
24 if (resp.status !== 200) {
25 throw new Error("Failed to delete question, status: " + resp.status);
26 }
27 } catch (e: any) {
28 div.style.opacity = "1";
29 div.style.pointerEvents = "auto";
30 div?.querySelectorAll("button").forEach((b) => {
31 b.disabled = false;
32 });
33 return alert(e.message);
34 }
35
36 div.remove();
37};
38// @ts-ignore
39globalThis.onDeleteFull = async (id: string) => {
40 const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement;
41 if (!div) return alert("Question not found");
42
43 // Confirmation
44 if (!confirm("Are you sure you want to delete this question?")) return;
45
46 // Pending State
47 div.style.opacity = "0.5";
48 div.style.pointerEvents = "none";
49 div?.querySelectorAll("button").forEach((b) => {
50 b.disabled = true;
51 });
52
53 try {
54 const resp = await fetch(`/admin/q+a/${id}`, {
55 method: "DELETE",
56 headers: {
57 Accept: "application/json",
58 "X-Delete-Full": "true",
59 },
60 });
61 if (resp.status !== 200) {
62 throw new Error("Failed to delete question, status: " + resp.status);
63 }
64 } catch (e: any) {
65 div.style.opacity = "1";
66 div.style.pointerEvents = "auto";
67 div?.querySelectorAll("button").forEach((b) => {
68 b.disabled = false;
69 });
70 return alert(e.message);
71 }
72
73 div.remove();
74};
1// @ts-ignore
2globalThis.onReply = (id: string) => {
3 location.href = `/admin/q+a/${id}?return=inbox`;
4};
5// @ts-ignore
6globalThis.onDelete = async (id: string) => {
7 const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement;
8 if (!div) return alert("Question not found");
9
10 // Pending State
11 div.style.opacity = "0.5";
12 div.style.pointerEvents = "none";
13 div?.querySelectorAll("button").forEach((b) => {
14 b.disabled = true;
15 });
16
17 try {
18 const resp = await fetch(`/admin/q+a/${id}`, {
19 method: "DELETE",
20 headers: {
21 Accept: "application/json",
22 },
23 });
24 if (resp.status !== 200) {
25 throw new Error("Failed to delete question, status: " + resp.status);
26 }
27 } catch (e: any) {
28 div.style.opacity = "1";
29 div.style.pointerEvents = "auto";
30 div?.querySelectorAll("button").forEach((b) => {
31 b.disabled = false;
32 });
33 return alert(e.message);
34 }
35
36 div.remove();
37};
38// @ts-ignore
39globalThis.onDeleteFull = async (id: string) => {
40 const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement;
41 if (!div) return alert("Question not found");
42
43 // Confirmation
44 if (!confirm("Are you sure you want to delete this question?")) return;
45
46 // Pending State
47 div.style.opacity = "0.5";
48 div.style.pointerEvents = "none";
49 div?.querySelectorAll("button").forEach((b) => {
50 b.disabled = true;
51 });
52
53 try {
54 const resp = await fetch(`/admin/q+a/${id}`, {
55 method: "DELETE",
56 headers: {
57 Accept: "application/json",
58 "X-Delete-Full": "true",
59 },
60 });
61 if (resp.status !== 200) {
62 throw new Error("Failed to delete question, status: " + resp.status);
63 }
64 } catch (e: any) {
65 div.style.opacity = "1";
66 div.style.pointerEvents = "auto";
67 div?.querySelectorAll("button").forEach((b) => {
68 b.disabled = false;
69 });
70 return alert(e.message);
71 }
72
73 div.remove();
74};
src/q+a/views/editor.css+39-39
......@@ -1,39 +1,39 @@
1#edit-grid {
2 display: grid;
3 grid-template-columns: 1fr 80ch;
4 grid-template-rows: 3rem 1fr;
5 grid-gap: 1em;
6 height: 100vh;
7
8 button {
9 margin-right: 1rem;
10 }
11 main {
12 padding: 0;
13 margin: 0;
14 }
15}
16#topleft, #topright {
17 padding: 1rem;
18}
19#topleft {
20 padding-right: 0rem;
21}
22#topright {
23 padding-left: 0rem;
24}
25#preview {
26 overflow-y: auto;
27 e- {
28 margin-top: 0;
29 }
30}
31#editor {
32 background-color: #303030;
33 overflow-y: scroll;
34 height: 100%;
35}
36.cm-scroller {
37 overflow-y: auto;
38 height: 100%;
39}
1#edit-grid {
2 display: grid;
3 grid-template-columns: 1fr 80ch;
4 grid-template-rows: 3rem 1fr;
5 grid-gap: 1em;
6 height: 100vh;
7
8 button {
9 margin-right: 1rem;
10 }
11 main {
12 padding: 0;
13 margin: 0;
14 }
15}
16#topleft, #topright {
17 padding: 1rem;
18}
19#topleft {
20 padding-right: 0rem;
21}
22#topright {
23 padding-left: 0rem;
24}
25#preview {
26 overflow-y: auto;
27 e- {
28 margin-top: 0;
29 }
30}
31#editor {
32 background-color: #303030;
33 overflow-y: scroll;
34 height: 100%;
35}
36.cm-scroller {
37 overflow-y: auto;
38 height: 100%;
39}
src/static/wurtzhome.html+6-2
......@@ -3,7 +3,9 @@
33 <title>paper clover</title>
44 </head>
55 <body bgcolor="black" style="word-wrap: initial">
6 <main style="display: flex; flex-direction: column; justify-content: center; align-items: center; height: 100vh">
6 <main
7 style="display: flex; flex-direction: column; justify-content: center; align-items: center; height: 100vh"
8 >
79 <div>
810 <p style="margin: 0.5rem 0">
911 <a
......@@ -56,7 +58,9 @@
5658 <font color="#FF8147">feed</font>
5759 </a>
5860 </p>
59 <h1 style="margin: -1.5rem 0 3rem 0; font-size: 7rem; font-weight: 400; font-family: times">
61 <h1
62 style="margin: -1.5rem 0 3rem 0; font-size: 7rem; font-weight: 400; font-family: times"
63 >
6064 <font color="#B8E1FF">paper</font>
6165 <font color="#E8F4FF">clover</font>
6266 </h1>
src/tags/Video.tsx+58-58
......@@ -1,58 +1,58 @@
1import "./video.css";
2import * as path from "node:path";
3import { addScript } from "#sitegen";
4import { PrecomputedBlurhash } from "./blurhash.tsx";
5
6export namespace Video {
7 export interface Props {
8 title: string;
9 width: number;
10 height: number;
11 sources: string[];
12 downloads: string[];
13 poster?: string;
14 posterHash?: string;
15 borderless?: boolean;
16 }
17}
18
19export function Video(
20 { title, sources, height, poster, posterHash, width, borderless }:
21 Video.Props,
22) {
23 addScript("./hls-polyfill.client.ts");
24 return (
25 <figure class={`video ${borderless ? "borderless" : ""}`}>
26 <figcaption>{title}</figcaption>
27 {posterHash && <PrecomputedBlurhash hash={posterHash} />}
28 {poster && <img src={poster} alt="waterfalls" />}
29 <video
30 controls
31 preload="none"
32 style={`width:100%;background:transparent;aspect-ratio:${
33 simplifyFraction(width, height)
34 }`}
35 poster="data:null"
36 >
37 {sources.map((src) => (
38 <source
39 src={src}
40 type={contentTypeFromExt(src)}
41 />
42 ))}
43 </video>
44 </figure>
45 );
46}
47export function contentTypeFromExt(src: string) {
48 if (src.endsWith(".m3u8")) return "application/x-mpegURL";
49 if (src.endsWith(".webm")) return "video/webm";
50 if (src.endsWith(".mp4")) return "video/mp4";
51 if (src.endsWith(".ogg")) return "video/ogg";
52 throw new Error("Unknown video extension: " + path.extname(src));
53}
54const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a;
55function simplifyFraction(n: number, d: number) {
56 const divisor = gcd(n, d);
57 return `${n / divisor}/${d / divisor}`;
58}
1import "./video.css";
2import * as path from "node:path";
3import { addScript } from "#sitegen";
4import { PrecomputedBlurhash } from "./blurhash.tsx";
5
6export namespace Video {
7 export interface Props {
8 title: string;
9 width: number;
10 height: number;
11 sources: string[];
12 downloads: string[];
13 poster?: string;
14 posterHash?: string;
15 borderless?: boolean;
16 }
17}
18
19export function Video(
20 { title, sources, height, poster, posterHash, width, borderless }:
21 Video.Props,
22) {
23 addScript("./hls-polyfill.client.ts");
24 return (
25 <figure class={`video ${borderless ? "borderless" : ""}`}>
26 <figcaption>{title}</figcaption>
27 {posterHash && <PrecomputedBlurhash hash={posterHash} />}
28 {poster && <img src={poster} alt="waterfalls" />}
29 <video
30 controls
31 preload="none"
32 style={`width:100%;background:transparent;aspect-ratio:${
33 simplifyFraction(width, height)
34 }`}
35 poster="data:null"
36 >
37 {sources.map((src) => (
38 <source
39 src={src}
40 type={contentTypeFromExt(src)}
41 />
42 ))}
43 </video>
44 </figure>
45 );
46}
47export function contentTypeFromExt(src: string) {
48 if (src.endsWith(".m3u8")) return "application/x-mpegURL";
49 if (src.endsWith(".webm")) return "video/webm";
50 if (src.endsWith(".mp4")) return "video/mp4";
51 if (src.endsWith(".ogg")) return "video/ogg";
52 throw new Error("Unknown video extension: " + path.extname(src));
53}
54const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a;
55function simplifyFraction(n: number, d: number) {
56 const divisor = gcd(n, d);
57 return `${n / divisor}/${d / divisor}`;
58}
src/tags/video.css+24-25
......@@ -1,25 +1,24 @@
1.video {
2 border: 4px solid var(--fg);
3 display: flex;
4 flex-direction: column;
5 position: relative;
6}
7
8.video > img,
9.video > span {
10 position: absolute;
11 left: 0;
12 top: 0;
13 width: 100%;
14 height: 100%;
15 z-index: -1;
16}
17
18.video figcaption {
19 background-color: var(--fg);
20 color: var(--bg);
21 width: 100%;
22 margin-top: -1px;
23 padding-bottom: 2px;
24}
25
1.video {
2 border: 4px solid var(--fg);
3 display: flex;
4 flex-direction: column;
5 position: relative;
6}
7
8.video > img,
9.video > span {
10 position: absolute;
11 left: 0;
12 top: 0;
13 width: 100%;
14 height: 100%;
15 z-index: -1;
16}
17
18.video figcaption {
19 background-color: var(--fg);
20 color: var(--bg);
21 width: 100%;
22 margin-top: -1px;
23 padding-bottom: 2px;
24}