authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-09-04 23:44:36-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:47-07:00
logfaaeaf2335c85cea2985085469b706f7b6e40939
tree97d4f5e63f5e6a8bee16af66001a336e7a5a792b
parentc8b7e2ef01711a96e294ade99c4c478a1a8f0be8
signature Commit is signed but in an unrecognized format.

feat(file-viewer): new cache layer

instead of compression modes (which do nothing helpful for video), i am embracing the derived asset system; each file gets a directory for arbitrary files to act as "derived assets", stored via the hash. in local mode, the cache layer reads directly the mounted network share. when unavailable, it fetches from the Source of Truth, caching its results on disk. one `Lru` is used to keep the most relevant files in the store, while another one is used to re-use file descriptors. the cache handles concurrent requests to the same disk entry by placing its file descriptor in "w+" mode for reading and writing. this way, the second request can read from the same file descriptor that the first request is writing into. large files are never buffered entirely in memory, which should keep memory usage low. as a measure to make this potentially more memory efficient, the buffer objects given by node:fs/node:http streams -- which are re-allocated for every read -- are weakly held when written and reused when reading. this somewhat avoids file reads in concurrent access situations.

14 files changed, 616 insertions(+), 629 deletions(-)

framework/definitions.d.ts deleted-4
...@@ -1,4 +0,0 @@
1declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
2declare function ASSERT(value: unknown, message?: string): asserts value;
3
4type Timer = ReturnType<typeof setTimeout>;
framework/esbuild-support.ts-2
...@@ -1,5 +1,3 @@...@@ -1,5 +1,3 @@
1type Awaitable<T> = T | Promise<T>;
2
3export function virtualFiles(1export function virtualFiles(
4 map: Record<2 map: Record<
5 string,3 string,
framework/global.d.ts created+5
...@@ -0,0 +1,5 @@
1declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
2declare function ASSERT(value: unknown, message?: string): asserts value;
3
4type Timer = ReturnType<typeof setTimeout>;
5type Awaitable<T> = T | Promise<T>;
framework/incremental.test.ts deleted-56
...@@ -1,56 +0,0 @@
1test("trivial case", async () => {
2 incr.reset();
3
4 const file1 = tmpFile("example.txt");
5 file1.write("one");
6
7 async function compilation() {
8 const first = incr.work({
9 label: "first compute",
10 async run(io) {
11 await setTimeout(1000);
12 const contents = await io.readFile(file1.path);
13 return [contents, Math.random()] as const;
14 },
15 });
16 const second = incr.work({
17 label: "second compute",
18 wait: first,
19 async run(io) {
20 await setTimeout(1000);
21 return io.readWork(first)[0].toUpperCase();
22 },
23 });
24 const third = incr.work({
25 label: "third compute",
26 wait: first,
27 async run(io) {
28 await setTimeout(1000);
29 return io.readWork(first)[1] * 1000;
30 },
31 });
32 return incr.work({
33 label: "last compute",
34 wait: [second, third],
35 async run(io) {
36 await setTimeout(1000);
37 return {
38 second: io.readWork(second),
39 third: io.readWork(third),
40 };
41 },
42 });
43 }
44 const { value: first } = await incr.compile(compilation);
45 const { value: second } = await incr.compile(compilation);
46 ASSERT(first === second);
47 incr.forceInvalidate(file1.path);
48 const { value: third } = await incr.compile(compilation);
49 ASSERT(first !== third);
50 ASSERT(first[0] === third[0]);
51});
52
53import * as incr from "./incremental2.ts";
54import { beforeEach, test } from "node:test";
55import { tmpFile } from "#sitegen/testing";
56import { setTimeout } from "node:timers/promises";
framework/lib/fs.ts+1
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1// Deprecated: use path.ts instead
1// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in2// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in
2// Node.js sync+promise fs methods. For convenince.3// Node.js sync+promise fs methods. For convenince.
3export {4export {
framework/lib/lru.ts+14-9
...@@ -49,7 +49,7 @@ export class Lru<K, V> extends Map<K, V> {...@@ -49,7 +49,7 @@ export class Lru<K, V> extends Map<K, V> {
49 ASSERT(super.delete(key));49 ASSERT(super.delete(key));
50 return true;50 return true;
51 }51 }
52 override get(key: K) {52 override get(key: K): V | undefined {
53 if (this.#head !== key) {53 if (this.#head !== key) {
54 const entry = this.#evict(key);54 const entry = this.#evict(key);
55 if (!entry) return undefined;55 if (!entry) return undefined;
...@@ -197,9 +197,11 @@ export class Lru<K, V> extends Map<K, V> {...@@ -197,9 +197,11 @@ export class Lru<K, V> extends Map<K, V> {
197 let head = this.#head;197 let head = this.#head;
198 while (head !== none) {198 while (head !== none) {
199 const entry = UNWRAP(order.get(head), head);199 const entry = UNWRAP(order.get(head), head);
200 const [key, value] = map(head, super.get(head) as V);200 const kv = map(head, super.get(head) as V);
201 result.push([key, value, entry[0]]);
202 head = entry[2];201 head = entry[2];
202 if (!kv) continue;
203 const [key, value] = kv;
204 result.push([key, value, entry[0]]);
203 }205 }
204 return result;206 return result;
205 }207 }
...@@ -213,7 +215,9 @@ export class Lru<K, V> extends Map<K, V> {...@@ -213,7 +215,9 @@ export class Lru<K, V> extends Map<K, V> {
213 let pk!: K;215 let pk!: K;
214 for (let i = entries.length - 1; i > 0; i -= 1) {216 for (let i = entries.length - 1; i > 0; i -= 1) {
215 const [ik, iv, size] = UNWRAP(entries[i]);217 const [ik, iv, size] = UNWRAP(entries[i]);
216 const [k, v] = map(ik, iv);218 const kv = map(ik, iv);
219 if (!kv) continue;
220 const [k, v] = kv;
217 super.set(k, v);221 super.set(k, v);
218 const current: Order<K> = [size, none, prev ? pk : none, 0];222 const current: Order<K> = [size, none, prev ? pk : none, 0];
219 if (prev) prev[1] = k;223 if (prev) prev[1] = k;
...@@ -227,13 +231,10 @@ export class Lru<K, V> extends Map<K, V> {...@@ -227,13 +231,10 @@ export class Lru<K, V> extends Map<K, V> {
227export type Serialized<K, V> = Array<[K, V, number]>;231export type Serialized<K, V> = Array<[K, V, number]>;
228232
229interface SerializeOptions<IK, IV, OK = IK, OV = IV> {233interface SerializeOptions<IK, IV, OK = IK, OV = IV> {
230 map?: (key: IK, value: IV) => [OK, OV];234 /** null = omit in serialization/serialization */
235 map?: (key: IK, value: IV) => [OK, OV] | null;
231}236}
232237
233/**
234 * To indicate the start or end of the list, set the key to
235 * itself. Otherwise, it is ambiguous with null or undefined keys.
236 */
237type Order<K> = [238type Order<K> = [
238 /** Number of units computed at insertion time. */239 /** Number of units computed at insertion time. */
239 size: number,240 size: number,
...@@ -249,5 +250,9 @@ interface Lock extends Disposable {...@@ -249,5 +250,9 @@ interface Lock extends Disposable {
249 (): void;250 (): void;
250}251}
251252
253export declare namespace Lru {
254 export type { Lock, Serialized, SerializeOptions };
255}
256
252const none: unique symbol = Symbol();257const none: unique symbol = Symbol();
253type none = typeof none;258type none = typeof none;
src/file-viewer/backend.tsx+91-129
...@@ -1,37 +1,5 @@...@@ -1,37 +1,5 @@
1export const app = new Hono();1export const app = new Hono();
22
3interface APIDirectoryList {
4 path: string;
5 readme: string | null;
6 files: APIFile[];
7}
8
9interface APIFile {
10 basename: string;
11 dir: boolean;
12 time: number;
13 size: number;
14 duration: number | null;
15}
16
17function checkCotyledonCookie(c: Context) {
18 const cookie = c.req.header("Cookie");
19 if (!cookie) return false;
20 const cookies = cookie.split("; ").map((x) => x.split("="));
21 return cookies.some(
22 (kv) => kv[0].trim() === "cotyledon" && kv[1].trim() === "agree",
23 );
24}
25
26function isCotyledonPath(path: string) {
27 if (path === "/cotyledon") return true;
28 const year = path.match(/^\/(\d{4})($|\/)/);
29 if (!year) return false;
30 const yearInt = parseInt(year[1]);
31 if (yearInt < 2025 && yearInt >= 2017) return true;
32 return false;
33}
34
35app.post("/file/cotyledon", async (c) => {3app.post("/file/cotyledon", async (c) => {
36 c.res = new Response(null, {4 c.res = new Response(null, {
37 status: 200,5 status: 200,
...@@ -46,31 +14,31 @@ app.get("/file/*", async (c, next) => {...@@ -46,31 +14,31 @@ app.get("/file/*", async (c, next) => {
46 const lofi = ua.includes("msie") || false;14 const lofi = ua.includes("msie") || false;
4715
48 // Discord ignores 'robots.txt' which violates the license agreement.16 // Discord ignores 'robots.txt' which violates the license agreement.
49 if (ua.includes("discordbot")) {17 if (ua.includes("discordbot")) return c.body("Forbidden", 403);
50 return next();
51 }
5218
53 let rawFilePath = c.req.path.slice(5) || "/";19 let rawFilePath = c.req.path.slice(5) || "/";
20
21 // The web file viewer sends urls suffixed with $partial to get HTML partials.
54 if (rawFilePath.endsWith("$partial")) {22 if (rawFilePath.endsWith("$partial")) {
55 return getPartialPage(c, rawFilePath.slice(0, -"$partial".length));23 return getPartialPage(c, rawFilePath.slice(0, -"$partial".length));
56 }24 }
25
26 // Cotyledon is gated behind a trivial cookie.
57 let hasCotyledonCookie = checkCotyledonCookie(c);27 let hasCotyledonCookie = checkCotyledonCookie(c);
58 if (isCotyledonPath(rawFilePath)) {28 if (isCotyledonPath(rawFilePath)) {
59 if (!hasCotyledonCookie) {29 if (!hasCotyledonCookie) {
60 return serveAsset(c, "/file/cotyledon_speedbump", 403);30 return serveAsset(c, "/file/cotyledon-speedbump", 403);
61 } else if (rawFilePath === "/cotyledon") {31 } else if (rawFilePath === "/cotyledon") {
62 return serveAsset(c, "/file/cotyledon_enterance", 200);32 return serveAsset(c, "/file/cotyledon-enterance", 200);
63 }33 }
64 }34 }
65 while (rawFilePath.length > 1 && rawFilePath.endsWith("/")) {35 while (rawFilePath.length > 1 && rawFilePath.endsWith("/")) {
66 rawFilePath = rawFilePath.slice(0, -1);36 rawFilePath = rawFilePath.slice(0, -1);
67 }37 }
68 const file = MediaFile.getByPath(rawFilePath);38 const file = MediaFile.getByPath(rawFilePath);
69 if (!file) {39 if (!file) return next();
70 // perhaps a specific 404 page for media files?
71 return next();
72 }
7340
41 // The permissions system is currently binary, using the friend auth.
74 const permissions = FilePermissions.getByPrefix(rawFilePath);42 const permissions = FilePermissions.getByPrefix(rawFilePath);
75 if (permissions !== 0) {43 if (permissions !== 0) {
76 const friendAuthChallenge = requireFriendAuth(c);44 const friendAuthChallenge = requireFriendAuth(c);
...@@ -79,20 +47,6 @@ app.get("/file/*", async (c, next) => {...@@ -79,20 +47,6 @@ app.get("/file/*", async (c, next) => {
7947
80 // File listings48 // File listings
81 if (file.kind === MediaFileKind.directory) {49 if (file.kind === MediaFileKind.directory) {
82 if (c.req.header("Accept")?.includes("application/json")) {
83 const json = {
84 path: file.path,
85 files: file.getPublicChildren().map((f) => ({
86 basename: f.basename,
87 dir: f.kind === MediaFileKind.directory,
88 time: f.date.getTime(),
89 size: f.size,
90 duration: f.duration ? f.duration : null,
91 })),
92 readme: file.contents ? file.contents : null,
93 } satisfies APIDirectoryList;
94 return c.json(json);
95 }
96 c.res = await view.serve(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {50 c.res = await view.serve(c, `file-viewer/${lofi ? "lofi" : "clofi"}`, {
97 file,51 file,
98 hasCotyledonCookie,52 hasCotyledonCookie,
...@@ -100,17 +54,16 @@ app.get("/file/*", async (c, next) => {...@@ -100,17 +54,16 @@ app.get("/file/*", async (c, next) => {
100 return;54 return;
101 }55 }
10256
103 // Redirect to directory list for regular files if client accepts HTML57 // Show a directory list for regular files if client accepts HTML.
58 // Exceptions:
59 // - `?view=download` or `?dl`
60 // - Old browsers like Internet Explorer act as `?view=dl`
104 let viewMode = c.req.query("view");61 let viewMode = c.req.query("view");
105 if (c.req.query("dl") !== undefined) {62 if (c.req.query("dl") != null) viewMode = "download";
106 viewMode = "download";63 if (lofi) viewMode = file.extension === ".html" ? "embed" : "download";
107 }64
108 if (65 if (viewMode == null && c.req.header("Accept")?.includes("text/html")) {
109 viewMode == undefined &&66 // cache.prefetch(file);
110 c.req.header("Accept")?.includes("text/html") &&
111 !lofi
112 ) {
113 prefetchFile(file.path);
114 c.res = await view.serve(c, "file-viewer/clofi", {67 c.res = await view.serve(c, "file-viewer/clofi", {
115 file,68 file,
116 hasCotyledonCookie,69 hasCotyledonCookie,
...@@ -120,15 +73,29 @@ app.get("/file/*", async (c, next) => {...@@ -120,15 +73,29 @@ app.get("/file/*", async (c, next) => {
120 const download = viewMode === "download";73 const download = viewMode === "download";
12174
122 const etag = file.hash;75 const etag = file.hash;
123 const filePath = file.path;76 // const filePath = file.path;
124 const expectedSize = file.size;77 const size = file.size;
12578
126 let encoding = decideEncoding(c.req.header("Accept-Encoding"));79 //let encoding = decideEncoding(c.req.header("Accept-Encoding"));
12780
128 let sizeHeader = encoding === "raw"81 // let sizeHeader = encoding === "raw"
129 ? expectedSize82 // ? expectedSize
130 // Size cannot be known because of compression modes83 // // Size cannot be known because of compression modes
131 : undefined;84 // : undefined;
85 //
86 const headers = new Headers({
87 Vary: "Accept-Encoding, Accept",
88 "Content-Type": contentTypeFor(file.path),
89 "Content-Length": size.toString(),
90 ETag: file.hash,
91 "Last-Modified": file.date.toUTCString(),
92 });
93 if (download) {
94 headers.set(
95 "Content-Disposition",
96 `attachment; filename="${file.basename}"`,
97 );
98 }
13299
133 // Etag100 // Etag
134 {101 {
...@@ -137,7 +104,7 @@ app.get("/file/*", async (c, next) => {...@@ -137,7 +104,7 @@ app.get("/file/*", async (c, next) => {
137 c.res = new Response(null, {104 c.res = new Response(null, {
138 status: 304,105 status: 304,
139 statusText: "Not Modified",106 statusText: "Not Modified",
140 headers: fileHeaders(file, download, sizeHeader),107 headers,
141 });108 });
142 return;109 return;
143 }110 }
...@@ -145,15 +112,12 @@ app.get("/file/*", async (c, next) => {...@@ -145,15 +112,12 @@ app.get("/file/*", async (c, next) => {
145112
146 // Head113 // Head
147 if (c.req.method === "HEAD") {114 if (c.req.method === "HEAD") {
148 c.res = new Response(null, {115 c.res = new Response(null, { headers });
149 headers: fileHeaders(file, download, sizeHeader),
150 });
151 return;116 return;
152 }117 }
153118
154 // Prevalidate range requests119 // Prevalidate range requests
155 let rangeHeader = c.req.header("Range") ?? null;120 let rangeHeader = c.req.header("Range") ?? null;
156 if (rangeHeader) encoding = "raw";
157121
158 const ifRangeHeader = c.req.header("If-Range");122 const ifRangeHeader = c.req.header("If-Range");
159 if (ifRangeHeader && ifRangeOutdated(file, ifRangeHeader)) {123 if (ifRangeHeader && ifRangeOutdated(file, ifRangeHeader)) {
...@@ -162,53 +126,36 @@ app.get("/file/*", async (c, next) => {...@@ -162,53 +126,36 @@ app.get("/file/*", async (c, next) => {
162 rangeHeader = null;126 rangeHeader = null;
163 }127 }
164128
165 let foundFile;129 const found = await cache.read(filePath
166 while (true) {130 // foundFile = await cache.read(filePath, encoding, {
167 let second = false;131 // start: rangeHeader ? rangeHeader[0] : 0,
168 try {132 // end: rangeHeader ? rangeHeader[1] : Infinity,
169 foundFile = await fetchFile(filePath, encoding);133 // });
170 if (second) {134 // const [streamOrBuffer, actualEncoding, src] = foundFile;
171 console.warn(`File ${filePath} has missing compression: ${encoding}`);135 // encoding = actualEncoding;
172 }136 //
173 break;137 // // Range requests
174 } catch (error) {138 // // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
175 if (encoding !== "raw") {139 // // Compression is skipped because it's a confusing, but solvable problem.
176 encoding = "raw";140 // // See https://stackoverflow.com/questions/33947562/is-it-possible-to-send-http-response-using-gzip-and-byte-ranges-at-the-same-time
177 sizeHeader = file.size;141 // if (rangeHeader) {
178 second = true;142 // const ranges = parseRange(rangeHeader, file.size);
179 continue;143 // // TODO: multiple ranges
180 }144 // if (ranges && ranges.length === 1) {
181145 // return (c.res = handleRanges(ranges, file, streamOrBuffer, download));
182 return c.text(146 // }
183 "internal server error: this file is present in the database but could not be fetched",147 // }
184 );148 //
185 }149 // // Respond in a streaming fashion
186 }150 // c.res = new Response(streamOrBuffer, {
187 const [streamOrBuffer, actualEncoding, src] = foundFile;151 // headers: {
188 encoding = actualEncoding;152 // ...fileHeaders(file, download, sizeHeader),
189153 // ...(encoding !== "raw" && {
190 // Range requests154 // "Content-Encoding": encoding,
191 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests155 // }),
192 // Compression is skipped because it's a confusing, but solvable problem.156 // "X-Cache": src,
193 // See https://stackoverflow.com/questions/33947562/is-it-possible-to-send-http-response-using-gzip-and-byte-ranges-at-the-same-time157 // },
194 if (rangeHeader) {158 // });
195 const ranges = parseRange(rangeHeader, file.size);
196 // TODO: multiple ranges
197 if (ranges && ranges.length === 1) {
198 return (c.res = handleRanges(ranges, file, streamOrBuffer, download));
199 }
200 }
201
202 // Respond in a streaming fashion
203 c.res = new Response(streamOrBuffer, {
204 headers: {
205 ...fileHeaders(file, download, sizeHeader),
206 ...(encoding !== "raw" && {
207 "Content-Encoding": encoding,
208 }),
209 "X-Cache": src,
210 },
211 });
212});159});
213160
214app.get("/canvas/:script", async (c, next) => {161app.get("/canvas/:script", async (c, next) => {
...@@ -374,6 +321,25 @@ function applySingleRangeToStream(...@@ -374,6 +321,25 @@ function applySingleRangeToStream(
374 },321 },
375 });322 });
376}323}
324function checkCotyledonCookie(c: Context) {
325 const cookie = c.req.header("Cookie");
326 if (!cookie) return false;
327 const cookies = cookie.split("; ").map((x) =>
328 x.split("=") as [k: string, v?: string]
329 );
330 return cookies.some(
331 (kv) => kv[0].trim() === "cotyledon" && kv[1]?.trim() === "agree",
332 );
333}
334
335function isCotyledonPath(path: string) {
336 if (path === "/cotyledon") return true;
337 const year = path.match(/^\/(\d{4})($|\/)/);
338 if (!year) return false;
339 const yearInt = parseInt(UNWRAP(year[1]));
340 if (yearInt < 2025 && yearInt >= 2017) return true;
341 return false;
342}
377343
378function getPartialPage(c: Context, rawFilePath: string) {344function getPartialPage(c: Context, rawFilePath: string) {
379 if (isCotyledonPath(rawFilePath)) {345 if (isCotyledonPath(rawFilePath)) {
...@@ -426,8 +392,4 @@ import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";...@@ -426,8 +392,4 @@ import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
426import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";392import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
427import { MediaPanel } from "@/file-viewer/views/clofi.tsx";393import { MediaPanel } from "@/file-viewer/views/clofi.tsx";
428import { Speedbump } from "@/file-viewer/cotyledon.tsx";394import { Speedbump } from "@/file-viewer/cotyledon.tsx";
429import {395import * as cache from "@/file-viewer/cache.ts";
430 type CompressionFormat,
431 fetchFile,
432 prefetchFile,
433} from "@/file-viewer/cache.ts";
src/file-viewer/cache.ts+450-371
...@@ -1,418 +1,497 @@...@@ -1,418 +1,497 @@
1import { Agent, get } from "node:https";1// implements the file viewer's cache. there are two primary exports:
2import * as fs from "node:fs";2// - `cache.read(...)` to read a file, caching it
3import * as path from "node:path";3// - TODO: `cache.prefetch(...)` to prefetch a file, placing it in the disk cache.
4import { Buffer } from "node:buffer";4//
5import type { ClientRequest } from "node:http";5// in local mode, the cache layer reads directly the mounted network share.
6import LRUCache from "lru-cache";6// when unavailable, it fetches from the Source of Truth, caching its
7import { open } from "node:fs/promises";7// results on disk. one `Lru` is used to keep the most relevant files in
8import { createHash } from "node:crypto";8// the store, while another one is used to re-use file descriptors.
9import { scoped } from "@paperclover/console";9//
10import { escapeUri } from "./format.ts";10// the cache handles concurrent requests to the same disk entry by placing
1111// its file descriptor in "w+" mode for reading and writing. this way, the
12declare const Deno: any;12// second request can read from the same file descriptor that the first
1313// request is writing into. large files are never buffered entirely in
14const sourceOfTruth = "https://nas.paperclover.net:43250";14// memory, which should keep memory usage low.
15// const caCert = fs.readFileSync("src/file-viewer/cert.pem");15//
1616// as a measure to make this potentially more memory efficient, the buffer
17const diskCacheRoot = path.join(import.meta.dirname, ".filecache/");17// objects given by node:fs/node:http streams -- which are re-allocated for
18const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB18// every read -- are weakly held when written and reused when reading. this
19const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB19// somewhat avoids file reads in concurrent access situations.
20const loadInProgress = new Map<20const log = console.scoped("cache");
21 string,21
22 Promise<{ stream: ReadableStream }> | { stream: ReadableStream }22export type Source =
23>();23 | "local" // local mode
24// Disk cache serializes the access times24 | "hit" // cache hit
25const diskCacheState: Record<string, [size: number, lastAccess: number]> =25 | "pending" // cache hit but the file is still being loaded
26 loadDiskCacheState();26 | "ignored" // file too big
27const diskCache = new LRUCache<string, number>({27 | "contended" // could not write disk because too many ongoing downloads
28 maxSize: diskCacheMaxSize,28 | "ratelimit" // the user is ratelimited for inserting files
29 ttl: 0,29 | "miss"; // cache miss, disk cache is created
30 sizeCalculation: (value) => value,30
31 dispose: (_, key) => {31// to download protected files from the source of truth -- for example files
32 delete diskCacheState[key];32// used on the private blog for friends -- its token is required.
33 },33const token = process.env.CLOVER_SOT_KEY ?? null;
34 onInsert: (size, key) => {34const sotUrl = new URL(
35 diskCacheState[key] = [size, Date.now()];35 process.env.CLOVER_SOT_URL ?? "https://db.paperclover.net",
36 },36);
37if (!token) console.warn("Missing CLOVER_SOT_KEY, cannot host protected files");
38
39const diskCacheRoot = openDiskCache();
40const diskCacheState = diskCacheRoot.join("state.json");
41const capacityInBytes = 8 * 1024 * 1024 * 1024;
42
43const derivedFileRoot = new Path(paths.derivedFileRoot).ifExistsSync();
44const rawFileRoot = new Path(paths.rawFileRoot).ifExistsSync();
45
46const diskCache = new Lru<Path, DiskFile>({
47 capacity: capacityInBytes,
48 sizeFn: ({ size }) => size,
49 delete: (_, path) => deleteCachedPath(path),
37});50});
38const ramCache = new LRUCache<string, Buffer>({51
39 maxSize: ramCacheMaxSize,52// When reading from the disk cache, file descriptors are ref-counted in `fds`,
40 ttl: 0,53// and held open in the `fdsCache` lru when no references.
41 sizeCalculation: (value) => value.byteLength,54const fdsCache = new Lru<Path, number>({
55 capacity: 64,
56 delete: (value) => void fsCallbacks.closeSync(value),
42});57});
43let diskCacheFlush: NodeJS.Timeout | undefined;58const fds = new Map<Path, Awaitable<RefCountFile>>();
44
45{
46 // Initialize the disk cache by validating all files exist, and then
47 // inserting them in last to start order. State is repaired pessimistically.
48 const toDelete = new Set(Object.keys(diskCacheState));
49 fs.mkdirSync(diskCacheRoot, { recursive: true });
50 for (
51 const file of fs.readdirSync(diskCacheRoot, {
52 recursive: true,
53 encoding: "utf-8",
54 })
55 ) {
56 const key = file.split("/").pop()!;
57 if (key.length !== 40) continue;
58 const entry = diskCacheState[key];
59 if (!entry) {
60 fs.rmSync(path.join(diskCacheRoot, file), {
61 recursive: true,
62 force: true,
63 });
64 delete diskCacheState[key];
65 continue;
66 }
67 toDelete.delete(key);
68 }
69 for (const key of toDelete) {
70 delete diskCacheState[key];
71 }
72 saveDiskCacheState();
73 const sorted = Object.keys(diskCacheState).sort((a, b) =>
74 diskCacheState[b][1] - diskCacheState[a][1]
75 );
76 for (const key of sorted) {
77 diskCache.set(key, diskCacheState[key][0]);
78 }
79}
8059
81export type CacheSource = "ram" | "disk" | "miss" | "lan" | "flight";60const parallelRequests = new Map<string, Promise<http.IncomingMessage>>();
82export type CompressionFormat = "gzip" | "zstd" | "raw";
83const compressionFormatMap = {
84 gzip: "gz",
85 zstd: "zstd",
86 raw: "file",
87} as const;
8861
89const log = scoped("file_cache");62// Revive disk cache state
63revive: {
64 const state = diskCacheState
65 .readIfExistsSync<Lru.Serialized<string, number>>("json");
66 if (!state) break revive;
67 const missing = new Set(diskCacheRoot.readDirSync().map((c) => c.base));
68 missing.delete(diskCacheState.base);
69 diskCache.revive(state, {
70 map(k: string, v: number) {
71 const path = new Path(k);
72 try {
73 const { size } = path.statSync();
74 if (size !== v) return null;
75 if (!missing.delete(k)) return null;
76 return [
77 path,
78 { size, written: new async.Watch(size), memory: new Map() },
79 ];
80 } catch {
81 return null;
82 }
83 },
84 });
85 // Delete unreferenced files
86 for (const missingFile of missing) {
87 try {
88 diskCacheRoot.join(missingFile).deleteSync();
89 } catch {}
90 }
91}
9092
91const lanMount = "/Volumes/clover/Published";93export interface Options {
92const hasLanMount = fs.existsSync(lanMount);94 file: MediaFile;
95 /** If set, fetches the derived asset instead. */
96 derived: string | null;
97 start: number;
98 /** exclusive. ex: [10, 30] skips ten bytes then grabs 20 */
99 end: number;
100 noWrite?: boolean;
101 localOnly?: boolean;
102}
93103
94/**104/**
95 * Fetches a file with the given compression format.105 * Read a value from file store, potentially fetching from the Source of Truth.
96 * Uncompressed files are never persisted to disk.106 * `null` if a derived asset does not exist
97 *
98 * Returns a promise to either:
99 * - Buffer: the data is from RAM cache
100 * - ReadableStream: the data is being streamed in from disk/server
101 *
102 * Additionally, returns a string indicating the source of the data, for debugging.
103 *
104 * Callers must be able to consume both output types.
105 */107 */
106export async function fetchFile(108export async function read(
107 pathname: string,109 { file, derived, start, end, noWrite, localOnly }: Options,
108 format: CompressionFormat = "raw",110): Promise<{ stream: ReadableStream; src: Source; size: number } | null> {
109): Promise<111 const rel = derived ? `${file.path}$/${derived}` : file.path;
110 [Buffer | ReadableStream, encoding: CompressionFormat, src: CacheSource]
111> {
112 // 1. Ram cache
113 const cacheKey = hashKey(`${pathname}:${format}`);
114 const ramCacheHit = ramCache.get(cacheKey);
115 if (ramCacheHit) {
116 log(`ram hit: ${format}${pathname}`);
117 return [ramCacheHit, format, "ram"];
118 }
119112
120 // 2. Tee an existing loading stream.113 // Serve from local network share if it is mounted.
121 const inProgress = loadInProgress.get(cacheKey);114 const localFile = derived
122 if (inProgress) {115 ? derivedFileRoot?.join(file.hash, derived)
123 const stream = await inProgress;116 : rawFileRoot?.join(file.path);
124 const [stream1, stream2] = stream.stream.tee();117 if (localFile) {
125 loadInProgress.set(cacheKey, { stream: stream2 });118 local: {
126 log(`in-flight copy: ${format}${pathname}`);119 let size: number;
127 return [stream1, format, "flight"];120 try {
128 }121 ({ size } = await localFile.stat());
129122 } catch (err) {
130 // 3. Disk cache + Load into ram cache.123 if (error.code(err) === "ENOENT") {
131 if (format !== "raw") {124 console.warn(`missing local file: ${localFile}`);
132 const diskCacheHit = diskCache.get(cacheKey);125 break local;
133 if (diskCacheHit) {126 }
134 diskCacheState[cacheKey] = [diskCacheHit, Date.now()];127 throw err;
135 saveDiskCacheStateLater();128 }
136 log(`disk hit: ${format}/${pathname}`);129 const fileStream = await readStream(localFile, { start, end });
137 return [130 if (!fileStream) break local;
138 startInProgress(131 log(`from local: ${rel}`);
139 cacheKey,132 return {
140 new ReadableStream({133 stream: stream.Readable.toWeb(fileStream),
141 start: async (controller) => {134 size,
142 const stream = fs.createReadStream(135 src: "local",
143 path.join(diskCacheRoot, cacheKey),136 };
144 );
145 const chunks: Buffer[] = [];
146 stream.on("data", (chunk) => {
147 controller.enqueue(chunk);
148 chunks.push(chunk as Buffer);
149 });
150 stream.on("end", () => {
151 controller.close();
152 ramCache.set(cacheKey, Buffer.concat(chunks));
153 finishInProgress(cacheKey);
154 });
155 stream.on("error", (error) => {
156 controller.error(error);
157 });
158 },
159 }),
160 ),
161 format,
162 "disk",
163 ];
164 }137 }
165 }138 }
166139
167 // 4. Lan Mount (access files that prod may not have)140 if (!localOnly) return null;
168 if (hasLanMount) {
169 log(`lan hit: ${format}/${pathname}`);
170 return [
171 startInProgress(
172 cacheKey,
173 new ReadableStream({
174 start: async (controller) => {
175 const stream = fs.createReadStream(
176 path.join(lanMount, pathname),
177 );
178 const chunks: Buffer[] = [];
179 stream.on("data", (chunk) => {
180 controller.enqueue(chunk);
181 chunks.push(chunk as Buffer);
182 });
183 stream.on("end", () => {
184 controller.close();
185 ramCache.set(cacheKey, Buffer.concat(chunks));
186 finishInProgress(cacheKey);
187 });
188 stream.on("error", (error) => {
189 controller.error(error);
190 });
191 },
192 }),
193 ),
194 "raw",
195 "lan",
196 ];
197 }
198141
199 // 4. Fetch from server142 const key = derived
200 const url = `${compressionFormatMap[format]}${escapeUri(pathname)}`;143 ? crypto.createHash("sha1").update(file.hash + derived).digest("hex")
201 log(`miss: ${format}${pathname}`);144 : file.hash;
202 const response = await startInProgress(cacheKey, fetchFileUncached(url));
203 const [stream1, stream2] = response.tee();
204 handleDownload(cacheKey, format, stream2);
205 return [stream1, format, "miss"];
206}
207145
208export async function prefetchFile(146 // Disk cache
209 pathname: string,147 const diskPath = diskCacheRoot.join(key);
210 format: CompressionFormat = "zstd",148 let diskFile = diskCache.get(diskPath);
211) {149 if (diskFile) {
212 const cacheKey = hashKey(`${pathname}:${format}`);150 const lock = diskCache.lock(diskPath);
213 const ramCacheHit = ramCache.get(cacheKey);151 const diskStream = await readStream(diskPath, {
214 if (ramCacheHit) {152 ref: diskFile,
215 return;153 start: start,
154 end: end,
155 lock,
156 });
157 if (diskStream) {
158 const isPending = diskFile.written.value !== diskFile.size;
159 log(`from ${isPending ? "pending " : ""}disk cache: ${rel}`);
160 return {
161 stream: stream.Readable.toWeb(diskStream),
162 src: isPending ? "pending" : "hit",
163 size: diskFile.size,
164 };
165 } else {
166 diskCache.delete(diskPath);
167 }
216 }168 }
217 if (hasLanMount) return;
218 const url = `${compressionFormatMap[format]}${pathname}`;
219 log(`prefetch: ${format}${pathname}`);
220 const stream2 = await startInProgress(cacheKey, fetchFileUncached(url));
221 handleDownload(cacheKey, format, stream2);
222}
223169
224async function handleDownload(170 // Fetch the source of truth
225 cacheKey: string,171 const url = `${sotUrl}/file${file.path}`;
226 format: CompressionFormat,172 let response;
227 stream2: ReadableStream,173 {
228) {174 let promise = parallelRequests.get(url);
229 let chunks: Buffer[] = [];175 if (promise) {
230 if (format !== "raw") {176 log(`from other response: ${rel}`);
231 const file = await open(path.join(diskCacheRoot, cacheKey), "w");177 response = await promise;
232 try {178 } else {
233 for await (const chunk of stream2) {179 log(`cache miss: ${rel}`);
234 await file.write(chunk);180 promise = httpGet(url, {
235 chunks.push(chunk);181 headers: token ? { Authorization: token } : {},
236 }182 });
237 } finally {183 parallelRequests.set(url, promise);
238 file.close();184 response = await promise;
239 }185 parallelRequests.delete(url);
240 } else {
241 for await (const chunk of stream2) {
242 chunks.push(chunk);
243 }186 }
244 }187 }
245 const final = Buffer.concat(chunks);188 const { statusCode, headers } = response;
246 chunks.length = 0;189 if (statusCode !== 200) {
247 ramCache.set(cacheKey, final);190 const text = await new Response(response as unknown as Blob).text();
248 if (format !== "raw") {191 const msg = `Source of Truth responded with status ${statusCode}: ${text}`;
249 diskCache.set(cacheKey, final.byteLength);192 console.error(msg);
193 throw new Error("Source of Truth is unavailable");
250 }194 }
251 finishInProgress(cacheKey);195 const size = Number(headers["content-length"]);
252}196 ASSERT(!Number.isNaN(size) && size >= 0);
253
254function hashKey(key: string): string {
255 return createHash("sha1").update(key).digest("hex");
256}
257197
258function startInProgress<T extends Promise<ReadableStream> | ReadableStream>(198 // Do not cache large assets
259 cacheKey: string,199 if (size > 1_000_000_000) {
260 promise: T,200 log(`skipping disk cache due to large asset: ${rel}`);
261): T {201 return { stream: stream.Readable.toWeb(response), size, src: "ignored" };
262 if (promise instanceof Promise) {
263 let resolve2: (stream: { stream: ReadableStream }) => void;
264 let reject2: (error: Error) => void;
265 const stream2Promise = new Promise<{ stream: ReadableStream }>(
266 (resolve, reject) => {
267 resolve2 = resolve;
268 reject2 = reject;
269 },
270 );
271 const stream1Promise = new Promise<ReadableStream>((resolve, reject) => {
272 promise.then((stream) => {
273 const [stream1, stream2] = stream.tee();
274 const stream2Obj = { stream: stream2 };
275 resolve2(stream2Obj);
276 loadInProgress.set(cacheKey, stream2Obj);
277 resolve(stream1);
278 }, reject);
279 });
280 loadInProgress.set(cacheKey, stream2Promise);
281 return stream1Promise as T;
282 } else {
283 const [stream1, stream2] = promise.tee();
284 loadInProgress.set(cacheKey, { stream: stream2 });
285 return stream1 as T;
286 }202 }
203 // Bail out of the disk cache if it is contended
204 if (diskCache.unlockedCapacity < size) {
205 log(`skipping disk cache due to contention: ${rel}`);
206 return { stream: stream.Readable.toWeb(response), size, src: "contended" };
207 }
208 // IPs which request too many cache misses are temporarily
209 // banned from writing into the disk cache to avoid abuse.
210 if (noWrite) {
211 log(`skipping disk cache due to config: ${rel}`);
212 return { stream: stream.Readable.toWeb(response), size, src: "ratelimit" };
213 }
214
215 // Create a disk cache entry;
216 // Tee the stream to both the request and the filesystem.
217 diskFile = {
218 size,
219 written: new async.Watch(0),
220 memory: new Map(),
221 };
222 diskCache.set(diskPath, diskFile);
223
224 const lock = diskCache.lock(diskPath);
225 const writer = await writeStream(diskPath, response, diskFile, lock);
226 writer.on("error", () => {
227 diskCache.delete(diskPath);
228 diskFile.written.value = Infinity;
229 });
230
231 return { stream: stream.Readable.toWeb(response), size, src: "miss" };
287}232}
288233
289function loadDiskCacheState(): Record<234interface DiskFile {
290 string,235 size: number;
291 [size: number, lastAccess: number]236 written: async.Watch<number>;
292> {237 memory: Map<number, WeakRef<Buffer>>;
293 try {
294 const state = JSON.parse(
295 fs.readFileSync(path.join(diskCacheRoot, "state.json"), "utf-8"),
296 );
297 return state;
298 } catch (error) {
299 return {};
300 }
301}238}
302239
303function saveDiskCacheStateLater() {240function httpGet(url: string, options: https.RequestOptions) {
304 if (diskCacheFlush) {241 return new Promise<http.IncomingMessage>((resolve, reject) => {
305 return;242 const request = https.get(url, options);
306 }243 request.on("response", resolve);
307 diskCacheFlush = setTimeout(() => {244 request.on("error", reject);
308 saveDiskCacheState();245 });
309 }, 60_000) as NodeJS.Timeout;
310 if (diskCacheFlush.unref) {
311 diskCacheFlush.unref();
312 }
313}246}
314247
315process.on("exit", () => {248const finalization = new FinalizationRegistry(
316 saveDiskCacheState();249 ([map, key]: [Map<number, unknown>, number]) => {
317});250 map.delete(key);
251 },
252);
318253
319function saveDiskCacheState() {254const openFile = util.promisify(fsCallbacks.open);
320 fs.writeFileSync(255
321 path.join(diskCacheRoot, "state.json"),256function deleteCachedPath(path: Path) {
322 JSON.stringify(diskCacheState),257 try {
323 );258 fdsCache.delete(path);
259 } catch {}
260 try {
261 path.deleteSync();
262 } catch {}
324}263}
325264
326function finishInProgress(cacheKey: string) {265interface RefCountFile {
327 loadInProgress.delete(cacheKey);266 fd: number;
267 refs: number;
328}268}
329269
330// Self signed certificate must be trusted to be able to request the above URL.270function openDiskCache() {
331//271 const clover = ".clover", fileCache = "file-viewer";
332// Unfortunately, Bun and Deno are both not node.js compatible, so those two272 const split = import.meta.filename.split(Path.sep);
333// runtimes need fallback implementations. The fallback implementations calls273 const index = split.indexOf(".clover");
334// fetch with the `agent` value as the RequestInit. Since `fetch` decompresses274 return index === -1
335// the body for you, it must be disabled.275 ? Path.createDirSync(clover, fileCache)
336const agent: any = typeof Bun !== "undefined"276 : Path.createDirSync(...split.slice(0, index + 1), fileCache);
337 ? {277}
338 // Bun has two non-standard fetch extensions278
339 decompress: false,279function writeDiskCache() {
340 tls: {280 const serialized = diskCache.serialize({
341 // ca: caCert,281 map(path, { size, written }) {
282 if (size !== written.value) return null;
283 return [path.toString(), size];
342 },284 },
343 }
344 // TODO: https://github.com/denoland/deno/issues/12291
345 // : typeof Deno !== "undefined"
346 // ? {
347 // // Deno configures through the non-standard `client` extension
348 // client: Deno.createHttpClient({
349 // caCerts: [caCert.toString()],
350 // }),
351 // }
352 // Node.js supports node:http
353 : new Agent({
354 // ca: caCert,
355 });285 });
286 diskCacheState.writeJsonFile(serialized)
287 .catch(() => {});
288}
356289
357function fetchFileNode(pathname: string): Promise<ReadableStream> {290async function getHandle(path: Path, flag: string, unlockOnFail?: Disposable) {
358 return new Promise((resolve, reject) => {291 let handle: RefCountFile | null = null;
359 const request: ClientRequest = get(`${sourceOfTruth}/${pathname}`, {292 try {
360 agent,293 handle = await fds.get(path) ?? null;
361 });294 if (!handle) {
362 request.on("response", (response) => {295 const cached = fdsCache.get(path);
363 if (response.statusCode !== 200) {296 if (cached) {
364 reject(new Error(`Failed to fetch ${pathname}`));297 fds.delete(path);
365 return;298 fds.set(path, handle = { fd: cached, refs: 0 });
299 } else {
300 const promise = openFile(path.raw, flag)
301 .then((fd) => ({ fd, refs: 0 }))
302 .catch((err) => {
303 fds.delete(path);
304 throw err;
305 });
306 fds.set(path, promise);
307 fds.set(path, handle = await promise);
366 }308 }
309 }
310 handle.refs += 1;
311 } catch (err: any) {
312 unlockOnFail?.[Symbol.dispose]();
313 if (err.code === "ENOENT") {
314 return null;
315 }
316 throw err;
317 }
318 return handle;
319}
367320
368 const stream = new ReadableStream({321interface ReadStreamOptions {
369 start(controller) {322 start: number;
370 response.on("data", (chunk) => {323 end: number;
371 controller.enqueue(chunk);324 ref?: DiskFile;
372 });325 lock?: Disposable;
326}
373327
374 response.on("end", () => {328async function readStream(path: Path, opts: ReadStreamOptions) {
375 controller.close();329 const handle = await getHandle(path, "r", opts?.lock);
376 });330 if (!handle) return null;
377331
378 response.on("error", (error) => {332 let read = fsCallbacks.read;
379 controller.error(error);333 if (opts?.ref) {
380 reject(error);334 const { memory, written, size } = opts.ref;
381 });335 let lastRead: number | null = null;
382 },336 read = function loop(
383 });337 fd: number,
338 buf: Buffer,
339 offset: 0,
340 len: number,
341 pos: number,
342 cb: (er: Error | null, bytesRead: number, buf: Buffer) => void,
343 ) {
344 ASSERT(pos != null);
345 ASSERT(offset === 0);
346 written.until((v) => v >= Math.min(size, pos + len)).then(
347 (newLength) => {
348 // Try to re-use uncollected buffers in memory
349 while (true) {
350 const weak = memory.get(lastRead ?? pos);
351 if (weak) {
352 let cachedBuf = weak.deref();
353 if (cachedBuf) {
354 if (lastRead) {
355 cachedBuf = cachedBuf.subarray(pos - lastRead);
356 }
357 const { byteLength } = cachedBuf;
384358
385 resolve(stream);359 if (byteLength === len && offset === 0) {
386 });360 cb(null, len, cachedBuf);
361 return;
362 }
363 if (byteLength <= len) {
364 buf.set(cachedBuf, offset);
365 lastRead = null;
366 } else {
367 buf.set(cachedBuf.subarray(0, len), offset);
368 lastRead = lastRead ?? pos;
369 }
370 offset += byteLength;
371 len -= byteLength;
372 pos += byteLength;
373 if (len > 0) continue;
374 cb(null, offset + len, buf);
375 return;
376 } else {
377 memory.delete(pos);
378 }
379 }
387380
388 request.on("error", (error) => {381 lastRead = null;
389 reject(error);382 break;
390 });383 }
391 });
392}
393384
394async function fetchFileDenoBun(pathname: string): Promise<ReadableStream> {385 // Read from disk
395 const req = await fetch(`${sourceOfTruth}/${pathname}`, agent);386 if (newLength > size) cb(new Error("Broken Pipe"), 0, buf);
396 if (!req.ok) {387 fsCallbacks.read(
397 throw new Error(`Failed to fetch ${pathname}`);388 fd,
389 buf,
390 offset,
391 len,
392 pos,
393 async (er, bytes, buf) => {
394 if (!er && bytes > 0) {
395 memory.set(pos - offset, new WeakRef(buf));
396 finalization.register(buf, [memory, pos]);
397 }
398 cb(er, offset + bytes, buf);
399 },
400 );
401 },
402 );
403 } as typeof fsCallbacks.read;
398 }404 }
399 return req.body!;405
406 let closed = false;
407 return fs.createReadStream(path.raw, {
408 fd: handle.fd,
409 start: opts?.start ?? 0,
410 end: opts?.end ?? Infinity,
411 fs: {
412 close: util.callbackify(async () => {
413 ASSERT(!closed);
414 if ((handle.refs -= 1) <= 0) {
415 fds.delete(path);
416 fdsCache.set(path, handle.fd);
417 }
418 closed = true;
419 opts?.lock?.[Symbol.dispose]();
420 }),
421 read,
422 },
423 });
400}424}
401425
402const fetchFileUncached =426async function writeStream(
403 typeof Bun !== "undefined" || typeof Deno !== "undefined"427 path: Path,
404 ? fetchFileDenoBun428 data: stream.Readable,
405 : fetchFileNode;429 file: DiskFile,
430 lock: Disposable,
431) {
432 const handle = UNWRAP(await getHandle(path, "w+", lock));
433 let closed = false;
406434
407export async function toBuffer(435 const { memory, written } = file;
408 stream: ReadableStream | Buffer,436 const writer = fs.createWriteStream(path.raw, {
409): Promise<Buffer> {437 start: 0,
410 if (!(stream instanceof ReadableStream)) {438 fd: handle.fd,
411 return stream;439 fs: {
412 }440 close: util.callbackify(async () => {
413 const chunks: Buffer[] = [];441 ASSERT(!closed);
414 for await (const chunk of stream) {442 if ((handle.refs -= 1) <= 0) {
415 chunks.push(chunk);443 fds.delete(path);
416 }444 fdsCache.set(path, handle.fd);
417 return Buffer.concat(chunks);445 }
446 closed = true;
447 lock[Symbol.dispose]();
448 writeDiskCache();
449 }),
450 write(
451 fd: number,
452 data: Buffer,
453 off: 0,
454 size: number,
455 pos: number | undefined,
456 cb: (er: Error | null, bytesWritten: number, buffer: Buffer) => void,
457 ) {
458 ASSERT(size === data.byteLength);
459 ASSERT(pos != null);
460 fsCallbacks.write(
461 fd,
462 data,
463 off,
464 size,
465 pos,
466 (er, bytes, buf) => {
467 if (!er && bytes) {
468 memory.set(pos, new WeakRef(buf));
469 finalization.register(buf, [memory, pos]);
470 written.value += bytes;
471 }
472 cb(er, bytes, buf);
473 },
474 );
475 },
476 },
477 });
478 data.pipe(writer);
479
480 return writer;
418}481}
482
483import * as console from "@paperclover/console";
484import * as util from "node:util";
485import * as fsCallbacks from "node:fs";
486import * as fs from "#sitegen/fs";
487import * as async from "#sitegen/async";
488import * as error from "#sitegen/error";
489import * as stream from "node:stream";
490import * as crypto from "node:crypto";
491import * as http from "node:http";
492import * as https from "node:https";
493import * as paths from "./paths.ts";
494import { MediaFile } from "./models/MediaFile.ts";
495import type { ReadableStream } from "node:stream/web";
496import { Lru } from "#sitegen/lru";
497import { Path } from "#sitegen/path";
src/file-viewer/models/MediaFile.ts+1-2
...@@ -370,8 +370,7 @@ const setProcessorsQuery = db.prepare<[{...@@ -370,8 +370,7 @@ const setProcessorsQuery = db.prepare<[{
370 processors: string;370 processors: string;
371}]>(/* SQL */ `371}]>(/* SQL */ `
372 update media_files set372 update media_files set
373 processed = $processed,373 processed = $processed
374 processors = $processors
375 where id = $id;374 where id = $id;
376`);375`);
377const setDurationQuery = db.prepare<[{376const setDurationQuery = db.prepare<[{
src/file-viewer/pages/file.cotyledon-enterance.tsx created+27
...@@ -0,0 +1,27 @@
1import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
2import { addScript } from "#sitegen";
3import { Readme } from "@/file-viewer/cotyledon.tsx";
4import { MediaPanel } from "../views/clofi.tsx";
5
6export const theme = {
7 bg: "#312652",
8 fg: "#f0f0ff",
9 primary: "#fabe32",
10};
11
12export const meta = { title: "living room" };
13
14export default function CotyledonPage() {
15 addScript("../scripts/canvas_cotyledon.client.ts");
16 return (
17 <div class="files ctld ctld-et">
18 <MediaPanel
19 file={MediaFile.getByPath("/")!}
20 isLast={false}
21 activeFilename={null}
22 hasCotyledonCookie={true}
23 />
24 <Readme />
25 </div>
26 );
27}
src/file-viewer/pages/file.cotyledon-speedbump.tsx created+27
...@@ -0,0 +1,27 @@
1import { MediaFile } from "../models/MediaFile.ts";
2import { MediaPanel } from "../views/clofi.tsx";
3import { addScript } from "#sitegen";
4import { Speedbump } from "../cotyledon.tsx";
5
6export const theme = {
7 bg: "#312652",
8 fg: "#f0f0ff",
9 primary: "#fabe32",
10};
11
12export const meta = { title: "the front door" };
13
14export default function CotyledonPage() {
15 addScript("../scripts/canvas_cotyledon.client.ts");
16 return (
17 <div class="files ctld ctld-sb">
18 <MediaPanel
19 file={MediaFile.getByPath("/")!}
20 isLast={false}
21 activeFilename={null}
22 hasCotyledonCookie={false}
23 />
24 <Speedbump />
25 </div>
26 );
27}
src/file-viewer/pages/file.cotyledon_enterance.tsx deleted-27
...@@ -1,27 +0,0 @@
1import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
2import { addScript } from "#sitegen";
3import { Readme } from "@/file-viewer/cotyledon.tsx";
4import { MediaPanel } from "../views/clofi.tsx";
5
6export const theme = {
7 bg: "#312652",
8 fg: "#f0f0ff",
9 primary: "#fabe32",
10};
11
12export const meta = { title: "living room" };
13
14export default function CotyledonPage() {
15 addScript("../scripts/canvas_cotyledon.client.ts");
16 return (
17 <div class="files ctld ctld-et">
18 <MediaPanel
19 file={MediaFile.getByPath("/")!}
20 isLast={false}
21 activeFilename={null}
22 hasCotyledonCookie={true}
23 />
24 <Readme />
25 </div>
26 );
27}
src/file-viewer/pages/file.cotyledon_speedbump.tsx deleted-27
...@@ -1,27 +0,0 @@
1import { MediaFile } from "../models/MediaFile.ts";
2import { MediaPanel } from "../views/clofi.tsx";
3import { addScript } from "#sitegen";
4import { Speedbump } from "../cotyledon.tsx";
5
6export const theme = {
7 bg: "#312652",
8 fg: "#f0f0ff",
9 primary: "#fabe32",
10};
11
12export const meta = { title: "the front door" };
13
14export default function CotyledonPage() {
15 addScript("../scripts/canvas_cotyledon.client.ts");
16 return (
17 <div class="files ctld ctld-sb">
18 <MediaPanel
19 file={MediaFile.getByPath("/")!}
20 isLast={false}
21 activeFilename={null}
22 hasCotyledonCookie={false}
23 />
24 <Speedbump />
25 </div>
26 );
27}
src/source-of-truth.ts-2
...@@ -29,8 +29,6 @@ if (!fs.existsSync(derivedFileRoot)) {...@@ -29,8 +29,6 @@ if (!fs.existsSync(derivedFileRoot)) {
29 throw new Error(`${derivedFileRoot} does not exist`);29 throw new Error(`${derivedFileRoot} does not exist`);
30}30}
3131
32type Awaitable<T> = T | Promise<T>;
33
34// Re-use file descriptors if the same file is being read twice.32// Re-use file descriptors if the same file is being read twice.
35const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();33const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();
3634