authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-06-27 19:40:19-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:45-07:00
logbba4a6d0d133e1513a1a4acbba1d6f1dd5fdd627
tree356cf1637de8b49e425918047eff931c7215e2d5
parente258365f5bba27170707594f3548c6a9753532a5
signature Commit is signed but in an unrecognized format.

stuff for file view

19 files changed, 1087 insertions(+), 241 deletions(-)

framework/bundle.ts+1-1
...@@ -250,7 +250,7 @@ export async function finalizeServerJavaScript(...@@ -250,7 +250,7 @@ export async function finalizeServerJavaScript(
250 // Replace the magic word250 // Replace the magic word
251 let text = files[fileWithMagicWord].toString("utf-8");251 let text = files[fileWithMagicWord].toString("utf-8");
252 text = text.replace(252 text = text.replace(
253 new RegExp(magicWord + "\\[(-?\\d)\\]", "gs"),253 new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"),
254 (_, i) => {254 (_, i) => {
255 i = Number(i);255 i = Number(i);
256 // Inline the styling data256 // Inline the styling data
framework/lib/async.ts+22-4
...@@ -34,6 +34,12 @@ export class Queue<T, R> {...@@ -34,6 +34,12 @@ export class Queue<T, R> {
34 this.#passive = options.passive ?? false; 34 this.#passive = options.passive ?? false;
35 } 35 }
36 36
37 cancel() {
38 const bar = this.#cachedProgress;
39 bar?.stop();
40 this.#queue = [];
41 }
42
37 get bar() { 43 get bar() {
38 const cached = this.#cachedProgress; 44 const cached = this.#cachedProgress;
39 if (!cached) { 45 if (!cached) {
...@@ -65,7 +71,7 @@ export class Queue<T, R> {...@@ -65,7 +71,7 @@ export class Queue<T, R> {
65 return cached; 71 return cached;
66 } 72 }
67 73
68 add(args: T) { 74 addReturn(args: T) {
69 this.#total += 1; 75 this.#total += 1;
70 this.updateTotal(); 76 this.updateTotal();
71 if (this.#active.length > this.#maxJobs) { 77 if (this.#active.length > this.#maxJobs) {
...@@ -76,6 +82,10 @@ export class Queue<T, R> {...@@ -76,6 +82,10 @@ export class Queue<T, R> {
76 return this.#run(args); 82 return this.#run(args);
77 } 83 }
78 84
85 add(args: T) {
86 return this.addReturn(args).then(() => {}, () => {});
87 }
88
79 addMany(items: T[]) { 89 addMany(items: T[]) {
80 this.#total += items.length; 90 this.#total += items.length;
81 this.updateTotal(); 91 this.updateTotal();
...@@ -92,10 +102,12 @@ export class Queue<T, R> {...@@ -92,10 +102,12 @@ export class Queue<T, R> {
92 const itemText = this.#getItemText(args); 102 const itemText = this.#getItemText(args);
93 const spinner = new Spinner(itemText); 103 const spinner = new Spinner(itemText);
94 spinner.stop(); 104 spinner.stop();
105 (spinner as any).redraw = () => (bar as any).redraw();
95 const active = this.#active; 106 const active = this.#active;
96 try { 107 try {
97 active.unshift(spinner); 108 active.unshift(spinner);
98 bar.props = { active }; 109 bar.props = { active };
110 console.log(this.#name + ": " + itemText);
99 const result = await this.#fn(args, spinner); 111 const result = await this.#fn(args, spinner);
100 this.#done++; 112 this.#done++;
101 return result; 113 return result;
...@@ -139,7 +151,7 @@ export class Queue<T, R> {...@@ -139,7 +151,7 @@ export class Queue<T, R> {
139 } 151 }
140 } 152 }
141 153
142 async done(o: { method: "success" | "stop" }) { 154 async done(o?: { method: "success" | "stop" }) {
143 if (this.#active.length === 0) { 155 if (this.#active.length === 0) {
144 this.#end(o); 156 this.#end(o);
145 return; 157 return;
...@@ -153,8 +165,8 @@ export class Queue<T, R> {...@@ -153,8 +165,8 @@ export class Queue<T, R> {
153 165
154 #end( 166 #end(
155 { method = this.#passive ? "stop" : "success" }: { 167 { method = this.#passive ? "stop" : "success" }: {
156 method: "success" | "stop"; 168 method?: "success" | "stop";
157 }, 169 } = {},
158 ) { 170 ) {
159 const bar = this.#cachedProgress; 171 const bar = this.#cachedProgress;
160 if (this.#errors.length > 0) { 172 if (this.#errors.length > 0) {
...@@ -171,6 +183,12 @@ export class Queue<T, R> {...@@ -171,6 +183,12 @@ export class Queue<T, R> {
171 get active(): boolean { 183 get active(): boolean {
172 return this.#active.length !== 0; 184 return this.#active.length !== 0;
173 } 185 }
186
187 [Symbol.dispose]() {
188 if (this.active) {
189 this.cancel();
190 }
191 }
174} 192}
175 193
176const cwd = process.cwd(); 194const cwd = process.cwd();
framework/lib/fs.ts+7
...@@ -1,7 +1,10 @@...@@ -1,7 +1,10 @@
1// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in1// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in
2// Node.js sync+promise fs methods. For convenince.2// Node.js sync+promise fs methods. For convenince.
3export {3export {
4 createReadStream,
5 createWriteStream,
4 existsSync,6 existsSync,
7 open,
5 readdir,8 readdir,
6 readdirSync,9 readdirSync,
7 readFile,10 readFile,
...@@ -57,6 +60,8 @@ export function readJsonSync<T>(file: string) {...@@ -57,6 +60,8 @@ export function readJsonSync<T>(file: string) {
5760
58import * as path from "node:path";61import * as path from "node:path";
59import {62import {
63 createReadStream,
64 createWriteStream,
60 existsSync,65 existsSync,
61 mkdirSync as nodeMkdirSync,66 mkdirSync as nodeMkdirSync,
62 readdirSync,67 readdirSync,
...@@ -67,9 +72,11 @@ import {...@@ -67,9 +72,11 @@ import {
67} from "node:fs";72} from "node:fs";
68import {73import {
69 mkdir as nodeMkdir,74 mkdir as nodeMkdir,
75 open,
70 readdir,76 readdir,
71 readFile,77 readFile,
72 rm,78 rm,
73 stat,79 stat,
74 writeFile,80 writeFile,
75} from "node:fs/promises";81} from "node:fs/promises";
82export { Stats } from "node:fs";
framework/lib/sqlite.ts+43-12
...@@ -48,6 +48,15 @@ export class WrappedDatabase {...@@ -48,6 +48,15 @@ export class WrappedDatabase {
48 prepare<Args extends unknown[] = [], Result = unknown>(48 prepare<Args extends unknown[] = [], Result = unknown>(
49 query: string,49 query: string,
50 ): Stmt<Args, Result> {50 ): Stmt<Args, Result> {
51 query = query.trim();
52 const lines = query.split("\n");
53 const trim = Math.min(
54 ...lines.map((line) =>
55 line.trim().length === 0 ? Infinity : line.match(/^\s*/)![0].length
56 ),
57 );
58 query = lines.map((x) => x.slice(trim)).join("\n");
59
51 let prepared;60 let prepared;
52 try {61 try {
53 prepared = this.node.prepare(query);62 prepared = this.node.prepare(query);
...@@ -62,42 +71,64 @@ export class WrappedDatabase {...@@ -62,42 +71,64 @@ export class WrappedDatabase {
62export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {71export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
63 #node: StatementSync;72 #node: StatementSync;
64 #class: any | null = null;73 #class: any | null = null;
74 query: string;
75
65 constructor(node: StatementSync) {76 constructor(node: StatementSync) {
66 this.#node = node;77 this.#node = node;
78 this.query = node.sourceSQL;
67 }79 }
6880
69 /** Get one row */81 /** Get one row */
70 get(...args: Args): Row | null {82 get(...args: Args): Row | null {
71 const item = this.#node.get(...args as any) as Row;83 return this.#wrap(args, () => {
72 if (!item) return null;84 const item = this.#node.get(...args as any) as Row;
73 const C = this.#class;85 if (!item) return null;
74 if (C) Object.setPrototypeOf(item, C.prototype);86 const C = this.#class;
75 return item;87 if (C) Object.setPrototypeOf(item, C.prototype);
88 return item;
89 });
76 }90 }
77 getNonNull(...args: Args) {91 getNonNull(...args: Args) {
78 const item = this.get(...args);92 const item = this.get(...args);
79 if (!item) throw new Error("Query returned no result");93 if (!item) {
94 throw this.#wrap(args, () => new Error("Query returned no result"));
95 }
80 return item;96 return item;
81 }97 }
82 iter(...args: Args): Iterator<Row> {98 iter(...args: Args): Iterator<Row> {
83 return this.array(...args)[Symbol.iterator]();99 return this.#wrap(args, () => this.array(...args)[Symbol.iterator]());
84 }100 }
85 /** Get all rows */101 /** Get all rows */
86 array(...args: Args): Row[] {102 array(...args: Args): Row[] {
87 const array = this.#node.all(...args as any) as Row[];103 return this.#wrap(args, () => {
88 const C = this.#class;104 const array = this.#node.all(...args as any) as Row[];
89 if (C) array.forEach((item) => Object.setPrototypeOf(item, C.prototype));105 const C = this.#class;
90 return array;106 if (C) array.forEach((item) => Object.setPrototypeOf(item, C.prototype));
107 return array;
108 });
91 }109 }
92 /** Return the number of changes / row ID */110 /** Return the number of changes / row ID */
93 run(...args: Args) {111 run(...args: Args) {
94 return this.#node.run(...args as any);112 return this.#wrap(args, () => this.#node.run(...args as any));
95 }113 }
96114
97 as<R>(Class: { new (): R }): Stmt<Args, R> {115 as<R>(Class: { new (): R }): Stmt<Args, R> {
98 this.#class = Class;116 this.#class = Class;
99 return this as any;117 return this as any;
100 }118 }
119
120 #wrap<T>(args: unknown[], fn: () => T) {
121 try {
122 return fn();
123 } catch (err: any) {
124 if (err && typeof err === "object") {
125 err.query = this.query;
126 args = args.flat(Infinity);
127 err.queryArgs = args.length === 1 ? args[0] : args;
128 }
129 throw err;
130 }
131 }
101}132}
102133
103import { DatabaseSync, StatementSync } from "node:sqlite";134import { DatabaseSync, StatementSync } from "node:sqlite";
package-lock.json+15-1
...@@ -17,7 +17,9 @@...@@ -17,7 +17,9 @@
17 "marko": "^6.0.20",17 "marko": "^6.0.20",
18 "puppeteer": "^24.10.1",18 "puppeteer": "^24.10.1",
19 "sharp": "^0.34.2",19 "sharp": "^0.34.2",
20 "unique-names-generator": "^4.7.1"20 "unique-names-generator": "^4.7.1",
21 "vscode-oniguruma": "^2.0.1",
22 "vscode-textmate": "^9.2.0"
21 },23 },
22 "devDependencies": {24 "devDependencies": {
23 "@types/node": "^22.15.29",25 "@types/node": "^22.15.29",
...@@ -4680,6 +4682,18 @@...@@ -4680,6 +4682,18 @@
4680 "url": "https://opencollective.com/unified"4682 "url": "https://opencollective.com/unified"
4681 }4683 }
4682 },4684 },
4685 "node_modules/vscode-oniguruma": {
4686 "version": "2.0.1",
4687 "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz",
4688 "integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==",
4689 "license": "MIT"
4690 },
4691 "node_modules/vscode-textmate": {
4692 "version": "9.2.0",
4693 "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.2.0.tgz",
4694 "integrity": "sha512-rkvG4SraZQaPSN/5XjwKswdU0OP9MF28QjrYzUBbhb8QyG3ljB1Ky996m++jiI7KdiAP2CkBiQZd9pqEDTClqA==",
4695 "license": "MIT"
4696 },
4683 "node_modules/w3c-keyname": {4697 "node_modules/w3c-keyname": {
4684 "version": "2.2.8",4698 "version": "2.2.8",
4685 "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz",4699 "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz",
package.json+3-1
...@@ -13,7 +13,9 @@...@@ -13,7 +13,9 @@
13 "marko": "^6.0.20",13 "marko": "^6.0.20",
14 "puppeteer": "^24.10.1",14 "puppeteer": "^24.10.1",
15 "sharp": "^0.34.2",15 "sharp": "^0.34.2",
16 "unique-names-generator": "^4.7.1"16 "unique-names-generator": "^4.7.1",
17 "vscode-oniguruma": "^2.0.1",
18 "vscode-textmate": "^9.2.0"
17 },19 },
18 "devDependencies": {20 "devDependencies": {
19 "@types/node": "^22.15.29",21 "@types/node": "^22.15.29",
repl.js+1-1
...@@ -18,7 +18,7 @@ const repl = hot.load("node:repl").start({...@@ -18,7 +18,7 @@ const repl = hot.load("node:repl").start({
18 .catch((err) => {18 .catch((err) => {
19 // TODO: improve @paperclover/console's ability to print AggregateError19 // TODO: improve @paperclover/console's ability to print AggregateError
20 // and errors with extra random properties20 // and errors with extra random properties
21 console.error(util.inspect(err));21 console.error(util.inspect(err, false, 10, true));
22 })22 })
23 .then((result) => done(null, result));23 .then((result) => done(null, result));
24 },24 },
src/backend.ts+7-4
...@@ -1,19 +1,22 @@...@@ -1,19 +1,22 @@
1const logHttp = scoped("http", { color: "magenta" }); 1// This is the main file for the backend
2
3const app = new Hono(); 2const app = new Hono();
3const logHttp = scoped("http", { color: "magenta" });
4 4
5app.notFound(assets.notFound); 5// Middleware
6
7app.use(trimTrailingSlash()); 6app.use(trimTrailingSlash());
8app.use(removeDuplicateSlashes); 7app.use(removeDuplicateSlashes);
9app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4)))); 8app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4))));
10app.use(admin.middleware); 9app.use(admin.middleware);
11 10
11// Backends
12app.route("", require("./q+a/backend.ts").app); 12app.route("", require("./q+a/backend.ts").app);
13app.route("", require("./file-viewer/backend.tsx").app); 13app.route("", require("./file-viewer/backend.tsx").app);
14 14
15// Asset middleware has least precedence
15app.use(assets.middleware); 16app.use(assets.middleware);
16 17
18// Handlers
19app.notFound(assets.notFound);
17if (process.argv.includes("--development")) { 20if (process.argv.includes("--development")) {
18 app.onError((err, c) => { 21 app.onError((err, c) => {
19 if (err instanceof HTTPException) { 22 if (err instanceof HTTPException) {
src/file-viewer/bin/extension-stats.ts deleted-83
...@@ -1,83 +0,0 @@
1import * as path from "node:path";
2import { cache, MediaFile } from "../db";
3
4// Function to get file extension statistics
5function getExtensionStats() {
6 // Get all files (not directories) from the database
7 const query = `
8 SELECT path FROM media_files
9 WHERE kind = ${MediaFile.Kind.file}
10 `;
11
12 // Use raw query to get all file paths
13 const rows = cache.query(query).all() as { path: string }[];
14
15 // Count extensions
16 const extensionCounts: Record<string, number> = {};
17
18 for (const row of rows) {
19 const extension = path.extname(row.path).toLowerCase();
20 extensionCounts[extension] = (extensionCounts[extension] || 0) + 1;
21 }
22
23 // Sort extensions by count (descending)
24 const sortedExtensions = Object.entries(extensionCounts)
25 .sort((a, b) => b[1] - a[1]);
26
27 return {
28 totalFiles: rows.length,
29 extensions: sortedExtensions,
30 };
31}
32
33// Function to print a visual table
34function printExtensionTable() {
35 const stats = getExtensionStats();
36
37 // Calculate column widths
38 const extensionColWidth = Math.max(
39 ...stats.extensions.map(([ext]) => ext.length),
40 "Extension".length,
41 ) + 2;
42
43 const countColWidth = Math.max(
44 ...stats.extensions.map(([_, count]) => count.toString().length),
45 "Count".length,
46 ) + 2;
47
48 const percentColWidth = "Percentage".length + 2;
49
50 // Print header
51 console.log("MediaFile Extension Statistics");
52 console.log(`Total files: ${stats.totalFiles}`);
53 console.log();
54
55 // Print table header
56 console.log(
57 "Extension".padEnd(extensionColWidth) +
58 "Count".padEnd(countColWidth) +
59 "Percentage".padEnd(percentColWidth),
60 );
61
62 // Print separator
63 console.log(
64 "-".repeat(extensionColWidth) +
65 "-".repeat(countColWidth) +
66 "-".repeat(percentColWidth),
67 );
68
69 // Print rows
70 for (const [extension, count] of stats.extensions) {
71 const percentage = ((count / stats.totalFiles) * 100).toFixed(2);
72 const ext = extension || "(no extension)";
73
74 console.log(
75 ext.padEnd(extensionColWidth) +
76 count.toString().padEnd(countColWidth) +
77 `${percentage}%`.padEnd(percentColWidth),
78 );
79 }
80}
81
82// Run the program
83printExtensionTable();
src/file-viewer/bin/list.ts created+8
...@@ -0,0 +1,8 @@
1export function main() {
2 const meows = MediaFile.db.prepare(`
3 select * from media_files;
4 `).as(MediaFile).array();
5 console.log(meows);
6}
7
8import { MediaFile } from "@/file-viewer/models/MediaFile.ts";
src/file-viewer/bin/scan3.ts+589-2
...@@ -1,2 +1,589 @@...@@ -1,2 +1,589 @@
1import "@/file-viewer/models/MediaFile.ts"; 1const root = path.resolve("C:/media");
2import "@/file-viewer/models/BlobAsset.ts"; 2const workDir = path.resolve(".clover/file-assets");
3
4export async function main() {
5 const start = performance.now();
6 const timerSpinner = new Spinner({
7 text: () =>
8 `paper clover's scan3 [${
9 ((performance.now() - start) / 1000).toFixed(1)
10 }s]`,
11 fps: 10,
12 });
13 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
14
15 // Read a directory or file stat and queue up changed files.
16 using qList = new async.Queue({
17 name: "Discover Tree",
18 async fn(absPath: string, spin) {
19 const stat = await fs.stat(absPath);
20
21 const publicPath = toPublicPath(absPath);
22 const mediaFile = MediaFile.getByPath(publicPath);
23
24 if (stat.isDirectory()) {
25 const items = await fs.readdir(absPath);
26 qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
27
28 if (mediaFile) {
29 const deleted = mediaFile.getChildren()
30 .filter((child) => !items.includes(child.basename))
31 .flatMap((child) =>
32 child.kind === MediaFileKind.directory
33 ? child.getRecursiveFileChildren()
34 : child
35 );
36
37 qMeta.addMany(deleted.map((mediaFile) => ({
38 absPath: path.join(root, mediaFile.path),
39 publicPath: mediaFile.path,
40 stat: null,
41 mediaFile,
42 })));
43 }
44
45 return;
46 }
47
48 // All processes must be performed again if there is no file.
49 if (
50 !mediaFile ||
51 stat.size !== mediaFile.size ||
52 stat.mtime.getTime() !== mediaFile.date.getTime()
53 ) {
54 qMeta.add({ absPath, publicPath, stat, mediaFile });
55 return;
56 }
57
58 // If the scanners changed, it may mean more processes should be run.
59 queueProcessors({ absPath, stat, mediaFile });
60 },
61 maxJobs: 24,
62 });
63 using qMeta = new async.Queue({
64 name: "Update Metadata",
65 async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) {
66 if (!stat) {
67 // File was deleted.
68 await runUndoProcessors(UNWRAP(mediaFile));
69 return;
70 }
71 // TODO: run scrubLocationMetadata first
72
73 const hash = await new Promise<string>((resolve, reject) => {
74 const reader = fs.createReadStream(absPath);
75 reader.on("error", reject);
76
77 const hasher = crypto.createHash("sha1").setEncoding("hex");
78 hasher.on("error", reject);
79 hasher.on("readable", () => resolve(hasher.read()));
80
81 reader.pipe(hasher);
82 });
83 let date = stat.mtime;
84 if (
85 mediaFile &&
86 mediaFile.date.getTime() < stat.mtime.getTime() &&
87 (Date.now() - stat.mtime.getTime()) < monthMilliseconds
88 ) {
89 date = mediaFile.date;
90 console.warn(
91 `M-time on ${publicPath} was likely corrupted. ${
92 formatDate(mediaFile.date)
93 } -> ${formatDate(stat.mtime)}`,
94 );
95 }
96 mediaFile = MediaFile.createFile({
97 path: publicPath,
98 date,
99 hash,
100 size: stat.size,
101 duration: mediaFile?.duration ?? 0,
102 dimensions: mediaFile?.dimensions ?? "",
103 contents: mediaFile?.contents ?? "",
104 });
105 await queueProcessors({ absPath, stat, mediaFile });
106 },
107 getItemText: (job) =>
108 job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"),
109 maxJobs: 2,
110 });
111 using qProcess = new async.Queue({
112 name: "Process Contents",
113 async fn(
114 { absPath, stat, mediaFile, processor, index, after }: ProcessJob,
115 spin,
116 ) {
117 await processor.run({ absPath, stat, mediaFile, spin });
118 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
119 for (const dependantJob of after) {
120 ASSERT(dependantJob.needs > 0);
121 dependantJob.needs -= 1;
122 if (dependantJob.needs == 0) qProcess.add(dependantJob);
123 }
124 },
125 getItemText: ({ mediaFile, processor }) =>
126 `${mediaFile.path.slice(1)} - ${processor.name}`,
127 maxJobs: 2,
128 });
129
130 function decodeProcessors(input: string) {
131 return input
132 .split(";")
133 .filter(Boolean)
134 .map(([a, b, c]) => ({
135 id: a,
136 hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0),
137 }));
138 }
139
140 async function queueProcessors(
141 { absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
142 ) {
143 const ext = mediaFile.extension.toLowerCase();
144 let possible = processors.filter((p) => p.include.has(ext));
145 if (possible.length === 0) return;
146
147 const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
148 ASSERT(hash <= 0xFFFF);
149 let processed = mediaFile.processed;
150
151 // If the hash has changed, migrate the bitfield over.
152 // This also runs when the processor hash is in it's initial 0 state.
153 const order = decodeProcessors(mediaFile.processors);
154 if ((processed & 0xFFFF) !== hash) {
155 const previous = order.filter((_, i) =>
156 (processed & (1 << (16 + i))) !== 0
157 );
158 processed = hash;
159 for (const { id, hash } of previous) {
160 const p = processors.find((p) => p.id === id);
161 if (!p) continue;
162 const index = possible.indexOf(p);
163 if (index !== -1 && p.hash === hash) {
164 processed |= 1 << (16 + index);
165 } else {
166 if (p.undo) await p.undo(mediaFile);
167 }
168 }
169 mediaFile.setProcessors(
170 processed,
171 possible.map((p) =>
172 p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF)
173 ).join(";"),
174 );
175 } else {
176 possible = order.map(({ id }) =>
177 UNWRAP(possible.find((p) => p.id === id))
178 );
179 }
180
181 // Queue needed processors.
182 const jobs: ProcessJob[] = [];
183 for (let i = 0, { length } = possible; i < length; i += 1) {
184 if ((processed & (1 << (16 + i))) === 0) {
185 const job: ProcessJob = {
186 absPath,
187 stat,
188 mediaFile,
189 processor: possible[i],
190 index: i,
191 after: [],
192 needs: possible[i].depends.length,
193 };
194 jobs.push(job);
195 if (job.needs === 0) qProcess.add(job);
196 }
197 }
198 for (const job of jobs) {
199 for (const dependId of job.processor.depends) {
200 const dependJob = jobs.find((j) => j.processor.id === dependId);
201 if (dependJob) {
202 dependJob.after.push(job);
203 } else {
204 ASSERT(job.needs > 0);
205 job.needs -= 1;
206 if (job.needs === 0) qProcess.add(job);
207 }
208 }
209 }
210 }
211
212 async function runUndoProcessors(mediaFile: MediaFile) {
213 const { processed } = mediaFile;
214 const previous = decodeProcessors(mediaFile.processors)
215 .filter((_, i) => (processed & (1 << (16 + i))) !== 0);
216 for (const { id } of previous) {
217 const p = processors.find((p) => p.id === id);
218 if (!p) continue;
219 if (p.undo) {
220 await p.undo(mediaFile);
221 }
222 }
223 mediaFile.delete();
224 }
225
226 // Add the root & recursively iterate!
227 qList.add(root);
228 await qList.done();
229 await qMeta.done();
230 await qProcess.done();
231
232 console.info(
233 "Updated file viewer index in " +
234 ((performance.now() - start) / 1000).toFixed(1) + "s",
235 );
236}
237
238interface Process {
239 name: string;
240 enable?: boolean;
241 include: Set<string>;
242 depends?: string[];
243 /* Perform an action. */
244 run(args: ProcessFileArgs): Promise<void>;
245 /* Should detect if `run` was never even run before before undoing state */
246 undo?(mediaFile: MediaFile): Promise<void>;
247}
248
249const execFileRaw = util.promisify(child_process.execFile);
250const execFile: typeof execFileRaw = ((
251 ...args: Parameters<typeof execFileRaw>
252) =>
253 execFileRaw(...args).catch((e: any) => {
254 if (e?.message?.startsWith?.("Command failed")) {
255 if (e.code > (2 ** 31)) e.code |= 0;
256 const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
257 e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
258 }
259 throw e;
260 })) as any;
261const ffprobe = testProgram("ffprobe", "--help");
262const ffmpeg = testProgram("ffmpeg", "--help");
263
264const ffmpegOptions = [
265 "-hide_banner",
266 "-loglevel",
267 "warning",
268];
269
270const imageSizes = [64, 128, 256, 512, 1024, 2048];
271
272const procDuration: Process = {
273 name: "calculate duration",
274 enable: ffprobe !== null,
275 include: rules.extsDuration,
276 async run({ absPath, mediaFile }) {
277 const { stdout } = await execFile(ffprobe!, [
278 "-v",
279 "error",
280 "-show_entries",
281 "format=duration",
282 "-of",
283 "default=noprint_wrappers=1:nokey=1",
284 absPath,
285 ]);
286
287 const duration = parseFloat(stdout.trim());
288 if (Number.isNaN(duration)) {
289 throw new Error("Could not extract duration from " + stdout);
290 }
291 mediaFile.setDuration(Math.ceil(duration));
292 },
293};
294
295// NOTE: Never re-order the processors. Add new ones at the end.
296const procDimensions: Process = {
297 name: "calculate dimensions",
298 enable: ffprobe != null,
299 include: rules.extsDimensions,
300 async run({ absPath, mediaFile }) {
301 const ext = path.extname(absPath);
302
303 let dimensions;
304
305 if (ext === ".svg") {
306 // Parse out of text data
307 const content = await fs.readFile(absPath, "utf8");
308 const widthMatch = content.match(/width="(\d+)"/);
309 const heightMatch = content.match(/height="(\d+)"/);
310
311 if (widthMatch && heightMatch) {
312 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
313 }
314 } else {
315 // Use ffprobe to observe streams
316 const { stdout } = await execFile("ffprobe", [
317 "-v",
318 "error",
319 "-select_streams",
320 "v:0",
321 "-show_entries",
322 "stream=width,height",
323 "-of",
324 "csv=s=x:p=0",
325 absPath,
326 ]);
327 if (stdout.includes("x")) {
328 dimensions = stdout.trim();
329 }
330 }
331
332 mediaFile.setDimensions(dimensions ?? "");
333 },
334};
335
336const procLoadTextContents: Process = {
337 name: "load text content",
338 include: rules.extsReadContents,
339 async run({ absPath, mediaFile, stat }) {
340 if (stat.size > 1_000_000) return;
341 const text = await fs.readFile(absPath, "utf-8");
342 mediaFile.setContents(text);
343 },
344};
345
346const procHighlightCode: Process = {
347 name: "highlight source code",
348 include: new Set(rules.extsCode.keys()),
349 async run({ absPath, mediaFile, stat }) {
350 const language = UNWRAP(
351 rules.extsCode.get(path.extname(absPath).toLowerCase()),
352 );
353 // An issue is that .ts is an overloaded extension, shared between
354 // 'transport stream' and 'typescript'.
355 //
356 // Filter used here is:
357 // - more than 1mb
358 // - invalid UTF-8
359 if (stat.size > 1_000_000) return;
360 let code;
361 const buf = await fs.readFile(absPath);
362 try {
363 code = new TextDecoder("utf-8", { fatal: true }).decode(buf);
364 } catch (error) {
365 mediaFile.setContents("");
366 return;
367 }
368 const content = await highlight.highlightCode(code, language);
369 mediaFile.setContents(content);
370 },
371};
372
373const imageSubsets = [
374 {
375 ext: ".webp",
376 // deno-fmt-disable-line
377 args: [
378 "-lossless",
379 "0",
380 "-compression_level",
381 "6",
382 "-quality",
383 "95",
384 "-method",
385 "6",
386 ],
387 },
388 {
389 ext: ".jxl",
390 args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
391 },
392];
393
394const procImageSubsets: Process = {
395 name: "encode image subsets",
396 include: rules.extsImage,
397 enable: false,
398 depends: ["calculate dimensions"],
399 async run({ absPath, mediaFile, stat, spin }) {
400 const { width, height } = UNWRAP(mediaFile.parseDimensions());
401 const targetSizes = imageSizes.filter((w) => w < width);
402 const baseStatus = spin.text;
403
404 using stack = new DisposableStack();
405 for (const size of targetSizes) {
406 const { w, h } = resizeDimensions(width, height, size);
407 for (const { ext, args } of imageSubsets) {
408 spin.text = baseStatus +
409 ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
410
411 stack.use(
412 await produceAsset(
413 `${mediaFile.hash}/${size}${ext}`,
414 async (out) => {
415 await fs.mkdir(path.dirname(out));
416 await fs.rm(out, { force: true });
417 await execFile(ffmpeg!, [
418 ...ffmpegOptions,
419 "-i",
420 absPath,
421 "-vf",
422 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
423 ...args,
424 out,
425 ]);
426 return [out];
427 },
428 ),
429 );
430 }
431 }
432
433 stack.move();
434 },
435 async undo(mediaFile) {
436 const { width } = UNWRAP(mediaFile.parseDimensions());
437 const targetSizes = imageSizes.filter((w) => w < width);
438 for (const size of targetSizes) {
439 for (const { ext } of imageSubsets) {
440 unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
441 }
442 }
443 },
444};
445
446const videoFormats = [
447 {
448 name: "webm",
449 },
450];
451
452const processors = [
453 procDimensions,
454 procDuration,
455 procLoadTextContents,
456 procHighlightCode,
457 procImageSubsets,
458]
459 .map((process, id, all) => {
460 const strIndex = (id: number) =>
461 String.fromCharCode("a".charCodeAt(0) + id);
462 return {
463 ...process as Process,
464 id: strIndex(id),
465 // Create a unique key.
466 hash: new Uint16Array(
467 crypto.createHash("sha1")
468 .update(process.run.toString())
469 .digest().buffer,
470 ).reduce((a, b) => a ^ b),
471 depends: (process.depends ?? []).map((depend) => {
472 const index = all.findIndex((p) => p.name === depend);
473 if (index === -1) throw new Error(`Cannot find depend '${depend}'`);
474 if (index === id) throw new Error(`Cannot depend on self: '${depend}'`);
475 return strIndex(index);
476 }),
477 };
478 });
479
480function resizeDimensions(w: number, h: number, desiredWidth: number) {
481 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
482 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
483}
484
485async function produceAsset(
486 key: string,
487 builder: (prefix: string) => Promise<string[]>,
488) {
489 const asset = AssetRef.putOrIncrement(key);
490 try {
491 if (asset.refs === 1) {
492 const paths = await builder(path.join(workDir, key));
493 asset.addFiles(
494 paths.map((file) =>
495 path.relative(workDir, file)
496 .replaceAll("\\", "/")
497 ),
498 );
499 }
500 return {
501 [Symbol.dispose]: () => asset.unref(),
502 };
503 } catch (err: any) {
504 if (err && typeof err === "object") err.assetKey = key;
505 asset.unref();
506 throw err;
507 }
508}
509
510async function unproduceAsset(key: string) {
511 const ref = AssetRef.get(key);
512 if (ref) {
513 ref.unref();
514 console.log(`unref ${key}`);
515 // TODO: remove associated files from target
516 }
517}
518
519interface UpdateMetadataJob {
520 absPath: string;
521 publicPath: string;
522 stat: fs.Stats | null;
523 mediaFile: MediaFile | null;
524}
525
526interface ProcessFileArgs {
527 absPath: string;
528 stat: fs.Stats;
529 mediaFile: MediaFile;
530 spin: Spinner;
531}
532
533interface ProcessJob {
534 absPath: string;
535 stat: fs.Stats;
536 mediaFile: MediaFile;
537 processor: typeof processors[0];
538 index: number;
539 after: ProcessJob[];
540 needs: number;
541}
542
543export function skipBasename(basename: string): boolean {
544 // dot files must be incrementally tracked
545 if (basename === ".dirsort") return true;
546 if (basename === ".friends") return true;
547
548 return (
549 basename.startsWith(".") ||
550 basename.startsWith("._") ||
551 basename.startsWith(".tmp") ||
552 basename === ".DS_Store" ||
553 basename.toLowerCase() === "thumbs.db" ||
554 basename.toLowerCase() === "desktop.ini"
555 );
556}
557
558export function toPublicPath(absPath: string) {
559 ASSERT(path.isAbsolute(absPath));
560 if (absPath === root) return "/";
561 return "/" + path.relative(root, absPath).replaceAll("\\", "/");
562}
563
564export function testProgram(name: string, helpArgument: string) {
565 try {
566 child_process.spawnSync(name, [helpArgument]);
567 return name;
568 } catch (err) {
569 console.warn(`Missing or corrupt executable '${name}'`);
570 }
571 return null;
572}
573
574const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
575
576import { Spinner } from "@paperclover/console/Spinner";
577import * as async from "#sitegen/async";
578import * as fs from "#sitegen/fs";
579
580import * as path from "node:path";
581import * as child_process from "node:child_process";
582import * as util from "node:util";
583import * as crypto from "node:crypto";
584
585import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
586import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
587import { formatDate } from "@/file-viewer/format.ts";
588import * as rules from "@/file-viewer/rules.ts";
589import * as highlight from "@/file-viewer/highlight.ts";
src/file-viewer/cotyledon.tsx+47
...@@ -1,3 +1,48 @@...@@ -1,3 +1,48 @@
1// WARNING
2// -------
3// This file contains spoilers for COTYLEDON
4// Consider reading through the entire archive before picking apart this
5// code, as this contains the beginning AND the ending sections, which
6// contains very percise storytelling. You've been warned...
7//
8// --> https://paperclover.net/file/cotyledon <--
9//
10//
11//
12//
13//
14//
15//
16//
17//
18//
19//
20//
21//
22//
23//
24//
25//
26//
27//
28//
29//
30//
31//
32//
33//
34//
35//
36//
37//
38//
39//
40//
41//
42//
43//
44
45// SPEEDBUMP
1export function Speedbump() {46export function Speedbump() {
2 return (47 return (
3 <div class="panel last">48 <div class="panel last">
...@@ -80,6 +125,7 @@ export function Speedbump() {...@@ -80,6 +125,7 @@ export function Speedbump() {
80 );125 );
81}126}
82127
128// OPENING
83export function Readme() {129export function Readme() {
84 return (130 return (
85 <div class="panel last">131 <div class="panel last">
...@@ -136,6 +182,7 @@ export function Readme() {...@@ -136,6 +182,7 @@ export function Readme() {
136 );182 );
137}183}
138184
185// TRUE ENDING. Written in Apple Notes.
139export function ForEveryone() {186export function ForEveryone() {
140 return (187 return (
141 <>188 <>
src/file-viewer/highlight.ts+15-16
...@@ -1,10 +1,3 @@...@@ -1,10 +1,3 @@
1import { onceAsync } from "../lib.ts";
2import * as fs from "node:fs/promises";
3import * as path from "node:path";
4import * as oniguruma from "vscode-oniguruma";
5import * as textmate from "vscode-textmate";
6import { escapeHTML } from "../framework/bun-polyfill.ts";
7
8const languages = [1const languages = [
9 "ts",2 "ts",
10 "tsx",3 "tsx",
...@@ -88,7 +81,6 @@ interface HighlightLinesOptions {...@@ -88,7 +81,6 @@ interface HighlightLinesOptions {
88}81}
8982
90export function getStyle(scopesToCheck: string[], langugage: Language) {83export function getStyle(scopesToCheck: string[], langugage: Language) {
91 if (import.meta.main) console.log(scopesToCheck);
92 for (const scope of scopes) {84 for (const scope of scopes) {
93 if (scope[2] && scope[2] !== langugage) continue;85 if (scope[2] && scope[2] !== langugage) continue;
94 const find = scopesToCheck.find((s) => s.startsWith(scope[0]));86 const find = scopesToCheck.find((s) => s.startsWith(scope[0]));
...@@ -98,6 +90,7 @@ export function getStyle(scopesToCheck: string[], langugage: Language) {...@@ -98,6 +90,7 @@ export function getStyle(scopesToCheck: string[], langugage: Language) {
98 }90 }
99 return null;91 return null;
100}92}
93
101function highlightLines({94function highlightLines({
102 lines,95 lines,
103 grammar,96 grammar,
...@@ -120,7 +113,7 @@ function highlightLines({...@@ -120,7 +113,7 @@ function highlightLines({
120 const str = lines[i].slice(token.startIndex, token.endIndex);113 const str = lines[i].slice(token.startIndex, token.endIndex);
121 if (str.trim().length === 0) {114 if (str.trim().length === 0) {
122 // Emit but do not consider scope changes115 // Emit but do not consider scope changes
123 html += escapeHTML(str);116 html += ssr.escapeHtml(str);
124 continue;117 continue;
125 }118 }
126119
...@@ -129,7 +122,7 @@ function highlightLines({...@@ -129,7 +122,7 @@ function highlightLines({
129 if (lastHtmlStyle) html += "</span>";122 if (lastHtmlStyle) html += "</span>";
130 if (style) html += `<span class='${style}'>`;123 if (style) html += `<span class='${style}'>`;
131 }124 }
132 html += escapeHTML(str);125 html += ssr.escapeHtml(str);
133 lastHtmlStyle = style;126 lastHtmlStyle = style;
134 }127 }
135 html += "\n";128 html += "\n";
...@@ -140,7 +133,7 @@ function highlightLines({...@@ -140,7 +133,7 @@ function highlightLines({
140 return { state, html };133 return { state, html };
141}134}
142135
143export const getRegistry = onceAsync(async () => {136export const getRegistry = async.once(async () => {
144 const wasmBin = await fs.readFile(137 const wasmBin = await fs.readFile(
145 path.join(138 path.join(
146 import.meta.dirname,139 import.meta.dirname,
...@@ -187,18 +180,24 @@ export async function highlightCode(code: string, language: Language) {...@@ -187,18 +180,24 @@ export async function highlightCode(code: string, language: Language) {
187 return html;180 return html;
188}181}
189182
190import { existsSync } from "node:fs";183export async function main() {
191if (import.meta.main) {
192 // validate exts184 // validate exts
193 for (const ext of languages) {185 for (const ext of languages) {
194 if (186 if (
195 !existsSync(187 !fs.existsSync(
196 path.join(import.meta.dirname, `highlight-grammar/${ext}.plist`),188 path.join(import.meta.dirname, `highlight-grammar/${ext}.plist`),
197 )189 )
198 ) {190 ) {
199 console.error(`Missing grammar for ${ext}`);191 console.error(`Missing grammar for ${ext}`);
200 }192 }
201 const html = await highlightCode("wwwwwwwwwwwaaaaaaaaaaaaaaaa", ext);193 // Sanity check
194 await highlightCode("wwwwwwwwwwwaaaaaaaaaaaaaaaa", ext);
202 }195 }
203 console.log(await highlightCode(`{"maps":"damn"`, "json"));
204}196}
197
198import * as async from "#sitegen/async";
199import * as fs from "#sitegen/fs";
200import * as path from "node:path";
201import * as oniguruma from "vscode-oniguruma";
202import * as textmate from "vscode-textmate";
203import * as ssr from "#ssr";
src/file-viewer/models/AssetRef.ts created+73
...@@ -0,0 +1,73 @@
1const db = getDb("cache.sqlite");
2db.table(
3 "asset_refs",
4 /* SQL */ `
5 create table if not exists asset_refs (
6 id integer primary key autoincrement,
7 key text not null UNIQUE,
8 refs integer not null
9 );
10 create table if not exists asset_ref_files (
11 file text not null,
12 id integer not null,
13 foreign key (id) references asset_refs(id)
14 );
15 create index asset_ref_files_id on asset_ref_files(id);
16`,
17);
18
19/**
20 * Uncompressed files are read directly from the media store root. Derivied
21 * assets like compressed files, optimized images, and streamable video are
22 * stored in the `derived` folder. After scanning, the derived assets are
23 * uploaded into the store (storage1/clofi-derived dataset on NAS). Since
24 * multiple files can share the same hash, the number of references is
25 * tracked, and the derived content is only produced once. This means if a
26 * file is deleted, it should only decrement a reference count; deleting it
27 * once all references are removed.
28 */
29export class AssetRef {
30 /** Key which aws referenced */
31 id!: number;
32 key!: string;
33 refs!: number;
34
35 unref() {
36 decrementQuery.run(this.key);
37 deleteUnreferencedQuery.run().changes > 0;
38 }
39
40 addFiles(files: string[]) {
41 for (const file of files) {
42 addFileQuery.run({ id: this.id, file });
43 }
44 }
45
46 static get(key: string) {
47 return getQuery.get(key);
48 }
49
50 static putOrIncrement(key: string) {
51 putOrIncrementQuery.get(key);
52 return UNWRAP(AssetRef.get(key));
53 }
54}
55
56const getQuery = db.prepare<[key: string]>(/* SQL */ `
57 select * from asset_refs where key = ?;
58`).as(AssetRef);
59const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ `
60 insert into asset_refs (key, refs) values (?, 1)
61 on conflict(key) do update set refs = refs + 1;
62`);
63const decrementQuery = db.prepare<[key: string]>(/* SQL */ `
64 update asset_refs set refs = refs - 1 where key = ? and refs > 0;
65`);
66const deleteUnreferencedQuery = db.prepare(/* SQL */ `
67 delete from asset_refs where refs <= 0;
68`);
69const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ `
70 insert into asset_ref_files (id, file) values ($id, $file);
71`);
72
73import { getDb } from "#sitegen/sqlite";
src/file-viewer/models/BlobAsset.ts deleted-57
...@@ -1,57 +0,0 @@
1const db = getDb("cache.sqlite");
2db.table(
3 "blob_assets",
4 /* SQL */ `
5 CREATE TABLE IF NOT EXISTS blob_assets (
6 hash TEXT PRIMARY KEY,
7 refs INTEGER NOT NULL DEFAULT 0
8 );
9`,
10);
11
12/**
13 * Uncompressed files are read directly from the media store root. Compressed
14 * files are stored as `<compress store>/<first 2 chars of hash>/<hash>` Since
15 * multiple files can share the same hash, the number of references is tracked
16 * so that when a file is deleted, the compressed data is only removed when all
17 * references are gone.
18 */
19export class BlobAsset {
20 /** sha1 of the contents */
21 hash!: string;
22 refs!: number;
23
24 decrementOrDelete() {
25 BlobAsset.decrementOrDelete(this.hash);
26 }
27
28 static get(hash: string) {
29 return getQuery.get(hash);
30 }
31 static putOrIncrement(hash: string) {
32 ASSERT(hash.length === 40);
33 putOrIncrementQuery.get(hash);
34 return BlobAsset.get(hash)!;
35 }
36 static decrementOrDelete(hash: string) {
37 ASSERT(hash.length === 40);
38 decrementQuery.run(hash);
39 return deleteQuery.run(hash).changes > 0;
40 }
41}
42
43const getQuery = db.prepare<[hash: string]>(/* SQL */ `
44 SELECT * FROM blob_assets WHERE hash = ?;
45`).as(BlobAsset);
46const putOrIncrementQuery = db.prepare<[hash: string]>(/* SQL */ `
47 INSERT INTO blob_assets (hash, refs) VALUES (?, 1)
48 ON CONFLICT(hash) DO UPDATE SET refs = refs + 1;
49`);
50const decrementQuery = db.prepare<[hash: string]>(/* SQL */ `
51 UPDATE blob_assets SET refs = refs - 1 WHERE hash = ? AND refs > 0;
52`);
53const deleteQuery = db.prepare<[hash: string]>(/* SQL */ `
54 DELETE FROM blob_assets WHERE hash = ? AND refs <= 0;
55`);
56
57import { getDb } from "#sitegen/sqlite";
src/file-viewer/models/MediaFile.ts+153-53
...@@ -2,26 +2,31 @@ const db = getDb("cache.sqlite");...@@ -2,26 +2,31 @@ const db = getDb("cache.sqlite");
2db.table( 2db.table(
3 "media_files", 3 "media_files",
4 /* SQL */ ` 4 /* SQL */ `
5 CREATE TABLE IF NOT EXISTS media_files ( 5 create table media_files (
6 id INTEGER PRIMARY KEY AUTOINCREMENT, 6 id integer primary key autoincrement,
7 parent_id INTEGER, 7 parent_id integer,
8 path TEXT UNIQUE, 8 path text unique,
9 kind INTEGER NOT NULL, 9 kind integer not null,
10 timestamp INTEGER NOT NULL, 10 timestamp integer not null,
11 timestamp_updated INTEGER NOT NULL DEFAULT NOW, 11 timestamp_updated integer not null default current_timestamp,
12 hash TEXT NOT NULL, 12 hash text not null,
13 size INTEGER NOT NULL, 13 size integer not null,
14 duration INTEGER NOT NULL DEFAULT 0, 14 duration integer not null default 0,
15 dimensions TEXT NOT NULL DEFAULT "", 15 dimensions text not null default "",
16 contents TEXT NOT NULL, 16 contents text not null,
17 dirsort TEXT, 17 dirsort text,
18 processed INTEGER NOT NULL, 18 processed integer not null,
19 FOREIGN KEY (parent_id) REFERENCES media_files(id) 19 processors text not null default "",
20 foreign key (parent_id) references media_files(id)
20 ); 21 );
21 -- Index for quickly looking up files by path 22 -- index for quickly looking up files by path
22 CREATE INDEX IF NOT EXISTS media_files_path ON media_files (path); 23 create index media_files_path on media_files (path);
23 -- Index for finding directories that need to be processed 24 -- index for quickly looking up children
24 CREATE INDEX IF NOT EXISTS media_files_directory_processed ON media_files (kind, processed); 25 create index media_files_parent_id on media_files (parent_id);
26 -- index for quickly looking up recursive file children
27 create index media_files_file_children on media_files (kind, path);
28 -- index for finding directories that need to be processed
29 create index media_files_directory_processed on media_files (kind, processed);
25`, 30`,
26); 31);
27 32
...@@ -31,7 +36,7 @@ export enum MediaFileKind {...@@ -31,7 +36,7 @@ export enum MediaFileKind {
31} 36}
32export class MediaFile { 37export class MediaFile {
33 id!: number; 38 id!: number;
34 parent_id!: number; 39 parent_id!: number | null;
35 /** 40 /**
36 * Has leading slash, does not have `/file` prefix. 41 * Has leading slash, does not have `/file` prefix.
37 * @example "/2025/waterfalls/waterfalls.mp3" 42 * @example "/2025/waterfalls/waterfalls.mp3"
...@@ -69,12 +74,13 @@ export class MediaFile {...@@ -69,12 +74,13 @@ export class MediaFile {
69 size!: number; 74 size!: number;
70 /** 75 /**
71 * 0 - not processed 76 * 0 - not processed
72 * 1 - processed 77 * non-zero - processed
73 * 78 *
74 * file: this is for compression 79 * file: a bit-field of the processors.
75 * directory: this is for re-indexing contents 80 * directory: this is for re-indexing contents
76 */ 81 */
77 processed!: number; 82 processed!: number;
83 processors!: string;
78 84
79 // -- instance ops -- 85 // -- instance ops --
80 get date() { 86 get date() {
...@@ -119,8 +125,33 @@ export class MediaFile {...@@ -119,8 +125,33 @@ export class MediaFile {
119 ASSERT(result.kind === MediaFileKind.directory); 125 ASSERT(result.kind === MediaFileKind.directory);
120 return result; 126 return result;
121 } 127 }
122 setCompressed(compressed: boolean) { 128 setProcessed(processed: number) {
123 MediaFile.markCompressed(this.id, compressed); 129 setProcessedQuery.run({ id: this.id, processed });
130 this.processed = processed;
131 }
132 setProcessors(processed: number, processors: string) {
133 setProcessorsQuery.run({ id: this.id, processed, processors });
134 this.processed = processed;
135 this.processors = processors;
136 }
137 setDuration(duration: number) {
138 setDurationQuery.run({ id: this.id, duration });
139 this.duration = duration;
140 }
141 setDimensions(dimensions: string) {
142 setDimensionsQuery.run({ id: this.id, dimensions });
143 this.dimensions = dimensions;
144 }
145 setContents(contents: string) {
146 setContentsQuery.run({ id: this.id, contents });
147 this.contents = contents;
148 }
149 getRecursiveFileChildren() {
150 if (this.kind !== MediaFileKind.directory) return [];
151 return getChildrenFilesRecursiveQuery.array(this.path + "/");
152 }
153 delete() {
154 deleteCascadeQuery.run({ id: this.id });
124 } 155 }
125 156
126 // -- static ops -- 157 // -- static ops --
...@@ -130,7 +161,7 @@ export class MediaFile {...@@ -130,7 +161,7 @@ export class MediaFile {
130 if (filePath === "/") { 161 if (filePath === "/") {
131 return Object.assign(new MediaFile(), { 162 return Object.assign(new MediaFile(), {
132 id: 0, 163 id: 0,
133 parent_id: 0, 164 parent_id: null,
134 path: "/", 165 path: "/",
135 kind: MediaFileKind.directory, 166 kind: MediaFileKind.directory,
136 timestamp: 0, 167 timestamp: 0,
...@@ -149,11 +180,15 @@ export class MediaFile {...@@ -149,11 +180,15 @@ export class MediaFile {
149 date, 180 date,
150 hash, 181 hash,
151 size, 182 size,
152 duration = 0, 183 duration,
153 dimensions = "", 184 dimensions,
154 content = "", 185 contents,
155 }: CreateFile) { 186 }: CreateFile) {
156 createFileQuery.get({ 187 ASSERT(
188 !filePath.includes("\\") && filePath.startsWith("/"),
189 `Invalid path: ${filePath}`,
190 );
191 return createFileQuery.getNonNull({
157 path: filePath, 192 path: filePath,
158 parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)), 193 parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)),
159 timestamp: date.getTime(), 194 timestamp: date.getTime(),
...@@ -162,37 +197,46 @@ export class MediaFile {...@@ -162,37 +197,46 @@ export class MediaFile {
162 size, 197 size,
163 duration, 198 duration,
164 dimensions, 199 dimensions,
165 contents: content, 200 contents,
166 }); 201 });
167 } 202 }
168 static getOrPutDirectoryId(filePath: string) { 203 static getOrPutDirectoryId(filePath: string) {
169 filePath = path.posix.normalize(filePath); 204 ASSERT(
170 const row = getDirectoryIdQuery.get(filePath) as { id: number }; 205 !filePath.includes("\\") && filePath.startsWith("/"),
206 `Invalid path: ${filePath}`,
207 );
208 filePath = path.normalize(filePath);
209 const row = getDirectoryIdQuery.get(filePath);
171 if (row) return row.id; 210 if (row) return row.id;
172 let current = filePath; 211 let current = filePath;
173 let parts = []; 212 let parts = [];
174 let parentId: null | number = 0; 213 let parentId: null | number = null;
175 if (filePath === "/") { 214 if (filePath === "/") {
176 return createDirectoryQuery.run(filePath, 0).lastInsertRowid as number; 215 return createDirectoryQuery.getNonNull({
216 path: filePath,
217 parentId,
218 }).id;
177 } 219 }
178 // walk down the path until we find a directory that exists 220 // walk up the path until we find a directory that exists
179 do { 221 do {
180 parts.unshift(path.basename(current)); 222 parts.unshift(path.basename(current));
181 current = path.dirname(current); 223 current = path.dirname(current);
182 parentId = (getDirectoryIdQuery.get(current) as { id: number })?.id; 224 parentId = getDirectoryIdQuery.get(current)?.id ?? null;
183 } while (parentId == undefined && current !== "/"); 225 } while (parentId == undefined && current !== "/");
184 if (parentId == undefined) { 226 if (parentId == undefined) {
185 parentId = createDirectoryQuery.run({ 227 parentId = createDirectoryQuery.getNonNull({
186 path: current, 228 path: current,
187 parentId: 0, 229 parentId,
188 }).lastInsertRowid as number; 230 }).id;
189 } 231 }
190 // walk back up the path, creating directories as needed 232 // walk back down the path, creating directories as needed
191 for (const part of parts) { 233 for (const part of parts) {
192 current = path.join(current, part); 234 current = path.join(current, part);
193 ASSERT(parentId != undefined); 235 ASSERT(parentId != undefined);
194 parentId = createDirectoryQuery.run({ path: current, parentId }) 236 parentId = createDirectoryQuery.getNonNull({
195 .lastInsertRowid as number; 237 path: current,
238 parentId,
239 }).id;
196 } 240 }
197 return parentId; 241 return parentId;
198 } 242 }
...@@ -213,8 +257,8 @@ export class MediaFile {...@@ -213,8 +257,8 @@ export class MediaFile {
213 size, 257 size,
214 }); 258 });
215 } 259 }
216 static markCompressed(id: number, compressed: boolean) { 260 static setProcessed(id: number, processed: number) {
217 markCompressedQuery.run({ id, processed: compressed ? 2 : 1 }); 261 setProcessedQuery.run({ id, processed });
218 } 262 }
219 static createOrUpdateDirectory(dirPath: string) { 263 static createOrUpdateDirectory(dirPath: string) {
220 const id = MediaFile.getOrPutDirectoryId(dirPath); 264 const id = MediaFile.getOrPutDirectoryId(dirPath);
...@@ -223,6 +267,7 @@ export class MediaFile {...@@ -223,6 +267,7 @@ export class MediaFile {
223 static getChildren(id: number) { 267 static getChildren(id: number) {
224 return getChildrenQuery.array(id); 268 return getChildrenQuery.array(id);
225 } 269 }
270 static db = db;
226} 271}
227 272
228// Create a `file` entry with a given path, date, file hash, size, and duration 273// Create a `file` entry with a given path, date, file hash, size, and duration
...@@ -233,9 +278,9 @@ interface CreateFile {...@@ -233,9 +278,9 @@ interface CreateFile {
233 date: Date; 278 date: Date;
234 hash: string; 279 hash: string;
235 size: number; 280 size: number;
236 duration?: number; 281 duration: number;
237 dimensions?: string; 282 dimensions: string;
238 content?: string; 283 contents: string;
239} 284}
240 285
241// Set the `processed` flag true and update the metadata for a directory 286// Set the `processed` flag true and update the metadata for a directory
...@@ -256,14 +301,18 @@ export interface DirConfig {...@@ -256,14 +301,18 @@ export interface DirConfig {
256// -- queries -- 301// -- queries --
257 302
258// Get a directory ID by path, creating it if it doesn't exist 303// Get a directory ID by path, creating it if it doesn't exist
259const createDirectoryQuery = db.prepare<[{ path: string; parentId: number }]>( 304const createDirectoryQuery = db.prepare<
305 [{ path: string; parentId: number | null }],
306 { id: number }
307>(
260 /* SQL */ ` 308 /* SQL */ `
261 insert into media_files ( 309 insert into media_files (
262 path, parent_id, kind, timestamp, hash, size, 310 path, parent_id, kind, timestamp, hash, size,
263 duration, dimensions, contents, dirsort, processed) 311 duration, dimensions, contents, dirsort, processed)
264 values ( 312 values (
265 $path, $parentId, ${MediaFileKind.directory}, 0, '', 0, 313 $path, $parentId, ${MediaFileKind.directory}, 0, '', 0,
266 0, '', '', '', 0); 314 0, '', '', '', 0)
315 returning id;
267`, 316`,
268); 317);
269const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ ` 318const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ `
...@@ -295,14 +344,43 @@ const createFileQuery = db.prepare<[{...@@ -295,14 +344,43 @@ const createFileQuery = db.prepare<[{
295 processed = case 344 processed = case
296 when media_files.hash != excluded.hash then 0 345 when media_files.hash != excluded.hash then 0
297 else media_files.processed 346 else media_files.processed
298 end; 347 end
299`); 348 returning *;
300const markCompressedQuery = db.prepare<[{ 349`).as(MediaFile);
350const setProcessedQuery = db.prepare<[{
301 id: number; 351 id: number;
302 processed: number; 352 processed: number;
303}]>(/* SQL */ ` 353}]>(/* SQL */ `
304 update media_files set processed = $processed where id = $id; 354 update media_files set processed = $processed where id = $id;
305`); 355`);
356const setProcessorsQuery = db.prepare<[{
357 id: number;
358 processed: number;
359 processors: string;
360}]>(/* SQL */ `
361 update media_files set
362 processed = $processed,
363 processors = $processors
364 where id = $id;
365`);
366const setDurationQuery = db.prepare<[{
367 id: number;
368 duration: number;
369}]>(/* SQL */ `
370 update media_files set duration = $duration where id = $id;
371`);
372const setDimensionsQuery = db.prepare<[{
373 id: number;
374 dimensions: string;
375}]>(/* SQL */ `
376 update media_files set dimensions = $dimensions where id = $id;
377`);
378const setContentsQuery = db.prepare<[{
379 id: number;
380 contents: string;
381}]>(/* SQL */ `
382 update media_files set contents = $contents where id = $id;
383`);
306const getByPathQuery = db.prepare<[string]>(/* SQL */ ` 384const getByPathQuery = db.prepare<[string]>(/* SQL */ `
307 select * from media_files where path = ?; 385 select * from media_files where path = ?;
308`).as(MediaFile); 386`).as(MediaFile);
...@@ -330,7 +408,29 @@ const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ `...@@ -330,7 +408,29 @@ const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ `
330const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ ` 408const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ `
331 select * from media_files where parent_id = ?; 409 select * from media_files where parent_id = ?;
332`).as(MediaFile); 410`).as(MediaFile);
411const getChildrenFilesRecursiveQuery = db.prepare<[dir: string]>(/* SQL */ `
412 select * from media_files
413 where path like ? || '%'
414 and kind = ${MediaFileKind.file}
415`).as(MediaFile);
416const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ `
417 with recursive items as (
418 select id, parent_id from media_files where id = $id
419 union all
420 select p.id, p.parent_id
421 from media_files p
422 join items c on p.id = c.parent_id
423 where p.parent_id is not null
424 and not exists (
425 select 1 from media_files child
426 where child.parent_id = p.id
427 and child.id <> c.id
428 )
429 )
430 delete from media_files
431 where id in (select id from items)
432`);
333 433
334import { getDb } from "#sitegen/sqlite"; 434import { getDb } from "#sitegen/sqlite";
335import * as path from "node:path"; 435import * as path from "node:path/posix";
336import { FilePermissions } from "./FilePermissions.ts"; 436import { FilePermissions } from "./FilePermissions.ts";
src/file-viewer/rules.ts+6-6
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1// -- file extension rules -- 1// -- file extension rules --
2 2
3/** Extensions that must have EXIF/etc data stripped */ 3/** Extensions that must have EXIF/etc data stripped */
4const extScrubExif = new Set([ 4export const extScrubExif = new Set([
5 ".jpg", 5 ".jpg",
6 ".jpeg", 6 ".jpeg",
7 ".png", 7 ".png",
...@@ -10,7 +10,7 @@ const extScrubExif = new Set([...@@ -10,7 +10,7 @@ const extScrubExif = new Set([
10 ".m4a", 10 ".m4a",
11]); 11]);
12/** Extensions that rendered syntax-highlighted code */ 12/** Extensions that rendered syntax-highlighted code */
13const extsCode = new Map<string, highlight.Language>(Object.entries({ 13export const extsCode = new Map<string, highlight.Language>(Object.entries({
14 ".json": "json", 14 ".json": "json",
15 ".toml": "toml", 15 ".toml": "toml",
16 ".ts": "ts", 16 ".ts": "ts",
...@@ -36,7 +36,7 @@ const extsCode = new Map<string, highlight.Language>(Object.entries({...@@ -36,7 +36,7 @@ const extsCode = new Map<string, highlight.Language>(Object.entries({
36 ".diff": "diff", 36 ".diff": "diff",
37})); 37}));
38/** These files show an audio embed. */ 38/** These files show an audio embed. */
39const extsAudio = new Set([ 39export const extsAudio = new Set([
40 ".mp3", 40 ".mp3",
41 ".flac", 41 ".flac",
42 ".wav", 42 ".wav",
...@@ -44,7 +44,7 @@ const extsAudio = new Set([...@@ -44,7 +44,7 @@ const extsAudio = new Set([
44 ".m4a", 44 ".m4a",
45]); 45]);
46/** These files show a video embed. */ 46/** These files show a video embed. */
47const extsVideo = new Set([ 47export const extsVideo = new Set([
48 ".mp4", 48 ".mp4",
49 ".mkv", 49 ".mkv",
50 ".webm", 50 ".webm",
...@@ -52,7 +52,7 @@ const extsVideo = new Set([...@@ -52,7 +52,7 @@ const extsVideo = new Set([
52 ".mov", 52 ".mov",
53]); 53]);
54/** These files show an image embed */ 54/** These files show an image embed */
55const extsImage = new Set([ 55export const extsImage = new Set([
56 ".jpg", 56 ".jpg",
57 ".jpeg", 57 ".jpeg",
58 ".png", 58 ".png",
...@@ -85,7 +85,7 @@ export const extsArchive = new Set([...@@ -85,7 +85,7 @@ export const extsArchive = new Set([
85 * Formats which are already compression formats, meaning a pass 85 * Formats which are already compression formats, meaning a pass
86 * through zstd would offer little to negative benefits 86 * through zstd would offer little to negative benefits
87 */ 87 */
88export const extsHaveCompression = new Set([ 88export const extsPreCompressed = new Set([
89 ...extsAudio, 89 ...extsAudio,
90 ...extsVideo, 90 ...extsVideo,
91 ...extsImage, 91 ...extsImage,
src/pages/resume.css created+47
...@@ -0,0 +1,47 @@
1body,html {
2 overflow: hidden;
3}
4h1 {
5 color: #f09;
6 margin-bottom: 0;
7}
8.job {
9 padding: 18px;
10 margin: 1em -18px;
11 border: 1px solid black;
12}
13.job *, footer * {
14 margin: 0;
15 padding: 0;
16}
17.job ul {
18 margin-left: 1em;
19}
20.job li {
21 line-height: 1.5em;
22}
23.job header, footer {
24 display: grid;
25 grid-template-columns: auto max-content;
26 grid-template-rows: 1fr 1fr;
27}
28footer {
29 margin-top: 1.5em;
30}
31footer h2 {
32 font-size: 1em;
33 margin-bottom: 0.5em;
34}
35
36.job header > em, footer > em {
37 margin-top: 2px;
38 font-size: 1.25em;
39}
40
41header h2, header em, footer h2, footer em {
42 display: inline-block;
43}
44 header em, footer em {
45 margin-left: 16px!important;
46 text-align: right;
47}
src/pages/resume.marko created+50
...@@ -0,0 +1,50 @@
1import "./resume.css";
2
3export const meta = { title: 'clover\'s resume' };
4
5<main>
6<h1>clover's resume</h1>
7<div>last updated: 2025</>
8
9<article.job>
10 <header>
11 <h2>web/backend engineer</h2>
12 <em>2025-now</em>
13 </>
14 <ul>
15 <i>(more details added as time goes on...)</i>
16 </ul>
17</>
18
19<article.job>
20 <header>
21 <h2>runtime/systems engineer</h2>
22 <em>2023-2025</em>
23 <p>developer tools company</p>
24 </>
25 <ul>
26 <li>hardcore engineering, elegant solutions</>
27 <li>platform compatibility & stability</>
28 <li>debugging and profiling across platforms</>
29 </ul>
30</>
31
32<article.job>
33 <header>
34 <h2>technician</h2>
35 <em>2023; part time</em>
36 <p>automotive maintainance company</p>
37 </>
38 <ul>
39 <li>pressed buttons on a computer</>
40 </ul>
41</>
42
43<footer>
44 <h2>eduation</h2> <em>2004-now</em>
45 <p>
46 my life on earth has taught me more than i expected. i <br/>
47 continue to learn new things daily, as if it was magic.
48 </p>
49</footer>
50</main>