authoravatarclover caruso<git@paperclover.net>2026-10-21 02:08:14-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:50:04-07:00
log4964d828660c6603c009e55b41e06c56b49c4bcf
tree45897fc85ad9ff81697d76a296abcdd0514b5c09
parent7f762445733ae31f3a199a06f643dcdd47db2e45
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Expose indexer status without a machine-name prefix

Serve stats and activity beside the existing scan endpoint. Assisted-by: gpt-6.1-sol

3 files changed, 153 insertions(+), 153 deletions(-)

src/file-viewer/indexer/status.ts created+148
...@@ -0,0 +1,148 @@
1import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
2import * as mime from "@clo/lib/mime";
3import * as progress from "@clo/lib/progress";
4import * as registry from "./registry.ts";
5import type { Service } from "./service.ts";
6
7type FileRow = {
8 id: number;
9 path: string;
10 kind: number;
11 size: number;
12 timestamp_updated: number | string;
13 duration: number;
14 dimensions: string;
15};
16type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null };
17
18const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000;
19const extension = (path: string) => {
20 const name = path.split("/").at(-1) ?? "";
21 const dot = name.lastIndexOf(".");
22 return (dot > 0 ? name.slice(dot) : name).toLowerCase();
23};
24
25export function stats(service: Service | null) {
26 const db = MediaFile.db.node;
27 const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files")
28 .all() as FileRow[];
29 const files = all.filter((file) => file.kind === 1);
30 const extensions = new Map(files.map((file) => [file.id, extension(file.path)]));
31 const recorded = db.prepare(`
32 select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error
33 from file_processors fp join processors p on p.id=fp.processor
34 `).all() as StateRow[];
35 const byFile = new Map<number, Map<string, StateRow>>();
36 for (const row of recorded) {
37 const states = byFile.get(row.file) ?? new Map<string, StateRow>();
38 states.set(row.name, row);
39 byFile.set(row.file, states);
40 }
41 const folders = new Map<string, { name: string; files: number; bytes: number }>();
42 const types = new Map<string, { ext: string; files: number; bytes: number }>();
43 for (const file of files) {
44 const top = file.path.split("/")[1] ?? "";
45 const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 };
46 folder.files++;
47 folder.bytes += file.size;
48 folders.set(top, folder);
49 const ext = extensions.get(file.id) ?? "";
50 const type = types.get(ext) ?? { ext, files: 0, bytes: 0 };
51 type.files++;
52 type.bytes += file.size;
53 types.set(ext, type);
54 }
55 const processors = registry.processors.map((p) => {
56 let applicable = 0;
57 let done = 0;
58 let failed = 0;
59 let stale = 0;
60 for (const file of files) {
61 const ext = extensions.get(file.id) ?? "";
62 if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue;
63 applicable++;
64 const state = byFile.get(file.id)?.get(p.name);
65 if (state?.version !== p.version) {
66 if (state) stale++;
67 } else if (state.status === 1) done++;
68 else if (state.status === 2) failed++;
69 }
70 return {
71 name: p.name,
72 title: p.title,
73 version: p.version,
74 runnable: registry.canExecute(p),
75 applicable,
76 done,
77 failed,
78 stale,
79 outputs: null,
80 bytes: null,
81 seconds: null,
82 };
83 });
84 const failures = files.flatMap((file) => {
85 const states = [...(byFile.get(file.id)?.values() ?? [])];
86 const errors = states.flatMap((state) =>
87 state.status === 2 && state.error !== null
88 ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }]
89 : []
90 );
91 return errors.length
92 ? [{
93 path: file.path,
94 size: file.size,
95 mime: mime.for(file.path),
96 made: states.filter((state) => state.status === 1).map((state) => state.name),
97 errors,
98 }]
99 : [];
100 });
101 const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated))
102 .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) }));
103 const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12)
104 .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions }));
105 const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as {
106 files: number;
107 bytes: number;
108 };
109 return {
110 indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null },
111 lastSweep: service?.lastSweep ?? null,
112 files: files.length,
113 directories: all.length - files.length,
114 bytes: files.reduce((sum, file) => sum + file.size, 0),
115 folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes),
116 types: [...types.values()].sort((a, b) => b.bytes - a.bytes),
117 processors,
118 derived,
119 failures,
120 recent,
121 largest,
122 };
123}
124
125const firstSeen = new WeakMap<progress.ReadOnlyNode, number>();
126
127export function activity(root: progress.Root) {
128 const convert = (node: progress.ReadOnlyNode): object => {
129 if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000);
130 const match = /^(\/.*?) - (.*)$/.exec(node.text);
131 const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children)
132 .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert);
133 return {
134 path: match?.[1] ?? null,
135 text: match?.[2] ?? node.text,
136 count: node.valueFormatter(node.value, node.total),
137 progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null,
138 eta: node.estimatedTime ? node.estimatedTime / 1000 : null,
139 since: firstSeen.get(node),
140 logs: node.logs.slice(-3).map((entry) => ({
141 level: entry.level ?? "info",
142 text: entry.text.replace(/\x1b\[[\d;]*m/g, ""),
143 })),
144 children,
145 };
146 };
147 return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert);
148}
src/file-viewer/indexer/studio.ts deleted-148
...@@ -1,148 +0,0 @@
1import { MediaFile } from "#src/file-viewer/models/MediaFile.ts";
2import * as mime from "@clo/lib/mime";
3import * as progress from "@clo/lib/progress";
4import * as registry from "./registry.ts";
5import type { Service } from "./service.ts";
6
7type FileRow = {
8 id: number;
9 path: string;
10 kind: number;
11 size: number;
12 timestamp_updated: number | string;
13 duration: number;
14 dimensions: string;
15};
16type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null };
17
18const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000;
19const extension = (path: string) => {
20 const name = path.split("/").at(-1) ?? "";
21 const dot = name.lastIndexOf(".");
22 return (dot > 0 ? name.slice(dot) : name).toLowerCase();
23};
24
25export function stats(service: Service | null) {
26 const db = MediaFile.db.node;
27 const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files")
28 .all() as FileRow[];
29 const files = all.filter((file) => file.kind === 1);
30 const extensions = new Map(files.map((file) => [file.id, extension(file.path)]));
31 const recorded = db.prepare(`
32 select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error
33 from file_processors fp join processors p on p.id=fp.processor
34 `).all() as StateRow[];
35 const byFile = new Map<number, Map<string, StateRow>>();
36 for (const row of recorded) {
37 const states = byFile.get(row.file) ?? new Map<string, StateRow>();
38 states.set(row.name, row);
39 byFile.set(row.file, states);
40 }
41 const folders = new Map<string, { name: string; files: number; bytes: number }>();
42 const types = new Map<string, { ext: string; files: number; bytes: number }>();
43 for (const file of files) {
44 const top = file.path.split("/")[1] ?? "";
45 const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 };
46 folder.files++;
47 folder.bytes += file.size;
48 folders.set(top, folder);
49 const ext = extensions.get(file.id) ?? "";
50 const type = types.get(ext) ?? { ext, files: 0, bytes: 0 };
51 type.files++;
52 type.bytes += file.size;
53 types.set(ext, type);
54 }
55 const processors = registry.processors.map((p) => {
56 let applicable = 0;
57 let done = 0;
58 let failed = 0;
59 let stale = 0;
60 for (const file of files) {
61 const ext = extensions.get(file.id) ?? "";
62 if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue;
63 applicable++;
64 const state = byFile.get(file.id)?.get(p.name);
65 if (state?.version !== p.version) {
66 if (state) stale++;
67 } else if (state.status === 1) done++;
68 else if (state.status === 2) failed++;
69 }
70 return {
71 name: p.name,
72 title: p.title,
73 version: p.version,
74 runnable: registry.canExecute(p),
75 applicable,
76 done,
77 failed,
78 stale,
79 outputs: null,
80 bytes: null,
81 seconds: null,
82 };
83 });
84 const failures = files.flatMap((file) => {
85 const states = [...(byFile.get(file.id)?.values() ?? [])];
86 const errors = states.flatMap((state) =>
87 state.status === 2 && state.error !== null
88 ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }]
89 : []
90 );
91 return errors.length
92 ? [{
93 path: file.path,
94 size: file.size,
95 mime: mime.for(file.path),
96 made: states.filter((state) => state.status === 1).map((state) => state.name),
97 errors,
98 }]
99 : [];
100 });
101 const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated))
102 .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) }));
103 const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12)
104 .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions }));
105 const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as {
106 files: number;
107 bytes: number;
108 };
109 return {
110 indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null },
111 lastSweep: service?.lastSweep ?? null,
112 files: files.length,
113 directories: all.length - files.length,
114 bytes: files.reduce((sum, file) => sum + file.size, 0),
115 folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes),
116 types: [...types.values()].sort((a, b) => b.bytes - a.bytes),
117 processors,
118 derived,
119 failures,
120 recent,
121 largest,
122 };
123}
124
125const firstSeen = new WeakMap<progress.ReadOnlyNode, number>();
126
127export function activity(root: progress.Root) {
128 const convert = (node: progress.ReadOnlyNode): object => {
129 if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000);
130 const match = /^(\/.*?) - (.*)$/.exec(node.text);
131 const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children)
132 .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert);
133 return {
134 path: match?.[1] ?? null,
135 text: match?.[2] ?? node.text,
136 count: node.valueFormatter(node.value, node.total),
137 progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null,
138 eta: node.estimatedTime ? node.estimatedTime / 1000 : null,
139 since: firstSeen.get(node),
140 logs: node.logs.slice(-3).map((entry) => ({
141 level: entry.level ?? "info",
142 text: entry.text.replace(/\x1b\[[\d;]*m/g, ""),
143 })),
144 children,
145 };
146 };
147 return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert);
148}
src/source-of-truth.ts+5-5
...@@ -189,14 +189,14 @@ app.get("/progress", (c) => {...@@ -189,14 +189,14 @@ app.get("/progress", (c) => {
189 return c.body(progress.encodeByteStream(indexer.root) as ReadableStream);189 return c.body(progress.encodeByteStream(indexer.root) as ReadableStream);
190});190});
191191
192app.get("/studio/stats", (c) => {192app.get("/stats", (c) => {
193 if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401);193 if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401);
194 return c.json(studio.stats(service));194 return c.json(status.stats(service));
195});195});
196196
197app.get("/studio/activity", (c) => {197app.get("/activity", (c) => {
198 if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401);198 if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401);
199 return c.json(studio.activity(indexer.root));199 return c.json(status.activity(indexer.root));
200});200});
201201
202// download a consistent snapshot of cache.sqlite. web nodes call this with202// download a consistent snapshot of cache.sqlite. web nodes call this with
...@@ -287,7 +287,7 @@ import * as fs from "#sitegen/fs";...@@ -287,7 +287,7 @@ import * as fs from "#sitegen/fs";
287import { Path } from "#sitegen/path";287import { Path } from "#sitegen/path";
288import { hashFile } from "#src/file-viewer/indexer/scan.ts";288import { hashFile } from "#src/file-viewer/indexer/scan.ts";
289import * as indexer from "#src/file-viewer/indexer/service.ts";289import * as indexer from "#src/file-viewer/indexer/service.ts";
290import * as studio from "#src/file-viewer/indexer/studio.ts";290import * as status from "#src/file-viewer/indexer/status.ts";
291import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";291import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts";
292import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";292import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts";
293import { ASSERT, UNWRAP } from "@clo/lib/assert";293import { ASSERT, UNWRAP } from "@clo/lib/assert";