| author | |
| committer | |
| log | 4964d828660c6603c009e55b41e06c56b49c4bcf |
| tree | 45897fc85ad9ff81697d76a296abcdd0514b5c09 |
| parent | 7f762445733ae31f3a199a06f643dcdd47db2e45 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Serve stats and activity beside the existing scan endpoint.
Assisted-by: gpt-6.1-sol3 files changed, 153 insertions(+), 153 deletions(-)
src/file-viewer/indexer/status.ts created+148| ... | ... | @@ -0,0 +1,148 @@ |
| 1 | import { MediaFile } from "#src/file-viewer/models/MediaFile.ts"; | |
| 2 | import * as mime from "@clo/lib/mime"; | |
| 3 | import * as progress from "@clo/lib/progress"; | |
| 4 | import * as registry from "./registry.ts"; | |
| 5 | import type { Service } from "./service.ts"; | |
| 6 | ||
| 7 | type FileRow = { | |
| 8 | id: number; | |
| 9 | path: string; | |
| 10 | kind: number; | |
| 11 | size: number; | |
| 12 | timestamp_updated: number | string; | |
| 13 | duration: number; | |
| 14 | dimensions: string; | |
| 15 | }; | |
| 16 | type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null }; | |
| 17 | ||
| 18 | const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000; | |
| 19 | const extension = (path: string) => { | |
| 20 | const name = path.split("/").at(-1) ?? ""; | |
| 21 | const dot = name.lastIndexOf("."); | |
| 22 | return (dot > 0 ? name.slice(dot) : name).toLowerCase(); | |
| 23 | }; | |
| 24 | ||
| 25 | export function stats(service: Service | null) { | |
| 26 | const db = MediaFile.db.node; | |
| 27 | const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files") | |
| 28 | .all() as FileRow[]; | |
| 29 | const files = all.filter((file) => file.kind === 1); | |
| 30 | const extensions = new Map(files.map((file) => [file.id, extension(file.path)])); | |
| 31 | const recorded = db.prepare(` | |
| 32 | select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error | |
| 33 | from file_processors fp join processors p on p.id=fp.processor | |
| 34 | `).all() as StateRow[]; | |
| 35 | const byFile = new Map<number, Map<string, StateRow>>(); | |
| 36 | for (const row of recorded) { | |
| 37 | const states = byFile.get(row.file) ?? new Map<string, StateRow>(); | |
| 38 | states.set(row.name, row); | |
| 39 | byFile.set(row.file, states); | |
| 40 | } | |
| 41 | const folders = new Map<string, { name: string; files: number; bytes: number }>(); | |
| 42 | const types = new Map<string, { ext: string; files: number; bytes: number }>(); | |
| 43 | for (const file of files) { | |
| 44 | const top = file.path.split("/")[1] ?? ""; | |
| 45 | const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 }; | |
| 46 | folder.files++; | |
| 47 | folder.bytes += file.size; | |
| 48 | folders.set(top, folder); | |
| 49 | const ext = extensions.get(file.id) ?? ""; | |
| 50 | const type = types.get(ext) ?? { ext, files: 0, bytes: 0 }; | |
| 51 | type.files++; | |
| 52 | type.bytes += file.size; | |
| 53 | types.set(ext, type); | |
| 54 | } | |
| 55 | const processors = registry.processors.map((p) => { | |
| 56 | let applicable = 0; | |
| 57 | let done = 0; | |
| 58 | let failed = 0; | |
| 59 | let stale = 0; | |
| 60 | for (const file of files) { | |
| 61 | const ext = extensions.get(file.id) ?? ""; | |
| 62 | if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue; | |
| 63 | applicable++; | |
| 64 | const state = byFile.get(file.id)?.get(p.name); | |
| 65 | if (state?.version !== p.version) { | |
| 66 | if (state) stale++; | |
| 67 | } else if (state.status === 1) done++; | |
| 68 | else if (state.status === 2) failed++; | |
| 69 | } | |
| 70 | return { | |
| 71 | name: p.name, | |
| 72 | title: p.title, | |
| 73 | version: p.version, | |
| 74 | runnable: registry.canExecute(p), | |
| 75 | applicable, | |
| 76 | done, | |
| 77 | failed, | |
| 78 | stale, | |
| 79 | outputs: null, | |
| 80 | bytes: null, | |
| 81 | seconds: null, | |
| 82 | }; | |
| 83 | }); | |
| 84 | const failures = files.flatMap((file) => { | |
| 85 | const states = [...(byFile.get(file.id)?.values() ?? [])]; | |
| 86 | const errors = states.flatMap((state) => | |
| 87 | state.status === 2 && state.error !== null | |
| 88 | ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }] | |
| 89 | : [] | |
| 90 | ); | |
| 91 | return errors.length | |
| 92 | ? [{ | |
| 93 | path: file.path, | |
| 94 | size: file.size, | |
| 95 | mime: mime.for(file.path), | |
| 96 | made: states.filter((state) => state.status === 1).map((state) => state.name), | |
| 97 | errors, | |
| 98 | }] | |
| 99 | : []; | |
| 100 | }); | |
| 101 | const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated)) | |
| 102 | .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) })); | |
| 103 | const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12) | |
| 104 | .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions })); | |
| 105 | const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as { | |
| 106 | files: number; | |
| 107 | bytes: number; | |
| 108 | }; | |
| 109 | return { | |
| 110 | indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null }, | |
| 111 | lastSweep: service?.lastSweep ?? null, | |
| 112 | files: files.length, | |
| 113 | directories: all.length - files.length, | |
| 114 | bytes: files.reduce((sum, file) => sum + file.size, 0), | |
| 115 | folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes), | |
| 116 | types: [...types.values()].sort((a, b) => b.bytes - a.bytes), | |
| 117 | processors, | |
| 118 | derived, | |
| 119 | failures, | |
| 120 | recent, | |
| 121 | largest, | |
| 122 | }; | |
| 123 | } | |
| 124 | ||
| 125 | const firstSeen = new WeakMap<progress.ReadOnlyNode, number>(); | |
| 126 | ||
| 127 | export function activity(root: progress.Root) { | |
| 128 | const convert = (node: progress.ReadOnlyNode): object => { | |
| 129 | if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000); | |
| 130 | const match = /^(\/.*?) - (.*)$/.exec(node.text); | |
| 131 | const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children) | |
| 132 | .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert); | |
| 133 | return { | |
| 134 | path: match?.[1] ?? null, | |
| 135 | text: match?.[2] ?? node.text, | |
| 136 | count: node.valueFormatter(node.value, node.total), | |
| 137 | progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null, | |
| 138 | eta: node.estimatedTime ? node.estimatedTime / 1000 : null, | |
| 139 | since: firstSeen.get(node), | |
| 140 | logs: node.logs.slice(-3).map((entry) => ({ | |
| 141 | level: entry.level ?? "info", | |
| 142 | text: entry.text.replace(/\x1b\[[\d;]*m/g, ""), | |
| 143 | })), | |
| 144 | children, | |
| 145 | }; | |
| 146 | }; | |
| 147 | return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert); | |
| 148 | } |
src/file-viewer/indexer/studio.ts deleted-148| ... | ... | @@ -1,148 +0,0 @@ |
| 1 | import { MediaFile } from "#src/file-viewer/models/MediaFile.ts"; | |
| 2 | import * as mime from "@clo/lib/mime"; | |
| 3 | import * as progress from "@clo/lib/progress"; | |
| 4 | import * as registry from "./registry.ts"; | |
| 5 | import type { Service } from "./service.ts"; | |
| 6 | ||
| 7 | type FileRow = { | |
| 8 | id: number; | |
| 9 | path: string; | |
| 10 | kind: number; | |
| 11 | size: number; | |
| 12 | timestamp_updated: number | string; | |
| 13 | duration: number; | |
| 14 | dimensions: string; | |
| 15 | }; | |
| 16 | type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null }; | |
| 17 | ||
| 18 | const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000; | |
| 19 | const extension = (path: string) => { | |
| 20 | const name = path.split("/").at(-1) ?? ""; | |
| 21 | const dot = name.lastIndexOf("."); | |
| 22 | return (dot > 0 ? name.slice(dot) : name).toLowerCase(); | |
| 23 | }; | |
| 24 | ||
| 25 | export function stats(service: Service | null) { | |
| 26 | const db = MediaFile.db.node; | |
| 27 | const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files") | |
| 28 | .all() as FileRow[]; | |
| 29 | const files = all.filter((file) => file.kind === 1); | |
| 30 | const extensions = new Map(files.map((file) => [file.id, extension(file.path)])); | |
| 31 | const recorded = db.prepare(` | |
| 32 | select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error | |
| 33 | from file_processors fp join processors p on p.id=fp.processor | |
| 34 | `).all() as StateRow[]; | |
| 35 | const byFile = new Map<number, Map<string, StateRow>>(); | |
| 36 | for (const row of recorded) { | |
| 37 | const states = byFile.get(row.file) ?? new Map<string, StateRow>(); | |
| 38 | states.set(row.name, row); | |
| 39 | byFile.set(row.file, states); | |
| 40 | } | |
| 41 | const folders = new Map<string, { name: string; files: number; bytes: number }>(); | |
| 42 | const types = new Map<string, { ext: string; files: number; bytes: number }>(); | |
| 43 | for (const file of files) { | |
| 44 | const top = file.path.split("/")[1] ?? ""; | |
| 45 | const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 }; | |
| 46 | folder.files++; | |
| 47 | folder.bytes += file.size; | |
| 48 | folders.set(top, folder); | |
| 49 | const ext = extensions.get(file.id) ?? ""; | |
| 50 | const type = types.get(ext) ?? { ext, files: 0, bytes: 0 }; | |
| 51 | type.files++; | |
| 52 | type.bytes += file.size; | |
| 53 | types.set(ext, type); | |
| 54 | } | |
| 55 | const processors = registry.processors.map((p) => { | |
| 56 | let applicable = 0; | |
| 57 | let done = 0; | |
| 58 | let failed = 0; | |
| 59 | let stale = 0; | |
| 60 | for (const file of files) { | |
| 61 | const ext = extensions.get(file.id) ?? ""; | |
| 62 | if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue; | |
| 63 | applicable++; | |
| 64 | const state = byFile.get(file.id)?.get(p.name); | |
| 65 | if (state?.version !== p.version) { | |
| 66 | if (state) stale++; | |
| 67 | } else if (state.status === 1) done++; | |
| 68 | else if (state.status === 2) failed++; | |
| 69 | } | |
| 70 | return { | |
| 71 | name: p.name, | |
| 72 | title: p.title, | |
| 73 | version: p.version, | |
| 74 | runnable: registry.canExecute(p), | |
| 75 | applicable, | |
| 76 | done, | |
| 77 | failed, | |
| 78 | stale, | |
| 79 | outputs: null, | |
| 80 | bytes: null, | |
| 81 | seconds: null, | |
| 82 | }; | |
| 83 | }); | |
| 84 | const failures = files.flatMap((file) => { | |
| 85 | const states = [...(byFile.get(file.id)?.values() ?? [])]; | |
| 86 | const errors = states.flatMap((state) => | |
| 87 | state.status === 2 && state.error !== null | |
| 88 | ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }] | |
| 89 | : [] | |
| 90 | ); | |
| 91 | return errors.length | |
| 92 | ? [{ | |
| 93 | path: file.path, | |
| 94 | size: file.size, | |
| 95 | mime: mime.for(file.path), | |
| 96 | made: states.filter((state) => state.status === 1).map((state) => state.name), | |
| 97 | errors, | |
| 98 | }] | |
| 99 | : []; | |
| 100 | }); | |
| 101 | const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated)) | |
| 102 | .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) })); | |
| 103 | const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12) | |
| 104 | .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions })); | |
| 105 | const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as { | |
| 106 | files: number; | |
| 107 | bytes: number; | |
| 108 | }; | |
| 109 | return { | |
| 110 | indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null }, | |
| 111 | lastSweep: service?.lastSweep ?? null, | |
| 112 | files: files.length, | |
| 113 | directories: all.length - files.length, | |
| 114 | bytes: files.reduce((sum, file) => sum + file.size, 0), | |
| 115 | folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes), | |
| 116 | types: [...types.values()].sort((a, b) => b.bytes - a.bytes), | |
| 117 | processors, | |
| 118 | derived, | |
| 119 | failures, | |
| 120 | recent, | |
| 121 | largest, | |
| 122 | }; | |
| 123 | } | |
| 124 | ||
| 125 | const firstSeen = new WeakMap<progress.ReadOnlyNode, number>(); | |
| 126 | ||
| 127 | export function activity(root: progress.Root) { | |
| 128 | const convert = (node: progress.ReadOnlyNode): object => { | |
| 129 | if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000); | |
| 130 | const match = /^(\/.*?) - (.*)$/.exec(node.text); | |
| 131 | const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children) | |
| 132 | .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert); | |
| 133 | return { | |
| 134 | path: match?.[1] ?? null, | |
| 135 | text: match?.[2] ?? node.text, | |
| 136 | count: node.valueFormatter(node.value, node.total), | |
| 137 | progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null, | |
| 138 | eta: node.estimatedTime ? node.estimatedTime / 1000 : null, | |
| 139 | since: firstSeen.get(node), | |
| 140 | logs: node.logs.slice(-3).map((entry) => ({ | |
| 141 | level: entry.level ?? "info", | |
| 142 | text: entry.text.replace(/\x1b\[[\d;]*m/g, ""), | |
| 143 | })), | |
| 144 | children, | |
| 145 | }; | |
| 146 | }; | |
| 147 | return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert); | |
| 148 | } |
src/source-of-truth.ts+5-5| ... | ... | @@ -189,14 +189,14 @@ app.get("/progress", (c) => { |
| 189 | 189 | return c.body(progress.encodeByteStream(indexer.root) as ReadableStream); |
| 190 | 190 | }); |
| 191 | 191 | |
| 192 | app.get("/studio/stats", (c) => { | |
| 192 | app.get("/stats", (c) => { | |
| 193 | 193 | if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401); |
| 194 | return c.json(studio.stats(service)); | |
| 194 | return c.json(status.stats(service)); | |
| 195 | 195 | }); |
| 196 | 196 | |
| 197 | app.get("/studio/activity", (c) => { | |
| 197 | app.get("/activity", (c) => { | |
| 198 | 198 | if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401); |
| 199 | return c.json(studio.activity(indexer.root)); | |
| 199 | return c.json(status.activity(indexer.root)); | |
| 200 | 200 | }); |
| 201 | 201 | |
| 202 | 202 | // download a consistent snapshot of cache.sqlite. web nodes call this with |
| ... | ... | @@ -287,7 +287,7 @@ import * as fs from "#sitegen/fs"; |
| 287 | 287 | import { Path } from "#sitegen/path"; |
| 288 | 288 | import { hashFile } from "#src/file-viewer/indexer/scan.ts"; |
| 289 | 289 | import * as indexer from "#src/file-viewer/indexer/service.ts"; |
| 290 | import * as studio from "#src/file-viewer/indexer/studio.ts"; | |
| 290 | import * as status from "#src/file-viewer/indexer/status.ts"; | |
| 291 | 291 | import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts"; |
| 292 | 292 | import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts"; |
| 293 | 293 | import { ASSERT, UNWRAP } from "@clo/lib/assert"; |