diff --git a/src/file-viewer/indexer/status.ts b/src/file-viewer/indexer/status.ts new file mode 100644 index 0000000000000000000000000000000000000000..4562912bcde9c0c4b411dfba9ff05217e301d68f --- /dev/null +++ b/src/file-viewer/indexer/status.ts @@ -0,0 +1,148 @@ +import { MediaFile } from "#src/file-viewer/models/MediaFile.ts"; +import * as mime from "@clo/lib/mime"; +import * as progress from "@clo/lib/progress"; +import * as registry from "./registry.ts"; +import type { Service } from "./service.ts"; + +type FileRow = { + id: number; + path: string; + kind: number; + size: number; + timestamp_updated: number | string; + duration: number; + dimensions: string; +}; +type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null }; + +const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000; +const extension = (path: string) => { + const name = path.split("/").at(-1) ?? ""; + const dot = name.lastIndexOf("."); + return (dot > 0 ? name.slice(dot) : name).toLowerCase(); +}; + +export function stats(service: Service | null) { + const db = MediaFile.db.node; + const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files") + .all() as FileRow[]; + const files = all.filter((file) => file.kind === 1); + const extensions = new Map(files.map((file) => [file.id, extension(file.path)])); + const recorded = db.prepare(` + select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error + from file_processors fp join processors p on p.id=fp.processor + `).all() as StateRow[]; + const byFile = new Map>(); + for (const row of recorded) { + const states = byFile.get(row.file) ?? new Map(); + states.set(row.name, row); + byFile.set(row.file, states); + } + const folders = new Map(); + const types = new Map(); + for (const file of files) { + const top = file.path.split("/")[1] ?? ""; + const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 }; + folder.files++; + folder.bytes += file.size; + folders.set(top, folder); + const ext = extensions.get(file.id) ?? ""; + const type = types.get(ext) ?? { ext, files: 0, bytes: 0 }; + type.files++; + type.bytes += file.size; + types.set(ext, type); + } + const processors = registry.processors.map((p) => { + let applicable = 0; + let done = 0; + let failed = 0; + let stale = 0; + for (const file of files) { + const ext = extensions.get(file.id) ?? ""; + if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue; + applicable++; + const state = byFile.get(file.id)?.get(p.name); + if (state?.version !== p.version) { + if (state) stale++; + } else if (state.status === 1) done++; + else if (state.status === 2) failed++; + } + return { + name: p.name, + title: p.title, + version: p.version, + runnable: registry.canExecute(p), + applicable, + done, + failed, + stale, + outputs: null, + bytes: null, + seconds: null, + }; + }); + const failures = files.flatMap((file) => { + const states = [...(byFile.get(file.id)?.values() ?? [])]; + const errors = states.flatMap((state) => + state.status === 2 && state.error !== null + ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }] + : [] + ); + return errors.length + ? [{ + path: file.path, + size: file.size, + mime: mime.for(file.path), + made: states.filter((state) => state.status === 1).map((state) => state.name), + errors, + }] + : []; + }); + const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated)) + .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) })); + const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12) + .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions })); + const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as { + files: number; + bytes: number; + }; + return { + indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null }, + lastSweep: service?.lastSweep ?? null, + files: files.length, + directories: all.length - files.length, + bytes: files.reduce((sum, file) => sum + file.size, 0), + folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes), + types: [...types.values()].sort((a, b) => b.bytes - a.bytes), + processors, + derived, + failures, + recent, + largest, + }; +} + +const firstSeen = new WeakMap(); + +export function activity(root: progress.Root) { + const convert = (node: progress.ReadOnlyNode): object => { + if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000); + const match = /^(\/.*?) - (.*)$/.exec(node.text); + const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children) + .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert); + return { + path: match?.[1] ?? null, + text: match?.[2] ?? node.text, + count: node.valueFormatter(node.value, node.total), + progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null, + eta: node.estimatedTime ? node.estimatedTime / 1000 : null, + since: firstSeen.get(node), + logs: node.logs.slice(-3).map((entry) => ({ + level: entry.level ?? "info", + text: entry.text.replace(/\x1b\[[\d;]*m/g, ""), + })), + children, + }; + }; + return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert); +} diff --git a/src/file-viewer/indexer/studio.ts b/src/file-viewer/indexer/studio.ts deleted file mode 100644 index 4562912bcde9c0c4b411dfba9ff05217e301d68f..0000000000000000000000000000000000000000 --- a/src/file-viewer/indexer/studio.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { MediaFile } from "#src/file-viewer/models/MediaFile.ts"; -import * as mime from "@clo/lib/mime"; -import * as progress from "@clo/lib/progress"; -import * as registry from "./registry.ts"; -import type { Service } from "./service.ts"; - -type FileRow = { - id: number; - path: string; - kind: number; - size: number; - timestamp_updated: number | string; - duration: number; - dimensions: string; -}; -type StateRow = { file: number; name: string; version: number; status: number; updated: number; error: string | null }; - -const seconds = (value: number | string) => (typeof value === "string" ? Date.parse(value) : value) / 1000; -const extension = (path: string) => { - const name = path.split("/").at(-1) ?? ""; - const dot = name.lastIndexOf("."); - return (dot > 0 ? name.slice(dot) : name).toLowerCase(); -}; - -export function stats(service: Service | null) { - const db = MediaFile.db.node; - const all = db.prepare("select id,path,kind,size,timestamp_updated,duration,dimensions from media_files") - .all() as FileRow[]; - const files = all.filter((file) => file.kind === 1); - const extensions = new Map(files.map((file) => [file.id, extension(file.path)])); - const recorded = db.prepare(` - select fp.file,p.name,fp.version,fp.status,fp.updated,fp.error - from file_processors fp join processors p on p.id=fp.processor - `).all() as StateRow[]; - const byFile = new Map>(); - for (const row of recorded) { - const states = byFile.get(row.file) ?? new Map(); - states.set(row.name, row); - byFile.set(row.file, states); - } - const folders = new Map(); - const types = new Map(); - for (const file of files) { - const top = file.path.split("/")[1] ?? ""; - const folder = folders.get(top) ?? { name: top, files: 0, bytes: 0 }; - folder.files++; - folder.bytes += file.size; - folders.set(top, folder); - const ext = extensions.get(file.id) ?? ""; - const type = types.get(ext) ?? { ext, files: 0, bytes: 0 }; - type.files++; - type.bytes += file.size; - types.set(ext, type); - } - const processors = registry.processors.map((p) => { - let applicable = 0; - let done = 0; - let failed = 0; - let stale = 0; - for (const file of files) { - const ext = extensions.get(file.id) ?? ""; - if (p.include ? !p.include.has(ext) : p.exclude?.has(ext)) continue; - applicable++; - const state = byFile.get(file.id)?.get(p.name); - if (state?.version !== p.version) { - if (state) stale++; - } else if (state.status === 1) done++; - else if (state.status === 2) failed++; - } - return { - name: p.name, - title: p.title, - version: p.version, - runnable: registry.canExecute(p), - applicable, - done, - failed, - stale, - outputs: null, - bytes: null, - seconds: null, - }; - }); - const failures = files.flatMap((file) => { - const states = [...(byFile.get(file.id)?.values() ?? [])]; - const errors = states.flatMap((state) => - state.status === 2 && state.error !== null - ? [{ processor: state.name, error: state.error, at: state.updated / 1000 }] - : [] - ); - return errors.length - ? [{ - path: file.path, - size: file.size, - mime: mime.for(file.path), - made: states.filter((state) => state.status === 1).map((state) => state.name), - errors, - }] - : []; - }); - const recent = [...files].sort((a, b) => seconds(b.timestamp_updated) - seconds(a.timestamp_updated)) - .slice(0, 12).map((file) => ({ path: file.path, size: file.size, at: seconds(file.timestamp_updated) })); - const largest = [...files].sort((a, b) => b.size - a.size).slice(0, 12) - .map((file) => ({ path: file.path, size: file.size, duration: file.duration, dimensions: file.dimensions })); - const derived = db.prepare("select count(*) as files,coalesce(sum(size),0) as bytes from derived_files").get() as { - files: number; - bytes: number; - }; - return { - indexer: { enabled: service !== null, watching: Boolean(service?.watcher), nextSweepAt: null }, - lastSweep: service?.lastSweep ?? null, - files: files.length, - directories: all.length - files.length, - bytes: files.reduce((sum, file) => sum + file.size, 0), - folders: [...folders.values()].sort((a, b) => b.bytes - a.bytes), - types: [...types.values()].sort((a, b) => b.bytes - a.bytes), - processors, - derived, - failures, - recent, - largest, - }; -} - -const firstSeen = new WeakMap(); - -export function activity(root: progress.Root) { - const convert = (node: progress.ReadOnlyNode): object => { - if (!firstSeen.has(node)) firstSeen.set(node, Date.now() / 1000); - const match = /^(\/.*?) - (.*)$/.exec(node.text); - const children = (node.sortChildren ? [...node.children].sort(node.sortChildren) : node.children) - .filter((child) => !child.hidden && (!child.passive || child.children.length)).map(convert); - return { - path: match?.[1] ?? null, - text: match?.[2] ?? node.text, - count: node.valueFormatter(node.value, node.total), - progress: node.showTotal && node.total && node.total > 0 ? node.value / node.total : null, - eta: node.estimatedTime ? node.estimatedTime / 1000 : null, - since: firstSeen.get(node), - logs: node.logs.slice(-3).map((entry) => ({ - level: entry.level ?? "info", - text: entry.text.replace(/\x1b\[[\d;]*m/g, ""), - })), - children, - }; - }; - return root.active.filter((node) => !node.hidden && (!node.passive || node.children.length)).map(convert); -} diff --git a/src/source-of-truth.ts b/src/source-of-truth.ts index 90754af552d387345b8427731f0c14b55bb30b2b..306852315ecf6d7765f89043f8e0737f38f9dc0d 100644 --- a/src/source-of-truth.ts +++ b/src/source-of-truth.ts @@ -189,14 +189,14 @@ app.get("/progress", (c) => { return c.body(progress.encodeByteStream(indexer.root) as ReadableStream); }); -app.get("/studio/stats", (c) => { +app.get("/stats", (c) => { if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401); - return c.json(studio.stats(service)); + return c.json(status.stats(service)); }); -app.get("/studio/activity", (c) => { +app.get("/activity", (c) => { if (c.req.header("Authorization") !== token) return c.json({ error: "invalid authorization header" }, 401); - return c.json(studio.activity(indexer.root)); + return c.json(status.activity(indexer.root)); }); // download a consistent snapshot of cache.sqlite. web nodes call this with @@ -287,7 +287,7 @@ import * as fs from "#sitegen/fs"; import { Path } from "#sitegen/path"; import { hashFile } from "#src/file-viewer/indexer/scan.ts"; import * as indexer from "#src/file-viewer/indexer/service.ts"; -import * as studio from "#src/file-viewer/indexer/studio.ts"; +import * as status from "#src/file-viewer/indexer/status.ts"; import { FilePermissions } from "#src/file-viewer/models/FilePermissions.ts"; import { MediaFile, MediaFileKind } from "#src/file-viewer/models/MediaFile.ts"; import { ASSERT, UNWRAP } from "@clo/lib/assert";