authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-09-27 04:29:21-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2025-10-14 02:40:47-07:00
log75a0bf20bbd4e1f6a785795b200db98cafae3a14
treef898e916a3b62680d38a3a962bb9b7811fa1012b
parent6e8e86f58f873d0aa6151ac60aef9a637b531f07
signature Commit is signed but in an unrecognized format.

chore(file-viewer): use lib/progress in scan3

holy fuck it's pretty as fuck.

4 files changed, 146 insertions(+), 162 deletions(-)

framework/lib/async.ts+1
...@@ -9,6 +9,7 @@ interface QueueOptions<T, R> {...@@ -9,6 +9,7 @@ interface QueueOptions<T, R> {
9}9}
1010
11// Process multiple items in parallel, queue up as many.11// Process multiple items in parallel, queue up as many.
12/** @deprecated */
12export class Queue<T, R> {13export class Queue<T, R> {
13 #name: string;14 #name: string;
14 #fn: (item: T, spin: Spinner) => Promise<R>;15 #fn: (item: T, spin: Spinner) => Promise<R>;
src/file-viewer/bin/scan3.ts+134-148
...@@ -14,30 +14,39 @@ const sotToken = process.env.CLOVER_SOT_KEY;...@@ -14,30 +14,39 @@ const sotToken = process.env.CLOVER_SOT_KEY;
1414
15export async function main() {15export async function main() {
16 const start = performance.now();16 const start = performance.now();
17 const timerSpinner = new Spinner({17 using _ = term.startWidget({
18 text: () =>18 format: (now) =>
19 `paper clover's scan3 [${19 `paper clover's scan3 [${((now - start) / 1000).toFixed(1)}s]`,
20 ((performance.now() - start) / 1000).toFixed(
21 1,
22 )
23 }s]`,
24 fps: 10,
25 });20 });
26 using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };21
22 const walkQueue = new queue.PriorityQueue(10);
23
24 const dirsNode = progress.start("Walk Tree", { estimate: 1 });
25 const fileNode = progress.start("Process File", { estimate: 0 });
26 fileNode.sortChildren = (a, b) => {
27 const ac = a.children.length > 0 ? 1 : 0;
28 const bc = b.children.length > 0 ? 1 : 0;
29 if (ac !== bc) return bc - ac;
30 return a.text.localeCompare(b.text);
31 };
2732
28 // Read a directory or file stat and queue up changed files.33 // Read a directory or file stat and queue up changed files.
29 using qList = new async.Queue({34 const scanDirectory = walkQueue.wrap(
30 name: "Discover Tree",35 async (path: Path) => {
31 async fn(absPath: string) {36 const publicPath = toPublicPath(path);
32 const stat = await fs.stat(absPath);37 using node = dirsNode.start(publicPath + " - stat");
38 using _ = ts.defer(() => dirsNode.inc());
39
40 const stat = await path.stat();
3341
34 const publicPath = toPublicPath(Path.resolve(absPath));
35 const mediaFile = MediaFile.getByPath(publicPath);42 const mediaFile = MediaFile.getByPath(publicPath);
3643
37 if (stat.isDirectory()) {44 if (stat.isDirectory()) {
38 const items = (await fs.readdir(absPath))45 node.text = publicPath + " - reading";
39 .filter((basename) => !skipBasename(basename));46 const items = (await path.readDir())
40 qList.addMany(items.map((subPath) => path.join(absPath, subPath)));47 .filter((child) => !skipBasename(child.base))
48 .map((child) => (scanDirectory(child), child.base));
49 dirsNode.estimate += items.length;
4150
42 if (mediaFile) {51 if (mediaFile) {
43 const deleted = mediaFile52 const deleted = mediaFile
...@@ -48,45 +57,39 @@ export async function main() {...@@ -48,45 +57,39 @@ export async function main() {
48 ? child.getRecursiveFileChildren()57 ? child.getRecursiveFileChildren()
49 : child58 : child
50 );59 );
5160 //deleted
52 qMeta.addMany(
53 deleted.map((mediaFile) => ({
54 path: Path.resolve(root, mediaFile.path),
55 publicPath: mediaFile.path,
56 stat: null,
57 mediaFile,
58 })),
59 );
60 }61 }
6162
62 return;63 return;
63 }64 }
6465
65 // All processes must be performed again if there is no file.
66 if (66 if (
67 !mediaFile ||67 !mediaFile || // All processes must be performed if there is no file.
68 // Rerun all processors if it changed
68 stat.size !== mediaFile.size ||69 stat.size !== mediaFile.size ||
69 stat.mtime.getTime() !== mediaFile.date.getTime()70 stat.mtime.getTime() !== mediaFile.date.getTime()
70 ) {71 ) {
71 qMeta.add({ path: new Path(absPath), publicPath, stat, mediaFile });72 updateMetadata({ path, publicPath, stat, mediaFile });
72 return;73 } else {
74 // If the scanners changed, it may mean more processes should be run.
75 queueProcessors({
76 path,
77 stat,
78 mediaFile,
79 node: fileNode.start(publicPath.slice(1)),
80 });
73 }81 }
74
75 // If the scanners changed, it may mean more processes should be run.
76 queueProcessors({ path: new Path(absPath), stat, mediaFile });
77 },82 },
78 maxJobs: 24,83 );
79 });84 const updateMetadata = queue.wrap(
80 using qMeta = new async.Queue({85 async ({ path, publicPath, stat, mediaFile }: UpdateMetadataArgs) => {
81 name: "Update Metadata",86 using errorHandler = new DisposableStack();
82 async fn({ path, publicPath, stat, mediaFile }: UpdateMetadataJob) {87 const label = publicPath.slice(1);
83 if (!stat) {88 const node = errorHandler.use(fileNode.start(label));
84 // File was deleted.
85 await runUndoProcessors(UNWRAP(mediaFile));
86 return;
87 }
88 await scrubLocationMetadata(path, stat);
8989
90 await scrubLocationMetadata(path, stat, node);
91
92 node.text = `${label} - hashing`;
90 const hash = await new Promise<string>((resolve, reject) => {93 const hash = await new Promise<string>((resolve, reject) => {
91 const reader = fs.createReadStream(path.toString());94 const reader = fs.createReadStream(path.toString());
92 reader.on("error", reject);95 reader.on("error", reject);
...@@ -121,42 +124,46 @@ export async function main() {...@@ -121,42 +124,46 @@ export async function main() {
121 dimensions: mediaFile?.dimensions ?? "",124 dimensions: mediaFile?.dimensions ?? "",
122 contents: mediaFile?.contents ?? "",125 contents: mediaFile?.contents ?? "",
123 });126 });
127
128 node.text = `${label}`;
124 await queueProcessors({129 await queueProcessors({
125 path,130 path,
126 stat,131 stat,
127 mediaFile,132 mediaFile,
133 node,
128 });134 });
135 errorHandler.move();
129 },136 },
130 getItemText: (job) =>137 () => ({ priority: -1 }),
131 job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"),138 );
132 maxJobs: 10,139 const queueFileProcessor = queue.wrap(async function ({
133 });140 path,
134 using qProcess = new async.Queue({141 stat,
135 name: "Process Contents",142 mediaFile,
136 async fn(143 processor,
137 { path, stat, mediaFile, processor, index, after }: ProcessJob,144 index,
138 spin,145 after,
139 ) {146 fileNode,
140 await processor.run({147 }: ProcessJob) {
141 path,148 using node = fileNode.start(processor.name);
142 stat,149 await processor.run({
143 mediaFile,150 path,
144 spin,151 stat,
145 });152 mediaFile,
146 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));153 node,
147 for (const dependantJob of after) {154 });
148 ASSERT(155 mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
149 dependantJob.needs > 0,156 for (const dependantJob of after) {
150 `dependantJob.needs > 0, ${dependantJob.needs}`,157 ASSERT(
151 );158 dependantJob.needs > 0,
152 dependantJob.needs -= 1;159 `dependantJob.needs > 0, ${dependantJob.needs}`,
153 if (dependantJob.needs == 0) qProcess.add(dependantJob);160 );
161 dependantJob.needs -= 1;
162 if (dependantJob.needs == 0) {
163 queueFileProcessor(dependantJob);
154 }164 }
155 },165 }
156 getItemText: ({ mediaFile, processor }) =>166 }, (job) => ({ cores: job.processor.cores }));
157 `${mediaFile.path.slice(1)} - ${processor.name}`,
158 maxJobs: 4,
159 });
160167
161 function decodeProcessors(input: string) {168 function decodeProcessors(input: string) {
162 return input169 return input
...@@ -172,7 +179,14 @@ export async function main() {...@@ -172,7 +179,14 @@ export async function main() {
172 path,179 path,
173 stat,180 stat,
174 mediaFile,181 mediaFile,
182 node,
175 }: Omit<ProcessFileArgs, "spin">) {183 }: Omit<ProcessFileArgs, "spin">) {
184 using errorHandler = new DisposableStack();
185 errorHandler.use(node);
186
187 node.showEstimate = false;
188 node.passive = true;
189
176 const ext = mediaFile.extensionNonEmpty.toLowerCase();190 const ext = mediaFile.extensionNonEmpty.toLowerCase();
177 let possible = processors.filter((p) =>191 let possible = processors.filter((p) =>
178 p.include ? p.include.has(ext) : !p.exclude?.has(ext)192 p.include ? p.include.has(ext) : !p.exclude?.has(ext)
...@@ -196,11 +210,7 @@ export async function main() {...@@ -196,11 +210,7 @@ export async function main() {
196 const p = processors.find((p) => p.id === id);210 const p = processors.find((p) => p.id === id);
197 if (!p) continue;211 if (!p) continue;
198 const index = possible.indexOf(p);212 const index = possible.indexOf(p);
199 if (index !== -1 && p.hash === hash) {213 if (index !== -1 && p.hash === hash) processed |= 1 << (16 + index);
200 processed |= 1 << (16 + index);
201 } else {
202 if (p.undo) await p.undo(mediaFile);
203 }
204 }214 }
205 mediaFile.setProcessors(215 mediaFile.setProcessors(
206 processed,216 processed,
...@@ -218,19 +228,22 @@ export async function main() {...@@ -218,19 +228,22 @@ export async function main() {
218 const jobs: ProcessJob[] = [];228 const jobs: ProcessJob[] = [];
219 for (let i = 0, { length } = possible; i < length; i += 1) {229 for (let i = 0, { length } = possible; i < length; i += 1) {
220 if ((processed & (1 << (16 + i))) === 0) {230 if ((processed & (1 << (16 + i))) === 0) {
231 const processor = UNWRAP(possible[i]);
221 const job: ProcessJob = {232 const job: ProcessJob = {
222 path,233 path,
223 stat,234 stat,
224 mediaFile,235 mediaFile,
225 processor: UNWRAP(possible[i]),236 processor,
226 index: i,237 index: i,
227 after: [],238 after: [],
228 needs: UNWRAP(possible[i]).depends.length,239 needs: processor.depends.length,
240 fileNode: node,
229 };241 };
230 jobs.push(job);242 jobs.push(job);
231 if (job.needs === 0) qProcess.add(job);243 if (job.needs === 0) queueFileProcessor(job);
232 }244 }
233 }245 }
246 node.estimate = jobs.length;
234 for (const job of jobs) {247 for (const job of jobs) {
235 for (const dependId of job.processor.depends) {248 for (const dependId of job.processor.depends) {
236 const dependJob = jobs.find((j) => j.processor.id === dependId);249 const dependJob = jobs.find((j) => j.processor.id === dependId);
...@@ -239,32 +252,19 @@ export async function main() {...@@ -239,32 +252,19 @@ export async function main() {
239 } else {252 } else {
240 ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);253 ASSERT(job.needs > 0, `job.needs !== 0, ${job.needs}`);
241 job.needs -= 1;254 job.needs -= 1;
242 if (job.needs === 0) qProcess.add(job);255 if (job.needs === 0) queueFileProcessor(job);
243 }256 }
244 }257 }
245 }258 }
246 }259 if (node.estimate > 0) errorHandler.move();
247
248 async function runUndoProcessors(mediaFile: MediaFile) {
249 const { processed } = mediaFile;
250 const previous = decodeProcessors(mediaFile.processors).filter(
251 (_, i) => (processed & (1 << (16 + i))) !== 0,
252 );
253 for (const { id } of previous) {
254 const p = processors.find((p) => p.id === id);
255 if (!p) continue;
256 if (p.undo) {
257 await p.undo(mediaFile);
258 }
259 }
260 mediaFile.delete();
261 }260 }
262261
263 // Add the root & recursively iterate!262 // Add the root & recursively iterate!
264 qList.add(root);263 scanDirectory(Path.resolve(root));
265 await qList.done();264 // await qList.done();
266 await qMeta.done();265 // await qMeta.done();
267 await qProcess.done();266 // await qProcess.done();
267 return;
268268
269 // Update directory metadata269 // Update directory metadata
270 const dirs = MediaFile.getDirectoriesToReindex()270 const dirs = MediaFile.getDirectoriesToReindex()
...@@ -401,6 +401,7 @@ export async function main() {...@@ -401,6 +401,7 @@ export async function main() {
401401
402interface Process {402interface Process {
403 name: string;403 name: string;
404 cores: number;
404 enable?: boolean;405 enable?: boolean;
405 include?: Set<string>;406 include?: Set<string>;
406 exclude?: Set<string>;407 exclude?: Set<string>;
...@@ -408,8 +409,6 @@ interface Process {...@@ -408,8 +409,6 @@ interface Process {
408 version?: number;409 version?: number;
409 /* Perform an action. */410 /* Perform an action. */
410 run(args: ProcessFileArgs): Promise<void>;411 run(args: ProcessFileArgs): Promise<void>;
411 /* Should detect if `run` was never even run before before undoing state */
412 undo?(mediaFile: MediaFile): Promise<void>;
413}412}
414413
415const ffprobeBin = testProgram("ffprobe", "--help");414const ffprobeBin = testProgram("ffprobe", "--help");
...@@ -422,6 +421,7 @@ const procDuration: Process = {...@@ -422,6 +421,7 @@ const procDuration: Process = {
422 name: "calculate duration",421 name: "calculate duration",
423 enable: ffprobeBin !== null,422 enable: ffprobeBin !== null,
424 include: rules.extsDuration,423 include: rules.extsDuration,
424 cores: 1,
425 async run({ path, mediaFile }) {425 async run({ path, mediaFile }) {
426 const { stdout } = await subprocess.exec(ffprobeBin!, [426 const { stdout } = await subprocess.exec(ffprobeBin!, [
427 "-v",427 "-v",
...@@ -445,6 +445,7 @@ const procDimensions: Process = {...@@ -445,6 +445,7 @@ const procDimensions: Process = {
445 name: "calculate dimensions",445 name: "calculate dimensions",
446 enable: ffprobeBin != null,446 enable: ffprobeBin != null,
447 include: rules.extsDimensions,447 include: rules.extsDimensions,
448 cores: 1,
448 async run({ path, mediaFile }) {449 async run({ path, mediaFile }) {
449 const { ext } = path;450 const { ext } = path;
450451
...@@ -484,6 +485,7 @@ const procDimensions: Process = {...@@ -484,6 +485,7 @@ const procDimensions: Process = {
484const procLoadTextContents: Process = {485const procLoadTextContents: Process = {
485 name: "load text content",486 name: "load text content",
486 include: rules.extsReadContents,487 include: rules.extsReadContents,
488 cores: 1,
487 async run({ path, mediaFile, stat }) {489 async run({ path, mediaFile, stat }) {
488 if (stat.size > 1_000_000) return;490 if (stat.size > 1_000_000) return;
489 const text = await path.read("utf-8");491 const text = await path.read("utf-8");
...@@ -494,6 +496,7 @@ const procLoadTextContents: Process = {...@@ -494,6 +496,7 @@ const procLoadTextContents: Process = {
494const procHighlightCode: Process = {496const procHighlightCode: Process = {
495 name: "highlight source code",497 name: "highlight source code",
496 include: new Set(rules.extsCode.keys()),498 include: new Set(rules.extsCode.keys()),
499 cores: 1,
497 async run({ path, mediaFile, stat }) {500 async run({ path, mediaFile, stat }) {
498 const language = UNWRAP(501 const language = UNWRAP(
499 rules.extsCode.get(path.ext.toLowerCase()),502 rules.extsCode.get(path.ext.toLowerCase()),
...@@ -523,16 +526,17 @@ const procImageSubsets: Process = {...@@ -523,16 +526,17 @@ const procImageSubsets: Process = {
523 include: rules.extsImage,526 include: rules.extsImage,
524 depends: [procDimensions.name],527 depends: [procDimensions.name],
525 version: 2,528 version: 2,
526 async run({ path, mediaFile, spin }) {529 cores: 2,
530 async run({ path, mediaFile, node }) {
527 const { width, height } = UNWRAP(mediaFile.parseDimensions());531 const { width, height } = UNWRAP(mediaFile.parseDimensions());
528 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);532 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
529 const baseStatus = spin.text;533 const baseStatus = node.text;
530534
531 using stack = new DisposableStack();535 using stack = new DisposableStack();
532 for (const size of targetSizes) {536 for (const size of targetSizes) {
533 const { w, h } = resizeDimensions(width, height, size);537 const { w, h } = resizeDimensions(width, height, size);
534 for (const { ext, args } of transcodeRules.imagePresets) {538 for (const { ext, args } of transcodeRules.imagePresets) {
535 spin.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;539 node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
536540
537 stack.use(541 stack.use(
538 await derived.produce(mediaFile, `${size}${ext}`, async (dir) => {542 await derived.produce(mediaFile, `${size}${ext}`, async (dir) => {
...@@ -570,22 +574,16 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({...@@ -570,22 +574,16 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
570 include: rules.extsVideo,574 include: rules.extsVideo,
571 enable: ffmpegBin != null,575 enable: ffmpegBin != null,
572 depends: [procDuration.name, procDimensions.name],576 depends: [procDuration.name, procDimensions.name],
573 async run({ path, mediaFile, spin }) {577 cores: 4,
578 async run({ path, mediaFile, node }) {
574 if ((mediaFile.duration ?? 0) < 5) return;579 if ((mediaFile.duration ?? 0) < 5) return;
575 await derived.produce(mediaFile, `av1-${preset.id}`, async (dir) => {580 await derived.produce(mediaFile, `av1-${preset.id}`, async (dir) => {
576 const input = await videoInputArgsCache.getOrRun(path);581 const input = await videoInputArgsCache.getOrRun(path);
577 const args = transcodeRules.getAv1VideoArgs(preset, input.video, dir);582 const args = transcodeRules.getAv1VideoArgs(preset, input.video, dir);
578583
579 const fakeProgress = new Progress({ text: spin.text, spinner: null });
580 fakeProgress.stop();
581 spin.format = (now: number) => fakeProgress.format(now);
582 // @ts-expect-error
583 fakeProgress.redraw = () => spin.redraw();
584
585 await ffmpeg.spawn({584 await ffmpeg.spawn({
586 ffmpeg: ffmpegBin!,585 ffmpeg: ffmpegBin!,
587 title: fakeProgress.text,586 progress: node,
588 progress: fakeProgress,
589 args,587 args,
590 cwd: dir,588 cwd: dir,
591 });589 });
...@@ -597,23 +595,17 @@ const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({...@@ -597,23 +595,17 @@ const procVideoAudios = transcodeRules.audioFormats.map<Process>((preset) => ({
597 include: rules.extsVideo,595 include: rules.extsVideo,
598 enable: ffmpegBin != null,596 enable: ffmpegBin != null,
599 depends: [procDuration.name, procDimensions.name],597 depends: [procDuration.name, procDimensions.name],
600 async run({ path, mediaFile, spin }) {598 cores: 2,
599 async run({ path, mediaFile, node }) {
601 if ((mediaFile.duration ?? 0) < 5) return;600 if ((mediaFile.duration ?? 0) < 5) return;
602 await derived.produce(mediaFile, `opus-${preset.id}`, async (dir) => {601 await derived.produce(mediaFile, `opus-${preset.id}`, async (dir) => {
603 const input = await videoInputArgsCache.getOrRun(path);602 const input = await videoInputArgsCache.getOrRun(path);
604 if (!input.audio) return;603 if (!input.audio) return;
605 const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);604 const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
606605
607 const fakeProgress = new Progress({ text: spin.text, spinner: null });
608 fakeProgress.stop();
609 spin.format = (now: number) => fakeProgress.format(now);
610 // @ts-expect-error
611 fakeProgress.redraw = () => spin.redraw();
612
613 await ffmpeg.spawn({606 await ffmpeg.spawn({
614 ffmpeg: ffmpegBin!,607 ffmpeg: ffmpegBin!,
615 title: fakeProgress.text,608 progress: node,
616 progress: fakeProgress,
617 args,609 args,
618 cwd: dir,610 cwd: dir,
619 });611 });
...@@ -625,7 +617,8 @@ const procDash: Process = {...@@ -625,7 +617,8 @@ const procDash: Process = {
625 include: rules.extsVideo,617 include: rules.extsVideo,
626 enable: ffmpegBin != null,618 enable: ffmpegBin != null,
627 depends: [...procVideos, ...procVideoAudios].map((x) => x.name),619 depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
628 async run({ path, mediaFile, spin }) {620 cores: 1,
621 async run({ path, mediaFile, node }) {
629 if ((mediaFile.duration ?? 0) < 5) return;622 if ((mediaFile.duration ?? 0) < 5) return;
630 await derived.produce(mediaFile, `dash-av1`, async (dir) => {623 await derived.produce(mediaFile, `dash-av1`, async (dir) => {
631 const input = await videoInputArgsCache.getOrRun(path);624 const input = await videoInputArgsCache.getOrRun(path);
...@@ -649,16 +642,9 @@ const procDash: Process = {...@@ -649,16 +642,9 @@ const procDash: Process = {
649642
650 await dir.join("d").makeDir();643 await dir.join("d").makeDir();
651644
652 const fakeProgress = new Progress({ text: spin.text, spinner: null });
653 fakeProgress.stop();
654 spin.format = (now: number) => fakeProgress.format(now);
655 // @ts-expect-error
656 fakeProgress.redraw = () => spin.redraw();
657
658 await ffmpeg.spawn({645 await ffmpeg.spawn({
659 ffmpeg: ffmpegBin!,646 ffmpeg: ffmpegBin!,
660 title: fakeProgress.text,647 progress: node,
661 progress: fakeProgress,
662 args,648 args,
663 cwd: dir,649 cwd: dir,
664 });650 });
...@@ -670,22 +656,16 @@ const procH264Hls: Process = {...@@ -670,22 +656,16 @@ const procH264Hls: Process = {
670 include: rules.extsVideo,656 include: rules.extsVideo,
671 enable: ffmpegBin != null,657 enable: ffmpegBin != null,
672 depends: [procDuration.name, procDimensions.name],658 depends: [procDuration.name, procDimensions.name],
673 async run({ path, mediaFile, spin }) {659 cores: 8,
660 async run({ path, mediaFile, node }) {
674 if ((mediaFile.duration ?? 0) < 5) return;661 if ((mediaFile.duration ?? 0) < 5) return;
675 await derived.produce(mediaFile, `hls`, async (dir) => {662 await derived.produce(mediaFile, `hls`, async (dir) => {
676 const input = await videoInputArgsCache.getOrRun(path);663 const input = await videoInputArgsCache.getOrRun(path);
677 const args = transcodeRules.getH264HlsArgs(input, dir);664 const args = transcodeRules.getH264HlsArgs(input, dir);
678665
679 const fakeProgress = new Progress({ text: spin.text, spinner: null });
680 fakeProgress.stop();
681 spin.format = (now: number) => fakeProgress.format(now);
682 // @ts-expect-error
683 fakeProgress.redraw = () => spin.redraw();
684
685 await ffmpeg.spawn({666 await ffmpeg.spawn({
686 ffmpeg: ffmpegBin!,667 ffmpeg: ffmpegBin!,
687 title: fakeProgress.text,668 progress: node,
688 progress: fakeProgress,
689 args,669 args,
690 cwd: dir,670 cwd: dir,
691 });671 });
...@@ -700,6 +680,7 @@ const procCompression = [...@@ -700,6 +680,7 @@ const procCompression = [
700 ({ name, fn }) => ({680 ({ name, fn }) => ({
701 name: `compress ${name}`,681 name: `compress ${name}`,
702 exclude: rules.extsPreCompressed,682 exclude: rules.extsPreCompressed,
683 cores: 1,
703 async run({ path, mediaFile }) {684 async run({ path, mediaFile }) {
704 await derived.produce(mediaFile, name, async (dir) => {685 await derived.produce(mediaFile, name, async (dir) => {
705 await stream.promises.pipeline(686 await stream.promises.pipeline(
...@@ -752,10 +733,10 @@ function resizeDimensions(w: number, h: number, desiredWidth: number) {...@@ -752,10 +733,10 @@ function resizeDimensions(w: number, h: number, desiredWidth: number) {
752 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };733 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
753}734}
754735
755interface UpdateMetadataJob {736interface UpdateMetadataArgs {
756 path: Path;737 path: Path;
757 publicPath: string;738 publicPath: string;
758 stat: fs.Stats | null;739 stat: fs.Stats;
759 mediaFile: MediaFile | null;740 mediaFile: MediaFile | null;
760}741}
761742
...@@ -763,7 +744,7 @@ interface ProcessFileArgs {...@@ -763,7 +744,7 @@ interface ProcessFileArgs {
763 path: Path;744 path: Path;
764 stat: fs.Stats;745 stat: fs.Stats;
765 mediaFile: MediaFile;746 mediaFile: MediaFile;
766 spin: Spinner;747 node: progress.Node;
767}748}
768749
769interface ProcessJob {750interface ProcessJob {
...@@ -774,6 +755,7 @@ interface ProcessJob {...@@ -774,6 +755,7 @@ interface ProcessJob {
774 index: number;755 index: number;
775 after: ProcessJob[];756 after: ProcessJob[];
776 needs: number;757 needs: number;
758 fileNode: progress.Node;
777}759}
778760
779function skipBasename(basename: string): boolean {761function skipBasename(basename: string): boolean {
...@@ -810,7 +792,9 @@ function testProgram(name: string, helpArgument: string) {...@@ -810,7 +792,9 @@ function testProgram(name: string, helpArgument: string) {
810async function scrubLocationMetadata(792async function scrubLocationMetadata(
811 path: Path,793 path: Path,
812 stats: fs.Stats,794 stats: fs.Stats,
795 progress: progress.Ref,
813): Promise<boolean> {796): Promise<boolean> {
797 using _ = progress.start("scrub exif metadata");
814 const ext = path.ext.toLowerCase();798 const ext = path.ext.toLowerCase();
815 if (!rules.extsScrubExif.has(ext)) return false;799 if (!rules.extsScrubExif.has(ext)) return false;
816800
...@@ -894,7 +878,7 @@ async function scrubLocationMetadata(...@@ -894,7 +878,7 @@ async function scrubLocationMetadata(
894 }878 }
895879
896 // Restore original timestamps880 // Restore original timestamps
897 await fsp.rename(tempOutput.toString(), path);881 await fsp.rename(tempOutput.toString(), path.toString());
898 await fsp.utimes(path.toString(), accessTime, modTime);882 await fsp.utimes(path.toString(), accessTime, modTime);
899883
900 // Backup is no longer needed884 // Backup is no longer needed
...@@ -920,12 +904,14 @@ async function scrubLocationMetadata(...@@ -920,12 +904,14 @@ async function scrubLocationMetadata(
920904
921const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;905const monthMilliseconds = 30 * 24 * 60 * 60 * 1000;
922906
923import { Progress } from "@paperclover/console/Progress";
924import { Spinner } from "@paperclover/console/Spinner";
925import * as async from "#sitegen/async";907import * as async from "#sitegen/async";
926import * as fs from "#sitegen/fs";908import * as fs from "#sitegen/fs";
927import * as subprocess from "#sitegen/subprocess";909import * as subprocess from "#sitegen/subprocess";
928import { Path } from "#sitegen/path";910import { Path } from "#sitegen/path";
911import * as queue from "../../../lib/queue.ts";
912import * as term from "../../../lib/term.ts";
913import * as progress from "../../../lib/term/progress.ts";
914import * as ts from "../../../lib/ts.ts";
929915
930import * as child_process from "node:child_process";916import * as child_process from "node:child_process";
931import * as crypto from "node:crypto";917import * as crypto from "node:crypto";
src/file-viewer/ffmpeg.ts+9-13
...@@ -20,21 +20,21 @@ export const defaultExtraOptions = [...@@ -20,21 +20,21 @@ export const defaultExtraOptions = [
2020
21export interface SpawnOptions {21export interface SpawnOptions {
22 args: string[];22 args: string[];
23 title: string;
24 ffmpeg?: string;23 ffmpeg?: string;
25 progress?: Progress;24 progress: progress.Node;
26 cwd: Path;25 cwd: Path;
27}26}
2827
29export async function spawn(options: SpawnOptions) {28export async function spawn(options: SpawnOptions) {
30 const { ffmpeg = "ffmpeg", args, title, cwd } = options;29 const { ffmpeg = "ffmpeg", args, cwd } = options;
30 using node = options.progress;
31 const title = node.text;
31 const proc = child_process.spawn(ffmpeg, [...defaultExtraOptions, ...args], {32 const proc = child_process.spawn(ffmpeg, [...defaultExtraOptions, ...args], {
32 stdio: ["ignore", "inherit", "pipe"],33 stdio: ["ignore", "inherit", "pipe"],
33 env: { ...process.env, SVT_LOG: "2" },34 env: { ...process.env, SVT_LOG: "2" },
34 cwd: cwd.toString(),35 cwd: cwd.toString(),
35 });36 });
36 const parser = new Parse();37 const parser = new Parse();
37 const bar = options.progress ?? new Progress({ text: title });
38 let running = true;38 let running = true;
39 const splitter = readline.createInterface({ input: proc.stderr });39 const splitter = readline.createInterface({ input: proc.stderr });
40 splitter.on("line", (line) => {40 splitter.on("line", (line) => {
...@@ -42,20 +42,18 @@ export async function spawn(options: SpawnOptions) {...@@ -42,20 +42,18 @@ export async function spawn(options: SpawnOptions) {
42 if (result.kind === "ignore") {42 if (result.kind === "ignore") {
43 return;43 return;
44 } else if (result.kind === "log") {44 } else if (result.kind === "log") {
45 console[result.level](result.message);45 node.log[result.level](result.message);
46 } else if (result.kind === "progress") {46 } else if (result.kind === "progress") {
47 if (!running) return;47 if (!running) return;
48 const { frame, totalFrames, fps, speed } = result;48 const { frame, totalFrames, fps, speed } = result;
49 bar.value = frame;49 node.value = frame;
50 bar.total = totalFrames;50 node.estimate = totalFrames;
51 const extras = [51 const extras = [
52 `${fps} fps`,52 `${fps} fps`,
53 speed,53 speed,
54 parser.hlsFile,54 parser.hlsFile,
55 ].filter(Boolean).join(", ");55 ].filter(Boolean).join(", ");
56 bar.text = `${title} ${frame}/${totalFrames} ${56 node.text = `${title}${extras.length > 0 ? ` (${extras})` : ""}`;
57 extras.length > 0 ? `(${extras})` : ""
58 }`;
59 } else result satisfies never;57 } else result satisfies never;
60 });58 });
61 const [code, signal] = await events.once(proc, "close");59 const [code, signal] = await events.once(proc, "close");
...@@ -66,10 +64,8 @@ export async function spawn(options: SpawnOptions) {...@@ -66,10 +64,8 @@ export async function spawn(options: SpawnOptions) {
66 e.args = [ffmpeg, ...args].join(" ");64 e.args = [ffmpeg, ...args].join(" ");
67 e.code = code;65 e.code = code;
68 e.signal = signal;66 e.signal = signal;
69 bar.stop();
70 throw e;67 throw e;
71 }68 }
72 bar.success(title);
73}69}
7470
75// this is janky71// this is janky
...@@ -181,5 +177,5 @@ import * as readline from "node:readline";...@@ -181,5 +177,5 @@ import * as readline from "node:readline";
181import * as process from "node:process";177import * as process from "node:process";
182import events from "node:events";178import events from "node:events";
183import * as path from "node:path";179import * as path from "node:path";
184import { Progress } from "@paperclover/console/Progress";180import * as progress from "../../lib/term/progress.ts";
185import type { Path } from "#sitegen/path";181import type { Path } from "#sitegen/path";
src/file-viewer/models/MediaFile.ts+2-1
...@@ -370,7 +370,8 @@ const setProcessorsQuery = db.prepare<[{...@@ -370,7 +370,8 @@ const setProcessorsQuery = db.prepare<[{
370 processors: string;370 processors: string;
371}]>(/* SQL */ `371}]>(/* SQL */ `
372 update media_files set372 update media_files set
373 processed = $processed373 processed = $processed,
374 processors = $processors
374 where id = $id;375 where id = $id;
375`);376`);
376const setDurationQuery = db.prepare<[{377const setDurationQuery = db.prepare<[{