| ... | ... | @@ -1,1015 +0,0 @@ |
| 1 | | // This file was started by AI and maintained by hand since. |
| 2 | | import "@paperclover/console/inject"; |
| 3 | | import { Progress } from "@paperclover/console/Progress"; |
| 4 | | import { Spinner } from "@paperclover/console/Spinner"; |
| 5 | | import assert from "node:assert"; |
| 6 | | import { execFile } from "node:child_process"; |
| 7 | | import { existsSync, Stats } from "node:fs"; |
| 8 | | import * as fsp from "node:fs/promises"; |
| 9 | | import * as path from "node:path"; |
| 10 | | import { promisify } from "node:util"; |
| 11 | | import { BlobAsset, cache, FilePermissions, MediaFile } from "../db.ts"; |
| 12 | | import { formatDate, formatSize } from "./share.ts"; |
| 13 | | import { highlightCode, type Language } from "./highlight.ts"; |
| 14 | | |
| 15 | | const execFileAsync = promisify(execFile); |
| 16 | | |
| 17 | | // Configuration |
| 18 | | const FILE_ROOT = process.env.SCAN_FILE_ROOT; |
| 19 | | if (!FILE_ROOT) { |
| 20 | | throw new Error( |
| 21 | | "FILE_ROOT environment variable not set (e.g. '/path/to/files')", |
| 22 | | ); |
| 23 | | } |
| 24 | | const LOCAL_DIR = path.resolve(FILE_ROOT); |
| 25 | | const DRY_RUN = process.argv.includes("--dry-run"); |
| 26 | | const SHOULD_COMPRESS = true; |
| 27 | | const VERBOSE = process.argv.includes("--verbose"); |
| 28 | | const SHOULD_SCRUB = true; |
| 29 | | const COMPRESS_STORE = process.env.COMPRESS_STORE || |
| 30 | | path.join(process.cwd(), ".clover/compressed"); |
| 31 | | |
| 32 | | // Helper function for logging that respects verbose flag |
| 33 | | function log(message: string, always = false): void { |
| 34 | | if (always || VERBOSE) { |
| 35 | | console.log(message); |
| 36 | | } |
| 37 | | } |
| 38 | | |
| 39 | | // File extensions that need duration metadata |
| 40 | | const MEDIA_EXTENSIONS = new Set([ |
| 41 | | ".mp4", |
| 42 | | ".mkv", |
| 43 | | ".webm", |
| 44 | | ".avi", |
| 45 | | ".mov", |
| 46 | | ".mp3", |
| 47 | | ".flac", |
| 48 | | ".wav", |
| 49 | | ".ogg", |
| 50 | | ".m4a", |
| 51 | | ]); |
| 52 | | |
| 53 | | // File extensions that need dimension metadata |
| 54 | | const IMAGE_EXTENSIONS = new Set([ |
| 55 | | ".jpg", |
| 56 | | ".jpeg", |
| 57 | | ".png", |
| 58 | | ".gif", |
| 59 | | ".webp", |
| 60 | | ".avif", |
| 61 | | ".heic", |
| 62 | | ".svg", |
| 63 | | ]); |
| 64 | | |
| 65 | | const VIDEO_EXTENSIONS = new Set([".mp4", ".mkv", ".webm", ".avi", ".mov"]); |
| 66 | | |
| 67 | | // File extensions that need metadata scrubbing |
| 68 | | const SCRUB_EXTENSIONS = new Set([ |
| 69 | | ".jpg", |
| 70 | | ".jpeg", |
| 71 | | ".png", |
| 72 | | ".mov", |
| 73 | | ".mp4", |
| 74 | | ".m4a", |
| 75 | | ]); |
| 76 | | |
| 77 | | const CODE_EXTENSIONS: Record<string, Language> = { |
| 78 | | ".json": "json", |
| 79 | | ".toml": "toml", |
| 80 | | ".ts": "ts", |
| 81 | | ".js": "ts", |
| 82 | | ".tsx": "tsx", |
| 83 | | ".jsx": "tsx", |
| 84 | | ".css": "css", |
| 85 | | ".py": "python", |
| 86 | | ".lua": "lua", |
| 87 | | ".sh": "shell", |
| 88 | | ".bat": "dosbatch", |
| 89 | | ".ps1": "powershell", |
| 90 | | ".cmd": "dosbatch", |
| 91 | | ".yaml": "yaml", |
| 92 | | ".yml": "yaml", |
| 93 | | ".zig": "zig", |
| 94 | | ".astro": "astro", |
| 95 | | ".mdx": "mdx", |
| 96 | | ".xml": "xml", |
| 97 | | ".jsonc": "json", |
| 98 | | ".php": "php", |
| 99 | | ".patch": "diff", |
| 100 | | ".diff": "diff", |
| 101 | | }; |
| 102 | | |
| 103 | | const READ_CONTENTS_EXTENSIONS = new Set([".txt", ".chat"]); |
| 104 | | |
| 105 | | // For files that have changed indexing logic, update the date here rescanning |
| 106 | | // will reconstruct the entire file object. This way you can incrementally |
| 107 | | // update new file types without having to reindex everything. |
| 108 | | const lastUpdateTypes: Record<string, Date> = {}; |
| 109 | | lastUpdateTypes[".lnk"] = new Date("2025-05-13 13:58:00"); |
| 110 | | for (const ext in CODE_EXTENSIONS) { |
| 111 | | lastUpdateTypes[ext] = new Date("2025-05-13 13:58:00"); |
| 112 | | } |
| 113 | | for (const ext of READ_CONTENTS_EXTENSIONS) { |
| 114 | | lastUpdateTypes[ext] = new Date("2025-05-13 13:58:00"); |
| 115 | | } |
| 116 | | lastUpdateTypes[".diff"] = new Date("2025-05-18 13:58:00"); |
| 117 | | lastUpdateTypes[".patch"] = new Date("2025-05-18 13:58:00"); |
| 118 | | |
| 119 | | // Helper functions for metadata extraction |
| 120 | | async function calculateHash(filePath: string): Promise<string> { |
| 121 | | try { |
| 122 | | const hash = await execFileAsync("sha1sum", [filePath]); |
| 123 | | return hash.stdout.split(" ")[0]; |
| 124 | | } catch (error) { |
| 125 | | console.error(`Error calculating hash for ${filePath}:`, error); |
| 126 | | throw error; |
| 127 | | } |
| 128 | | } |
| 129 | | |
| 130 | | async function calculateDuration(filePath: string): Promise<number> { |
| 131 | | try { |
| 132 | | const ext = path.extname(filePath).toLowerCase(); |
| 133 | | if (!MEDIA_EXTENSIONS.has(ext)) return 0; |
| 134 | | |
| 135 | | const { stdout } = await execFileAsync("ffprobe", [ |
| 136 | | "-v", |
| 137 | | "error", |
| 138 | | "-show_entries", |
| 139 | | "format=duration", |
| 140 | | "-of", |
| 141 | | "default=noprint_wrappers=1:nokey=1", |
| 142 | | filePath, |
| 143 | | ]); |
| 144 | | return Math.ceil(parseFloat(stdout.trim())); |
| 145 | | } catch (error) { |
| 146 | | console.error(`Error calculating duration for ${filePath}:`, error); |
| 147 | | return 0; // Return 0 for duration on error |
| 148 | | } |
| 149 | | } |
| 150 | | |
| 151 | | async function calculateDimensions(filePath: string): Promise<string> { |
| 152 | | const ext = path.extname(filePath).toLowerCase(); |
| 153 | | if (!IMAGE_EXTENSIONS.has(ext) && !VIDEO_EXTENSIONS.has(ext)) return ""; |
| 154 | | |
| 155 | | try { |
| 156 | | if (ext === ".svg") { |
| 157 | | // For SVG files, parse the file and extract width/height |
| 158 | | const content = await fsp.readFile(filePath, "utf8"); |
| 159 | | const widthMatch = content.match(/width="(\d+)"/); |
| 160 | | const heightMatch = content.match(/height="(\d+)"/); |
| 161 | | |
| 162 | | if (widthMatch && heightMatch) { |
| 163 | | return `${widthMatch[1]}x${heightMatch[1]}`; |
| 164 | | } |
| 165 | | } else if (IMAGE_EXTENSIONS.has(ext) || VIDEO_EXTENSIONS.has(ext)) { |
| 166 | | // Use ffprobe for images and videos |
| 167 | | const { stdout } = await execFileAsync("ffprobe", [ |
| 168 | | "-v", |
| 169 | | "error", |
| 170 | | "-select_streams", |
| 171 | | "v:0", |
| 172 | | "-show_entries", |
| 173 | | "stream=width,height", |
| 174 | | "-of", |
| 175 | | "csv=s=x:p=0", |
| 176 | | filePath, |
| 177 | | ]); |
| 178 | | return stdout.trim(); |
| 179 | | } |
| 180 | | } catch (error) { |
| 181 | | console.error(`Error calculating dimensions for ${filePath}:`, error); |
| 182 | | } |
| 183 | | |
| 184 | | return ""; |
| 185 | | } |
| 186 | | |
| 187 | | // Helper function to check and remove location metadata |
| 188 | | async function scrubLocationMetadata( |
| 189 | | filePath: string, |
| 190 | | stats: Stats, |
| 191 | | ): Promise<boolean> { |
| 192 | | try { |
| 193 | | const ext = path.extname(filePath).toLowerCase(); |
| 194 | | if (!SCRUB_EXTENSIONS.has(ext)) return false; |
| 195 | | |
| 196 | | let hasLocation = false; |
| 197 | | let args: string[] = []; |
| 198 | | |
| 199 | | // Check for location metadata based on file type |
| 200 | | const tempOutput = path.join( |
| 201 | | path.dirname(filePath), |
| 202 | | `.tmp.${path.basename(filePath)}`, |
| 203 | | ); |
| 204 | | switch (ext) { |
| 205 | | case ".jpg": |
| 206 | | case ".jpeg": |
| 207 | | case ".png": |
| 208 | | // Check for GPS tags in EXIF |
| 209 | | const { stdout: gpsCheck } = await execFileAsync("exiftool", [ |
| 210 | | "-gps:all", |
| 211 | | filePath, |
| 212 | | ]); |
| 213 | | hasLocation = gpsCheck.trim().length > 0; |
| 214 | | args = ["-gps:all=", filePath, "-o", tempOutput]; |
| 215 | | break; |
| 216 | | case ".mov": |
| 217 | | case ".mp4": |
| 218 | | // Check for GPS metadata in video files |
| 219 | | const { stdout: videoCheck } = await execFileAsync("exiftool", [ |
| 220 | | "-ee", |
| 221 | | "-G3", |
| 222 | | "-s", |
| 223 | | filePath, |
| 224 | | ]); |
| 225 | | hasLocation = videoCheck.includes("GPS") || |
| 226 | | videoCheck.includes("Location"); |
| 227 | | args = ["-gps:all=", "-xmp:all=", filePath, "-o", tempOutput]; |
| 228 | | break; |
| 229 | | case ".m4a": |
| 230 | | // Check for location and other metadata in m4a files |
| 231 | | const { stdout: m4aCheck } = await execFileAsync("exiftool", [ |
| 232 | | "-ee", |
| 233 | | "-G3", |
| 234 | | "-s", |
| 235 | | filePath, |
| 236 | | ]); |
| 237 | | hasLocation = m4aCheck.includes("GPS") || |
| 238 | | m4aCheck.includes("Location") || |
| 239 | | m4aCheck.includes("Filename") || |
| 240 | | m4aCheck.includes("Title"); |
| 241 | | |
| 242 | | if (hasLocation) { |
| 243 | | args = [ |
| 244 | | "-gps:all=", |
| 245 | | "-location:all=", |
| 246 | | "-filename:all=", |
| 247 | | "-title=", |
| 248 | | "-m4a:all=", |
| 249 | | filePath, |
| 250 | | "-o", |
| 251 | | tempOutput, |
| 252 | | ]; |
| 253 | | } |
| 254 | | break; |
| 255 | | } |
| 256 | | |
| 257 | | const accessTime = stats.atime; |
| 258 | | const modTime = stats.mtime; |
| 259 | | |
| 260 | | let backup: string | null = null; |
| 261 | | try { |
| 262 | | if (hasLocation) { |
| 263 | | if (DRY_RUN) return true; |
| 264 | | |
| 265 | | // Prepare a backup |
| 266 | | const tmp = path.join( |
| 267 | | path.dirname(filePath), |
| 268 | | `.tmp.backup.${path.basename(filePath)}`, |
| 269 | | ); |
| 270 | | await fsp.copyFile(filePath, tmp); |
| 271 | | await fsp.utimes(tmp, accessTime, modTime); |
| 272 | | backup = tmp; |
| 273 | | |
| 274 | | // Remove metadata |
| 275 | | await execFileAsync("exiftool", args); |
| 276 | | if (!existsSync(tempOutput)) { |
| 277 | | throw new Error(`Failed to create output file: ${tempOutput}`); |
| 278 | | } |
| 279 | | |
| 280 | | // Restore original timestamps |
| 281 | | await fsp.rename(tempOutput, filePath); |
| 282 | | await fsp.utimes(filePath, accessTime, modTime); |
| 283 | | |
| 284 | | // Backup is no longer needed |
| 285 | | await fsp.unlink(backup); |
| 286 | | |
| 287 | | log( |
| 288 | | `Scrubbed location metadata in ${path.relative(LOCAL_DIR, filePath)}`, |
| 289 | | true, |
| 290 | | ); |
| 291 | | return true; |
| 292 | | } |
| 293 | | } catch (error) { |
| 294 | | if (backup) { |
| 295 | | await fsp.rename(backup, filePath); |
| 296 | | } |
| 297 | | if (existsSync(tempOutput)) { |
| 298 | | await fsp.unlink(tempOutput); |
| 299 | | } |
| 300 | | throw error; |
| 301 | | } |
| 302 | | } catch (error) { |
| 303 | | console.error(`Error scrubbing metadata for ${filePath}:`, error); |
| 304 | | } |
| 305 | | |
| 306 | | return false; |
| 307 | | } |
| 308 | | |
| 309 | | // Queue implementation for parallel processing |
| 310 | | type AsyncQueueProcessor<T> = (s: Spinner, item: T) => Promise<void>; |
| 311 | | class AsyncQueue<T> { |
| 312 | | private queue: T[] = []; |
| 313 | | private running = 0; |
| 314 | | private maxConcurrent: number; |
| 315 | | private processed = 0; |
| 316 | | private progress?: Progress<{ active: Spinner[] }>; |
| 317 | | private name: string; |
| 318 | | private estimate?: number; |
| 319 | | |
| 320 | | constructor(name: string, maxConcurrent: number) { |
| 321 | | this.maxConcurrent = maxConcurrent; |
| 322 | | this.name = name; |
| 323 | | } |
| 324 | | |
| 325 | | setEstimate(estimate: number) { |
| 326 | | this.estimate = estimate; |
| 327 | | if (this.progress) { |
| 328 | | this.progress.total = Math.max( |
| 329 | | this.processed + this.queue.length, |
| 330 | | estimate, |
| 331 | | ); |
| 332 | | } |
| 333 | | } |
| 334 | | |
| 335 | | getProgress() { |
| 336 | | if (!this.progress) { |
| 337 | | this.progress = new Progress({ |
| 338 | | spinner: null, |
| 339 | | text: ({ active }) => { |
| 340 | | const now = performance.now(); |
| 341 | | let text = `[${this.processed}/${ |
| 342 | | this.processed + this.queue.length |
| 343 | | }] ${this.name}`; |
| 344 | | let n = 0; |
| 345 | | for (const item of active) { |
| 346 | | let itemText = "- " + item.format(now); |
| 347 | | text += `\n` + |
| 348 | | itemText.slice(0, Math.max(0, process.stdout.columns - 1)); |
| 349 | | if (n > 10) { |
| 350 | | text += `\n ... + ${active.length - n} more`; |
| 351 | | break; |
| 352 | | } |
| 353 | | n++; |
| 354 | | } |
| 355 | | return text; |
| 356 | | }, |
| 357 | | props: { |
| 358 | | active: [] as Spinner[], |
| 359 | | }, |
| 360 | | }); |
| 361 | | this.progress.total = this.estimate ?? 0; |
| 362 | | this.progress.value = 0; |
| 363 | | this.progress.fps = 30; |
| 364 | | } |
| 365 | | return this.progress; |
| 366 | | } |
| 367 | | |
| 368 | | async add(item: T, processor: AsyncQueueProcessor<T>): Promise<void> { |
| 369 | | this.queue.push(item); |
| 370 | | this.getProgress().total = Math.max( |
| 371 | | this.processed + this.queue.length, |
| 372 | | this.estimate ?? 0, |
| 373 | | ); |
| 374 | | return this.processNext(processor); |
| 375 | | } |
| 376 | | |
| 377 | | async addBatch(items: T[], processor: AsyncQueueProcessor<T>): Promise<void> { |
| 378 | | this.queue.push(...items); |
| 379 | | this.getProgress().total = Math.max( |
| 380 | | this.processed + this.queue.length, |
| 381 | | this.estimate ?? 0, |
| 382 | | ); |
| 383 | | return this.processNext(processor); |
| 384 | | } |
| 385 | | |
| 386 | | private async processNext(processor: AsyncQueueProcessor<T>): Promise<void> { |
| 387 | | if (this.running >= this.maxConcurrent || this.queue.length === 0) { |
| 388 | | return; |
| 389 | | } |
| 390 | | |
| 391 | | const item = this.queue.shift(); |
| 392 | | if (!item) return; |
| 393 | | |
| 394 | | this.running++; |
| 395 | | |
| 396 | | try { |
| 397 | | const progress = this.getProgress(); |
| 398 | | |
| 399 | | let itemText = ""; |
| 400 | | if (typeof item === "string") { |
| 401 | | itemText = item; |
| 402 | | } else if (typeof item === "object" && item !== null && "path" in item) { |
| 403 | | itemText = "" + item.path; |
| 404 | | } else { |
| 405 | | itemText = JSON.stringify(item); |
| 406 | | } |
| 407 | | if (itemText.startsWith(LOCAL_DIR)) { |
| 408 | | itemText = path.relative(LOCAL_DIR, itemText); |
| 409 | | } |
| 410 | | |
| 411 | | const spinner = new Spinner(itemText); |
| 412 | | spinner.stop(); |
| 413 | | progress.props.active.unshift(spinner); |
| 414 | | await processor(spinner, item); |
| 415 | | progress.props = { |
| 416 | | active: progress.props.active.filter((s) => s !== spinner), |
| 417 | | }; |
| 418 | | this.processed++; |
| 419 | | progress.value = this.processed; |
| 420 | | } catch (error) { |
| 421 | | console.error(`Error processing ${this.name} queue item:`, error); |
| 422 | | this.processed++; |
| 423 | | this.getProgress().value = this.processed; |
| 424 | | } finally { |
| 425 | | this.running--; |
| 426 | | await this.processNext(processor); |
| 427 | | } |
| 428 | | } |
| 429 | | |
| 430 | | async waitForCompletion(): Promise<void> { |
| 431 | | if (this.queue.length === 0 && this.running === 0) { |
| 432 | | if (this.processed > 0) { |
| 433 | | this.#success(); |
| 434 | | } |
| 435 | | return; |
| 436 | | } |
| 437 | | |
| 438 | | return new Promise((resolve) => { |
| 439 | | const checkInterval = setInterval(() => { |
| 440 | | if (this.queue.length === 0 && this.running === 0) { |
| 441 | | clearInterval(checkInterval); |
| 442 | | this.#success(); |
| 443 | | resolve(); |
| 444 | | } |
| 445 | | }, 100); |
| 446 | | }); |
| 447 | | } |
| 448 | | |
| 449 | | #success() { |
| 450 | | this.getProgress().success(`${this.processed} ${this.name}`); |
| 451 | | } |
| 452 | | } |
| 453 | | |
| 454 | | function skipBasename(basename: string): boolean { |
| 455 | | // dot files must be incrementally tracked |
| 456 | | if (basename === ".dirsort") return true; |
| 457 | | if (basename === ".friends") return true; |
| 458 | | |
| 459 | | return ( |
| 460 | | basename.startsWith(".") || |
| 461 | | basename.startsWith("._") || |
| 462 | | basename.startsWith(".tmp") || |
| 463 | | basename === ".DS_Store" || |
| 464 | | basename.toLowerCase() === "thumbs.db" || |
| 465 | | basename.toLowerCase() === "desktop.ini" |
| 466 | | ); |
| 467 | | } |
| 468 | | |
| 469 | | // File system scanner |
| 470 | | class FileSystemScanner { |
| 471 | | private visitedPaths = new Set<string>(); |
| 472 | | private previousPaths = new Set<string>(); |
| 473 | | private dirQueue = new AsyncQueue<string>("Scan Directories", 10); |
| 474 | | private fileQueue = new AsyncQueue<{ path: string; stat: any }>( |
| 475 | | "File metadata", |
| 476 | | 20, |
| 477 | | ); |
| 478 | | private compressQueue: AsyncQueue<{ file: MediaFile; path: string }> | null = |
| 479 | | SHOULD_COMPRESS ? new AsyncQueue("Compress Assets", 10) : null; |
| 480 | | |
| 481 | | private getDbPath(localPath: string): string { |
| 482 | | // Convert local file system path to database path |
| 483 | | const relativePath = path.relative(LOCAL_DIR, localPath); |
| 484 | | return "/" + relativePath.split(path.sep).join(path.posix.sep); |
| 485 | | } |
| 486 | | |
| 487 | | private getLocalPath(dbPath: string): string { |
| 488 | | // Convert database path to local file system path |
| 489 | | return path.join(LOCAL_DIR, dbPath.slice(1)); |
| 490 | | } |
| 491 | | |
| 492 | | async scanFile(s: Spinner, filePath: string, stat: any): Promise<void> { |
| 493 | | const dbPath = this.getDbPath(filePath); |
| 494 | | |
| 495 | | // Skip hidden files |
| 496 | | const basename = path.basename(filePath); |
| 497 | | if (skipBasename(basename)) { |
| 498 | | return; |
| 499 | | } |
| 500 | | |
| 501 | | this.visitedPaths.add(dbPath); |
| 502 | | |
| 503 | | // Get existing file info from db |
| 504 | | const existingFile = MediaFile.getByPath(dbPath); |
| 505 | | |
| 506 | | // Determine which date to use (for date protection) |
| 507 | | let dateToUse = stat.mtime; |
| 508 | | const year2025Start = new Date("2025-01-01T00:00:00Z"); |
| 509 | | |
| 510 | | if ( |
| 511 | | existingFile && |
| 512 | | existingFile.date < year2025Start && |
| 513 | | stat.mtime >= year2025Start |
| 514 | | ) { |
| 515 | | console.error( |
| 516 | | `Error: ${dbPath} is ${ |
| 517 | | formatDate( |
| 518 | | existingFile.date, |
| 519 | | ) |
| 520 | | }, got modified to ${formatDate(stat.mtime)}`, |
| 521 | | ); |
| 522 | | dateToUse = existingFile.date; |
| 523 | | } |
| 524 | | |
| 525 | | // Check if we need to reprocess the file |
| 526 | | if (existingFile && existingFile.size === stat.size && existingFile.hash) { |
| 527 | | maybe_skip: { |
| 528 | | const lastUpdateDate = lastUpdateTypes[path.extname(filePath)]; |
| 529 | | if (lastUpdateDate && existingFile.lastUpdateDate < lastUpdateDate) { |
| 530 | | console.log( |
| 531 | | `Reprocessing ${dbPath} because indexing logic changed after ${ |
| 532 | | formatDate( |
| 533 | | lastUpdateDate, |
| 534 | | ) |
| 535 | | }`, |
| 536 | | ); |
| 537 | | break maybe_skip; |
| 538 | | } |
| 539 | | |
| 540 | | if (SHOULD_COMPRESS && existingFile.processed !== 2) { |
| 541 | | this.compressQueue!.add( |
| 542 | | { file: existingFile, path: dbPath }, |
| 543 | | this.compressFile.bind(this), |
| 544 | | ); |
| 545 | | } |
| 546 | | |
| 547 | | // File hasn't changed, no need to reprocess |
| 548 | | MediaFile.createFile({ |
| 549 | | path: dbPath, |
| 550 | | date: dateToUse, |
| 551 | | hash: existingFile.hash, |
| 552 | | size: stat.size, |
| 553 | | duration: existingFile.duration, |
| 554 | | dimensions: existingFile.dimensions, |
| 555 | | content: existingFile.contents, |
| 556 | | }); |
| 557 | | return; |
| 558 | | } |
| 559 | | } |
| 560 | | |
| 561 | | // Process the file |
| 562 | | log(`Processing file: ${dbPath}`); |
| 563 | | |
| 564 | | // Scrub location metadata if needed |
| 565 | | if (SHOULD_SCRUB) { |
| 566 | | if (await scrubLocationMetadata(filePath, stat)) { |
| 567 | | // Re-stat the file in case it was modified |
| 568 | | const newStat = await fsp.stat(filePath); |
| 569 | | stat.size = newStat.size; |
| 570 | | } |
| 571 | | } |
| 572 | | |
| 573 | | // Extract content |
| 574 | | const hash = await calculateHash(filePath); |
| 575 | | let content = ""; |
| 576 | | if (filePath.endsWith(".lnk")) { |
| 577 | | content = (await fsp.readFile(filePath, "utf8")).trim(); |
| 578 | | } |
| 579 | | const language = CODE_EXTENSIONS[path.extname(filePath)]; |
| 580 | | if (language) { |
| 581 | | read_code: { |
| 582 | | // An issue is that .ts is an overloaded extension, shared between |
| 583 | | // 'transport stream' and 'typescript'. |
| 584 | | // |
| 585 | | // Filter used here is: |
| 586 | | // - more than 1mb |
| 587 | | // - invalid UTF-8 |
| 588 | | if (stat.size > 1_000_000) break read_code; |
| 589 | | let code; |
| 590 | | const buf = await fsp.readFile(filePath); |
| 591 | | try { |
| 592 | | code = new TextDecoder("utf-8", { fatal: true }).decode(buf); |
| 593 | | } catch (error) { |
| 594 | | break read_code; |
| 595 | | } |
| 596 | | content = await highlightCode(code, language); |
| 597 | | } |
| 598 | | } |
| 599 | | if (!content && READ_CONTENTS_EXTENSIONS.has(path.extname(filePath))) { |
| 600 | | content = await fsp.readFile(filePath, "utf8"); |
| 601 | | } |
| 602 | | // End extract content |
| 603 | | |
| 604 | | if (hash === existingFile?.hash) { |
| 605 | | MediaFile.createFile({ |
| 606 | | path: dbPath, |
| 607 | | date: dateToUse, |
| 608 | | hash, |
| 609 | | size: stat.size, |
| 610 | | duration: existingFile.duration, |
| 611 | | dimensions: existingFile.dimensions, |
| 612 | | content, |
| 613 | | }); |
| 614 | | return; |
| 615 | | } else if (existingFile) { |
| 616 | | if (existingFile.processed === 2) { |
| 617 | | if (BlobAsset.decrementOrDelete(existingFile.hash)) { |
| 618 | | log( |
| 619 | | `Deleted compressed asset ${existingFile.hash}.{gzip, zstd}`, |
| 620 | | true, |
| 621 | | ); |
| 622 | | await fsp.unlink( |
| 623 | | path.join( |
| 624 | | COMPRESS_STORE, |
| 625 | | existingFile.hash.substring(0, 2), |
| 626 | | existingFile.hash + ".gz", |
| 627 | | ), |
| 628 | | ); |
| 629 | | await fsp.unlink( |
| 630 | | path.join( |
| 631 | | COMPRESS_STORE, |
| 632 | | existingFile.hash.substring(0, 2), |
| 633 | | existingFile.hash + ".zstd", |
| 634 | | ), |
| 635 | | ); |
| 636 | | } |
| 637 | | } |
| 638 | | } |
| 639 | | const [duration, dimensions] = await Promise.all([ |
| 640 | | calculateDuration(filePath), |
| 641 | | calculateDimensions(filePath), |
| 642 | | ]); |
| 643 | | |
| 644 | | // Update database with all metadata |
| 645 | | MediaFile.createFile({ |
| 646 | | path: dbPath, |
| 647 | | date: dateToUse, |
| 648 | | hash, |
| 649 | | size: stat.size, |
| 650 | | duration, |
| 651 | | dimensions, |
| 652 | | content, |
| 653 | | }); |
| 654 | | |
| 655 | | if (SHOULD_COMPRESS) { |
| 656 | | this.compressQueue!.add( |
| 657 | | { |
| 658 | | file: MediaFile.getByPath(dbPath)!, |
| 659 | | path: dbPath, |
| 660 | | }, |
| 661 | | this.compressFile.bind(this), |
| 662 | | ); |
| 663 | | } |
| 664 | | } |
| 665 | | |
| 666 | | async compressFile(s: Spinner, { file }: { file: MediaFile }): Promise<void> { |
| 667 | | log(`Compressing file: ${file.path}`); |
| 668 | | if (DRY_RUN) return; |
| 669 | | |
| 670 | | const filePath = path.join(FILE_ROOT!, file.path); |
| 671 | | |
| 672 | | const hash = file.hash; |
| 673 | | const firstTwoChars = hash.substring(0, 2); |
| 674 | | const compressDir = `${COMPRESS_STORE}/${firstTwoChars}`; |
| 675 | | const compressPath = `${compressDir}/${hash}`; |
| 676 | | |
| 677 | | // Create directory structure if it doesn't exist |
| 678 | | await fsp.mkdir(compressDir, { recursive: true }); |
| 679 | | |
| 680 | | // Compress the file with gzip |
| 681 | | const blob = BlobAsset.putOrIncrement(hash); |
| 682 | | if (blob.refs > 1) { |
| 683 | | log( |
| 684 | | `Skipping compression of ${filePath} because it already exists in ${compressPath}`, |
| 685 | | ); |
| 686 | | return; |
| 687 | | } |
| 688 | | // Check if already exists |
| 689 | | if (existsSync(compressPath + ".gz")) { |
| 690 | | file.setCompressed(true); |
| 691 | | return; |
| 692 | | } |
| 693 | | try { |
| 694 | | const gzipProcess = Bun.spawn(["gzip", "-c", filePath, "-9"], { |
| 695 | | stdout: Bun.file(compressPath + ".gz"), |
| 696 | | }); |
| 697 | | const zstdProcess = Bun.spawn(["zstd", "-c", filePath, "-9"], { |
| 698 | | stdout: Bun.file(compressPath + ".zstd"), |
| 699 | | }); |
| 700 | | const [gzipExited, zstdExited] = await Promise.all([ |
| 701 | | gzipProcess.exited, |
| 702 | | zstdProcess.exited, |
| 703 | | ]); |
| 704 | | assert(gzipExited === 0); |
| 705 | | assert(zstdExited === 0); |
| 706 | | assert(existsSync(compressPath + ".gz")); |
| 707 | | assert(existsSync(compressPath + ".zstd")); |
| 708 | | file.setCompressed(true); |
| 709 | | } catch (error) { |
| 710 | | console.error(`Error compressing file ${filePath}:`, error); |
| 711 | | BlobAsset.decrementOrDelete(hash); |
| 712 | | file.setCompressed(false); |
| 713 | | } |
| 714 | | } |
| 715 | | |
| 716 | | async scanDirectory(s: Spinner, dirPath: string): Promise<void> { |
| 717 | | const dbPath = this.getDbPath(dirPath); |
| 718 | | |
| 719 | | this.visitedPaths.add(dbPath); |
| 720 | | |
| 721 | | // Create or update directory entry |
| 722 | | log(`Scanning directory: ${dbPath}`); |
| 723 | | if (!DRY_RUN) { |
| 724 | | MediaFile.createOrUpdateDirectory(dbPath); |
| 725 | | } |
| 726 | | |
| 727 | | try { |
| 728 | | const entries = await fsp.readdir(dirPath, { withFileTypes: true }); |
| 729 | | |
| 730 | | // Process files and subdirectories |
| 731 | | for (const entry of entries) { |
| 732 | | const entryPath = path.join(dirPath, entry.name); |
| 733 | | |
| 734 | | // Skip hidden files and system files |
| 735 | | if (skipBasename(entry.name)) { |
| 736 | | continue; |
| 737 | | } |
| 738 | | |
| 739 | | if (entry.isDirectory()) { |
| 740 | | // Queue subdirectory for scanning |
| 741 | | this.dirQueue.add(entryPath, this.scanDirectory.bind(this)); |
| 742 | | } else if (entry.isFile()) { |
| 743 | | // Queue file for processing |
| 744 | | const stat = await fsp.stat(entryPath); |
| 745 | | |
| 746 | | this.fileQueue.add( |
| 747 | | { path: entryPath, stat }, |
| 748 | | async (s, item) => await this.scanFile(s, item.path, item.stat), |
| 749 | | ); |
| 750 | | } |
| 751 | | } |
| 752 | | } catch (error) { |
| 753 | | console.error(`Error scanning directory ${dirPath}:`, error); |
| 754 | | } |
| 755 | | } |
| 756 | | |
| 757 | | async processDirectoryMetadata(dirPath: string): Promise<void> { |
| 758 | | const dbPath = this.getDbPath(dirPath); |
| 759 | | const dir = MediaFile.getByPath(dbPath); |
| 760 | | |
| 761 | | if (!dir || dir.kind !== MediaFile.Kind.directory) { |
| 762 | | return; |
| 763 | | } |
| 764 | | |
| 765 | | if (DRY_RUN) return; |
| 766 | | |
| 767 | | const children = dir.getChildren(); |
| 768 | | |
| 769 | | // Calculate directory metadata |
| 770 | | let totalSize = 0; |
| 771 | | let newestDate = new Date(0); |
| 772 | | let allHashes = ""; |
| 773 | | |
| 774 | | // Check for readme.txt |
| 775 | | let readmeContent = ""; |
| 776 | | |
| 777 | | try { |
| 778 | | readmeContent = await fsp.readFile( |
| 779 | | path.join(dirPath, "readme.txt"), |
| 780 | | "utf8", |
| 781 | | ); |
| 782 | | } catch (error: any) { |
| 783 | | console.info(`no readme ${dirPath}`); |
| 784 | | if (error.code !== "ENOENT") { |
| 785 | | console.error(`Error reading readme.txt in ${dirPath}:`, error); |
| 786 | | } |
| 787 | | } |
| 788 | | |
| 789 | | let dirsort: string[] | null = null; |
| 790 | | try { |
| 791 | | dirsort = (await fsp.readFile(path.join(dirPath, ".dirsort"), "utf8")) |
| 792 | | .split("\n") |
| 793 | | .map((x) => x.trim()) |
| 794 | | .filter(Boolean); |
| 795 | | } catch (error: any) { |
| 796 | | if (error.code !== "ENOENT") { |
| 797 | | console.error(`Error reading .dirsort in ${dirPath}:`, error); |
| 798 | | } |
| 799 | | } |
| 800 | | |
| 801 | | if (await fsp.exists(path.join(dirPath, ".friends"))) { |
| 802 | | FilePermissions.setPermissions(dbPath, 1); |
| 803 | | } else { |
| 804 | | FilePermissions.setPermissions(dbPath, 0); |
| 805 | | } |
| 806 | | |
| 807 | | // Process children |
| 808 | | for (const child of children) { |
| 809 | | totalSize += child.size; |
| 810 | | allHashes += child.hash; |
| 811 | | |
| 812 | | // Update newest date, ignoring readme.txt |
| 813 | | if (!child.path.endsWith("/readme.txt") && child.date > newestDate) { |
| 814 | | newestDate = child.date; |
| 815 | | } |
| 816 | | } |
| 817 | | |
| 818 | | // Create a hash for the directory |
| 819 | | const dirHash = new Bun.CryptoHasher("sha1") |
| 820 | | .update(dbPath + allHashes) |
| 821 | | .digest("hex"); |
| 822 | | |
| 823 | | // Update directory metadata |
| 824 | | MediaFile.markDirectoryProcessed({ |
| 825 | | id: dir.id, |
| 826 | | timestamp: newestDate, |
| 827 | | contents: readmeContent, |
| 828 | | size: totalSize, |
| 829 | | hash: dirHash, |
| 830 | | dirsort, |
| 831 | | }); |
| 832 | | } |
| 833 | | |
| 834 | | async findDeletedFiles(): Promise<void> { |
| 835 | | if (DRY_RUN) return; |
| 836 | | |
| 837 | | // Find all paths that exist in the DB but not in the filesystem |
| 838 | | const deletedPaths = Array.from(this.previousPaths).filter( |
| 839 | | (path) => !this.visitedPaths.has(path), |
| 840 | | ); |
| 841 | | |
| 842 | | for (const dbPath of deletedPaths) { |
| 843 | | const file = MediaFile.getByPath(dbPath); |
| 844 | | if (!file) continue; |
| 845 | | |
| 846 | | log(`Item Deleted: ${dbPath}`, true); |
| 847 | | if (file.processed === 2) { |
| 848 | | if (BlobAsset.decrementOrDelete(file.hash)) { |
| 849 | | log(`Deleted compressed asset ${file.hash}.{gzip, zstd}`, true); |
| 850 | | await fsp.unlink( |
| 851 | | path.join( |
| 852 | | COMPRESS_STORE, |
| 853 | | file.hash.substring(0, 2), |
| 854 | | file.hash + ".gz", |
| 855 | | ), |
| 856 | | ); |
| 857 | | await fsp.unlink( |
| 858 | | path.join( |
| 859 | | COMPRESS_STORE, |
| 860 | | file.hash.substring(0, 2), |
| 861 | | file.hash + ".zstd", |
| 862 | | ), |
| 863 | | ); |
| 864 | | } |
| 865 | | } |
| 866 | | MediaFile.deleteByPath(dbPath); |
| 867 | | } |
| 868 | | } |
| 869 | | |
| 870 | | async loadPreviousPaths(): Promise<void> { |
| 871 | | // Get all files and directories from the database |
| 872 | | // This uses a custom query to get all paths at once |
| 873 | | const getAllPathsQuery = cache |
| 874 | | .prepare(`SELECT path, kind FROM media_files`) |
| 875 | | .all() as { |
| 876 | | path: string; |
| 877 | | kind: MediaFile.Kind; |
| 878 | | }[]; |
| 879 | | |
| 880 | | let dirs = 0; |
| 881 | | let files = 0; |
| 882 | | for (const row of getAllPathsQuery) { |
| 883 | | this.previousPaths.add(row.path); |
| 884 | | if (row.kind === MediaFile.Kind.directory) { |
| 885 | | dirs++; |
| 886 | | } else { |
| 887 | | files++; |
| 888 | | } |
| 889 | | } |
| 890 | | |
| 891 | | this.dirQueue.setEstimate(dirs); |
| 892 | | this.fileQueue.setEstimate(files); |
| 893 | | |
| 894 | | // log(`Loaded ${this.previousPaths.size} paths from database`, true); |
| 895 | | } |
| 896 | | |
| 897 | | async scan(): Promise<void> { |
| 898 | | log(`Starting file system scan in ${LOCAL_DIR}`, true); |
| 899 | | |
| 900 | | // Check if the root directory exists and is accessible |
| 901 | | try { |
| 902 | | const rootStat = await fsp.stat(LOCAL_DIR); |
| 903 | | if (!rootStat.isDirectory()) { |
| 904 | | throw new Error(`${LOCAL_DIR} is not a directory`); |
| 905 | | } |
| 906 | | } catch (error) { |
| 907 | | console.error(`Error: Cannot access root directory ${LOCAL_DIR}`, error); |
| 908 | | console.error( |
| 909 | | `Aborting scan to prevent database corruption. Please check if the volume is mounted.`, |
| 910 | | ); |
| 911 | | process.exit(1); |
| 912 | | } |
| 913 | | |
| 914 | | await this.loadPreviousPaths(); |
| 915 | | |
| 916 | | await this.dirQueue.add(LOCAL_DIR, this.scanDirectory.bind(this)); |
| 917 | | |
| 918 | | await this.dirQueue.waitForCompletion(); |
| 919 | | await this.fileQueue.waitForCompletion(); |
| 920 | | |
| 921 | | await this.findDeletedFiles(); |
| 922 | | |
| 923 | | const allDirs = Array.from(this.visitedPaths) |
| 924 | | .filter((path) => { |
| 925 | | const file = MediaFile.getByPath(path); |
| 926 | | return file && file.kind === MediaFile.Kind.directory; |
| 927 | | }) |
| 928 | | .sort((a, b) => b.length - a.length); |
| 929 | | |
| 930 | | const dirMetadataQueue = new AsyncQueue<string>("Directory Metadata", 10); |
| 931 | | for (const dirPath of allDirs) { |
| 932 | | await this.processDirectoryMetadata(this.getLocalPath(dirPath)); |
| 933 | | } |
| 934 | | |
| 935 | | await dirMetadataQueue.waitForCompletion(); |
| 936 | | |
| 937 | | if (SHOULD_COMPRESS) { |
| 938 | | await this.compressQueue!.waitForCompletion(); |
| 939 | | } |
| 940 | | |
| 941 | | log("Scan completed successfully!", true); |
| 942 | | } |
| 943 | | } |
| 944 | | |
| 945 | | // Main execution |
| 946 | | function showHelp() { |
| 947 | | console.log(` |
| 948 | | MediaFile Scanner - Index filesystem content for paperclover.net |
| 949 | | |
| 950 | | Environment variables: |
| 951 | | FILE_ROOT Required. Path to the directory to scan |
| 952 | | COMPRESS_STORE Optional. Path to store compressed files (default: .clover/compressed) |
| 953 | | |
| 954 | | Options: |
| 955 | | --help Show this help message |
| 956 | | --dry-run Don't make any changes to the database |
| 957 | | --verbose Show detailed output |
| 958 | | |
| 959 | | Usage: |
| 960 | | bun ./media/scan.ts [options] |
| 961 | | |
| 962 | | `); |
| 963 | | process.exit(0); |
| 964 | | } |
| 965 | | |
| 966 | | { |
| 967 | | // Show help if requested |
| 968 | | if (process.argv.includes("--help")) { |
| 969 | | showHelp(); |
| 970 | | process.exit(0); |
| 971 | | } |
| 972 | | |
| 973 | | // Check if the root directory exists before starting |
| 974 | | if (!existsSync(LOCAL_DIR)) { |
| 975 | | console.error( |
| 976 | | `Error: Root directory ${LOCAL_DIR} does not exist or is not accessible.`, |
| 977 | | ); |
| 978 | | console.error(`Please check if the volume is mounted correctly.`); |
| 979 | | process.exit(1); |
| 980 | | } |
| 981 | | |
| 982 | | const startTime = Date.now(); |
| 983 | | |
| 984 | | try { |
| 985 | | const scanner = new FileSystemScanner(); |
| 986 | | await scanner.scan(); |
| 987 | | |
| 988 | | const endTime = Date.now(); |
| 989 | | log(`Scan completed in ${(endTime - startTime) / 1000} seconds`, true); |
| 990 | | |
| 991 | | const rootDir = MediaFile.getByPath("/")!; |
| 992 | | const totalEntries = cache |
| 993 | | .prepare(`SELECT COUNT(*) as count FROM media_files`) |
| 994 | | .get() as { count: number }; |
| 995 | | const totalDuration = cache |
| 996 | | .prepare(`SELECT SUM(duration) as duration FROM media_files`) |
| 997 | | .get() as { duration: number }; |
| 998 | | console.log(); |
| 999 | | console.log("Global Stats"); |
| 1000 | | console.log(` Entry count: ${totalEntries.count}`); |
| 1001 | | console.log(` Uncompressed size: ${formatSize(rootDir.size)}`); |
| 1002 | | console.log( |
| 1003 | | ` Total audio/video duration: ${ |
| 1004 | | ( |
| 1005 | | totalDuration.duration / |
| 1006 | | 60 / |
| 1007 | | 60 |
| 1008 | | ).toFixed(1) |
| 1009 | | } hours`, |
| 1010 | | ); |
| 1011 | | } catch (error) { |
| 1012 | | console.error("Error during scan:", error); |
| 1013 | | process.exit(1); |
| 1014 | | } |
| 1015 | | } |
clover caruso <git@paperclover.net> 2025-07-08 23:10:41-07:00