package library import ( "context" "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "sync" "time" ) // audioExts are the formats Sonda reads. var audioExts = map[string]bool{".mp3": true, ".flac": true} // unsupportedExts are audio formats Sonda does not read yet. They are counted, // never opened. Video containers such as .mpg or .mp4 are deliberately absent. var unsupportedExts = map[string]bool{ ".m4a": true, ".m4b": true, ".aac": true, ".alac": true, ".ogg": true, ".oga": true, ".opus": true, ".wav": true, ".wma": true, ".ape": true, ".wv": true, ".aif": true, ".aiff": true, ".mpc": true, ".mka": true, ".dsf": true, ".dff": true, } // systemDirs are housekeeping folders that NAS boxes and operating systems // create inside shares: thumbnails, recycle bins, indexes. What they hold is // never part of a library. Only the exact folder name matches, ignoring case. var systemDirs = []string{ ".@__thumb", "@Recycle", "#recycle", "@eaDir", "$RECYCLE.BIN", "System Volume Information", } func isSystemDir(name string) bool { for _, d := range systemDirs { if strings.EqualFold(name, d) { return true } } return false } // Scanner indexes the audio files of a set of libraries. // // Per library it walks the tree (cheap, stat only), decides what changed, then // reads tags with a pool of workers. A single goroutine writes to the store. type Scanner struct { store Store reader TagReader hub *Hub workers int now func() time.Time } // NewScanner builds a scanner that parses files with the given number of workers. func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner { if workers < 1 { workers = 1 } return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now} } // ScanOption changes how a scan runs. type ScanOption func(*scanOptions) type scanOptions struct{ full bool } // FullRescan makes a scan read the tags of every file again, not only of the files // that are new or changed. It is how what the reader learned since (MusicBrainz // ids, genres) reaches music that was indexed before. Nothing else changes: files // keep their identity, so listening history is kept. func FullRescan(o *scanOptions) { o.full = true } func buildOptions(opts []ScanOption) scanOptions { var o scanOptions for _, f := range opts { f(&o) } return o } // Scan indexes every library in turn. An unreadable library is reported in the // returned error and the other libraries are still scanned. func (s *Scanner) Scan(ctx context.Context, libs []Library, opts ...ScanOption) (Summary, error) { o := buildOptions(opts) total := Summary{Libraries: len(libs)} s.publish(Event{Type: ScanStarted}) var errs []error for _, lib := range libs { sum, err := s.scanLibrary(ctx, lib, o) total.add(sum) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err)) if ctx.Err() != nil { break } } } err := errors.Join(errs...) fin := Event{Type: ScanFinished, Summary: &total} if err != nil { fin.Error = err.Error() } s.publish(fin) return total, err } func (s *Scanner) publish(e Event) { e.Time = s.now() s.hub.Publish(e) } type parseJob struct { file FileInfo existingID int64 } type parseResult struct { job parseJob meta Metadata err error } func (s *Scanner) scanLibrary(ctx context.Context, lib Library, o scanOptions) (Summary, error) { var sum Summary walked, err := s.walk(ctx, lib, &sum) if err != nil { // The root itself is unreadable (for example an unmounted NAS). // Do not touch the database: nothing is known about its files. return sum, err } sum.Found = len(walked.files) existing, err := s.store.ExistingFiles(ctx, lib.ID) if err != nil { return sum, fmt.Errorf("list indexed files: %w", err) } // An unmounted NAS usually leaves an empty but readable folder. Treat // "nothing found although files are indexed" as a problem, not as a mass // deletion. Delete the library to really empty it. if len(walked.files) == 0 && len(walked.failedDirs) == 0 { if n := countPresent(existing); n > 0 { return sum, fmt.Errorf("directory is empty but %d indexed files exist: is the volume mounted? "+ "refusing to mark them missing", n) } } p := makePlan(walked, existing, o) sum.Unchanged = p.unchanged if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil { return sum, err } if err := ctx.Err(); err != nil { return sum, err // interrupted: never mark files missing on a partial scan } for _, r := range p.renames { if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil { s.fail(lib, r.newRel, err, &sum) continue } sum.Renamed++ } if len(p.restore) > 0 { if err := s.store.Restore(ctx, p.restore); err != nil { return sum, fmt.Errorf("restore files: %w", err) } sum.Restored = len(p.restore) } if len(p.missing) > 0 { if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil { return sum, fmt.Errorf("mark missing: %w", err) } sum.Missing = len(p.missing) } return sum, nil } func (s *Scanner) fail(lib Library, rel string, err error, sum *Summary) { sum.Failed++ s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error()}) } // parseAndSave reads tags in parallel and stores results from this goroutine. func (s *Scanner) parseAndSave(ctx context.Context, lib Library, jobs []parseJob, sum *Summary) error { if len(jobs) == 0 { return nil } jobCh := make(chan parseJob) resCh := make(chan parseResult) var wg sync.WaitGroup for i := 0; i < s.workers; i++ { wg.Add(1) go func() { defer wg.Done() for j := range jobCh { meta, err := s.reader.Read(filepath.Join(lib.Path, filepath.FromSlash(j.file.RelPath))) resCh <- parseResult{job: j, meta: meta, err: err} } }() } go func() { defer func() { close(jobCh) wg.Wait() close(resCh) }() for _, j := range jobs { select { case jobCh <- j: case <-ctx.Done(): return } } }() done := 0 for r := range resCh { // always drain, so no worker is left blocked if ctx.Err() != nil { continue } done++ rel := r.job.file.RelPath if r.err != nil { sum.Failed++ s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)}) continue } added, err := s.store.SaveFile(ctx, lib.ID, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID) if err != nil { sum.Failed++ s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)}) continue } if added { sum.Added++ } else { sum.Updated++ } s.publish(Event{Type: FileScanned, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Done: done, Total: len(jobs)}) } return ctx.Err() } // ---- walking ---- type walkResult struct { files []FileInfo // failedDirs are slash-separated relative paths (with a trailing slash) // that could not be read. Files below them are unknown, not gone. failedDirs []string // failedFiles are relative paths of files that could not be stat'ed for a // reason other than "does not exist". They are unknown, not gone. failedFiles map[string]bool } func (s *Scanner) walk(ctx context.Context, lib Library, sum *Summary) (walkResult, error) { var out walkResult // The library path may itself be a symlink (for example /srv/music -> // /mnt/nas/music). WalkDir would treat it as a plain file and find nothing, // so walk the resolved directory. Symlinks *inside* the root are still skipped. walkRoot, err := filepath.EvalSymlinks(lib.Path) if err != nil { return out, err } info, err := os.Stat(walkRoot) if err != nil { return out, err } if !info.IsDir() { return out, fmt.Errorf("%s is not a directory", lib.Path) } err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error { if cerr := ctx.Err(); cerr != nil { return cerr } if path == walkRoot { return err // nil, or the reason the root cannot be read } rel, rerr := filepath.Rel(walkRoot, path) if rerr != nil { return rerr } rel = filepath.ToSlash(rel) if err != nil { s.fail(lib, rel, err, sum) if d != nil && d.IsDir() { out.failedDirs = append(out.failedDirs, rel+"/") } else { out.markFileFailed(rel) } return nil // keep going, do not descend into an unreadable dir } if d.Type()&fs.ModeSymlink != 0 { sum.Skipped++ // never follow symlinks: they could point outside the root return nil } if d.IsDir() { if isSystemDir(d.Name()) { return filepath.SkipDir } return nil } ext := strings.ToLower(filepath.Ext(rel)) if !audioExts[ext] { if unsupportedExts[ext] { sum.Unsupported++ } return nil } if !d.Type().IsRegular() { // FIFOs, sockets and devices: opening one can block forever. sum.Skipped++ return nil } info, ierr := d.Info() if ierr != nil { s.fail(lib, rel, ierr, sum) if !errors.Is(ierr, fs.ErrNotExist) { out.markFileFailed(rel) } return nil } if !info.Mode().IsRegular() { // Replaced by a FIFO or symlink between the directory read and now. sum.Skipped++ return nil } out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}) return nil }) return out, err } func (w *walkResult) markFileFailed(rel string) { if w.failedFiles == nil { w.failedFiles = map[string]bool{} } w.failedFiles[rel] = true } // ---- planning ---- type renameOp struct { id int64 newRel string } type plan struct { parse []parseJob renames []renameOp restore []int64 missing []int64 unchanged int } type fileKey struct{ size, mtime int64 } // makePlan compares what is on disk with what is indexed. // // A new file and a vanished file with the same size and modification time are // treated as a rename only when each is the sole candidate for that pair, so // look-alike files are never guessed. func makePlan(w walkResult, existing []ExistingFile, o scanOptions) plan { var p plan byPath := make(map[string]ExistingFile, len(existing)) for _, e := range existing { byPath[e.RelPath] = e } seen := make(map[string]bool, len(w.files)) var added []FileInfo for _, f := range w.files { e, ok := byPath[f.RelPath] if !ok { added = append(added, f) continue } seen[f.RelPath] = true changed := e.Size != f.Size || e.MTimeUnix != f.MTimeUnix switch { case changed || o.full: p.parse = append(p.parse, parseJob{file: f, existingID: e.ID}) if e.Missing && !changed { // Back, and read again. It is back even if reading fails this time. p.restore = append(p.restore, e.ID) } case e.Missing: p.restore = append(p.restore, e.ID) default: p.unchanged++ } } var gone []ExistingFile for _, e := range existing { if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) { continue } gone = append(gone, e) } goneByKey := map[fileKey][]ExistingFile{} for _, e := range gone { k := fileKey{e.Size, e.MTimeUnix} goneByKey[k] = append(goneByKey[k], e) } addedByKey := map[fileKey]int{} for _, f := range added { addedByKey[fileKey{f.Size, f.MTimeUnix}]++ } renamed := map[int64]bool{} for _, f := range added { k := fileKey{f.Size, f.MTimeUnix} if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 { p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath}) renamed[cands[0].ID] = true continue } p.parse = append(p.parse, parseJob{file: f}) } for _, e := range gone { if !renamed[e.ID] && !e.Missing { p.missing = append(p.missing, e.ID) } } return p } // countPresent counts indexed files that were not already flagged missing. func countPresent(existing []ExistingFile) int { n := 0 for _, e := range existing { if !e.Missing { n++ } } return n } func underAny(rel string, prefixes []string) bool { for _, p := range prefixes { if strings.HasPrefix(rel, p) { return true } } return false }