sonda/internal/library/scanner.go
tgiachi b0a7a562a6 fix(discovery): purge lock, cleanup chains and rescans that cannot read
- fix: on PostgreSQL a purge could delete a discovered artist while a scan was
  adopting it. The scan then failed on the missing artist for the whole file,
  and the two could also deadlock. Purge now takes the same artist lock as the
  scanner and discovery before it removes anything. Saving a list or tags does
  too, and an artist that is gone by then is reported as not found instead of a
  foreign key error, so a user who opened an artist that was removed meanwhile
  gets a 404
- fix: a discovered artist that was only the source of a list stayed forever and
  kept the artists of its own list alive, so chains built up. Cleanup now removes
  such an artist and then follows the chain until nothing more goes
- fix: in a full rescan a file that came back but could not be read stayed
  marked missing. It is restored first, as in a normal scan, and read as well
- fix: ?full given twice is refused, so a typo cannot start a scan of another
  kind
- fix: genre cleaning lives in one place, used by the reader and the store
- docs: the files genres count the tracks the artist is credited on, and the spec
  says every files tag is a genre
- test: a purge colliding with a scan on both databases (it fails without the
  lock), a chain of discovered artists, saving for an artist that is gone, a
  file that returns unreadable during a full scan
2026-11-11 17:36:14 +01:00

457 lines
12 KiB
Go

package library
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
// audioExts are the formats Sonda reads.
var audioExts = map[string]bool{".mp3": true, ".flac": true}
// unsupportedExts are audio formats Sonda does not read yet. They are counted,
// never opened. Video containers such as .mpg or .mp4 are deliberately absent.
var unsupportedExts = map[string]bool{
".m4a": true, ".m4b": true, ".aac": true, ".alac": true, ".ogg": true, ".oga": true,
".opus": true, ".wav": true, ".wma": true, ".ape": true, ".wv": true, ".aif": true,
".aiff": true, ".mpc": true, ".mka": true, ".dsf": true, ".dff": true,
}
// systemDirs are housekeeping folders that NAS boxes and operating systems
// create inside shares: thumbnails, recycle bins, indexes. What they hold is
// never part of a library. Only the exact folder name matches, ignoring case.
var systemDirs = []string{
".@__thumb", "@Recycle", "#recycle", "@eaDir", "$RECYCLE.BIN", "System Volume Information",
}
func isSystemDir(name string) bool {
for _, d := range systemDirs {
if strings.EqualFold(name, d) {
return true
}
}
return false
}
// Scanner indexes the audio files of a set of libraries.
//
// Per library it walks the tree (cheap, stat only), decides what changed, then
// reads tags with a pool of workers. A single goroutine writes to the store.
type Scanner struct {
store Store
reader TagReader
hub *Hub
workers int
now func() time.Time
}
// NewScanner builds a scanner that parses files with the given number of workers.
func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner {
if workers < 1 {
workers = 1
}
return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now}
}
// ScanOption changes how a scan runs.
type ScanOption func(*scanOptions)
type scanOptions struct{ full bool }
// FullRescan makes a scan read the tags of every file again, not only of the files
// that are new or changed. It is how what the reader learned since (MusicBrainz
// ids, genres) reaches music that was indexed before. Nothing else changes: files
// keep their identity, so listening history is kept.
func FullRescan(o *scanOptions) { o.full = true }
func buildOptions(opts []ScanOption) scanOptions {
var o scanOptions
for _, f := range opts {
f(&o)
}
return o
}
// Scan indexes every library in turn. An unreadable library is reported in the
// returned error and the other libraries are still scanned.
func (s *Scanner) Scan(ctx context.Context, libs []Library, opts ...ScanOption) (Summary, error) {
o := buildOptions(opts)
total := Summary{Libraries: len(libs)}
s.publish(Event{Type: ScanStarted})
var errs []error
for _, lib := range libs {
sum, err := s.scanLibrary(ctx, lib, o)
total.add(sum)
if err != nil {
errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err))
if ctx.Err() != nil {
break
}
}
}
err := errors.Join(errs...)
fin := Event{Type: ScanFinished, Summary: &total}
if err != nil {
fin.Error = err.Error()
}
s.publish(fin)
return total, err
}
func (s *Scanner) publish(e Event) {
e.Time = s.now()
s.hub.Publish(e)
}
type parseJob struct {
file FileInfo
existingID int64
}
type parseResult struct {
job parseJob
meta Metadata
err error
}
func (s *Scanner) scanLibrary(ctx context.Context, lib Library, o scanOptions) (Summary, error) {
var sum Summary
walked, err := s.walk(ctx, lib, &sum)
if err != nil {
// The root itself is unreadable (for example an unmounted NAS).
// Do not touch the database: nothing is known about its files.
return sum, err
}
sum.Found = len(walked.files)
existing, err := s.store.ExistingFiles(ctx, lib.ID)
if err != nil {
return sum, fmt.Errorf("list indexed files: %w", err)
}
// An unmounted NAS usually leaves an empty but readable folder. Treat
// "nothing found although files are indexed" as a problem, not as a mass
// deletion. Delete the library to really empty it.
if len(walked.files) == 0 && len(walked.failedDirs) == 0 {
if n := countPresent(existing); n > 0 {
return sum, fmt.Errorf("directory is empty but %d indexed files exist: is the volume mounted? "+
"refusing to mark them missing", n)
}
}
p := makePlan(walked, existing, o)
sum.Unchanged = p.unchanged
if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil {
return sum, err
}
if err := ctx.Err(); err != nil {
return sum, err // interrupted: never mark files missing on a partial scan
}
for _, r := range p.renames {
if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil {
s.fail(lib, r.newRel, err, &sum)
continue
}
sum.Renamed++
}
if len(p.restore) > 0 {
if err := s.store.Restore(ctx, p.restore); err != nil {
return sum, fmt.Errorf("restore files: %w", err)
}
sum.Restored = len(p.restore)
}
if len(p.missing) > 0 {
if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil {
return sum, fmt.Errorf("mark missing: %w", err)
}
sum.Missing = len(p.missing)
}
return sum, nil
}
func (s *Scanner) fail(lib Library, rel string, err error, sum *Summary) {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error()})
}
// parseAndSave reads tags in parallel and stores results from this goroutine.
func (s *Scanner) parseAndSave(ctx context.Context, lib Library, jobs []parseJob, sum *Summary) error {
if len(jobs) == 0 {
return nil
}
jobCh := make(chan parseJob)
resCh := make(chan parseResult)
var wg sync.WaitGroup
for i := 0; i < s.workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := range jobCh {
meta, err := s.reader.Read(filepath.Join(lib.Path, filepath.FromSlash(j.file.RelPath)))
resCh <- parseResult{job: j, meta: meta, err: err}
}
}()
}
go func() {
defer func() {
close(jobCh)
wg.Wait()
close(resCh)
}()
for _, j := range jobs {
select {
case jobCh <- j:
case <-ctx.Done():
return
}
}
}()
done := 0
for r := range resCh { // always drain, so no worker is left blocked
if ctx.Err() != nil {
continue
}
done++
rel := r.job.file.RelPath
if r.err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
continue
}
added, err := s.store.SaveFile(ctx, lib.ID, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
if err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
continue
}
if added {
sum.Added++
} else {
sum.Updated++
}
s.publish(Event{Type: FileScanned, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Done: done, Total: len(jobs)})
}
return ctx.Err()
}
// ---- walking ----
type walkResult struct {
files []FileInfo
// failedDirs are slash-separated relative paths (with a trailing slash)
// that could not be read. Files below them are unknown, not gone.
failedDirs []string
// failedFiles are relative paths of files that could not be stat'ed for a
// reason other than "does not exist". They are unknown, not gone.
failedFiles map[string]bool
}
func (s *Scanner) walk(ctx context.Context, lib Library, sum *Summary) (walkResult, error) {
var out walkResult
// The library path may itself be a symlink (for example /srv/music ->
// /mnt/nas/music). WalkDir would treat it as a plain file and find nothing,
// so walk the resolved directory. Symlinks *inside* the root are still skipped.
walkRoot, err := filepath.EvalSymlinks(lib.Path)
if err != nil {
return out, err
}
info, err := os.Stat(walkRoot)
if err != nil {
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf("%s is not a directory", lib.Path)
}
err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error {
if cerr := ctx.Err(); cerr != nil {
return cerr
}
if path == walkRoot {
return err // nil, or the reason the root cannot be read
}
rel, rerr := filepath.Rel(walkRoot, path)
if rerr != nil {
return rerr
}
rel = filepath.ToSlash(rel)
if err != nil {
s.fail(lib, rel, err, sum)
if d != nil && d.IsDir() {
out.failedDirs = append(out.failedDirs, rel+"/")
} else {
out.markFileFailed(rel)
}
return nil // keep going, do not descend into an unreadable dir
}
if d.Type()&fs.ModeSymlink != 0 {
sum.Skipped++ // never follow symlinks: they could point outside the root
return nil
}
if d.IsDir() {
if isSystemDir(d.Name()) {
return filepath.SkipDir
}
return nil
}
ext := strings.ToLower(filepath.Ext(rel))
if !audioExts[ext] {
if unsupportedExts[ext] {
sum.Unsupported++
}
return nil
}
if !d.Type().IsRegular() {
// FIFOs, sockets and devices: opening one can block forever.
sum.Skipped++
return nil
}
info, ierr := d.Info()
if ierr != nil {
s.fail(lib, rel, ierr, sum)
if !errors.Is(ierr, fs.ErrNotExist) {
out.markFileFailed(rel)
}
return nil
}
if !info.Mode().IsRegular() {
// Replaced by a FIFO or symlink between the directory read and now.
sum.Skipped++
return nil
}
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
return nil
})
return out, err
}
func (w *walkResult) markFileFailed(rel string) {
if w.failedFiles == nil {
w.failedFiles = map[string]bool{}
}
w.failedFiles[rel] = true
}
// ---- planning ----
type renameOp struct {
id int64
newRel string
}
type plan struct {
parse []parseJob
renames []renameOp
restore []int64
missing []int64
unchanged int
}
type fileKey struct{ size, mtime int64 }
// makePlan compares what is on disk with what is indexed.
//
// A new file and a vanished file with the same size and modification time are
// treated as a rename only when each is the sole candidate for that pair, so
// look-alike files are never guessed.
func makePlan(w walkResult, existing []ExistingFile, o scanOptions) plan {
var p plan
byPath := make(map[string]ExistingFile, len(existing))
for _, e := range existing {
byPath[e.RelPath] = e
}
seen := make(map[string]bool, len(w.files))
var added []FileInfo
for _, f := range w.files {
e, ok := byPath[f.RelPath]
if !ok {
added = append(added, f)
continue
}
seen[f.RelPath] = true
changed := e.Size != f.Size || e.MTimeUnix != f.MTimeUnix
switch {
case changed || o.full:
p.parse = append(p.parse, parseJob{file: f, existingID: e.ID})
if e.Missing && !changed {
// Back, and read again. It is back even if reading fails this time.
p.restore = append(p.restore, e.ID)
}
case e.Missing:
p.restore = append(p.restore, e.ID)
default:
p.unchanged++
}
}
var gone []ExistingFile
for _, e := range existing {
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
continue
}
gone = append(gone, e)
}
goneByKey := map[fileKey][]ExistingFile{}
for _, e := range gone {
k := fileKey{e.Size, e.MTimeUnix}
goneByKey[k] = append(goneByKey[k], e)
}
addedByKey := map[fileKey]int{}
for _, f := range added {
addedByKey[fileKey{f.Size, f.MTimeUnix}]++
}
renamed := map[int64]bool{}
for _, f := range added {
k := fileKey{f.Size, f.MTimeUnix}
if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 {
p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath})
renamed[cands[0].ID] = true
continue
}
p.parse = append(p.parse, parseJob{file: f})
}
for _, e := range gone {
if !renamed[e.ID] && !e.Missing {
p.missing = append(p.missing, e.ID)
}
}
return p
}
// countPresent counts indexed files that were not already flagged missing.
func countPresent(existing []ExistingFile) int {
n := 0
for _, e := range existing {
if !e.Missing {
n++
}
}
return n
}
func underAny(rel string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(rel, p) {
return true
}
}
return false
}