sonda/internal/library/scanner.go
tgiachi 11aa47e79f fix: refresh match keys at startup and re-check file type after stat
- fix(storage): RefreshMatchKeys recomputes name_key and title_key for rows
  created before migration 0003, whose SQL lower() key is ASCII-only on SQLite;
  idempotent, runs at every start
- fix(library): re-check that the stat result is a regular file, closing the
  window where a path is swapped for a FIFO or symlink after the directory read
- docs(library): note that OnFinish callbacks of consecutive scans can overlap
- test: old rows with approximate keys are matched after the refresh
2026-11-11 12:30:43 +01:00

397 lines
9.8 KiB
Go

package library
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var audioExts = map[string]bool{".mp3": true, ".flac": true}
// Scanner indexes the audio files under a set of roots.
//
// Per root it walks the tree (cheap, stat only), decides what changed, then
// reads tags with a pool of workers. A single goroutine writes to the store.
type Scanner struct {
store Store
reader TagReader
hub *Hub
workers int
now func() time.Time
}
// NewScanner builds a scanner that parses files with the given number of workers.
func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner {
if workers < 1 {
workers = 1
}
return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now}
}
// Scan indexes every root in turn. An unreadable root is reported in the
// returned error and the other roots are still scanned.
func (s *Scanner) Scan(ctx context.Context, roots []string) (Summary, error) {
total := Summary{Roots: len(roots)}
s.publish(Event{Type: ScanStarted})
var errs []error
for _, root := range roots {
sum, err := s.scanRoot(ctx, root)
total.add(sum)
if err != nil {
errs = append(errs, fmt.Errorf("%s: %w", root, err))
if ctx.Err() != nil {
break
}
}
}
err := errors.Join(errs...)
fin := Event{Type: ScanFinished, Summary: &total}
if err != nil {
fin.Error = err.Error()
}
s.publish(fin)
return total, err
}
func (s *Scanner) publish(e Event) {
e.Time = s.now()
s.hub.Publish(e)
}
type parseJob struct {
file FileInfo
existingID int64
}
type parseResult struct {
job parseJob
meta Metadata
err error
}
func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) {
var sum Summary
walked, err := s.walk(ctx, root, &sum)
if err != nil {
// The root itself is unreadable (for example an unmounted NAS).
// Do not touch the database: nothing is known about its files.
return sum, err
}
sum.Found = len(walked.files)
existing, err := s.store.ExistingFiles(ctx, root)
if err != nil {
return sum, fmt.Errorf("list indexed files: %w", err)
}
// An unmounted NAS usually leaves an empty but readable folder. Treat
// "nothing found although files are indexed" as a problem, not as a mass
// deletion. Remove the root from the config to really empty a library.
if len(walked.files) == 0 && len(walked.failedDirs) == 0 {
if n := countPresent(existing); n > 0 {
return sum, fmt.Errorf("root is empty but %d indexed files exist: is the volume mounted? "+
"refusing to mark them missing", n)
}
}
p := makePlan(walked, existing)
sum.Unchanged = p.unchanged
if err := s.parseAndSave(ctx, root, p.parse, &sum); err != nil {
return sum, err
}
if err := ctx.Err(); err != nil {
return sum, err // interrupted: never mark files missing on a partial scan
}
for _, r := range p.renames {
if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil {
s.fail(root, r.newRel, err, &sum)
continue
}
sum.Renamed++
}
if len(p.restore) > 0 {
if err := s.store.Restore(ctx, p.restore); err != nil {
return sum, fmt.Errorf("restore files: %w", err)
}
sum.Restored = len(p.restore)
}
if len(p.missing) > 0 {
if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil {
return sum, fmt.Errorf("mark missing: %w", err)
}
sum.Missing = len(p.missing)
}
return sum, nil
}
func (s *Scanner) fail(root, rel string, err error, sum *Summary) {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error()})
}
// parseAndSave reads tags in parallel and stores results from this goroutine.
func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob, sum *Summary) error {
if len(jobs) == 0 {
return nil
}
jobCh := make(chan parseJob)
resCh := make(chan parseResult)
var wg sync.WaitGroup
for i := 0; i < s.workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := range jobCh {
meta, err := s.reader.Read(filepath.Join(root, filepath.FromSlash(j.file.RelPath)))
resCh <- parseResult{job: j, meta: meta, err: err}
}
}()
}
go func() {
defer func() {
close(jobCh)
wg.Wait()
close(resCh)
}()
for _, j := range jobs {
select {
case jobCh <- j:
case <-ctx.Done():
return
}
}
}()
done := 0
for r := range resCh { // always drain, so no worker is left blocked
if ctx.Err() != nil {
continue
}
done++
rel := r.job.file.RelPath
if r.err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
continue
}
added, err := s.store.SaveFile(ctx, root, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
if err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
continue
}
if added {
sum.Added++
} else {
sum.Updated++
}
s.publish(Event{Type: FileScanned, Root: root, RelPath: rel, Done: done, Total: len(jobs)})
}
return ctx.Err()
}
// ---- walking ----
type walkResult struct {
files []FileInfo
// failedDirs are slash-separated relative paths (with a trailing slash)
// that could not be read. Files below them are unknown, not gone.
failedDirs []string
// failedFiles are relative paths of files that could not be stat'ed for a
// reason other than "does not exist". They are unknown, not gone.
failedFiles map[string]bool
}
func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) {
var out walkResult
// The configured root may itself be a symlink (for example /srv/music ->
// /mnt/nas/music). WalkDir would treat it as a plain file and find nothing,
// so walk the resolved directory. Symlinks *inside* the root are still skipped.
walkRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return out, err
}
info, err := os.Stat(walkRoot)
if err != nil {
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf("%s is not a directory", root)
}
err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error {
if cerr := ctx.Err(); cerr != nil {
return cerr
}
if path == walkRoot {
return err // nil, or the reason the root cannot be read
}
rel, rerr := filepath.Rel(walkRoot, path)
if rerr != nil {
return rerr
}
rel = filepath.ToSlash(rel)
if err != nil {
s.fail(root, rel, err, sum)
if d != nil && d.IsDir() {
out.failedDirs = append(out.failedDirs, rel+"/")
} else {
out.markFileFailed(rel)
}
return nil // keep going, do not descend into an unreadable dir
}
if d.Type()&fs.ModeSymlink != 0 {
sum.Skipped++ // never follow symlinks: they could point outside the root
return nil
}
if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] {
return nil
}
if !d.Type().IsRegular() {
// FIFOs, sockets and devices: opening one can block forever.
sum.Skipped++
return nil
}
info, ierr := d.Info()
if ierr != nil {
s.fail(root, rel, ierr, sum)
if !errors.Is(ierr, fs.ErrNotExist) {
out.markFileFailed(rel)
}
return nil
}
if !info.Mode().IsRegular() {
// Replaced by a FIFO or symlink between the directory read and now.
sum.Skipped++
return nil
}
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
return nil
})
return out, err
}
func (w *walkResult) markFileFailed(rel string) {
if w.failedFiles == nil {
w.failedFiles = map[string]bool{}
}
w.failedFiles[rel] = true
}
// ---- planning ----
type renameOp struct {
id int64
newRel string
}
type plan struct {
parse []parseJob
renames []renameOp
restore []int64
missing []int64
unchanged int
}
type fileKey struct{ size, mtime int64 }
// makePlan compares what is on disk with what is indexed.
//
// A new file and a vanished file with the same size and modification time are
// treated as a rename only when each is the sole candidate for that pair, so
// look-alike files are never guessed.
func makePlan(w walkResult, existing []ExistingFile) plan {
var p plan
byPath := make(map[string]ExistingFile, len(existing))
for _, e := range existing {
byPath[e.RelPath] = e
}
seen := make(map[string]bool, len(w.files))
var added []FileInfo
for _, f := range w.files {
e, ok := byPath[f.RelPath]
if !ok {
added = append(added, f)
continue
}
seen[f.RelPath] = true
switch {
case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix:
p.parse = append(p.parse, parseJob{file: f, existingID: e.ID})
case e.Missing:
p.restore = append(p.restore, e.ID)
default:
p.unchanged++
}
}
var gone []ExistingFile
for _, e := range existing {
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
continue
}
gone = append(gone, e)
}
goneByKey := map[fileKey][]ExistingFile{}
for _, e := range gone {
k := fileKey{e.Size, e.MTimeUnix}
goneByKey[k] = append(goneByKey[k], e)
}
addedByKey := map[fileKey]int{}
for _, f := range added {
addedByKey[fileKey{f.Size, f.MTimeUnix}]++
}
renamed := map[int64]bool{}
for _, f := range added {
k := fileKey{f.Size, f.MTimeUnix}
if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 {
p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath})
renamed[cands[0].ID] = true
continue
}
p.parse = append(p.parse, parseJob{file: f})
}
for _, e := range gone {
if !renamed[e.ID] && !e.Missing {
p.missing = append(p.missing, e.ID)
}
}
return p
}
// countPresent counts indexed files that were not already flagged missing.
func countPresent(existing []ExistingFile) int {
n := 0
for _, e := range existing {
if !e.Missing {
n++
}
}
return n
}
func underAny(rel string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(rel, p) {
return true
}
}
return false
}