sonda/internal/library/scanner.go
tgiachi 46463846b3 feat(library): named media libraries managed through the API
- feat(storage): libraries table (name, path); audio_files points at a library
  by id instead of repeating the root path; migration 0005 for both drivers
  turns every existing root into a library and keeps file ids, tracks and the
  missing flag
- feat(library): Libraries service validates names and paths: absolute, an
  existing directory, never the filesystem root, never inside or around
  another library (checked on real paths, so symlinks cannot hide an overlap)
- feat(api): admin-only GET/POST /libraries, PATCH/DELETE /libraries/{id} and
  POST /libraries/{id}/scan; deleting a library drops its index entries and
  keeps tracks and listening history
- feat(library): the scanner and the scan manager work on libraries; scanning
  with no libraries answers 409 no_libraries
- feat(streaming): the library directory comes from the database instead of
  a configured list of roots
- breaking(config): library.roots is removed; an old config fails with a hint
  to use POST /api/v1/libraries
- test: CRUD and duplicates on both databases, path change keeps files,
  upgrade of a database written before libraries existed
2026-11-11 13:30:25 +01:00

397 lines
9.9 KiB
Go

package library
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var audioExts = map[string]bool{".mp3": true, ".flac": true}
// Scanner indexes the audio files of a set of libraries.
//
// Per library it walks the tree (cheap, stat only), decides what changed, then
// reads tags with a pool of workers. A single goroutine writes to the store.
type Scanner struct {
store Store
reader TagReader
hub *Hub
workers int
now func() time.Time
}
// NewScanner builds a scanner that parses files with the given number of workers.
func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner {
if workers < 1 {
workers = 1
}
return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now}
}
// Scan indexes every library in turn. An unreadable library is reported in the
// returned error and the other libraries are still scanned.
func (s *Scanner) Scan(ctx context.Context, libs []Library) (Summary, error) {
total := Summary{Libraries: len(libs)}
s.publish(Event{Type: ScanStarted})
var errs []error
for _, lib := range libs {
sum, err := s.scanLibrary(ctx, lib)
total.add(sum)
if err != nil {
errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err))
if ctx.Err() != nil {
break
}
}
}
err := errors.Join(errs...)
fin := Event{Type: ScanFinished, Summary: &total}
if err != nil {
fin.Error = err.Error()
}
s.publish(fin)
return total, err
}
func (s *Scanner) publish(e Event) {
e.Time = s.now()
s.hub.Publish(e)
}
type parseJob struct {
file FileInfo
existingID int64
}
type parseResult struct {
job parseJob
meta Metadata
err error
}
func (s *Scanner) scanLibrary(ctx context.Context, lib Library) (Summary, error) {
var sum Summary
walked, err := s.walk(ctx, lib, &sum)
if err != nil {
// The root itself is unreadable (for example an unmounted NAS).
// Do not touch the database: nothing is known about its files.
return sum, err
}
sum.Found = len(walked.files)
existing, err := s.store.ExistingFiles(ctx, lib.ID)
if err != nil {
return sum, fmt.Errorf("list indexed files: %w", err)
}
// An unmounted NAS usually leaves an empty but readable folder. Treat
// "nothing found although files are indexed" as a problem, not as a mass
// deletion. Delete the library to really empty it.
if len(walked.files) == 0 && len(walked.failedDirs) == 0 {
if n := countPresent(existing); n > 0 {
return sum, fmt.Errorf("directory is empty but %d indexed files exist: is the volume mounted? "+
"refusing to mark them missing", n)
}
}
p := makePlan(walked, existing)
sum.Unchanged = p.unchanged
if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil {
return sum, err
}
if err := ctx.Err(); err != nil {
return sum, err // interrupted: never mark files missing on a partial scan
}
for _, r := range p.renames {
if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil {
s.fail(lib, r.newRel, err, &sum)
continue
}
sum.Renamed++
}
if len(p.restore) > 0 {
if err := s.store.Restore(ctx, p.restore); err != nil {
return sum, fmt.Errorf("restore files: %w", err)
}
sum.Restored = len(p.restore)
}
if len(p.missing) > 0 {
if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil {
return sum, fmt.Errorf("mark missing: %w", err)
}
sum.Missing = len(p.missing)
}
return sum, nil
}
func (s *Scanner) fail(lib Library, rel string, err error, sum *Summary) {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error()})
}
// parseAndSave reads tags in parallel and stores results from this goroutine.
func (s *Scanner) parseAndSave(ctx context.Context, lib Library, jobs []parseJob, sum *Summary) error {
if len(jobs) == 0 {
return nil
}
jobCh := make(chan parseJob)
resCh := make(chan parseResult)
var wg sync.WaitGroup
for i := 0; i < s.workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := range jobCh {
meta, err := s.reader.Read(filepath.Join(lib.Path, filepath.FromSlash(j.file.RelPath)))
resCh <- parseResult{job: j, meta: meta, err: err}
}
}()
}
go func() {
defer func() {
close(jobCh)
wg.Wait()
close(resCh)
}()
for _, j := range jobs {
select {
case jobCh <- j:
case <-ctx.Done():
return
}
}
}()
done := 0
for r := range resCh { // always drain, so no worker is left blocked
if ctx.Err() != nil {
continue
}
done++
rel := r.job.file.RelPath
if r.err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
continue
}
added, err := s.store.SaveFile(ctx, lib.ID, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
if err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
continue
}
if added {
sum.Added++
} else {
sum.Updated++
}
s.publish(Event{Type: FileScanned, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Done: done, Total: len(jobs)})
}
return ctx.Err()
}
// ---- walking ----
type walkResult struct {
files []FileInfo
// failedDirs are slash-separated relative paths (with a trailing slash)
// that could not be read. Files below them are unknown, not gone.
failedDirs []string
// failedFiles are relative paths of files that could not be stat'ed for a
// reason other than "does not exist". They are unknown, not gone.
failedFiles map[string]bool
}
func (s *Scanner) walk(ctx context.Context, lib Library, sum *Summary) (walkResult, error) {
var out walkResult
// The library path may itself be a symlink (for example /srv/music ->
// /mnt/nas/music). WalkDir would treat it as a plain file and find nothing,
// so walk the resolved directory. Symlinks *inside* the root are still skipped.
walkRoot, err := filepath.EvalSymlinks(lib.Path)
if err != nil {
return out, err
}
info, err := os.Stat(walkRoot)
if err != nil {
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf("%s is not a directory", lib.Path)
}
err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error {
if cerr := ctx.Err(); cerr != nil {
return cerr
}
if path == walkRoot {
return err // nil, or the reason the root cannot be read
}
rel, rerr := filepath.Rel(walkRoot, path)
if rerr != nil {
return rerr
}
rel = filepath.ToSlash(rel)
if err != nil {
s.fail(lib, rel, err, sum)
if d != nil && d.IsDir() {
out.failedDirs = append(out.failedDirs, rel+"/")
} else {
out.markFileFailed(rel)
}
return nil // keep going, do not descend into an unreadable dir
}
if d.Type()&fs.ModeSymlink != 0 {
sum.Skipped++ // never follow symlinks: they could point outside the root
return nil
}
if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] {
return nil
}
if !d.Type().IsRegular() {
// FIFOs, sockets and devices: opening one can block forever.
sum.Skipped++
return nil
}
info, ierr := d.Info()
if ierr != nil {
s.fail(lib, rel, ierr, sum)
if !errors.Is(ierr, fs.ErrNotExist) {
out.markFileFailed(rel)
}
return nil
}
if !info.Mode().IsRegular() {
// Replaced by a FIFO or symlink between the directory read and now.
sum.Skipped++
return nil
}
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
return nil
})
return out, err
}
func (w *walkResult) markFileFailed(rel string) {
if w.failedFiles == nil {
w.failedFiles = map[string]bool{}
}
w.failedFiles[rel] = true
}
// ---- planning ----
type renameOp struct {
id int64
newRel string
}
type plan struct {
parse []parseJob
renames []renameOp
restore []int64
missing []int64
unchanged int
}
type fileKey struct{ size, mtime int64 }
// makePlan compares what is on disk with what is indexed.
//
// A new file and a vanished file with the same size and modification time are
// treated as a rename only when each is the sole candidate for that pair, so
// look-alike files are never guessed.
func makePlan(w walkResult, existing []ExistingFile) plan {
var p plan
byPath := make(map[string]ExistingFile, len(existing))
for _, e := range existing {
byPath[e.RelPath] = e
}
seen := make(map[string]bool, len(w.files))
var added []FileInfo
for _, f := range w.files {
e, ok := byPath[f.RelPath]
if !ok {
added = append(added, f)
continue
}
seen[f.RelPath] = true
switch {
case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix:
p.parse = append(p.parse, parseJob{file: f, existingID: e.ID})
case e.Missing:
p.restore = append(p.restore, e.ID)
default:
p.unchanged++
}
}
var gone []ExistingFile
for _, e := range existing {
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
continue
}
gone = append(gone, e)
}
goneByKey := map[fileKey][]ExistingFile{}
for _, e := range gone {
k := fileKey{e.Size, e.MTimeUnix}
goneByKey[k] = append(goneByKey[k], e)
}
addedByKey := map[fileKey]int{}
for _, f := range added {
addedByKey[fileKey{f.Size, f.MTimeUnix}]++
}
renamed := map[int64]bool{}
for _, f := range added {
k := fileKey{f.Size, f.MTimeUnix}
if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 {
p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath})
renamed[cands[0].ID] = true
continue
}
p.parse = append(p.parse, parseJob{file: f})
}
for _, e := range gone {
if !renamed[e.ID] && !e.Missing {
p.missing = append(p.missing, e.ID)
}
}
return p
}
// countPresent counts indexed files that were not already flagged missing.
func countPresent(existing []ExistingFile) int {
n := 0
for _, e := range existing {
if !e.Missing {
n++
}
}
return n
}
func underAny(rel string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(rel, p) {
return true
}
}
return false
}