sonda/internal/library/scanner_test.go
tgiachi d6e9e09ce7 fix(library): harden the scanner against special files, stat errors and lossy logging
- fix: skip FIFOs, sockets and devices named like audio files; opening a named
  pipe blocked a scan forever and left the manager stuck in "running"
- fix: a file that cannot be stat'ed for a transient reason is unknown, not
  gone, so it is never marked missing or used as a rename source
- fix: log the scan result from Manager.OnFinish instead of the lossy event
  hub, so the line can no longer be dropped
- fix(storage): match artists and albums on a Unicode-aware key computed in Go
  (migration 0003, both drivers), so SQLite and PostgreSQL agree
- test: FIFO, stat errors, OnFinish, Unicode artist and album matching
2026-11-11 12:28:44 +01:00

708 lines
19 KiB
Go

package library
import (
"context"
"errors"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
)
// ---- fakes ----
type fakeRow struct {
ExistingFile
Parsed ParsedFile
}
type fakeStore struct {
mu sync.Mutex
nextID int64
rows map[string]map[string]*fakeRow // root -> rel path -> row
saves int
}
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]map[string]*fakeRow{}} }
func (s *fakeStore) byID(id int64) *fakeRow {
for _, files := range s.rows {
for _, r := range files {
if r.ID == id {
return r
}
}
}
return nil
}
func (s *fakeStore) ExistingFiles(_ context.Context, root string) ([]ExistingFile, error) {
s.mu.Lock()
defer s.mu.Unlock()
var out []ExistingFile
for _, r := range s.rows[root] {
out = append(out, r.ExistingFile)
}
return out, nil
}
func (s *fakeStore) SaveFile(_ context.Context, root string, f ParsedFile, existingID int64) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.saves++
if existingID != 0 {
r := s.byID(existingID)
r.Size, r.MTimeUnix, r.Missing, r.Parsed = f.Size, f.MTimeUnix, false, f
return false, nil
}
if s.rows[root] == nil {
s.rows[root] = map[string]*fakeRow{}
}
s.nextID++
s.rows[root][f.RelPath] = &fakeRow{
ExistingFile: ExistingFile{ID: s.nextID, FileInfo: f.FileInfo},
Parsed: f,
}
return true, nil
}
func (s *fakeStore) RenameFile(_ context.Context, id int64, newRel string) error {
s.mu.Lock()
defer s.mu.Unlock()
for _, files := range s.rows {
for old, r := range files {
if r.ID == id {
delete(files, old)
r.RelPath, r.Missing = newRel, false
files[newRel] = r
return nil
}
}
}
return errors.New("no such file id")
}
func (s *fakeStore) MarkMissing(_ context.Context, ids []int64, _ time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
for _, id := range ids {
s.byID(id).Missing = true
}
return nil
}
func (s *fakeStore) Restore(_ context.Context, ids []int64) error {
s.mu.Lock()
defer s.mu.Unlock()
for _, id := range ids {
s.byID(id).Missing = false
}
return nil
}
func (s *fakeStore) row(root, rel string) *fakeRow {
s.mu.Lock()
defer s.mu.Unlock()
return s.rows[root][rel]
}
// fakeReader returns metadata derived from the file name and counts calls.
type fakeReader struct {
calls atomic.Int64
delay time.Duration
fail map[string]bool // base names that fail
running atomic.Int64
peak atomic.Int64
}
func (r *fakeReader) Read(path string) (Metadata, error) {
r.calls.Add(1)
n := r.running.Add(1)
for {
p := r.peak.Load()
if n <= p || r.peak.CompareAndSwap(p, n) {
break
}
}
defer r.running.Add(-1)
if r.delay > 0 {
time.Sleep(r.delay)
}
if r.fail[filepath.Base(path)] {
return Metadata{}, ErrUnreadable
}
return Metadata{
Tags: Tags{Title: filepath.Base(path), Artists: []string{"Artist"}},
Properties: Properties{Format: "mp3", DurationMs: 1000, SampleRate: 44100},
}, nil
}
// ---- helpers ----
func write(t *testing.T, path, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
func newScanner(store Store, reader TagReader, workers int) (*Scanner, *Hub) {
hub := NewHub()
return NewScanner(store, reader, hub, workers), hub
}
func mustScan(t *testing.T, s *Scanner, roots ...string) Summary {
t.Helper()
sum, err := s.Scan(context.Background(), roots)
if err != nil {
t.Fatal(err)
}
return sum
}
// ---- tests ----
func TestScanIndexesAudioFilesOnly(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a/one.mp3"), "1")
write(t, filepath.Join(root, "a/two.FLAC"), "22")
write(t, filepath.Join(root, "a/cover.jpg"), "x")
write(t, filepath.Join(root, "notes.txt"), "x")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 2)
sum := mustScan(t, s, root)
if sum.Found != 2 || sum.Added != 2 || sum.Failed != 0 {
t.Fatalf("summary = %+v", sum)
}
if store.row(root, "a/one.mp3") == nil || store.row(root, "a/two.FLAC") == nil {
t.Error("audio files not stored with root-relative slash paths")
}
if store.row(root, "a/cover.jpg") != nil {
t.Error("non-audio file was stored")
}
}
func TestScanSkipsSymlinks(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
write(t, filepath.Join(outside, "secret.mp3"), "s")
write(t, filepath.Join(root, "real.mp3"), "r")
if err := os.Symlink(filepath.Join(outside, "secret.mp3"), filepath.Join(root, "link.mp3")); err != nil {
t.Skip("symlinks not supported:", err)
}
if err := os.Symlink(outside, filepath.Join(root, "dirlink")); err != nil {
t.Skip("symlinks not supported:", err)
}
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 2)
sum := mustScan(t, s, root)
if sum.Found != 1 || store.row(root, "real.mp3") == nil {
t.Fatalf("summary = %+v", sum)
}
if sum.Skipped < 1 {
t.Errorf("skipped = %d, want symlinks counted", sum.Skipped)
}
}
func TestRescanSkipsUnchangedFiles(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
write(t, filepath.Join(root, "b.mp3"), "22")
store, reader := newFakeStore(), &fakeReader{}
s, _ := newScanner(store, reader, 2)
mustScan(t, s, root)
before := reader.calls.Load()
sum := mustScan(t, s, root)
if sum.Unchanged != 2 || sum.Added != 0 || sum.Updated != 0 {
t.Fatalf("rescan summary = %+v", sum)
}
if reader.calls.Load() != before {
t.Error("unchanged files were parsed again")
}
if store.saves != 2 {
t.Errorf("saves = %d, want 2 (no duplicates)", store.saves)
}
}
func TestRescanUpdatesChangedFile(t *testing.T) {
root := t.TempDir()
p := filepath.Join(root, "a.mp3")
write(t, p, "1")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
id := store.row(root, "a.mp3").ID
write(t, p, "longer content")
later := time.Now().Add(time.Hour)
if err := os.Chtimes(p, later, later); err != nil {
t.Fatal(err)
}
sum := mustScan(t, s, root)
if sum.Updated != 1 || sum.Added != 0 {
t.Fatalf("summary = %+v", sum)
}
if got := store.row(root, "a.mp3").ID; got != id {
t.Errorf("id changed %d -> %d", id, got)
}
}
func TestRenamedFileKeepsIdentity(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "old.mp3"), "content")
store, reader := newFakeStore(), &fakeReader{}
s, _ := newScanner(store, reader, 1)
mustScan(t, s, root)
id := store.row(root, "old.mp3").ID
calls := reader.calls.Load()
if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Rename(filepath.Join(root, "old.mp3"), filepath.Join(root, "sub/new.mp3")); err != nil {
t.Fatal(err)
}
sum := mustScan(t, s, root)
if sum.Renamed != 1 || sum.Added != 0 || sum.Missing != 0 {
t.Fatalf("summary = %+v", sum)
}
r := store.row(root, "sub/new.mp3")
if r == nil || r.ID != id {
t.Fatalf("renamed row = %+v, want id %d", r, id)
}
if reader.calls.Load() != calls {
t.Error("renamed file was parsed again")
}
}
func TestAmbiguousRenameIsNotGuessed(t *testing.T) {
root := t.TempDir()
ts := time.Unix(1_700_000_000, 0)
for _, n := range []string{"a.mp3", "b.mp3"} {
p := filepath.Join(root, n)
write(t, p, "same")
_ = os.Chtimes(p, ts, ts)
}
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
// Both vanish and two look-alike files appear: size and mtime cannot tell which is which.
_ = os.Remove(filepath.Join(root, "a.mp3"))
_ = os.Remove(filepath.Join(root, "b.mp3"))
for _, n := range []string{"c.mp3", "d.mp3"} {
p := filepath.Join(root, n)
write(t, p, "same")
_ = os.Chtimes(p, ts, ts)
}
sum := mustScan(t, s, root)
if sum.Renamed != 0 || sum.Added != 2 || sum.Missing != 2 {
t.Fatalf("summary = %+v", sum)
}
}
func TestMissingFileIsMarkedThenRestored(t *testing.T) {
root := t.TempDir()
p := filepath.Join(root, "a.mp3")
write(t, p, "1")
write(t, filepath.Join(root, "keep.mp3"), "22") // the library must not become empty
ts := time.Unix(1_700_000_000, 0)
_ = os.Chtimes(p, ts, ts)
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
if err := os.Remove(p); err != nil {
t.Fatal(err)
}
sum := mustScan(t, s, root)
if sum.Missing != 1 {
t.Fatalf("summary = %+v", sum)
}
r := store.row(root, "a.mp3")
if r == nil || !r.Missing {
t.Fatal("row should be kept and marked missing, not deleted")
}
// A second scan must not count it as newly missing.
if sum = mustScan(t, s, root); sum.Missing != 0 {
t.Errorf("already-missing file counted again: %+v", sum)
}
write(t, p, "1")
_ = os.Chtimes(p, ts, ts)
sum = mustScan(t, s, root)
if sum.Restored != 1 || store.row(root, "a.mp3").Missing {
t.Fatalf("summary = %+v, row = %+v", sum, store.row(root, "a.mp3"))
}
}
func TestUnreadableRootDoesNotMarkFilesMissing(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
// Simulate an unmounted NAS: the root path no longer exists.
if err := os.RemoveAll(root); err != nil {
t.Fatal(err)
}
if _, err := s.Scan(context.Background(), []string{root}); err == nil {
t.Fatal("expected an error for an unreadable root")
}
if store.row(root, "a.mp3").Missing {
t.Fatal("files were marked missing although the root could not be read")
}
}
func TestSymlinkedRootIsFollowed(t *testing.T) {
real := t.TempDir()
write(t, filepath.Join(real, "a.mp3"), "1")
write(t, filepath.Join(real, "sub/b.mp3"), "22")
link := filepath.Join(t.TempDir(), "music")
if err := os.Symlink(real, link); err != nil {
t.Skip("symlinks not supported:", err)
}
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 2)
sum := mustScan(t, s, link)
if sum.Found != 2 || sum.Added != 2 {
t.Fatalf("first scan through a symlinked root: %+v", sum)
}
if store.row(link, "sub/b.mp3") == nil {
t.Error("files must be stored under the configured root path")
}
// A rescan must see the same files, not an empty tree.
sum = mustScan(t, s, link)
if sum.Unchanged != 2 || sum.Missing != 0 {
t.Fatalf("rescan through a symlinked root: %+v", sum)
}
}
func TestSymlinkRootToAFileIsRejected(t *testing.T) {
dir := t.TempDir()
write(t, filepath.Join(dir, "x.mp3"), "1")
link := filepath.Join(t.TempDir(), "music")
if err := os.Symlink(filepath.Join(dir, "x.mp3"), link); err != nil {
t.Skip("symlinks not supported:", err)
}
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
if _, err := s.Scan(context.Background(), []string{link}); err == nil {
t.Fatal("a root that is not a directory must be an error")
}
}
func TestEmptyRootDoesNotMarkIndexedFilesMissing(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
write(t, filepath.Join(root, "b.mp3"), "22")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
// An unmounted NAS typically leaves an empty, perfectly readable folder.
_ = os.Remove(filepath.Join(root, "a.mp3"))
_ = os.Remove(filepath.Join(root, "b.mp3"))
sum, err := s.Scan(context.Background(), []string{root})
if err == nil {
t.Fatal("expected an error: root is empty but files are indexed")
}
if sum.Missing != 0 || store.row(root, "a.mp3").Missing || store.row(root, "b.mp3").Missing {
t.Fatalf("an empty root must not flag the library as missing: %+v", sum)
}
}
func TestEmptyRootWithNothingIndexedIsFine(t *testing.T) {
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
sum, err := s.Scan(context.Background(), []string{t.TempDir()})
if err != nil || sum.Found != 0 {
t.Fatalf("fresh empty library: %+v err=%v", sum, err)
}
}
func TestUnreadableSubdirDoesNotMarkItsFilesMissing(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("permissions are not enforced for root")
}
root := t.TempDir()
write(t, filepath.Join(root, "ok.mp3"), "1")
write(t, filepath.Join(root, "locked/hidden.mp3"), "22")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
locked := filepath.Join(root, "locked")
if err := os.Chmod(locked, 0); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
sum := mustScan(t, s, root)
if sum.Failed < 1 {
t.Errorf("failed = %d, want the unreadable dir reported", sum.Failed)
}
if sum.Missing != 0 || store.row(root, "locked/hidden.mp3").Missing {
t.Fatalf("file in unreadable dir marked missing: %+v", sum)
}
}
func TestUnreadableFileIsReportedAndScanContinues(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "good.mp3"), "1")
write(t, filepath.Join(root, "bad.mp3"), "22")
store := newFakeStore()
s, hub := newScanner(store, &fakeReader{fail: map[string]bool{"bad.mp3": true}}, 2)
events, cancel := hub.Subscribe(64)
defer cancel()
sum := mustScan(t, s, root)
if sum.Added != 1 || sum.Failed != 1 {
t.Fatalf("summary = %+v", sum)
}
if store.row(root, "bad.mp3") != nil {
t.Error("unreadable file was stored")
}
var failed *Event
for len(events) > 0 {
if e := <-events; e.Type == FileFailed {
failed = &e
}
}
if failed == nil || failed.RelPath != "bad.mp3" || failed.Error == "" {
t.Fatalf("file_failed event = %+v", failed)
}
}
func TestScanRunsWorkersInParallel(t *testing.T) {
root := t.TempDir()
for i := 0; i < 12; i++ {
write(t, filepath.Join(root, string(rune('a'+i))+".mp3"), string(rune('a'+i)))
}
reader := &fakeReader{delay: 30 * time.Millisecond}
s, _ := newScanner(newFakeStore(), reader, 4)
start := time.Now()
mustScan(t, s, root)
elapsed := time.Since(start)
if reader.peak.Load() < 2 {
t.Errorf("peak concurrency = %d, want parallel reads", reader.peak.Load())
}
if reader.peak.Load() > 4 {
t.Errorf("peak concurrency = %d, exceeds 4 workers", reader.peak.Load())
}
if elapsed > 12*30*time.Millisecond {
t.Errorf("scan took %v, no faster than sequential", elapsed)
}
}
func TestCancelledScanDoesNotMarkMissing(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
mustScan(t, s, root)
_ = os.Remove(filepath.Join(root, "a.mp3"))
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.Scan(ctx, []string{root}); !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
if store.row(root, "a.mp3").Missing {
t.Fatal("an interrupted scan must not mark files missing")
}
}
func TestScanPublishesProgressEvents(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
write(t, filepath.Join(root, "b.mp3"), "22")
s, hub := newScanner(newFakeStore(), &fakeReader{}, 2)
events, cancel := hub.Subscribe(64)
defer cancel()
mustScan(t, s, root)
var types []EventType
var last Event
scanned := 0
for len(events) > 0 {
e := <-events
types = append(types, e.Type)
if e.Type == FileScanned {
scanned++
if e.Total != 2 || e.Done < 1 || e.Done > 2 {
t.Errorf("progress = %d/%d", e.Done, e.Total)
}
}
last = e
}
if types[0] != ScanStarted || last.Type != ScanFinished || scanned != 2 {
t.Fatalf("event types = %v", types)
}
if last.Summary == nil || last.Summary.Added != 2 {
t.Errorf("final summary = %+v", last.Summary)
}
}
func TestMultipleRootsAreIndependent(t *testing.T) {
a, b := t.TempDir(), t.TempDir()
write(t, filepath.Join(a, "same/name.mp3"), "1")
write(t, filepath.Join(b, "same/name.mp3"), "1")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 2)
sum := mustScan(t, s, a, b)
if sum.Added != 2 || sum.Roots != 2 {
t.Fatalf("summary = %+v", sum)
}
if store.row(a, "same/name.mp3") == nil || store.row(b, "same/name.mp3") == nil {
t.Error("same relative path under two roots must be two files")
}
}
func TestOneBadRootDoesNotStopTheOthers(t *testing.T) {
good := t.TempDir()
write(t, filepath.Join(good, "a.mp3"), "1")
bad := filepath.Join(t.TempDir(), "gone")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
sum, err := s.Scan(context.Background(), []string{bad, good})
if err == nil {
t.Fatal("expected an error mentioning the bad root")
}
if sum.Added != 1 || store.row(good, "a.mp3") == nil {
t.Fatalf("good root not scanned: %+v", sum)
}
}
// ---- hub ----
func TestHubSlowSubscriberDoesNotBlock(t *testing.T) {
hub := NewHub()
slow, cancelSlow := hub.Subscribe(1)
defer cancelSlow()
fast, cancelFast := hub.Subscribe(100)
defer cancelFast()
done := make(chan struct{})
go func() {
for i := 0; i < 50; i++ {
hub.Publish(Event{Type: FileScanned, Done: i})
}
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Publish blocked on a slow subscriber")
}
if len(fast) != 50 {
t.Errorf("fast subscriber got %d events, want 50", len(fast))
}
if len(slow) != 1 {
t.Errorf("slow subscriber buffer = %d, want 1 (rest dropped)", len(slow))
}
}
func TestHubCancelClosesChannel(t *testing.T) {
hub := NewHub()
ch, cancel := hub.Subscribe(1)
cancel()
cancel() // idempotent
if _, ok := <-ch; ok {
t.Fatal("channel should be closed after cancel")
}
hub.Publish(Event{Type: FileScanned}) // must not panic
}
// ---- manager ----
func TestManagerRunsOneScanAtATime(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
reader := &fakeReader{delay: 100 * time.Millisecond}
s, _ := newScanner(newFakeStore(), reader, 1)
m := NewManager(s, []string{root})
if err := m.Start(context.Background()); err != nil {
t.Fatal(err)
}
if err := m.Start(context.Background()); !errors.Is(err, ErrScanRunning) {
t.Fatalf("second Start: err = %v, want ErrScanRunning", err)
}
if !m.Status().Running {
t.Error("status should report running")
}
deadline := time.Now().Add(3 * time.Second)
for m.Status().Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
st := m.Status()
if st.Running || st.Last == nil || st.Last.Added != 1 || st.LastError != "" {
t.Fatalf("status = %+v", st)
}
if err := m.Start(context.Background()); err != nil {
t.Fatalf("Start after finish: %v", err)
}
}
func TestManagerCallsOnFinishOnce(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
m := NewManager(s, []string{root})
type finished struct {
sum Summary
err error
}
got := make(chan finished, 4)
m.OnFinish(func(sum Summary, err error) { got <- finished{sum, err} })
if err := m.Start(context.Background()); err != nil {
t.Fatal(err)
}
select {
case f := <-got:
if f.err != nil || f.sum.Added != 1 {
t.Fatalf("finished = %+v", f)
}
case <-time.After(3 * time.Second):
t.Fatal("OnFinish was not called")
}
select {
case <-got:
t.Fatal("OnFinish called twice for one scan")
case <-time.After(100 * time.Millisecond):
}
}