diff --git a/cmd/sondad/scan.go b/cmd/sondad/scan.go index 4845d6e..908c7fb 100644 --- a/cmd/sondad/scan.go +++ b/cmd/sondad/scan.go @@ -21,20 +21,17 @@ func (c scanControl) Status() library.Status { return c.m.Status() } // logScanResults writes one log line per finished scan. File paths are left // out on purpose: audio paths are private. func logScanResults(logger *zap.Logger, m *library.Manager) { - events, _ := m.Subscribe(16) - go func() { - for e := range events { - if e.Type != library.ScanFinished || e.Summary == nil { - continue - } - s := e.Summary - logger.Info("library scan finished", - zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated), - zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed), - zap.Int("restored", s.Restored), zap.Int("missing", s.Missing), - zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped), - zap.String("error", e.Error), - ) + m.OnFinish(func(s library.Summary, err error) { + fields := []zap.Field{ + zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated), + zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed), + zap.Int("restored", s.Restored), zap.Int("missing", s.Missing), + zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped), } - }() + if err != nil { + logger.Warn("library scan finished with errors", append(fields, zap.Error(err))...) + return + } + logger.Info("library scan finished", fields...) + }) } diff --git a/internal/library/manager.go b/internal/library/manager.go index e7579f3..455386c 100644 --- a/internal/library/manager.go +++ b/internal/library/manager.go @@ -24,8 +24,17 @@ type Manager struct { scanner *Scanner roots []string - mu sync.Mutex - status Status + mu sync.Mutex + status Status + onFinish func(Summary, error) +} + +// OnFinish registers a function called once after every scan, with the result +// of that scan. Unlike hub events it is never dropped. Set it before Start. +func (m *Manager) OnFinish(fn func(Summary, error)) { + m.mu.Lock() + defer m.mu.Unlock() + m.onFinish = fn } // NewManager builds a Manager for the given roots. @@ -51,7 +60,6 @@ func (m *Manager) run(ctx context.Context) { sum, err := m.scanner.Scan(ctx, m.roots) m.mu.Lock() - defer m.mu.Unlock() m.status.Running = false m.status.LastFinishedAt = time.Now() m.status.Last = &sum @@ -59,6 +67,12 @@ func (m *Manager) run(ctx context.Context) { if err != nil { m.status.LastError = err.Error() } + fn := m.onFinish + m.mu.Unlock() + + if fn != nil { + fn(sum, err) + } } // Status returns a snapshot of the scan state. diff --git a/internal/library/plan_test.go b/internal/library/plan_test.go new file mode 100644 index 0000000..0af7096 --- /dev/null +++ b/internal/library/plan_test.go @@ -0,0 +1,31 @@ +package library + +import "testing" + +// A file whose stat failed for a transient reason is unknown, not gone. +func TestPlanDoesNotFlagFilesWithStatErrors(t *testing.T) { + existing := []ExistingFile{ + {ID: 1, FileInfo: FileInfo{RelPath: "flaky.mp3", Size: 10, MTimeUnix: 5}}, + {ID: 2, FileInfo: FileInfo{RelPath: "gone.mp3", Size: 20, MTimeUnix: 6}}, + } + w := walkResult{ + files: []FileInfo{{RelPath: "new.mp3", Size: 10, MTimeUnix: 5}}, // same size+mtime as flaky.mp3 + failedFiles: map[string]bool{"flaky.mp3": true}, + } + p := makePlan(w, existing) + + for _, id := range p.missing { + if id == 1 { + t.Error("flaky.mp3 was marked missing after a stat error") + } + } + if len(p.renames) != 0 { + t.Errorf("renames = %+v: a file with a stat error must not be a rename source", p.renames) + } + if len(p.missing) != 1 || p.missing[0] != 2 { + t.Errorf("missing = %v, want only gone.mp3", p.missing) + } + if len(p.parse) != 1 || p.parse[0].file.RelPath != "new.mp3" { + t.Errorf("parse = %+v, want new.mp3 as a new file", p.parse) + } +} diff --git a/internal/library/scanner.go b/internal/library/scanner.go index 44710d3..79c5888 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -208,6 +208,9 @@ type walkResult struct { // failedDirs are slash-separated relative paths (with a trailing slash) // that could not be read. Files below them are unknown, not gone. failedDirs []string + // failedFiles are relative paths of files that could not be stat'ed for a + // reason other than "does not exist". They are unknown, not gone. + failedFiles map[string]bool } func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) { @@ -245,6 +248,8 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu s.fail(root, rel, err, sum) if d != nil && d.IsDir() { out.failedDirs = append(out.failedDirs, rel+"/") + } else { + out.markFileFailed(rel) } return nil // keep going, do not descend into an unreadable dir } @@ -255,9 +260,17 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] { return nil } + if !d.Type().IsRegular() { + // FIFOs, sockets and devices: opening one can block forever. + sum.Skipped++ + return nil + } info, ierr := d.Info() if ierr != nil { s.fail(root, rel, ierr, sum) + if !errors.Is(ierr, fs.ErrNotExist) { + out.markFileFailed(rel) + } return nil } out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}) @@ -266,6 +279,13 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu return out, err } +func (w *walkResult) markFileFailed(rel string) { + if w.failedFiles == nil { + w.failedFiles = map[string]bool{} + } + w.failedFiles[rel] = true +} + // ---- planning ---- type renameOp struct { @@ -316,7 +336,7 @@ func makePlan(w walkResult, existing []ExistingFile) plan { var gone []ExistingFile for _, e := range existing { - if seen[e.RelPath] || underAny(e.RelPath, w.failedDirs) { + if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) { continue } gone = append(gone, e) diff --git a/internal/library/scanner_test.go b/internal/library/scanner_test.go index 0104ea9..dd7a46a 100644 --- a/internal/library/scanner_test.go +++ b/internal/library/scanner_test.go @@ -675,3 +675,34 @@ func TestManagerRunsOneScanAtATime(t *testing.T) { t.Fatalf("Start after finish: %v", err) } } + +func TestManagerCallsOnFinishOnce(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + s, _ := newScanner(newFakeStore(), &fakeReader{}, 1) + m := NewManager(s, []string{root}) + + type finished struct { + sum Summary + err error + } + got := make(chan finished, 4) + m.OnFinish(func(sum Summary, err error) { got <- finished{sum, err} }) + + if err := m.Start(context.Background()); err != nil { + t.Fatal(err) + } + select { + case f := <-got: + if f.err != nil || f.sum.Added != 1 { + t.Fatalf("finished = %+v", f) + } + case <-time.After(3 * time.Second): + t.Fatal("OnFinish was not called") + } + select { + case <-got: + t.Fatal("OnFinish called twice for one scan") + case <-time.After(100 * time.Millisecond): + } +} diff --git a/internal/library/special_unix_test.go b/internal/library/special_unix_test.go new file mode 100644 index 0000000..04f9940 --- /dev/null +++ b/internal/library/special_unix_test.go @@ -0,0 +1,48 @@ +//go:build unix + +package library + +import ( + "context" + "path/filepath" + "syscall" + "testing" + "time" +) + +// A named pipe called *.mp3 blocks forever on open if anything reads it. +func TestSpecialFilesAreSkipped(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "real.mp3"), "1") + if err := syscall.Mkfifo(filepath.Join(root, "pipe.mp3"), 0o644); err != nil { + t.Skip("cannot create a FIFO:", err) + } + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 2) + + type result struct { + sum Summary + err error + } + done := make(chan result, 1) + go func() { + sum, err := s.Scan(context.Background(), []string{root}) + done <- result{sum, err} + }() + + select { + case r := <-done: + if r.err != nil { + t.Fatal(r.err) + } + if r.sum.Found != 1 || r.sum.Added != 1 || r.sum.Skipped != 1 { + t.Fatalf("summary = %+v, want the FIFO skipped", r.sum) + } + case <-time.After(5 * time.Second): + t.Fatal("scan hung on a FIFO") + } + if store.row(root, "pipe.mp3") != nil { + t.Error("FIFO was indexed") + } +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 1666b91..37860fa 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strings" "time" "github.com/uptrace/bun" @@ -15,8 +16,9 @@ import ( type artistModel struct { bun.BaseModel `bun:"table:artists"` - ID int64 `bun:"id,pk,autoincrement"` - Name string `bun:"name,notnull"` + ID int64 `bun:"id,pk,autoincrement"` + Name string `bun:"name,notnull"` + NameKey string `bun:"name_key,notnull"` } type albumModel struct { @@ -24,6 +26,7 @@ type albumModel struct { ID int64 `bun:"id,pk,autoincrement"` Title string `bun:"title,notnull"` + TitleKey string `bun:"title_key,notnull"` ArtistID int64 `bun:"artist_id,nullzero"` } @@ -206,14 +209,14 @@ func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error { func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) { var a artistModel - err := tx.NewSelect().Model(&a).Where("lower(name) = lower(?)", name).OrderExpr("id").Limit(1).Scan(ctx) + err := tx.NewSelect().Model(&a).Where("name_key = ?", matchKey(name)).OrderExpr("id").Limit(1).Scan(ctx) if err == nil { return a.ID, nil } if !errors.Is(err, sql.ErrNoRows) { return 0, fmt.Errorf("find artist: %w", err) } - a = artistModel{Name: name} + a = artistModel{Name: name, NameKey: matchKey(name)} if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { return 0, fmt.Errorf("insert artist: %w", err) } @@ -223,7 +226,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, err func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) { var a albumModel err := tx.NewSelect().Model(&a). - Where("lower(title) = lower(?)", title).Where("artist_id = ?", artistID). + Where("title_key = ?", matchKey(title)).Where("artist_id = ?", artistID). OrderExpr("id").Limit(1).Scan(ctx) if err == nil { return a.ID, nil @@ -231,7 +234,7 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in if !errors.Is(err, sql.ErrNoRows) { return 0, fmt.Errorf("find album: %w", err) } - a = albumModel{Title: title, ArtistID: artistID} + a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID} if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { return 0, fmt.Errorf("insert album: %w", err) } @@ -251,6 +254,12 @@ func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []i return nil } +// matchKey is the case-insensitive key used to match artist and album names. +// It is computed here, not in SQL, so SQLite and PostgreSQL agree on Unicode. +func matchKey(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + func firstNonEmpty(values ...string) string { for _, v := range values { if v != "" { diff --git a/internal/storage/library_test.go b/internal/storage/library_test.go index 6171947..53ae88c 100644 --- a/internal/storage/library_test.go +++ b/internal/storage/library_test.go @@ -80,6 +80,33 @@ func TestLibraryRepoSaveNewFile(t *testing.T) { }) } +// SQLite's lower() only folds ASCII, so matching is done in Go on a stored key. +func TestLibraryRepoMatchesUnicodeNamesCaseInsensitively(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + if _, err := repo.SaveFile(ctx, "/m", parsed("1.flac", "Hyperballad", "Post", "Björk"), 0); err != nil { + t.Fatal(err) + } + if _, err := repo.SaveFile(ctx, "/m", parsed("2.flac", "Army of Me", "POST", "BJÖRK"), 0); err != nil { + t.Fatal(err) + } + if _, err := repo.SaveFile(ctx, "/m", parsed("3.flac", "Isobel", "Ünïcode Album", "Åsa"), 0); err != nil { + t.Fatal(err) + } + if _, err := repo.SaveFile(ctx, "/m", parsed("4.flac", "Other", "ÜNÏCODE ALBUM", "ÅSA"), 0); err != nil { + t.Fatal(err) + } + + if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 2 { + t.Errorf("artists = %d, want 2 (Björk and Åsa, whatever the case)", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 2 { + t.Errorf("albums = %d, want 2", n) + } + }) +} + func TestLibraryRepoTrackWithoutAlbum(t *testing.T) { forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { repo := NewLibraryRepo(db) diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index af3d006..8801c88 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 2 { - t.Errorf("applied migrations = %d, want 2", applied) + if applied != 3 { + t.Errorf("applied migrations = %d, want 3", applied) } }) } diff --git a/migrations/postgres/0003_name_keys.sql b/migrations/postgres/0003_name_keys.sql new file mode 100644 index 0000000..3d67538 --- /dev/null +++ b/migrations/postgres/0003_name_keys.sql @@ -0,0 +1,13 @@ +-- Case-insensitive matching keys for artists and albums. +-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds +-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL. + +ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT ''; + +-- Best effort for rows that already exist; the scanner keeps them up to date. +UPDATE artists SET name_key = lower(name); +UPDATE albums SET title_key = lower(title); + +CREATE INDEX idx_artists_name_key ON artists(name_key); +CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id); diff --git a/migrations/sqlite/0003_name_keys.sql b/migrations/sqlite/0003_name_keys.sql new file mode 100644 index 0000000..3d67538 --- /dev/null +++ b/migrations/sqlite/0003_name_keys.sql @@ -0,0 +1,13 @@ +-- Case-insensitive matching keys for artists and albums. +-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds +-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL. + +ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT ''; +ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT ''; + +-- Best effort for rows that already exist; the scanner keeps them up to date. +UPDATE artists SET name_key = lower(name); +UPDATE albums SET title_key = lower(title); + +CREATE INDEX idx_artists_name_key ON artists(name_key); +CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id);