fix(library): harden the scanner against special files, stat errors and lossy logging

- fix: skip FIFOs, sockets and devices named like audio files; opening a named
  pipe blocked a scan forever and left the manager stuck in "running"
- fix: a file that cannot be stat'ed for a transient reason is unknown, not
  gone, so it is never marked missing or used as a rename source
- fix: log the scan result from Manager.OnFinish instead of the lossy event
  hub, so the line can no longer be dropped
- fix(storage): match artists and albums on a Unicode-aware key computed in Go
  (migration 0003, both drivers), so SQLite and PostgreSQL agree
- test: FIFO, stat errors, OnFinish, Unicode artist and album matching
This commit is contained in:
tgiachi 2026-11-11 12:28:44 +01:00
commit d6e9e09ce7
11 changed files with 230 additions and 27 deletions

View file

@ -21,20 +21,17 @@ func (c scanControl) Status() library.Status { return c.m.Status() }
// logScanResults writes one log line per finished scan. File paths are left
// out on purpose: audio paths are private.
func logScanResults(logger *zap.Logger, m *library.Manager) {
events, _ := m.Subscribe(16)
go func() {
for e := range events {
if e.Type != library.ScanFinished || e.Summary == nil {
continue
}
s := e.Summary
logger.Info("library scan finished",
zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated),
zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed),
zap.Int("restored", s.Restored), zap.Int("missing", s.Missing),
zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped),
zap.String("error", e.Error),
)
m.OnFinish(func(s library.Summary, err error) {
fields := []zap.Field{
zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated),
zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed),
zap.Int("restored", s.Restored), zap.Int("missing", s.Missing),
zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped),
}
}()
if err != nil {
logger.Warn("library scan finished with errors", append(fields, zap.Error(err))...)
return
}
logger.Info("library scan finished", fields...)
})
}

View file

@ -24,8 +24,17 @@ type Manager struct {
scanner *Scanner
roots []string
mu sync.Mutex
status Status
mu sync.Mutex
status Status
onFinish func(Summary, error)
}
// OnFinish registers a function called once after every scan, with the result
// of that scan. Unlike hub events it is never dropped. Set it before Start.
func (m *Manager) OnFinish(fn func(Summary, error)) {
m.mu.Lock()
defer m.mu.Unlock()
m.onFinish = fn
}
// NewManager builds a Manager for the given roots.
@ -51,7 +60,6 @@ func (m *Manager) run(ctx context.Context) {
sum, err := m.scanner.Scan(ctx, m.roots)
m.mu.Lock()
defer m.mu.Unlock()
m.status.Running = false
m.status.LastFinishedAt = time.Now()
m.status.Last = &sum
@ -59,6 +67,12 @@ func (m *Manager) run(ctx context.Context) {
if err != nil {
m.status.LastError = err.Error()
}
fn := m.onFinish
m.mu.Unlock()
if fn != nil {
fn(sum, err)
}
}
// Status returns a snapshot of the scan state.

View file

@ -0,0 +1,31 @@
package library
import "testing"
// A file whose stat failed for a transient reason is unknown, not gone.
func TestPlanDoesNotFlagFilesWithStatErrors(t *testing.T) {
existing := []ExistingFile{
{ID: 1, FileInfo: FileInfo{RelPath: "flaky.mp3", Size: 10, MTimeUnix: 5}},
{ID: 2, FileInfo: FileInfo{RelPath: "gone.mp3", Size: 20, MTimeUnix: 6}},
}
w := walkResult{
files: []FileInfo{{RelPath: "new.mp3", Size: 10, MTimeUnix: 5}}, // same size+mtime as flaky.mp3
failedFiles: map[string]bool{"flaky.mp3": true},
}
p := makePlan(w, existing)
for _, id := range p.missing {
if id == 1 {
t.Error("flaky.mp3 was marked missing after a stat error")
}
}
if len(p.renames) != 0 {
t.Errorf("renames = %+v: a file with a stat error must not be a rename source", p.renames)
}
if len(p.missing) != 1 || p.missing[0] != 2 {
t.Errorf("missing = %v, want only gone.mp3", p.missing)
}
if len(p.parse) != 1 || p.parse[0].file.RelPath != "new.mp3" {
t.Errorf("parse = %+v, want new.mp3 as a new file", p.parse)
}
}

View file

@ -208,6 +208,9 @@ type walkResult struct {
// failedDirs are slash-separated relative paths (with a trailing slash)
// that could not be read. Files below them are unknown, not gone.
failedDirs []string
// failedFiles are relative paths of files that could not be stat'ed for a
// reason other than "does not exist". They are unknown, not gone.
failedFiles map[string]bool
}
func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) {
@ -245,6 +248,8 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
s.fail(root, rel, err, sum)
if d != nil && d.IsDir() {
out.failedDirs = append(out.failedDirs, rel+"/")
} else {
out.markFileFailed(rel)
}
return nil // keep going, do not descend into an unreadable dir
}
@ -255,9 +260,17 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] {
return nil
}
if !d.Type().IsRegular() {
// FIFOs, sockets and devices: opening one can block forever.
sum.Skipped++
return nil
}
info, ierr := d.Info()
if ierr != nil {
s.fail(root, rel, ierr, sum)
if !errors.Is(ierr, fs.ErrNotExist) {
out.markFileFailed(rel)
}
return nil
}
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
@ -266,6 +279,13 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
return out, err
}
func (w *walkResult) markFileFailed(rel string) {
if w.failedFiles == nil {
w.failedFiles = map[string]bool{}
}
w.failedFiles[rel] = true
}
// ---- planning ----
type renameOp struct {
@ -316,7 +336,7 @@ func makePlan(w walkResult, existing []ExistingFile) plan {
var gone []ExistingFile
for _, e := range existing {
if seen[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
continue
}
gone = append(gone, e)

View file

@ -675,3 +675,34 @@ func TestManagerRunsOneScanAtATime(t *testing.T) {
t.Fatalf("Start after finish: %v", err)
}
}
func TestManagerCallsOnFinishOnce(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
m := NewManager(s, []string{root})
type finished struct {
sum Summary
err error
}
got := make(chan finished, 4)
m.OnFinish(func(sum Summary, err error) { got <- finished{sum, err} })
if err := m.Start(context.Background()); err != nil {
t.Fatal(err)
}
select {
case f := <-got:
if f.err != nil || f.sum.Added != 1 {
t.Fatalf("finished = %+v", f)
}
case <-time.After(3 * time.Second):
t.Fatal("OnFinish was not called")
}
select {
case <-got:
t.Fatal("OnFinish called twice for one scan")
case <-time.After(100 * time.Millisecond):
}
}

View file

@ -0,0 +1,48 @@
//go:build unix
package library
import (
"context"
"path/filepath"
"syscall"
"testing"
"time"
)
// A named pipe called *.mp3 blocks forever on open if anything reads it.
func TestSpecialFilesAreSkipped(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "real.mp3"), "1")
if err := syscall.Mkfifo(filepath.Join(root, "pipe.mp3"), 0o644); err != nil {
t.Skip("cannot create a FIFO:", err)
}
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 2)
type result struct {
sum Summary
err error
}
done := make(chan result, 1)
go func() {
sum, err := s.Scan(context.Background(), []string{root})
done <- result{sum, err}
}()
select {
case r := <-done:
if r.err != nil {
t.Fatal(r.err)
}
if r.sum.Found != 1 || r.sum.Added != 1 || r.sum.Skipped != 1 {
t.Fatalf("summary = %+v, want the FIFO skipped", r.sum)
}
case <-time.After(5 * time.Second):
t.Fatal("scan hung on a FIFO")
}
if store.row(root, "pipe.mp3") != nil {
t.Error("FIFO was indexed")
}
}

View file

@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
"github.com/uptrace/bun"
@ -15,8 +16,9 @@ import (
type artistModel struct {
bun.BaseModel `bun:"table:artists"`
ID int64 `bun:"id,pk,autoincrement"`
Name string `bun:"name,notnull"`
ID int64 `bun:"id,pk,autoincrement"`
Name string `bun:"name,notnull"`
NameKey string `bun:"name_key,notnull"`
}
type albumModel struct {
@ -24,6 +26,7 @@ type albumModel struct {
ID int64 `bun:"id,pk,autoincrement"`
Title string `bun:"title,notnull"`
TitleKey string `bun:"title_key,notnull"`
ArtistID int64 `bun:"artist_id,nullzero"`
}
@ -206,14 +209,14 @@ func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error {
func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) {
var a artistModel
err := tx.NewSelect().Model(&a).Where("lower(name) = lower(?)", name).OrderExpr("id").Limit(1).Scan(ctx)
err := tx.NewSelect().Model(&a).Where("name_key = ?", matchKey(name)).OrderExpr("id").Limit(1).Scan(ctx)
if err == nil {
return a.ID, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, fmt.Errorf("find artist: %w", err)
}
a = artistModel{Name: name}
a = artistModel{Name: name, NameKey: matchKey(name)}
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
return 0, fmt.Errorf("insert artist: %w", err)
}
@ -223,7 +226,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, err
func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) {
var a albumModel
err := tx.NewSelect().Model(&a).
Where("lower(title) = lower(?)", title).Where("artist_id = ?", artistID).
Where("title_key = ?", matchKey(title)).Where("artist_id = ?", artistID).
OrderExpr("id").Limit(1).Scan(ctx)
if err == nil {
return a.ID, nil
@ -231,7 +234,7 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in
if !errors.Is(err, sql.ErrNoRows) {
return 0, fmt.Errorf("find album: %w", err)
}
a = albumModel{Title: title, ArtistID: artistID}
a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID}
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
return 0, fmt.Errorf("insert album: %w", err)
}
@ -251,6 +254,12 @@ func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []i
return nil
}
// matchKey is the case-insensitive key used to match artist and album names.
// It is computed here, not in SQL, so SQLite and PostgreSQL agree on Unicode.
func matchKey(s string) string {
return strings.ToLower(strings.TrimSpace(s))
}
func firstNonEmpty(values ...string) string {
for _, v := range values {
if v != "" {

View file

@ -80,6 +80,33 @@ func TestLibraryRepoSaveNewFile(t *testing.T) {
})
}
// SQLite's lower() only folds ASCII, so matching is done in Go on a stored key.
func TestLibraryRepoMatchesUnicodeNamesCaseInsensitively(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
if _, err := repo.SaveFile(ctx, "/m", parsed("1.flac", "Hyperballad", "Post", "Björk"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("2.flac", "Army of Me", "POST", "BJÖRK"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("3.flac", "Isobel", "Ünïcode Album", "Åsa"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("4.flac", "Other", "ÜNÏCODE ALBUM", "ÅSA"), 0); err != nil {
t.Fatal(err)
}
if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 2 {
t.Errorf("artists = %d, want 2 (Björk and Åsa, whatever the case)", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 2 {
t.Errorf("albums = %d, want 2", n)
}
})
}
func TestLibraryRepoTrackWithoutAlbum(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
repo := NewLibraryRepo(db)

View file

@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
t.Fatal(err)
}
if applied != 2 {
t.Errorf("applied migrations = %d, want 2", applied)
if applied != 3 {
t.Errorf("applied migrations = %d, want 3", applied)
}
})
}

View file

@ -0,0 +1,13 @@
-- Case-insensitive matching keys for artists and albums.
-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds
-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL.
ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT '';
ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT '';
-- Best effort for rows that already exist; the scanner keeps them up to date.
UPDATE artists SET name_key = lower(name);
UPDATE albums SET title_key = lower(title);
CREATE INDEX idx_artists_name_key ON artists(name_key);
CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id);

View file

@ -0,0 +1,13 @@
-- Case-insensitive matching keys for artists and albums.
-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds
-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL.
ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT '';
ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT '';
-- Best effort for rows that already exist; the scanner keeps them up to date.
UPDATE artists SET name_key = lower(name);
UPDATE albums SET title_key = lower(title);
CREATE INDEX idx_artists_name_key ON artists(name_key);
CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id);