fix(library): harden the scanner against special files, stat errors and lossy logging
- fix: skip FIFOs, sockets and devices named like audio files; opening a named pipe blocked a scan forever and left the manager stuck in "running" - fix: a file that cannot be stat'ed for a transient reason is unknown, not gone, so it is never marked missing or used as a rename source - fix: log the scan result from Manager.OnFinish instead of the lossy event hub, so the line can no longer be dropped - fix(storage): match artists and albums on a Unicode-aware key computed in Go (migration 0003, both drivers), so SQLite and PostgreSQL agree - test: FIFO, stat errors, OnFinish, Unicode artist and album matching
This commit is contained in:
parent
7bdd799907
commit
d6e9e09ce7
11 changed files with 230 additions and 27 deletions
|
|
@ -21,20 +21,17 @@ func (c scanControl) Status() library.Status { return c.m.Status() }
|
|||
// logScanResults writes one log line per finished scan. File paths are left
|
||||
// out on purpose: audio paths are private.
|
||||
func logScanResults(logger *zap.Logger, m *library.Manager) {
|
||||
events, _ := m.Subscribe(16)
|
||||
go func() {
|
||||
for e := range events {
|
||||
if e.Type != library.ScanFinished || e.Summary == nil {
|
||||
continue
|
||||
}
|
||||
s := e.Summary
|
||||
logger.Info("library scan finished",
|
||||
zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated),
|
||||
zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed),
|
||||
zap.Int("restored", s.Restored), zap.Int("missing", s.Missing),
|
||||
zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped),
|
||||
zap.String("error", e.Error),
|
||||
)
|
||||
m.OnFinish(func(s library.Summary, err error) {
|
||||
fields := []zap.Field{
|
||||
zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated),
|
||||
zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed),
|
||||
zap.Int("restored", s.Restored), zap.Int("missing", s.Missing),
|
||||
zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped),
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
logger.Warn("library scan finished with errors", append(fields, zap.Error(err))...)
|
||||
return
|
||||
}
|
||||
logger.Info("library scan finished", fields...)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,8 +24,17 @@ type Manager struct {
|
|||
scanner *Scanner
|
||||
roots []string
|
||||
|
||||
mu sync.Mutex
|
||||
status Status
|
||||
mu sync.Mutex
|
||||
status Status
|
||||
onFinish func(Summary, error)
|
||||
}
|
||||
|
||||
// OnFinish registers a function called once after every scan, with the result
|
||||
// of that scan. Unlike hub events it is never dropped. Set it before Start.
|
||||
func (m *Manager) OnFinish(fn func(Summary, error)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.onFinish = fn
|
||||
}
|
||||
|
||||
// NewManager builds a Manager for the given roots.
|
||||
|
|
@ -51,7 +60,6 @@ func (m *Manager) run(ctx context.Context) {
|
|||
sum, err := m.scanner.Scan(ctx, m.roots)
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.status.Running = false
|
||||
m.status.LastFinishedAt = time.Now()
|
||||
m.status.Last = &sum
|
||||
|
|
@ -59,6 +67,12 @@ func (m *Manager) run(ctx context.Context) {
|
|||
if err != nil {
|
||||
m.status.LastError = err.Error()
|
||||
}
|
||||
fn := m.onFinish
|
||||
m.mu.Unlock()
|
||||
|
||||
if fn != nil {
|
||||
fn(sum, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Status returns a snapshot of the scan state.
|
||||
|
|
|
|||
31
internal/library/plan_test.go
Normal file
31
internal/library/plan_test.go
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
package library
|
||||
|
||||
import "testing"
|
||||
|
||||
// A file whose stat failed for a transient reason is unknown, not gone.
|
||||
func TestPlanDoesNotFlagFilesWithStatErrors(t *testing.T) {
|
||||
existing := []ExistingFile{
|
||||
{ID: 1, FileInfo: FileInfo{RelPath: "flaky.mp3", Size: 10, MTimeUnix: 5}},
|
||||
{ID: 2, FileInfo: FileInfo{RelPath: "gone.mp3", Size: 20, MTimeUnix: 6}},
|
||||
}
|
||||
w := walkResult{
|
||||
files: []FileInfo{{RelPath: "new.mp3", Size: 10, MTimeUnix: 5}}, // same size+mtime as flaky.mp3
|
||||
failedFiles: map[string]bool{"flaky.mp3": true},
|
||||
}
|
||||
p := makePlan(w, existing)
|
||||
|
||||
for _, id := range p.missing {
|
||||
if id == 1 {
|
||||
t.Error("flaky.mp3 was marked missing after a stat error")
|
||||
}
|
||||
}
|
||||
if len(p.renames) != 0 {
|
||||
t.Errorf("renames = %+v: a file with a stat error must not be a rename source", p.renames)
|
||||
}
|
||||
if len(p.missing) != 1 || p.missing[0] != 2 {
|
||||
t.Errorf("missing = %v, want only gone.mp3", p.missing)
|
||||
}
|
||||
if len(p.parse) != 1 || p.parse[0].file.RelPath != "new.mp3" {
|
||||
t.Errorf("parse = %+v, want new.mp3 as a new file", p.parse)
|
||||
}
|
||||
}
|
||||
|
|
@ -208,6 +208,9 @@ type walkResult struct {
|
|||
// failedDirs are slash-separated relative paths (with a trailing slash)
|
||||
// that could not be read. Files below them are unknown, not gone.
|
||||
failedDirs []string
|
||||
// failedFiles are relative paths of files that could not be stat'ed for a
|
||||
// reason other than "does not exist". They are unknown, not gone.
|
||||
failedFiles map[string]bool
|
||||
}
|
||||
|
||||
func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) {
|
||||
|
|
@ -245,6 +248,8 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
|
|||
s.fail(root, rel, err, sum)
|
||||
if d != nil && d.IsDir() {
|
||||
out.failedDirs = append(out.failedDirs, rel+"/")
|
||||
} else {
|
||||
out.markFileFailed(rel)
|
||||
}
|
||||
return nil // keep going, do not descend into an unreadable dir
|
||||
}
|
||||
|
|
@ -255,9 +260,17 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
|
|||
if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] {
|
||||
return nil
|
||||
}
|
||||
if !d.Type().IsRegular() {
|
||||
// FIFOs, sockets and devices: opening one can block forever.
|
||||
sum.Skipped++
|
||||
return nil
|
||||
}
|
||||
info, ierr := d.Info()
|
||||
if ierr != nil {
|
||||
s.fail(root, rel, ierr, sum)
|
||||
if !errors.Is(ierr, fs.ErrNotExist) {
|
||||
out.markFileFailed(rel)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
|
||||
|
|
@ -266,6 +279,13 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
|
|||
return out, err
|
||||
}
|
||||
|
||||
func (w *walkResult) markFileFailed(rel string) {
|
||||
if w.failedFiles == nil {
|
||||
w.failedFiles = map[string]bool{}
|
||||
}
|
||||
w.failedFiles[rel] = true
|
||||
}
|
||||
|
||||
// ---- planning ----
|
||||
|
||||
type renameOp struct {
|
||||
|
|
@ -316,7 +336,7 @@ func makePlan(w walkResult, existing []ExistingFile) plan {
|
|||
|
||||
var gone []ExistingFile
|
||||
for _, e := range existing {
|
||||
if seen[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
|
||||
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
|
||||
continue
|
||||
}
|
||||
gone = append(gone, e)
|
||||
|
|
|
|||
|
|
@ -675,3 +675,34 @@ func TestManagerRunsOneScanAtATime(t *testing.T) {
|
|||
t.Fatalf("Start after finish: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerCallsOnFinishOnce(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write(t, filepath.Join(root, "a.mp3"), "1")
|
||||
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
|
||||
m := NewManager(s, []string{root})
|
||||
|
||||
type finished struct {
|
||||
sum Summary
|
||||
err error
|
||||
}
|
||||
got := make(chan finished, 4)
|
||||
m.OnFinish(func(sum Summary, err error) { got <- finished{sum, err} })
|
||||
|
||||
if err := m.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case f := <-got:
|
||||
if f.err != nil || f.sum.Added != 1 {
|
||||
t.Fatalf("finished = %+v", f)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("OnFinish was not called")
|
||||
}
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("OnFinish called twice for one scan")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
|
|
|||
48
internal/library/special_unix_test.go
Normal file
48
internal/library/special_unix_test.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
//go:build unix
|
||||
|
||||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A named pipe called *.mp3 blocks forever on open if anything reads it.
|
||||
func TestSpecialFilesAreSkipped(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write(t, filepath.Join(root, "real.mp3"), "1")
|
||||
if err := syscall.Mkfifo(filepath.Join(root, "pipe.mp3"), 0o644); err != nil {
|
||||
t.Skip("cannot create a FIFO:", err)
|
||||
}
|
||||
|
||||
store := newFakeStore()
|
||||
s, _ := newScanner(store, &fakeReader{}, 2)
|
||||
|
||||
type result struct {
|
||||
sum Summary
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
sum, err := s.Scan(context.Background(), []string{root})
|
||||
done <- result{sum, err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-done:
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
if r.sum.Found != 1 || r.sum.Added != 1 || r.sum.Skipped != 1 {
|
||||
t.Fatalf("summary = %+v, want the FIFO skipped", r.sum)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("scan hung on a FIFO")
|
||||
}
|
||||
if store.row(root, "pipe.mp3") != nil {
|
||||
t.Error("FIFO was indexed")
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
|
@ -15,8 +16,9 @@ import (
|
|||
type artistModel struct {
|
||||
bun.BaseModel `bun:"table:artists"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Name string `bun:"name,notnull"`
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Name string `bun:"name,notnull"`
|
||||
NameKey string `bun:"name_key,notnull"`
|
||||
}
|
||||
|
||||
type albumModel struct {
|
||||
|
|
@ -24,6 +26,7 @@ type albumModel struct {
|
|||
|
||||
ID int64 `bun:"id,pk,autoincrement"`
|
||||
Title string `bun:"title,notnull"`
|
||||
TitleKey string `bun:"title_key,notnull"`
|
||||
ArtistID int64 `bun:"artist_id,nullzero"`
|
||||
}
|
||||
|
||||
|
|
@ -206,14 +209,14 @@ func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error {
|
|||
|
||||
func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) {
|
||||
var a artistModel
|
||||
err := tx.NewSelect().Model(&a).Where("lower(name) = lower(?)", name).OrderExpr("id").Limit(1).Scan(ctx)
|
||||
err := tx.NewSelect().Model(&a).Where("name_key = ?", matchKey(name)).OrderExpr("id").Limit(1).Scan(ctx)
|
||||
if err == nil {
|
||||
return a.ID, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, fmt.Errorf("find artist: %w", err)
|
||||
}
|
||||
a = artistModel{Name: name}
|
||||
a = artistModel{Name: name, NameKey: matchKey(name)}
|
||||
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
|
||||
return 0, fmt.Errorf("insert artist: %w", err)
|
||||
}
|
||||
|
|
@ -223,7 +226,7 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, err
|
|||
func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) {
|
||||
var a albumModel
|
||||
err := tx.NewSelect().Model(&a).
|
||||
Where("lower(title) = lower(?)", title).Where("artist_id = ?", artistID).
|
||||
Where("title_key = ?", matchKey(title)).Where("artist_id = ?", artistID).
|
||||
OrderExpr("id").Limit(1).Scan(ctx)
|
||||
if err == nil {
|
||||
return a.ID, nil
|
||||
|
|
@ -231,7 +234,7 @@ func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID in
|
|||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, fmt.Errorf("find album: %w", err)
|
||||
}
|
||||
a = albumModel{Title: title, ArtistID: artistID}
|
||||
a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID}
|
||||
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
|
||||
return 0, fmt.Errorf("insert album: %w", err)
|
||||
}
|
||||
|
|
@ -251,6 +254,12 @@ func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []i
|
|||
return nil
|
||||
}
|
||||
|
||||
// matchKey is the case-insensitive key used to match artist and album names.
|
||||
// It is computed here, not in SQL, so SQLite and PostgreSQL agree on Unicode.
|
||||
func matchKey(s string) string {
|
||||
return strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if v != "" {
|
||||
|
|
|
|||
|
|
@ -80,6 +80,33 @@ func TestLibraryRepoSaveNewFile(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// SQLite's lower() only folds ASCII, so matching is done in Go on a stored key.
|
||||
func TestLibraryRepoMatchesUnicodeNamesCaseInsensitively(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
ctx := context.Background()
|
||||
repo := NewLibraryRepo(db)
|
||||
if _, err := repo.SaveFile(ctx, "/m", parsed("1.flac", "Hyperballad", "Post", "Björk"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, "/m", parsed("2.flac", "Army of Me", "POST", "BJÖRK"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, "/m", parsed("3.flac", "Isobel", "Ünïcode Album", "Åsa"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := repo.SaveFile(ctx, "/m", parsed("4.flac", "Other", "ÜNÏCODE ALBUM", "ÅSA"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 2 {
|
||||
t.Errorf("artists = %d, want 2 (Björk and Åsa, whatever the case)", n)
|
||||
}
|
||||
if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 2 {
|
||||
t.Errorf("albums = %d, want 2", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLibraryRepoTrackWithoutAlbum(t *testing.T) {
|
||||
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
|
||||
repo := NewLibraryRepo(db)
|
||||
|
|
|
|||
|
|
@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) {
|
|||
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applied != 2 {
|
||||
t.Errorf("applied migrations = %d, want 2", applied)
|
||||
if applied != 3 {
|
||||
t.Errorf("applied migrations = %d, want 3", applied)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
13
migrations/postgres/0003_name_keys.sql
Normal file
13
migrations/postgres/0003_name_keys.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
-- Case-insensitive matching keys for artists and albums.
|
||||
-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds
|
||||
-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL.
|
||||
|
||||
ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT '';
|
||||
|
||||
-- Best effort for rows that already exist; the scanner keeps them up to date.
|
||||
UPDATE artists SET name_key = lower(name);
|
||||
UPDATE albums SET title_key = lower(title);
|
||||
|
||||
CREATE INDEX idx_artists_name_key ON artists(name_key);
|
||||
CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id);
|
||||
13
migrations/sqlite/0003_name_keys.sql
Normal file
13
migrations/sqlite/0003_name_keys.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
-- Case-insensitive matching keys for artists and albums.
|
||||
-- The scanner computes them in Go (Unicode-aware); SQLite's lower() only folds
|
||||
-- ASCII, so matching in SQL would differ between SQLite and PostgreSQL.
|
||||
|
||||
ALTER TABLE artists ADD COLUMN name_key TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE albums ADD COLUMN title_key TEXT NOT NULL DEFAULT '';
|
||||
|
||||
-- Best effort for rows that already exist; the scanner keeps them up to date.
|
||||
UPDATE artists SET name_key = lower(name);
|
||||
UPDATE albums SET title_key = lower(title);
|
||||
|
||||
CREATE INDEX idx_artists_name_key ON artists(name_key);
|
||||
CREATE INDEX idx_albums_title_key ON albums(title_key, artist_id);
|
||||
Loading…
Reference in a new issue