From 4c7e99bd6a6089a321db4648b604d6e41ff940f8 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 12:16:45 +0100 Subject: [PATCH] feat(library): add concurrent read-only scanner with progress events - feat(library): walk each root, plan the changes, read tags with N parallel workers and write from a single goroutine - feat(library): detect renames by size and mtime only when the match is unambiguous; mark vanished files missing instead of deleting them - fix-safe: an unreadable root, an unreadable subdirectory or an interrupted scan never marks files as missing; symlinks are skipped - feat(library): event hub that fans scan progress out to subscribers without ever blocking the scan (ready for a websocket client) - feat(library): TagLib (WASM, no CGO) reader for MP3 and FLAC tags and audio properties, with title and artist fallbacks - feat(storage): library repository for SQLite and PostgreSQL - feat(api): POST/GET /api/v1/library/scan to start a scan and read its status - feat(config): library.scan_workers --- cmd/sondad/main.go | 6 + cmd/sondad/scan.go | 40 ++ go.mod | 2 + go.sum | 4 + internal/api/http/api_test.go | 6 + internal/api/http/handlers_library.go | 70 +++ internal/api/http/library_test.go | 80 ++++ internal/api/http/router.go | 11 + internal/config/config.go | 7 +- internal/config/config_test.go | 4 + internal/library/hub.go | 53 +++ internal/library/manager.go | 74 ++++ internal/library/scanner.go | 335 ++++++++++++++ internal/library/scanner_test.go | 608 ++++++++++++++++++++++++++ internal/library/taglib.go | 93 ++++ internal/library/taglib_test.go | 123 ++++++ internal/library/types.go | 128 ++++++ internal/storage/library.go | 261 +++++++++++ internal/storage/library_test.go | 246 +++++++++++ sonda.example.toml | 1 + 20 files changed, 2151 insertions(+), 1 deletion(-) create mode 100644 cmd/sondad/scan.go create mode 100644 internal/api/http/handlers_library.go create mode 100644 internal/api/http/library_test.go create mode 100644 internal/library/hub.go create mode 100644 internal/library/manager.go create mode 100644 internal/library/scanner.go create mode 100644 internal/library/scanner_test.go create mode 100644 internal/library/taglib.go create mode 100644 internal/library/taglib_test.go create mode 100644 internal/library/types.go create mode 100644 internal/storage/library.go create mode 100644 internal/storage/library_test.go diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 10f2897..1ac52cf 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -17,6 +17,7 @@ import ( httpapi "codeberg.org/squidd/sonda/internal/api/http" "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/config" + "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" "codeberg.org/squidd/sonda/internal/storage" "codeberg.org/squidd/sonda/migrations" @@ -82,11 +83,16 @@ func run() error { if cfg.Log.Env == "production" { gin.SetMode(gin.ReleaseMode) } + scanner := library.NewScanner(storage.NewLibraryRepo(db), library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers) + scans := library.NewManager(scanner, cfg.Library.Roots) + logScanResults(logger, scans) + router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Auth: auth.NewService(storage.NewUserRepo(db), tokens), Tokens: tokens, Listening: listening.NewService(storage.NewListeningRepo(db)), + Scan: scanControl{ctx: ctx, m: scans}, }) // No WriteTimeout: audio streaming (later) needs long responses. diff --git a/cmd/sondad/scan.go b/cmd/sondad/scan.go new file mode 100644 index 0000000..4845d6e --- /dev/null +++ b/cmd/sondad/scan.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/library" +) + +// scanControl lets the HTTP layer start scans that live as long as the +// application, not as long as a single request. +type scanControl struct { + ctx context.Context + m *library.Manager +} + +func (c scanControl) Start() error { return c.m.Start(c.ctx) } +func (c scanControl) Status() library.Status { return c.m.Status() } + +// logScanResults writes one log line per finished scan. File paths are left +// out on purpose: audio paths are private. +func logScanResults(logger *zap.Logger, m *library.Manager) { + events, _ := m.Subscribe(16) + go func() { + for e := range events { + if e.Type != library.ScanFinished || e.Summary == nil { + continue + } + s := e.Summary + logger.Info("library scan finished", + zap.Int("found", s.Found), zap.Int("added", s.Added), zap.Int("updated", s.Updated), + zap.Int("unchanged", s.Unchanged), zap.Int("renamed", s.Renamed), + zap.Int("restored", s.Restored), zap.Int("missing", s.Missing), + zap.Int("failed", s.Failed), zap.Int("skipped", s.Skipped), + zap.String("error", e.Error), + ) + } + }() +} diff --git a/go.mod b/go.mod index 0f3fc09..1d89fc3 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/uptrace/bun v1.3.0 github.com/uptrace/bun/dialect/pgdialect v1.3.0 github.com/uptrace/bun/dialect/sqlitedialect v1.3.0 + go.senan.xyz/taglib v0.15.0 go.uber.org/zap v1.28.0 golang.org/x/crypto v0.58.0 modernc.org/sqlite v1.62.0 @@ -44,6 +45,7 @@ require ( github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/tetratelabs/wazero v1.11.1-0.20260428013916-2bbd517b7633 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect diff --git a/go.sum b/go.sum index 387926c..59a8911 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tetratelabs/wazero v1.11.1-0.20260428013916-2bbd517b7633 h1:6GN/lazdqr69FIzz1U6c4TF/ppE2dInMR4GzU9QKxjg= +github.com/tetratelabs/wazero v1.11.1-0.20260428013916-2bbd517b7633/go.mod h1:3ghOSSWYnzX0zd/3Ns4ni2tKxcXDE9/QgkwuH1PW3Rs= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= @@ -107,6 +109,8 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.senan.xyz/taglib v0.15.0 h1:8z25qOiNm4o58kV37INltLGRKHelz1QqGBIZ/HWMyjM= +go.senan.xyz/taglib v0.15.0/go.mod h1:+k5CamBu88xgydgNGJjugYVeafoCCswoGjpw5w5CvD4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 8c3ed37..bb047a8 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -26,6 +26,11 @@ func init() { gin.SetMode(gin.TestMode) } // newTestAPI wires real services on a temporary SQLite database. func newTestAPI(t *testing.T) http.Handler { + t.Helper() + return newTestAPIWithScan(t, &fakeScan{}) +} + +func newTestAPIWithScan(t *testing.T, scan scanService) http.Handler { t.Helper() db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "api.db")) if err != nil { @@ -49,6 +54,7 @@ func newTestAPI(t *testing.T) http.Handler { Auth: auth.NewService(storage.NewUserRepo(db), tokens), Tokens: tokens, Listening: listening.NewService(storage.NewListeningRepo(db)), + Scan: scan, }) } diff --git a/internal/api/http/handlers_library.go b/internal/api/http/handlers_library.go new file mode 100644 index 0000000..39041a3 --- /dev/null +++ b/internal/api/http/handlers_library.go @@ -0,0 +1,70 @@ +package httpapi + +import ( + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/library" +) + +type libraryHandlers struct{ scan scanService } + +type summaryJSON struct { + Roots int `json:"roots"` + Found int `json:"found"` + Added int `json:"added"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Renamed int `json:"renamed"` + Restored int `json:"restored"` + Missing int `json:"missing"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` +} + +type scanStatusJSON struct { + Running bool `json:"running"` + StartedAt *string `json:"started_at,omitempty"` + LastFinishedAt *string `json:"last_finished_at,omitempty"` + Last *summaryJSON `json:"last,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +// startScan launches a background scan of the configured roots. +func (h *libraryHandlers) startScan(c *gin.Context) { + err := h.scan.Start() + switch { + case errors.Is(err, library.ErrScanRunning): + writeError(c, http.StatusConflict, "scan_running", err.Error()) + case err != nil: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + default: + c.JSON(http.StatusAccepted, gin.H{"status": "started"}) + } +} + +// scanStatus reports whether a scan is running and how the last one ended. +func (h *libraryHandlers) scanStatus(c *gin.Context) { + st := h.scan.Status() + out := scanStatusJSON{Running: st.Running, LastError: st.LastError} + if !st.StartedAt.IsZero() { + out.StartedAt = rfc3339(st.StartedAt) + } + if !st.LastFinishedAt.IsZero() { + out.LastFinishedAt = rfc3339(st.LastFinishedAt) + } + if st.Last != nil { + s := summaryJSON(*st.Last) + out.Last = &s + } + c.JSON(http.StatusOK, out) +} + +func rfc3339(t time.Time) *string { + s := t.UTC().Format(time.RFC3339) + return &s +} diff --git a/internal/api/http/library_test.go b/internal/api/http/library_test.go new file mode 100644 index 0000000..b9824c6 --- /dev/null +++ b/internal/api/http/library_test.go @@ -0,0 +1,80 @@ +package httpapi + +import ( + "net/http" + "testing" + "time" + + "codeberg.org/squidd/sonda/internal/library" +) + +type fakeScan struct { + running bool + starts int + status library.Status +} + +func (f *fakeScan) Start() error { + if f.running { + return library.ErrScanRunning + } + f.running = true + f.starts++ + return nil +} + +func (f *fakeScan) Status() library.Status { + s := f.status + s.Running = f.running + return s +} + +func TestScanEndpointsRequireAuth(t *testing.T) { + h := newTestAPIWithScan(t, &fakeScan{}) + for _, method := range []string{"POST", "GET"} { + if r := do(t, h, method, "/api/v1/library/scan", "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("%s without token: %d", method, r.Code) + } + } +} + +func TestScanStartAndConflict(t *testing.T) { + scan := &fakeScan{} + h := newTestAPIWithScan(t, scan) + tok := registerAndLogin(t, h) + + r := do(t, h, "POST", "/api/v1/library/scan", tok, nil) + if r.Code != http.StatusAccepted || r.Body["status"] != "started" { + t.Fatalf("start: %d %v", r.Code, r.Body) + } + r = do(t, h, "POST", "/api/v1/library/scan", tok, nil) + if r.Code != http.StatusConflict || errCode(r) != "scan_running" { + t.Fatalf("second start: %d %v", r.Code, r.Body) + } + if scan.starts != 1 { + t.Errorf("starts = %d, want 1", scan.starts) + } +} + +func TestScanStatus(t *testing.T) { + scan := &fakeScan{status: library.Status{ + LastFinishedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Last: &library.Summary{Roots: 1, Found: 10, Added: 7, Failed: 1}, + LastError: "boom", + }} + h := newTestAPIWithScan(t, scan) + tok := registerAndLogin(t, h) + + r := do(t, h, "GET", "/api/v1/library/scan", tok, nil) + if r.Code != http.StatusOK { + t.Fatalf("status: %d %v", r.Code, r.Body) + } + if r.Body["running"] != false || r.Body["last_error"] != "boom" || + r.Body["last_finished_at"] != "2026-01-02T03:04:05Z" { + t.Errorf("body = %v", r.Body) + } + last, _ := r.Body["last"].(map[string]any) + if last["found"] != float64(10) || last["added"] != float64(7) || last["failed"] != float64(1) { + t.Errorf("last = %v", last) + } +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 7c4b190..1d85add 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -11,6 +11,7 @@ import ( "go.uber.org/zap" "codeberg.org/squidd/sonda/internal/auth" + "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" ) @@ -30,12 +31,18 @@ type listenService interface { Record(ctx context.Context, e listening.Event) (bool, error) } +type scanService interface { + Start() error + Status() library.Status +} + // Deps are the services the API needs. type Deps struct { Logger *zap.Logger Auth authService Tokens tokenParser Listening listenService + Scan scanService } // NewRouter builds the gin engine with all routes. @@ -56,6 +63,10 @@ func NewRouter(d Deps) *gin.Engine { l := &listenHandlers{svc: d.Listening} protected.POST("/listens", l.record) + lib := &libraryHandlers{scan: d.Scan} + protected.POST("/library/scan", lib.startScan) + protected.GET("/library/scan", lib.scanStatus) + return r } diff --git a/internal/config/config.go b/internal/config/config.go index 17b31a6..72c43f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,7 +32,8 @@ type Server struct { // Library holds the music directories. They are treated as read-only. type Library struct { - Roots []string `toml:"roots"` + Roots []string `toml:"roots"` + ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan } // Database selects and configures the storage backend. @@ -75,6 +76,7 @@ type LastFM struct { func Default() Config { return Config{ Server: Server{Listen: "127.0.0.1:8080"}, + Library: Library{ScanWorkers: 4}, Database: Database{Driver: "sqlite", Path: "sonda.db"}, Log: Log{Env: "production"}, Auth: Auth{TokenTTLHours: 168}, // one week @@ -154,6 +156,9 @@ func (c Config) Validate() error { if c.Auth.TokenTTLHours <= 0 { errs = append(errs, errors.New("auth.token_ttl_hours must be greater than 0")) } + if c.Library.ScanWorkers < 1 { + errs = append(errs, errors.New("library.scan_workers must be at least 1")) + } if len(c.Library.Roots) == 0 { errs = append(errs, errors.New("library.roots needs at least one directory")) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d2e40bb..0533d8b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -34,6 +34,9 @@ func TestLoadEmptyPathReturnsDefaults(t *testing.T) { if cfg.Log.Env != "production" { t.Errorf("env = %q", cfg.Log.Env) } + if cfg.Library.ScanWorkers != 4 { + t.Errorf("scan workers = %d", cfg.Library.ScanWorkers) + } if cfg.Auth.TokenTTL() != 168*time.Hour { t.Errorf("token ttl = %v", cfg.Auth.TokenTTL()) } @@ -134,6 +137,7 @@ func TestValidate(t *testing.T) { }, ""}, {"no secret", func(c *Config) { c.Auth.JWTSecret = "" }, "auth.jwt_secret"}, {"short secret", func(c *Config) { c.Auth.JWTSecret = "short" }, "auth.jwt_secret"}, + {"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"}, {"bad ttl", func(c *Config) { c.Auth.TokenTTLHours = 0 }, "auth.token_ttl_hours"}, } for _, tt := range tests { diff --git a/internal/library/hub.go b/internal/library/hub.go new file mode 100644 index 0000000..b38cb28 --- /dev/null +++ b/internal/library/hub.go @@ -0,0 +1,53 @@ +package library + +import "sync" + +// Hub fans scan events out to any number of subscribers, for example one +// websocket connection each. Publishing never blocks: a subscriber whose +// buffer is full simply misses events. +type Hub struct { + mu sync.Mutex + next int + subs map[int]chan Event +} + +// NewHub creates an empty hub. +func NewHub() *Hub { return &Hub{subs: map[int]chan Event{}} } + +// Subscribe returns a channel of events and a function that stops the +// subscription and closes the channel. The function is safe to call twice. +func (h *Hub) Subscribe(buffer int) (<-chan Event, func()) { + if buffer < 1 { + buffer = 1 + } + ch := make(chan Event, buffer) + + h.mu.Lock() + id := h.next + h.next++ + h.subs[id] = ch + h.mu.Unlock() + + var once sync.Once + cancel := func() { + once.Do(func() { + h.mu.Lock() + delete(h.subs, id) + close(ch) + h.mu.Unlock() + }) + } + return ch, cancel +} + +// Publish delivers e to every subscriber that has room for it. +func (h *Hub) Publish(e Event) { + h.mu.Lock() + defer h.mu.Unlock() + for _, ch := range h.subs { + select { + case ch <- e: + default: // slow subscriber: drop instead of stalling the scan + } + } +} diff --git a/internal/library/manager.go b/internal/library/manager.go new file mode 100644 index 0000000..e7579f3 --- /dev/null +++ b/internal/library/manager.go @@ -0,0 +1,74 @@ +package library + +import ( + "context" + "errors" + "sync" + "time" +) + +// ErrScanRunning is returned by Start while a scan is in progress. +var ErrScanRunning = errors.New("a scan is already running") + +// Status describes the scan state for the API. +type Status struct { + Running bool + StartedAt time.Time + LastFinishedAt time.Time + Last *Summary // result of the last finished scan + LastError string +} + +// Manager runs at most one background scan of the configured roots. +type Manager struct { + scanner *Scanner + roots []string + + mu sync.Mutex + status Status +} + +// NewManager builds a Manager for the given roots. +func NewManager(scanner *Scanner, roots []string) *Manager { + return &Manager{scanner: scanner, roots: roots} +} + +// Start launches a scan in the background. ctx bounds the scan, so pass a +// context that lives as long as the application, not an HTTP request. +func (m *Manager) Start(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.status.Running { + return ErrScanRunning + } + m.status.Running = true + m.status.StartedAt = time.Now() + go m.run(ctx) + return nil +} + +func (m *Manager) run(ctx context.Context) { + sum, err := m.scanner.Scan(ctx, m.roots) + + m.mu.Lock() + defer m.mu.Unlock() + m.status.Running = false + m.status.LastFinishedAt = time.Now() + m.status.Last = &sum + m.status.LastError = "" + if err != nil { + m.status.LastError = err.Error() + } +} + +// Status returns a snapshot of the scan state. +func (m *Manager) Status() Status { + m.mu.Lock() + defer m.mu.Unlock() + return m.status +} + +// Subscribe returns a channel of scan events (see Hub.Subscribe). +func (m *Manager) Subscribe(buffer int) (<-chan Event, func()) { + return m.scanner.hub.Subscribe(buffer) +} diff --git a/internal/library/scanner.go b/internal/library/scanner.go new file mode 100644 index 0000000..28b04e5 --- /dev/null +++ b/internal/library/scanner.go @@ -0,0 +1,335 @@ +package library + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + "sync" + "time" +) + +var audioExts = map[string]bool{".mp3": true, ".flac": true} + +// Scanner indexes the audio files under a set of roots. +// +// Per root it walks the tree (cheap, stat only), decides what changed, then +// reads tags with a pool of workers. A single goroutine writes to the store. +type Scanner struct { + store Store + reader TagReader + hub *Hub + workers int + now func() time.Time +} + +// NewScanner builds a scanner that parses files with the given number of workers. +func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner { + if workers < 1 { + workers = 1 + } + return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now} +} + +// Scan indexes every root in turn. An unreadable root is reported in the +// returned error and the other roots are still scanned. +func (s *Scanner) Scan(ctx context.Context, roots []string) (Summary, error) { + total := Summary{Roots: len(roots)} + s.publish(Event{Type: ScanStarted}) + + var errs []error + for _, root := range roots { + sum, err := s.scanRoot(ctx, root) + total.add(sum) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", root, err)) + if ctx.Err() != nil { + break + } + } + } + + err := errors.Join(errs...) + fin := Event{Type: ScanFinished, Summary: &total} + if err != nil { + fin.Error = err.Error() + } + s.publish(fin) + return total, err +} + +func (s *Scanner) publish(e Event) { + e.Time = s.now() + s.hub.Publish(e) +} + +type parseJob struct { + file FileInfo + existingID int64 +} + +type parseResult struct { + job parseJob + meta Metadata + err error +} + +func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) { + var sum Summary + + walked, err := s.walk(ctx, root, &sum) + if err != nil { + // The root itself is unreadable (for example an unmounted NAS). + // Do not touch the database: nothing is known about its files. + return sum, err + } + sum.Found = len(walked.files) + + existing, err := s.store.ExistingFiles(ctx, root) + if err != nil { + return sum, fmt.Errorf("list indexed files: %w", err) + } + p := makePlan(walked, existing) + sum.Unchanged = p.unchanged + + if err := s.parseAndSave(ctx, root, p.parse, &sum); err != nil { + return sum, err + } + if err := ctx.Err(); err != nil { + return sum, err // interrupted: never mark files missing on a partial scan + } + + for _, r := range p.renames { + if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil { + s.fail(root, r.newRel, err, &sum) + continue + } + sum.Renamed++ + } + if len(p.restore) > 0 { + if err := s.store.Restore(ctx, p.restore); err != nil { + return sum, fmt.Errorf("restore files: %w", err) + } + sum.Restored = len(p.restore) + } + if len(p.missing) > 0 { + if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil { + return sum, fmt.Errorf("mark missing: %w", err) + } + sum.Missing = len(p.missing) + } + return sum, nil +} + +func (s *Scanner) fail(root, rel string, err error, sum *Summary) { + sum.Failed++ + s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error()}) +} + +// parseAndSave reads tags in parallel and stores results from this goroutine. +func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob, sum *Summary) error { + if len(jobs) == 0 { + return nil + } + + jobCh := make(chan parseJob) + resCh := make(chan parseResult) + + var wg sync.WaitGroup + for i := 0; i < s.workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobCh { + meta, err := s.reader.Read(filepath.Join(root, filepath.FromSlash(j.file.RelPath))) + resCh <- parseResult{job: j, meta: meta, err: err} + } + }() + } + go func() { + defer func() { + close(jobCh) + wg.Wait() + close(resCh) + }() + for _, j := range jobs { + select { + case jobCh <- j: + case <-ctx.Done(): + return + } + } + }() + + done := 0 + for r := range resCh { // always drain, so no worker is left blocked + if ctx.Err() != nil { + continue + } + done++ + rel := r.job.file.RelPath + if r.err != nil { + sum.Failed++ + s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)}) + continue + } + added, err := s.store.SaveFile(ctx, root, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID) + if err != nil { + sum.Failed++ + s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)}) + continue + } + if added { + sum.Added++ + } else { + sum.Updated++ + } + s.publish(Event{Type: FileScanned, Root: root, RelPath: rel, Done: done, Total: len(jobs)}) + } + return ctx.Err() +} + +// ---- walking ---- + +type walkResult struct { + files []FileInfo + // failedDirs are slash-separated relative paths (with a trailing slash) + // that could not be read. Files below them are unknown, not gone. + failedDirs []string +} + +func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) { + var out walkResult + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + if path == root { + return err // nil, or the reason the root cannot be read + } + rel, rerr := filepath.Rel(root, path) + if rerr != nil { + return rerr + } + rel = filepath.ToSlash(rel) + + if err != nil { + s.fail(root, rel, err, sum) + if d != nil && d.IsDir() { + out.failedDirs = append(out.failedDirs, rel+"/") + } + return nil // keep going, do not descend into an unreadable dir + } + if d.Type()&fs.ModeSymlink != 0 { + sum.Skipped++ // never follow symlinks: they could point outside the root + return nil + } + if d.IsDir() || !audioExts[strings.ToLower(filepath.Ext(rel))] { + return nil + } + info, ierr := d.Info() + if ierr != nil { + s.fail(root, rel, ierr, sum) + return nil + } + out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}) + return nil + }) + return out, err +} + +// ---- planning ---- + +type renameOp struct { + id int64 + newRel string +} + +type plan struct { + parse []parseJob + renames []renameOp + restore []int64 + missing []int64 + unchanged int +} + +type fileKey struct{ size, mtime int64 } + +// makePlan compares what is on disk with what is indexed. +// +// A new file and a vanished file with the same size and modification time are +// treated as a rename only when each is the sole candidate for that pair, so +// look-alike files are never guessed. +func makePlan(w walkResult, existing []ExistingFile) plan { + var p plan + byPath := make(map[string]ExistingFile, len(existing)) + for _, e := range existing { + byPath[e.RelPath] = e + } + + seen := make(map[string]bool, len(w.files)) + var added []FileInfo + for _, f := range w.files { + e, ok := byPath[f.RelPath] + if !ok { + added = append(added, f) + continue + } + seen[f.RelPath] = true + switch { + case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix: + p.parse = append(p.parse, parseJob{file: f, existingID: e.ID}) + case e.Missing: + p.restore = append(p.restore, e.ID) + default: + p.unchanged++ + } + } + + var gone []ExistingFile + for _, e := range existing { + if seen[e.RelPath] || underAny(e.RelPath, w.failedDirs) { + continue + } + gone = append(gone, e) + } + + goneByKey := map[fileKey][]ExistingFile{} + for _, e := range gone { + k := fileKey{e.Size, e.MTimeUnix} + goneByKey[k] = append(goneByKey[k], e) + } + addedByKey := map[fileKey]int{} + for _, f := range added { + addedByKey[fileKey{f.Size, f.MTimeUnix}]++ + } + + renamed := map[int64]bool{} + for _, f := range added { + k := fileKey{f.Size, f.MTimeUnix} + if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 { + p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath}) + renamed[cands[0].ID] = true + continue + } + p.parse = append(p.parse, parseJob{file: f}) + } + + for _, e := range gone { + if !renamed[e.ID] && !e.Missing { + p.missing = append(p.missing, e.ID) + } + } + return p +} + +func underAny(rel string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(rel, p) { + return true + } + } + return false +} diff --git a/internal/library/scanner_test.go b/internal/library/scanner_test.go new file mode 100644 index 0000000..8a8418e --- /dev/null +++ b/internal/library/scanner_test.go @@ -0,0 +1,608 @@ +package library + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +// ---- fakes ---- + +type fakeRow struct { + ExistingFile + Parsed ParsedFile +} + +type fakeStore struct { + mu sync.Mutex + nextID int64 + rows map[string]map[string]*fakeRow // root -> rel path -> row + saves int +} + +func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]map[string]*fakeRow{}} } + +func (s *fakeStore) byID(id int64) *fakeRow { + for _, files := range s.rows { + for _, r := range files { + if r.ID == id { + return r + } + } + } + return nil +} + +func (s *fakeStore) ExistingFiles(_ context.Context, root string) ([]ExistingFile, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []ExistingFile + for _, r := range s.rows[root] { + out = append(out, r.ExistingFile) + } + return out, nil +} + +func (s *fakeStore) SaveFile(_ context.Context, root string, f ParsedFile, existingID int64) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.saves++ + if existingID != 0 { + r := s.byID(existingID) + r.Size, r.MTimeUnix, r.Missing, r.Parsed = f.Size, f.MTimeUnix, false, f + return false, nil + } + if s.rows[root] == nil { + s.rows[root] = map[string]*fakeRow{} + } + s.nextID++ + s.rows[root][f.RelPath] = &fakeRow{ + ExistingFile: ExistingFile{ID: s.nextID, FileInfo: f.FileInfo}, + Parsed: f, + } + return true, nil +} + +func (s *fakeStore) RenameFile(_ context.Context, id int64, newRel string) error { + s.mu.Lock() + defer s.mu.Unlock() + for _, files := range s.rows { + for old, r := range files { + if r.ID == id { + delete(files, old) + r.RelPath, r.Missing = newRel, false + files[newRel] = r + return nil + } + } + } + return errors.New("no such file id") +} + +func (s *fakeStore) MarkMissing(_ context.Context, ids []int64, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range ids { + s.byID(id).Missing = true + } + return nil +} + +func (s *fakeStore) Restore(_ context.Context, ids []int64) error { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range ids { + s.byID(id).Missing = false + } + return nil +} + +func (s *fakeStore) row(root, rel string) *fakeRow { + s.mu.Lock() + defer s.mu.Unlock() + return s.rows[root][rel] +} + +// fakeReader returns metadata derived from the file name and counts calls. +type fakeReader struct { + calls atomic.Int64 + delay time.Duration + fail map[string]bool // base names that fail + running atomic.Int64 + peak atomic.Int64 +} + +func (r *fakeReader) Read(path string) (Metadata, error) { + r.calls.Add(1) + n := r.running.Add(1) + for { + p := r.peak.Load() + if n <= p || r.peak.CompareAndSwap(p, n) { + break + } + } + defer r.running.Add(-1) + if r.delay > 0 { + time.Sleep(r.delay) + } + if r.fail[filepath.Base(path)] { + return Metadata{}, ErrUnreadable + } + return Metadata{ + Tags: Tags{Title: filepath.Base(path), Artists: []string{"Artist"}}, + Properties: Properties{Format: "mp3", DurationMs: 1000, SampleRate: 44100}, + }, nil +} + +// ---- helpers ---- + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func newScanner(store Store, reader TagReader, workers int) (*Scanner, *Hub) { + hub := NewHub() + return NewScanner(store, reader, hub, workers), hub +} + +func mustScan(t *testing.T, s *Scanner, roots ...string) Summary { + t.Helper() + sum, err := s.Scan(context.Background(), roots) + if err != nil { + t.Fatal(err) + } + return sum +} + +// ---- tests ---- + +func TestScanIndexesAudioFilesOnly(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a/one.mp3"), "1") + write(t, filepath.Join(root, "a/two.FLAC"), "22") + write(t, filepath.Join(root, "a/cover.jpg"), "x") + write(t, filepath.Join(root, "notes.txt"), "x") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 2) + sum := mustScan(t, s, root) + + if sum.Found != 2 || sum.Added != 2 || sum.Failed != 0 { + t.Fatalf("summary = %+v", sum) + } + if store.row(root, "a/one.mp3") == nil || store.row(root, "a/two.FLAC") == nil { + t.Error("audio files not stored with root-relative slash paths") + } + if store.row(root, "a/cover.jpg") != nil { + t.Error("non-audio file was stored") + } +} + +func TestScanSkipsSymlinks(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + write(t, filepath.Join(outside, "secret.mp3"), "s") + write(t, filepath.Join(root, "real.mp3"), "r") + if err := os.Symlink(filepath.Join(outside, "secret.mp3"), filepath.Join(root, "link.mp3")); err != nil { + t.Skip("symlinks not supported:", err) + } + if err := os.Symlink(outside, filepath.Join(root, "dirlink")); err != nil { + t.Skip("symlinks not supported:", err) + } + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 2) + sum := mustScan(t, s, root) + + if sum.Found != 1 || store.row(root, "real.mp3") == nil { + t.Fatalf("summary = %+v", sum) + } + if sum.Skipped < 1 { + t.Errorf("skipped = %d, want symlinks counted", sum.Skipped) + } +} + +func TestRescanSkipsUnchangedFiles(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + write(t, filepath.Join(root, "b.mp3"), "22") + + store, reader := newFakeStore(), &fakeReader{} + s, _ := newScanner(store, reader, 2) + mustScan(t, s, root) + before := reader.calls.Load() + + sum := mustScan(t, s, root) + if sum.Unchanged != 2 || sum.Added != 0 || sum.Updated != 0 { + t.Fatalf("rescan summary = %+v", sum) + } + if reader.calls.Load() != before { + t.Error("unchanged files were parsed again") + } + if store.saves != 2 { + t.Errorf("saves = %d, want 2 (no duplicates)", store.saves) + } +} + +func TestRescanUpdatesChangedFile(t *testing.T) { + root := t.TempDir() + p := filepath.Join(root, "a.mp3") + write(t, p, "1") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + id := store.row(root, "a.mp3").ID + + write(t, p, "longer content") + later := time.Now().Add(time.Hour) + if err := os.Chtimes(p, later, later); err != nil { + t.Fatal(err) + } + sum := mustScan(t, s, root) + + if sum.Updated != 1 || sum.Added != 0 { + t.Fatalf("summary = %+v", sum) + } + if got := store.row(root, "a.mp3").ID; got != id { + t.Errorf("id changed %d -> %d", id, got) + } +} + +func TestRenamedFileKeepsIdentity(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "old.mp3"), "content") + + store, reader := newFakeStore(), &fakeReader{} + s, _ := newScanner(store, reader, 1) + mustScan(t, s, root) + id := store.row(root, "old.mp3").ID + calls := reader.calls.Load() + + if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(root, "old.mp3"), filepath.Join(root, "sub/new.mp3")); err != nil { + t.Fatal(err) + } + sum := mustScan(t, s, root) + + if sum.Renamed != 1 || sum.Added != 0 || sum.Missing != 0 { + t.Fatalf("summary = %+v", sum) + } + r := store.row(root, "sub/new.mp3") + if r == nil || r.ID != id { + t.Fatalf("renamed row = %+v, want id %d", r, id) + } + if reader.calls.Load() != calls { + t.Error("renamed file was parsed again") + } +} + +func TestAmbiguousRenameIsNotGuessed(t *testing.T) { + root := t.TempDir() + ts := time.Unix(1_700_000_000, 0) + for _, n := range []string{"a.mp3", "b.mp3"} { + p := filepath.Join(root, n) + write(t, p, "same") + _ = os.Chtimes(p, ts, ts) + } + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + + // Both vanish and two look-alike files appear: size and mtime cannot tell which is which. + _ = os.Remove(filepath.Join(root, "a.mp3")) + _ = os.Remove(filepath.Join(root, "b.mp3")) + for _, n := range []string{"c.mp3", "d.mp3"} { + p := filepath.Join(root, n) + write(t, p, "same") + _ = os.Chtimes(p, ts, ts) + } + sum := mustScan(t, s, root) + + if sum.Renamed != 0 || sum.Added != 2 || sum.Missing != 2 { + t.Fatalf("summary = %+v", sum) + } +} + +func TestMissingFileIsMarkedThenRestored(t *testing.T) { + root := t.TempDir() + p := filepath.Join(root, "a.mp3") + write(t, p, "1") + ts := time.Unix(1_700_000_000, 0) + _ = os.Chtimes(p, ts, ts) + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + sum := mustScan(t, s, root) + if sum.Missing != 1 { + t.Fatalf("summary = %+v", sum) + } + r := store.row(root, "a.mp3") + if r == nil || !r.Missing { + t.Fatal("row should be kept and marked missing, not deleted") + } + + // A second scan must not count it as newly missing. + if sum = mustScan(t, s, root); sum.Missing != 0 { + t.Errorf("already-missing file counted again: %+v", sum) + } + + write(t, p, "1") + _ = os.Chtimes(p, ts, ts) + sum = mustScan(t, s, root) + if sum.Restored != 1 || store.row(root, "a.mp3").Missing { + t.Fatalf("summary = %+v, row = %+v", sum, store.row(root, "a.mp3")) + } +} + +func TestUnreadableRootDoesNotMarkFilesMissing(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + + // Simulate an unmounted NAS: the root path no longer exists. + if err := os.RemoveAll(root); err != nil { + t.Fatal(err) + } + if _, err := s.Scan(context.Background(), []string{root}); err == nil { + t.Fatal("expected an error for an unreadable root") + } + if store.row(root, "a.mp3").Missing { + t.Fatal("files were marked missing although the root could not be read") + } +} + +func TestUnreadableSubdirDoesNotMarkItsFilesMissing(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced for root") + } + root := t.TempDir() + write(t, filepath.Join(root, "ok.mp3"), "1") + write(t, filepath.Join(root, "locked/hidden.mp3"), "22") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + + locked := filepath.Join(root, "locked") + if err := os.Chmod(locked, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) + + sum := mustScan(t, s, root) + if sum.Failed < 1 { + t.Errorf("failed = %d, want the unreadable dir reported", sum.Failed) + } + if sum.Missing != 0 || store.row(root, "locked/hidden.mp3").Missing { + t.Fatalf("file in unreadable dir marked missing: %+v", sum) + } +} + +func TestUnreadableFileIsReportedAndScanContinues(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "good.mp3"), "1") + write(t, filepath.Join(root, "bad.mp3"), "22") + + store := newFakeStore() + s, hub := newScanner(store, &fakeReader{fail: map[string]bool{"bad.mp3": true}}, 2) + events, cancel := hub.Subscribe(64) + defer cancel() + + sum := mustScan(t, s, root) + if sum.Added != 1 || sum.Failed != 1 { + t.Fatalf("summary = %+v", sum) + } + if store.row(root, "bad.mp3") != nil { + t.Error("unreadable file was stored") + } + + var failed *Event + for len(events) > 0 { + if e := <-events; e.Type == FileFailed { + failed = &e + } + } + if failed == nil || failed.RelPath != "bad.mp3" || failed.Error == "" { + t.Fatalf("file_failed event = %+v", failed) + } +} + +func TestScanRunsWorkersInParallel(t *testing.T) { + root := t.TempDir() + for i := 0; i < 12; i++ { + write(t, filepath.Join(root, string(rune('a'+i))+".mp3"), string(rune('a'+i))) + } + reader := &fakeReader{delay: 30 * time.Millisecond} + s, _ := newScanner(newFakeStore(), reader, 4) + + start := time.Now() + mustScan(t, s, root) + elapsed := time.Since(start) + + if reader.peak.Load() < 2 { + t.Errorf("peak concurrency = %d, want parallel reads", reader.peak.Load()) + } + if reader.peak.Load() > 4 { + t.Errorf("peak concurrency = %d, exceeds 4 workers", reader.peak.Load()) + } + if elapsed > 12*30*time.Millisecond { + t.Errorf("scan took %v, no faster than sequential", elapsed) + } +} + +func TestCancelledScanDoesNotMarkMissing(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + mustScan(t, s, root) + + _ = os.Remove(filepath.Join(root, "a.mp3")) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.Scan(ctx, []string{root}); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if store.row(root, "a.mp3").Missing { + t.Fatal("an interrupted scan must not mark files missing") + } +} + +func TestScanPublishesProgressEvents(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + write(t, filepath.Join(root, "b.mp3"), "22") + + s, hub := newScanner(newFakeStore(), &fakeReader{}, 2) + events, cancel := hub.Subscribe(64) + defer cancel() + mustScan(t, s, root) + + var types []EventType + var last Event + scanned := 0 + for len(events) > 0 { + e := <-events + types = append(types, e.Type) + if e.Type == FileScanned { + scanned++ + if e.Total != 2 || e.Done < 1 || e.Done > 2 { + t.Errorf("progress = %d/%d", e.Done, e.Total) + } + } + last = e + } + if types[0] != ScanStarted || last.Type != ScanFinished || scanned != 2 { + t.Fatalf("event types = %v", types) + } + if last.Summary == nil || last.Summary.Added != 2 { + t.Errorf("final summary = %+v", last.Summary) + } +} + +func TestMultipleRootsAreIndependent(t *testing.T) { + a, b := t.TempDir(), t.TempDir() + write(t, filepath.Join(a, "same/name.mp3"), "1") + write(t, filepath.Join(b, "same/name.mp3"), "1") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 2) + sum := mustScan(t, s, a, b) + + if sum.Added != 2 || sum.Roots != 2 { + t.Fatalf("summary = %+v", sum) + } + if store.row(a, "same/name.mp3") == nil || store.row(b, "same/name.mp3") == nil { + t.Error("same relative path under two roots must be two files") + } +} + +func TestOneBadRootDoesNotStopTheOthers(t *testing.T) { + good := t.TempDir() + write(t, filepath.Join(good, "a.mp3"), "1") + bad := filepath.Join(t.TempDir(), "gone") + + store := newFakeStore() + s, _ := newScanner(store, &fakeReader{}, 1) + sum, err := s.Scan(context.Background(), []string{bad, good}) + if err == nil { + t.Fatal("expected an error mentioning the bad root") + } + if sum.Added != 1 || store.row(good, "a.mp3") == nil { + t.Fatalf("good root not scanned: %+v", sum) + } +} + +// ---- hub ---- + +func TestHubSlowSubscriberDoesNotBlock(t *testing.T) { + hub := NewHub() + slow, cancelSlow := hub.Subscribe(1) + defer cancelSlow() + fast, cancelFast := hub.Subscribe(100) + defer cancelFast() + + done := make(chan struct{}) + go func() { + for i := 0; i < 50; i++ { + hub.Publish(Event{Type: FileScanned, Done: i}) + } + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Publish blocked on a slow subscriber") + } + if len(fast) != 50 { + t.Errorf("fast subscriber got %d events, want 50", len(fast)) + } + if len(slow) != 1 { + t.Errorf("slow subscriber buffer = %d, want 1 (rest dropped)", len(slow)) + } +} + +func TestHubCancelClosesChannel(t *testing.T) { + hub := NewHub() + ch, cancel := hub.Subscribe(1) + cancel() + cancel() // idempotent + if _, ok := <-ch; ok { + t.Fatal("channel should be closed after cancel") + } + hub.Publish(Event{Type: FileScanned}) // must not panic +} + +// ---- manager ---- + +func TestManagerRunsOneScanAtATime(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "a.mp3"), "1") + reader := &fakeReader{delay: 100 * time.Millisecond} + s, _ := newScanner(newFakeStore(), reader, 1) + m := NewManager(s, []string{root}) + + if err := m.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := m.Start(context.Background()); !errors.Is(err, ErrScanRunning) { + t.Fatalf("second Start: err = %v, want ErrScanRunning", err) + } + if !m.Status().Running { + t.Error("status should report running") + } + + deadline := time.Now().Add(3 * time.Second) + for m.Status().Running && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + st := m.Status() + if st.Running || st.Last == nil || st.Last.Added != 1 || st.LastError != "" { + t.Fatalf("status = %+v", st) + } + if err := m.Start(context.Background()); err != nil { + t.Fatalf("Start after finish: %v", err) + } +} diff --git a/internal/library/taglib.go b/internal/library/taglib.go new file mode 100644 index 0000000..620d270 --- /dev/null +++ b/internal/library/taglib.go @@ -0,0 +1,93 @@ +package library + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + "go.senan.xyz/taglib" +) + +// UnknownArtist is used when a file has no artist tag. +const UnknownArtist = "Unknown Artist" + +// TaglibReader reads tags and audio properties with TagLib (compiled to +// WebAssembly, so there is no CGO). It is safe for concurrent use. +type TaglibReader struct{} + +// Read implements TagReader. +func (TaglibReader) Read(path string) (Metadata, error) { + tags, err := taglib.ReadTags(path) + if err != nil { + return Metadata{}, fmt.Errorf("%w: %v", ErrUnreadable, err) + } + props, err := taglib.ReadProperties(path) + if err != nil { + return Metadata{}, fmt.Errorf("%w: %v", ErrUnreadable, err) + } + // TagLib does not fail on random bytes named .mp3; it just finds no audio. + if props.SampleRate == 0 || props.Length <= 0 { + return Metadata{}, fmt.Errorf("%w: no audio stream found", ErrUnreadable) + } + + m := Metadata{ + Tags: Tags{ + Title: first(tags, "TITLE"), + Album: first(tags, "ALBUM"), + AlbumArtist: first(tags, "ALBUMARTIST"), + Artists: nonEmpty(tags["ARTIST"]), + TrackNo: leadingInt(first(tags, "TRACKNUMBER")), + DiscNo: leadingInt(first(tags, "DISCNUMBER")), + }, + Properties: Properties{ + Format: strings.ToLower(props.Format), + DurationMs: props.Length.Milliseconds(), + SampleRate: int(props.SampleRate), + BitRate: int(props.BitRate), + }, + } + + if m.Title == "" { + base := filepath.Base(path) + m.Title = strings.TrimSuffix(base, filepath.Ext(base)) + } + if len(m.Artists) == 0 { + m.Artists = []string{UnknownArtist} + } + if m.AlbumArtist == "" { + m.AlbumArtist = m.Artists[0] + } + return m, nil +} + +func first(tags map[string][]string, key string) string { + for _, v := range tags[key] { + if v = strings.TrimSpace(v); v != "" { + return v + } + } + return "" +} + +func nonEmpty(values []string) []string { + var out []string + for _, v := range values { + if v = strings.TrimSpace(v); v != "" { + out = append(out, v) + } + } + return out +} + +// leadingInt parses "3" or "3/12" and returns 0 when there is no number. +func leadingInt(s string) int { + if i := strings.IndexByte(s, '/'); i >= 0 { + s = s[:i] + } + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil || n < 0 { + return 0 + } + return n +} diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go new file mode 100644 index 0000000..092387f --- /dev/null +++ b/internal/library/taglib_test.go @@ -0,0 +1,123 @@ +package library + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// makeAudio creates a short tagged file with ffmpeg. Tests that need real +// audio are skipped where ffmpeg is not installed. +func makeAudio(t *testing.T, dir, name string, meta map[string]string) string { + t.Helper() + ffmpeg, err := exec.LookPath("ffmpeg") + if err != nil { + t.Skip("ffmpeg not installed") + } + args := []string{"-loglevel", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=2"} + for k, v := range meta { + args = append(args, "-metadata", k+"="+v) + } + path := filepath.Join(dir, name) + args = append(args, path) + if out, err := exec.Command(ffmpeg, args...).CombinedOutput(); err != nil { + t.Fatalf("ffmpeg: %v: %s", err, out) + } + return path +} + +func TestTaglibReaderMP3(t *testing.T) { + p := makeAudio(t, t.TempDir(), "song.mp3", map[string]string{ + "title": "Perché è così", "artist": "Björk", "album": "Vespertine", + "album_artist": "Björk", "track": "3/12", "disc": "1/2", + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if m.Title != "Perché è così" || m.Album != "Vespertine" || m.AlbumArtist != "Björk" { + t.Errorf("tags = %+v", m.Tags) + } + if len(m.Artists) != 1 || m.Artists[0] != "Björk" { + t.Errorf("artists = %v", m.Artists) + } + if m.TrackNo != 3 || m.DiscNo != 1 { + t.Errorf("track/disc = %d/%d", m.TrackNo, m.DiscNo) + } + if m.Format != "mpeg" && m.Format != "mp3" { + t.Errorf("format = %q", m.Format) + } + if m.DurationMs < 1500 || m.DurationMs > 2500 || m.SampleRate != 44100 { + t.Errorf("properties = %+v", m.Properties) + } +} + +func TestTaglibReaderFLAC(t *testing.T) { + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "Hello", "artist": "Radiohead", "album": "OK Computer", "track": "7", + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if m.Title != "Hello" || m.Album != "OK Computer" || m.TrackNo != 7 || m.Format != "flac" { + t.Errorf("metadata = %+v", m) + } + // Without an album artist tag, the first artist stands in. + if m.AlbumArtist != "Radiohead" { + t.Errorf("album artist = %q, want fallback to artist", m.AlbumArtist) + } +} + +func TestTaglibReaderFallbacks(t *testing.T) { + p := makeAudio(t, t.TempDir(), "No Tags Here.mp3", nil) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if m.Title != "No Tags Here" { + t.Errorf("title = %q, want file name without extension", m.Title) + } + if len(m.Artists) != 1 || m.Artists[0] != UnknownArtist { + t.Errorf("artists = %v, want [%s]", m.Artists, UnknownArtist) + } +} + +func TestTaglibReaderRejectsBadFiles(t *testing.T) { + dir := t.TempDir() + garbage := filepath.Join(dir, "garbage.mp3") + if err := os.WriteFile(garbage, make([]byte, 500), 0o644); err != nil { + t.Fatal(err) + } + empty := filepath.Join(dir, "empty.flac") + if err := os.WriteFile(empty, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, p := range []string{garbage, empty, filepath.Join(dir, "missing.mp3")} { + if _, err := (TaglibReader{}).Read(p); !errors.Is(err, ErrUnreadable) { + t.Errorf("%s: err = %v, want ErrUnreadable", filepath.Base(p), err) + } + } +} + +func TestScanWithRealFiles(t *testing.T) { + root := t.TempDir() + makeAudio(t, root, "a.mp3", map[string]string{"title": "A", "artist": "X"}) + makeAudio(t, root, "b.flac", map[string]string{"title": "B", "artist": "Y"}) + if err := os.WriteFile(filepath.Join(root, "broken.mp3"), []byte("not audio"), 0o644); err != nil { + t.Fatal(err) + } + + store := newFakeStore() + s, _ := newScanner(store, TaglibReader{}, 4) + sum := mustScan(t, s, root) + + if sum.Added != 2 || sum.Failed != 1 { + t.Fatalf("summary = %+v", sum) + } + if r := store.row(root, "a.mp3"); r == nil || r.Parsed.Title != "A" || r.Parsed.Artists[0] != "X" { + t.Errorf("a.mp3 row = %+v", r) + } +} diff --git a/internal/library/types.go b/internal/library/types.go new file mode 100644 index 0000000..1d6c47c --- /dev/null +++ b/internal/library/types.go @@ -0,0 +1,128 @@ +// Package library scans the music directories and indexes audio files. +// +// The scanner only reads: it never writes, renames or deletes music files. +package library + +import ( + "context" + "errors" + "time" +) + +// ErrUnreadable means a file could not be parsed as audio. +var ErrUnreadable = errors.New("unreadable audio file") + +// Tags are the descriptive tags of a track. Fields the file lacks are empty. +type Tags struct { + Title string + Album string + AlbumArtist string + Artists []string + TrackNo int + DiscNo int +} + +// Properties are the technical properties of an audio file. +type Properties struct { + Format string // "mp3", "flac", ... + DurationMs int64 + SampleRate int + BitRate int // kbit/s +} + +// Metadata is everything read from one audio file. +type Metadata struct { + Tags + Properties +} + +// TagReader reads metadata from a file on disk. It must be safe for +// concurrent use and return ErrUnreadable for files that are not valid audio. +type TagReader interface { + Read(path string) (Metadata, error) +} + +// FileInfo identifies a file inside a library root. +type FileInfo struct { + RelPath string // slash-separated, relative to the root + Size int64 + MTimeUnix int64 +} + +// ExistingFile is a file already indexed in the database. +type ExistingFile struct { + ID int64 + FileInfo + Missing bool // last scan did not find it +} + +// ParsedFile is a file together with its metadata, ready to be stored. +type ParsedFile struct { + FileInfo + Metadata +} + +// Store is what the scanner needs from persistence. All writes come from a +// single goroutine, so implementations need no extra locking for the scanner. +type Store interface { + ExistingFiles(ctx context.Context, root string) ([]ExistingFile, error) + // SaveFile creates the file (existingID == 0) or updates it, together with + // its track, album and artists. It reports whether a new file was created. + SaveFile(ctx context.Context, root string, f ParsedFile, existingID int64) (added bool, err error) + // RenameFile points an indexed file at a new relative path and clears its missing flag. + RenameFile(ctx context.Context, id int64, newRelPath string) error + // MarkMissing flags files that were not found. Rows are never deleted. + MarkMissing(ctx context.Context, ids []int64, at time.Time) error + // Restore clears the missing flag of files that reappeared. + Restore(ctx context.Context, ids []int64) error +} + +// EventType names a scan progress event. +type EventType string + +// Scan progress events. +const ( + ScanStarted EventType = "scan_started" + FileScanned EventType = "file_scanned" + FileFailed EventType = "file_failed" + ScanFinished EventType = "scan_finished" +) + +// Event is one progress notification. Done and Total count files being +// parsed in the current root. +type Event struct { + Type EventType + Time time.Time + Root string + RelPath string + Error string + Done int + Total int + Summary *Summary // only on ScanFinished +} + +// Summary counts what a scan did. +type Summary struct { + Roots int + Found int // audio files seen on disk + Added int + Updated int + Unchanged int + Renamed int + Restored int + Missing int // newly marked missing + Failed int + Skipped int // symlinks +} + +func (s *Summary) add(o Summary) { + s.Found += o.Found + s.Added += o.Added + s.Updated += o.Updated + s.Unchanged += o.Unchanged + s.Renamed += o.Renamed + s.Restored += o.Restored + s.Missing += o.Missing + s.Failed += o.Failed + s.Skipped += o.Skipped +} diff --git a/internal/storage/library.go b/internal/storage/library.go new file mode 100644 index 0000000..1666b91 --- /dev/null +++ b/internal/storage/library.go @@ -0,0 +1,261 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/library" +) + +type artistModel struct { + bun.BaseModel `bun:"table:artists"` + + ID int64 `bun:"id,pk,autoincrement"` + Name string `bun:"name,notnull"` +} + +type albumModel struct { + bun.BaseModel `bun:"table:albums"` + + ID int64 `bun:"id,pk,autoincrement"` + Title string `bun:"title,notnull"` + ArtistID int64 `bun:"artist_id,nullzero"` +} + +type trackModel struct { + bun.BaseModel `bun:"table:tracks"` + + ID int64 `bun:"id,pk,autoincrement"` + Title string `bun:"title,notnull"` + DurationMs int64 `bun:"duration_ms,nullzero"` + TrackNo int `bun:"track_no,nullzero"` + DiscNo int `bun:"disc_no,nullzero"` + AlbumID int64 `bun:"album_id,nullzero"` +} + +type trackArtistModel struct { + bun.BaseModel `bun:"table:track_artists"` + + TrackID int64 `bun:"track_id,pk"` + ArtistID int64 `bun:"artist_id,pk"` + Role string `bun:"role,pk"` +} + +type audioFileModel struct { + bun.BaseModel `bun:"table:audio_files"` + + ID int64 `bun:"id,pk,autoincrement"` + TrackID int64 `bun:"track_id,notnull"` + Root string `bun:"root,notnull"` + RelPath string `bun:"rel_path,notnull"` + Format string `bun:"format,notnull"` + SizeBytes int64 `bun:"size_bytes,notnull"` + MTimeUnix int64 `bun:"mtime_unix,notnull"` + SampleRate int `bun:"sample_rate,nullzero"` + BitRate int `bun:"bitrate,nullzero"` + MissingAt sql.NullInt64 `bun:"missing_at"` +} + +// LibraryRepo stores the scanned catalog. It implements library.Store. +type LibraryRepo struct{ db *bun.DB } + +// NewLibraryRepo builds a LibraryRepo. +func NewLibraryRepo(db *bun.DB) *LibraryRepo { return &LibraryRepo{db: db} } + +// ExistingFiles lists the files indexed under root, including missing ones. +func (r *LibraryRepo) ExistingFiles(ctx context.Context, root string) ([]library.ExistingFile, error) { + var rows []audioFileModel + if err := r.db.NewSelect().Model(&rows).Where("root = ?", root).Scan(ctx); err != nil { + return nil, fmt.Errorf("select audio files: %w", err) + } + out := make([]library.ExistingFile, len(rows)) + for i, row := range rows { + out[i] = library.ExistingFile{ + ID: row.ID, + FileInfo: library.FileInfo{RelPath: row.RelPath, Size: row.SizeBytes, MTimeUnix: row.MTimeUnix}, + Missing: row.MissingAt.Valid, + } + } + return out, nil +} + +// SaveFile creates or updates one file with its track, album and artists in a +// single transaction. MusicBrainz ids are never invented: they stay NULL. +func (r *LibraryRepo) SaveFile(ctx context.Context, root string, f library.ParsedFile, existingID int64) (bool, error) { + var added bool + err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + artistIDs := make([]int64, 0, len(f.Artists)) + seen := map[int64]bool{} + for _, name := range f.Artists { + id, err := findOrCreateArtist(ctx, tx, name) + if err != nil { + return err + } + if !seen[id] { + seen[id] = true + artistIDs = append(artistIDs, id) + } + } + + var albumID int64 + if f.Album != "" { + albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist)) + if err != nil { + return err + } + if albumID, err = findOrCreateAlbum(ctx, tx, f.Album, albumArtistID); err != nil { + return err + } + } + + track := &trackModel{ + Title: f.Title, DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID, + } + + if existingID == 0 { + if _, err := tx.NewInsert().Model(track).Exec(ctx); err != nil { + return fmt.Errorf("insert track: %w", err) + } + file := newAudioFile(root, f, track.ID) + if _, err := tx.NewInsert().Model(file).Exec(ctx); err != nil { + return fmt.Errorf("insert audio file: %w", err) + } + added = true + return replaceCredits(ctx, tx, track.ID, artistIDs) + } + + var file audioFileModel + if err := tx.NewSelect().Model(&file).Where("id = ?", existingID).Scan(ctx); err != nil { + return fmt.Errorf("load audio file %d: %w", existingID, err) + } + track.ID = file.TrackID + if _, err := tx.NewUpdate().Model(track). + Column("title", "duration_ms", "track_no", "disc_no", "album_id"). + WherePK().Exec(ctx); err != nil { + return fmt.Errorf("update track: %w", err) + } + file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix + file.SampleRate, file.BitRate = f.SampleRate, f.BitRate + file.MissingAt = sql.NullInt64{} + if _, err := tx.NewUpdate().Model(&file). + Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "missing_at"). + WherePK().Exec(ctx); err != nil { + return fmt.Errorf("update audio file: %w", err) + } + return replaceCredits(ctx, tx, track.ID, artistIDs) + }) + if err != nil { + return false, err + } + return added, nil +} + +func newAudioFile(root string, f library.ParsedFile, trackID int64) *audioFileModel { + return &audioFileModel{ + TrackID: trackID, Root: root, RelPath: f.RelPath, Format: f.Format, + SizeBytes: f.Size, MTimeUnix: f.MTimeUnix, SampleRate: f.SampleRate, BitRate: f.BitRate, + } +} + +// RenameFile points an indexed file at a new relative path and clears its missing flag. +func (r *LibraryRepo) RenameFile(ctx context.Context, id int64, newRelPath string) error { + _, err := r.db.NewUpdate().Model((*audioFileModel)(nil)). + Set("rel_path = ?", newRelPath). + Set("missing_at = NULL"). + Where("id = ?", id).Exec(ctx) + if err != nil { + return fmt.Errorf("rename audio file: %w", err) + } + return nil +} + +// MarkMissing flags files that were not found. Rows and tracks are kept. +func (r *LibraryRepo) MarkMissing(ctx context.Context, ids []int64, at time.Time) error { + if len(ids) == 0 { + return nil + } + _, err := r.db.NewUpdate().Model((*audioFileModel)(nil)). + Set("missing_at = ?", at.Unix()). + Where("id IN (?)", bun.In(ids)).Exec(ctx) + if err != nil { + return fmt.Errorf("mark missing: %w", err) + } + return nil +} + +// Restore clears the missing flag of files that reappeared. +func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + _, err := r.db.NewUpdate().Model((*audioFileModel)(nil)). + Set("missing_at = NULL"). + Where("id IN (?)", bun.In(ids)).Exec(ctx) + if err != nil { + return fmt.Errorf("restore files: %w", err) + } + return nil +} + +// ---- helpers (run inside a transaction) ---- + +func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) { + var a artistModel + err := tx.NewSelect().Model(&a).Where("lower(name) = lower(?)", name).OrderExpr("id").Limit(1).Scan(ctx) + if err == nil { + return a.ID, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("find artist: %w", err) + } + a = artistModel{Name: name} + if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { + return 0, fmt.Errorf("insert artist: %w", err) + } + return a.ID, nil +} + +func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) { + var a albumModel + err := tx.NewSelect().Model(&a). + Where("lower(title) = lower(?)", title).Where("artist_id = ?", artistID). + OrderExpr("id").Limit(1).Scan(ctx) + if err == nil { + return a.ID, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("find album: %w", err) + } + a = albumModel{Title: title, ArtistID: artistID} + if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { + return 0, fmt.Errorf("insert album: %w", err) + } + return a.ID, nil +} + +func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []int64) error { + if _, err := tx.NewDelete().Model((*trackArtistModel)(nil)).Where("track_id = ?", trackID).Exec(ctx); err != nil { + return fmt.Errorf("clear credits: %w", err) + } + for _, id := range artistIDs { + row := &trackArtistModel{TrackID: trackID, ArtistID: id, Role: "main"} + if _, err := tx.NewInsert().Model(row).Exec(ctx); err != nil { + return fmt.Errorf("insert credit: %w", err) + } + } + return nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} diff --git a/internal/storage/library_test.go b/internal/storage/library_test.go new file mode 100644 index 0000000..6171947 --- /dev/null +++ b/internal/storage/library_test.go @@ -0,0 +1,246 @@ +package storage + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/library" +) + +func parsed(rel, title, album string, artists ...string) library.ParsedFile { + albumArtist := "" + if len(artists) > 0 { + albumArtist = artists[0] + } + return library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: 100, MTimeUnix: 1_700_000_000}, + Metadata: library.Metadata{ + Tags: library.Tags{ + Title: title, Album: album, AlbumArtist: albumArtist, + Artists: artists, TrackNo: 2, DiscNo: 1, + }, + Properties: library.Properties{Format: "flac", DurationMs: 180000, SampleRate: 44100, BitRate: 900}, + }, + } +} + +func count(t *testing.T, db *bun.DB, query string, args ...any) int { + t.Helper() + var n int + if err := db.QueryRow(query, args...).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestLibraryRepoSaveNewFile(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + + added, err := repo.SaveFile(ctx, "/music", parsed("a/1.flac", "Song", "Album", "Radiohead"), 0) + if err != nil || !added { + t.Fatalf("added=%v err=%v", added, err) + } + // Same artist (different case) and same album are reused, not duplicated. + if _, err := repo.SaveFile(ctx, "/music", parsed("a/2.flac", "Other", "album", "RADIOHEAD"), 0); err != nil { + t.Fatal(err) + } + + if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 1 { + t.Errorf("artists = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 1 { + t.Errorf("albums = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 2 { + t.Errorf("tracks = %d, want 2", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE root = ?", "/music"); n != 2 { + t.Errorf("audio_files = %d, want 2", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM track_artists"); n != 2 { + t.Errorf("track_artists = %d, want 2", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE mbid IS NOT NULL"); n != 0 { + t.Errorf("a MusicBrainz id was invented for %d artists", n) + } + + var title string + var dur, trackNo int64 + err = db.QueryRow("SELECT title, duration_ms, track_no FROM tracks WHERE title = 'Song'").Scan(&title, &dur, &trackNo) + if err != nil || dur != 180000 || trackNo != 2 { + t.Errorf("track row: %q %d %d err=%v", title, dur, trackNo, err) + } + }) +} + +func TestLibraryRepoTrackWithoutAlbum(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + repo := NewLibraryRepo(db) + if _, err := repo.SaveFile(context.Background(), "/m", parsed("x.flac", "Loose", "", "Someone"), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 0 { + t.Errorf("albums = %d, want 0", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE album_id IS NULL"); n != 1 { + t.Errorf("tracks without album = %d, want 1", n) + } + }) +} + +func TestLibraryRepoUpdateKeepsIdentityAndReplacesCredits(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + if _, err := repo.SaveFile(ctx, "/m", parsed("s.flac", "Old", "Album", "Artist A"), 0); err != nil { + t.Fatal(err) + } + existing, err := repo.ExistingFiles(ctx, "/m") + if err != nil || len(existing) != 1 { + t.Fatalf("existing = %v err=%v", existing, err) + } + id := existing[0].ID + + upd := parsed("s.flac", "New", "Album", "Artist B") + upd.Size, upd.MTimeUnix = 200, 1_800_000_000 + added, err := repo.SaveFile(ctx, "/m", upd, id) + if err != nil || added { + t.Fatalf("update: added=%v err=%v", added, err) + } + + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 1 { + t.Errorf("tracks = %d, want 1 (no duplicate on update)", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM audio_files"); n != 1 { + t.Errorf("audio_files = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE title = 'New'"); n != 1 { + t.Error("title was not updated") + } + if n := count(t, db, `SELECT COUNT(*) FROM track_artists ta JOIN artists a ON a.id = ta.artist_id WHERE a.name = 'Artist B'`); n != 1 { + t.Error("credits were not replaced") + } + if n := count(t, db, `SELECT COUNT(*) FROM track_artists ta JOIN artists a ON a.id = ta.artist_id WHERE a.name = 'Artist A'`); n != 0 { + t.Error("old credit still present") + } + after, _ := repo.ExistingFiles(ctx, "/m") + if after[0].ID != id || after[0].Size != 200 || after[0].MTimeUnix != 1_800_000_000 { + t.Errorf("after update = %+v", after[0]) + } + }) +} + +func TestLibraryRepoSameRelPathUnderTwoRoots(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + for _, root := range []string{"/a", "/b"} { + if _, err := repo.SaveFile(ctx, root, parsed("same.flac", "S", "", "X"), 0); err != nil { + t.Fatalf("root %s: %v", root, err) + } + } + for _, root := range []string{"/a", "/b"} { + if ex, _ := repo.ExistingFiles(ctx, root); len(ex) != 1 { + t.Errorf("root %s has %d files, want 1", root, len(ex)) + } + } + }) +} + +func TestLibraryRepoMissingRestoreRename(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + repo := NewLibraryRepo(db) + if _, err := repo.SaveFile(ctx, "/m", parsed("a.flac", "A", "", "X"), 0); err != nil { + t.Fatal(err) + } + ex, _ := repo.ExistingFiles(ctx, "/m") + id := ex[0].ID + + if err := repo.MarkMissing(ctx, []int64{id}, time.Unix(1_900_000_000, 0)); err != nil { + t.Fatal(err) + } + ex, _ = repo.ExistingFiles(ctx, "/m") + if len(ex) != 1 || !ex[0].Missing { + t.Fatalf("after MarkMissing: %+v (row must be kept, flagged)", ex) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 1 { + t.Errorf("tracks = %d: marking a file missing must keep the track", n) + } + + if err := repo.Restore(ctx, []int64{id}); err != nil { + t.Fatal(err) + } + if ex, _ = repo.ExistingFiles(ctx, "/m"); ex[0].Missing { + t.Error("Restore did not clear the missing flag") + } + + _ = repo.MarkMissing(ctx, []int64{id}, time.Now()) + if err := repo.RenameFile(ctx, id, "sub/b.flac"); err != nil { + t.Fatal(err) + } + ex, _ = repo.ExistingFiles(ctx, "/m") + if ex[0].ID != id || ex[0].RelPath != "sub/b.flac" || ex[0].Missing { + t.Errorf("after rename: %+v", ex[0]) + } + }) +} + +// stubReader lets the end-to-end test run without real audio. +type stubReader struct{} + +func (stubReader) Read(path string) (library.Metadata, error) { + return library.Metadata{ + Tags: library.Tags{Title: filepath.Base(path), Artists: []string{"Band"}, Album: "LP", AlbumArtist: "Band"}, + Properties: library.Properties{Format: "mp3", DurationMs: 1000, SampleRate: 44100}, + }, nil +} + +func TestScannerEndToEndOnBothBackends(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + root := t.TempDir() + for _, n := range []string{"one.mp3", "two.mp3"} { + if err := os.WriteFile(filepath.Join(root, n), []byte(n), 0o644); err != nil { + t.Fatal(err) + } + } + s := library.NewScanner(NewLibraryRepo(db), stubReader{}, library.NewHub(), 4) + scan := func() library.Summary { + sum, err := s.Scan(context.Background(), []string{root}) + if err != nil { + t.Fatal(err) + } + return sum + } + + if sum := scan(); sum.Added != 2 { + t.Fatalf("first scan: %+v", sum) + } + if sum := scan(); sum.Unchanged != 2 || sum.Added != 0 { + t.Fatalf("rescan: %+v", sum) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 2 { + t.Errorf("tracks = %d after rescan, want 2 (no duplicates)", n) + } + + if err := os.Remove(filepath.Join(root, "one.mp3")); err != nil { + t.Fatal(err) + } + if sum := scan(); sum.Missing != 1 { + t.Fatalf("after delete: %+v", sum) + } + if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE missing_at IS NOT NULL"); n != 1 { + t.Errorf("missing files = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 2 { + t.Errorf("tracks = %d: history must survive a missing file", n) + } + }) +} diff --git a/sonda.example.toml b/sonda.example.toml index 56375dd..4f26291 100644 --- a/sonda.example.toml +++ b/sonda.example.toml @@ -8,6 +8,7 @@ listen = "127.0.0.1:8080" # localhost by default [library] roots = ["/path/to/music"] # read-only, at least one required +scan_workers = 4 # parallel tag readers during a scan [database] driver = "sqlite" # "sqlite" or "postgres"