diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 076f1d3..5c93fed 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -92,13 +92,17 @@ func run() error { scans := library.NewManager(scanner, libraryRepo) logScanResults(logger, scans) + libraries := library.NewLibraries(libraryRepo) + libraries.SetBusy(func() bool { return scans.Status().Running }) + warnAboutLibraries(ctx, logger, libraries) + router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()), Tokens: tokens, Listening: listening.NewService(storage.NewListeningRepo(db)), Scan: scanControl{ctx: ctx, m: scans}, - Libraries: library.NewLibraries(libraryRepo), + Libraries: libraries, Streams: streaming.NewService(libraryRepo), }) diff --git a/cmd/sondad/scan.go b/cmd/sondad/scan.go index a4d0207..a26efbf 100644 --- a/cmd/sondad/scan.go +++ b/cmd/sondad/scan.go @@ -36,3 +36,17 @@ func logScanResults(logger *zap.Logger, m *library.Manager) { logger.Info("library scan finished", fields...) }) } + +// warnAboutLibraries logs libraries that look wrong. Rows created by the +// upgrade from the roots-in-config versions skip validation, so a stale root +// can show up as a missing directory or as a library inside another one. +func warnAboutLibraries(ctx context.Context, logger *zap.Logger, libs *library.Libraries) { + problems, err := libs.Problems(ctx) + if err != nil { + logger.Warn("could not check the libraries", zap.Error(err)) + return + } + for _, p := range problems { + logger.Warn("library problem: review GET /api/v1/libraries", zap.String("problem", p)) + } +} diff --git a/internal/api/http/handlers_libraries.go b/internal/api/http/handlers_libraries.go index ecfefb0..46f65e9 100644 --- a/internal/api/http/handlers_libraries.go +++ b/internal/api/http/handlers_libraries.go @@ -32,6 +32,8 @@ func writeLibraryError(c *gin.Context, err error) { writeError(c, http.StatusConflict, "library_exists", err.Error()) case errors.Is(err, library.ErrLibraryNotFound): writeError(c, http.StatusNotFound, "library_not_found", err.Error()) + case errors.Is(err, library.ErrScanRunning): + writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish") default: _ = c.Error(err) writeError(c, http.StatusInternalServerError, "internal", "internal error") diff --git a/internal/api/http/libraries_test.go b/internal/api/http/libraries_test.go index ffa9afd..ceee8ff 100644 --- a/internal/api/http/libraries_test.go +++ b/internal/api/http/libraries_test.go @@ -1,11 +1,14 @@ package httpapi import ( + "context" "net/http" "os" "path/filepath" "strconv" "testing" + + "codeberg.org/squidd/sonda/internal/library" ) func libPath(id float64) string { @@ -176,3 +179,29 @@ func TestScanAllWithoutLibraries(t *testing.T) { t.Fatalf("%d %v", r.Code, r.Body) } } + +// busyLibraries is a libraryService whose directory changes are refused. +type busyLibraries struct{} + +func (busyLibraries) List(context.Context) ([]library.Library, error) { return nil, nil } +func (busyLibraries) Create(context.Context, string, string) (library.Library, error) { + return library.Library{}, nil +} +func (busyLibraries) Update(context.Context, int64, *string, *string) (library.Library, error) { + return library.Library{}, library.ErrScanRunning +} +func (busyLibraries) Delete(context.Context, int64) error { return library.ErrScanRunning } + +func TestLibraryChangesDuringAScanAreConflicts(t *testing.T) { + h := newTestAPIWith(t, func(d *Deps) { d.Libraries = busyLibraries{} }) + admin := registerAndLogin(t, h) + + r := do(t, h, "PATCH", "/api/v1/libraries/1", admin, map[string]string{"path": "/tmp"}) + if r.Code != http.StatusConflict || errCode(r) != "scan_running" { + t.Errorf("PATCH: %d %v", r.Code, r.Body) + } + r = do(t, h, "DELETE", "/api/v1/libraries/1", admin, nil) + if r.Code != http.StatusConflict || errCode(r) != "scan_running" { + t.Errorf("DELETE: %d %v", r.Code, r.Body) + } +} diff --git a/internal/library/libraries.go b/internal/library/libraries.go index ffe5757..2be9375 100644 --- a/internal/library/libraries.go +++ b/internal/library/libraries.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "unicode" "unicode/utf8" @@ -47,11 +48,30 @@ type LibraryStore interface { // Libraries manages the list of libraries and keeps it sane: every path is an // absolute, existing directory, and no library sits inside or around another // (that would index the same files twice). -type Libraries struct{ store LibraryStore } +type Libraries struct { + store LibraryStore + + // mu makes validate-then-write atomic, so two admins creating libraries at + // the same moment cannot both pass the overlap check. Sonda is a + // single-instance server, so an in-process lock is enough. + mu sync.Mutex + busy func() bool +} // NewLibraries builds the service. func NewLibraries(store LibraryStore) *Libraries { return &Libraries{store: store} } +// SetBusy registers a function that tells whether a scan is running. While it +// is, a library's directory cannot change and a library cannot be deleted: the +// scan works from the list it started with and would write under a stale path. +func (s *Libraries) SetBusy(fn func() bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.busy = fn +} + +func (s *Libraries) scanning() bool { return s.busy != nil && s.busy() } + // List returns every library. func (s *Libraries) List(ctx context.Context) ([]Library, error) { return s.store.ListLibraries(ctx) @@ -59,7 +79,10 @@ func (s *Libraries) List(ctx context.Context) ([]Library, error) { // Create adds a library. func (s *Libraries) Create(ctx context.Context, name, path string) (Library, error) { - name, path, err := s.validate(ctx, name, path, 0) + s.mu.Lock() + defer s.mu.Unlock() + + name, path, err := s.validate(ctx, name, path, 0, true) if err != nil { return Library{}, err } @@ -69,6 +92,9 @@ func (s *Libraries) Create(ctx context.Context, name, path string) (Library, err // Update changes the name and/or the path of a library. A nil argument keeps // the current value. Files keep their identity when the path changes. func (s *Libraries) Update(ctx context.Context, id int64, name, path *string) (Library, error) { + s.mu.Lock() + defer s.mu.Unlock() + cur, err := s.store.GetLibrary(ctx, id) if err != nil { return Library{}, err @@ -80,28 +106,42 @@ func (s *Libraries) Update(ctx context.Context, id int64, name, path *string) (L if path != nil { newPath = *path } - newName, newPath, err = s.validate(ctx, newName, newPath, id) + // An untouched name is not validated again: a migrated library is named + // after its path, which can be longer than the limit for new names. + newName, newPath, err = s.validate(ctx, newName, newPath, id, name != nil) if err != nil { return Library{}, err } + if newPath != cur.Path && s.scanning() { + return Library{}, ErrScanRunning + } return s.store.UpdateLibrary(ctx, id, newName, newPath) } // Delete removes a library from the index. The music on disk is untouched. func (s *Libraries) Delete(ctx context.Context, id int64) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.scanning() { + return ErrScanRunning + } return s.store.DeleteLibrary(ctx, id) } // validate checks a name and a path, ignoring the library skipID when it -// looks for overlaps. It returns the trimmed name and the cleaned path. -func (s *Libraries) validate(ctx context.Context, name, path string, skipID int64) (string, string, error) { +// looks for overlaps. It returns the trimmed name and the cleaned path. The +// name is only checked when checkName is true. +func (s *Libraries) validate(ctx context.Context, name, path string, skipID int64, checkName bool) (string, string, error) { name = strings.TrimSpace(name) - if name == "" || utf8.RuneCountInString(name) > maxLibraryNameLen { - return "", "", fmt.Errorf("%w: name must be 1-%d characters", ErrInvalidLibrary, maxLibraryNameLen) - } - for _, r := range name { - if unicode.IsControl(r) { - return "", "", fmt.Errorf("%w: name has control characters", ErrInvalidLibrary) + if checkName { + if name == "" || utf8.RuneCountInString(name) > maxLibraryNameLen { + return "", "", fmt.Errorf("%w: name must be 1-%d characters", ErrInvalidLibrary, maxLibraryNameLen) + } + for _, r := range name { + if unicode.IsControl(r) { + return "", "", fmt.Errorf("%w: name has control characters", ErrInvalidLibrary) + } } } @@ -135,6 +175,11 @@ func (s *Libraries) validate(ctx context.Context, name, path string, skipID int6 if o.Path == path { return "", "", ErrLibraryExists } + // Done here, not only by the database: SQLite's NOCASE folds ASCII + // letters only, PostgreSQL's lower() folds accents too. + if checkName && strings.ToLower(o.Name) == strings.ToLower(name) { + return "", "", ErrLibraryExists + } // Compare real paths so a symlink cannot hide an overlap. A library // whose directory has vanished is compared by its stored path. otherReal, err := filepath.EvalSymlinks(o.Path) @@ -155,3 +200,40 @@ func (s *Libraries) validate(ctx context.Context, name, path string, skipID int6 func within(child, parent string) bool { return strings.HasPrefix(child, parent+string(filepath.Separator)) } + +// Problems lists libraries that look wrong: a directory that cannot be read, or +// libraries that overlap and would index the same files twice. Migrated data +// skips validation, so this is meant to run at startup and be logged. +func (s *Libraries) Problems(ctx context.Context) ([]string, error) { + libs, err := s.store.ListLibraries(ctx) + if err != nil { + return nil, fmt.Errorf("list libraries: %w", err) + } + + var problems []string + reals := make([]string, len(libs)) + for i, l := range libs { + real, err := filepath.EvalSymlinks(l.Path) + if err != nil { + problems = append(problems, fmt.Sprintf("library %q: directory %s cannot be read: %v", l.Name, l.Path, err)) + continue + } + reals[i] = real + } + for i := range libs { + for j := i + 1; j < len(libs); j++ { + a, b := reals[i], reals[j] + switch { + case a == "" || b == "": + // already reported as unreadable + case a == b: + problems = append(problems, fmt.Sprintf("libraries %q and %q point at the same directory: files are indexed twice", libs[i].Name, libs[j].Name)) + case within(a, b): + problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[i].Name, libs[j].Name)) + case within(b, a): + problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[j].Name, libs[i].Name)) + } + } + } + return problems, nil +} diff --git a/internal/library/libraries_test.go b/internal/library/libraries_test.go index ac1e8e2..d6b4116 100644 --- a/internal/library/libraries_test.go +++ b/internal/library/libraries_test.go @@ -13,15 +13,18 @@ import ( // memLibraries is an in-memory LibraryStore that enforces unique names and paths. type memLibraries struct { - mu sync.Mutex - next int64 - libs []Library + mu sync.Mutex + next int64 + libs []Library + delay time.Duration // makes ListLibraries slow, to widen race windows in tests } func (m *memLibraries) ListLibraries(context.Context) ([]Library, error) { m.mu.Lock() - defer m.mu.Unlock() - return append([]Library(nil), m.libs...), nil + out := append([]Library(nil), m.libs...) + m.mu.Unlock() + time.Sleep(m.delay) // the snapshot is taken before the pause, like a slow query + return out, nil } func (m *memLibraries) GetLibrary(_ context.Context, id int64) (Library, error) { @@ -270,3 +273,138 @@ func TestListLibraries(t *testing.T) { t.Fatalf("libs = %+v err=%v", libs, err) } } + +// Names must clash on accents and case the same way on every database. +func TestLibraryNamesClashCaseInsensitivelyOnUnicode(t *testing.T) { + s, _ := newLibraries() + ctx := context.Background() + base := t.TempDir() + if _, err := s.Create(ctx, "Música", mkdir(t, filepath.Join(base, "a"))); err != nil { + t.Fatal(err) + } + if _, err := s.Create(ctx, "MÚSICA", mkdir(t, filepath.Join(base, "b"))); !errors.Is(err, ErrLibraryExists) { + t.Fatalf("err = %v, want ErrLibraryExists", err) + } +} + +// Two admins creating libraries at the same moment must not both pass the +// overlap check. +func TestConcurrentCreatesCannotOverlap(t *testing.T) { + for round := 0; round < 5; round++ { + s, st := newLibraries() + st.delay = 5 * time.Millisecond + parent := mkdir(t, filepath.Join(t.TempDir(), "music")) + child := mkdir(t, filepath.Join(parent, "a")) + + var wg sync.WaitGroup + for i, p := range []string{parent, child} { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = s.Create(context.Background(), "lib"+string(rune('a'+i)), p) + }() + } + wg.Wait() + if len(st.libs) != 1 { + t.Fatalf("round %d: %d libraries created, want 1 (the two paths overlap)", round, len(st.libs)) + } + } +} + +// A migrated library is named after its path, which can be longer than the +// name limit. Changing only the path must still work. +func TestUpdatingThePathDoesNotRevalidateAnUntouchedName(t *testing.T) { + s, st := newLibraries() + ctx := context.Background() + base := t.TempDir() + a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b")) + long := strings.Repeat("x", 150) + st.libs = []Library{{ID: 1, Name: long, Path: a}} + st.next = 1 + + got, err := s.Update(ctx, 1, nil, &b) + if err != nil || got.Path != b || got.Name != long { + t.Fatalf("path-only update: %+v err=%v", got, err) + } + short := "short" + if _, err := s.Update(ctx, 1, &short, nil); err != nil { + t.Fatalf("renaming to a valid name: %v", err) + } + tooLong := strings.Repeat("y", 101) + if _, err := s.Update(ctx, 1, &tooLong, nil); !errors.Is(err, ErrInvalidLibrary) { + t.Fatalf("a new name is still validated: err = %v", err) + } +} + +// Changing a library's directory or deleting it while a scan runs would leave +// the scan writing under a path that no longer matches. +func TestLibraryPathChangeAndDeleteAreRefusedWhileScanning(t *testing.T) { + s, _ := newLibraries() + ctx := context.Background() + base := t.TempDir() + a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b")) + lib, _ := s.Create(ctx, "Music", a) + + scanning := true + s.SetBusy(func() bool { return scanning }) + + if _, err := s.Update(ctx, lib.ID, nil, &b); !errors.Is(err, ErrScanRunning) { + t.Errorf("moving during a scan: err = %v, want ErrScanRunning", err) + } + if err := s.Delete(ctx, lib.ID); !errors.Is(err, ErrScanRunning) { + t.Errorf("deleting during a scan: err = %v, want ErrScanRunning", err) + } + // A rename, or "moving" to the same path, changes nothing under the scan. + name := "Renamed" + if _, err := s.Update(ctx, lib.ID, &name, nil); err != nil { + t.Errorf("renaming during a scan: %v", err) + } + if _, err := s.Update(ctx, lib.ID, nil, &a); err != nil { + t.Errorf("same path during a scan: %v", err) + } + + scanning = false + if _, err := s.Update(ctx, lib.ID, nil, &b); err != nil { + t.Errorf("moving after the scan: %v", err) + } + if err := s.Delete(ctx, lib.ID); err != nil { + t.Errorf("deleting after the scan: %v", err) + } +} + +func TestProblemsReportsOverlapAndMissingDirectories(t *testing.T) { + s, st := newLibraries() + ctx := context.Background() + base := t.TempDir() + parent := mkdir(t, filepath.Join(base, "music")) + child := mkdir(t, filepath.Join(parent, "flac")) + gone := filepath.Join(base, "gone") + fine := mkdir(t, filepath.Join(base, "fine")) + + // Rows as a migration could leave them, bypassing validation. + st.libs = []Library{ + {ID: 1, Name: "music", Path: parent}, + {ID: 2, Name: "flac", Path: child}, + {ID: 3, Name: "gone", Path: gone}, + {ID: 4, Name: "fine", Path: fine}, + } + + problems, err := s.Problems(ctx) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(problems, "\n") + for _, want := range []string{`"gone"`, `"music"`, `"flac"`} { + if !strings.Contains(joined, want) { + t.Errorf("problems %q do not mention %s", joined, want) + } + } + if strings.Contains(joined, `"fine"`) { + t.Errorf("a healthy library was reported: %q", joined) + } + + st.libs = st.libs[3:] + if problems, _ := s.Problems(ctx); len(problems) != 0 { + t.Errorf("a healthy setup reported problems: %v", problems) + } +} diff --git a/internal/storage/upgrade_test.go b/internal/storage/upgrade_test.go index ef98e92..387ead9 100644 --- a/internal/storage/upgrade_test.go +++ b/internal/storage/upgrade_test.go @@ -116,3 +116,46 @@ func TestMigration0005OnAnEmptyDatabase(t *testing.T) { }) } } + +// On Linux /Music and /music are different directories. Both can hold rows, +// and library names must be unique ignoring case, so the migration has to +// give them different names instead of failing. +func TestMigration0005RootsDifferingOnlyInCase(t *testing.T) { + for _, driver := range availableDrivers() { + t.Run(driver, func(t *testing.T) { + ctx := context.Background() + db := openBackend(t, driver) + if err := Migrate(ctx, db, upTo(t, driver, 4)); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO tracks (title) VALUES ('a'), ('b'), ('c')"); err != nil { + t.Fatal(err) + } + rows := `INSERT INTO audio_files (track_id, root, rel_path, format, size_bytes, mtime_unix) VALUES + (1, '/Music', 'x.mp3', 'mp3', 10, 100), + (2, '/music', 'x.mp3', 'mp3', 20, 200), + (3, '/MUSIC', 'x.mp3', 'mp3', 30, 300)` + if _, err := db.Exec(rows); err != nil { + t.Fatal(err) + } + + full, _ := migrations.For(driver) + if err := Migrate(ctx, db, full); err != nil { + t.Fatalf("the upgrade must not fail on roots that differ only in case: %v", err) + } + + if n := count(t, db, "SELECT COUNT(*) FROM libraries"); n != 3 { + t.Errorf("libraries = %d, want 3", n) + } + if n := count(t, db, "SELECT COUNT(DISTINCT lower(name)) FROM libraries"); n != 3 { + t.Error("migrated names are not unique ignoring case") + } + // Every file ends up in the library of its own path. + if n := count(t, db, `SELECT COUNT(*) FROM audio_files f JOIN libraries l ON l.id = f.library_id + WHERE (l.path = '/Music' AND f.size_bytes = 10) OR (l.path = '/music' AND f.size_bytes = 20) + OR (l.path = '/MUSIC' AND f.size_bytes = 30)`); n != 3 { + t.Errorf("correctly linked files = %d, want 3", n) + } + }) + } +} diff --git a/migrations/postgres/0005_libraries.sql b/migrations/postgres/0005_libraries.sql index af29ad4..7bcc3ee 100644 --- a/migrations/postgres/0005_libraries.sql +++ b/migrations/postgres/0005_libraries.sql @@ -11,10 +11,14 @@ CREATE TABLE libraries ( -- Case-insensitive uniqueness (SQLite uses COLLATE NOCASE for the same effect). CREATE UNIQUE INDEX libraries_name_lower ON libraries (lower(name)); --- Existing data: every distinct root becomes a library. The path doubles as --- the name, which is always unique; admins can rename it afterwards. +-- Existing data: every distinct root becomes a library, named after its path. +-- Names are unique ignoring case, and on Linux /Music and /music are two +-- directories, so the second one gets a suffix. Admins can rename them later. INSERT INTO libraries (name, path, created_at) - SELECT DISTINCT root, root, extract(epoch FROM now())::bigint FROM audio_files; + SELECT CASE WHEN n = 1 THEN root ELSE root || ' (' || n || ')' END, root, + extract(epoch FROM now())::bigint + FROM (SELECT root, ROW_NUMBER() OVER (PARTITION BY lower(root) ORDER BY root) AS n + FROM (SELECT DISTINCT root FROM audio_files) AS roots) AS numbered; ALTER TABLE audio_files ADD COLUMN library_id BIGINT REFERENCES libraries(id) ON DELETE CASCADE; UPDATE audio_files f SET library_id = l.id FROM libraries l WHERE l.path = f.root; diff --git a/migrations/sqlite/0005_libraries.sql b/migrations/sqlite/0005_libraries.sql index aa6cad4..2ec5965 100644 --- a/migrations/sqlite/0005_libraries.sql +++ b/migrations/sqlite/0005_libraries.sql @@ -8,10 +8,13 @@ CREATE TABLE libraries ( created_at INTEGER NOT NULL -- unix seconds ); --- Existing data: every distinct root becomes a library. The path doubles as --- the name, which is always unique; admins can rename it afterwards. +-- Existing data: every distinct root becomes a library, named after its path. +-- Names are unique ignoring case, and on Linux /Music and /music are two +-- directories, so the second one gets a suffix. Admins can rename them later. INSERT INTO libraries (name, path, created_at) - SELECT DISTINCT root, root, unixepoch() FROM audio_files; + SELECT CASE WHEN n = 1 THEN root ELSE root || ' (' || n || ')' END, root, unixepoch() + FROM (SELECT root, ROW_NUMBER() OVER (PARTITION BY lower(root) ORDER BY root) AS n + FROM (SELECT DISTINCT root FROM audio_files)); -- SQLite cannot drop a column that is part of a UNIQUE constraint, so the -- table is rebuilt.