fix(library): make the libraries upgrade and validation robust

- fix(migration): roots that differ only in case (/Music and /music are two
  directories on Linux) used to make migration 0005 fail on the unique name;
  the later ones now get a numeric suffix
- feat(library): at startup, log a warning for every library whose directory
  cannot be read or that overlaps another, because migrated rows skip
  validation and may still hold roots that were removed from the old config
- fix(library): creating or updating a library is now serialised, so two
  concurrent requests cannot both pass the overlap check
- fix(library): changing a library's directory or deleting it is refused with
  409 scan_running while a scan runs; the scan works from the list it started
  with and would write under a stale path
- fix(library): library names clash ignoring case and accents the same way on
  SQLite and PostgreSQL, checked in Go
- fix(library): changing only the path no longer revalidates an untouched
  name, which can be longer than the limit after a migration
- test: roots differing in case on both databases; concurrent creates with a
  slow store, verified to fail without the lock; the busy guard; Problems
This commit is contained in:
tgiachi 2026-11-11 13:34:44 +01:00
commit eb23ee426a
9 changed files with 342 additions and 23 deletions

View file

@ -92,13 +92,17 @@ func run() error {
scans := library.NewManager(scanner, libraryRepo)
logScanResults(logger, scans)
libraries := library.NewLibraries(libraryRepo)
libraries.SetBusy(func() bool { return scans.Status().Running })
warnAboutLibraries(ctx, logger, libraries)
router := httpapi.NewRouter(httpapi.Deps{
Logger: logger,
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
Tokens: tokens,
Listening: listening.NewService(storage.NewListeningRepo(db)),
Scan: scanControl{ctx: ctx, m: scans},
Libraries: library.NewLibraries(libraryRepo),
Libraries: libraries,
Streams: streaming.NewService(libraryRepo),
})

View file

@ -36,3 +36,17 @@ func logScanResults(logger *zap.Logger, m *library.Manager) {
logger.Info("library scan finished", fields...)
})
}
// warnAboutLibraries logs libraries that look wrong. Rows created by the
// upgrade from the roots-in-config versions skip validation, so a stale root
// can show up as a missing directory or as a library inside another one.
func warnAboutLibraries(ctx context.Context, logger *zap.Logger, libs *library.Libraries) {
problems, err := libs.Problems(ctx)
if err != nil {
logger.Warn("could not check the libraries", zap.Error(err))
return
}
for _, p := range problems {
logger.Warn("library problem: review GET /api/v1/libraries", zap.String("problem", p))
}
}

View file

@ -32,6 +32,8 @@ func writeLibraryError(c *gin.Context, err error) {
writeError(c, http.StatusConflict, "library_exists", err.Error())
case errors.Is(err, library.ErrLibraryNotFound):
writeError(c, http.StatusNotFound, "library_not_found", err.Error())
case errors.Is(err, library.ErrScanRunning):
writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish")
default:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")

View file

@ -1,11 +1,14 @@
package httpapi
import (
"context"
"net/http"
"os"
"path/filepath"
"strconv"
"testing"
"codeberg.org/squidd/sonda/internal/library"
)
func libPath(id float64) string {
@ -176,3 +179,29 @@ func TestScanAllWithoutLibraries(t *testing.T) {
t.Fatalf("%d %v", r.Code, r.Body)
}
}
// busyLibraries is a libraryService whose directory changes are refused.
type busyLibraries struct{}
func (busyLibraries) List(context.Context) ([]library.Library, error) { return nil, nil }
func (busyLibraries) Create(context.Context, string, string) (library.Library, error) {
return library.Library{}, nil
}
func (busyLibraries) Update(context.Context, int64, *string, *string) (library.Library, error) {
return library.Library{}, library.ErrScanRunning
}
func (busyLibraries) Delete(context.Context, int64) error { return library.ErrScanRunning }
func TestLibraryChangesDuringAScanAreConflicts(t *testing.T) {
h := newTestAPIWith(t, func(d *Deps) { d.Libraries = busyLibraries{} })
admin := registerAndLogin(t, h)
r := do(t, h, "PATCH", "/api/v1/libraries/1", admin, map[string]string{"path": "/tmp"})
if r.Code != http.StatusConflict || errCode(r) != "scan_running" {
t.Errorf("PATCH: %d %v", r.Code, r.Body)
}
r = do(t, h, "DELETE", "/api/v1/libraries/1", admin, nil)
if r.Code != http.StatusConflict || errCode(r) != "scan_running" {
t.Errorf("DELETE: %d %v", r.Code, r.Body)
}
}

View file

@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
@ -47,11 +48,30 @@ type LibraryStore interface {
// Libraries manages the list of libraries and keeps it sane: every path is an
// absolute, existing directory, and no library sits inside or around another
// (that would index the same files twice).
type Libraries struct{ store LibraryStore }
type Libraries struct {
store LibraryStore
// mu makes validate-then-write atomic, so two admins creating libraries at
// the same moment cannot both pass the overlap check. Sonda is a
// single-instance server, so an in-process lock is enough.
mu sync.Mutex
busy func() bool
}
// NewLibraries builds the service.
func NewLibraries(store LibraryStore) *Libraries { return &Libraries{store: store} }
// SetBusy registers a function that tells whether a scan is running. While it
// is, a library's directory cannot change and a library cannot be deleted: the
// scan works from the list it started with and would write under a stale path.
func (s *Libraries) SetBusy(fn func() bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.busy = fn
}
func (s *Libraries) scanning() bool { return s.busy != nil && s.busy() }
// List returns every library.
func (s *Libraries) List(ctx context.Context) ([]Library, error) {
return s.store.ListLibraries(ctx)
@ -59,7 +79,10 @@ func (s *Libraries) List(ctx context.Context) ([]Library, error) {
// Create adds a library.
func (s *Libraries) Create(ctx context.Context, name, path string) (Library, error) {
name, path, err := s.validate(ctx, name, path, 0)
s.mu.Lock()
defer s.mu.Unlock()
name, path, err := s.validate(ctx, name, path, 0, true)
if err != nil {
return Library{}, err
}
@ -69,6 +92,9 @@ func (s *Libraries) Create(ctx context.Context, name, path string) (Library, err
// Update changes the name and/or the path of a library. A nil argument keeps
// the current value. Files keep their identity when the path changes.
func (s *Libraries) Update(ctx context.Context, id int64, name, path *string) (Library, error) {
s.mu.Lock()
defer s.mu.Unlock()
cur, err := s.store.GetLibrary(ctx, id)
if err != nil {
return Library{}, err
@ -80,28 +106,42 @@ func (s *Libraries) Update(ctx context.Context, id int64, name, path *string) (L
if path != nil {
newPath = *path
}
newName, newPath, err = s.validate(ctx, newName, newPath, id)
// An untouched name is not validated again: a migrated library is named
// after its path, which can be longer than the limit for new names.
newName, newPath, err = s.validate(ctx, newName, newPath, id, name != nil)
if err != nil {
return Library{}, err
}
if newPath != cur.Path && s.scanning() {
return Library{}, ErrScanRunning
}
return s.store.UpdateLibrary(ctx, id, newName, newPath)
}
// Delete removes a library from the index. The music on disk is untouched.
func (s *Libraries) Delete(ctx context.Context, id int64) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.scanning() {
return ErrScanRunning
}
return s.store.DeleteLibrary(ctx, id)
}
// validate checks a name and a path, ignoring the library skipID when it
// looks for overlaps. It returns the trimmed name and the cleaned path.
func (s *Libraries) validate(ctx context.Context, name, path string, skipID int64) (string, string, error) {
// looks for overlaps. It returns the trimmed name and the cleaned path. The
// name is only checked when checkName is true.
func (s *Libraries) validate(ctx context.Context, name, path string, skipID int64, checkName bool) (string, string, error) {
name = strings.TrimSpace(name)
if name == "" || utf8.RuneCountInString(name) > maxLibraryNameLen {
return "", "", fmt.Errorf("%w: name must be 1-%d characters", ErrInvalidLibrary, maxLibraryNameLen)
}
for _, r := range name {
if unicode.IsControl(r) {
return "", "", fmt.Errorf("%w: name has control characters", ErrInvalidLibrary)
if checkName {
if name == "" || utf8.RuneCountInString(name) > maxLibraryNameLen {
return "", "", fmt.Errorf("%w: name must be 1-%d characters", ErrInvalidLibrary, maxLibraryNameLen)
}
for _, r := range name {
if unicode.IsControl(r) {
return "", "", fmt.Errorf("%w: name has control characters", ErrInvalidLibrary)
}
}
}
@ -135,6 +175,11 @@ func (s *Libraries) validate(ctx context.Context, name, path string, skipID int6
if o.Path == path {
return "", "", ErrLibraryExists
}
// Done here, not only by the database: SQLite's NOCASE folds ASCII
// letters only, PostgreSQL's lower() folds accents too.
if checkName && strings.ToLower(o.Name) == strings.ToLower(name) {
return "", "", ErrLibraryExists
}
// Compare real paths so a symlink cannot hide an overlap. A library
// whose directory has vanished is compared by its stored path.
otherReal, err := filepath.EvalSymlinks(o.Path)
@ -155,3 +200,40 @@ func (s *Libraries) validate(ctx context.Context, name, path string, skipID int6
func within(child, parent string) bool {
return strings.HasPrefix(child, parent+string(filepath.Separator))
}
// Problems lists libraries that look wrong: a directory that cannot be read, or
// libraries that overlap and would index the same files twice. Migrated data
// skips validation, so this is meant to run at startup and be logged.
func (s *Libraries) Problems(ctx context.Context) ([]string, error) {
libs, err := s.store.ListLibraries(ctx)
if err != nil {
return nil, fmt.Errorf("list libraries: %w", err)
}
var problems []string
reals := make([]string, len(libs))
for i, l := range libs {
real, err := filepath.EvalSymlinks(l.Path)
if err != nil {
problems = append(problems, fmt.Sprintf("library %q: directory %s cannot be read: %v", l.Name, l.Path, err))
continue
}
reals[i] = real
}
for i := range libs {
for j := i + 1; j < len(libs); j++ {
a, b := reals[i], reals[j]
switch {
case a == "" || b == "":
// already reported as unreadable
case a == b:
problems = append(problems, fmt.Sprintf("libraries %q and %q point at the same directory: files are indexed twice", libs[i].Name, libs[j].Name))
case within(a, b):
problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[i].Name, libs[j].Name))
case within(b, a):
problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[j].Name, libs[i].Name))
}
}
}
return problems, nil
}

View file

@ -13,15 +13,18 @@ import (
// memLibraries is an in-memory LibraryStore that enforces unique names and paths.
type memLibraries struct {
mu sync.Mutex
next int64
libs []Library
mu sync.Mutex
next int64
libs []Library
delay time.Duration // makes ListLibraries slow, to widen race windows in tests
}
func (m *memLibraries) ListLibraries(context.Context) ([]Library, error) {
m.mu.Lock()
defer m.mu.Unlock()
return append([]Library(nil), m.libs...), nil
out := append([]Library(nil), m.libs...)
m.mu.Unlock()
time.Sleep(m.delay) // the snapshot is taken before the pause, like a slow query
return out, nil
}
func (m *memLibraries) GetLibrary(_ context.Context, id int64) (Library, error) {
@ -270,3 +273,138 @@ func TestListLibraries(t *testing.T) {
t.Fatalf("libs = %+v err=%v", libs, err)
}
}
// Names must clash on accents and case the same way on every database.
func TestLibraryNamesClashCaseInsensitivelyOnUnicode(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
if _, err := s.Create(ctx, "Música", mkdir(t, filepath.Join(base, "a"))); err != nil {
t.Fatal(err)
}
if _, err := s.Create(ctx, "MÚSICA", mkdir(t, filepath.Join(base, "b"))); !errors.Is(err, ErrLibraryExists) {
t.Fatalf("err = %v, want ErrLibraryExists", err)
}
}
// Two admins creating libraries at the same moment must not both pass the
// overlap check.
func TestConcurrentCreatesCannotOverlap(t *testing.T) {
for round := 0; round < 5; round++ {
s, st := newLibraries()
st.delay = 5 * time.Millisecond
parent := mkdir(t, filepath.Join(t.TempDir(), "music"))
child := mkdir(t, filepath.Join(parent, "a"))
var wg sync.WaitGroup
for i, p := range []string{parent, child} {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = s.Create(context.Background(), "lib"+string(rune('a'+i)), p)
}()
}
wg.Wait()
if len(st.libs) != 1 {
t.Fatalf("round %d: %d libraries created, want 1 (the two paths overlap)", round, len(st.libs))
}
}
}
// A migrated library is named after its path, which can be longer than the
// name limit. Changing only the path must still work.
func TestUpdatingThePathDoesNotRevalidateAnUntouchedName(t *testing.T) {
s, st := newLibraries()
ctx := context.Background()
base := t.TempDir()
a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b"))
long := strings.Repeat("x", 150)
st.libs = []Library{{ID: 1, Name: long, Path: a}}
st.next = 1
got, err := s.Update(ctx, 1, nil, &b)
if err != nil || got.Path != b || got.Name != long {
t.Fatalf("path-only update: %+v err=%v", got, err)
}
short := "short"
if _, err := s.Update(ctx, 1, &short, nil); err != nil {
t.Fatalf("renaming to a valid name: %v", err)
}
tooLong := strings.Repeat("y", 101)
if _, err := s.Update(ctx, 1, &tooLong, nil); !errors.Is(err, ErrInvalidLibrary) {
t.Fatalf("a new name is still validated: err = %v", err)
}
}
// Changing a library's directory or deleting it while a scan runs would leave
// the scan writing under a path that no longer matches.
func TestLibraryPathChangeAndDeleteAreRefusedWhileScanning(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b"))
lib, _ := s.Create(ctx, "Music", a)
scanning := true
s.SetBusy(func() bool { return scanning })
if _, err := s.Update(ctx, lib.ID, nil, &b); !errors.Is(err, ErrScanRunning) {
t.Errorf("moving during a scan: err = %v, want ErrScanRunning", err)
}
if err := s.Delete(ctx, lib.ID); !errors.Is(err, ErrScanRunning) {
t.Errorf("deleting during a scan: err = %v, want ErrScanRunning", err)
}
// A rename, or "moving" to the same path, changes nothing under the scan.
name := "Renamed"
if _, err := s.Update(ctx, lib.ID, &name, nil); err != nil {
t.Errorf("renaming during a scan: %v", err)
}
if _, err := s.Update(ctx, lib.ID, nil, &a); err != nil {
t.Errorf("same path during a scan: %v", err)
}
scanning = false
if _, err := s.Update(ctx, lib.ID, nil, &b); err != nil {
t.Errorf("moving after the scan: %v", err)
}
if err := s.Delete(ctx, lib.ID); err != nil {
t.Errorf("deleting after the scan: %v", err)
}
}
func TestProblemsReportsOverlapAndMissingDirectories(t *testing.T) {
s, st := newLibraries()
ctx := context.Background()
base := t.TempDir()
parent := mkdir(t, filepath.Join(base, "music"))
child := mkdir(t, filepath.Join(parent, "flac"))
gone := filepath.Join(base, "gone")
fine := mkdir(t, filepath.Join(base, "fine"))
// Rows as a migration could leave them, bypassing validation.
st.libs = []Library{
{ID: 1, Name: "music", Path: parent},
{ID: 2, Name: "flac", Path: child},
{ID: 3, Name: "gone", Path: gone},
{ID: 4, Name: "fine", Path: fine},
}
problems, err := s.Problems(ctx)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(problems, "\n")
for _, want := range []string{`"gone"`, `"music"`, `"flac"`} {
if !strings.Contains(joined, want) {
t.Errorf("problems %q do not mention %s", joined, want)
}
}
if strings.Contains(joined, `"fine"`) {
t.Errorf("a healthy library was reported: %q", joined)
}
st.libs = st.libs[3:]
if problems, _ := s.Problems(ctx); len(problems) != 0 {
t.Errorf("a healthy setup reported problems: %v", problems)
}
}

View file

@ -116,3 +116,46 @@ func TestMigration0005OnAnEmptyDatabase(t *testing.T) {
})
}
}
// On Linux /Music and /music are different directories. Both can hold rows,
// and library names must be unique ignoring case, so the migration has to
// give them different names instead of failing.
func TestMigration0005RootsDifferingOnlyInCase(t *testing.T) {
for _, driver := range availableDrivers() {
t.Run(driver, func(t *testing.T) {
ctx := context.Background()
db := openBackend(t, driver)
if err := Migrate(ctx, db, upTo(t, driver, 4)); err != nil {
t.Fatal(err)
}
if _, err := db.Exec("INSERT INTO tracks (title) VALUES ('a'), ('b'), ('c')"); err != nil {
t.Fatal(err)
}
rows := `INSERT INTO audio_files (track_id, root, rel_path, format, size_bytes, mtime_unix) VALUES
(1, '/Music', 'x.mp3', 'mp3', 10, 100),
(2, '/music', 'x.mp3', 'mp3', 20, 200),
(3, '/MUSIC', 'x.mp3', 'mp3', 30, 300)`
if _, err := db.Exec(rows); err != nil {
t.Fatal(err)
}
full, _ := migrations.For(driver)
if err := Migrate(ctx, db, full); err != nil {
t.Fatalf("the upgrade must not fail on roots that differ only in case: %v", err)
}
if n := count(t, db, "SELECT COUNT(*) FROM libraries"); n != 3 {
t.Errorf("libraries = %d, want 3", n)
}
if n := count(t, db, "SELECT COUNT(DISTINCT lower(name)) FROM libraries"); n != 3 {
t.Error("migrated names are not unique ignoring case")
}
// Every file ends up in the library of its own path.
if n := count(t, db, `SELECT COUNT(*) FROM audio_files f JOIN libraries l ON l.id = f.library_id
WHERE (l.path = '/Music' AND f.size_bytes = 10) OR (l.path = '/music' AND f.size_bytes = 20)
OR (l.path = '/MUSIC' AND f.size_bytes = 30)`); n != 3 {
t.Errorf("correctly linked files = %d, want 3", n)
}
})
}
}

View file

@ -11,10 +11,14 @@ CREATE TABLE libraries (
-- Case-insensitive uniqueness (SQLite uses COLLATE NOCASE for the same effect).
CREATE UNIQUE INDEX libraries_name_lower ON libraries (lower(name));
-- Existing data: every distinct root becomes a library. The path doubles as
-- the name, which is always unique; admins can rename it afterwards.
-- Existing data: every distinct root becomes a library, named after its path.
-- Names are unique ignoring case, and on Linux /Music and /music are two
-- directories, so the second one gets a suffix. Admins can rename them later.
INSERT INTO libraries (name, path, created_at)
SELECT DISTINCT root, root, extract(epoch FROM now())::bigint FROM audio_files;
SELECT CASE WHEN n = 1 THEN root ELSE root || ' (' || n || ')' END, root,
extract(epoch FROM now())::bigint
FROM (SELECT root, ROW_NUMBER() OVER (PARTITION BY lower(root) ORDER BY root) AS n
FROM (SELECT DISTINCT root FROM audio_files) AS roots) AS numbered;
ALTER TABLE audio_files ADD COLUMN library_id BIGINT REFERENCES libraries(id) ON DELETE CASCADE;
UPDATE audio_files f SET library_id = l.id FROM libraries l WHERE l.path = f.root;

View file

@ -8,10 +8,13 @@ CREATE TABLE libraries (
created_at INTEGER NOT NULL -- unix seconds
);
-- Existing data: every distinct root becomes a library. The path doubles as
-- the name, which is always unique; admins can rename it afterwards.
-- Existing data: every distinct root becomes a library, named after its path.
-- Names are unique ignoring case, and on Linux /Music and /music are two
-- directories, so the second one gets a suffix. Admins can rename them later.
INSERT INTO libraries (name, path, created_at)
SELECT DISTINCT root, root, unixepoch() FROM audio_files;
SELECT CASE WHEN n = 1 THEN root ELSE root || ' (' || n || ')' END, root, unixepoch()
FROM (SELECT root, ROW_NUMBER() OVER (PARTITION BY lower(root) ORDER BY root) AS n
FROM (SELECT DISTINCT root FROM audio_files));
-- SQLite cannot drop a column that is part of a UNIQUE constraint, so the
-- table is rebuilt.