Merge pull request 'feat(library): named media libraries managed through the API' (#10) from feat/media-libraries into main

This commit is contained in:
squidd 2026-11-11 13:35:19 +01:00
commit a28d028a67
28 changed files with 1861 additions and 236 deletions

View file

@ -89,16 +89,21 @@ func run() error {
return err
}
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
scans := library.NewManager(scanner, cfg.Library.Roots)
scans := library.NewManager(scanner, libraryRepo)
logScanResults(logger, scans)
libraries := library.NewLibraries(libraryRepo)
libraries.SetBusy(func() bool { return scans.Status().Running })
warnAboutLibraries(ctx, logger, libraries)
router := httpapi.NewRouter(httpapi.Deps{
Logger: logger,
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
Tokens: tokens,
Listening: listening.NewService(storage.NewListeningRepo(db)),
Scan: scanControl{ctx: ctx, m: scans},
Streams: streaming.NewService(libraryRepo, cfg.Library.Roots),
Libraries: libraries,
Streams: streaming.NewService(libraryRepo),
})
// No WriteTimeout: audio streaming (later) needs long responses.
@ -113,7 +118,6 @@ func run() error {
zap.String("config", path),
zap.String("env", cfg.Log.Env),
zap.String("listen", cfg.Server.Listen),
zap.Strings("library_roots", cfg.Library.Roots),
zap.String("database_driver", cfg.Database.Driver), // never log the DSN: it holds the password
)

View file

@ -15,8 +15,9 @@ type scanControl struct {
m *library.Manager
}
func (c scanControl) Start() error { return c.m.Start(c.ctx) }
func (c scanControl) Status() library.Status { return c.m.Status() }
func (c scanControl) Start() error { return c.m.Start(c.ctx) }
func (c scanControl) StartLibrary(id int64) error { return c.m.StartLibrary(c.ctx, id) }
func (c scanControl) Status() library.Status { return c.m.Status() }
// logScanResults writes one log line per finished scan. File paths are left
// out on purpose: audio paths are private.
@ -35,3 +36,17 @@ func logScanResults(logger *zap.Logger, m *library.Manager) {
logger.Info("library scan finished", fields...)
})
}
// warnAboutLibraries logs libraries that look wrong. Rows created by the
// upgrade from the roots-in-config versions skip validation, so a stale root
// can show up as a missing directory or as a library inside another one.
func warnAboutLibraries(ctx context.Context, logger *zap.Logger, libs *library.Libraries) {
problems, err := libs.Problems(ctx)
if err != nil {
logger.Warn("could not check the libraries", zap.Error(err))
return
}
for _, p := range problems {
logger.Warn("library problem: review GET /api/v1/libraries", zap.String("problem", p))
}
}

View file

@ -15,6 +15,7 @@ import (
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/auth"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/storage"
"codeberg.org/squidd/sonda/internal/streaming"
@ -62,7 +63,8 @@ func newTestAPIWith(t *testing.T, mutate func(*Deps)) http.Handler {
Tokens: tokens,
Listening: listening.NewService(storage.NewListeningRepo(db)),
Scan: &fakeScan{},
Streams: streaming.NewService(noFiles{}, nil),
Libraries: library.NewLibraries(storage.NewLibraryRepo(db)),
Streams: streaming.NewService(noFiles{}),
}
mutate(&deps)
return NewRouter(deps)

View file

@ -0,0 +1,110 @@
package httpapi
import (
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
"codeberg.org/squidd/sonda/internal/library"
)
// librariesHandlers are the admin-only routes that manage libraries.
type librariesHandlers struct{ svc libraryService }
type libraryJSON struct {
ID int64 `json:"id"`
Name string `json:"name"`
Path string `json:"path"`
CreatedAt string `json:"created_at"`
}
func toLibraryJSON(l library.Library) libraryJSON {
return libraryJSON{ID: l.ID, Name: l.Name, Path: l.Path, CreatedAt: l.CreatedAt.UTC().Format(time.RFC3339)}
}
func writeLibraryError(c *gin.Context, err error) {
switch {
case errors.Is(err, library.ErrInvalidLibrary):
writeError(c, http.StatusBadRequest, "invalid_input", err.Error())
case errors.Is(err, library.ErrLibraryExists):
writeError(c, http.StatusConflict, "library_exists", err.Error())
case errors.Is(err, library.ErrLibraryNotFound):
writeError(c, http.StatusNotFound, "library_not_found", err.Error())
case errors.Is(err, library.ErrScanRunning):
writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish")
default:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
}
}
func (h *librariesHandlers) list(c *gin.Context) {
libs, err := h.svc.List(c.Request.Context())
if err != nil {
writeLibraryError(c, err)
return
}
out := make([]libraryJSON, len(libs))
for i, l := range libs {
out[i] = toLibraryJSON(l)
}
c.JSON(http.StatusOK, gin.H{"libraries": out})
}
type createLibraryRequest struct {
Name string `json:"name"`
Path string `json:"path"`
}
func (h *librariesHandlers) create(c *gin.Context) {
var req createLibraryRequest
if !bindJSON(c, &req) {
return
}
lib, err := h.svc.Create(c.Request.Context(), req.Name, req.Path)
if err != nil {
writeLibraryError(c, err)
return
}
c.JSON(http.StatusCreated, toLibraryJSON(lib))
}
type updateLibraryRequest struct {
Name *string `json:"name"` // pointers: an absent field keeps the current value
Path *string `json:"path"`
}
func (h *librariesHandlers) update(c *gin.Context) {
id, ok := pathID(c)
if !ok {
return
}
var req updateLibraryRequest
if !bindJSON(c, &req) {
return
}
if req.Name == nil && req.Path == nil {
writeError(c, http.StatusBadRequest, "invalid_input", "send name, path or both")
return
}
lib, err := h.svc.Update(c.Request.Context(), id, req.Name, req.Path)
if err != nil {
writeLibraryError(c, err)
return
}
c.JSON(http.StatusOK, toLibraryJSON(lib))
}
func (h *librariesHandlers) remove(c *gin.Context) {
id, ok := pathID(c)
if !ok {
return
}
if err := h.svc.Delete(c.Request.Context(), id); err != nil {
writeLibraryError(c, err)
return
}
c.Status(http.StatusNoContent)
}

View file

@ -13,7 +13,7 @@ import (
type libraryHandlers struct{ scan scanService }
type summaryJSON struct {
Roots int `json:"roots"`
Libraries int `json:"libraries"`
Found int `json:"found"`
Added int `json:"added"`
Updated int `json:"updated"`
@ -35,10 +35,26 @@ type scanStatusJSON struct {
// startScan launches a background scan of the configured roots.
func (h *libraryHandlers) startScan(c *gin.Context) {
err := h.scan.Start()
writeScanStarted(c, h.scan.Start())
}
// startLibraryScan launches a background scan of one library.
func (h *libraryHandlers) startLibraryScan(c *gin.Context) {
id, ok := pathID(c)
if !ok {
return
}
writeScanStarted(c, h.scan.StartLibrary(id))
}
func writeScanStarted(c *gin.Context, err error) {
switch {
case errors.Is(err, library.ErrScanRunning):
writeError(c, http.StatusConflict, "scan_running", err.Error())
case errors.Is(err, library.ErrNoLibraries):
writeError(c, http.StatusConflict, "no_libraries", "add a library before scanning")
case errors.Is(err, library.ErrLibraryNotFound):
writeError(c, http.StatusNotFound, "library_not_found", err.Error())
case err != nil:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")

View file

@ -0,0 +1,207 @@
package httpapi
import (
"context"
"net/http"
"os"
"path/filepath"
"strconv"
"testing"
"codeberg.org/squidd/sonda/internal/library"
)
func libPath(id float64) string {
return "/api/v1/libraries/" + strconv.FormatInt(int64(id), 10)
}
func mkdirT(t *testing.T, path string) string {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatal(err)
}
return path
}
func TestLibraryRoutesAreAdminOnly(t *testing.T) {
h := newTestAPIWithScan(t, &fakeScan{})
admin := registerAndLogin(t, h)
_, bob := addUser(t, h, admin, "bob", false)
calls := []struct{ method, path string }{
{"GET", "/api/v1/libraries"},
{"POST", "/api/v1/libraries"},
{"PATCH", "/api/v1/libraries/1"},
{"DELETE", "/api/v1/libraries/1"},
{"POST", "/api/v1/libraries/1/scan"},
}
for _, c := range calls {
if r := do(t, h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("%s %s without token: %d", c.method, c.path, r.Code)
}
r := do(t, h, c.method, c.path, bob, map[string]any{"name": "x", "path": "/tmp"})
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
t.Errorf("%s %s as non-admin: %d %v", c.method, c.path, r.Code, r.Body)
}
}
}
func TestLibraryLifecycle(t *testing.T) {
h := newTestAPI(t)
admin := registerAndLogin(t, h)
base := t.TempDir()
jazz := mkdirT(t, filepath.Join(base, "jazz"))
rock := mkdirT(t, filepath.Join(base, "rock"))
r := do(t, h, "POST", "/api/v1/libraries", admin, map[string]string{"name": " Jazz ", "path": jazz})
if r.Code != http.StatusCreated || r.Body["name"] != "Jazz" || r.Body["path"] != jazz || r.Body["id"] == nil {
t.Fatalf("create: %d %v", r.Code, r.Body)
}
id, _ := r.Body["id"].(float64)
r = do(t, h, "GET", "/api/v1/libraries", admin, nil)
libs, _ := r.Body["libraries"].([]any)
if r.Code != http.StatusOK || len(libs) != 1 {
t.Fatalf("list: %d %v", r.Code, r.Body)
}
if first := libs[0].(map[string]any); first["created_at"] == nil || first["name"] != "Jazz" {
t.Errorf("listed library = %v", first)
}
// Rename only, then move only, then both.
r = do(t, h, "PATCH", libPath(id), admin, map[string]string{"name": "Jazz Collection"})
if r.Code != http.StatusOK || r.Body["name"] != "Jazz Collection" || r.Body["path"] != jazz {
t.Fatalf("rename: %d %v", r.Code, r.Body)
}
r = do(t, h, "PATCH", libPath(id), admin, map[string]string{"path": rock})
if r.Code != http.StatusOK || r.Body["path"] != rock || r.Body["name"] != "Jazz Collection" {
t.Fatalf("move: %d %v", r.Code, r.Body)
}
if r := do(t, h, "DELETE", libPath(id), admin, nil); r.Code != http.StatusNoContent {
t.Fatalf("delete: %d %v", r.Code, r.Body)
}
if r := do(t, h, "DELETE", libPath(id), admin, nil); r.Code != http.StatusNotFound || errCode(r) != "library_not_found" {
t.Errorf("delete twice: %d %v", r.Code, r.Body)
}
r = do(t, h, "GET", "/api/v1/libraries", admin, nil)
if libs, _ := r.Body["libraries"].([]any); len(libs) != 0 {
t.Errorf("libraries after delete: %v", libs)
}
}
func TestLibraryValidationErrors(t *testing.T) {
h := newTestAPI(t)
admin := registerAndLogin(t, h)
base := t.TempDir()
parent := mkdirT(t, filepath.Join(base, "music"))
child := mkdirT(t, filepath.Join(parent, "jazz"))
other := mkdirT(t, filepath.Join(base, "other"))
file := filepath.Join(base, "file.txt")
if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if r := do(t, h, "POST", "/api/v1/libraries", admin, map[string]string{"name": "Music", "path": parent}); r.Code != http.StatusCreated {
t.Fatal(r.Code, r.Body)
}
tests := []struct {
name string
body any
status int
code string
}{
{"empty name", map[string]string{"name": " ", "path": other}, 400, "invalid_input"},
{"relative path", map[string]string{"name": "x", "path": "music/jazz"}, 400, "invalid_input"},
{"filesystem root", map[string]string{"name": "x", "path": "/"}, 400, "invalid_input"},
{"missing directory", map[string]string{"name": "x", "path": filepath.Join(base, "nope")}, 400, "invalid_input"},
{"a file", map[string]string{"name": "x", "path": file}, 400, "invalid_input"},
{"inside another library", map[string]string{"name": "x", "path": child}, 400, "invalid_input"},
{"around another library", map[string]string{"name": "x", "path": base}, 400, "invalid_input"},
{"same name", map[string]string{"name": "MUSIC", "path": other}, 409, "library_exists"},
{"same path", map[string]string{"name": "x", "path": parent}, 409, "library_exists"},
{"malformed json", "{oops", 400, "invalid_input"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := do(t, h, "POST", "/api/v1/libraries", admin, tt.body)
if r.Code != tt.status || errCode(r) != tt.code {
t.Fatalf("%d %v, want %d %s", r.Code, r.Body, tt.status, tt.code)
}
})
}
}
func TestLibraryPatchValidation(t *testing.T) {
h := newTestAPI(t)
admin := registerAndLogin(t, h)
base := t.TempDir()
a := mkdirT(t, filepath.Join(base, "a"))
r := do(t, h, "POST", "/api/v1/libraries", admin, map[string]string{"name": "A", "path": a})
id, _ := r.Body["id"].(float64)
if r := do(t, h, "PATCH", libPath(id), admin, map[string]any{}); r.Code != http.StatusBadRequest {
t.Errorf("empty patch: %d", r.Code)
}
if r := do(t, h, "PATCH", libPath(id), admin, map[string]string{"name": ""}); r.Code != http.StatusBadRequest {
t.Errorf("empty name: %d", r.Code)
}
if r := do(t, h, "PATCH", "/api/v1/libraries/999", admin, map[string]string{"name": "x"}); r.Code != http.StatusNotFound || errCode(r) != "library_not_found" {
t.Errorf("unknown library: %d %v", r.Code, r.Body)
}
if r := do(t, h, "PATCH", "/api/v1/libraries/abc", admin, map[string]string{"name": "x"}); r.Code != http.StatusBadRequest {
t.Errorf("bad id: %d", r.Code)
}
}
func TestScanOneLibrary(t *testing.T) {
scan := &fakeScan{}
h := newTestAPIWithScan(t, scan)
admin := registerAndLogin(t, h)
if r := do(t, h, "POST", "/api/v1/libraries/7/scan", admin, nil); r.Code != http.StatusAccepted || scan.lastOnly != 7 {
t.Fatalf("scan one: %d %v (library %d)", r.Code, r.Body, scan.lastOnly)
}
if r := do(t, h, "POST", "/api/v1/libraries/7/scan", admin, nil); r.Code != http.StatusConflict || errCode(r) != "scan_running" {
t.Errorf("second scan: %d %v", r.Code, r.Body)
}
if r := do(t, h, "POST", "/api/v1/libraries/999/scan", admin, nil); r.Code != http.StatusNotFound || errCode(r) != "library_not_found" {
t.Errorf("unknown library: %d %v", r.Code, r.Body)
}
}
func TestScanAllWithoutLibraries(t *testing.T) {
h := newTestAPIWithScan(t, &fakeScan{noLibs: true})
admin := registerAndLogin(t, h)
if r := do(t, h, "POST", "/api/v1/library/scan", admin, nil); r.Code != http.StatusConflict || errCode(r) != "no_libraries" {
t.Fatalf("%d %v", r.Code, r.Body)
}
}
// busyLibraries is a libraryService whose directory changes are refused.
type busyLibraries struct{}
func (busyLibraries) List(context.Context) ([]library.Library, error) { return nil, nil }
func (busyLibraries) Create(context.Context, string, string) (library.Library, error) {
return library.Library{}, nil
}
func (busyLibraries) Update(context.Context, int64, *string, *string) (library.Library, error) {
return library.Library{}, library.ErrScanRunning
}
func (busyLibraries) Delete(context.Context, int64) error { return library.ErrScanRunning }
func TestLibraryChangesDuringAScanAreConflicts(t *testing.T) {
h := newTestAPIWith(t, func(d *Deps) { d.Libraries = busyLibraries{} })
admin := registerAndLogin(t, h)
r := do(t, h, "PATCH", "/api/v1/libraries/1", admin, map[string]string{"path": "/tmp"})
if r.Code != http.StatusConflict || errCode(r) != "scan_running" {
t.Errorf("PATCH: %d %v", r.Code, r.Body)
}
r = do(t, h, "DELETE", "/api/v1/libraries/1", admin, nil)
if r.Code != http.StatusConflict || errCode(r) != "scan_running" {
t.Errorf("DELETE: %d %v", r.Code, r.Body)
}
}

View file

@ -9,12 +9,17 @@ import (
)
type fakeScan struct {
running bool
starts int
status library.Status
running bool
noLibs bool
starts int
lastOnly int64 // library of the last StartLibrary call
status library.Status
}
func (f *fakeScan) Start() error {
if f.noLibs {
return library.ErrNoLibraries
}
if f.running {
return library.ErrScanRunning
}
@ -23,6 +28,19 @@ func (f *fakeScan) Start() error {
return nil
}
func (f *fakeScan) StartLibrary(id int64) error {
if id == 999 {
return library.ErrLibraryNotFound
}
if f.running {
return library.ErrScanRunning
}
f.running = true
f.starts++
f.lastOnly = id
return nil
}
func (f *fakeScan) Status() library.Status {
s := f.status
s.Running = f.running
@ -59,7 +77,7 @@ func TestScanStartAndConflict(t *testing.T) {
func TestScanStatus(t *testing.T) {
scan := &fakeScan{status: library.Status{
LastFinishedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
Last: &library.Summary{Roots: 1, Found: 10, Added: 7, Failed: 1},
Last: &library.Summary{Libraries: 1, Found: 10, Added: 7, Failed: 1},
LastError: "boom",
}}
h := newTestAPIWithScan(t, scan)

View file

@ -47,9 +47,17 @@ type streamService interface {
type scanService interface {
Start() error
StartLibrary(id int64) error
Status() library.Status
}
type libraryService interface {
List(ctx context.Context) ([]library.Library, error)
Create(ctx context.Context, name, path string) (library.Library, error)
Update(ctx context.Context, id int64, name, path *string) (library.Library, error)
Delete(ctx context.Context, id int64) error
}
// Deps are the services the API needs.
type Deps struct {
Logger *zap.Logger
@ -57,6 +65,7 @@ type Deps struct {
Tokens tokenParser
Listening listenService
Scan scanService
Libraries libraryService
Streams streamService
}
@ -96,6 +105,12 @@ func NewRouter(d Deps) *gin.Engine {
// Admins only.
admin := protected.Group("", requireAdmin(d.Auth))
admin.POST("/library/scan", lib.startScan)
libs := &librariesHandlers{svc: d.Libraries}
admin.GET("/libraries", libs.list)
admin.POST("/libraries", libs.create)
admin.PATCH("/libraries/:id", libs.update)
admin.DELETE("/libraries/:id", libs.remove)
admin.POST("/libraries/:id/scan", lib.startLibraryScan)
u := &usersHandlers{svc: d.Auth}
admin.GET("/users", u.list)
admin.POST("/users", u.create)

View file

@ -47,7 +47,7 @@ func newStreamFixture(t *testing.T) streamFixture {
2: {Root: root, RelPath: "gone.mp3"},
3: {Root: root, RelPath: "../escape.mp3"},
}
h := newTestAPIWith(t, func(d *Deps) { d.Streams = streaming.NewService(src, []string{root}) })
h := newTestAPIWith(t, func(d *Deps) { d.Streams = streaming.NewService(src) })
return streamFixture{h: h, token: registerAndLogin(t, h), data: data}
}

View file

@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/pelletier/go-toml/v2"
@ -31,10 +30,10 @@ type Server struct {
Listen string `toml:"listen"`
}
// Library holds the music directories. They are treated as read-only.
// Library holds scanner settings. The libraries themselves (name and path) live
// in the database and are managed through the API.
type Library struct {
Roots []string `toml:"roots"`
ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan
ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan
}
// Database selects and configures the storage backend.
@ -103,7 +102,7 @@ func Load(path string) (Config, error) {
dec := toml.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
if err := dec.Decode(&cfg); err != nil {
return Config{}, fmt.Errorf("parse config %s: %w", path, err)
return Config{}, explainDecodeError(path, err)
}
}
@ -169,24 +168,22 @@ func (c Config) Validate() error {
if c.Library.ScanWorkers < 1 {
errs = append(errs, errors.New("library.scan_workers must be at least 1"))
}
if len(c.Library.Roots) == 0 {
errs = append(errs, errors.New("library.roots needs at least one directory"))
}
for _, root := range c.Library.Roots {
if !filepath.IsAbs(root) {
// Roots are stored with every indexed file and used to serve it:
// a relative one would depend on the directory sondad starts in.
errs = append(errs, fmt.Errorf("library.roots: %q must be an absolute path", root))
continue
}
info, err := os.Stat(root)
switch {
case err != nil:
errs = append(errs, fmt.Errorf("library.roots: %w", err))
case !info.IsDir():
errs = append(errs, fmt.Errorf("library.roots: %s is not a directory", root))
}
}
return errors.Join(errs...)
}
// explainDecodeError turns the strict decoder's "unknown key" into something
// useful for the one key that used to exist.
func explainDecodeError(path string, err error) error {
var missing *toml.StrictMissingError
if errors.As(err, &missing) {
for _, e := range missing.Errors {
key := e.Key()
if len(key) == 2 && key[0] == "library" && key[1] == "roots" {
return fmt.Errorf("parse config %s: library.roots was removed; "+
"remove it and add each directory as a library with POST /api/v1/libraries", path)
}
}
}
return fmt.Errorf("parse config %s: %w", path, err)
}

View file

@ -52,7 +52,7 @@ func TestLoadFileOverridesDefaults(t *testing.T) {
listen = "0.0.0.0:9000"
[library]
roots = ["/a", "/b"]
scan_workers = 9
[providers.lastfm]
api_key = "k"
@ -64,8 +64,8 @@ api_key = "k"
if cfg.Server.Listen != "0.0.0.0:9000" {
t.Errorf("listen = %q", cfg.Server.Listen)
}
if len(cfg.Library.Roots) != 2 {
t.Errorf("roots = %v", cfg.Library.Roots)
if cfg.Library.ScanWorkers != 9 {
t.Errorf("scan workers = %d", cfg.Library.ScanWorkers)
}
if cfg.Providers.LastFM.APIKey != "k" {
t.Errorf("api key = %q", cfg.Providers.LastFM.APIKey)
@ -113,22 +113,12 @@ func TestLoadMissingFile(t *testing.T) {
}
func TestValidate(t *testing.T) {
good := t.TempDir()
file := filepath.Join(good, "f")
if err := os.WriteFile(file, nil, 0o600); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
mutate func(*Config)
wantErr string
}{
{"ok", func(c *Config) {}, ""},
{"no roots", func(c *Config) { c.Library.Roots = nil }, "library.roots"},
{"root missing", func(c *Config) { c.Library.Roots = []string{filepath.Join(good, "nope")} }, "library.roots"},
{"relative root", func(c *Config) { c.Library.Roots = []string{"music"} }, "absolute"},
{"root is file", func(c *Config) { c.Library.Roots = []string{file} }, "not a directory"},
{"bad env", func(c *Config) { c.Log.Env = "staging" }, "log.env"},
{"empty listen", func(c *Config) { c.Server.Listen = "" }, "server.listen"},
{"empty db", func(c *Config) { c.Database.Path = "" }, "database.path"},
@ -151,7 +141,6 @@ func TestValidate(t *testing.T) {
if err != nil {
t.Fatal(err)
}
cfg.Library.Roots = []string{good}
cfg.Auth.JWTSecret = strings.Repeat("s", 32)
tt.mutate(&cfg)
err = cfg.Validate()
@ -167,3 +156,16 @@ func TestValidate(t *testing.T) {
})
}
}
// library.roots moved into the database; an old config must fail with a hint,
// not with a bare "unknown key".
func TestLoadExplainsRemovedLibraryRoots(t *testing.T) {
p := writeFile(t, t.TempDir(), "[library]\nroots = [\"/music\"]\n")
_, err := Load(p)
if err == nil {
t.Fatal("expected an error for library.roots")
}
if !strings.Contains(err.Error(), "library.roots") || !strings.Contains(err.Error(), "/api/v1/libraries") {
t.Errorf("error = %v, want a pointer to the libraries API", err)
}
}

View file

@ -0,0 +1,239 @@
package library
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
)
// Errors returned by the Libraries service.
var (
ErrLibraryNotFound = errors.New("library not found")
ErrLibraryExists = errors.New("a library with this name or path already exists")
ErrInvalidLibrary = errors.New("invalid library")
)
const maxLibraryNameLen = 100
// Library is a named directory of music.
type Library struct {
ID int64
Name string
Path string // absolute directory
CreatedAt time.Time
}
// LibraryStore persists libraries.
type LibraryStore interface {
ListLibraries(ctx context.Context) ([]Library, error)
// GetLibrary returns ErrLibraryNotFound for an unknown id.
GetLibrary(ctx context.Context, id int64) (Library, error)
// CreateLibrary and UpdateLibrary return ErrLibraryExists when the name
// (case-insensitive) or the path is already used by another library.
CreateLibrary(ctx context.Context, name, path string) (Library, error)
UpdateLibrary(ctx context.Context, id int64, name, path string) (Library, error)
// DeleteLibrary removes the library and its indexed files, never the music
// on disk. It returns ErrLibraryNotFound for an unknown id.
DeleteLibrary(ctx context.Context, id int64) error
}
// Libraries manages the list of libraries and keeps it sane: every path is an
// absolute, existing directory, and no library sits inside or around another
// (that would index the same files twice).
type Libraries struct {
store LibraryStore
// mu makes validate-then-write atomic, so two admins creating libraries at
// the same moment cannot both pass the overlap check. Sonda is a
// single-instance server, so an in-process lock is enough.
mu sync.Mutex
busy func() bool
}
// NewLibraries builds the service.
func NewLibraries(store LibraryStore) *Libraries { return &Libraries{store: store} }
// SetBusy registers a function that tells whether a scan is running. While it
// is, a library's directory cannot change and a library cannot be deleted: the
// scan works from the list it started with and would write under a stale path.
func (s *Libraries) SetBusy(fn func() bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.busy = fn
}
func (s *Libraries) scanning() bool { return s.busy != nil && s.busy() }
// List returns every library.
func (s *Libraries) List(ctx context.Context) ([]Library, error) {
return s.store.ListLibraries(ctx)
}
// Create adds a library.
func (s *Libraries) Create(ctx context.Context, name, path string) (Library, error) {
s.mu.Lock()
defer s.mu.Unlock()
name, path, err := s.validate(ctx, name, path, 0, true)
if err != nil {
return Library{}, err
}
return s.store.CreateLibrary(ctx, name, path)
}
// Update changes the name and/or the path of a library. A nil argument keeps
// the current value. Files keep their identity when the path changes.
func (s *Libraries) Update(ctx context.Context, id int64, name, path *string) (Library, error) {
s.mu.Lock()
defer s.mu.Unlock()
cur, err := s.store.GetLibrary(ctx, id)
if err != nil {
return Library{}, err
}
newName, newPath := cur.Name, cur.Path
if name != nil {
newName = *name
}
if path != nil {
newPath = *path
}
// An untouched name is not validated again: a migrated library is named
// after its path, which can be longer than the limit for new names.
newName, newPath, err = s.validate(ctx, newName, newPath, id, name != nil)
if err != nil {
return Library{}, err
}
if newPath != cur.Path && s.scanning() {
return Library{}, ErrScanRunning
}
return s.store.UpdateLibrary(ctx, id, newName, newPath)
}
// Delete removes a library from the index. The music on disk is untouched.
func (s *Libraries) Delete(ctx context.Context, id int64) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.scanning() {
return ErrScanRunning
}
return s.store.DeleteLibrary(ctx, id)
}
// validate checks a name and a path, ignoring the library skipID when it
// looks for overlaps. It returns the trimmed name and the cleaned path. The
// name is only checked when checkName is true.
func (s *Libraries) validate(ctx context.Context, name, path string, skipID int64, checkName bool) (string, string, error) {
name = strings.TrimSpace(name)
if checkName {
if name == "" || utf8.RuneCountInString(name) > maxLibraryNameLen {
return "", "", fmt.Errorf("%w: name must be 1-%d characters", ErrInvalidLibrary, maxLibraryNameLen)
}
for _, r := range name {
if unicode.IsControl(r) {
return "", "", fmt.Errorf("%w: name has control characters", ErrInvalidLibrary)
}
}
}
if path == "" || !filepath.IsAbs(path) {
return "", "", fmt.Errorf("%w: path must be absolute", ErrInvalidLibrary)
}
path = filepath.Clean(path)
if filepath.Dir(path) == path {
return "", "", fmt.Errorf("%w: the filesystem root cannot be a library", ErrInvalidLibrary)
}
real, err := filepath.EvalSymlinks(path)
if errors.Is(err, fs.ErrNotExist) {
return "", "", fmt.Errorf("%w: directory does not exist", ErrInvalidLibrary)
}
if err != nil {
return "", "", fmt.Errorf("%w: cannot read the directory: %v", ErrInvalidLibrary, err)
}
info, err := os.Stat(real)
if err != nil || !info.IsDir() {
return "", "", fmt.Errorf("%w: path is not a directory", ErrInvalidLibrary)
}
others, err := s.store.ListLibraries(ctx)
if err != nil {
return "", "", fmt.Errorf("list libraries: %w", err)
}
for _, o := range others {
if o.ID == skipID {
continue
}
if o.Path == path {
return "", "", ErrLibraryExists
}
// Done here, not only by the database: SQLite's NOCASE folds ASCII
// letters only, PostgreSQL's lower() folds accents too.
if checkName && strings.ToLower(o.Name) == strings.ToLower(name) {
return "", "", ErrLibraryExists
}
// Compare real paths so a symlink cannot hide an overlap. A library
// whose directory has vanished is compared by its stored path.
otherReal, err := filepath.EvalSymlinks(o.Path)
if err != nil {
otherReal = o.Path
}
if real == otherReal {
return "", "", ErrLibraryExists
}
if within(real, otherReal) || within(otherReal, real) {
return "", "", fmt.Errorf("%w: overlaps the library %q", ErrInvalidLibrary, o.Name)
}
}
return name, path, nil
}
// within reports whether child is strictly inside parent.
func within(child, parent string) bool {
return strings.HasPrefix(child, parent+string(filepath.Separator))
}
// Problems lists libraries that look wrong: a directory that cannot be read, or
// libraries that overlap and would index the same files twice. Migrated data
// skips validation, so this is meant to run at startup and be logged.
func (s *Libraries) Problems(ctx context.Context) ([]string, error) {
libs, err := s.store.ListLibraries(ctx)
if err != nil {
return nil, fmt.Errorf("list libraries: %w", err)
}
var problems []string
reals := make([]string, len(libs))
for i, l := range libs {
real, err := filepath.EvalSymlinks(l.Path)
if err != nil {
problems = append(problems, fmt.Sprintf("library %q: directory %s cannot be read: %v", l.Name, l.Path, err))
continue
}
reals[i] = real
}
for i := range libs {
for j := i + 1; j < len(libs); j++ {
a, b := reals[i], reals[j]
switch {
case a == "" || b == "":
// already reported as unreadable
case a == b:
problems = append(problems, fmt.Sprintf("libraries %q and %q point at the same directory: files are indexed twice", libs[i].Name, libs[j].Name))
case within(a, b):
problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[i].Name, libs[j].Name))
case within(b, a):
problems = append(problems, fmt.Sprintf("library %q is inside library %q: files are indexed twice", libs[j].Name, libs[i].Name))
}
}
}
return problems, nil
}

View file

@ -0,0 +1,410 @@
package library
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// memLibraries is an in-memory LibraryStore that enforces unique names and paths.
type memLibraries struct {
mu sync.Mutex
next int64
libs []Library
delay time.Duration // makes ListLibraries slow, to widen race windows in tests
}
func (m *memLibraries) ListLibraries(context.Context) ([]Library, error) {
m.mu.Lock()
out := append([]Library(nil), m.libs...)
m.mu.Unlock()
time.Sleep(m.delay) // the snapshot is taken before the pause, like a slow query
return out, nil
}
func (m *memLibraries) GetLibrary(_ context.Context, id int64) (Library, error) {
m.mu.Lock()
defer m.mu.Unlock()
for _, l := range m.libs {
if l.ID == id {
return l, nil
}
}
return Library{}, ErrLibraryNotFound
}
func (m *memLibraries) clash(name, path string, skip int64) bool {
for _, l := range m.libs {
if l.ID != skip && (strings.EqualFold(l.Name, name) || l.Path == path) {
return true
}
}
return false
}
func (m *memLibraries) CreateLibrary(_ context.Context, name, path string) (Library, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.clash(name, path, 0) {
return Library{}, ErrLibraryExists
}
m.next++
l := Library{ID: m.next, Name: name, Path: path, CreatedAt: time.Now()}
m.libs = append(m.libs, l)
return l, nil
}
func (m *memLibraries) UpdateLibrary(_ context.Context, id int64, name, path string) (Library, error) {
m.mu.Lock()
defer m.mu.Unlock()
for i, l := range m.libs {
if l.ID == id {
if m.clash(name, path, id) {
return Library{}, ErrLibraryExists
}
m.libs[i].Name, m.libs[i].Path = name, path
return m.libs[i], nil
}
}
return Library{}, ErrLibraryNotFound
}
func (m *memLibraries) DeleteLibrary(_ context.Context, id int64) error {
m.mu.Lock()
defer m.mu.Unlock()
for i, l := range m.libs {
if l.ID == id {
m.libs = append(m.libs[:i], m.libs[i+1:]...)
return nil
}
}
return ErrLibraryNotFound
}
func newLibraries() (*Libraries, *memLibraries) {
st := &memLibraries{}
return NewLibraries(st), st
}
func mkdir(t *testing.T, path string) string {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatal(err)
}
return path
}
func TestCreateLibrary(t *testing.T) {
s, _ := newLibraries()
dir := t.TempDir()
lib, err := s.Create(context.Background(), " Jazz ", dir+"/")
if err != nil {
t.Fatal(err)
}
if lib.Name != "Jazz" || lib.Path != dir || lib.ID == 0 {
t.Errorf("library = %+v, want trimmed name and cleaned path %q", lib, dir)
}
}
func TestCreateLibraryValidation(t *testing.T) {
dir := t.TempDir()
file := filepath.Join(dir, "file.txt")
if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
tests := []struct {
name, libName, path string
}{
{"empty name", "", dir},
{"blank name", " ", dir},
{"long name", strings.Repeat("n", 101), dir},
{"control character in name", "bad\x00name", dir},
{"empty path", "x", ""},
{"relative path", "x", "music/jazz"},
{"filesystem root", "x", "/"},
{"path does not exist", "x", filepath.Join(dir, "nope")},
{"path is a file", "x", file},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, _ := newLibraries()
if _, err := s.Create(context.Background(), tt.libName, tt.path); !errors.Is(err, ErrInvalidLibrary) {
t.Fatalf("err = %v, want ErrInvalidLibrary", err)
}
})
}
}
func TestCreateLibraryDuplicates(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
a, b := mkdir(t, filepath.Join(t.TempDir(), "a")), mkdir(t, filepath.Join(t.TempDir(), "b"))
if _, err := s.Create(ctx, "Music", a); err != nil {
t.Fatal(err)
}
if _, err := s.Create(ctx, "MUSIC", b); !errors.Is(err, ErrLibraryExists) {
t.Errorf("same name, other case: err = %v", err)
}
if _, err := s.Create(ctx, "Other", a); !errors.Is(err, ErrLibraryExists) {
t.Errorf("same path: err = %v", err)
}
if _, err := s.Create(ctx, "Other", a+"/"); !errors.Is(err, ErrLibraryExists) {
t.Errorf("same path with a trailing slash: err = %v", err)
}
}
// Nested libraries would index the same files twice.
func TestCreateLibraryRejectsOverlap(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
parent := mkdir(t, filepath.Join(base, "music"))
child := mkdir(t, filepath.Join(parent, "jazz"))
sibling := mkdir(t, filepath.Join(base, "music2")) // shares a string prefix, not a directory
if _, err := s.Create(ctx, "Music", parent); err != nil {
t.Fatal(err)
}
if _, err := s.Create(ctx, "Jazz", child); !errors.Is(err, ErrInvalidLibrary) {
t.Errorf("inside an existing library: err = %v", err)
}
if _, err := s.Create(ctx, "Sibling", sibling); err != nil {
t.Errorf("a sibling with a common prefix must be allowed: %v", err)
}
if _, err := s.Create(ctx, "Everything", base); !errors.Is(err, ErrInvalidLibrary) {
t.Errorf("around existing libraries: err = %v", err)
}
}
func TestCreateLibraryRejectsOverlapThroughSymlinks(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
parent := mkdir(t, filepath.Join(base, "music"))
child := mkdir(t, filepath.Join(parent, "jazz"))
link := filepath.Join(base, "shortcut")
if err := os.Symlink(child, link); err != nil {
t.Skip("symlinks not supported:", err)
}
if _, err := s.Create(ctx, "Music", parent); err != nil {
t.Fatal(err)
}
if _, err := s.Create(ctx, "Shortcut", link); !errors.Is(err, ErrInvalidLibrary) {
t.Errorf("a symlink into an existing library: err = %v", err)
}
}
func TestUpdateLibrary(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b"))
inside := mkdir(t, filepath.Join(a, "inside"))
first, _ := s.Create(ctx, "First", a)
second, _ := s.Create(ctx, "Second", b)
str := func(v string) *string { return &v }
// Rename only.
got, err := s.Update(ctx, first.ID, str("Renamed"), nil)
if err != nil || got.Name != "Renamed" || got.Path != a {
t.Fatalf("rename: %+v err=%v", got, err)
}
// Moving to its own current path is not an overlap with itself.
if _, err := s.Update(ctx, first.ID, nil, str(a)); err != nil {
t.Errorf("same path: %v", err)
}
// A new path in an unrelated place.
c := mkdir(t, filepath.Join(base, "c"))
if got, err = s.Update(ctx, first.ID, nil, str(c)); err != nil || got.Path != c {
t.Fatalf("move: %+v err=%v", got, err)
}
// Into another library, or onto its name or path.
if _, err := s.Update(ctx, first.ID, nil, str(b)); !errors.Is(err, ErrLibraryExists) {
t.Errorf("onto another library's path: err = %v", err)
}
if _, err := s.Update(ctx, second.ID, str("renamed"), nil); !errors.Is(err, ErrLibraryExists) {
t.Errorf("onto another library's name: err = %v", err)
}
if _, err := s.Update(ctx, first.ID, nil, str(filepath.Join(b, "deeper"))); !errors.Is(err, ErrInvalidLibrary) {
t.Errorf("a missing directory: err = %v", err)
}
_ = inside
if _, err := s.Update(ctx, first.ID, str(""), nil); !errors.Is(err, ErrInvalidLibrary) {
t.Errorf("empty name: err = %v", err)
}
if _, err := s.Update(ctx, 999, str("x"), nil); !errors.Is(err, ErrLibraryNotFound) {
t.Errorf("unknown id: err = %v", err)
}
}
func TestDeleteLibrary(t *testing.T) {
s, st := newLibraries()
ctx := context.Background()
lib, _ := s.Create(ctx, "Music", t.TempDir())
if err := s.Delete(ctx, lib.ID); err != nil {
t.Fatal(err)
}
if len(st.libs) != 0 {
t.Error("library not deleted")
}
if err := s.Delete(ctx, lib.ID); !errors.Is(err, ErrLibraryNotFound) {
t.Errorf("deleting twice: err = %v", err)
}
}
func TestListLibraries(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
_, _ = s.Create(ctx, "A", mkdir(t, filepath.Join(base, "a")))
_, _ = s.Create(ctx, "B", mkdir(t, filepath.Join(base, "b")))
libs, err := s.List(ctx)
if err != nil || len(libs) != 2 {
t.Fatalf("libs = %+v err=%v", libs, err)
}
}
// Names must clash on accents and case the same way on every database.
func TestLibraryNamesClashCaseInsensitivelyOnUnicode(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
if _, err := s.Create(ctx, "Música", mkdir(t, filepath.Join(base, "a"))); err != nil {
t.Fatal(err)
}
if _, err := s.Create(ctx, "MÚSICA", mkdir(t, filepath.Join(base, "b"))); !errors.Is(err, ErrLibraryExists) {
t.Fatalf("err = %v, want ErrLibraryExists", err)
}
}
// Two admins creating libraries at the same moment must not both pass the
// overlap check.
func TestConcurrentCreatesCannotOverlap(t *testing.T) {
for round := 0; round < 5; round++ {
s, st := newLibraries()
st.delay = 5 * time.Millisecond
parent := mkdir(t, filepath.Join(t.TempDir(), "music"))
child := mkdir(t, filepath.Join(parent, "a"))
var wg sync.WaitGroup
for i, p := range []string{parent, child} {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = s.Create(context.Background(), "lib"+string(rune('a'+i)), p)
}()
}
wg.Wait()
if len(st.libs) != 1 {
t.Fatalf("round %d: %d libraries created, want 1 (the two paths overlap)", round, len(st.libs))
}
}
}
// A migrated library is named after its path, which can be longer than the
// name limit. Changing only the path must still work.
func TestUpdatingThePathDoesNotRevalidateAnUntouchedName(t *testing.T) {
s, st := newLibraries()
ctx := context.Background()
base := t.TempDir()
a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b"))
long := strings.Repeat("x", 150)
st.libs = []Library{{ID: 1, Name: long, Path: a}}
st.next = 1
got, err := s.Update(ctx, 1, nil, &b)
if err != nil || got.Path != b || got.Name != long {
t.Fatalf("path-only update: %+v err=%v", got, err)
}
short := "short"
if _, err := s.Update(ctx, 1, &short, nil); err != nil {
t.Fatalf("renaming to a valid name: %v", err)
}
tooLong := strings.Repeat("y", 101)
if _, err := s.Update(ctx, 1, &tooLong, nil); !errors.Is(err, ErrInvalidLibrary) {
t.Fatalf("a new name is still validated: err = %v", err)
}
}
// Changing a library's directory or deleting it while a scan runs would leave
// the scan writing under a path that no longer matches.
func TestLibraryPathChangeAndDeleteAreRefusedWhileScanning(t *testing.T) {
s, _ := newLibraries()
ctx := context.Background()
base := t.TempDir()
a, b := mkdir(t, filepath.Join(base, "a")), mkdir(t, filepath.Join(base, "b"))
lib, _ := s.Create(ctx, "Music", a)
scanning := true
s.SetBusy(func() bool { return scanning })
if _, err := s.Update(ctx, lib.ID, nil, &b); !errors.Is(err, ErrScanRunning) {
t.Errorf("moving during a scan: err = %v, want ErrScanRunning", err)
}
if err := s.Delete(ctx, lib.ID); !errors.Is(err, ErrScanRunning) {
t.Errorf("deleting during a scan: err = %v, want ErrScanRunning", err)
}
// A rename, or "moving" to the same path, changes nothing under the scan.
name := "Renamed"
if _, err := s.Update(ctx, lib.ID, &name, nil); err != nil {
t.Errorf("renaming during a scan: %v", err)
}
if _, err := s.Update(ctx, lib.ID, nil, &a); err != nil {
t.Errorf("same path during a scan: %v", err)
}
scanning = false
if _, err := s.Update(ctx, lib.ID, nil, &b); err != nil {
t.Errorf("moving after the scan: %v", err)
}
if err := s.Delete(ctx, lib.ID); err != nil {
t.Errorf("deleting after the scan: %v", err)
}
}
func TestProblemsReportsOverlapAndMissingDirectories(t *testing.T) {
s, st := newLibraries()
ctx := context.Background()
base := t.TempDir()
parent := mkdir(t, filepath.Join(base, "music"))
child := mkdir(t, filepath.Join(parent, "flac"))
gone := filepath.Join(base, "gone")
fine := mkdir(t, filepath.Join(base, "fine"))
// Rows as a migration could leave them, bypassing validation.
st.libs = []Library{
{ID: 1, Name: "music", Path: parent},
{ID: 2, Name: "flac", Path: child},
{ID: 3, Name: "gone", Path: gone},
{ID: 4, Name: "fine", Path: fine},
}
problems, err := s.Problems(ctx)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(problems, "\n")
for _, want := range []string{`"gone"`, `"music"`, `"flac"`} {
if !strings.Contains(joined, want) {
t.Errorf("problems %q do not mention %s", joined, want)
}
}
if strings.Contains(joined, `"fine"`) {
t.Errorf("a healthy library was reported: %q", joined)
}
st.libs = st.libs[3:]
if problems, _ := s.Problems(ctx); len(problems) != 0 {
t.Errorf("a healthy setup reported problems: %v", problems)
}
}

View file

@ -3,12 +3,22 @@ package library
import (
"context"
"errors"
"fmt"
"sync"
"time"
)
// ErrScanRunning is returned by Start while a scan is in progress.
var ErrScanRunning = errors.New("a scan is already running")
// Errors returned when a scan cannot start.
var (
ErrScanRunning = errors.New("a scan is already running")
ErrNoLibraries = errors.New("there are no libraries to scan")
)
// LibrarySource lists the libraries a scan can cover. LibraryStore satisfies it.
type LibrarySource interface {
ListLibraries(ctx context.Context) ([]Library, error)
GetLibrary(ctx context.Context, id int64) (Library, error)
}
// Status describes the scan state for the API.
type Status struct {
@ -19,16 +29,21 @@ type Status struct {
LastError string
}
// Manager runs at most one background scan of the configured roots.
// Manager runs at most one background scan at a time.
type Manager struct {
scanner *Scanner
roots []string
scanner *Scanner
libraries LibrarySource
mu sync.Mutex
status Status
onFinish func(Summary, error)
}
// NewManager builds a Manager that scans libraries from the given source.
func NewManager(scanner *Scanner, libraries LibrarySource) *Manager {
return &Manager{scanner: scanner, libraries: libraries}
}
// OnFinish registers a function called once after every scan, with the result
// of that scan. Unlike hub events it is never dropped. Set it before Start.
// A new scan may start while a callback is still running, so callbacks of
@ -39,14 +54,31 @@ func (m *Manager) OnFinish(fn func(Summary, error)) {
m.onFinish = fn
}
// NewManager builds a Manager for the given roots.
func NewManager(scanner *Scanner, roots []string) *Manager {
return &Manager{scanner: scanner, roots: roots}
// Start scans every library in the background. ctx bounds the scan, so pass a
// context that lives as long as the application, not an HTTP request.
// It returns ErrNoLibraries when none are defined.
func (m *Manager) Start(ctx context.Context) error {
libs, err := m.libraries.ListLibraries(ctx)
if err != nil {
return fmt.Errorf("list libraries: %w", err)
}
if len(libs) == 0 {
return ErrNoLibraries
}
return m.start(ctx, libs)
}
// Start launches a scan in the background. ctx bounds the scan, so pass a
// context that lives as long as the application, not an HTTP request.
func (m *Manager) Start(ctx context.Context) error {
// StartLibrary scans a single library in the background.
// It returns ErrLibraryNotFound for an unknown id.
func (m *Manager) StartLibrary(ctx context.Context, id int64) error {
lib, err := m.libraries.GetLibrary(ctx, id)
if err != nil {
return err
}
return m.start(ctx, []Library{lib})
}
func (m *Manager) start(ctx context.Context, libs []Library) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.status.Running {
@ -54,12 +86,12 @@ func (m *Manager) Start(ctx context.Context) error {
}
m.status.Running = true
m.status.StartedAt = time.Now()
go m.run(ctx)
go m.run(ctx, libs)
return nil
}
func (m *Manager) run(ctx context.Context) {
sum, err := m.scanner.Scan(ctx, m.roots)
func (m *Manager) run(ctx context.Context, libs []Library) {
sum, err := m.scanner.Scan(ctx, libs)
m.mu.Lock()
m.status.Running = false

View file

@ -14,9 +14,9 @@ import (
var audioExts = map[string]bool{".mp3": true, ".flac": true}
// Scanner indexes the audio files under a set of roots.
// Scanner indexes the audio files of a set of libraries.
//
// Per root it walks the tree (cheap, stat only), decides what changed, then
// Per library it walks the tree (cheap, stat only), decides what changed, then
// reads tags with a pool of workers. A single goroutine writes to the store.
type Scanner struct {
store Store
@ -34,18 +34,18 @@ func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner {
return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now}
}
// Scan indexes every root in turn. An unreadable root is reported in the
// returned error and the other roots are still scanned.
func (s *Scanner) Scan(ctx context.Context, roots []string) (Summary, error) {
total := Summary{Roots: len(roots)}
// Scan indexes every library in turn. An unreadable library is reported in the
// returned error and the other libraries are still scanned.
func (s *Scanner) Scan(ctx context.Context, libs []Library) (Summary, error) {
total := Summary{Libraries: len(libs)}
s.publish(Event{Type: ScanStarted})
var errs []error
for _, root := range roots {
sum, err := s.scanRoot(ctx, root)
for _, lib := range libs {
sum, err := s.scanLibrary(ctx, lib)
total.add(sum)
if err != nil {
errs = append(errs, fmt.Errorf("%s: %w", root, err))
errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err))
if ctx.Err() != nil {
break
}
@ -77,10 +77,10 @@ type parseResult struct {
err error
}
func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) {
func (s *Scanner) scanLibrary(ctx context.Context, lib Library) (Summary, error) {
var sum Summary
walked, err := s.walk(ctx, root, &sum)
walked, err := s.walk(ctx, lib, &sum)
if err != nil {
// The root itself is unreadable (for example an unmounted NAS).
// Do not touch the database: nothing is known about its files.
@ -88,23 +88,23 @@ func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) {
}
sum.Found = len(walked.files)
existing, err := s.store.ExistingFiles(ctx, root)
existing, err := s.store.ExistingFiles(ctx, lib.ID)
if err != nil {
return sum, fmt.Errorf("list indexed files: %w", err)
}
// An unmounted NAS usually leaves an empty but readable folder. Treat
// "nothing found although files are indexed" as a problem, not as a mass
// deletion. Remove the root from the config to really empty a library.
// deletion. Delete the library to really empty it.
if len(walked.files) == 0 && len(walked.failedDirs) == 0 {
if n := countPresent(existing); n > 0 {
return sum, fmt.Errorf("root is empty but %d indexed files exist: is the volume mounted? "+
return sum, fmt.Errorf("directory is empty but %d indexed files exist: is the volume mounted? "+
"refusing to mark them missing", n)
}
}
p := makePlan(walked, existing)
sum.Unchanged = p.unchanged
if err := s.parseAndSave(ctx, root, p.parse, &sum); err != nil {
if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil {
return sum, err
}
if err := ctx.Err(); err != nil {
@ -113,7 +113,7 @@ func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) {
for _, r := range p.renames {
if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil {
s.fail(root, r.newRel, err, &sum)
s.fail(lib, r.newRel, err, &sum)
continue
}
sum.Renamed++
@ -133,13 +133,13 @@ func (s *Scanner) scanRoot(ctx context.Context, root string) (Summary, error) {
return sum, nil
}
func (s *Scanner) fail(root, rel string, err error, sum *Summary) {
func (s *Scanner) fail(lib Library, rel string, err error, sum *Summary) {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error()})
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error()})
}
// parseAndSave reads tags in parallel and stores results from this goroutine.
func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob, sum *Summary) error {
func (s *Scanner) parseAndSave(ctx context.Context, lib Library, jobs []parseJob, sum *Summary) error {
if len(jobs) == 0 {
return nil
}
@ -153,7 +153,7 @@ func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob
go func() {
defer wg.Done()
for j := range jobCh {
meta, err := s.reader.Read(filepath.Join(root, filepath.FromSlash(j.file.RelPath)))
meta, err := s.reader.Read(filepath.Join(lib.Path, filepath.FromSlash(j.file.RelPath)))
resCh <- parseResult{job: j, meta: meta, err: err}
}
}()
@ -182,13 +182,13 @@ func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob
rel := r.job.file.RelPath
if r.err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
continue
}
added, err := s.store.SaveFile(ctx, root, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
added, err := s.store.SaveFile(ctx, lib.ID, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
if err != nil {
sum.Failed++
s.publish(Event{Type: FileFailed, Root: root, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
continue
}
if added {
@ -196,7 +196,7 @@ func (s *Scanner) parseAndSave(ctx context.Context, root string, jobs []parseJob
} else {
sum.Updated++
}
s.publish(Event{Type: FileScanned, Root: root, RelPath: rel, Done: done, Total: len(jobs)})
s.publish(Event{Type: FileScanned, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Done: done, Total: len(jobs)})
}
return ctx.Err()
}
@ -213,13 +213,13 @@ type walkResult struct {
failedFiles map[string]bool
}
func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResult, error) {
func (s *Scanner) walk(ctx context.Context, lib Library, sum *Summary) (walkResult, error) {
var out walkResult
// The configured root may itself be a symlink (for example /srv/music ->
// The library path may itself be a symlink (for example /srv/music ->
// /mnt/nas/music). WalkDir would treat it as a plain file and find nothing,
// so walk the resolved directory. Symlinks *inside* the root are still skipped.
walkRoot, err := filepath.EvalSymlinks(root)
walkRoot, err := filepath.EvalSymlinks(lib.Path)
if err != nil {
return out, err
}
@ -228,7 +228,7 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
return out, err
}
if !info.IsDir() {
return out, fmt.Errorf("%s is not a directory", root)
return out, fmt.Errorf("%s is not a directory", lib.Path)
}
err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error {
@ -245,7 +245,7 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
rel = filepath.ToSlash(rel)
if err != nil {
s.fail(root, rel, err, sum)
s.fail(lib, rel, err, sum)
if d != nil && d.IsDir() {
out.failedDirs = append(out.failedDirs, rel+"/")
} else {
@ -267,7 +267,7 @@ func (s *Scanner) walk(ctx context.Context, root string, sum *Summary) (walkResu
}
info, ierr := d.Info()
if ierr != nil {
s.fail(root, rel, ierr, sum)
s.fail(lib, rel, ierr, sum)
if !errors.Is(ierr, fs.ErrNotExist) {
out.markFileFailed(rel)
}

View file

@ -21,11 +21,11 @@ type fakeRow struct {
type fakeStore struct {
mu sync.Mutex
nextID int64
rows map[string]map[string]*fakeRow // root -> rel path -> row
rows map[int64]map[string]*fakeRow // library id -> rel path -> row
saves int
}
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]map[string]*fakeRow{}} }
func newFakeStore() *fakeStore { return &fakeStore{rows: map[int64]map[string]*fakeRow{}} }
func (s *fakeStore) byID(id int64) *fakeRow {
for _, files := range s.rows {
@ -38,17 +38,17 @@ func (s *fakeStore) byID(id int64) *fakeRow {
return nil
}
func (s *fakeStore) ExistingFiles(_ context.Context, root string) ([]ExistingFile, error) {
func (s *fakeStore) ExistingFiles(_ context.Context, libraryID int64) ([]ExistingFile, error) {
s.mu.Lock()
defer s.mu.Unlock()
var out []ExistingFile
for _, r := range s.rows[root] {
for _, r := range s.rows[libraryID] {
out = append(out, r.ExistingFile)
}
return out, nil
}
func (s *fakeStore) SaveFile(_ context.Context, root string, f ParsedFile, existingID int64) (bool, error) {
func (s *fakeStore) SaveFile(_ context.Context, libraryID int64, f ParsedFile, existingID int64) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.saves++
@ -57,11 +57,11 @@ func (s *fakeStore) SaveFile(_ context.Context, root string, f ParsedFile, exist
r.Size, r.MTimeUnix, r.Missing, r.Parsed = f.Size, f.MTimeUnix, false, f
return false, nil
}
if s.rows[root] == nil {
s.rows[root] = map[string]*fakeRow{}
if s.rows[libraryID] == nil {
s.rows[libraryID] = map[string]*fakeRow{}
}
s.nextID++
s.rows[root][f.RelPath] = &fakeRow{
s.rows[libraryID][f.RelPath] = &fakeRow{
ExistingFile: ExistingFile{ID: s.nextID, FileInfo: f.FileInfo},
Parsed: f,
}
@ -102,10 +102,52 @@ func (s *fakeStore) Restore(_ context.Context, ids []int64) error {
return nil
}
// row looks a file up by the library path used in the test.
func (s *fakeStore) row(root, rel string) *fakeRow {
s.mu.Lock()
defer s.mu.Unlock()
return s.rows[root][rel]
return s.rows[libOf(root).ID][rel]
}
// ---- libraries ----
var (
libMu sync.Mutex
libIDs = map[string]int64{}
)
// libOf gives every directory used in a test a stable library.
func libOf(root string) Library {
libMu.Lock()
defer libMu.Unlock()
id, ok := libIDs[root]
if !ok {
id = int64(len(libIDs) + 1)
libIDs[root] = id
}
return Library{ID: id, Name: filepath.Base(root), Path: root}
}
func libsOf(roots ...string) []Library {
out := make([]Library, len(roots))
for i, r := range roots {
out[i] = libOf(r)
}
return out
}
// fakeSource is a LibrarySource over a fixed list.
type fakeSource struct{ libs []Library }
func (f fakeSource) ListLibraries(context.Context) ([]Library, error) { return f.libs, nil }
func (f fakeSource) GetLibrary(_ context.Context, id int64) (Library, error) {
for _, l := range f.libs {
if l.ID == id {
return l, nil
}
}
return Library{}, ErrLibraryNotFound
}
// fakeReader returns metadata derived from the file name and counts calls.
@ -158,7 +200,7 @@ func newScanner(store Store, reader TagReader, workers int) (*Scanner, *Hub) {
func mustScan(t *testing.T, s *Scanner, roots ...string) Summary {
t.Helper()
sum, err := s.Scan(context.Background(), roots)
sum, err := s.Scan(context.Background(), libsOf(roots...))
if err != nil {
t.Fatal(err)
}
@ -366,7 +408,7 @@ func TestUnreadableRootDoesNotMarkFilesMissing(t *testing.T) {
if err := os.RemoveAll(root); err != nil {
t.Fatal(err)
}
if _, err := s.Scan(context.Background(), []string{root}); err == nil {
if _, err := s.Scan(context.Background(), libsOf(root)); err == nil {
t.Fatal("expected an error for an unreadable root")
}
if store.row(root, "a.mp3").Missing {
@ -408,7 +450,7 @@ func TestSymlinkRootToAFileIsRejected(t *testing.T) {
t.Skip("symlinks not supported:", err)
}
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
if _, err := s.Scan(context.Background(), []string{link}); err == nil {
if _, err := s.Scan(context.Background(), libsOf(link)); err == nil {
t.Fatal("a root that is not a directory must be an error")
}
}
@ -425,7 +467,7 @@ func TestEmptyRootDoesNotMarkIndexedFilesMissing(t *testing.T) {
// An unmounted NAS typically leaves an empty, perfectly readable folder.
_ = os.Remove(filepath.Join(root, "a.mp3"))
_ = os.Remove(filepath.Join(root, "b.mp3"))
sum, err := s.Scan(context.Background(), []string{root})
sum, err := s.Scan(context.Background(), libsOf(root))
if err == nil {
t.Fatal("expected an error: root is empty but files are indexed")
}
@ -436,7 +478,7 @@ func TestEmptyRootDoesNotMarkIndexedFilesMissing(t *testing.T) {
func TestEmptyRootWithNothingIndexedIsFine(t *testing.T) {
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
sum, err := s.Scan(context.Background(), []string{t.TempDir()})
sum, err := s.Scan(context.Background(), libsOf(t.TempDir()))
if err != nil || sum.Found != 0 {
t.Fatalf("fresh empty library: %+v err=%v", sum, err)
}
@ -531,7 +573,7 @@ func TestCancelledScanDoesNotMarkMissing(t *testing.T) {
_ = os.Remove(filepath.Join(root, "a.mp3"))
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.Scan(ctx, []string{root}); !errors.Is(err, context.Canceled) {
if _, err := s.Scan(ctx, libsOf(root)); !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
if store.row(root, "a.mp3").Missing {
@ -580,7 +622,7 @@ func TestMultipleRootsAreIndependent(t *testing.T) {
s, _ := newScanner(store, &fakeReader{}, 2)
sum := mustScan(t, s, a, b)
if sum.Added != 2 || sum.Roots != 2 {
if sum.Added != 2 || sum.Libraries != 2 {
t.Fatalf("summary = %+v", sum)
}
if store.row(a, "same/name.mp3") == nil || store.row(b, "same/name.mp3") == nil {
@ -595,7 +637,7 @@ func TestOneBadRootDoesNotStopTheOthers(t *testing.T) {
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
sum, err := s.Scan(context.Background(), []string{bad, good})
sum, err := s.Scan(context.Background(), libsOf(bad, good))
if err == nil {
t.Fatal("expected an error mentioning the bad root")
}
@ -651,7 +693,7 @@ func TestManagerRunsOneScanAtATime(t *testing.T) {
write(t, filepath.Join(root, "a.mp3"), "1")
reader := &fakeReader{delay: 100 * time.Millisecond}
s, _ := newScanner(newFakeStore(), reader, 1)
m := NewManager(s, []string{root})
m := NewManager(s, fakeSource{libs: libsOf(root)})
if err := m.Start(context.Background()); err != nil {
t.Fatal(err)
@ -680,7 +722,7 @@ func TestManagerCallsOnFinishOnce(t *testing.T) {
root := t.TempDir()
write(t, filepath.Join(root, "a.mp3"), "1")
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
m := NewManager(s, []string{root})
m := NewManager(s, fakeSource{libs: libsOf(root)})
type finished struct {
sum Summary
@ -706,3 +748,48 @@ func TestManagerCallsOnFinishOnce(t *testing.T) {
case <-time.After(100 * time.Millisecond):
}
}
func TestManagerWithoutLibraries(t *testing.T) {
s, _ := newScanner(newFakeStore(), &fakeReader{}, 1)
m := NewManager(s, fakeSource{})
if err := m.Start(context.Background()); !errors.Is(err, ErrNoLibraries) {
t.Fatalf("err = %v, want ErrNoLibraries", err)
}
if m.Status().Running {
t.Error("a scan that could not start must not look running")
}
}
func TestManagerScansOneLibrary(t *testing.T) {
a, b := t.TempDir(), t.TempDir()
write(t, filepath.Join(a, "a.mp3"), "1")
write(t, filepath.Join(b, "b.mp3"), "22")
store := newFakeStore()
s, _ := newScanner(store, &fakeReader{}, 1)
m := NewManager(s, fakeSource{libs: libsOf(a, b)})
if err := m.StartLibrary(context.Background(), 999); !errors.Is(err, ErrLibraryNotFound) {
t.Fatalf("unknown library: err = %v", err)
}
if m.Status().Running {
t.Fatal("an unknown library must not start a scan")
}
if err := m.StartLibrary(context.Background(), libOf(a).ID); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(3 * time.Second)
for m.Status().Running && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if store.row(a, "a.mp3") == nil {
t.Error("the requested library was not scanned")
}
if store.row(b, "b.mp3") != nil {
t.Error("another library was scanned too")
}
if st := m.Status(); st.Last == nil || st.Last.Libraries != 1 {
t.Errorf("status = %+v", st)
}
}

View file

@ -27,7 +27,7 @@ func TestSpecialFilesAreSkipped(t *testing.T) {
}
done := make(chan result, 1)
go func() {
sum, err := s.Scan(context.Background(), []string{root})
sum, err := s.Scan(context.Background(), libsOf(root))
done <- result{sum, err}
}()

View file

@ -65,10 +65,10 @@ type ParsedFile struct {
// Store is what the scanner needs from persistence. All writes come from a
// single goroutine, so implementations need no extra locking for the scanner.
type Store interface {
ExistingFiles(ctx context.Context, root string) ([]ExistingFile, error)
ExistingFiles(ctx context.Context, libraryID int64) ([]ExistingFile, error)
// SaveFile creates the file (existingID == 0) or updates it, together with
// its track, album and artists. It reports whether a new file was created.
SaveFile(ctx context.Context, root string, f ParsedFile, existingID int64) (added bool, err error)
SaveFile(ctx context.Context, libraryID int64, f ParsedFile, existingID int64) (added bool, err error)
// RenameFile points an indexed file at a new relative path and clears its missing flag.
RenameFile(ctx context.Context, id int64, newRelPath string) error
// MarkMissing flags files that were not found. Rows are never deleted.
@ -89,21 +89,22 @@ const (
)
// Event is one progress notification. Done and Total count files being
// parsed in the current root.
// parsed in the current library.
type Event struct {
Type EventType
Time time.Time
Root string
RelPath string
Error string
Done int
Total int
Summary *Summary // only on ScanFinished
Type EventType
Time time.Time
LibraryID int64
Library string // library name
RelPath string
Error string
Done int
Total int
Summary *Summary // only on ScanFinished
}
// Summary counts what a scan did.
type Summary struct {
Roots int
Libraries int
Found int // audio files seen on disk
Added int
Updated int

View file

@ -55,7 +55,7 @@ type audioFileModel struct {
ID int64 `bun:"id,pk,autoincrement"`
TrackID int64 `bun:"track_id,notnull"`
Root string `bun:"root,notnull"`
LibraryID int64 `bun:"library_id,notnull"`
RelPath string `bun:"rel_path,notnull"`
Format string `bun:"format,notnull"`
SizeBytes int64 `bun:"size_bytes,notnull"`
@ -65,16 +65,30 @@ type audioFileModel struct {
MissingAt sql.NullInt64 `bun:"missing_at"`
}
// LibraryRepo stores the scanned catalog. It implements library.Store.
type libraryModel struct {
bun.BaseModel `bun:"table:libraries"`
ID int64 `bun:"id,pk,autoincrement"`
Name string `bun:"name,notnull"`
Path string `bun:"path,notnull"`
CreatedAt int64 `bun:"created_at,notnull"`
}
func (m libraryModel) toLibrary() library.Library {
return library.Library{ID: m.ID, Name: m.Name, Path: m.Path, CreatedAt: time.Unix(m.CreatedAt, 0)}
}
// LibraryRepo stores the scanned catalog and the libraries themselves. It
// implements library.Store, library.LibraryStore and streaming.Source.
type LibraryRepo struct{ db *bun.DB }
// NewLibraryRepo builds a LibraryRepo.
func NewLibraryRepo(db *bun.DB) *LibraryRepo { return &LibraryRepo{db: db} }
// ExistingFiles lists the files indexed under root, including missing ones.
func (r *LibraryRepo) ExistingFiles(ctx context.Context, root string) ([]library.ExistingFile, error) {
// ExistingFiles lists the files indexed in a library, including missing ones.
func (r *LibraryRepo) ExistingFiles(ctx context.Context, libraryID int64) ([]library.ExistingFile, error) {
var rows []audioFileModel
if err := r.db.NewSelect().Model(&rows).Where("root = ?", root).Scan(ctx); err != nil {
if err := r.db.NewSelect().Model(&rows).Where("library_id = ?", libraryID).Scan(ctx); err != nil {
return nil, fmt.Errorf("select audio files: %w", err)
}
out := make([]library.ExistingFile, len(rows))
@ -90,7 +104,7 @@ func (r *LibraryRepo) ExistingFiles(ctx context.Context, root string) ([]library
// SaveFile creates or updates one file with its track, album and artists in a
// single transaction. MusicBrainz ids are never invented: they stay NULL.
func (r *LibraryRepo) SaveFile(ctx context.Context, root string, f library.ParsedFile, existingID int64) (bool, error) {
func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.ParsedFile, existingID int64) (bool, error) {
var added bool
err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
artistIDs := make([]int64, 0, len(f.Artists))
@ -125,7 +139,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, root string, f library.Parse
if _, err := tx.NewInsert().Model(track).Exec(ctx); err != nil {
return fmt.Errorf("insert track: %w", err)
}
file := newAudioFile(root, f, track.ID)
file := newAudioFile(libraryID, f, track.ID)
if _, err := tx.NewInsert().Model(file).Exec(ctx); err != nil {
return fmt.Errorf("insert audio file: %w", err)
}
@ -159,9 +173,9 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, root string, f library.Parse
return added, nil
}
func newAudioFile(root string, f library.ParsedFile, trackID int64) *audioFileModel {
func newAudioFile(libraryID int64, f library.ParsedFile, trackID int64) *audioFileModel {
return &audioFileModel{
TrackID: trackID, Root: root, RelPath: f.RelPath, Format: f.Format,
TrackID: trackID, LibraryID: libraryID, RelPath: f.RelPath, Format: f.Format,
SizeBytes: f.Size, MTimeUnix: f.MTimeUnix, SampleRate: f.SampleRate, BitRate: f.BitRate,
}
}
@ -207,21 +221,99 @@ func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error {
}
// PlayableFile returns the file to play for a track: the oldest one that was
// not flagged missing. It returns streaming.ErrNotFound when the track does not
// exist or has no such file. It implements streaming.Source.
// not flagged missing, with the path of its library. It returns
// streaming.ErrNotFound when the track does not exist or has no such file. It
// implements streaming.Source.
func (r *LibraryRepo) PlayableFile(ctx context.Context, trackID int64) (streaming.File, error) {
var row audioFileModel
err := r.db.NewSelect().Model(&row).
Where("track_id = ?", trackID).
Where("missing_at IS NULL").
OrderExpr("id").Limit(1).Scan(ctx)
var row struct {
Path string `bun:"path"`
RelPath string `bun:"rel_path"`
}
err := r.db.NewSelect().
TableExpr("audio_files AS f").
ColumnExpr("l.path AS path, f.rel_path AS rel_path").
Join("JOIN libraries AS l ON l.id = f.library_id").
Where("f.track_id = ?", trackID).
Where("f.missing_at IS NULL").
OrderExpr("f.id").Limit(1).
Scan(ctx, &row)
if errors.Is(err, sql.ErrNoRows) {
return streaming.File{}, streaming.ErrNotFound
}
if err != nil {
return streaming.File{}, fmt.Errorf("select playable file: %w", err)
}
return streaming.File{Root: row.Root, RelPath: row.RelPath}, nil
return streaming.File{Root: row.Path, RelPath: row.RelPath}, nil
}
// ---- libraries ----
// ListLibraries returns every library, oldest first.
func (r *LibraryRepo) ListLibraries(ctx context.Context) ([]library.Library, error) {
var rows []libraryModel
if err := r.db.NewSelect().Model(&rows).OrderExpr("id").Scan(ctx); err != nil {
return nil, fmt.Errorf("list libraries: %w", err)
}
out := make([]library.Library, len(rows))
for i, row := range rows {
out[i] = row.toLibrary()
}
return out, nil
}
// GetLibrary returns one library, or library.ErrLibraryNotFound.
func (r *LibraryRepo) GetLibrary(ctx context.Context, id int64) (library.Library, error) {
var row libraryModel
err := r.db.NewSelect().Model(&row).Where("id = ?", id).Limit(1).Scan(ctx)
if errors.Is(err, sql.ErrNoRows) {
return library.Library{}, library.ErrLibraryNotFound
}
if err != nil {
return library.Library{}, fmt.Errorf("select library: %w", err)
}
return row.toLibrary(), nil
}
// CreateLibrary adds a library. Names are unique case-insensitively, paths exactly.
func (r *LibraryRepo) CreateLibrary(ctx context.Context, name, path string) (library.Library, error) {
row := &libraryModel{Name: name, Path: path, CreatedAt: time.Now().Unix()}
if _, err := r.db.NewInsert().Model(row).Exec(ctx); err != nil {
if isUniqueViolation(err) {
return library.Library{}, library.ErrLibraryExists
}
return library.Library{}, fmt.Errorf("insert library: %w", err)
}
return row.toLibrary(), nil
}
// UpdateLibrary changes the name and path of a library. Its files keep their
// identity because they point at the library by id.
func (r *LibraryRepo) UpdateLibrary(ctx context.Context, id int64, name, path string) (library.Library, error) {
res, err := r.db.NewUpdate().Model((*libraryModel)(nil)).
Set("name = ?", name).Set("path = ?", path).Where("id = ?", id).Exec(ctx)
if err != nil {
if isUniqueViolation(err) {
return library.Library{}, library.ErrLibraryExists
}
return library.Library{}, fmt.Errorf("update library: %w", err)
}
if n, _ := res.RowsAffected(); n == 0 {
return library.Library{}, library.ErrLibraryNotFound
}
return r.GetLibrary(ctx, id)
}
// DeleteLibrary removes a library and, through the foreign key, its indexed
// files. Tracks stay, so listening history is kept. Music on disk is untouched.
func (r *LibraryRepo) DeleteLibrary(ctx context.Context, id int64) error {
res, err := r.db.NewDelete().Model((*libraryModel)(nil)).Where("id = ?", id).Exec(ctx)
if err != nil {
return fmt.Errorf("delete library: %w", err)
}
if n, _ := res.RowsAffected(); n == 0 {
return library.ErrLibraryNotFound
}
return nil
}
// RefreshMatchKeys recomputes name_key and title_key for every artist and album

View file

@ -31,6 +31,16 @@ func parsed(rel, title, album string, artists ...string) library.ParsedFile {
}
}
// mkLib creates a library row and returns its id.
func mkLib(t *testing.T, repo *LibraryRepo, name string) int64 {
t.Helper()
lib, err := repo.CreateLibrary(context.Background(), name, "/lib/"+name)
if err != nil {
t.Fatal(err)
}
return lib.ID
}
func count(t *testing.T, db *bun.DB, query string, args ...any) int {
t.Helper()
var n int
@ -44,13 +54,14 @@ func TestLibraryRepoSaveNewFile(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "music")
added, err := repo.SaveFile(ctx, "/music", parsed("a/1.flac", "Song", "Album", "Radiohead"), 0)
added, err := repo.SaveFile(ctx, lib, parsed("a/1.flac", "Song", "Album", "Radiohead"), 0)
if err != nil || !added {
t.Fatalf("added=%v err=%v", added, err)
}
// Same artist (different case) and same album are reused, not duplicated.
if _, err := repo.SaveFile(ctx, "/music", parsed("a/2.flac", "Other", "album", "RADIOHEAD"), 0); err != nil {
if _, err := repo.SaveFile(ctx, lib, parsed("a/2.flac", "Other", "album", "RADIOHEAD"), 0); err != nil {
t.Fatal(err)
}
@ -63,7 +74,7 @@ func TestLibraryRepoSaveNewFile(t *testing.T) {
if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 2 {
t.Errorf("tracks = %d, want 2", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE root = ?", "/music"); n != 2 {
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE library_id = ?", lib); n != 2 {
t.Errorf("audio_files = %d, want 2", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM track_artists"); n != 2 {
@ -87,16 +98,17 @@ func TestLibraryRepoMatchesUnicodeNamesCaseInsensitively(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
if _, err := repo.SaveFile(ctx, "/m", parsed("1.flac", "Hyperballad", "Post", "Björk"), 0); err != nil {
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Hyperballad", "Post", "Björk"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("2.flac", "Army of Me", "POST", "BJÖRK"), 0); err != nil {
if _, err := repo.SaveFile(ctx, lib, parsed("2.flac", "Army of Me", "POST", "BJÖRK"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("3.flac", "Isobel", "Ünïcode Album", "Åsa"), 0); err != nil {
if _, err := repo.SaveFile(ctx, lib, parsed("3.flac", "Isobel", "Ünïcode Album", "Åsa"), 0); err != nil {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("4.flac", "Other", "ÜNÏCODE ALBUM", "ÅSA"), 0); err != nil {
if _, err := repo.SaveFile(ctx, lib, parsed("4.flac", "Other", "ÜNÏCODE ALBUM", "ÅSA"), 0); err != nil {
t.Fatal(err)
}
@ -115,6 +127,7 @@ func TestLibraryRepoRefreshMatchKeysFixesOldRows(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "m")
// Simulate what the migration backfill leaves behind on SQLite.
if _, err := db.Exec("INSERT INTO artists (name, name_key) VALUES ('BJÖRK', 'bjÖrk')"); err != nil {
@ -131,7 +144,7 @@ func TestLibraryRepoRefreshMatchKeysFixesOldRows(t *testing.T) {
t.Fatal(err)
}
if _, err := repo.SaveFile(ctx, "/m", parsed("1.flac", "Hyperballad", "post", "Björk"), 0); err != nil {
if _, err := repo.SaveFile(ctx, lib, parsed("1.flac", "Hyperballad", "post", "Björk"), 0); err != nil {
t.Fatal(err)
}
if n := count(t, db, "SELECT COUNT(*) FROM artists"); n != 1 {
@ -146,7 +159,8 @@ func TestLibraryRepoRefreshMatchKeysFixesOldRows(t *testing.T) {
func TestLibraryRepoTrackWithoutAlbum(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
repo := NewLibraryRepo(db)
if _, err := repo.SaveFile(context.Background(), "/m", parsed("x.flac", "Loose", "", "Someone"), 0); err != nil {
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(context.Background(), lib, parsed("x.flac", "Loose", "", "Someone"), 0); err != nil {
t.Fatal(err)
}
if n := count(t, db, "SELECT COUNT(*) FROM albums"); n != 0 {
@ -162,10 +176,11 @@ func TestLibraryRepoUpdateKeepsIdentityAndReplacesCredits(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
if _, err := repo.SaveFile(ctx, "/m", parsed("s.flac", "Old", "Album", "Artist A"), 0); err != nil {
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(ctx, lib, parsed("s.flac", "Old", "Album", "Artist A"), 0); err != nil {
t.Fatal(err)
}
existing, err := repo.ExistingFiles(ctx, "/m")
existing, err := repo.ExistingFiles(ctx, lib)
if err != nil || len(existing) != 1 {
t.Fatalf("existing = %v err=%v", existing, err)
}
@ -173,7 +188,7 @@ func TestLibraryRepoUpdateKeepsIdentityAndReplacesCredits(t *testing.T) {
upd := parsed("s.flac", "New", "Album", "Artist B")
upd.Size, upd.MTimeUnix = 200, 1_800_000_000
added, err := repo.SaveFile(ctx, "/m", upd, id)
added, err := repo.SaveFile(ctx, lib, upd, id)
if err != nil || added {
t.Fatalf("update: added=%v err=%v", added, err)
}
@ -193,25 +208,26 @@ func TestLibraryRepoUpdateKeepsIdentityAndReplacesCredits(t *testing.T) {
if n := count(t, db, `SELECT COUNT(*) FROM track_artists ta JOIN artists a ON a.id = ta.artist_id WHERE a.name = 'Artist A'`); n != 0 {
t.Error("old credit still present")
}
after, _ := repo.ExistingFiles(ctx, "/m")
after, _ := repo.ExistingFiles(ctx, lib)
if after[0].ID != id || after[0].Size != 200 || after[0].MTimeUnix != 1_800_000_000 {
t.Errorf("after update = %+v", after[0])
}
})
}
func TestLibraryRepoSameRelPathUnderTwoRoots(t *testing.T) {
func TestLibraryRepoSameRelPathUnderTwoLibraries(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
for _, root := range []string{"/a", "/b"} {
if _, err := repo.SaveFile(ctx, root, parsed("same.flac", "S", "", "X"), 0); err != nil {
t.Fatalf("root %s: %v", root, err)
a, b := mkLib(t, repo, "a"), mkLib(t, repo, "b")
for _, lib := range []int64{a, b} {
if _, err := repo.SaveFile(ctx, lib, parsed("same.flac", "S", "", "X"), 0); err != nil {
t.Fatalf("library %d: %v", lib, err)
}
}
for _, root := range []string{"/a", "/b"} {
if ex, _ := repo.ExistingFiles(ctx, root); len(ex) != 1 {
t.Errorf("root %s has %d files, want 1", root, len(ex))
for _, lib := range []int64{a, b} {
if ex, _ := repo.ExistingFiles(ctx, lib); len(ex) != 1 {
t.Errorf("library %d has %d files, want 1", lib, len(ex))
}
}
})
@ -221,16 +237,17 @@ func TestLibraryRepoMissingRestoreRename(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
if _, err := repo.SaveFile(ctx, "/m", parsed("a.flac", "A", "", "X"), 0); err != nil {
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(ctx, lib, parsed("a.flac", "A", "", "X"), 0); err != nil {
t.Fatal(err)
}
ex, _ := repo.ExistingFiles(ctx, "/m")
ex, _ := repo.ExistingFiles(ctx, lib)
id := ex[0].ID
if err := repo.MarkMissing(ctx, []int64{id}, time.Unix(1_900_000_000, 0)); err != nil {
t.Fatal(err)
}
ex, _ = repo.ExistingFiles(ctx, "/m")
ex, _ = repo.ExistingFiles(ctx, lib)
if len(ex) != 1 || !ex[0].Missing {
t.Fatalf("after MarkMissing: %+v (row must be kept, flagged)", ex)
}
@ -241,7 +258,7 @@ func TestLibraryRepoMissingRestoreRename(t *testing.T) {
if err := repo.Restore(ctx, []int64{id}); err != nil {
t.Fatal(err)
}
if ex, _ = repo.ExistingFiles(ctx, "/m"); ex[0].Missing {
if ex, _ = repo.ExistingFiles(ctx, lib); ex[0].Missing {
t.Error("Restore did not clear the missing flag")
}
@ -249,7 +266,7 @@ func TestLibraryRepoMissingRestoreRename(t *testing.T) {
if err := repo.RenameFile(ctx, id, "sub/b.flac"); err != nil {
t.Fatal(err)
}
ex, _ = repo.ExistingFiles(ctx, "/m")
ex, _ = repo.ExistingFiles(ctx, lib)
if ex[0].ID != id || ex[0].RelPath != "sub/b.flac" || ex[0].Missing {
t.Errorf("after rename: %+v", ex[0])
}
@ -265,7 +282,8 @@ func TestLibraryRepoPlayableFile(t *testing.T) {
t.Fatalf("unknown track: err = %v, want ErrNotFound", err)
}
if _, err := repo.SaveFile(ctx, "/music", parsed("a/1.flac", "Song", "", "X"), 0); err != nil {
lib := mkLib(t, repo, "music") // path /lib/music
if _, err := repo.SaveFile(ctx, lib, parsed("a/1.flac", "Song", "", "X"), 0); err != nil {
t.Fatal(err)
}
var trackID, fileID int64
@ -274,13 +292,13 @@ func TestLibraryRepoPlayableFile(t *testing.T) {
}
f, err := repo.PlayableFile(ctx, trackID)
if err != nil || f.Root != "/music" || f.RelPath != "a/1.flac" {
if err != nil || f.Root != "/lib/music" || f.RelPath != "a/1.flac" {
t.Fatalf("PlayableFile = %+v err=%v", f, err)
}
// A second encoding of the same track.
if _, err := db.Exec(`INSERT INTO audio_files (track_id, root, rel_path, format, size_bytes, mtime_unix)
VALUES (?, '/music', 'a/1.mp3', 'mp3', 1, 1)`, trackID); err != nil {
if _, err := db.Exec(`INSERT INTO audio_files (track_id, library_id, rel_path, format, size_bytes, mtime_unix)
VALUES (?, ?, 'a/1.mp3', 'mp3', 1, 1)`, trackID, lib); err != nil {
t.Fatal(err)
}
// The first file goes missing: the other one is served.
@ -320,9 +338,14 @@ func TestScannerEndToEndOnBothBackends(t *testing.T) {
t.Fatal(err)
}
}
s := library.NewScanner(NewLibraryRepo(db), stubReader{}, library.NewHub(), 4)
repo := NewLibraryRepo(db)
lib, err := repo.CreateLibrary(context.Background(), "e2e", root)
if err != nil {
t.Fatal(err)
}
s := library.NewScanner(repo, stubReader{}, library.NewHub(), 4)
scan := func() library.Summary {
sum, err := s.Scan(context.Background(), []string{root})
sum, err := s.Scan(context.Background(), []library.Library{lib})
if err != nil {
t.Fatal(err)
}
@ -353,3 +376,123 @@ func TestScannerEndToEndOnBothBackends(t *testing.T) {
}
})
}
func TestLibraryRepoCRUD(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
if libs, err := repo.ListLibraries(ctx); err != nil || len(libs) != 0 {
t.Fatalf("empty list: %v err=%v", libs, err)
}
a, err := repo.CreateLibrary(ctx, "Jazz", "/music/jazz")
if err != nil || a.ID == 0 || a.Name != "Jazz" || a.Path != "/music/jazz" || a.CreatedAt.IsZero() {
t.Fatalf("create: %+v err=%v", a, err)
}
b, _ := repo.CreateLibrary(ctx, "Rock", "/music/rock")
libs, err := repo.ListLibraries(ctx)
if err != nil || len(libs) != 2 || libs[0].ID != a.ID || libs[1].ID != b.ID {
t.Fatalf("list: %+v err=%v", libs, err)
}
got, err := repo.GetLibrary(ctx, a.ID)
if err != nil || got.Name != "Jazz" {
t.Fatalf("get: %+v err=%v", got, err)
}
upd, err := repo.UpdateLibrary(ctx, a.ID, "Jazz Collection", "/mnt/jazz")
if err != nil || upd.Name != "Jazz Collection" || upd.Path != "/mnt/jazz" || upd.ID != a.ID {
t.Fatalf("update: %+v err=%v", upd, err)
}
})
}
func TestLibraryRepoRejectsDuplicatesAndUnknownIDs(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
a, _ := repo.CreateLibrary(ctx, "Jazz", "/music/jazz")
b, _ := repo.CreateLibrary(ctx, "Rock", "/music/rock")
if _, err := repo.CreateLibrary(ctx, "jazz", "/other"); !errors.Is(err, library.ErrLibraryExists) {
t.Errorf("same name in another case: %v", err)
}
if _, err := repo.CreateLibrary(ctx, "Other", "/music/jazz"); !errors.Is(err, library.ErrLibraryExists) {
t.Errorf("same path: %v", err)
}
if _, err := repo.UpdateLibrary(ctx, b.ID, "JAZZ", "/music/rock"); !errors.Is(err, library.ErrLibraryExists) {
t.Errorf("update onto another name: %v", err)
}
if _, err := repo.UpdateLibrary(ctx, b.ID, "Rock", "/music/jazz"); !errors.Is(err, library.ErrLibraryExists) {
t.Errorf("update onto another path: %v", err)
}
if _, err := repo.UpdateLibrary(ctx, a.ID, "Jazz", "/music/jazz"); err != nil {
t.Errorf("update to its own values: %v", err)
}
if _, err := repo.GetLibrary(ctx, 999); !errors.Is(err, library.ErrLibraryNotFound) {
t.Errorf("get: %v", err)
}
if _, err := repo.UpdateLibrary(ctx, 999, "x", "/x"); !errors.Is(err, library.ErrLibraryNotFound) {
t.Errorf("update: %v", err)
}
if err := repo.DeleteLibrary(ctx, 999); !errors.Is(err, library.ErrLibraryNotFound) {
t.Errorf("delete: %v", err)
}
})
}
// Moving a library's folder is a one-row change: its files keep their identity.
func TestLibraryRepoPathChangeKeepsFiles(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
lib := mkLib(t, repo, "m")
if _, err := repo.SaveFile(ctx, lib, parsed("a.flac", "A", "", "X"), 0); err != nil {
t.Fatal(err)
}
before, _ := repo.ExistingFiles(ctx, lib)
if _, err := repo.UpdateLibrary(ctx, lib, "m", "/moved/here"); err != nil {
t.Fatal(err)
}
after, _ := repo.ExistingFiles(ctx, lib)
if len(after) != 1 || after[0].ID != before[0].ID {
t.Fatalf("files after the move: %+v, want the same one", after)
}
var trackID int64
_ = db.QueryRow("SELECT track_id FROM audio_files").Scan(&trackID)
f, err := repo.PlayableFile(ctx, trackID)
if err != nil || f.Root != "/moved/here" {
t.Errorf("playable file after the move: %+v err=%v", f, err)
}
})
}
// Deleting a library drops its index entries; tracks (and with them the
// listening history) stay.
func TestLibraryRepoDeleteRemovesFilesButKeepsTracks(t *testing.T) {
forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) {
ctx := context.Background()
repo := NewLibraryRepo(db)
keep, drop := mkLib(t, repo, "keep"), mkLib(t, repo, "drop")
for _, lib := range []int64{keep, drop} {
if _, err := repo.SaveFile(ctx, lib, parsed("x.flac", "T", "", "X"), 0); err != nil {
t.Fatal(err)
}
}
if err := repo.DeleteLibrary(ctx, drop); err != nil {
t.Fatal(err)
}
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE library_id = ?", drop); n != 0 {
t.Errorf("files of the deleted library = %d, want 0", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE library_id = ?", keep); n != 1 {
t.Errorf("files of the other library = %d, want 1", n)
}
if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 2 {
t.Errorf("tracks = %d, want 2: history must survive", n)
}
})
}

Some files were not shown because too many files have changed in this diff Show more