feat: scan the libraries by themselves when their music changed

Every 15 minutes (library.auto_scan_minutes, SONDA_AUTO_SCAN_MINUTES, 0 turns it off)
Sonda takes the signature of each library, a hash of the names of everything in it folder
by folder, which costs only the names of the files and so suits a NAS that sleeps. A
library whose signature changed (a song or a folder came, went or was renamed) is
scanned the usual way; every library is also scanned at least once a day, to find a song
tagged again where it stands.

It does not scan while another scan runs, tries again at the next look when a scan failed,
leaves alone a folder that cannot be read (a NAS that is not mounted), and ignores the
housekeeping folders of a NAS.
This commit is contained in:
tgiachi 2026-11-16 13:05:09 +01:00
commit 89f1b5275a
4 changed files with 580 additions and 2 deletions

View file

@ -42,6 +42,9 @@ type Server struct {
// in the database and are managed through the API.
type Library struct {
ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan
// AutoScanMinutes is how often Sonda looks at the folders of the libraries and scans the ones whose music changed;
// 0 turns it off. Looking reads the names of the files and nothing else.
AutoScanMinutes int `toml:"auto_scan_minutes"`
// AllowDelete lets an admin delete the extra copies of a song from the music folders. It is the
// one thing that makes Sonda write there, so it is off until the operator turns it on (and it
// needs the folders mounted writable).
@ -142,7 +145,7 @@ type Storage struct {
func Default() Config {
return Config{
Server: Server{Listen: "127.0.0.1:8080"},
Library: Library{ScanWorkers: 4},
Library: Library{ScanWorkers: 4, AutoScanMinutes: 15},
Database: Database{Driver: "sqlite", Path: "sonda.db"},
Log: Log{Env: "production"},
Auth: Auth{AccessTTLMinutes: 15, RefreshTTLHours: 720}, // 15 minutes, 30 days
@ -185,6 +188,13 @@ func Load(path string) (Config, error) {
}
cfg.Library.AllowDelete = allow
}
if v := os.Getenv("SONDA_AUTO_SCAN_MINUTES"); v != "" {
n, err := strconv.Atoi(v)
if err != nil {
return Config{}, fmt.Errorf("SONDA_AUTO_SCAN_MINUTES must be a whole number of minutes, got %q", v)
}
cfg.Library.AutoScanMinutes = n
}
if cfg.Storage.DataDir == "" {
cfg.Storage.DataDir = defaultDataDir(cfg.Database)
}
@ -285,6 +295,9 @@ func (c Config) Validate() error {
if c.Storage.DataDir == "" {
errs = append(errs, errors.New("storage.data_dir must not be empty"))
}
if c.Library.AutoScanMinutes < 0 || c.Library.AutoScanMinutes > 10080 {
errs = append(errs, errors.New("library.auto_scan_minutes must be between 0 (off) and 10080 (a week)"))
}
if c.Library.ScanWorkers < 1 {
errs = append(errs, errors.New("library.scan_workers must be at least 1"))
}

View file

@ -37,6 +37,9 @@ func TestLoadEmptyPathReturnsDefaults(t *testing.T) {
if cfg.Library.ScanWorkers != 4 {
t.Errorf("scan workers = %d", cfg.Library.ScanWorkers)
}
if cfg.Library.AutoScanMinutes != 15 {
t.Errorf("auto scan = %d minutes, want 15", cfg.Library.AutoScanMinutes)
}
if cfg.Listening.MinListen() != 20*time.Second {
t.Errorf("min listen = %v, want 20s", cfg.Listening.MinListen())
}
@ -146,6 +149,9 @@ func TestValidate(t *testing.T) {
{"two wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"it", "en"} }, ""},
{"empty data dir", func(c *Config) { c.Storage.DataDir = "" }, "storage.data_dir"},
{"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"},
{"a negative auto scan", func(c *Config) { c.Library.AutoScanMinutes = -1 }, "library.auto_scan_minutes"},
{"an auto scan of a year", func(c *Config) { c.Library.AutoScanMinutes = 525600 }, "library.auto_scan_minutes"},
{"auto scan off", func(c *Config) { c.Library.AutoScanMinutes = 0 }, ""},
{"bad access ttl", func(c *Config) { c.Auth.AccessTTLMinutes = 0 }, "auth.access_ttl_minutes"},
{"bad refresh ttl", func(c *Config) { c.Auth.RefreshTTLHours = 0 }, "auth.refresh_ttl_hours"},
}
@ -341,3 +347,20 @@ func TestAllowDeleteEnvironmentVariableMustBeABoolean(t *testing.T) {
t.Errorf("got %v, want an error naming SONDA_ALLOW_DELETE", err)
}
}
func TestTheAutoScanComesFromTheFileOrTheEnvironment(t *testing.T) {
dir := t.TempDir()
p := writeFile(t, dir, "[library]\nauto_scan_minutes = 5\n")
cfg, err := Load(p)
if err != nil || cfg.Library.AutoScanMinutes != 5 {
t.Fatalf("file: %d %v", cfg.Library.AutoScanMinutes, err)
}
t.Setenv("SONDA_AUTO_SCAN_MINUTES", "0")
if cfg, err = Load(p); err != nil || cfg.Library.AutoScanMinutes != 0 {
t.Fatalf("environment over file: %d %v", cfg.Library.AutoScanMinutes, err)
}
t.Setenv("SONDA_AUTO_SCAN_MINUTES", "often")
if _, err = Load(""); err == nil {
t.Fatal("a value that is not a number was taken")
}
}

View file

@ -0,0 +1,199 @@
package library
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"time"
)
// forceEvery is how long a library may go without a scan even when its folders look the same: a song that is tagged
// again keeps its place and its name, so only a scan of the file finds it.
const forceEvery = 24 * time.Hour
// Signature says how a folder looks: a hash of the names of everything in it, folder by folder, and of the time of each
// folder. It reads the names and nothing else (no file is opened or measured), which is why it is cheap enough to
// ask often of a NAS, and it changes when a song or a folder comes, goes or is renamed. It does not change when a
// file is changed where it stands. The housekeeping folders of a NAS (see isSystemDir) and the links inside the folder
// are left out, as the scanner leaves them out; a link as the folder itself is followed. A folder that cannot be read is
// an error.
func Signature(ctx context.Context, root string) (string, error) {
top, err := filepath.EvalSymlinks(root)
if err != nil {
return "", err
}
info, err := os.Stat(top)
if err != nil {
return "", err
}
if !info.IsDir() {
return "", fmt.Errorf("%s is not a directory", root)
}
h := sha256.New()
var walk func(dir, rel string) error
walk = func(dir, rel string) error {
if err := ctx.Err(); err != nil {
return err
}
entries, err := os.ReadDir(dir)
if err != nil {
if rel == "" {
return err
}
fmt.Fprintf(h, "! %s\n", rel) // a folder that cannot be read is a state of its own
return nil
}
var mtime int64
if st, err := os.Stat(dir); err == nil {
mtime = st.ModTime().UnixNano()
}
fmt.Fprintf(h, "d %s %d\n", rel, mtime)
for _, e := range entries { // os.ReadDir sorts them by name
if e.IsDir() && isSystemDir(e.Name()) {
continue
}
kind := 'f'
switch {
case e.Type()&os.ModeSymlink != 0:
kind = 'l'
case e.IsDir():
kind = 'd'
}
fmt.Fprintf(h, "%c %s\n", kind, e.Name())
}
for _, e := range entries {
if e.IsDir() && e.Type()&os.ModeSymlink == 0 && !isSystemDir(e.Name()) {
if err := walk(filepath.Join(dir, e.Name()), rel+e.Name()+"/"); err != nil {
return err
}
}
}
return nil
}
if err := walk(top, ""); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// scans is what the watcher asks of the manager of the scans: *Manager.
type scans interface {
StartLibrary(ctx context.Context, id int64, opts ...ScanOption) error
Status() Status
}
// AutoScanner scans the libraries by itself when their music changed. Every `interval` it takes the Signature of each
// library and scans the ones that do not look as they did the last time (the first time, all of them), and every library
// at least once a day. A scan is the usual one: it reads only the files that are new or changed. Nothing is scanned
// while another scan runs, and a scan that failed is tried again at the next look.
type AutoScanner struct {
scans scans
libs LibrarySource
interval time.Duration
// first is how long after the start the first look comes: the server is not busy with it while it starts.
first time.Duration
now func() time.Time
poll time.Duration // how often it asks whether a scan it started is over
// OnScan, when set, is told about each library it scans and about each one it could not look at.
OnScan func(lib Library, signatureErr, scanErr error)
seen map[int64]seenLibrary
}
type seenLibrary struct {
sig string
at time.Time
}
// NewAutoScanner builds an AutoScanner that looks every interval.
func NewAutoScanner(s scans, libs LibrarySource, interval time.Duration) *AutoScanner {
return &AutoScanner{
scans: s, libs: libs, interval: interval, first: 30 * time.Second,
now: time.Now, poll: time.Second, seen: map[int64]seenLibrary{},
}
}
// Run looks at the libraries until ctx ends.
func (a *AutoScanner) Run(ctx context.Context) {
timer := time.NewTimer(a.first)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
a.tick(ctx)
timer.Reset(a.interval)
}
}
}
// tick looks at every library once.
func (a *AutoScanner) tick(ctx context.Context) {
libs, err := a.libs.ListLibraries(ctx)
if err != nil {
return
}
present := map[int64]bool{}
for _, l := range libs {
present[l.ID] = true
}
for id := range a.seen {
if !present[id] {
delete(a.seen, id) // a library that is gone: a new one with its number starts from nothing
}
}
for _, lib := range libs {
if ctx.Err() != nil {
return
}
sig, err := Signature(ctx, lib.Path)
if err != nil {
// Not there (a NAS that is not mounted): nothing is known about it, and the scan would only say so.
a.told(lib, err, nil)
continue
}
prev, ok := a.seen[lib.ID]
if ok && prev.sig == sig && a.now().Sub(prev.at) < forceEvery {
continue
}
switch err := a.scans.StartLibrary(ctx, lib.ID); {
case errors.Is(err, ErrScanRunning):
return // another scan has the turn: the rest wait for the next look
case err != nil:
a.told(lib, nil, err)
continue
}
scanErr := a.wait(ctx)
a.told(lib, nil, scanErr)
if scanErr == nil && ctx.Err() == nil {
// The signature is the one taken before the scan: what changed while it ran is seen at the next look.
a.seen[lib.ID] = seenLibrary{sig: sig, at: a.now()}
}
}
}
// wait holds until the scan that was started is over, and says how it ended.
func (a *AutoScanner) wait(ctx context.Context) error {
for a.scans.Status().Running {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(a.poll):
}
}
if msg := a.scans.Status().LastError; msg != "" {
return errors.New(msg)
}
return nil
}
func (a *AutoScanner) told(lib Library, sigErr, scanErr error) {
if a.OnScan != nil {
a.OnScan(lib, sigErr, scanErr)
}
}

View file

@ -0,0 +1,328 @@
package library
import (
"context"
"errors"
"os"
"path/filepath"
"sync"
"testing"
"time"
)
// ---- the signature of a folder ----
func tree(t *testing.T) string {
t.Helper()
root := t.TempDir()
for _, f := range []string{"a/one.mp3", "a/two.flac", "b/c/three.mp3"} {
touch(t, root, f)
}
settle(t, root)
return root
}
func touch(t *testing.T, root, rel string) {
t.Helper()
p := filepath.Join(root, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
// settle gives every folder of the tree a moment far in the past, so that a change made after it moves the time of the
// folder it touches, whatever the resolution of the file system.
func settle(t *testing.T, root string) {
t.Helper()
old := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) // always the same, so that two calls leave the same time
var dirs []string
_ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
if err == nil && d.IsDir() {
dirs = append(dirs, p)
}
return nil
})
for i := len(dirs) - 1; i >= 0; i-- {
if err := os.Chtimes(dirs[i], old, old); err != nil {
t.Fatal(err)
}
}
}
func sig(t *testing.T, root string) string {
t.Helper()
s, err := Signature(context.Background(), root)
if err != nil {
t.Fatal(err)
}
return s
}
func TestTheSignatureOfAFolderChangesWhenMusicComesOrGoes(t *testing.T) {
root := tree(t)
base := sig(t, root)
if sig(t, root) != base {
t.Fatal("the same folder, two signatures")
}
for name, change := range map[string]func(){
"a song comes": func() { touch(t, root, "a/new.mp3") },
"a song goes": func() { _ = os.Remove(filepath.Join(root, "a/one.mp3")) },
"a song is renamed": func() { _ = os.Rename(filepath.Join(root, "a/two.flac"), filepath.Join(root, "a/deux.flac")) },
"a folder comes": func() { touch(t, root, "d/new.mp3") },
"a folder goes": func() { _ = os.RemoveAll(filepath.Join(root, "b")) },
"something is dropped": func() { touch(t, root, "b/c/cover.jpg") },
"a deep folder changes": func() { touch(t, root, "b/c/more.flac") },
} {
fresh := tree(t)
before := sig(t, fresh)
root = fresh
change()
if sig(t, fresh) == before {
t.Errorf("%s: the signature did not change", name)
}
}
}
func TestTheSignatureLeavesOutTheHousekeepingFoldersOfANAS(t *testing.T) {
root := tree(t)
base := sig(t, root)
// A NAS writes thumbnails in @eaDir all the time: what happens inside it is not a change of the music.
if err := os.MkdirAll(filepath.Join(root, "a", "@eaDir"), 0o755); err != nil {
t.Fatal(err)
}
settle(t, root)
withDir := sig(t, root)
touch(t, root, "a/@eaDir/thumb.jpg")
touch(t, root, "a/@eaDir/deeper/thumb2.jpg")
settle(t, root)
if got := sig(t, root); got != withDir {
t.Errorf("a change inside @eaDir moved the signature")
}
_ = base
}
func TestTheSignatureOfAFolderThatCannotBeReadIsAnError(t *testing.T) {
if _, err := Signature(context.Background(), filepath.Join(t.TempDir(), "nothing")); err == nil {
t.Fatal("no error for a folder that is not there")
}
file := filepath.Join(t.TempDir(), "f")
_ = os.WriteFile(file, nil, 0o644)
if _, err := Signature(context.Background(), file); err == nil {
t.Fatal("no error for a file")
}
}
func TestTheSignatureFollowsTheLinkOfTheRootButNotOthers(t *testing.T) {
root := tree(t)
link := filepath.Join(t.TempDir(), "music")
if err := os.Symlink(root, link); err != nil {
t.Skip("no symlinks here")
}
if sig(t, link) != sig(t, root) {
t.Error("a library that is a link has another signature than its target")
}
outside := tree(t)
if err := os.Symlink(outside, filepath.Join(root, "outside")); err != nil {
t.Fatal(err)
}
settle(t, root)
before := sig(t, root)
touch(t, outside, "a/changed.mp3")
if sig(t, root) != before {
t.Error("a link inside the folder was followed")
}
}
func TestTheSignatureStopsWithTheContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := Signature(ctx, tree(t)); !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v", err)
}
}
// ---- the watcher ----
type fakeScans struct {
mu sync.Mutex
started []int64
running bool
busy bool // StartLibrary answers ErrScanRunning
lastErr string // what the scan that finished says
finishes int // how many Status calls say "running" before it is over
left int
}
func (f *fakeScans) StartLibrary(_ context.Context, id int64, _ ...ScanOption) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.busy {
return ErrScanRunning
}
f.started = append(f.started, id)
f.left = f.finishes
return nil
}
func (f *fakeScans) Status() Status {
f.mu.Lock()
defer f.mu.Unlock()
if f.left > 0 {
f.left--
return Status{Running: true}
}
return Status{LastError: f.lastErr}
}
func (f *fakeScans) count() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.started)
}
type fakeLibs struct{ libs []Library }
func (f *fakeLibs) ListLibraries(context.Context) ([]Library, error) { return f.libs, nil }
func (f *fakeLibs) GetLibrary(_ context.Context, id int64) (Library, error) {
for _, l := range f.libs {
if l.ID == id {
return l, nil
}
}
return Library{}, ErrLibraryNotFound
}
func newWatcher(scans *fakeScans, libs ...Library) (*AutoScanner, *time.Time) {
now := time.Unix(1_700_000_000, 0)
a := NewAutoScanner(scans, &fakeLibs{libs: libs}, time.Minute)
a.now = func() time.Time { return now }
a.poll = time.Millisecond
return a, &now
}
func TestTheWatcherScansALibraryThatChangedAndLeavesOneThatDidNot(t *testing.T) {
ctx := context.Background()
rock, jazz := tree(t), tree(t)
scans := &fakeScans{}
a, _ := newWatcher(scans, Library{ID: 1, Path: rock}, Library{ID: 2, Path: jazz})
a.tick(ctx) // the first time nothing is known: each library is looked at
if scans.count() != 2 {
t.Fatalf("first tick: %v", scans.started)
}
a.tick(ctx)
if scans.count() != 2 {
t.Fatalf("nothing changed, yet %v", scans.started)
}
touch(t, jazz, "a/new.flac")
a.tick(ctx)
if got := scans.started; len(got) != 3 || got[2] != 2 {
t.Fatalf("only jazz changed: %v", got)
}
a.tick(ctx)
if scans.count() != 3 {
t.Fatalf("after the scan nothing changed: %v", scans.started)
}
}
func TestAScanThatCouldNotStartOrFailedIsTriedAgain(t *testing.T) {
ctx := context.Background()
root := tree(t)
scans := &fakeScans{busy: true}
a, _ := newWatcher(scans, Library{ID: 1, Path: root})
a.tick(ctx) // somebody else is scanning
scans.busy = false
a.tick(ctx)
if scans.count() != 1 {
t.Fatalf("a scan that did not start was forgotten: %v", scans.started)
}
scans.lastErr = "directory is empty but 10 indexed files exist"
touch(t, root, "a/new.mp3")
a.tick(ctx) // starts, and fails
a.tick(ctx) // the same change is still not scanned: try again
if scans.count() != 3 {
t.Fatalf("a failed scan was not tried again: %v", scans.started)
}
scans.lastErr = ""
a.tick(ctx)
a.tick(ctx)
if scans.count() != 4 {
t.Fatalf("after it worked it was scanned again: %v", scans.started)
}
}
func TestAnUnreadableLibraryIsLeftAlone(t *testing.T) {
scans := &fakeScans{}
a, _ := newWatcher(scans, Library{ID: 1, Path: filepath.Join(t.TempDir(), "unmounted")})
a.tick(context.Background())
if scans.count() != 0 {
t.Fatalf("a library that cannot be read was scanned: %v", scans.started)
}
}
func TestEveryLibraryIsScannedOnceADayEvenIfNothingMovedItsFolders(t *testing.T) {
// A song that is tagged again keeps its folder as it was: only the file knows. A scan once a day finds it.
ctx := context.Background()
scans := &fakeScans{}
a, now := newWatcher(scans, Library{ID: 1, Path: tree(t)})
a.tick(ctx)
*now = now.Add(23 * time.Hour)
a.tick(ctx)
if scans.count() != 1 {
t.Fatalf("after 23 hours: %v", scans.started)
}
*now = now.Add(2 * time.Hour)
a.tick(ctx)
if scans.count() != 2 {
t.Fatalf("after 25 hours: %v", scans.started)
}
}
func TestTheWatcherWaitsForTheScanToEndBeforeItGoesOn(t *testing.T) {
ctx := context.Background()
scans := &fakeScans{finishes: 3}
a, _ := newWatcher(scans, Library{ID: 1, Path: tree(t)}, Library{ID: 2, Path: tree(t)})
a.tick(ctx)
if scans.count() != 2 || scans.left != 0 {
t.Fatalf("%v, %d left", scans.started, scans.left)
}
}
func TestADeletedLibraryIsForgottenAndANewOneIsLookedAt(t *testing.T) {
ctx := context.Background()
scans := &fakeScans{}
libs := &fakeLibs{libs: []Library{{ID: 1, Path: tree(t)}}}
a := NewAutoScanner(scans, libs, time.Minute)
a.poll = time.Millisecond
a.tick(ctx)
libs.libs = nil
a.tick(ctx)
if len(a.seen) != 0 {
t.Fatalf("a library that is gone is still remembered: %v", a.seen)
}
libs.libs = []Library{{ID: 1, Path: tree(t)}} // another one with the same id, later
a.tick(ctx)
if scans.count() != 2 {
t.Fatalf("a new library with an old id: %v", scans.started)
}
}
func TestRunStopsWithItsContext(t *testing.T) {
scans := &fakeScans{}
a := NewAutoScanner(scans, &fakeLibs{}, time.Hour)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { a.Run(ctx); close(done) }()
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Run did not stop")
}
}