- feat: the scan summary has an unsupported counter for audio files in formats Sonda cannot read yet (m4a, ogg, opus, wav, wma, ...); they are counted and never opened, instead of disappearing without a trace. Video containers such as .mpg are not counted - feat: folders named .@__thumb, @Recycle, #recycle, @eaDir, $RECYCLE.BIN and System Volume Information are skipped, matched by exact name ignoring case, so a NAS recycle bin does not end up in the library; a library whose own folder carries one of these names is still scanned, and files indexed before from such a folder are flagged missing like any vanished file - docs: the README lists the new counter and the skipped folders - test: system folders at any depth, near-miss names, a library inside a system-named folder, entries flagged missing, unsupported counting Closes #14
432 lines
11 KiB
Go
432 lines
11 KiB
Go
package library
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// audioExts are the formats Sonda reads.
|
|
var audioExts = map[string]bool{".mp3": true, ".flac": true}
|
|
|
|
// unsupportedExts are audio formats Sonda does not read yet. They are counted,
|
|
// never opened. Video containers such as .mpg or .mp4 are deliberately absent.
|
|
var unsupportedExts = map[string]bool{
|
|
".m4a": true, ".m4b": true, ".aac": true, ".alac": true, ".ogg": true, ".oga": true,
|
|
".opus": true, ".wav": true, ".wma": true, ".ape": true, ".wv": true, ".aif": true,
|
|
".aiff": true, ".mpc": true, ".mka": true, ".dsf": true, ".dff": true,
|
|
}
|
|
|
|
// systemDirs are housekeeping folders that NAS boxes and operating systems
|
|
// create inside shares: thumbnails, recycle bins, indexes. What they hold is
|
|
// never part of a library. Only the exact folder name matches, ignoring case.
|
|
var systemDirs = []string{
|
|
".@__thumb", "@Recycle", "#recycle", "@eaDir", "$RECYCLE.BIN", "System Volume Information",
|
|
}
|
|
|
|
func isSystemDir(name string) bool {
|
|
for _, d := range systemDirs {
|
|
if strings.EqualFold(name, d) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Scanner indexes the audio files of a set of libraries.
|
|
//
|
|
// Per library it walks the tree (cheap, stat only), decides what changed, then
|
|
// reads tags with a pool of workers. A single goroutine writes to the store.
|
|
type Scanner struct {
|
|
store Store
|
|
reader TagReader
|
|
hub *Hub
|
|
workers int
|
|
now func() time.Time
|
|
}
|
|
|
|
// NewScanner builds a scanner that parses files with the given number of workers.
|
|
func NewScanner(store Store, reader TagReader, hub *Hub, workers int) *Scanner {
|
|
if workers < 1 {
|
|
workers = 1
|
|
}
|
|
return &Scanner{store: store, reader: reader, hub: hub, workers: workers, now: time.Now}
|
|
}
|
|
|
|
// Scan indexes every library in turn. An unreadable library is reported in the
|
|
// returned error and the other libraries are still scanned.
|
|
func (s *Scanner) Scan(ctx context.Context, libs []Library) (Summary, error) {
|
|
total := Summary{Libraries: len(libs)}
|
|
s.publish(Event{Type: ScanStarted})
|
|
|
|
var errs []error
|
|
for _, lib := range libs {
|
|
sum, err := s.scanLibrary(ctx, lib)
|
|
total.add(sum)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("%s: %w", lib.Name, err))
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
err := errors.Join(errs...)
|
|
fin := Event{Type: ScanFinished, Summary: &total}
|
|
if err != nil {
|
|
fin.Error = err.Error()
|
|
}
|
|
s.publish(fin)
|
|
return total, err
|
|
}
|
|
|
|
func (s *Scanner) publish(e Event) {
|
|
e.Time = s.now()
|
|
s.hub.Publish(e)
|
|
}
|
|
|
|
type parseJob struct {
|
|
file FileInfo
|
|
existingID int64
|
|
}
|
|
|
|
type parseResult struct {
|
|
job parseJob
|
|
meta Metadata
|
|
err error
|
|
}
|
|
|
|
func (s *Scanner) scanLibrary(ctx context.Context, lib Library) (Summary, error) {
|
|
var sum Summary
|
|
|
|
walked, err := s.walk(ctx, lib, &sum)
|
|
if err != nil {
|
|
// The root itself is unreadable (for example an unmounted NAS).
|
|
// Do not touch the database: nothing is known about its files.
|
|
return sum, err
|
|
}
|
|
sum.Found = len(walked.files)
|
|
|
|
existing, err := s.store.ExistingFiles(ctx, lib.ID)
|
|
if err != nil {
|
|
return sum, fmt.Errorf("list indexed files: %w", err)
|
|
}
|
|
// An unmounted NAS usually leaves an empty but readable folder. Treat
|
|
// "nothing found although files are indexed" as a problem, not as a mass
|
|
// deletion. Delete the library to really empty it.
|
|
if len(walked.files) == 0 && len(walked.failedDirs) == 0 {
|
|
if n := countPresent(existing); n > 0 {
|
|
return sum, fmt.Errorf("directory is empty but %d indexed files exist: is the volume mounted? "+
|
|
"refusing to mark them missing", n)
|
|
}
|
|
}
|
|
p := makePlan(walked, existing)
|
|
sum.Unchanged = p.unchanged
|
|
|
|
if err := s.parseAndSave(ctx, lib, p.parse, &sum); err != nil {
|
|
return sum, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return sum, err // interrupted: never mark files missing on a partial scan
|
|
}
|
|
|
|
for _, r := range p.renames {
|
|
if err := s.store.RenameFile(ctx, r.id, r.newRel); err != nil {
|
|
s.fail(lib, r.newRel, err, &sum)
|
|
continue
|
|
}
|
|
sum.Renamed++
|
|
}
|
|
if len(p.restore) > 0 {
|
|
if err := s.store.Restore(ctx, p.restore); err != nil {
|
|
return sum, fmt.Errorf("restore files: %w", err)
|
|
}
|
|
sum.Restored = len(p.restore)
|
|
}
|
|
if len(p.missing) > 0 {
|
|
if err := s.store.MarkMissing(ctx, p.missing, s.now()); err != nil {
|
|
return sum, fmt.Errorf("mark missing: %w", err)
|
|
}
|
|
sum.Missing = len(p.missing)
|
|
}
|
|
return sum, nil
|
|
}
|
|
|
|
func (s *Scanner) fail(lib Library, rel string, err error, sum *Summary) {
|
|
sum.Failed++
|
|
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error()})
|
|
}
|
|
|
|
// parseAndSave reads tags in parallel and stores results from this goroutine.
|
|
func (s *Scanner) parseAndSave(ctx context.Context, lib Library, jobs []parseJob, sum *Summary) error {
|
|
if len(jobs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
jobCh := make(chan parseJob)
|
|
resCh := make(chan parseResult)
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < s.workers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := range jobCh {
|
|
meta, err := s.reader.Read(filepath.Join(lib.Path, filepath.FromSlash(j.file.RelPath)))
|
|
resCh <- parseResult{job: j, meta: meta, err: err}
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
defer func() {
|
|
close(jobCh)
|
|
wg.Wait()
|
|
close(resCh)
|
|
}()
|
|
for _, j := range jobs {
|
|
select {
|
|
case jobCh <- j:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
done := 0
|
|
for r := range resCh { // always drain, so no worker is left blocked
|
|
if ctx.Err() != nil {
|
|
continue
|
|
}
|
|
done++
|
|
rel := r.job.file.RelPath
|
|
if r.err != nil {
|
|
sum.Failed++
|
|
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: r.err.Error(), Done: done, Total: len(jobs)})
|
|
continue
|
|
}
|
|
added, err := s.store.SaveFile(ctx, lib.ID, ParsedFile{FileInfo: r.job.file, Metadata: r.meta}, r.job.existingID)
|
|
if err != nil {
|
|
sum.Failed++
|
|
s.publish(Event{Type: FileFailed, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Error: err.Error(), Done: done, Total: len(jobs)})
|
|
continue
|
|
}
|
|
if added {
|
|
sum.Added++
|
|
} else {
|
|
sum.Updated++
|
|
}
|
|
s.publish(Event{Type: FileScanned, LibraryID: lib.ID, Library: lib.Name, RelPath: rel, Done: done, Total: len(jobs)})
|
|
}
|
|
return ctx.Err()
|
|
}
|
|
|
|
// ---- walking ----
|
|
|
|
type walkResult struct {
|
|
files []FileInfo
|
|
// failedDirs are slash-separated relative paths (with a trailing slash)
|
|
// that could not be read. Files below them are unknown, not gone.
|
|
failedDirs []string
|
|
// failedFiles are relative paths of files that could not be stat'ed for a
|
|
// reason other than "does not exist". They are unknown, not gone.
|
|
failedFiles map[string]bool
|
|
}
|
|
|
|
func (s *Scanner) walk(ctx context.Context, lib Library, sum *Summary) (walkResult, error) {
|
|
var out walkResult
|
|
|
|
// The library path may itself be a symlink (for example /srv/music ->
|
|
// /mnt/nas/music). WalkDir would treat it as a plain file and find nothing,
|
|
// so walk the resolved directory. Symlinks *inside* the root are still skipped.
|
|
walkRoot, err := filepath.EvalSymlinks(lib.Path)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
info, err := os.Stat(walkRoot)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
if !info.IsDir() {
|
|
return out, fmt.Errorf("%s is not a directory", lib.Path)
|
|
}
|
|
|
|
err = filepath.WalkDir(walkRoot, func(path string, d fs.DirEntry, err error) error {
|
|
if cerr := ctx.Err(); cerr != nil {
|
|
return cerr
|
|
}
|
|
if path == walkRoot {
|
|
return err // nil, or the reason the root cannot be read
|
|
}
|
|
rel, rerr := filepath.Rel(walkRoot, path)
|
|
if rerr != nil {
|
|
return rerr
|
|
}
|
|
rel = filepath.ToSlash(rel)
|
|
|
|
if err != nil {
|
|
s.fail(lib, rel, err, sum)
|
|
if d != nil && d.IsDir() {
|
|
out.failedDirs = append(out.failedDirs, rel+"/")
|
|
} else {
|
|
out.markFileFailed(rel)
|
|
}
|
|
return nil // keep going, do not descend into an unreadable dir
|
|
}
|
|
if d.Type()&fs.ModeSymlink != 0 {
|
|
sum.Skipped++ // never follow symlinks: they could point outside the root
|
|
return nil
|
|
}
|
|
if d.IsDir() {
|
|
if isSystemDir(d.Name()) {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
ext := strings.ToLower(filepath.Ext(rel))
|
|
if !audioExts[ext] {
|
|
if unsupportedExts[ext] {
|
|
sum.Unsupported++
|
|
}
|
|
return nil
|
|
}
|
|
if !d.Type().IsRegular() {
|
|
// FIFOs, sockets and devices: opening one can block forever.
|
|
sum.Skipped++
|
|
return nil
|
|
}
|
|
info, ierr := d.Info()
|
|
if ierr != nil {
|
|
s.fail(lib, rel, ierr, sum)
|
|
if !errors.Is(ierr, fs.ErrNotExist) {
|
|
out.markFileFailed(rel)
|
|
}
|
|
return nil
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
// Replaced by a FIFO or symlink between the directory read and now.
|
|
sum.Skipped++
|
|
return nil
|
|
}
|
|
out.files = append(out.files, FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()})
|
|
return nil
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
func (w *walkResult) markFileFailed(rel string) {
|
|
if w.failedFiles == nil {
|
|
w.failedFiles = map[string]bool{}
|
|
}
|
|
w.failedFiles[rel] = true
|
|
}
|
|
|
|
// ---- planning ----
|
|
|
|
type renameOp struct {
|
|
id int64
|
|
newRel string
|
|
}
|
|
|
|
type plan struct {
|
|
parse []parseJob
|
|
renames []renameOp
|
|
restore []int64
|
|
missing []int64
|
|
unchanged int
|
|
}
|
|
|
|
type fileKey struct{ size, mtime int64 }
|
|
|
|
// makePlan compares what is on disk with what is indexed.
|
|
//
|
|
// A new file and a vanished file with the same size and modification time are
|
|
// treated as a rename only when each is the sole candidate for that pair, so
|
|
// look-alike files are never guessed.
|
|
func makePlan(w walkResult, existing []ExistingFile) plan {
|
|
var p plan
|
|
byPath := make(map[string]ExistingFile, len(existing))
|
|
for _, e := range existing {
|
|
byPath[e.RelPath] = e
|
|
}
|
|
|
|
seen := make(map[string]bool, len(w.files))
|
|
var added []FileInfo
|
|
for _, f := range w.files {
|
|
e, ok := byPath[f.RelPath]
|
|
if !ok {
|
|
added = append(added, f)
|
|
continue
|
|
}
|
|
seen[f.RelPath] = true
|
|
switch {
|
|
case e.Size != f.Size || e.MTimeUnix != f.MTimeUnix:
|
|
p.parse = append(p.parse, parseJob{file: f, existingID: e.ID})
|
|
case e.Missing:
|
|
p.restore = append(p.restore, e.ID)
|
|
default:
|
|
p.unchanged++
|
|
}
|
|
}
|
|
|
|
var gone []ExistingFile
|
|
for _, e := range existing {
|
|
if seen[e.RelPath] || w.failedFiles[e.RelPath] || underAny(e.RelPath, w.failedDirs) {
|
|
continue
|
|
}
|
|
gone = append(gone, e)
|
|
}
|
|
|
|
goneByKey := map[fileKey][]ExistingFile{}
|
|
for _, e := range gone {
|
|
k := fileKey{e.Size, e.MTimeUnix}
|
|
goneByKey[k] = append(goneByKey[k], e)
|
|
}
|
|
addedByKey := map[fileKey]int{}
|
|
for _, f := range added {
|
|
addedByKey[fileKey{f.Size, f.MTimeUnix}]++
|
|
}
|
|
|
|
renamed := map[int64]bool{}
|
|
for _, f := range added {
|
|
k := fileKey{f.Size, f.MTimeUnix}
|
|
if cands := goneByKey[k]; len(cands) == 1 && addedByKey[k] == 1 {
|
|
p.renames = append(p.renames, renameOp{id: cands[0].ID, newRel: f.RelPath})
|
|
renamed[cands[0].ID] = true
|
|
continue
|
|
}
|
|
p.parse = append(p.parse, parseJob{file: f})
|
|
}
|
|
|
|
for _, e := range gone {
|
|
if !renamed[e.ID] && !e.Missing {
|
|
p.missing = append(p.missing, e.ID)
|
|
}
|
|
}
|
|
return p
|
|
}
|
|
|
|
// countPresent counts indexed files that were not already flagged missing.
|
|
func countPresent(existing []ExistingFile) int {
|
|
n := 0
|
|
for _, e := range existing {
|
|
if !e.Missing {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func underAny(rel string, prefixes []string) bool {
|
|
for _, p := range prefixes {
|
|
if strings.HasPrefix(rel, p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|