diff --git a/internal/duplicates/files.go b/internal/duplicates/files.go new file mode 100644 index 0000000..69a992f --- /dev/null +++ b/internal/duplicates/files.go @@ -0,0 +1,80 @@ +package duplicates + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" +) + +// openRoot opens the folder of a library so that nothing done through it can leave it: a ../ or a +// link that points outside is refused by the system at the moment of the call, with no gap between +// checking a path and using it. It returns ErrChanged when the folder is not there (a disk that is +// not mounted). +func openRoot(root string) (*os.Root, error) { + clean := filepath.Clean(root) + if !filepath.IsAbs(clean) { + return nil, fmt.Errorf("%w: the library folder is not an absolute path", ErrChanged) + } + r, err := os.OpenRoot(clean) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w: the library folder is not there", ErrChanged) + } + if err != nil { + return nil, fmt.Errorf("open the library folder: %w", err) + } + return r, nil +} + +// relPath is the path of a file inside its library, in the system's own form. An empty or absolute +// path is not one. +func relPath(rec FileRecord) (string, error) { + rel := filepath.FromSlash(rec.RelPath) + if rel == "" || filepath.IsAbs(rel) { + return "", fmt.Errorf("%w: the path of the file is not inside the library", ErrChanged) + } + return rel, nil +} + +// isEscape says whether an error from an os.Root is the refusal to leave the root (it has no exported value). +func isEscape(err error) bool { + return err != nil && strings.Contains(err.Error(), "path escapes from parent") +} + +// sameAsScanned checks, through the root, that the path is a regular file (not a link, not a +// folder) with the size and date the index has for it. Anything else is ErrChanged. +func sameAsScanned(root *os.Root, rel string, rec FileRecord) error { + info, err := root.Lstat(rel) + switch { + case errors.Is(err, fs.ErrNotExist), isEscape(err): + return fmt.Errorf("%w: the file is not there", ErrChanged) + case err != nil: + return fmt.Errorf("look at the file: %w", err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: it is not a regular file any more", ErrChanged) + } + if info.Size() != rec.SizeBytes || info.ModTime().Unix() != rec.MTimeUnix { + return fmt.Errorf("%w: its size or date is not what the scan saw", ErrChanged) + } + return nil +} + +// removeFile deletes one file inside the root, and turns the ways it can fail into the answers the API gives. +func removeFile(root *os.Root, rel string) error { + err := root.Remove(rel) + switch { + case err == nil: + return nil + case errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("%w: the file is gone", ErrChanged) + case isEscape(err): + return fmt.Errorf("%w: the path leaves the library", ErrChanged) + case errors.Is(err, fs.ErrPermission), errors.Is(err, syscall.EROFS): + return fmt.Errorf("%w: %v", ErrReadOnly, err) + } + return fmt.Errorf("remove the file: %w", err) +} diff --git a/internal/duplicates/hashjob.go b/internal/duplicates/hashjob.go new file mode 100644 index 0000000..7c8223f --- /dev/null +++ b/internal/duplicates/hashjob.go @@ -0,0 +1,120 @@ +package duplicates + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "sync" + "time" + + "codeberg.org/squidd/sonda/internal/jobs" +) + +// OutcomeHashed is what the check counts for a file it read and fingerprinted. +const OutcomeHashed = "hashed" + +// NewHashJob builds the job that reads the files of the groups of copies and keeps their SHA-256, so +// that a group of files that are byte for byte the same can be said to be identical. It is started by +// hand like the other bulk jobs: it reads whole files, which on a NAS takes a while. +func NewHashJob(s *Service) *jobs.Job { + t := &hashTask{s: s} + return jobs.New(jobs.Task{Noun: "files", Count: t.count, List: t.list, Do: t.do}, s.now) +} + +// hashTask keeps, between Count and the Lists that follow, the files to read: those of the groups +// that have no fingerprint yet (all of them to do everything again). +type hashTask struct { + s *Service + + mu sync.Mutex + ids []int64 +} + +func (t *hashTask) count(ctx context.Context, refresh bool, _ time.Time) (int, error) { + groups, err := t.s.groups(ctx) + if err != nil { + return 0, err + } + var ids []int64 + for _, g := range groups { + for _, f := range g.Files { + if refresh || f.Hash == "" { + ids = append(ids, f.FileID) + } + } + } + slices.Sort(ids) + t.mu.Lock() + t.ids = ids + t.mu.Unlock() + return len(ids), nil +} + +func (t *hashTask) list(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) { + t.mu.Lock() + defer t.mu.Unlock() + start := sort.Search(len(t.ids), func(i int) bool { return t.ids[i] > after }) + return slices.Clone(t.ids[start:min(start+limit, len(t.ids))]), nil +} + +// do reads one file and keeps its fingerprint, if the file is still the one the scan saw. +func (t *hashTask) do(ctx context.Context, fileID int64, _ bool) (string, error) { + rec, err := t.s.store.FileByID(ctx, fileID) + if errors.Is(err, ErrTrackNotFound) || err == nil && rec.Missing { + return "", jobs.ErrSkip + } + if err != nil { + return "", err + } + root, err := openRoot(rec.Root) + if err != nil { + return "", err // a disk that is not mounted: every file fails the same way, and the job gives up + } + defer root.Close() + rel, err := relPath(rec) + if err != nil { + return "", jobs.ErrSkip + } + if err := sameAsScanned(root, rel, rec); errors.Is(err, ErrChanged) { + return "", jobs.ErrSkip + } else if err != nil { + return "", err + } + + f, err := root.Open(rel) + if err != nil { + return "", fmt.Errorf("open the file: %w", err) + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, &contextReader{ctx: ctx, r: f}); err != nil { + return "", fmt.Errorf("read the file: %w", err) + } + saved, err := t.s.store.SaveHash(ctx, fileID, rec.SizeBytes, rec.MTimeUnix, hex.EncodeToString(h.Sum(nil))) + if err != nil { + return "", err + } + if !saved { + return "", jobs.ErrSkip // it changed while it was read + } + return OutcomeHashed, nil +} + +// contextReader stops reading when the job is stopped, instead of finishing a long file first. +type contextReader struct { + ctx context.Context + r *os.File +} + +func (c *contextReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/internal/duplicates/service.go b/internal/duplicates/service.go new file mode 100644 index 0000000..185f309 --- /dev/null +++ b/internal/duplicates/service.go @@ -0,0 +1,236 @@ +package duplicates + +import ( + "context" + "fmt" + "os" + "slices" + "time" + + "go.uber.org/zap" +) + +// Store is what the service reads and writes in the index. internal/storage implements it. +type Store interface { + Candidates(ctx context.Context) ([]Candidate, error) + FileOf(ctx context.Context, trackID int64) (FileRecord, error) + FileByID(ctx context.Context, fileID int64) (FileRecord, error) + SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error) + RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error +} + +// Options say what the service may do. +type Options struct { + // AllowDelete is library.allow_delete. Without it nothing is ever removed. + AllowDelete bool + // ScanRunning says whether a scan is running; a removal waits for it to end. Nil means never. + ScanRunning func() bool + // Logger records each removal, by track and without the path: audio paths are private. + Logger *zap.Logger +} + +// Service finds duplicate songs and removes the extra copies. +type Service struct { + store Store + allow bool + scanning func() bool + log *zap.Logger + now func() time.Time +} + +// NewService builds a Service. +func NewService(store Store, o Options) *Service { + log := o.Logger + if log == nil { + log = zap.NewNop() + } + return &Service{store: store, allow: o.AllowDelete, scanning: o.ScanRunning, log: log, now: time.Now} +} + +// Page is some of the groups, with how many there are in all. +type Page struct { + Groups []Group + Total int + // DeletionAllowed says whether Clean can work: with it off, the groups can still be seen. + DeletionAllowed bool +} + +// Groups returns the groups of copies in the library, worked out from the index and never from the disk. +func (s *Service) Groups(ctx context.Context, limit, offset int) (Page, error) { + all, err := s.groups(ctx) + if err != nil { + return Page{}, err + } + page := Page{Total: len(all), DeletionAllowed: s.allow} + if offset < len(all) { + page.Groups = all[offset:min(offset+limit, len(all))] + } + return page, nil +} + +func (s *Service) groups(ctx context.Context) ([]Group, error) { + candidates, err := s.store.Candidates(ctx) + if err != nil { + return nil, err + } + return Build(candidates, DurationTolerance), nil +} + +// maxRemovals is the most copies of one song that one request may remove. +const maxRemovals = 100 + +// Result says what a removal did. +type Result struct { + Removed int + FreedBytes int64 +} + +// Clean removes the copies `remove` of a song from the disk and the index, keeping `keep`. Nothing +// is removed unless every check passes, and the checks are made here, again, whatever the client +// saw: deleting is allowed, no scan is running, the tracks are copies of one song by the rule of +// Build, the copy to keep is still the file that was scanned (so it is never the last copy that is +// removed), and so is every copy to remove (a file that changed, a link, or a path that leaves the +// library is refused). The files are checked all together before the first is removed. Each removed +// file takes its track out of the index, and its listens go to the track that stays. +// +// On an error part way (the folder turned out to be read-only on the second file), the Result says +// what was already removed. +func (s *Service) Clean(ctx context.Context, keep int64, remove []int64) (Result, error) { + var res Result + if !s.allow { + return res, ErrDisabled + } + if s.scanning != nil && s.scanning() { + return res, ErrScanRunning + } + if err := checkRequest(keep, remove); err != nil { + return res, err + } + + keepFile, err := s.store.FileOf(ctx, keep) + if err != nil { + return res, err + } + files := make(map[int64]FileRecord, len(remove)) + for _, id := range remove { + rec, err := s.store.FileOf(ctx, id) + if err != nil { + return res, err + } + files[id] = rec + } + if keepFile.Missing { + return res, fmt.Errorf("%w: the copy to keep is flagged as missing", ErrChanged) + } + for _, rec := range files { + if rec.Missing { + return res, fmt.Errorf("%w: a copy to remove is flagged as missing", ErrChanged) + } + } + if err := s.checkSameSong(ctx, keep, remove); err != nil { + return res, err + } + + // Look at every file before touching any. + roots := map[string]*os.Root{} + defer func() { + for _, r := range roots { + _ = r.Close() + } + }() + rootOf := func(path string) (*os.Root, error) { + if r, ok := roots[path]; ok { + return r, nil + } + r, err := openRoot(path) + if err != nil { + return nil, err + } + roots[path] = r + return r, nil + } + inspect := func(rec FileRecord) (*os.Root, string, error) { + root, err := rootOf(rec.Root) + if err != nil { + return nil, "", err + } + rel, err := relPath(rec) + if err != nil { + return nil, "", err + } + return root, rel, sameAsScanned(root, rel, rec) + } + if _, _, err := inspect(keepFile); err != nil { + return res, err + } + type target struct { + id int64 + root *os.Root + rel string + } + targets := make([]target, 0, len(remove)) + for _, id := range remove { + root, rel, err := inspect(files[id]) + if err != nil { + return res, err + } + targets = append(targets, target{id, root, rel}) + } + + for _, t := range targets { + if err := removeFile(t.root, t.rel); err != nil { + return res, err + } + // The file is gone. If the index cannot follow, the next scan flags the path as missing, and + // nothing is left broken; the error says so. + if err := s.store.RemoveCopy(ctx, keep, t.id); err != nil { + s.log.Error("a duplicate file was removed but its track is still in the index; a scan will flag it as missing", + zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep), zap.Error(err)) + return res, fmt.Errorf("the file was removed but the index could not be updated: %w", err) + } + res.Removed++ + res.FreedBytes += files[t.id].SizeBytes + s.log.Info("a duplicate file was removed", zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep)) + } + return res, nil +} + +// checkRequest refuses a request that makes no sense, before anything is read: nothing to remove, +// too much, the copy to keep among those to remove, or a copy named twice. +func checkRequest(keep int64, remove []int64) error { + if len(remove) == 0 || len(remove) > maxRemovals { + return fmt.Errorf("%w: name between 1 and %d copies to remove", ErrInvalid, maxRemovals) + } + seen := map[int64]bool{} + for _, id := range remove { + if id == keep { + return fmt.Errorf("%w: the copy to keep is among those to remove", ErrInvalid) + } + if seen[id] { + return fmt.Errorf("%w: a copy is named twice", ErrInvalid) + } + seen[id] = true + } + return nil +} + +// checkSameSong works out the groups again and checks that every copy to remove is in the group of +// the copy to keep. +func (s *Service) checkSameSong(ctx context.Context, keep int64, remove []int64) error { + groups, err := s.groups(ctx) + if err != nil { + return err + } + for _, g := range groups { + if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == keep }) { + continue + } + for _, id := range remove { + if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == id }) { + return ErrInvalid + } + } + return nil + } + return ErrInvalid +} diff --git a/internal/duplicates/service_test.go b/internal/duplicates/service_test.go new file mode 100644 index 0000000..576c8d7 --- /dev/null +++ b/internal/duplicates/service_test.go @@ -0,0 +1,497 @@ +package duplicates_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/storage" + "codeberg.org/squidd/sonda/migrations" +) + +// env is a real library on a real folder, indexed in a real database. +type env struct { + t *testing.T + ctx context.Context + db *bun.DB + dir string + libID int64 + lib *storage.LibraryRepo + store *storage.DuplicatesRepo + scanning bool +} + +func newEnv(t *testing.T) *env { + t.Helper() + db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + fsys, err := migrations.For(storage.DriverSQLite) + if err != nil { + t.Fatal(err) + } + if err := storage.Migrate(context.Background(), db, fsys); err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)} + l, err := e.lib.CreateLibrary(e.ctx, "music", dir) + if err != nil { + t.Fatal(err) + } + e.libID = l.ID + return e +} + +func (e *env) service(allow bool) *duplicates.Service { + return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()}) +} + +// add writes a file and indexes it exactly as it is on the disk, and returns its track. +func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + e.t.Helper() + path := filepath.Join(e.dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + e.t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + e.t.Fatal(err) + } + return e.index(rel, path, title, album, durationMs, format, bitrate, artists...) +} + +// index puts a file that is on the disk into the index with the size and date it has. +func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + e.t.Helper() + info, err := os.Stat(path) + if err != nil { + e.t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate}, + }, + } + if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil { + e.t.Fatal(err) + } + var id int64 + if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil { + e.t.Fatal(err) + } + return id +} + +func (e *env) exists(rel string) bool { + _, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel))) + return err == nil +} + +func (e *env) tracks() int { + var n int + if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil { + e.t.Fatal(err) + } + return n +} + +// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums. +func twoCopies(e *env) (flac, mp3 int64) { + flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann") + mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann") + return flac, mp3 +} + +// --- finding --------------------------------------------------------------------------------------- + +func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann") + + page, err := e.service(true).Groups(e.ctx, 10, 0) + if err != nil { + t.Fatal(err) + } + if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed { + t.Fatalf("page = %+v, want one group and deleting allowed", page) + } + g := page.Groups[0] + if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) { + t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g) + } + if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" { + t.Errorf("files = %+v", g.Files) + } +} + +func TestGroupsReportsThatDeletingIsOff(t *testing.T) { + e := newEnv(t) + twoCopies(e) + page, err := e.service(false).Groups(e.ctx, 10, 0) + if err != nil { + t.Fatal(err) + } + if page.DeletionAllowed || len(page.Groups) != 1 { + t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page) + } +} + +func TestGroupsArePaged(t *testing.T) { + e := newEnv(t) + for _, title := range []string{"A song", "B song", "C song"} { + e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann") + e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann") + } + svc := e.service(true) + first, _ := svc.Groups(e.ctx, 2, 0) + second, _ := svc.Groups(e.ctx, 2, 2) + if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 { + t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total) + } + if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" { + t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title) + } + if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 { + t.Errorf("past the end: %+v", beyond) + } +} + +// --- fingerprints ------------------------------------------------------------------------------------ + +func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status { + t.Helper() + if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if st := j.Status(); !st.Running { + return st + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("the job did not finish") + return jobs.Status{} +} + +func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) { + e := newEnv(t) + e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann") + e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann") + e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann") + e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann") + e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann") + svc := e.service(true) + + st := runJob(t, duplicates.NewHashJob(svc), false) + if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" { + t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st) + } + + page, _ := svc.Groups(e.ctx, 10, 0) + byTitle := map[string]duplicates.Group{} + for _, g := range page.Groups { + byTitle[g.Title] = g + } + if g := byTitle["Same"]; !g.Hashed || !g.Identical { + t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical) + } + if g := byTitle["Alike"]; !g.Hashed || g.Identical { + t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical) + } +} + +func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) { + e := newEnv(t) + twoCopies(e) + job := duplicates.NewHashJob(e.service(true)) + if st := runJob(t, job, false); st.Last.Checked != 2 { + t.Fatalf("first run checked %d, want 2", st.Last.Checked) + } + if st := runJob(t, job, false); st.Last.Checked != 0 { + t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked) + } + if st := runJob(t, job, true); st.Last.Checked != 2 { + t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked) + } +} + +func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) { + e := newEnv(t) + twoCopies(e) + if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil { + t.Fatal(err) + } + st := runJob(t, duplicates.NewHashJob(e.service(true)), false) + if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 { + t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last) + } + page, _ := e.service(true).Groups(e.ctx, 10, 0) + if page.Groups[0].Hashed { + t.Error("the group claims every file was read, but one was not") + } +} + +func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) { + e := newEnv(t) + twoCopies(e) + if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Fatal(err) + } + st := runJob(t, duplicates.NewHashJob(e.service(true)), false) + if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 { + t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last) + } +} + +// --- removing ----------------------------------------------------------------------------------------- + +func TestWithDeletingOffNothingIsTouched(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + _, err := e.service(false).Clean(e.ctx, flac, []int64{mp3}) + if !errors.Is(err, duplicates.ErrDisabled) { + t.Fatalf("got %v, want ErrDisabled", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +func TestNothingIsRemovedWhileAScanRuns(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + e.scanning = true + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) { + t.Fatalf("got %v, want ErrScanRunning", err) + } + if !e.exists("b/song.mp3") { + t.Error("a file was removed during a scan") + } +} + +func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}) + if err != nil { + t.Fatal(err) + } + if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) { + t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res) + } + if e.exists("b/song.mp3") { + t.Error("the copy is still on the disk") + } + if !e.exists("a/song.flac") { + t.Error("the copy to keep is gone") + } + if e.tracks() != 1 { + t.Errorf("%d tracks, want 1", e.tracks()) + } + if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 { + t.Errorf("%d groups are left, want none", page.Total) + } + if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("the removed track is still known: %v", err) + } +} + +func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) { + e := newEnv(t) + keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann") + b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann") + c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann") + res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}) + if err != nil { + t.Fatal(err) + } + if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") { + t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac")) + } +} + +func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} { + if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) { + t.Errorf("%s: got %v, want ErrInvalid", name, err) + } + } + if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +// What the client sends is not trusted: the server works out again that these are copies of one song. +func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) { + e := newEnv(t) + flac, _ := twoCopies(e) + other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann") + otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob") + live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann") + for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} { + if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) { + t.Errorf("%s: got %v, want ErrInvalid", name, err) + } + } + for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} { + if !e.exists(rel) { + t.Errorf("%s was removed", rel) + } + } +} + +func TestATrackThatDoesNotExistIsRefused(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err) + } + if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err) + } + if !e.exists("b/song.mp3") { + t.Error("a file was removed") + } +} + +// If the copy to keep is not there any more, the other one is the last, and the last is never removed. +func TestTheLastCopyIsNeverRemoved(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil { + t.Fatal(err) + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("the only copy left was removed") + } +} + +func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) { + e := newEnv(t) + keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann") + b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann") + c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann") + // c is edited after the scan: it is not the file the admin saw. + if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + // Nothing at all was removed, not even b, which was fine: every file is checked before any is removed. + if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 { + t.Error("something was removed although a file in the request had changed") + } +} + +func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + target := filepath.Join(e.dir, "precious.txt") + if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Skipf("no symlinks here: %v", err) + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + if !e.exists("precious.txt") || !e.exists("b/song.mp3") { + t.Error("the link or what it points at was removed") + } +} + +// A path that was recorded as leaving the library is never followed out of it. +func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) { + e := newEnv(t) + flac, _ := twoCopies(e) + outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir)) + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(outside) }) + victim := filepath.Join(outside, "song.mp3") + if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil { + t.Fatal(err) + } + rel := "../" + filepath.Base(outside) + "/song.mp3" + escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann") + if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil { + t.Fatal("a file outside the library was removed without an error") + } + if _, err := os.Stat(victim); err != nil { + t.Errorf("the file outside the library was removed: %v", err) + } +} + +func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("permissions are not enforced here") + } + e := newEnv(t) + flac, mp3 := twoCopies(e) + dir := filepath.Join(e.dir, "b") + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) { + t.Fatalf("got %v, want ErrReadOnly", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +// Listening history must survive: it moves to the copy that stays. +func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false) + if err != nil { + t.Fatal(err) + } + plays := storage.NewListeningRepo(e.db) + for i, id := range []int64{mp3, mp3, flac} { + ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))} + if _, err := plays.Record(e.ctx, ev); err != nil { + t.Fatal(err) + } + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil { + t.Fatal(err) + } + var n int + if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 { + t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err) + } +}