feat(duplicates): find the copies, fingerprint their files and remove the extra ones

Groups come from the index. The check job reads the files of the groups and keeps their SHA-256, skipping a file that changed since the scan. Clean refuses unless deleting is on and no scan runs, works out again that the tracks are copies of one song, checks every file is still what the scan saw (so the last copy is never removed, and a link or a path that leaves the library is refused) before removing any, removes each file through an os.Root, then takes its track out of the index and moves its listens. Tested with real files, including the refusals; three of the safety rules were broken on purpose to see the tests fail.
This commit is contained in:
tgiachi 2026-11-12 16:40:43 +01:00
commit d3e3bde2e4
4 changed files with 933 additions and 0 deletions

View file

@ -0,0 +1,80 @@
package duplicates
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
)
// openRoot opens the folder of a library so that nothing done through it can leave it: a ../ or a
// link that points outside is refused by the system at the moment of the call, with no gap between
// checking a path and using it. It returns ErrChanged when the folder is not there (a disk that is
// not mounted).
func openRoot(root string) (*os.Root, error) {
clean := filepath.Clean(root)
if !filepath.IsAbs(clean) {
return nil, fmt.Errorf("%w: the library folder is not an absolute path", ErrChanged)
}
r, err := os.OpenRoot(clean)
if errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("%w: the library folder is not there", ErrChanged)
}
if err != nil {
return nil, fmt.Errorf("open the library folder: %w", err)
}
return r, nil
}
// relPath is the path of a file inside its library, in the system's own form. An empty or absolute
// path is not one.
func relPath(rec FileRecord) (string, error) {
rel := filepath.FromSlash(rec.RelPath)
if rel == "" || filepath.IsAbs(rel) {
return "", fmt.Errorf("%w: the path of the file is not inside the library", ErrChanged)
}
return rel, nil
}
// isEscape says whether an error from an os.Root is the refusal to leave the root (it has no exported value).
func isEscape(err error) bool {
return err != nil && strings.Contains(err.Error(), "path escapes from parent")
}
// sameAsScanned checks, through the root, that the path is a regular file (not a link, not a
// folder) with the size and date the index has for it. Anything else is ErrChanged.
func sameAsScanned(root *os.Root, rel string, rec FileRecord) error {
info, err := root.Lstat(rel)
switch {
case errors.Is(err, fs.ErrNotExist), isEscape(err):
return fmt.Errorf("%w: the file is not there", ErrChanged)
case err != nil:
return fmt.Errorf("look at the file: %w", err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("%w: it is not a regular file any more", ErrChanged)
}
if info.Size() != rec.SizeBytes || info.ModTime().Unix() != rec.MTimeUnix {
return fmt.Errorf("%w: its size or date is not what the scan saw", ErrChanged)
}
return nil
}
// removeFile deletes one file inside the root, and turns the ways it can fail into the answers the API gives.
func removeFile(root *os.Root, rel string) error {
err := root.Remove(rel)
switch {
case err == nil:
return nil
case errors.Is(err, fs.ErrNotExist):
return fmt.Errorf("%w: the file is gone", ErrChanged)
case isEscape(err):
return fmt.Errorf("%w: the path leaves the library", ErrChanged)
case errors.Is(err, fs.ErrPermission), errors.Is(err, syscall.EROFS):
return fmt.Errorf("%w: %v", ErrReadOnly, err)
}
return fmt.Errorf("remove the file: %w", err)
}

View file

@ -0,0 +1,120 @@
package duplicates
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"slices"
"sort"
"sync"
"time"
"codeberg.org/squidd/sonda/internal/jobs"
)
// OutcomeHashed is what the check counts for a file it read and fingerprinted.
const OutcomeHashed = "hashed"
// NewHashJob builds the job that reads the files of the groups of copies and keeps their SHA-256, so
// that a group of files that are byte for byte the same can be said to be identical. It is started by
// hand like the other bulk jobs: it reads whole files, which on a NAS takes a while.
func NewHashJob(s *Service) *jobs.Job {
t := &hashTask{s: s}
return jobs.New(jobs.Task{Noun: "files", Count: t.count, List: t.list, Do: t.do}, s.now)
}
// hashTask keeps, between Count and the Lists that follow, the files to read: those of the groups
// that have no fingerprint yet (all of them to do everything again).
type hashTask struct {
s *Service
mu sync.Mutex
ids []int64
}
func (t *hashTask) count(ctx context.Context, refresh bool, _ time.Time) (int, error) {
groups, err := t.s.groups(ctx)
if err != nil {
return 0, err
}
var ids []int64
for _, g := range groups {
for _, f := range g.Files {
if refresh || f.Hash == "" {
ids = append(ids, f.FileID)
}
}
}
slices.Sort(ids)
t.mu.Lock()
t.ids = ids
t.mu.Unlock()
return len(ids), nil
}
func (t *hashTask) list(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) {
t.mu.Lock()
defer t.mu.Unlock()
start := sort.Search(len(t.ids), func(i int) bool { return t.ids[i] > after })
return slices.Clone(t.ids[start:min(start+limit, len(t.ids))]), nil
}
// do reads one file and keeps its fingerprint, if the file is still the one the scan saw.
func (t *hashTask) do(ctx context.Context, fileID int64, _ bool) (string, error) {
rec, err := t.s.store.FileByID(ctx, fileID)
if errors.Is(err, ErrTrackNotFound) || err == nil && rec.Missing {
return "", jobs.ErrSkip
}
if err != nil {
return "", err
}
root, err := openRoot(rec.Root)
if err != nil {
return "", err // a disk that is not mounted: every file fails the same way, and the job gives up
}
defer root.Close()
rel, err := relPath(rec)
if err != nil {
return "", jobs.ErrSkip
}
if err := sameAsScanned(root, rel, rec); errors.Is(err, ErrChanged) {
return "", jobs.ErrSkip
} else if err != nil {
return "", err
}
f, err := root.Open(rel)
if err != nil {
return "", fmt.Errorf("open the file: %w", err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, &contextReader{ctx: ctx, r: f}); err != nil {
return "", fmt.Errorf("read the file: %w", err)
}
saved, err := t.s.store.SaveHash(ctx, fileID, rec.SizeBytes, rec.MTimeUnix, hex.EncodeToString(h.Sum(nil)))
if err != nil {
return "", err
}
if !saved {
return "", jobs.ErrSkip // it changed while it was read
}
return OutcomeHashed, nil
}
// contextReader stops reading when the job is stopped, instead of finishing a long file first.
type contextReader struct {
ctx context.Context
r *os.File
}
func (c *contextReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}

View file

@ -0,0 +1,236 @@
package duplicates
import (
"context"
"fmt"
"os"
"slices"
"time"
"go.uber.org/zap"
)
// Store is what the service reads and writes in the index. internal/storage implements it.
type Store interface {
Candidates(ctx context.Context) ([]Candidate, error)
FileOf(ctx context.Context, trackID int64) (FileRecord, error)
FileByID(ctx context.Context, fileID int64) (FileRecord, error)
SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error)
RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error
}
// Options say what the service may do.
type Options struct {
// AllowDelete is library.allow_delete. Without it nothing is ever removed.
AllowDelete bool
// ScanRunning says whether a scan is running; a removal waits for it to end. Nil means never.
ScanRunning func() bool
// Logger records each removal, by track and without the path: audio paths are private.
Logger *zap.Logger
}
// Service finds duplicate songs and removes the extra copies.
type Service struct {
store Store
allow bool
scanning func() bool
log *zap.Logger
now func() time.Time
}
// NewService builds a Service.
func NewService(store Store, o Options) *Service {
log := o.Logger
if log == nil {
log = zap.NewNop()
}
return &Service{store: store, allow: o.AllowDelete, scanning: o.ScanRunning, log: log, now: time.Now}
}
// Page is some of the groups, with how many there are in all.
type Page struct {
Groups []Group
Total int
// DeletionAllowed says whether Clean can work: with it off, the groups can still be seen.
DeletionAllowed bool
}
// Groups returns the groups of copies in the library, worked out from the index and never from the disk.
func (s *Service) Groups(ctx context.Context, limit, offset int) (Page, error) {
all, err := s.groups(ctx)
if err != nil {
return Page{}, err
}
page := Page{Total: len(all), DeletionAllowed: s.allow}
if offset < len(all) {
page.Groups = all[offset:min(offset+limit, len(all))]
}
return page, nil
}
func (s *Service) groups(ctx context.Context) ([]Group, error) {
candidates, err := s.store.Candidates(ctx)
if err != nil {
return nil, err
}
return Build(candidates, DurationTolerance), nil
}
// maxRemovals is the most copies of one song that one request may remove.
const maxRemovals = 100
// Result says what a removal did.
type Result struct {
Removed int
FreedBytes int64
}
// Clean removes the copies `remove` of a song from the disk and the index, keeping `keep`. Nothing
// is removed unless every check passes, and the checks are made here, again, whatever the client
// saw: deleting is allowed, no scan is running, the tracks are copies of one song by the rule of
// Build, the copy to keep is still the file that was scanned (so it is never the last copy that is
// removed), and so is every copy to remove (a file that changed, a link, or a path that leaves the
// library is refused). The files are checked all together before the first is removed. Each removed
// file takes its track out of the index, and its listens go to the track that stays.
//
// On an error part way (the folder turned out to be read-only on the second file), the Result says
// what was already removed.
func (s *Service) Clean(ctx context.Context, keep int64, remove []int64) (Result, error) {
var res Result
if !s.allow {
return res, ErrDisabled
}
if s.scanning != nil && s.scanning() {
return res, ErrScanRunning
}
if err := checkRequest(keep, remove); err != nil {
return res, err
}
keepFile, err := s.store.FileOf(ctx, keep)
if err != nil {
return res, err
}
files := make(map[int64]FileRecord, len(remove))
for _, id := range remove {
rec, err := s.store.FileOf(ctx, id)
if err != nil {
return res, err
}
files[id] = rec
}
if keepFile.Missing {
return res, fmt.Errorf("%w: the copy to keep is flagged as missing", ErrChanged)
}
for _, rec := range files {
if rec.Missing {
return res, fmt.Errorf("%w: a copy to remove is flagged as missing", ErrChanged)
}
}
if err := s.checkSameSong(ctx, keep, remove); err != nil {
return res, err
}
// Look at every file before touching any.
roots := map[string]*os.Root{}
defer func() {
for _, r := range roots {
_ = r.Close()
}
}()
rootOf := func(path string) (*os.Root, error) {
if r, ok := roots[path]; ok {
return r, nil
}
r, err := openRoot(path)
if err != nil {
return nil, err
}
roots[path] = r
return r, nil
}
inspect := func(rec FileRecord) (*os.Root, string, error) {
root, err := rootOf(rec.Root)
if err != nil {
return nil, "", err
}
rel, err := relPath(rec)
if err != nil {
return nil, "", err
}
return root, rel, sameAsScanned(root, rel, rec)
}
if _, _, err := inspect(keepFile); err != nil {
return res, err
}
type target struct {
id int64
root *os.Root
rel string
}
targets := make([]target, 0, len(remove))
for _, id := range remove {
root, rel, err := inspect(files[id])
if err != nil {
return res, err
}
targets = append(targets, target{id, root, rel})
}
for _, t := range targets {
if err := removeFile(t.root, t.rel); err != nil {
return res, err
}
// The file is gone. If the index cannot follow, the next scan flags the path as missing, and
// nothing is left broken; the error says so.
if err := s.store.RemoveCopy(ctx, keep, t.id); err != nil {
s.log.Error("a duplicate file was removed but its track is still in the index; a scan will flag it as missing",
zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep), zap.Error(err))
return res, fmt.Errorf("the file was removed but the index could not be updated: %w", err)
}
res.Removed++
res.FreedBytes += files[t.id].SizeBytes
s.log.Info("a duplicate file was removed", zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep))
}
return res, nil
}
// checkRequest refuses a request that makes no sense, before anything is read: nothing to remove,
// too much, the copy to keep among those to remove, or a copy named twice.
func checkRequest(keep int64, remove []int64) error {
if len(remove) == 0 || len(remove) > maxRemovals {
return fmt.Errorf("%w: name between 1 and %d copies to remove", ErrInvalid, maxRemovals)
}
seen := map[int64]bool{}
for _, id := range remove {
if id == keep {
return fmt.Errorf("%w: the copy to keep is among those to remove", ErrInvalid)
}
if seen[id] {
return fmt.Errorf("%w: a copy is named twice", ErrInvalid)
}
seen[id] = true
}
return nil
}
// checkSameSong works out the groups again and checks that every copy to remove is in the group of
// the copy to keep.
func (s *Service) checkSameSong(ctx context.Context, keep int64, remove []int64) error {
groups, err := s.groups(ctx)
if err != nil {
return err
}
for _, g := range groups {
if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == keep }) {
continue
}
for _, id := range remove {
if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == id }) {
return ErrInvalid
}
}
return nil
}
return ErrInvalid
}

View file

@ -0,0 +1,497 @@
package duplicates_test
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/storage"
"codeberg.org/squidd/sonda/migrations"
)
// env is a real library on a real folder, indexed in a real database.
type env struct {
t *testing.T
ctx context.Context
db *bun.DB
dir string
libID int64
lib *storage.LibraryRepo
store *storage.DuplicatesRepo
scanning bool
}
func newEnv(t *testing.T) *env {
t.Helper()
db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
fsys, err := migrations.For(storage.DriverSQLite)
if err != nil {
t.Fatal(err)
}
if err := storage.Migrate(context.Background(), db, fsys); err != nil {
t.Fatal(err)
}
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)}
l, err := e.lib.CreateLibrary(e.ctx, "music", dir)
if err != nil {
t.Fatal(err)
}
e.libID = l.ID
return e
}
func (e *env) service(allow bool) *duplicates.Service {
return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()})
}
// add writes a file and indexes it exactly as it is on the disk, and returns its track.
func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
path := filepath.Join(e.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
e.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
e.t.Fatal(err)
}
return e.index(rel, path, title, album, durationMs, format, bitrate, artists...)
}
// index puts a file that is on the disk into the index with the size and date it has.
func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
info, err := os.Stat(path)
if err != nil {
e.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil {
e.t.Fatal(err)
}
var id int64
if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
e.t.Fatal(err)
}
return id
}
func (e *env) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel)))
return err == nil
}
func (e *env) tracks() int {
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil {
e.t.Fatal(err)
}
return n
}
// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums.
func twoCopies(e *env) (flac, mp3 int64) {
flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
// --- finding ---------------------------------------------------------------------------------------
func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
page, err := e.service(true).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed {
t.Fatalf("page = %+v, want one group and deleting allowed", page)
}
g := page.Groups[0]
if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) {
t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g)
}
if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" {
t.Errorf("files = %+v", g.Files)
}
}
func TestGroupsReportsThatDeletingIsOff(t *testing.T) {
e := newEnv(t)
twoCopies(e)
page, err := e.service(false).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.DeletionAllowed || len(page.Groups) != 1 {
t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page)
}
}
func TestGroupsArePaged(t *testing.T) {
e := newEnv(t)
for _, title := range []string{"A song", "B song", "C song"} {
e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
svc := e.service(true)
first, _ := svc.Groups(e.ctx, 2, 0)
second, _ := svc.Groups(e.ctx, 2, 2)
if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 {
t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total)
}
if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" {
t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title)
}
if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 {
t.Errorf("past the end: %+v", beyond)
}
}
// --- fingerprints ------------------------------------------------------------------------------------
func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status {
t.Helper()
if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if st := j.Status(); !st.Running {
return st
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the job did not finish")
return jobs.Status{}
}
func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) {
e := newEnv(t)
e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann")
e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann")
e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann")
svc := e.service(true)
st := runJob(t, duplicates.NewHashJob(svc), false)
if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" {
t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st)
}
page, _ := svc.Groups(e.ctx, 10, 0)
byTitle := map[string]duplicates.Group{}
for _, g := range page.Groups {
byTitle[g.Title] = g
}
if g := byTitle["Same"]; !g.Hashed || !g.Identical {
t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical)
}
if g := byTitle["Alike"]; !g.Hashed || g.Identical {
t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
}
}
func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) {
e := newEnv(t)
twoCopies(e)
job := duplicates.NewHashJob(e.service(true))
if st := runJob(t, job, false); st.Last.Checked != 2 {
t.Fatalf("first run checked %d, want 2", st.Last.Checked)
}
if st := runJob(t, job, false); st.Last.Checked != 0 {
t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked)
}
if st := runJob(t, job, true); st.Last.Checked != 2 {
t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked)
}
}
func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 {
t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last)
}
page, _ := e.service(true).Groups(e.ctx, 10, 0)
if page.Groups[0].Hashed {
t.Error("the group claims every file was read, but one was not")
}
}
func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 {
t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last)
}
}
// --- removing -----------------------------------------------------------------------------------------
func TestWithDeletingOffNothingIsTouched(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
_, err := e.service(false).Clean(e.ctx, flac, []int64{mp3})
if !errors.Is(err, duplicates.ErrDisabled) {
t.Fatalf("got %v, want ErrDisabled", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
func TestNothingIsRemovedWhileAScanRuns(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.scanning = true
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) {
t.Fatalf("got %v, want ErrScanRunning", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed during a scan")
}
}
func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3})
if err != nil {
t.Fatal(err)
}
if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) {
t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if !e.exists("a/song.flac") {
t.Error("the copy to keep is gone")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1", e.tracks())
}
if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 {
t.Errorf("%d groups are left, want none", page.Total)
}
if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("the removed track is still known: %v", err)
}
}
func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c})
if err != nil {
t.Fatal(err)
}
if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") {
t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac"))
}
}
func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} {
if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// What the client sends is not trusted: the server works out again that these are copies of one song.
func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob")
live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann")
for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} {
if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} {
if !e.exists(rel) {
t.Errorf("%s was removed", rel)
}
}
}
func TestATrackThatDoesNotExistIsRefused(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err)
}
if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed")
}
}
// If the copy to keep is not there any more, the other one is the last, and the last is never removed.
func TestTheLastCopyIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("the only copy left was removed")
}
}
func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
// c is edited after the scan: it is not the file the admin saw.
if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
// Nothing at all was removed, not even b, which was fine: every file is checked before any is removed.
if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 {
t.Error("something was removed although a file in the request had changed")
}
}
func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
target := filepath.Join(e.dir, "precious.txt")
if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Skipf("no symlinks here: %v", err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("precious.txt") || !e.exists("b/song.mp3") {
t.Error("the link or what it points at was removed")
}
}
// A path that was recorded as leaving the library is never followed out of it.
func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir))
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(outside) })
victim := filepath.Join(outside, "song.mp3")
if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil {
t.Fatal(err)
}
rel := "../" + filepath.Base(outside) + "/song.mp3"
escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann")
if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil {
t.Fatal("a file outside the library was removed without an error")
}
if _, err := os.Stat(victim); err != nil {
t.Errorf("the file outside the library was removed: %v", err)
}
}
func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
e := newEnv(t)
flac, mp3 := twoCopies(e)
dir := filepath.Join(e.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) {
t.Fatalf("got %v, want ErrReadOnly", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// Listening history must survive: it moves to the copy that stays.
func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false)
if err != nil {
t.Fatal(err)
}
plays := storage.NewListeningRepo(e.db)
for i, id := range []int64{mp3, mp3, flac} {
ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))}
if _, err := plays.Record(e.ctx, ev); err != nil {
t.Fatal(err)
}
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil {
t.Fatal(err)
}
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 {
t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err)
}
}