sonda/internal/duplicates/service_test.go
tgiachi 05f4e33470 fix(duplicates): finish updating the index even if the client leaves after the file is deleted
Found reviewing the branch: a request cancelled between deleting the file and updating the index left a track whose file does not exist. The index step no longer depends on the request's context. Test first: it cancels the request at exactly that moment.
2026-11-12 16:52:38 +01:00

531 lines
19 KiB
Go

package duplicates_test
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/listening"
"codeberg.org/squidd/sonda/internal/storage"
"codeberg.org/squidd/sonda/migrations"
)
// env is a real library on a real folder, indexed in a real database.
type env struct {
t *testing.T
ctx context.Context
db *bun.DB
dir string
libID int64
lib *storage.LibraryRepo
store *storage.DuplicatesRepo
scanning bool
}
func newEnv(t *testing.T) *env {
t.Helper()
db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
fsys, err := migrations.For(storage.DriverSQLite)
if err != nil {
t.Fatal(err)
}
if err := storage.Migrate(context.Background(), db, fsys); err != nil {
t.Fatal(err)
}
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)}
l, err := e.lib.CreateLibrary(e.ctx, "music", dir)
if err != nil {
t.Fatal(err)
}
e.libID = l.ID
return e
}
func (e *env) service(allow bool) *duplicates.Service {
return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()})
}
// add writes a file and indexes it exactly as it is on the disk, and returns its track.
func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
path := filepath.Join(e.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
e.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
e.t.Fatal(err)
}
return e.index(rel, path, title, album, durationMs, format, bitrate, artists...)
}
// index puts a file that is on the disk into the index with the size and date it has.
func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
info, err := os.Stat(path)
if err != nil {
e.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil {
e.t.Fatal(err)
}
var id int64
if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
e.t.Fatal(err)
}
return id
}
func (e *env) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel)))
return err == nil
}
func (e *env) tracks() int {
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil {
e.t.Fatal(err)
}
return n
}
// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums.
func twoCopies(e *env) (flac, mp3 int64) {
flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
// --- finding ---------------------------------------------------------------------------------------
func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
page, err := e.service(true).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed {
t.Fatalf("page = %+v, want one group and deleting allowed", page)
}
g := page.Groups[0]
if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) {
t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g)
}
if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" {
t.Errorf("files = %+v", g.Files)
}
}
func TestGroupsReportsThatDeletingIsOff(t *testing.T) {
e := newEnv(t)
twoCopies(e)
page, err := e.service(false).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.DeletionAllowed || len(page.Groups) != 1 {
t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page)
}
}
func TestGroupsArePaged(t *testing.T) {
e := newEnv(t)
for _, title := range []string{"A song", "B song", "C song"} {
e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
svc := e.service(true)
first, _ := svc.Groups(e.ctx, 2, 0)
second, _ := svc.Groups(e.ctx, 2, 2)
if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 {
t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total)
}
if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" {
t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title)
}
if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 {
t.Errorf("past the end: %+v", beyond)
}
}
// --- fingerprints ------------------------------------------------------------------------------------
func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status {
t.Helper()
if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if st := j.Status(); !st.Running {
return st
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the job did not finish")
return jobs.Status{}
}
func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) {
e := newEnv(t)
e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann")
e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann")
e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann")
svc := e.service(true)
st := runJob(t, duplicates.NewHashJob(svc), false)
if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" {
t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st)
}
page, _ := svc.Groups(e.ctx, 10, 0)
byTitle := map[string]duplicates.Group{}
for _, g := range page.Groups {
byTitle[g.Title] = g
}
if g := byTitle["Same"]; !g.Hashed || !g.Identical {
t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical)
}
if g := byTitle["Alike"]; !g.Hashed || g.Identical {
t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
}
}
func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) {
e := newEnv(t)
twoCopies(e)
job := duplicates.NewHashJob(e.service(true))
if st := runJob(t, job, false); st.Last.Checked != 2 {
t.Fatalf("first run checked %d, want 2", st.Last.Checked)
}
if st := runJob(t, job, false); st.Last.Checked != 0 {
t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked)
}
if st := runJob(t, job, true); st.Last.Checked != 2 {
t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked)
}
}
func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 {
t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last)
}
page, _ := e.service(true).Groups(e.ctx, 10, 0)
if page.Groups[0].Hashed {
t.Error("the group claims every file was read, but one was not")
}
}
func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 {
t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last)
}
}
// --- removing -----------------------------------------------------------------------------------------
func TestWithDeletingOffNothingIsTouched(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
_, err := e.service(false).Clean(e.ctx, flac, []int64{mp3})
if !errors.Is(err, duplicates.ErrDisabled) {
t.Fatalf("got %v, want ErrDisabled", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
func TestNothingIsRemovedWhileAScanRuns(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.scanning = true
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) {
t.Fatalf("got %v, want ErrScanRunning", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed during a scan")
}
}
func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3})
if err != nil {
t.Fatal(err)
}
if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) {
t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if !e.exists("a/song.flac") {
t.Error("the copy to keep is gone")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1", e.tracks())
}
if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 {
t.Errorf("%d groups are left, want none", page.Total)
}
if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("the removed track is still known: %v", err)
}
}
func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c})
if err != nil {
t.Fatal(err)
}
if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") {
t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac"))
}
}
func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} {
if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// What the client sends is not trusted: the server works out again that these are copies of one song.
func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob")
live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann")
for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} {
if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} {
if !e.exists(rel) {
t.Errorf("%s was removed", rel)
}
}
}
func TestATrackThatDoesNotExistIsRefused(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err)
}
if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed")
}
}
// If the copy to keep is not there any more, the other one is the last, and the last is never removed.
func TestTheLastCopyIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("the only copy left was removed")
}
}
func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
// c is edited after the scan: it is not the file the admin saw.
if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
// Nothing at all was removed, not even b, which was fine: every file is checked before any is removed.
if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 {
t.Error("something was removed although a file in the request had changed")
}
}
func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
target := filepath.Join(e.dir, "precious.txt")
if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Skipf("no symlinks here: %v", err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("precious.txt") || !e.exists("b/song.mp3") {
t.Error("the link or what it points at was removed")
}
}
// A path that was recorded as leaving the library is never followed out of it.
func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir))
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(outside) })
victim := filepath.Join(outside, "song.mp3")
if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil {
t.Fatal(err)
}
rel := "../" + filepath.Base(outside) + "/song.mp3"
escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann")
if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil {
t.Fatal("a file outside the library was removed without an error")
}
if _, err := os.Stat(victim); err != nil {
t.Errorf("the file outside the library was removed: %v", err)
}
}
func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
e := newEnv(t)
flac, mp3 := twoCopies(e)
dir := filepath.Join(e.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) {
t.Fatalf("got %v, want ErrReadOnly", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// Listening history must survive: it moves to the copy that stays.
func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false)
if err != nil {
t.Fatal(err)
}
plays := storage.NewListeningRepo(e.db)
for i, id := range []int64{mp3, mp3, flac} {
ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))}
if _, err := plays.Record(e.ctx, ev); err != nil {
t.Fatal(err)
}
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil {
t.Fatal(err)
}
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 {
t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err)
}
}
// cancelsBeforeUpdating ends the request at the worst moment: after the file was deleted, when the
// index is about to follow. The context the service hands to the index is the one that is cancelled,
// unless the service has made sure that what it began is finished.
type cancelsBeforeUpdating struct {
duplicates.Store
cancel context.CancelFunc
}
func (s cancelsBeforeUpdating) RemoveCopy(ctx context.Context, keep, remove int64) error {
s.cancel()
return s.Store.RemoveCopy(ctx, keep, remove)
}
// A client that disconnects once the file is deleted must not leave the index behind: the file is gone,
// so the track has to go too, whether or not anyone is still waiting for the answer.
func TestTheIndexIsUpdatedEvenIfTheClientLeavesAfterTheFileIsDeleted(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
svc := duplicates.NewService(cancelsBeforeUpdating{Store: e.store, cancel: cancel},
duplicates.Options{AllowDelete: true, Logger: zap.NewNop()})
if _, err := svc.Clean(ctx, flac, []int64{mp3}); err != nil {
t.Fatalf("got %v: the request was cancelled, but the work had begun and has to be finished", err)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1: the file is gone but its track is still in the index", e.tracks())
}
}