sonda/internal/duplicates/service_test.go
tgiachi a880b87a9e chore: move the module path to git.kmalloc.org/squid/sonda
Migrating hosting from Codeberg to git.kmalloc.org. Renames the
module path in go.mod, every internal import, and the OCI image
source label in the Dockerfile. Build and full test suite verified
green after the rename.
2026-11-15 17:30:29 +01:00

531 lines
19 KiB
Go

package duplicates_test
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"git.kmalloc.org/squid/sonda/internal/duplicates"
"git.kmalloc.org/squid/sonda/internal/jobs"
"git.kmalloc.org/squid/sonda/internal/library"
"git.kmalloc.org/squid/sonda/internal/listening"
"git.kmalloc.org/squid/sonda/internal/storage"
"git.kmalloc.org/squid/sonda/migrations"
)
// env is a real library on a real folder, indexed in a real database.
type env struct {
t *testing.T
ctx context.Context
db *bun.DB
dir string
libID int64
lib *storage.LibraryRepo
store *storage.DuplicatesRepo
scanning bool
}
func newEnv(t *testing.T) *env {
t.Helper()
db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
fsys, err := migrations.For(storage.DriverSQLite)
if err != nil {
t.Fatal(err)
}
if err := storage.Migrate(context.Background(), db, fsys); err != nil {
t.Fatal(err)
}
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)}
l, err := e.lib.CreateLibrary(e.ctx, "music", dir)
if err != nil {
t.Fatal(err)
}
e.libID = l.ID
return e
}
func (e *env) service(allow bool) *duplicates.Service {
return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()})
}
// add writes a file and indexes it exactly as it is on the disk, and returns its track.
func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
path := filepath.Join(e.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
e.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
e.t.Fatal(err)
}
return e.index(rel, path, title, album, durationMs, format, bitrate, artists...)
}
// index puts a file that is on the disk into the index with the size and date it has.
func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
e.t.Helper()
info, err := os.Stat(path)
if err != nil {
e.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil {
e.t.Fatal(err)
}
var id int64
if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
e.t.Fatal(err)
}
return id
}
func (e *env) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel)))
return err == nil
}
func (e *env) tracks() int {
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil {
e.t.Fatal(err)
}
return n
}
// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums.
func twoCopies(e *env) (flac, mp3 int64) {
flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
// --- finding ---------------------------------------------------------------------------------------
func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
page, err := e.service(true).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed {
t.Fatalf("page = %+v, want one group and deleting allowed", page)
}
g := page.Groups[0]
if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) {
t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g)
}
if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" {
t.Errorf("files = %+v", g.Files)
}
}
func TestGroupsReportsThatDeletingIsOff(t *testing.T) {
e := newEnv(t)
twoCopies(e)
page, err := e.service(false).Groups(e.ctx, 10, 0)
if err != nil {
t.Fatal(err)
}
if page.DeletionAllowed || len(page.Groups) != 1 {
t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page)
}
}
func TestGroupsArePaged(t *testing.T) {
e := newEnv(t)
for _, title := range []string{"A song", "B song", "C song"} {
e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
svc := e.service(true)
first, _ := svc.Groups(e.ctx, 2, 0)
second, _ := svc.Groups(e.ctx, 2, 2)
if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 {
t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total)
}
if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" {
t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title)
}
if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 {
t.Errorf("past the end: %+v", beyond)
}
}
// --- fingerprints ------------------------------------------------------------------------------------
func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status {
t.Helper()
if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if st := j.Status(); !st.Running {
return st
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the job did not finish")
return jobs.Status{}
}
func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) {
e := newEnv(t)
e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann")
e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann")
e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann")
svc := e.service(true)
st := runJob(t, duplicates.NewHashJob(svc), false)
if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" {
t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st)
}
page, _ := svc.Groups(e.ctx, 10, 0)
byTitle := map[string]duplicates.Group{}
for _, g := range page.Groups {
byTitle[g.Title] = g
}
if g := byTitle["Same"]; !g.Hashed || !g.Identical {
t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical)
}
if g := byTitle["Alike"]; !g.Hashed || g.Identical {
t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
}
}
func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) {
e := newEnv(t)
twoCopies(e)
job := duplicates.NewHashJob(e.service(true))
if st := runJob(t, job, false); st.Last.Checked != 2 {
t.Fatalf("first run checked %d, want 2", st.Last.Checked)
}
if st := runJob(t, job, false); st.Last.Checked != 0 {
t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked)
}
if st := runJob(t, job, true); st.Last.Checked != 2 {
t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked)
}
}
func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 {
t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last)
}
page, _ := e.service(true).Groups(e.ctx, 10, 0)
if page.Groups[0].Hashed {
t.Error("the group claims every file was read, but one was not")
}
}
func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) {
e := newEnv(t)
twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
st := runJob(t, duplicates.NewHashJob(e.service(true)), false)
if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 {
t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last)
}
}
// --- removing -----------------------------------------------------------------------------------------
func TestWithDeletingOffNothingIsTouched(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
_, err := e.service(false).Clean(e.ctx, flac, []int64{mp3})
if !errors.Is(err, duplicates.ErrDisabled) {
t.Fatalf("got %v, want ErrDisabled", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
func TestNothingIsRemovedWhileAScanRuns(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
e.scanning = true
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) {
t.Fatalf("got %v, want ErrScanRunning", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed during a scan")
}
}
func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3})
if err != nil {
t.Fatal(err)
}
if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) {
t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if !e.exists("a/song.flac") {
t.Error("the copy to keep is gone")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1", e.tracks())
}
if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 {
t.Errorf("%d groups are left, want none", page.Total)
}
if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("the removed track is still known: %v", err)
}
}
func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c})
if err != nil {
t.Fatal(err)
}
if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") {
t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac"))
}
}
func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} {
if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// What the client sends is not trusted: the server works out again that these are copies of one song.
func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann")
otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob")
live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann")
for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} {
if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) {
t.Errorf("%s: got %v, want ErrInvalid", name, err)
}
}
for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} {
if !e.exists(rel) {
t.Errorf("%s was removed", rel)
}
}
}
func TestATrackThatDoesNotExistIsRefused(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err)
}
if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) {
t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err)
}
if !e.exists("b/song.mp3") {
t.Error("a file was removed")
}
}
// If the copy to keep is not there any more, the other one is the last, and the last is never removed.
func TestTheLastCopyIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("the only copy left was removed")
}
}
func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) {
e := newEnv(t)
keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann")
b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann")
c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann")
// c is edited after the scan: it is not the file the admin saw.
if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
// Nothing at all was removed, not even b, which was fine: every file is checked before any is removed.
if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 {
t.Error("something was removed although a file in the request had changed")
}
}
func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
target := filepath.Join(e.dir, "precious.txt")
if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil {
t.Skipf("no symlinks here: %v", err)
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) {
t.Fatalf("got %v, want ErrChanged", err)
}
if !e.exists("precious.txt") || !e.exists("b/song.mp3") {
t.Error("the link or what it points at was removed")
}
}
// A path that was recorded as leaving the library is never followed out of it.
func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) {
e := newEnv(t)
flac, _ := twoCopies(e)
outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir))
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(outside) })
victim := filepath.Join(outside, "song.mp3")
if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil {
t.Fatal(err)
}
rel := "../" + filepath.Base(outside) + "/song.mp3"
escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann")
if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil {
t.Fatal("a file outside the library was removed without an error")
}
if _, err := os.Stat(victim); err != nil {
t.Errorf("the file outside the library was removed: %v", err)
}
}
func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
e := newEnv(t)
flac, mp3 := twoCopies(e)
dir := filepath.Join(e.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) {
t.Fatalf("got %v, want ErrReadOnly", err)
}
if !e.exists("b/song.mp3") || e.tracks() != 2 {
t.Error("something was removed")
}
}
// Listening history must survive: it moves to the copy that stays.
func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false)
if err != nil {
t.Fatal(err)
}
plays := storage.NewListeningRepo(e.db)
for i, id := range []int64{mp3, mp3, flac} {
ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))}
if _, err := plays.Record(e.ctx, ev); err != nil {
t.Fatal(err)
}
}
if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil {
t.Fatal(err)
}
var n int
if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 {
t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err)
}
}
// cancelsBeforeUpdating ends the request at the worst moment: after the file was deleted, when the
// index is about to follow. The context the service hands to the index is the one that is cancelled,
// unless the service has made sure that what it began is finished.
type cancelsBeforeUpdating struct {
duplicates.Store
cancel context.CancelFunc
}
func (s cancelsBeforeUpdating) RemoveCopy(ctx context.Context, keep, remove int64) error {
s.cancel()
return s.Store.RemoveCopy(ctx, keep, remove)
}
// A client that disconnects once the file is deleted must not leave the index behind: the file is gone,
// so the track has to go too, whether or not anyone is still waiting for the answer.
func TestTheIndexIsUpdatedEvenIfTheClientLeavesAfterTheFileIsDeleted(t *testing.T) {
e := newEnv(t)
flac, mp3 := twoCopies(e)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
svc := duplicates.NewService(cancelsBeforeUpdating{Store: e.store, cancel: cancel},
duplicates.Options{AllowDelete: true, Logger: zap.NewNop()})
if _, err := svc.Clean(ctx, flac, []int64{mp3}); err != nil {
t.Fatalf("got %v: the request was cancelled, but the work had begun and has to be finished", err)
}
if e.exists("b/song.mp3") {
t.Error("the copy is still on the disk")
}
if e.tracks() != 1 {
t.Errorf("%d tracks, want 1: the file is gone but its track is still in the index", e.tracks())
}
}