From 309160951eb758ce23fc5adb71aedf73988225e2 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:33:11 +0100 Subject: [PATCH 1/8] feat(config): library.allow_delete, off by default, for deleting duplicate files The one setting that lets Sonda delete files from the music folders. Off until the operator turns it on, in the file or with SONDA_ALLOW_DELETE; a value that is not true or false is an error, not ignored. --- internal/config/config.go | 13 +++++++++ internal/config/config_test.go | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/internal/config/config.go b/internal/config/config.go index 0d61d69..f20f5bf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "time" "github.com/pelletier/go-toml/v2" @@ -41,6 +42,10 @@ type Server struct { // in the database and are managed through the API. type Library struct { ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan + // AllowDelete lets an admin delete the extra copies of a song from the music folders. It is the + // one thing that makes Sonda write there, so it is off until the operator turns it on (and it + // needs the folders mounted writable). + AllowDelete bool `toml:"allow_delete"` } // Database selects and configures the storage backend. @@ -172,6 +177,14 @@ func Load(path string) (Config, error) { } applyEnv(&cfg) + // A typo here must not be ignored in silence: it decides whether Sonda may delete files. + if v := os.Getenv("SONDA_ALLOW_DELETE"); v != "" { + allow, err := strconv.ParseBool(v) + if err != nil { + return Config{}, fmt.Errorf("SONDA_ALLOW_DELETE must be true or false, got %q", v) + } + cfg.Library.AllowDelete = allow + } if cfg.Storage.DataDir == "" { cfg.Storage.DataDir = defaultDataDir(cfg.Database) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index db8ed49..5f72e19 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -292,3 +292,52 @@ func TestLoadRefusesTheOldLastFMEnvironmentVariable(t *testing.T) { t.Errorf("the error repeats the key: %v", err) } } + +// Deleting files from the music folders is the one thing Sonda does not do unless it is told to. +func TestDeletingIsOffByDefault(t *testing.T) { + cfg, err := Load("") + if err != nil { + t.Fatal(err) + } + if cfg.Library.AllowDelete { + t.Error("library.allow_delete is on by default") + } +} + +func TestAllowDeleteIsReadFromTheFile(t *testing.T) { + cfg, err := Load(writeFile(t, t.TempDir(), "[library]\nallow_delete = true\n")) + if err != nil { + t.Fatal(err) + } + if !cfg.Library.AllowDelete { + t.Error("allow_delete = true in the file was not read") + } +} + +func TestAllowDeleteEnvironmentVariableWinsOverTheFile(t *testing.T) { + p := writeFile(t, t.TempDir(), "[library]\nallow_delete = true\n") + t.Setenv("SONDA_ALLOW_DELETE", "false") + cfg, err := Load(p) + if err != nil { + t.Fatal(err) + } + if cfg.Library.AllowDelete { + t.Error("SONDA_ALLOW_DELETE=false did not turn it off") + } + t.Setenv("SONDA_ALLOW_DELETE", "true") + cfg, err = Load("") + if err != nil { + t.Fatal(err) + } + if !cfg.Library.AllowDelete { + t.Error("SONDA_ALLOW_DELETE=true did not turn it on") + } +} + +// A typo in a setting that lets Sonda delete files must not be ignored in silence. +func TestAllowDeleteEnvironmentVariableMustBeABoolean(t *testing.T) { + t.Setenv("SONDA_ALLOW_DELETE", "yes please") + if _, err := Load(""); err == nil || !strings.Contains(err.Error(), "SONDA_ALLOW_DELETE") { + t.Errorf("got %v, want an error naming SONDA_ALLOW_DELETE", err) + } +} From 7e1e63bdd2cda33c334e3a32c678548b533f0409 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:34:12 +0100 Subject: [PATCH 2/8] feat(duplicates): group the copies of a song and propose the one to keep The same song is the same set of main artists, the same title and durations within 3 seconds, every pair within it; a file with no duration or no artist never matches. A group says whether its files were all read and are byte for byte the same. The copy to keep is the lossless one, then the higher bitrate, then the bigger file, then the oldest, whatever the order of the input. --- internal/duplicates/group.go | 175 ++++++++++++++++++++++++++ internal/duplicates/group_test.go | 198 ++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 internal/duplicates/group.go create mode 100644 internal/duplicates/group_test.go diff --git a/internal/duplicates/group.go b/internal/duplicates/group.go new file mode 100644 index 0000000..52e7dfd --- /dev/null +++ b/internal/duplicates/group.go @@ -0,0 +1,175 @@ +// Package duplicates finds the same song by the same artist in more than one file, and removes +// the extra copies when an admin says so. Finding only reads the index. Removing is the one thing +// Sonda does to the music folders, so it is off until the operator turns it on, and every +// removal is checked again at the moment it is made. +package duplicates + +import ( + "slices" + "strconv" + "strings" + "time" +) + +// DurationTolerance is how far apart two durations may be for two files to be the same song. It +// keeps a live version that is minutes longer apart from the studio one. +const DurationTolerance = 3 * time.Second + +// ArtistRef names an artist. +type ArtistRef struct { + ID int64 + Name string +} + +// Candidate is one playable file as the grouping sees it: its track, its main artists and the +// facts that tell the copies apart. +type Candidate struct { + TrackID int64 + FileID int64 + Artists []ArtistRef // the main artists of the track + Title string + TitleKey string // the key Sonda matches titles by + DurationMs int64 + AlbumID int64 + Album string + Format string + BitRate int + SizeBytes int64 + Hash string // SHA-256 of the file, empty until it has been read +} + +// Group is a song that is in more than one file. +type Group struct { + Artists []ArtistRef + Title string + Files []Candidate // by track id + // Hashed is true when every file has been read; Identical when, besides, they are byte for byte the same. + Hashed, Identical bool + // KeepTrackID is the copy to keep, by Suggest; FreedBytes is the size of all the others. + KeepTrackID int64 + FreedBytes int64 +} + +// artistsKey identifies a set of artists, whatever the order they are credited in. +func artistsKey(artists []ArtistRef) string { + ids := make([]int64, len(artists)) + for i, a := range artists { + ids[i] = a.ID + } + slices.Sort(ids) + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = strconv.FormatInt(id, 10) + } + return strings.Join(parts, ",") +} + +// Build finds the groups. Two files are the same song when they have the same set of main +// artists, the same title and durations no more than tolerance apart (every pair in a group, not +// a chain of near ones). A file with no known duration or no artist never matches. The groups +// and the files in them come in a fixed order, whatever the order of cands. +func Build(cands []Candidate, tolerance time.Duration) []Group { + buckets := map[string][]Candidate{} + for _, c := range cands { + if c.DurationMs <= 0 || len(c.Artists) == 0 { + continue + } + key := artistsKey(c.Artists) + "\x00" + c.TitleKey + buckets[key] = append(buckets[key], c) + } + + tolMs := tolerance.Milliseconds() + var groups []Group + for _, bucket := range buckets { + if len(bucket) < 2 { + continue + } + slices.SortFunc(bucket, func(a, b Candidate) int { + if a.DurationMs != b.DurationMs { + return int(a.DurationMs - b.DurationMs) + } + return int(a.TrackID - b.TrackID) + }) + for i := 0; i < len(bucket); { + j := i + 1 + for j < len(bucket) && bucket[j].DurationMs-bucket[i].DurationMs <= tolMs { + j++ + } + if j-i >= 2 { + groups = append(groups, newGroup(bucket[i:j])) + } + i = j + } + } + + slices.SortFunc(groups, func(a, b Group) int { + if c := strings.Compare(a.Files[0].TitleKey, b.Files[0].TitleKey); c != 0 { + return c + } + if c := strings.Compare(artistsKey(a.Artists), artistsKey(b.Artists)); c != 0 { + return c + } + return int(a.Files[0].TrackID - b.Files[0].TrackID) + }) + return groups +} + +func newGroup(files []Candidate) Group { + files = slices.Clone(files) + slices.SortFunc(files, func(a, b Candidate) int { return int(a.TrackID - b.TrackID) }) + artists := slices.Clone(files[0].Artists) + slices.SortFunc(artists, func(a, b ArtistRef) int { return int(a.ID - b.ID) }) + + g := Group{Artists: artists, Title: files[0].Title, Files: files, KeepTrackID: Suggest(files)} + g.Hashed = true + for _, f := range files { + if f.Hash == "" { + g.Hashed = false + } + if f.TrackID != g.KeepTrackID { + g.FreedBytes += f.SizeBytes + } + } + g.Identical = g.Hashed + for _, f := range files[1:] { + if f.Hash != files[0].Hash { + g.Identical = false + } + } + return g +} + +// lossless says whether a format keeps everything the recording had. +func lossless(format string) bool { + switch strings.ToLower(format) { + case "flac", "wav", "alac", "ape", "wv", "aiff": + return true + } + return false +} + +// Suggest picks the copy to keep: one without loss before one with loss, then the higher bitrate, +// then the bigger file, and at a tie the oldest track. The answer does not depend on the order. +func Suggest(files []Candidate) int64 { + best := files[0] + for _, f := range files[1:] { + if better(f, best) { + best = f + } + } + return best.TrackID +} + +// better says whether a should be kept over b. +func better(a, b Candidate) bool { + if la, lb := lossless(a.Format), lossless(b.Format); la != lb { + return la + } + if a.BitRate != b.BitRate { + return a.BitRate > b.BitRate + } + if a.SizeBytes != b.SizeBytes { + return a.SizeBytes > b.SizeBytes + } + return a.TrackID < b.TrackID +} diff --git a/internal/duplicates/group_test.go b/internal/duplicates/group_test.go new file mode 100644 index 0000000..8df408d --- /dev/null +++ b/internal/duplicates/group_test.go @@ -0,0 +1,198 @@ +package duplicates + +import ( + "reflect" + "testing" + "time" +) + +const tol = 3 * time.Second + +func cand(track int64, title string, durationMs int64, artists ...int64) Candidate { + refs := make([]ArtistRef, len(artists)) + for i, id := range artists { + refs[i] = ArtistRef{ID: id, Name: "Artist"} + } + return Candidate{ + TrackID: track, FileID: track * 10, Artists: refs, Title: title, TitleKey: title, + DurationMs: durationMs, AlbumID: 1, Album: "Album", Format: "mp3", BitRate: 192, SizeBytes: 5_000_000, + } +} + +func trackIDs(g Group) []int64 { + var out []int64 + for _, f := range g.Files { + out = append(out, f.TrackID) + } + return out +} + +func TestTheSameSongBySameArtistsIsAGroup(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 200_000, 7), + cand(2, "song", 201_000, 7), + }, tol) + if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) { + t.Fatalf("groups = %+v, want one group of tracks 1 and 2", groups) + } +} + +func TestASongThatIsAloneIsNotAGroup(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7)}, tol); len(got) != 0 { + t.Errorf("a single copy made a group: %+v", got) + } +} + +func TestSameTitleByAnotherArtistIsAnotherSong(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 8)}, tol); len(got) != 0 { + t.Errorf("two artists made a group: %+v", got) + } +} + +func TestTheSetOfArtistsMustBeTheSame(t *testing.T) { + // A duet is not the song of one of the two. + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7, 8)}, tol); len(got) != 0 { + t.Errorf("a solo and a duet made a group: %+v", got) + } + // The order in which the artists are credited does not matter. + got := Build([]Candidate{cand(1, "song", 200_000, 7, 8), cand(2, "song", 200_000, 8, 7)}, tol) + if len(got) != 1 { + t.Errorf("the same artists in another order did not make a group: %+v", got) + } +} + +func TestADifferentTitleIsAnotherSong(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song (live)", 200_000, 7)}, tol); len(got) != 0 { + t.Errorf("two titles made a group: %+v", got) + } +} + +func TestDurationsMustBeWithinTheTolerance(t *testing.T) { + // 3 seconds apart is the same song, 3.001 is not. + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_000, 7)}, tol); len(got) != 1 { + t.Errorf("3 s apart: %+v, want a group", got) + } + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_001, 7)}, tol); len(got) != 0 { + t.Errorf("3.001 s apart: %+v, want none", got) + } +} + +func TestALongLiveVersionIsNotMixedWithTheStudioOnes(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 220_000, 7), + cand(2, "song", 221_000, 7), + cand(3, "song", 370_000, 7), // the same title, but 2.5 minutes longer + }, tol) + if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) { + t.Fatalf("groups = %+v, want only tracks 1 and 2 together", groups) + } +} + +// Every pair in a group is within the tolerance, so a chain of songs each a little longer than the last +// does not become one group. +func TestAChainOfSlightlyLongerCopiesDoesNotGrowOneGroup(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 200_000, 7), + cand(2, "song", 202_000, 7), + cand(3, "song", 204_000, 7), + cand(4, "song", 206_000, 7), + }, tol) + for _, g := range groups { + ids := trackIDs(g) + var lo, hi int64 = 1 << 60, 0 + for _, f := range g.Files { + lo, hi = min(lo, f.DurationMs), max(hi, f.DurationMs) + } + if hi-lo > 3000 { + t.Errorf("group %v spans %d ms", ids, hi-lo) + } + } +} + +func TestATrackWithNoDurationOrNoArtistNeverMatches(t *testing.T) { + noDuration := cand(2, "song", 0, 7) + noArtist := cand(3, "song", 200_000) + if got := Build([]Candidate{cand(1, "song", 200_000, 7), noDuration, noArtist}, tol); len(got) != 0 { + t.Errorf("groups = %+v, want none", got) + } +} + +func TestTheSameSongOnDifferentAlbumsIsAGroup(t *testing.T) { + other := cand(2, "song", 200_000, 7) + other.AlbumID, other.Album = 2, "Greatest hits" + groups := Build([]Candidate{cand(1, "song", 200_000, 7), other}, tol) + if len(groups) != 1 || groups[0].Files[0].Album == groups[0].Files[1].Album { + t.Fatalf("groups = %+v, want one group with two albums", groups) + } +} + +func TestSeveralGroupsComeInAStableOrder(t *testing.T) { + a := []Candidate{cand(1, "b song", 200_000, 7), cand(2, "b song", 200_000, 7), cand(3, "a song", 100_000, 7), cand(4, "a song", 100_000, 7)} + b := []Candidate{a[3], a[2], a[1], a[0]} + first, second := Build(a, tol), Build(b, tol) + if !reflect.DeepEqual(first, second) { + t.Errorf("the order of the input changes the answer:\n%+v\n%+v", first, second) + } + if first[0].Title != "a song" { + t.Errorf("first group is %q, want the titles in order", first[0].Title) + } +} + +func TestAGroupIsIdenticalOnlyWhenEveryFileHasTheSameHash(t *testing.T) { + x, y := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7) + if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical { + t.Errorf("no hashes: hashed=%v identical=%v, want both false", g.Hashed, g.Identical) + } + x.Hash = "aa" + if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical { + t.Errorf("one hash of two: hashed=%v identical=%v, want both false", g.Hashed, g.Identical) + } + y.Hash = "aa" + if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || !g.Identical { + t.Errorf("same hashes: hashed=%v identical=%v, want both true", g.Hashed, g.Identical) + } + y.Hash = "bb" + if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || g.Identical { + t.Errorf("different hashes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical) + } +} + +func TestFreedBytesAreTheSizeOfEverythingButTheCopyToKeep(t *testing.T) { + big, small := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7) + big.Format, big.SizeBytes = "flac", 30_000_000 + small.SizeBytes = 5_000_000 + g := Build([]Candidate{big, small}, tol)[0] + if g.KeepTrackID != 1 || g.FreedBytes != 5_000_000 { + t.Errorf("keep %d, freed %d; want 1 and 5000000", g.KeepTrackID, g.FreedBytes) + } +} + +// --- which copy to keep --------------------------------------------------------------------------- + +func TestSuggestPrefersLosslessThenBitrateThenSizeThenTheOldest(t *testing.T) { + flac := Candidate{TrackID: 5, Format: "flac", BitRate: 800, SizeBytes: 20_000_000} + hiMP3 := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000} + loMP3 := Candidate{TrackID: 1, Format: "mp3", BitRate: 128, SizeBytes: 4_000_000} + if got := Suggest([]Candidate{loMP3, hiMP3, flac}); got != 5 { + t.Errorf("flac among mp3: kept %d, want 5", got) + } + if got := Suggest([]Candidate{loMP3, hiMP3}); got != 2 { + t.Errorf("320 against 128: kept %d, want 2", got) + } + bigger := Candidate{TrackID: 7, Format: "mp3", BitRate: 320, SizeBytes: 9_500_000} + if got := Suggest([]Candidate{hiMP3, bigger}); got != 7 { + t.Errorf("same bitrate, bigger file: kept %d, want 7", got) + } + twin := Candidate{TrackID: 9, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000} + if got := Suggest([]Candidate{twin, hiMP3}); got != 2 { + t.Errorf("all equal: kept %d, want the oldest (2)", got) + } +} + +func TestSuggestIsTheSameWhateverTheOrder(t *testing.T) { + a := Candidate{TrackID: 1, Format: "mp3", BitRate: 320, SizeBytes: 9} + b := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9} + if Suggest([]Candidate{a, b}) != Suggest([]Candidate{b, a}) { + t.Error("the answer depends on the order") + } +} From 65edfc49188f8a6054f87146b8393ee8fd9dbf10 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:37:15 +0100 Subject: [PATCH 3/8] feat(storage): candidates for duplicates, file fingerprints and removing a copy from the index Candidates are the playable files whose title another file shares. A fingerprint is kept only for the file that was read (same size and date), and a scan that finds the file changed forgets it. Removing a copy moves its listens to the track that stays, because they would be deleted with the track, then removes the track with its file, credits and genres. Tested on SQLite and PostgreSQL. --- internal/duplicates/types.go | 32 ++++ internal/storage/duplicates.go | 208 +++++++++++++++++++++ internal/storage/duplicates_test.go | 269 ++++++++++++++++++++++++++++ internal/storage/library.go | 28 +-- 4 files changed, 526 insertions(+), 11 deletions(-) create mode 100644 internal/duplicates/types.go create mode 100644 internal/storage/duplicates.go create mode 100644 internal/storage/duplicates_test.go diff --git a/internal/duplicates/types.go b/internal/duplicates/types.go new file mode 100644 index 0000000..d6f6acd --- /dev/null +++ b/internal/duplicates/types.go @@ -0,0 +1,32 @@ +package duplicates + +import "errors" + +// Why a removal is refused. The API turns each into its own answer. +var ( + // ErrTrackNotFound: a track that was named does not exist (any more). + ErrTrackNotFound = errors.New("no such track") + // ErrDisabled: the operator has not allowed Sonda to delete files (library.allow_delete). + ErrDisabled = errors.New("deleting files is not allowed: turn on library.allow_delete") + // ErrScanRunning: a scan is running, and what it reads and what is removed must not meet. + ErrScanRunning = errors.New("a scan is running") + // ErrInvalid: the tracks named are not copies of one song, or the request makes no sense. + ErrInvalid = errors.New("these tracks are not copies of the same song") + // ErrChanged: a file is not what the index says (another size or date, a link, gone), or the + // copy to keep is not there any more. Nothing is removed; a new scan brings the index up to date. + ErrChanged = errors.New("a file changed since it was scanned") + // ErrReadOnly: the folder cannot be written to. + ErrReadOnly = errors.New("the music folder is read-only") +) + +// FileRecord is a file as the index knows it: where it is and what it looked like when it was scanned. +type FileRecord struct { + FileID int64 + TrackID int64 + Root string // the path of its library + RelPath string // relative to the library + SizeBytes int64 + MTimeUnix int64 + Hash string // SHA-256, empty until read + Missing bool // flagged as not found by a scan +} diff --git a/internal/storage/duplicates.go b/internal/storage/duplicates.go new file mode 100644 index 0000000..93965ec --- /dev/null +++ b/internal/storage/duplicates.go @@ -0,0 +1,208 @@ +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/duplicates" +) + +// DuplicatesRepo reads the files that may be copies of one song, keeps their fingerprints, and +// removes a copy from the index. It implements duplicates.Store. +type DuplicatesRepo struct{ db *bun.DB } + +// NewDuplicatesRepo builds a DuplicatesRepo. +func NewDuplicatesRepo(db *bun.DB) *DuplicatesRepo { return &DuplicatesRepo{db: db} } + +type candidateRow struct { + TrackID int64 `bun:"track_id"` + FileID int64 `bun:"file_id"` + Title string `bun:"title"` + TitleKey string `bun:"title_key"` + DurationMs int64 `bun:"duration_ms"` + AlbumID int64 `bun:"album_id"` + Album string `bun:"album"` + Format string `bun:"format"` + BitRate int `bun:"bitrate"` + SizeBytes int64 `bun:"size_bytes"` + Hash string `bun:"hash"` +} + +type creditRow struct { + TrackID int64 `bun:"track_id"` + ArtistID int64 `bun:"artist_id"` + Name string `bun:"name"` +} + +// idsPerQuery keeps a list of ids well under what a database accepts in one statement. +const idsPerQuery = 500 + +// Candidates returns the playable files whose title is the title of another playable file: the only +// ones that can have a copy. A track that has no file that is there, or more than one, is left out. +// What makes two of them the same song (the artists, the durations) is decided by duplicates.Build. +func (r *DuplicatesRepo) Candidates(ctx context.Context) ([]duplicates.Candidate, error) { + var rows []candidateRow + err := r.db.NewRaw(` + SELECT t.id AS track_id, f.id AS file_id, t.title AS title, t.title_key AS title_key, + COALESCE(t.duration_ms, 0) AS duration_ms, COALESCE(t.album_id, 0) AS album_id, + COALESCE(al.title, '') AS album, f.format AS format, COALESCE(f.bitrate, 0) AS bitrate, + f.size_bytes AS size_bytes, COALESCE(f.content_hash, '') AS hash + FROM tracks t + JOIN audio_files f ON f.track_id = t.id AND f.missing_at IS NULL + LEFT JOIN albums al ON al.id = t.album_id + WHERE t.id IN (SELECT track_id FROM audio_files WHERE missing_at IS NULL GROUP BY track_id HAVING COUNT(*) = 1) + AND t.title_key IN ( + SELECT t2.title_key FROM tracks t2 + JOIN audio_files f2 ON f2.track_id = t2.id AND f2.missing_at IS NULL + GROUP BY t2.title_key HAVING COUNT(*) > 1) + ORDER BY t.id`).Scan(ctx, &rows) + if err != nil { + return nil, fmt.Errorf("select the candidates: %w", err) + } + if len(rows) == 0 { + return nil, nil + } + + ids := make([]int64, len(rows)) + for i, row := range rows { + ids[i] = row.TrackID + } + artists, err := r.mainArtists(ctx, ids) + if err != nil { + return nil, err + } + + out := make([]duplicates.Candidate, len(rows)) + for i, row := range rows { + out[i] = duplicates.Candidate{ + TrackID: row.TrackID, FileID: row.FileID, Artists: artists[row.TrackID], Title: row.Title, TitleKey: row.TitleKey, + DurationMs: row.DurationMs, AlbumID: row.AlbumID, Album: row.Album, Format: row.Format, + BitRate: row.BitRate, SizeBytes: row.SizeBytes, Hash: row.Hash, + } + } + return out, nil +} + +// mainArtists reads the main artists of some tracks. +func (r *DuplicatesRepo) mainArtists(ctx context.Context, trackIDs []int64) (map[int64][]duplicates.ArtistRef, error) { + out := map[int64][]duplicates.ArtistRef{} + for start := 0; start < len(trackIDs); start += idsPerQuery { + chunk := trackIDs[start:min(start+idsPerQuery, len(trackIDs))] + var rows []creditRow + err := r.db.NewRaw(` + SELECT ta.track_id AS track_id, a.id AS artist_id, a.name AS name + FROM track_artists ta JOIN artists a ON a.id = ta.artist_id + WHERE ta.role = 'main' AND ta.track_id IN (?) + ORDER BY ta.track_id, a.id`, bun.In(chunk)).Scan(ctx, &rows) + if err != nil { + return nil, fmt.Errorf("select the artists of the candidates: %w", err) + } + for _, row := range rows { + out[row.TrackID] = append(out[row.TrackID], duplicates.ArtistRef{ID: row.ArtistID, Name: row.Name}) + } + } + return out, nil +} + +type fileRow struct { + FileID int64 `bun:"file_id"` + TrackID int64 `bun:"track_id"` + Root string `bun:"root"` + RelPath string `bun:"rel_path"` + SizeBytes int64 `bun:"size_bytes"` + MTimeUnix int64 `bun:"mtime_unix"` + Hash string `bun:"hash"` + Missing bool `bun:"missing"` +} + +func (row fileRow) record() duplicates.FileRecord { + return duplicates.FileRecord{ + FileID: row.FileID, TrackID: row.TrackID, Root: row.Root, RelPath: row.RelPath, + SizeBytes: row.SizeBytes, MTimeUnix: row.MTimeUnix, Hash: row.Hash, Missing: row.Missing, + } +} + +const fileSelect = ` + SELECT f.id AS file_id, f.track_id AS track_id, l.path AS root, f.rel_path AS rel_path, + f.size_bytes AS size_bytes, f.mtime_unix AS mtime_unix, COALESCE(f.content_hash, '') AS hash, + (f.missing_at IS NOT NULL) AS missing + FROM audio_files f JOIN libraries l ON l.id = f.library_id` + +// FileOf returns the file of a track: the oldest one that is there, else the oldest one. It returns +// duplicates.ErrTrackNotFound when the track has no file. +func (r *DuplicatesRepo) FileOf(ctx context.Context, trackID int64) (duplicates.FileRecord, error) { + var row fileRow + err := r.db.NewRaw(fileSelect+` WHERE f.track_id = ? ORDER BY (f.missing_at IS NOT NULL), f.id LIMIT 1`, trackID).Scan(ctx, &row) + if errors.Is(err, sql.ErrNoRows) { + return duplicates.FileRecord{}, duplicates.ErrTrackNotFound + } + if err != nil { + return duplicates.FileRecord{}, fmt.Errorf("select the file of a track: %w", err) + } + return row.record(), nil +} + +// FileByID returns one file, or duplicates.ErrTrackNotFound. +func (r *DuplicatesRepo) FileByID(ctx context.Context, fileID int64) (duplicates.FileRecord, error) { + var row fileRow + err := r.db.NewRaw(fileSelect+` WHERE f.id = ?`, fileID).Scan(ctx, &row) + if errors.Is(err, sql.ErrNoRows) { + return duplicates.FileRecord{}, duplicates.ErrTrackNotFound + } + if err != nil { + return duplicates.FileRecord{}, fmt.Errorf("select a file: %w", err) + } + return row.record(), nil +} + +// SaveHash keeps the fingerprint of a file, but only if the file is still the one that was read: the +// size and date are what they were. It says whether it was kept. +func (r *DuplicatesRepo) SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error) { + res, err := r.db.NewUpdate().Model((*audioFileModel)(nil)). + Set("content_hash = ?", hash). + Where("id = ?", fileID).Where("size_bytes = ?", sizeBytes).Where("mtime_unix = ?", mtimeUnix). + Exec(ctx) + if err != nil { + return false, fmt.Errorf("save the hash: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("save the hash: %w", err) + } + return n > 0, nil +} + +// RemoveCopy takes a track out of the index in favour of another that stays: the listens of the +// removed track become listens of the one that stays (they would be deleted with it otherwise), and +// the track goes, with its file, credits and genres. The covers of the albums involved are built +// again. It is one transaction. +func (r *DuplicatesRepo) RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error { + if keepTrackID == removeTrackID { + return duplicates.ErrInvalid + } + return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + albums := map[int64]int64{} + for _, id := range []int64{keepTrackID, removeTrackID} { + var album sql.NullInt64 + err := tx.QueryRowContext(ctx, "SELECT album_id FROM tracks WHERE id = ?", id).Scan(&album) + if errors.Is(err, sql.ErrNoRows) { + return duplicates.ErrTrackNotFound + } + if err != nil { + return fmt.Errorf("read a track: %w", err) + } + albums[id] = album.Int64 + } + if _, err := tx.ExecContext(ctx, "UPDATE listening_events SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil { + return fmt.Errorf("move the listens: %w", err) + } + if _, err := tx.ExecContext(ctx, "DELETE FROM tracks WHERE id = ?", removeTrackID); err != nil { + return fmt.Errorf("remove the track: %w", err) + } + return clearAlbumCovers(ctx, tx, albums[keepTrackID], albums[removeTrackID]) + }) +} diff --git a/internal/storage/duplicates_test.go b/internal/storage/duplicates_test.go new file mode 100644 index 0000000..d303d33 --- /dev/null +++ b/internal/storage/duplicates_test.go @@ -0,0 +1,269 @@ +package storage + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/uptrace/bun" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/listening" +) + +// song is a scanned file: a title on an album by artists, with the facts that tell copies apart. +func song(rel, title, album string, durationMs, size int64, format string, bitrate int, artists ...string) library.ParsedFile { + p := parsed(rel, title, album, artists...) + p.DurationMs, p.Size, p.Format, p.BitRate = durationMs, size, format, bitrate + return p +} + +func saved(t *testing.T, repo *LibraryRepo, lib int64, f library.ParsedFile) { + t.Helper() + if _, err := repo.SaveFile(context.Background(), lib, f, 0); err != nil { + t.Fatal(err) + } +} + +func trackOf(t *testing.T, db *bun.DB, rel string) int64 { + t.Helper() + var id int64 + if err := db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +func candidateByRel(t *testing.T, db *bun.DB, cs []duplicates.Candidate, rel string) (duplicates.Candidate, bool) { + t.Helper() + id := trackOf(t, db, rel) + for _, c := range cs { + if c.TrackID == id { + return c, true + } + } + return duplicates.Candidate{}, false +} + +func TestDuplicateCandidatesAreTheFilesWhoseTitleIsSharedWithAnotherFile(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "Album X", 200_000, 30_000_000, "flac", 900, "Ann")) + saved(t, lib, l, song("b/2.mp3", "Song", "Album Y", 201_000, 5_000_000, "mp3", 192, "Ann")) + saved(t, lib, l, song("c/3.flac", "Song", "Album Z", 200_000, 30_000_000, "flac", 900, "Bob")) + saved(t, lib, l, song("d/4.flac", "Song", "Album W", 200_000, 30_000_000, "flac", 900, "Ann", "Bob")) + saved(t, lib, l, song("e/5.flac", "Other", "Album X", 200_000, 30_000_000, "flac", 900, "Ann")) + + cs, err := NewDuplicatesRepo(db).Candidates(ctx) + if err != nil { + t.Fatal(err) + } + if _, ok := candidateByRel(t, db, cs, "e/5.flac"); ok { + t.Error("a title nobody shares is a candidate") + } + if len(cs) != 4 { + t.Fatalf("%d candidates, want the 4 files called Song", len(cs)) + } + + c, ok := candidateByRel(t, db, cs, "b/2.mp3") + if !ok { + t.Fatal("the mp3 is not a candidate") + } + if c.Title != "Song" || c.TitleKey != "song" || c.Album != "Album Y" || c.Format != "mp3" || c.BitRate != 192 || + c.SizeBytes != 5_000_000 || c.DurationMs != 201_000 || c.FileID == 0 || c.Hash != "" { + t.Errorf("candidate = %+v", c) + } + if len(c.Artists) != 1 || c.Artists[0].Name != "Ann" { + t.Errorf("artists = %+v, want Ann", c.Artists) + } + duo, _ := candidateByRel(t, db, cs, "d/4.flac") + if len(duo.Artists) != 2 { + t.Errorf("a duet has %d artists, want 2", len(duo.Artists)) + } + + // Seen through the grouping, the two by Ann alone are the same song, the others are not. + groups := duplicates.Build(cs, duplicates.DurationTolerance) + if len(groups) != 1 || len(groups[0].Files) != 2 { + t.Errorf("groups = %+v, want one of the two copies by Ann", groups) + } + }) +} + +func TestAFileFlaggedMissingIsNotACandidate(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann")) + saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1, "flac", 900, "Ann")) + repo := NewDuplicatesRepo(db) + if cs, _ := repo.Candidates(ctx); len(cs) != 2 { + t.Fatalf("%d candidates before, want 2", len(cs)) + } + var id int64 + if err := db.QueryRow("SELECT id FROM audio_files WHERE rel_path = 'b/2.flac'").Scan(&id); err != nil { + t.Fatal(err) + } + if err := lib.MarkMissing(ctx, []int64{id}, time.Now()); err != nil { + t.Fatal(err) + } + if cs, _ := repo.Candidates(ctx); len(cs) != 0 { + t.Errorf("%d candidates after one went missing, want 0", len(cs)) + } + }) +} + +func TestFileOfSaysWhereTheFileIsAndWhatItLookedLike(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann")) + repo := NewDuplicatesRepo(db) + rec, err := repo.FileOf(ctx, trackOf(t, db, "a/1.flac")) + if err != nil { + t.Fatal(err) + } + if rec.Root != "/lib/music" || rec.RelPath != "a/1.flac" || rec.SizeBytes != 1234 || rec.MTimeUnix != 1_700_000_000 || rec.Hash != "" || rec.Missing { + t.Errorf("record = %+v", rec) + } + if _, err := repo.FileOf(ctx, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("an unknown track gave %v, want ErrTrackNotFound", err) + } + }) +} + +func TestAHashIsStoredOnlyForTheFileThatWasRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann")) + saved(t, lib, l, song("b/2.flac", "Song", "B", 200_000, 1234, "flac", 900, "Ann")) + repo := NewDuplicatesRepo(db) + rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac")) + + // The file changed after it was read: its hash is no hash of it. + if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes+1, rec.MTimeUnix, "aa"); err != nil || ok { + t.Errorf("a hash of a file that changed in size was stored: ok=%v err=%v", ok, err) + } + if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix+1, "aa"); err != nil || ok { + t.Errorf("a hash of a file that changed in date was stored: ok=%v err=%v", ok, err) + } + if ok, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil || !ok { + t.Fatalf("the hash was not stored: ok=%v err=%v", ok, err) + } + cs, _ := repo.Candidates(ctx) + c, _ := candidateByRel(t, db, cs, "a/1.flac") + other, _ := candidateByRel(t, db, cs, "b/2.flac") + if c.Hash != "aa" || other.Hash != "" { + t.Errorf("hashes = %q and %q, want aa and nothing", c.Hash, other.Hash) + } + }) +} + +// A hash says "this is what the file held when it was read". A scan that finds the file changed forgets it. +func TestAScanThatFindsAFileChangedForgetsItsHash(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann")) + repo := NewDuplicatesRepo(db) + rec, _ := repo.FileOf(ctx, trackOf(t, db, "a/1.flac")) + if _, err := repo.SaveHash(ctx, rec.FileID, rec.SizeBytes, rec.MTimeUnix, "aa"); err != nil { + t.Fatal(err) + } + + // The same file read again, unchanged (a full rescan): the hash is still true. + if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 1234, "flac", 900, "Ann"), rec.FileID); err != nil { + t.Fatal(err) + } + if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "aa" { + t.Errorf("an unchanged file lost its hash: %q", got.Hash) + } + + // The file was edited: another size. + if _, err := lib.SaveFile(ctx, l, song("a/1.flac", "Song", "A", 200_000, 4321, "flac", 900, "Ann"), rec.FileID); err != nil { + t.Fatal(err) + } + if got, _ := repo.FileOf(ctx, rec.TrackID); got.Hash != "" || got.SizeBytes != 4321 { + t.Errorf("after a change: hash %q size %d, want no hash and the new size", got.Hash, got.SizeBytes) + } + }) +} + +func TestRemovingACopyMovesItsListensToTheOneThatStays(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann")) + saved(t, lib, l, song("b/2.mp3", "Song", "B", 200_000, 1, "mp3", 192, "Ann")) + keep, gone := trackOf(t, db, "a/1.flac"), trackOf(t, db, "b/2.mp3") + + user, err := NewUserRepo(db).CreateUser(ctx, "alice", "h", false) + if err != nil { + t.Fatal(err) + } + plays := NewListeningRepo(db) + for _, e := range []struct { + track int64 + key string + }{{keep, "k1"}, {gone, "g1"}, {gone, "g2"}} { + if _, err := plays.Record(ctx, listening.Event{UserID: user.ID, TrackID: e.track, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: e.key}); err != nil { + t.Fatal(err) + } + } + + if err := NewDuplicatesRepo(db).RemoveCopy(ctx, keep, gone); err != nil { + t.Fatal(err) + } + + if n := count(t, db, "SELECT COUNT(*) FROM listening_events WHERE track_id = ?", keep); n != 3 { + t.Errorf("the track that stays has %d listens, want its own and the 2 moved (3)", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM listening_events"); n != 3 { + t.Errorf("%d listens in all, want 3: none lost", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", gone); n != 0 { + t.Error("the removed track is still there") + } + if n := count(t, db, "SELECT COUNT(*) FROM audio_files WHERE track_id = ?", gone); n != 0 { + t.Error("the file row of the removed track is still there") + } + if n := count(t, db, "SELECT COUNT(*) FROM track_artists WHERE track_id = ?", gone); n != 0 { + t.Error("the credits of the removed track are still there") + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks WHERE id = ?", keep); n != 1 { + t.Error("the track that stays is gone") + } + }) +} + +func TestRemovingACopyNeedsTwoTracksThatExistAndAreDifferent(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann")) + keep := trackOf(t, db, "a/1.flac") + repo := NewDuplicatesRepo(db) + if err := repo.RemoveCopy(ctx, keep, 9999); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("a track that does not exist: %v, want ErrTrackNotFound", err) + } + if err := repo.RemoveCopy(ctx, 9999, keep); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("a track to keep that does not exist: %v, want ErrTrackNotFound", err) + } + if err := repo.RemoveCopy(ctx, keep, keep); !errors.Is(err, duplicates.ErrInvalid) { + t.Errorf("a track removed in favour of itself: %v, want ErrInvalid", err) + } + if n := count(t, db, "SELECT COUNT(*) FROM tracks"); n != 1 { + t.Errorf("%d tracks left, want the one that was there", n) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index c4a24e6..b5ae035 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -60,16 +60,18 @@ type trackArtistModel struct { type audioFileModel struct { bun.BaseModel `bun:"table:audio_files"` - ID int64 `bun:"id,pk,autoincrement"` - TrackID int64 `bun:"track_id,notnull"` - LibraryID int64 `bun:"library_id,notnull"` - RelPath string `bun:"rel_path,notnull"` - Format string `bun:"format,notnull"` - SizeBytes int64 `bun:"size_bytes,notnull"` - MTimeUnix int64 `bun:"mtime_unix,notnull"` - SampleRate int `bun:"sample_rate,nullzero"` - BitRate int `bun:"bitrate,nullzero"` - MissingAt sql.NullInt64 `bun:"missing_at"` + ID int64 `bun:"id,pk,autoincrement"` + TrackID int64 `bun:"track_id,notnull"` + LibraryID int64 `bun:"library_id,notnull"` + RelPath string `bun:"rel_path,notnull"` + Format string `bun:"format,notnull"` + SizeBytes int64 `bun:"size_bytes,notnull"` + MTimeUnix int64 `bun:"mtime_unix,notnull"` + SampleRate int `bun:"sample_rate,nullzero"` + BitRate int `bun:"bitrate,nullzero"` + // ContentHash is the SHA-256 of the file as it was when it was read for duplicates; empty until then. + ContentHash sql.NullString `bun:"content_hash"` + MissingAt sql.NullInt64 `bun:"missing_at"` } type libraryModel struct { @@ -198,11 +200,15 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P return err } } + // A hash says what the file held when it was read: a file that changed has no hash any more. + if file.SizeBytes != f.Size || file.MTimeUnix != f.MTimeUnix { + file.ContentHash = sql.NullString{} + } file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix file.SampleRate, file.BitRate = f.SampleRate, f.BitRate file.MissingAt = sql.NullInt64{} if _, err := tx.NewUpdate().Model(&file). - Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "missing_at"). + Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "content_hash", "missing_at"). WherePK().Exec(ctx); err != nil { return fmt.Errorf("update audio file: %w", err) } From d3e3bde2e4fa407e9f60d9c121d4d8e0550ac1ec Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:40:43 +0100 Subject: [PATCH 4/8] feat(duplicates): find the copies, fingerprint their files and remove the extra ones Groups come from the index. The check job reads the files of the groups and keeps their SHA-256, skipping a file that changed since the scan. Clean refuses unless deleting is on and no scan runs, works out again that the tracks are copies of one song, checks every file is still what the scan saw (so the last copy is never removed, and a link or a path that leaves the library is refused) before removing any, removes each file through an os.Root, then takes its track out of the index and moves its listens. Tested with real files, including the refusals; three of the safety rules were broken on purpose to see the tests fail. --- internal/duplicates/files.go | 80 +++++ internal/duplicates/hashjob.go | 120 +++++++ internal/duplicates/service.go | 236 +++++++++++++ internal/duplicates/service_test.go | 497 ++++++++++++++++++++++++++++ 4 files changed, 933 insertions(+) create mode 100644 internal/duplicates/files.go create mode 100644 internal/duplicates/hashjob.go create mode 100644 internal/duplicates/service.go create mode 100644 internal/duplicates/service_test.go diff --git a/internal/duplicates/files.go b/internal/duplicates/files.go new file mode 100644 index 0000000..69a992f --- /dev/null +++ b/internal/duplicates/files.go @@ -0,0 +1,80 @@ +package duplicates + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" +) + +// openRoot opens the folder of a library so that nothing done through it can leave it: a ../ or a +// link that points outside is refused by the system at the moment of the call, with no gap between +// checking a path and using it. It returns ErrChanged when the folder is not there (a disk that is +// not mounted). +func openRoot(root string) (*os.Root, error) { + clean := filepath.Clean(root) + if !filepath.IsAbs(clean) { + return nil, fmt.Errorf("%w: the library folder is not an absolute path", ErrChanged) + } + r, err := os.OpenRoot(clean) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w: the library folder is not there", ErrChanged) + } + if err != nil { + return nil, fmt.Errorf("open the library folder: %w", err) + } + return r, nil +} + +// relPath is the path of a file inside its library, in the system's own form. An empty or absolute +// path is not one. +func relPath(rec FileRecord) (string, error) { + rel := filepath.FromSlash(rec.RelPath) + if rel == "" || filepath.IsAbs(rel) { + return "", fmt.Errorf("%w: the path of the file is not inside the library", ErrChanged) + } + return rel, nil +} + +// isEscape says whether an error from an os.Root is the refusal to leave the root (it has no exported value). +func isEscape(err error) bool { + return err != nil && strings.Contains(err.Error(), "path escapes from parent") +} + +// sameAsScanned checks, through the root, that the path is a regular file (not a link, not a +// folder) with the size and date the index has for it. Anything else is ErrChanged. +func sameAsScanned(root *os.Root, rel string, rec FileRecord) error { + info, err := root.Lstat(rel) + switch { + case errors.Is(err, fs.ErrNotExist), isEscape(err): + return fmt.Errorf("%w: the file is not there", ErrChanged) + case err != nil: + return fmt.Errorf("look at the file: %w", err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: it is not a regular file any more", ErrChanged) + } + if info.Size() != rec.SizeBytes || info.ModTime().Unix() != rec.MTimeUnix { + return fmt.Errorf("%w: its size or date is not what the scan saw", ErrChanged) + } + return nil +} + +// removeFile deletes one file inside the root, and turns the ways it can fail into the answers the API gives. +func removeFile(root *os.Root, rel string) error { + err := root.Remove(rel) + switch { + case err == nil: + return nil + case errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("%w: the file is gone", ErrChanged) + case isEscape(err): + return fmt.Errorf("%w: the path leaves the library", ErrChanged) + case errors.Is(err, fs.ErrPermission), errors.Is(err, syscall.EROFS): + return fmt.Errorf("%w: %v", ErrReadOnly, err) + } + return fmt.Errorf("remove the file: %w", err) +} diff --git a/internal/duplicates/hashjob.go b/internal/duplicates/hashjob.go new file mode 100644 index 0000000..7c8223f --- /dev/null +++ b/internal/duplicates/hashjob.go @@ -0,0 +1,120 @@ +package duplicates + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "sync" + "time" + + "codeberg.org/squidd/sonda/internal/jobs" +) + +// OutcomeHashed is what the check counts for a file it read and fingerprinted. +const OutcomeHashed = "hashed" + +// NewHashJob builds the job that reads the files of the groups of copies and keeps their SHA-256, so +// that a group of files that are byte for byte the same can be said to be identical. It is started by +// hand like the other bulk jobs: it reads whole files, which on a NAS takes a while. +func NewHashJob(s *Service) *jobs.Job { + t := &hashTask{s: s} + return jobs.New(jobs.Task{Noun: "files", Count: t.count, List: t.list, Do: t.do}, s.now) +} + +// hashTask keeps, between Count and the Lists that follow, the files to read: those of the groups +// that have no fingerprint yet (all of them to do everything again). +type hashTask struct { + s *Service + + mu sync.Mutex + ids []int64 +} + +func (t *hashTask) count(ctx context.Context, refresh bool, _ time.Time) (int, error) { + groups, err := t.s.groups(ctx) + if err != nil { + return 0, err + } + var ids []int64 + for _, g := range groups { + for _, f := range g.Files { + if refresh || f.Hash == "" { + ids = append(ids, f.FileID) + } + } + } + slices.Sort(ids) + t.mu.Lock() + t.ids = ids + t.mu.Unlock() + return len(ids), nil +} + +func (t *hashTask) list(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) { + t.mu.Lock() + defer t.mu.Unlock() + start := sort.Search(len(t.ids), func(i int) bool { return t.ids[i] > after }) + return slices.Clone(t.ids[start:min(start+limit, len(t.ids))]), nil +} + +// do reads one file and keeps its fingerprint, if the file is still the one the scan saw. +func (t *hashTask) do(ctx context.Context, fileID int64, _ bool) (string, error) { + rec, err := t.s.store.FileByID(ctx, fileID) + if errors.Is(err, ErrTrackNotFound) || err == nil && rec.Missing { + return "", jobs.ErrSkip + } + if err != nil { + return "", err + } + root, err := openRoot(rec.Root) + if err != nil { + return "", err // a disk that is not mounted: every file fails the same way, and the job gives up + } + defer root.Close() + rel, err := relPath(rec) + if err != nil { + return "", jobs.ErrSkip + } + if err := sameAsScanned(root, rel, rec); errors.Is(err, ErrChanged) { + return "", jobs.ErrSkip + } else if err != nil { + return "", err + } + + f, err := root.Open(rel) + if err != nil { + return "", fmt.Errorf("open the file: %w", err) + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, &contextReader{ctx: ctx, r: f}); err != nil { + return "", fmt.Errorf("read the file: %w", err) + } + saved, err := t.s.store.SaveHash(ctx, fileID, rec.SizeBytes, rec.MTimeUnix, hex.EncodeToString(h.Sum(nil))) + if err != nil { + return "", err + } + if !saved { + return "", jobs.ErrSkip // it changed while it was read + } + return OutcomeHashed, nil +} + +// contextReader stops reading when the job is stopped, instead of finishing a long file first. +type contextReader struct { + ctx context.Context + r *os.File +} + +func (c *contextReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/internal/duplicates/service.go b/internal/duplicates/service.go new file mode 100644 index 0000000..185f309 --- /dev/null +++ b/internal/duplicates/service.go @@ -0,0 +1,236 @@ +package duplicates + +import ( + "context" + "fmt" + "os" + "slices" + "time" + + "go.uber.org/zap" +) + +// Store is what the service reads and writes in the index. internal/storage implements it. +type Store interface { + Candidates(ctx context.Context) ([]Candidate, error) + FileOf(ctx context.Context, trackID int64) (FileRecord, error) + FileByID(ctx context.Context, fileID int64) (FileRecord, error) + SaveHash(ctx context.Context, fileID, sizeBytes, mtimeUnix int64, hash string) (bool, error) + RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error +} + +// Options say what the service may do. +type Options struct { + // AllowDelete is library.allow_delete. Without it nothing is ever removed. + AllowDelete bool + // ScanRunning says whether a scan is running; a removal waits for it to end. Nil means never. + ScanRunning func() bool + // Logger records each removal, by track and without the path: audio paths are private. + Logger *zap.Logger +} + +// Service finds duplicate songs and removes the extra copies. +type Service struct { + store Store + allow bool + scanning func() bool + log *zap.Logger + now func() time.Time +} + +// NewService builds a Service. +func NewService(store Store, o Options) *Service { + log := o.Logger + if log == nil { + log = zap.NewNop() + } + return &Service{store: store, allow: o.AllowDelete, scanning: o.ScanRunning, log: log, now: time.Now} +} + +// Page is some of the groups, with how many there are in all. +type Page struct { + Groups []Group + Total int + // DeletionAllowed says whether Clean can work: with it off, the groups can still be seen. + DeletionAllowed bool +} + +// Groups returns the groups of copies in the library, worked out from the index and never from the disk. +func (s *Service) Groups(ctx context.Context, limit, offset int) (Page, error) { + all, err := s.groups(ctx) + if err != nil { + return Page{}, err + } + page := Page{Total: len(all), DeletionAllowed: s.allow} + if offset < len(all) { + page.Groups = all[offset:min(offset+limit, len(all))] + } + return page, nil +} + +func (s *Service) groups(ctx context.Context) ([]Group, error) { + candidates, err := s.store.Candidates(ctx) + if err != nil { + return nil, err + } + return Build(candidates, DurationTolerance), nil +} + +// maxRemovals is the most copies of one song that one request may remove. +const maxRemovals = 100 + +// Result says what a removal did. +type Result struct { + Removed int + FreedBytes int64 +} + +// Clean removes the copies `remove` of a song from the disk and the index, keeping `keep`. Nothing +// is removed unless every check passes, and the checks are made here, again, whatever the client +// saw: deleting is allowed, no scan is running, the tracks are copies of one song by the rule of +// Build, the copy to keep is still the file that was scanned (so it is never the last copy that is +// removed), and so is every copy to remove (a file that changed, a link, or a path that leaves the +// library is refused). The files are checked all together before the first is removed. Each removed +// file takes its track out of the index, and its listens go to the track that stays. +// +// On an error part way (the folder turned out to be read-only on the second file), the Result says +// what was already removed. +func (s *Service) Clean(ctx context.Context, keep int64, remove []int64) (Result, error) { + var res Result + if !s.allow { + return res, ErrDisabled + } + if s.scanning != nil && s.scanning() { + return res, ErrScanRunning + } + if err := checkRequest(keep, remove); err != nil { + return res, err + } + + keepFile, err := s.store.FileOf(ctx, keep) + if err != nil { + return res, err + } + files := make(map[int64]FileRecord, len(remove)) + for _, id := range remove { + rec, err := s.store.FileOf(ctx, id) + if err != nil { + return res, err + } + files[id] = rec + } + if keepFile.Missing { + return res, fmt.Errorf("%w: the copy to keep is flagged as missing", ErrChanged) + } + for _, rec := range files { + if rec.Missing { + return res, fmt.Errorf("%w: a copy to remove is flagged as missing", ErrChanged) + } + } + if err := s.checkSameSong(ctx, keep, remove); err != nil { + return res, err + } + + // Look at every file before touching any. + roots := map[string]*os.Root{} + defer func() { + for _, r := range roots { + _ = r.Close() + } + }() + rootOf := func(path string) (*os.Root, error) { + if r, ok := roots[path]; ok { + return r, nil + } + r, err := openRoot(path) + if err != nil { + return nil, err + } + roots[path] = r + return r, nil + } + inspect := func(rec FileRecord) (*os.Root, string, error) { + root, err := rootOf(rec.Root) + if err != nil { + return nil, "", err + } + rel, err := relPath(rec) + if err != nil { + return nil, "", err + } + return root, rel, sameAsScanned(root, rel, rec) + } + if _, _, err := inspect(keepFile); err != nil { + return res, err + } + type target struct { + id int64 + root *os.Root + rel string + } + targets := make([]target, 0, len(remove)) + for _, id := range remove { + root, rel, err := inspect(files[id]) + if err != nil { + return res, err + } + targets = append(targets, target{id, root, rel}) + } + + for _, t := range targets { + if err := removeFile(t.root, t.rel); err != nil { + return res, err + } + // The file is gone. If the index cannot follow, the next scan flags the path as missing, and + // nothing is left broken; the error says so. + if err := s.store.RemoveCopy(ctx, keep, t.id); err != nil { + s.log.Error("a duplicate file was removed but its track is still in the index; a scan will flag it as missing", + zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep), zap.Error(err)) + return res, fmt.Errorf("the file was removed but the index could not be updated: %w", err) + } + res.Removed++ + res.FreedBytes += files[t.id].SizeBytes + s.log.Info("a duplicate file was removed", zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep)) + } + return res, nil +} + +// checkRequest refuses a request that makes no sense, before anything is read: nothing to remove, +// too much, the copy to keep among those to remove, or a copy named twice. +func checkRequest(keep int64, remove []int64) error { + if len(remove) == 0 || len(remove) > maxRemovals { + return fmt.Errorf("%w: name between 1 and %d copies to remove", ErrInvalid, maxRemovals) + } + seen := map[int64]bool{} + for _, id := range remove { + if id == keep { + return fmt.Errorf("%w: the copy to keep is among those to remove", ErrInvalid) + } + if seen[id] { + return fmt.Errorf("%w: a copy is named twice", ErrInvalid) + } + seen[id] = true + } + return nil +} + +// checkSameSong works out the groups again and checks that every copy to remove is in the group of +// the copy to keep. +func (s *Service) checkSameSong(ctx context.Context, keep int64, remove []int64) error { + groups, err := s.groups(ctx) + if err != nil { + return err + } + for _, g := range groups { + if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == keep }) { + continue + } + for _, id := range remove { + if !slices.ContainsFunc(g.Files, func(f Candidate) bool { return f.TrackID == id }) { + return ErrInvalid + } + } + return nil + } + return ErrInvalid +} diff --git a/internal/duplicates/service_test.go b/internal/duplicates/service_test.go new file mode 100644 index 0000000..576c8d7 --- /dev/null +++ b/internal/duplicates/service_test.go @@ -0,0 +1,497 @@ +package duplicates_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/listening" + "codeberg.org/squidd/sonda/internal/storage" + "codeberg.org/squidd/sonda/migrations" +) + +// env is a real library on a real folder, indexed in a real database. +type env struct { + t *testing.T + ctx context.Context + db *bun.DB + dir string + libID int64 + lib *storage.LibraryRepo + store *storage.DuplicatesRepo + scanning bool +} + +func newEnv(t *testing.T) *env { + t.Helper() + db, err := storage.Open(storage.DriverSQLite, filepath.Join(t.TempDir(), "d.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + fsys, err := migrations.For(storage.DriverSQLite) + if err != nil { + t.Fatal(err) + } + if err := storage.Migrate(context.Background(), db, fsys); err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + e := &env{t: t, ctx: context.Background(), db: db, dir: dir, lib: storage.NewLibraryRepo(db), store: storage.NewDuplicatesRepo(db)} + l, err := e.lib.CreateLibrary(e.ctx, "music", dir) + if err != nil { + t.Fatal(err) + } + e.libID = l.ID + return e +} + +func (e *env) service(allow bool) *duplicates.Service { + return duplicates.NewService(e.store, duplicates.Options{AllowDelete: allow, ScanRunning: func() bool { return e.scanning }, Logger: zap.NewNop()}) +} + +// add writes a file and indexes it exactly as it is on the disk, and returns its track. +func (e *env) add(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + e.t.Helper() + path := filepath.Join(e.dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + e.t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + e.t.Fatal(err) + } + return e.index(rel, path, title, album, durationMs, format, bitrate, artists...) +} + +// index puts a file that is on the disk into the index with the size and date it has. +func (e *env) index(rel, path, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + e.t.Helper() + info, err := os.Stat(path) + if err != nil { + e.t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate}, + }, + } + if _, err := e.lib.SaveFile(e.ctx, e.libID, f, 0); err != nil { + e.t.Fatal(err) + } + var id int64 + if err := e.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil { + e.t.Fatal(err) + } + return id +} + +func (e *env) exists(rel string) bool { + _, err := os.Lstat(filepath.Join(e.dir, filepath.FromSlash(rel))) + return err == nil +} + +func (e *env) tracks() int { + var n int + if err := e.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&n); err != nil { + e.t.Fatal(err) + } + return n +} + +// twoCopies is the usual case: the same song twice, an mp3 and a flac, on different albums. +func twoCopies(e *env) (flac, mp3 int64) { + flac = e.add("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann") + mp3 = e.add("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann") + return flac, mp3 +} + +// --- finding --------------------------------------------------------------------------------------- + +func TestGroupsListsTheCopiesWithWhatTheAdminNeedsToChoose(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann") + + page, err := e.service(true).Groups(e.ctx, 10, 0) + if err != nil { + t.Fatal(err) + } + if page.Total != 1 || len(page.Groups) != 1 || !page.DeletionAllowed { + t.Fatalf("page = %+v, want one group and deleting allowed", page) + } + g := page.Groups[0] + if g.Title != "Song" || len(g.Files) != 2 || g.KeepTrackID != flac || g.FreedBytes != int64(len("mp3 content")) { + t.Errorf("group = %+v, want the flac kept and the size of the mp3 freed", g) + } + if g.Files[1].TrackID != mp3 || g.Files[1].Album != "Greatest hits" || g.Files[1].Format != "mp3" { + t.Errorf("files = %+v", g.Files) + } +} + +func TestGroupsReportsThatDeletingIsOff(t *testing.T) { + e := newEnv(t) + twoCopies(e) + page, err := e.service(false).Groups(e.ctx, 10, 0) + if err != nil { + t.Fatal(err) + } + if page.DeletionAllowed || len(page.Groups) != 1 { + t.Errorf("page = %+v: the groups are visible and deleting is not allowed", page) + } +} + +func TestGroupsArePaged(t *testing.T) { + e := newEnv(t) + for _, title := range []string{"A song", "B song", "C song"} { + e.add("x/"+title+"-1.mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann") + e.add("y/"+title+"-2.mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann") + } + svc := e.service(true) + first, _ := svc.Groups(e.ctx, 2, 0) + second, _ := svc.Groups(e.ctx, 2, 2) + if first.Total != 3 || len(first.Groups) != 2 || len(second.Groups) != 1 { + t.Fatalf("pages of %d and %d of %d, want 2, 1 of 3", len(first.Groups), len(second.Groups), first.Total) + } + if first.Groups[0].Title != "A song" || second.Groups[0].Title != "C song" { + t.Errorf("titles %q then %q, want them in order", first.Groups[0].Title, second.Groups[0].Title) + } + if beyond, _ := svc.Groups(e.ctx, 2, 10); len(beyond.Groups) != 0 || beyond.Total != 3 { + t.Errorf("past the end: %+v", beyond) + } +} + +// --- fingerprints ------------------------------------------------------------------------------------ + +func runJob(t *testing.T, j *jobs.Job, refresh bool) jobs.Status { + t.Helper() + if err := j.Start(context.Background(), jobs.Options{Refresh: refresh}); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if st := j.Status(); !st.Running { + return st + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("the job did not finish") + return jobs.Status{} +} + +func TestTheCheckReadsTheFilesOfTheGroupsAndSaysWhichAreIdentical(t *testing.T) { + e := newEnv(t) + e.add("a/1.mp3", "the very same bytes", "Same", "A", 100_000, "mp3", 192, "Ann") + e.add("b/2.mp3", "the very same bytes", "Same", "B", 100_000, "mp3", 192, "Ann") + e.add("a/3.mp3", "one version", "Alike", "A", 100_000, "mp3", 192, "Ann") + e.add("b/4.mp3", "another version!", "Alike", "B", 100_000, "mp3", 192, "Ann") + e.add("c/5.mp3", "alone", "Alone", "A", 100_000, "mp3", 192, "Ann") + svc := e.service(true) + + st := runJob(t, duplicates.NewHashJob(svc), false) + if st.Last == nil || st.Last.Checked != 4 || st.Last.Outcomes[duplicates.OutcomeHashed] != 4 || st.LastError != "" { + t.Fatalf("status = %+v, want the 4 files of the groups hashed and the one alone left", st) + } + + page, _ := svc.Groups(e.ctx, 10, 0) + byTitle := map[string]duplicates.Group{} + for _, g := range page.Groups { + byTitle[g.Title] = g + } + if g := byTitle["Same"]; !g.Hashed || !g.Identical { + t.Errorf("the same bytes: hashed=%v identical=%v, want both", g.Hashed, g.Identical) + } + if g := byTitle["Alike"]; !g.Hashed || g.Identical { + t.Errorf("different bytes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical) + } +} + +func TestTheCheckSkipsWhatIsHashedUnlessAskedToDoItAgain(t *testing.T) { + e := newEnv(t) + twoCopies(e) + job := duplicates.NewHashJob(e.service(true)) + if st := runJob(t, job, false); st.Last.Checked != 2 { + t.Fatalf("first run checked %d, want 2", st.Last.Checked) + } + if st := runJob(t, job, false); st.Last.Checked != 0 { + t.Errorf("second run checked %d, want 0: everything is hashed", st.Last.Checked) + } + if st := runJob(t, job, true); st.Last.Checked != 2 { + t.Errorf("a run that does everything again checked %d, want 2", st.Last.Checked) + } +} + +func TestTheCheckSkipsAFileThatChangedAfterTheScan(t *testing.T) { + e := newEnv(t) + twoCopies(e) + if err := os.WriteFile(filepath.Join(e.dir, "b/song.mp3"), []byte("the file was edited, so it is longer now"), 0o644); err != nil { + t.Fatal(err) + } + st := runJob(t, duplicates.NewHashJob(e.service(true)), false) + if st.Last.Outcomes[duplicates.OutcomeHashed] != 1 || st.Last.Skipped != 1 { + t.Errorf("summary = %+v, want the file that is as scanned hashed and the edited one skipped", st.Last) + } + page, _ := e.service(true).Groups(e.ctx, 10, 0) + if page.Groups[0].Hashed { + t.Error("the group claims every file was read, but one was not") + } +} + +func TestTheCheckFailsOnAFileItCannotRead(t *testing.T) { + e := newEnv(t) + twoCopies(e) + if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Fatal(err) + } + st := runJob(t, duplicates.NewHashJob(e.service(true)), false) + if st.Last.Skipped+st.Last.Failed != 1 || st.Last.Outcomes[duplicates.OutcomeHashed] != 1 { + t.Errorf("summary = %+v, want one file hashed and the missing one not counted as read", st.Last) + } +} + +// --- removing ----------------------------------------------------------------------------------------- + +func TestWithDeletingOffNothingIsTouched(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + _, err := e.service(false).Clean(e.ctx, flac, []int64{mp3}) + if !errors.Is(err, duplicates.ErrDisabled) { + t.Fatalf("got %v, want ErrDisabled", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +func TestNothingIsRemovedWhileAScanRuns(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + e.scanning = true + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrScanRunning) { + t.Fatalf("got %v, want ErrScanRunning", err) + } + if !e.exists("b/song.mp3") { + t.Error("a file was removed during a scan") + } +} + +func TestCleaningRemovesTheCopyFromTheDiskAndTheIndexAndKeepsTheOther(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + res, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}) + if err != nil { + t.Fatal(err) + } + if res.Removed != 1 || res.FreedBytes != int64(len("mp3 content")) { + t.Errorf("result = %+v, want 1 removed and the size of the mp3 freed", res) + } + if e.exists("b/song.mp3") { + t.Error("the copy is still on the disk") + } + if !e.exists("a/song.flac") { + t.Error("the copy to keep is gone") + } + if e.tracks() != 1 { + t.Errorf("%d tracks, want 1", e.tracks()) + } + if page, _ := e.service(true).Groups(e.ctx, 10, 0); page.Total != 0 { + t.Errorf("%d groups are left, want none", page.Total) + } + if _, err := e.store.FileOf(e.ctx, mp3); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("the removed track is still known: %v", err) + } +} + +func TestSeveralCopiesAreRemovedAtOnce(t *testing.T) { + e := newEnv(t) + keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann") + b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann") + c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann") + res, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}) + if err != nil { + t.Fatal(err) + } + if res.Removed != 2 || e.exists("b/2.mp3") || e.exists("c/3.mp3") || !e.exists("a/1.flac") { + t.Errorf("result %+v; b exists=%v c exists=%v keep exists=%v", res, e.exists("b/2.mp3"), e.exists("c/3.mp3"), e.exists("a/1.flac")) + } +} + +func TestTheCopyToKeepCannotBeInTheListToRemove(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + for name, remove := range map[string][]int64{"both": {flac, mp3}, "only the one to keep": {flac}, "nothing": {}, "the same twice": {mp3, mp3}} { + if _, err := e.service(true).Clean(e.ctx, flac, remove); !errors.Is(err, duplicates.ErrInvalid) { + t.Errorf("%s: got %v, want ErrInvalid", name, err) + } + } + if !e.exists("a/song.flac") || !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +// What the client sends is not trusted: the server works out again that these are copies of one song. +func TestOnlyCopiesOfTheSameSongCanBeRemovedInFavourOfEachOther(t *testing.T) { + e := newEnv(t) + flac, _ := twoCopies(e) + other := e.add("c/other.flac", "something else", "Other", "Album", 100_000, "flac", 900, "Ann") + otherArtist := e.add("d/song.mp3", "by someone else", "Song", "Album", 200_000, "mp3", 192, "Bob") + live := e.add("e/song-live.mp3", "a long live version", "Song", "Live", 420_000, "mp3", 192, "Ann") + for name, id := range map[string]int64{"another song": other, "another artist": otherArtist, "a much longer version": live} { + if _, err := e.service(true).Clean(e.ctx, flac, []int64{id}); !errors.Is(err, duplicates.ErrInvalid) { + t.Errorf("%s: got %v, want ErrInvalid", name, err) + } + } + for _, rel := range []string{"c/other.flac", "d/song.mp3", "e/song-live.mp3", "a/song.flac", "b/song.mp3"} { + if !e.exists(rel) { + t.Errorf("%s was removed", rel) + } + } +} + +func TestATrackThatDoesNotExistIsRefused(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + if _, err := e.service(true).Clean(e.ctx, flac, []int64{9999}); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("an unknown copy: %v, want ErrTrackNotFound", err) + } + if _, err := e.service(true).Clean(e.ctx, 9999, []int64{mp3}); !errors.Is(err, duplicates.ErrTrackNotFound) { + t.Errorf("an unknown copy to keep: %v, want ErrTrackNotFound", err) + } + if !e.exists("b/song.mp3") { + t.Error("a file was removed") + } +} + +// If the copy to keep is not there any more, the other one is the last, and the last is never removed. +func TestTheLastCopyIsNeverRemoved(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + if err := os.Remove(filepath.Join(e.dir, "a/song.flac")); err != nil { + t.Fatal(err) + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("the only copy left was removed") + } +} + +func TestACopyThatChangedAfterTheScanIsNotRemoved(t *testing.T) { + e := newEnv(t) + keep := e.add("a/1.flac", "best", "Song", "A", 200_000, "flac", 900, "Ann") + b := e.add("b/2.mp3", "two", "Song", "B", 200_000, "mp3", 192, "Ann") + c := e.add("c/3.mp3", "three!", "Song", "C", 200_000, "mp3", 128, "Ann") + // c is edited after the scan: it is not the file the admin saw. + if err := os.WriteFile(filepath.Join(e.dir, "c/3.mp3"), []byte("edited and longer than before"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := e.service(true).Clean(e.ctx, keep, []int64{b, c}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + // Nothing at all was removed, not even b, which was fine: every file is checked before any is removed. + if !e.exists("b/2.mp3") || !e.exists("c/3.mp3") || e.tracks() != 3 { + t.Error("something was removed although a file in the request had changed") + } +} + +func TestACopyThatBecameALinkIsNotFollowedOrRemoved(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + target := filepath.Join(e.dir, "precious.txt") + if err := os.WriteFile(target, []byte("do not touch"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(e.dir, "b/song.mp3")); err != nil { + t.Skipf("no symlinks here: %v", err) + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrChanged) { + t.Fatalf("got %v, want ErrChanged", err) + } + if !e.exists("precious.txt") || !e.exists("b/song.mp3") { + t.Error("the link or what it points at was removed") + } +} + +// A path that was recorded as leaving the library is never followed out of it. +func TestAPathThatLeavesTheLibraryIsNeverRemoved(t *testing.T) { + e := newEnv(t) + flac, _ := twoCopies(e) + outside := filepath.Join(filepath.Dir(e.dir), "outside-"+filepath.Base(e.dir)) + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(outside) }) + victim := filepath.Join(outside, "song.mp3") + if err := os.WriteFile(victim, []byte("not in the library"), 0o644); err != nil { + t.Fatal(err) + } + rel := "../" + filepath.Base(outside) + "/song.mp3" + escaped := e.index(rel, victim, "Song", "Escaped", 200_000, "mp3", 192, "Ann") + if _, err := e.service(true).Clean(e.ctx, flac, []int64{escaped}); err == nil { + t.Fatal("a file outside the library was removed without an error") + } + if _, err := os.Stat(victim); err != nil { + t.Errorf("the file outside the library was removed: %v", err) + } +} + +func TestAReadOnlyFolderGivesAClearErrorAndRemovesNothing(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("permissions are not enforced here") + } + e := newEnv(t) + flac, mp3 := twoCopies(e) + dir := filepath.Join(e.dir, "b") + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); !errors.Is(err, duplicates.ErrReadOnly) { + t.Fatalf("got %v, want ErrReadOnly", err) + } + if !e.exists("b/song.mp3") || e.tracks() != 2 { + t.Error("something was removed") + } +} + +// Listening history must survive: it moves to the copy that stays. +func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + user, err := storage.NewUserRepo(e.db).CreateUser(e.ctx, "a@example.org", "h", false) + if err != nil { + t.Fatal(err) + } + plays := storage.NewListeningRepo(e.db) + for i, id := range []int64{mp3, mp3, flac} { + ev := listening.Event{UserID: user.ID, TrackID: id, Type: listening.Complete, DurationPlayedMs: 200_000, IdempotencyKey: string(rune('a' + i))} + if _, err := plays.Record(e.ctx, ev); err != nil { + t.Fatal(err) + } + } + if _, err := e.service(true).Clean(e.ctx, flac, []int64{mp3}); err != nil { + t.Fatal(err) + } + var n int + if err := e.db.QueryRow("SELECT COUNT(*) FROM listening_events WHERE track_id = ?", flac).Scan(&n); err != nil || n != 3 { + t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err) + } +} From 30f45bb3834ca33a526f59763cbedcfa2a471a42 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:46:50 +0100 Subject: [PATCH 5/8] feat(api): list the duplicate songs, check their files and delete the extra copies Admin only. GET /library/duplicates lists the groups with what is needed to choose and never a path; POST|GET|DELETE /library/duplicates/check is the job that fingerprints the files; POST /library/duplicates/clean deletes the chosen copies of one song, each refusal with its own status and code (deletion_disabled, scan_running, invalid_input, track_not_found, duplicate_changed, library_read_only). In the spec with the contract test, and wired in main, where an info line says when library.allow_delete is on. --- cmd/sondad/main.go | 14 + internal/api/http/api_test.go | 9 + internal/api/http/contract_test.go | 136 ++++++++ internal/api/http/duplicates_test.go | 375 +++++++++++++++++++++++ internal/api/http/handlers_duplicates.go | 169 ++++++++++ internal/api/http/router.go | 16 + internal/api/openapi/openapi.yaml | 258 ++++++++++++++++ 7 files changed, 977 insertions(+) create mode 100644 internal/api/http/duplicates_test.go create mode 100644 internal/api/http/handlers_duplicates.go diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 0007097..854e2d5 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -21,6 +21,7 @@ import ( "codeberg.org/squidd/sonda/internal/config" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -261,6 +262,17 @@ func run() error { Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery, } + // Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders: + // off unless the operator turned it on, and said in the log when it is on. + duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: cfg.Library.AllowDelete, + ScanRunning: func() bool { return scans.Status().Running }, + Logger: logger, + }) + if cfg.Library.AllowDelete { + logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders") + } + router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()), @@ -281,6 +293,8 @@ func run() error { CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)}, ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)}, DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)}, + Duplicates: duplicateSvc, + DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)}, ArtistImages: pictures, About: descriptions, Settings: userSettings, diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 82a9eb2..9a8d6ab 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -20,6 +20,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -115,6 +116,14 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun if deps.DescriptionsJob == nil { deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))} } + // Deleting is off unless a test turns it on. + if deps.Duplicates == nil { + svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{Logger: zap.NewNop()}) + deps.Duplicates = svc + if deps.DuplicatesJob == nil { + deps.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)} + } + } // Built from the final services, so a test that swaps one gets a Discover that uses it. if deps.Discover == nil { deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar), diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 90bd447..3b79aff 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -26,7 +26,9 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/images" + "codeberg.org/squidd/sonda/internal/jobs" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/lastfm" @@ -140,6 +142,11 @@ func newContract(t *testing.T) *contract { libs := library.NewLibraries(repo) libs.SetBusy(func() bool { return mgr.Status().Running }) d.Scan = scanAdapter{mgr} + dupSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: true, ScanRunning: func() bool { return mgr.Status().Running }, Logger: zap.NewNop(), + }) + d.Duplicates = dupSvc + d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(dupSvc)} d.Libraries = libs d.Catalog = catalog.NewService(storage.NewCatalogRepo(db)) d.Streams = streaming.NewService(repo) @@ -767,6 +774,9 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"nope": true}), 400, "a setting that does not exist", "invalid_input") c.expect(c.call("GET", "/me/settings", "", nil), 401, "settings without a token", "unauthorized") + // ---- duplicate songs ---- + c.duplicates(t, token, libID) + // ---- every operation in the spec was exercised ---- var missing []string for _, op := range specOperations(loadSpec(t)) { @@ -779,6 +789,132 @@ func TestResponsesMatchTheSpec(t *testing.T) { } } +// duplicates exercises the duplicate songs: the same song as a.mp3 is added to the library, as real +// files, so that what is checked against the spec is what the real services answer. +func (c *contract) duplicates(t *testing.T, token, libID string) { + t.Helper() + repo := storage.NewLibraryRepo(c.db) + libInt, err := strconv.ParseInt(libID, 10, 64) + if err != nil { + t.Fatal(err) + } + // addCopy puts a second file with the tags of Song A in the library, as the scanner would. + addCopy := func(name string) { + t.Helper() + path := filepath.Join(c.music, name) + if err := os.WriteFile(path, bytes.Repeat([]byte{0xAB}, 400), 0o644); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: name, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: "Song A", Album: "Compilation", AlbumArtist: "Artist X", Artists: []string{"Artist X"}, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: "mpeg", DurationMs: 180000, SampleRate: 44100}, + }, + } + if _, err := repo.SaveFile(context.Background(), libInt, f, 0); err != nil { + t.Fatal(err) + } + } + trackOf := func(name string) int64 { + t.Helper() + var id int64 + if err := c.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", name).Scan(&id); err != nil { + t.Fatal(err) + } + return id + } + exists := func(name string) bool { _, err := os.Lstat(filepath.Join(c.music, name)); return err == nil } + body := func(keep int64, remove ...int64) map[string]any { + return map[string]any{"keep_track_id": keep, "remove_track_ids": remove} + } + + none := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates when there are none") + if none["total"] != float64(0) || none["groups"] == nil || none["deletion_allowed"] != true { + t.Fatalf("duplicates = %v, want no group, an empty list and deleting allowed", none) + } + + addCopy("dup.mp3") + keep, copyID := trackOf("a.mp3"), trackOf("dup.mp3") + list := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates") + groups, _ := list["groups"].([]any) + if list["total"] != float64(1) || len(groups) != 1 { + t.Fatalf("duplicates = %v, want the one group of Song A", list) + } + if g := groups[0].(map[string]any); g["title"] != "Song A" || g["suggested_keep_track_id"] != float64(keep) || g["hashed"] != false { + t.Fatalf("group = %v, want Song A with a.mp3 to keep and nothing read yet", g) + } + c.expect(c.call("GET", "/library/duplicates?limit=-1", token, nil), 400, "duplicates with a bad limit", "invalid_input") + c.expect(c.call("GET", "/library/duplicates", "", nil), 401, "duplicates without a token", "unauthorized") + + // Not an admin: nothing of this is theirs. + c.expect(c.call("POST", "/users", token, map[string]any{"email": "carol@example.org", "password": "carols long pass"}), 201, "create a plain user") + carol := c.expect(c.call("POST", "/auth/login", "", map[string]string{"email": "carol@example.org", "password": "carols long pass"}), 200, "carol logs in")["access_token"].(string) + c.expect(c.call("GET", "/library/duplicates", carol, nil), 403, "duplicates as a plain user", "admin_required") + c.expect(c.call("POST", "/library/duplicates/clean", carol, body(keep, copyID)), 403, "cleaning as a plain user", "admin_required") + + // The check reads the files, and the group is then identical. + if idle := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check before it ran"); idle["running"] != false { + t.Fatalf("check = %v", idle) + } + c.expect(c.call("POST", "/library/duplicates/check?refresh=maybe", token, nil), 400, "a check with a bad refresh", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/check", "", nil), 401, "a check without a token", "unauthorized") + c.expect(c.call("POST", "/library/duplicates/check", token, nil), 202, "start the check") + for i := 0; ; i++ { + st := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check running") + if st["running"] == false { + if last, _ := st["last"].(map[string]any); last == nil || last["checked"] != float64(2) || last["hashed"] != float64(2) { + t.Fatalf("check = %v, want 2 files checked and hashed", st) + } + break + } + if i > 400 { + t.Fatal("the check did not finish") + } + time.Sleep(25 * time.Millisecond) + } + c.expect(c.call("DELETE", "/library/duplicates/check", token, nil), 204, "stop the check when none runs") + after := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates after the check") + if g := after["groups"].([]any)[0].(map[string]any); g["hashed"] != true || g["identical"] != true { + t.Fatalf("group = %v, want hashed and identical", g) + } + + // Refusals first: nothing is removed by any of them. + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, keep)), 400, "cleaning the copy to keep", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/clean", token, map[string]any{"keep_track_id": keep}), 400, "cleaning with nothing to remove", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, 99999)), 404, "cleaning a track that does not exist", "track_not_found") + c.expect(c.call("POST", "/library/duplicates/clean", "", body(keep, copyID)), 401, "cleaning without a token", "unauthorized") + other := trackOf("b.flac") + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, other)), 400, "cleaning something that is not a copy", "invalid_input") + if !exists("dup.mp3") || !exists("b.flac") { + t.Fatal("a refused request removed a file") + } + + // A copy that changed since the scan is not removed. + addCopy("dup2.mp3") + dup2 := trackOf("dup2.mp3") + if err := os.WriteFile(filepath.Join(c.music, "dup2.mp3"), bytes.Repeat([]byte{0xCD}, 500), 0o644); err != nil { + t.Fatal(err) + } + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, dup2)), 409, "cleaning a copy that changed", "duplicate_changed") + if !exists("dup2.mp3") { + t.Fatal("a copy that changed was removed") + } + + // The real thing: the extra copy leaves the disk and the index, the other stays. + done := c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, copyID)), 200, "clean the duplicate") + if done["removed"] != float64(1) || done["freed_bytes"] != float64(400) { + t.Fatalf("clean = %v, want 1 file and 400 bytes", done) + } + if exists("dup.mp3") || !exists("a.mp3") { + t.Fatalf("after the clean dup.mp3 exists=%v a.mp3 exists=%v", exists("dup.mp3"), exists("a.mp3")) + } +} + type orphanIDs struct{ album, track string } // addOrphan inserts an album with no album artist and a track in it whose file diff --git a/internal/api/http/duplicates_test.go b/internal/api/http/duplicates_test.go new file mode 100644 index 0000000..ac38afd --- /dev/null +++ b/internal/api/http/duplicates_test.go @@ -0,0 +1,375 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/storage" +) + +// dupAPI is the API over a real library on a real folder. +type dupAPI struct { + t *testing.T + h http.Handler + db *bun.DB + dir string + token string + scanning bool +} + +func newDupAPI(t *testing.T, allow bool) *dupAPI { + t.Helper() + a := &dupAPI{t: t} + a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { + svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(), + }) + d.Duplicates = svc + d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)} + }) + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + a.dir = dir + a.token = registerAndLogin(t, a.h) + if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil { + t.Fatal(err) + } + return a +} + +// seed writes a file and indexes it as it is, and returns its track. +func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + a.t.Helper() + path := filepath.Join(a.dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + a.t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + a.t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + a.t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate}, + }, + } + if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil { + a.t.Fatal(err) + } + var id int64 + if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil { + a.t.Fatal(err) + } + return id +} + +func (a *dupAPI) twoCopies() (flac, mp3 int64) { + flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann") + mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann") + return flac, mp3 +} + +func (a *dupAPI) exists(rel string) bool { + _, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel))) + return err == nil +} + +func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) } + +func (a *dupAPI) clean(keep int64, remove ...int64) resp { + return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove}) +} + +func TestDuplicateRoutesAreAdminOnly(t *testing.T) { + a := newDupAPI(t, true) + _, bob := addUser(t, a.h, a.token, "bob", false) + calls := []struct{ method, path string }{ + {"GET", "/api/v1/library/duplicates"}, + {"POST", "/api/v1/library/duplicates/clean"}, + {"GET", "/api/v1/library/duplicates/check"}, + {"POST", "/api/v1/library/duplicates/check"}, + {"DELETE", "/api/v1/library/duplicates/check"}, + } + for _, c := range calls { + if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code) + } + r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}}) + if r.Code != http.StatusForbidden || errCode(r) != "admin_required" { + t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body) + } + } +} + +func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann") + + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil) + req.Header.Set("Authorization", "Bearer "+a.token) + a.h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body) + } + raw := rec.Body.String() + for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} { + if strings.Contains(raw, secret) { + t.Errorf("the answer contains %q: file paths are private", secret) + } + } + + var out struct { + DeletionAllowed bool `json:"deletion_allowed"` + Total, Limit int + Offset int + Groups []struct { + Title string `json:"title"` + Artists []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"artists"` + Files []struct { + TrackID int64 `json:"track_id"` + Album *struct { + ID int64 `json:"id"` + Title string `json:"title"` + } `json:"album"` + Format string `json:"format"` + Bitrate int `json:"bitrate"` + SizeBytes int64 `json:"size_bytes"` + DurationMs int64 `json:"duration_ms"` + Hashed bool `json:"hashed"` + } `json:"files"` + Hashed bool `json:"hashed"` + Identical bool `json:"identical"` + Keep int64 `json:"suggested_keep_track_id"` + FreedBytes int64 `json:"freed_bytes"` + } `json:"groups"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 { + t.Fatalf("answer = %+v", out) + } + g := out.Groups[0] + if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical { + t.Errorf("group = %+v", g) + } + if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" || + g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed { + t.Errorf("files = %+v", g.Files) + } +} + +func TestTheListSaysWhenDeletingIsOff(t *testing.T) { + a := newDupAPI(t, false) + a.twoCopies() + r := a.get("/api/v1/library/duplicates") + if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) { + t.Errorf("answer = %d %v", r.Code, r.Body) + } +} + +func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) { + a := newDupAPI(t, true) + for _, title := range []string{"A song", "B song"} { + a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann") + a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann") + } + first := a.get("/api/v1/library/duplicates?limit=1") + second := a.get("/api/v1/library/duplicates?limit=1&offset=1") + if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 { + t.Fatalf("pages = %v and %v", first.Body, second.Body) + } + if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) { + t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body) + } + if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil { + t.Errorf("past the end: %v, want an empty list and not null", past.Body) + } + if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) { + t.Errorf("a huge limit became %v, want 200", big.Body["limit"]) + } + for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} { + if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body) + } + } +} + +func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + r := a.clean(flac, mp3) + if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) { + t.Fatalf("answer = %d %v", r.Code, r.Body) + } + if a.exists("b/song.mp3") || !a.exists("a/song.flac") { + t.Error("the wrong file was removed") + } + if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) { + t.Errorf("the list still shows %v groups", list.Body["total"]) + } +} + +func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) { + t.Run("deleting is off", func(t *testing.T) { + a := newDupAPI(t, false) + flac, mp3 := a.twoCopies() + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a scan is running", func(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + a.scanning = true + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("not copies of one song", func(t *testing.T) { + a := newDupAPI(t, true) + flac, _ := a.twoCopies() + other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann") + if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a track that does not exist", func(t *testing.T) { + a := newDupAPI(t, true) + flac, _ := a.twoCopies() + if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a file that changed since the scan", func(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil { + t.Fatal(err) + } + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a read-only folder", func(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("permissions are not enforced here") + } + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + dir := filepath.Join(a.dir, "b") + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) +} + +func TestACleanRequestMustBeWellFormed(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + bodies := map[string]any{ + "malformed JSON": `{"keep_track_id":`, + "no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}}, + "nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}}, + "the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}}, + "a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}}, + "ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}}, + } + for name, body := range bodies { + r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body) + if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body) + } + } + if !a.exists("b/song.mp3") || !a.exists("a/song.flac") { + t.Error("a file was removed by a bad request") + } +} + +// --- the check job --------------------------------------------------------------------------------- + +func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) { + a := newDupAPI(t, true) + a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann") + a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann") + + if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false { + t.Fatalf("before it ran: %d %v", idle.Code, idle.Body) + } + if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" { + t.Errorf("a bad refresh: %d %v", r.Code, r.Body) + } + if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" { + t.Fatalf("start: %d %v", r.Code, r.Body) + } + var st resp + for i := 0; ; i++ { + st = a.get("/api/v1/library/duplicates/check") + if st.Body["running"] == false { + break + } + if i > 400 { + t.Fatal("the job did not finish") + } + time.Sleep(25 * time.Millisecond) + } + last, _ := st.Body["last"].(map[string]any) + if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false { + t.Errorf("last = %v, want 2 checked and 2 hashed", last) + } + if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true { + t.Errorf("the group is not called identical after the check: %v", list.Body) + } + if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent { + t.Errorf("stop: %d", r.Code) + } +} + +type alreadyRunning struct{} + +func (alreadyRunning) Start(bool) error { return jobs.ErrRunning } +func (alreadyRunning) Stop() bool { return false } +func (alreadyRunning) Status() jobs.Status { return jobs.Status{} } + +func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) { + a := &dupAPI{t: t} + a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { + d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{}) + d.DuplicatesJob = alreadyRunning{} + }) + a.token = registerAndLogin(t, a.h) + r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil) + if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" { + t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body) + } +} diff --git a/internal/api/http/handlers_duplicates.go b/internal/api/http/handlers_duplicates.go new file mode 100644 index 0000000..969f048 --- /dev/null +++ b/internal/api/http/handlers_duplicates.go @@ -0,0 +1,169 @@ +package httpapi + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" +) + +// duplicateHandlers serve the admin's view of the songs that are in more than one file, and the +// removal of the extra copies. File paths never appear in any answer. +type duplicateHandlers struct{ svc duplicateService } + +const ( + defaultDuplicatesLimit = 50 + maxDuplicatesLimit = 200 +) + +type duplicateFileJSON struct { + TrackID int64 `json:"track_id"` + Album *albumRefJSON `json:"album"` + Format string `json:"format"` + Bitrate int `json:"bitrate"` + SizeBytes int64 `json:"size_bytes"` + DurationMs int64 `json:"duration_ms"` + Hashed bool `json:"hashed"` +} + +type duplicateGroupJSON struct { + Title string `json:"title"` + Artists []artistRefJSON `json:"artists"` + Files []duplicateFileJSON `json:"files"` + Hashed bool `json:"hashed"` + Identical bool `json:"identical"` + SuggestedID int64 `json:"suggested_keep_track_id"` + FreedBytes int64 `json:"freed_bytes"` +} + +type duplicatePageJSON struct { + DeletionAllowed bool `json:"deletion_allowed"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Groups []duplicateGroupJSON `json:"groups"` +} + +// list gives the groups of copies, a page at a time. +func (h *duplicateHandlers) list(c *gin.Context) { + limit, ok := queryInt(c, "limit") + if !ok { + return + } + offset, ok := queryInt(c, "offset") + if !ok { + return + } + if limit < 0 || offset < 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "limit and offset cannot be negative") + return + } + size := clampInt(limit) + if size == 0 { + size = defaultDuplicatesLimit + } + size = min(size, maxDuplicatesLimit) + + page, err := h.svc.Groups(c.Request.Context(), size, clampInt(offset)) + if err != nil { + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + out := duplicatePageJSON{ + DeletionAllowed: page.DeletionAllowed, Total: page.Total, Limit: size, Offset: clampInt(offset), + Groups: make([]duplicateGroupJSON, len(page.Groups)), + } + for i, g := range page.Groups { + out.Groups[i] = toDuplicateGroupJSON(g) + } + c.JSON(http.StatusOK, out) +} + +func toDuplicateGroupJSON(g duplicates.Group) duplicateGroupJSON { + out := duplicateGroupJSON{ + Title: g.Title, Artists: make([]artistRefJSON, len(g.Artists)), Files: make([]duplicateFileJSON, len(g.Files)), + Hashed: g.Hashed, Identical: g.Identical, SuggestedID: g.KeepTrackID, FreedBytes: g.FreedBytes, + } + for i, a := range g.Artists { + out.Artists[i] = artistRefJSON{ID: a.ID, Name: a.Name} + } + for i, f := range g.Files { + file := duplicateFileJSON{ + TrackID: f.TrackID, Format: f.Format, Bitrate: f.BitRate, SizeBytes: f.SizeBytes, DurationMs: f.DurationMs, Hashed: f.Hash != "", + } + if f.AlbumID != 0 { + file.Album = &albumRefJSON{ID: f.AlbumID, Title: f.Album} + } + out.Files[i] = file + } + return out +} + +type cleanRequest struct { + KeepTrackID int64 `json:"keep_track_id"` + RemoveTrackIDs []int64 `json:"remove_track_ids"` +} + +// clean removes the copies named of one song, keeping the one named. +func (h *duplicateHandlers) clean(c *gin.Context) { + var req cleanRequest + if !bindJSON(c, &req) { + return + } + if req.KeepTrackID <= 0 || len(req.RemoveTrackIDs) == 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "name the copy to keep and at least one copy to remove") + return + } + for _, id := range req.RemoveTrackIDs { + if id <= 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "track ids must be positive numbers") + return + } + } + res, err := h.svc.Clean(c.Request.Context(), req.KeepTrackID, req.RemoveTrackIDs) + if err != nil { + writeDuplicatesError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"removed": res.Removed, "freed_bytes": res.FreedBytes}) +} + +// writeDuplicatesError gives each way a removal is refused its own status and code, so a client can tell them apart. +func writeDuplicatesError(c *gin.Context, err error) { + switch { + case errors.Is(err, duplicates.ErrDisabled): + writeError(c, http.StatusConflict, "deletion_disabled", "deleting files is not turned on: set library.allow_delete") + case errors.Is(err, duplicates.ErrScanRunning): + writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish") + case errors.Is(err, duplicates.ErrInvalid): + writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + case errors.Is(err, duplicates.ErrTrackNotFound): + writeError(c, http.StatusNotFound, "track_not_found", "no such track") + case errors.Is(err, duplicates.ErrChanged): + writeError(c, http.StatusConflict, "duplicate_changed", "a file is not what the last scan saw: scan again") + case errors.Is(err, duplicates.ErrReadOnly): + writeError(c, http.StatusConflict, "library_read_only", "the music folder cannot be written to") + default: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + } +} + +type duplicatesCheckSummaryJSON struct { + Checked int `json:"checked"` + Hashed int `json:"hashed"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + Stopped bool `json:"stopped"` +} + +// duplicatesCheckSummary is the summary of the job that fingerprints the files of the groups. +func duplicatesCheckSummary(s *jobs.Summary) any { + return duplicatesCheckSummaryJSON{ + Checked: s.Checked, Hashed: s.Outcomes[duplicates.OutcomeHashed], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped, + } +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 41e517c..f611a83 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -18,6 +18,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -105,6 +106,11 @@ type jobService interface { Status() jobs.Status } +type duplicateService interface { + Groups(ctx context.Context, limit, offset int) (duplicates.Page, error) + Clean(ctx context.Context, keep int64, remove []int64) (duplicates.Result, error) +} + type coverService interface { AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error) } @@ -155,6 +161,8 @@ type Deps struct { CoversJob jobService ArtistImagesJob jobService DescriptionsJob jobService + Duplicates duplicateService + DuplicatesJob jobService About aboutService ArtistImages artistImageService Settings userSettingsService @@ -274,6 +282,14 @@ func NewRouter(d Deps) *gin.Engine { admin.DELETE("/library/artist-images", aj.stop) admin.POST("/library/descriptions", dj.start) admin.DELETE("/library/descriptions", dj.stop) + + dup := &duplicateHandlers{svc: d.Duplicates} + dupJob := &jobHandlers{job: d.DuplicatesJob, runningCode: "duplicates_check_running", summary: duplicatesCheckSummary} + admin.GET("/library/duplicates", dup.list) + admin.POST("/library/duplicates/clean", dup.clean) + admin.GET("/library/duplicates/check", dupJob.status) + admin.POST("/library/duplicates/check", dupJob.start) + admin.DELETE("/library/duplicates/check", dupJob.stop) libs := &librariesHandlers{svc: d.Libraries} admin.GET("/libraries", libs.list) admin.POST("/libraries", libs.create) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 2b15625..8d9b8ea 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -715,6 +715,162 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/AdminRequired'} + /library/duplicates: + get: + tags: [Scan] + operationId: listDuplicates + summary: The songs that are in more than one file + description: | + Admin only. Groups of files that are the same song by the same artists, worked out from the index and + never from the disk, so it is quick and touches no file. Two files are the same song when they have the + same set of main artists, the same title (the key Sonda matches titles by) and durations no more than + 3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches, + a file flagged as missing does not count, and a copy may be on a different album: `files` says which. + + Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC) + before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when + every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when, + besides, they are byte for byte the same. No file path is ever returned. + + `deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete` + off (the default) the groups can still be seen. + security: [{bearerAuth: []}] + parameters: + - {$ref: '#/components/parameters/Limit'} + - {$ref: '#/components/parameters/Offset'} + responses: + '200': + description: A page of groups, in a fixed order (by title). + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesPage'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + + /library/duplicates/check: + get: + tags: [Scan] + operationId: getDuplicatesCheck + summary: Progress of the check that reads the files of the duplicates + description: | + Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended. + The check is described at `POST /library/duplicates/check`. + security: [{bearerAuth: []}] + responses: + '200': + description: The job. + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesCheckStatus'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + post: + tags: [Scan] + operationId: startDuplicatesCheck + summary: Read the files of the duplicates, to tell which are identical + description: | + Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in + a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be + called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a + network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the + fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file + again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files + cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it. + security: [{bearerAuth: []}] + parameters: + - name: refresh + in: query + description: "`true` reads every file again, also those that have a fingerprint. `false` when absent." + schema: {type: boolean, default: false} + responses: + '202': + description: The check was started in the background. + content: + application/json: + schema: + type: object + additionalProperties: false + required: [status] + properties: + status: {type: string, enum: [started]} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '409': + description: The check is already running (`duplicates_check_running`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '500': + description: The groups could not be worked out (`internal`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + delete: + tags: [Scan] + operationId: stopDuplicatesCheck + summary: Stop the check of the duplicates + description: | + Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It + answers the same when no check is running. + security: [{bearerAuth: []}] + responses: + '204': {description: "Done, also when no check was running."} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + + /library/duplicates/clean: + post: + tags: [Scan] + operationId: cleanDuplicates + summary: Delete the extra copies of one song + description: | + Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it + is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder + has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies + `remove_track_ids` of one song and keeps `keep_track_id`. + + The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is + running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of + `GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to + remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to + remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed` + for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of + a song is never removed. Every file is checked before the first is removed. + + Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a + link) and its track leaves the index; **the listens of the removed track become listens of the track that + stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for + example the folder turns out to be read-only for the second file), what was removed stays removed. + security: [{bearerAuth: []}] + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/CleanDuplicates'} + responses: + '200': + description: The extra copies were removed. + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesCleaned'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '404': + description: A track that was named does not exist (`track_not_found`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '409': + description: | + Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last + scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /library/scan: get: tags: [Scan] @@ -2800,6 +2956,108 @@ components: stopped: {type: boolean, description: "It was stopped, or the application was."} last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."} + DuplicatesPage: + type: object + additionalProperties: false + required: [deletion_allowed, total, limit, offset, groups] + properties: + deletion_allowed: {type: boolean, description: "Whether `POST /library/duplicates/clean` can work (`library.allow_delete`)."} + total: {type: integer, description: "Groups in all, not in this page."} + limit: {type: integer} + offset: {type: integer} + groups: + type: array + items: {$ref: '#/components/schemas/DuplicateGroup'} + + DuplicateGroup: + type: object + additionalProperties: false + required: [title, artists, files, hashed, identical, suggested_keep_track_id, freed_bytes] + properties: + title: {type: string} + artists: + type: array + description: The main artists the copies share. + items: {$ref: '#/components/schemas/ArtistRef'} + files: + type: array + description: The copies, by track id; at least two. + items: {$ref: '#/components/schemas/DuplicateFile'} + hashed: {type: boolean, description: "Every file of the group has been read by the check."} + identical: {type: boolean, description: "Hashed, and byte for byte the same."} + suggested_keep_track_id: {type: integer, format: int64, description: "The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest."} + freed_bytes: {type: integer, format: int64, description: "The size of every copy but the one to keep."} + + DuplicateFile: + type: object + additionalProperties: false + required: [track_id, album, format, bitrate, size_bytes, duration_ms, hashed] + properties: + track_id: {type: integer, format: int64} + album: + type: object + nullable: true + additionalProperties: false + required: [id, title] + properties: + id: {type: integer, format: int64} + title: {type: string} + format: {type: string, description: "`mp3`, `flac`..."} + bitrate: {type: integer, description: "0 when unknown."} + size_bytes: {type: integer, format: int64} + duration_ms: {type: integer, format: int64} + hashed: {type: boolean, description: "This file has been read by the check."} + + CleanDuplicates: + type: object + additionalProperties: false + required: [keep_track_id, remove_track_ids] + properties: + keep_track_id: {type: integer, format: int64, minimum: 1, description: "The copy that stays."} + remove_track_ids: + type: array + minItems: 1 + maxItems: 100 + description: "The copies to delete, of the same song as the one that stays, none twice." + items: {type: integer, format: int64, minimum: 1} + + DuplicatesCleaned: + type: object + additionalProperties: false + required: [removed, freed_bytes] + properties: + removed: {type: integer, description: "Files deleted."} + freed_bytes: {type: integer, format: int64} + + DuplicatesCheckStatus: + type: object + additionalProperties: false + required: [running] + properties: + running: {type: boolean} + started_at: {type: string, format: date-time, description: "When the last check started."} + last_finished_at: {type: string, format: date-time} + progress: + type: object + description: Only while the check runs. + additionalProperties: false + required: [done, total] + properties: + done: {type: integer} + total: {type: integer, description: "Files the check set out to read."} + last: + type: object + description: How the last finished check ended. + additionalProperties: false + required: [checked, hashed, skipped, failed, stopped] + properties: + checked: {type: integer, description: "Files looked at."} + hashed: {type: integer, description: "Files read and fingerprinted."} + skipped: {type: integer, description: "Files that changed since the scan, or are gone."} + failed: {type: integer, description: "Files that could not be read just then."} + stopped: {type: boolean, description: "It was stopped, or the application was."} + last_error: {type: string, description: "Why the last check ended before the end, for example that the disk is not mounted."} + UserSettings: type: object additionalProperties: false From 7180a747e9bf24d295e81cfea0ac496bf76a79f9 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:50:11 +0100 Subject: [PATCH 6/8] docs: duplicates, library.allow_delete and the one exception to read-only music folders README: the Duplicates feature, the new setting and its environment variable, the endpoints, and the two places that promised the music folders are never written. The example config and the compose file say how to turn it on and that the folder must then be writable. --- README.md | 9 ++++++--- compose.yaml | 2 +- sonda.example.toml | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a0ffba6..34db973 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr ## What works today - **Accounts.** An account is an email address and a password. The first user to register becomes the admin and registration then closes; admins create the other users. The address only identifies the account: Sonda sends no mail, does not check that the address exists, and keeps it on your server. It is trimmed and lowercased, and unique. Accounts made before addresses were asked for keep the plain name they had as their login. Short-lived access tokens (JWT) plus rotating refresh tokens, so a stolen refresh token is detected. -- **Libraries.** A library is a name and a folder, managed through the API. Sonda only ever reads your music folders: it never writes, renames or deletes a file. +- **Libraries.** A library is a name and a folder, managed through the API. Sonda only reads your music folders: it never writes or renames a file, and it deletes one only when an admin asks to remove the extra copy of a song and you turned on `library.allow_delete` (see **Duplicates**), which is off by default. - **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped. - **Catalog.** Browse and search artists, albums and tracks, with paging, by library, artist or album. Only tracks you can actually play are listed. +- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens of a deleted copy move to the one that stays, so your taste is not lost. Folders are never removed, and nothing is sent anywhere. - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. @@ -60,7 +61,7 @@ docker run -d --name sonda --restart unless-stopped --stop-timeout 15 \ sonda ``` -Mount your music read-only wherever you like, then add the library through the API with the path *inside* the container, `/music` above (not the folder of your host): the [Demo](#demo) works the same way with `BASE=http://127.0.0.1:8080/api/v1`, except that step 4 creates the library with `"path":"/music"`. Keep `sonda.env` and pass it every time you run a new container, for example after an upgrade: a new secret signs out everybody. The container's health check asks the port in `SONDA_LISTEN`, so it works for `0.0.0.0`, `[::]` and a port alone, but not if you bind the server to one interface address. Published like this the server answers only on this machine; to serve your network, publish the port without `127.0.0.1:` and put a reverse proxy with https in front. The other settings are the environment variables of the [table below](#configure) (`SONDA_DB_DRIVER` and `SONDA_DB_DSN` for PostgreSQL); the image sets `SONDA_LISTEN`, `SONDA_DB_PATH` and `SONDA_DATA_DIR` for the volume. +Mount your music read-only wherever you like (only if you want Sonda to delete duplicate files, mount it writable and set `SONDA_ALLOW_DELETE=true`: see **Duplicates**), then add the library through the API with the path *inside* the container, `/music` above (not the folder of your host): the [Demo](#demo) works the same way with `BASE=http://127.0.0.1:8080/api/v1`, except that step 4 creates the library with `"path":"/music"`. Keep `sonda.env` and pass it every time you run a new container, for example after an upgrade: a new secret signs out everybody. The container's health check asks the port in `SONDA_LISTEN`, so it works for `0.0.0.0`, `[::]` and a port alone, but not if you bind the server to one interface address. Published like this the server answers only on this machine; to serve your network, publish the port without `127.0.0.1:` and put a reverse proxy with https in front. The other settings are the environment variables of the [table below](#configure) (`SONDA_DB_DRIVER` and `SONDA_DB_DSN` for PostgreSQL); the image sets `SONDA_LISTEN`, `SONDA_DB_PATH` and `SONDA_DATA_DIR` for the volume. To keep the data in a folder of the host instead of a volume, run the container as the owner of that folder, or it cannot write there: add `--user "$(id -u):$(id -g)"` and `-v /path/to/sonda-data:/data`. Back up `sonda.db` (and treat the backup like a secret: it holds the API keys of the providers); the rest of `/data` is built from your music. @@ -86,6 +87,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An |---|---|---|---| | `server.listen` | `SONDA_LISTEN` | `127.0.0.1:8080` | Address to listen on. Localhost by default. | | `library.scan_workers` | | `4` | Files whose tags are read in parallel during a scan. | +| `library.allow_delete` | `SONDA_ALLOW_DELETE` | `false` | Lets an admin delete the extra copy of a song from the music folders (see **Duplicates**). It is the one thing that makes Sonda write there, so it needs the folders mounted writable. A value that is not `true` or `false` is an error. | | `database.driver` | `SONDA_DB_DRIVER` | `sqlite` | `sqlite` or `postgres`. | | `database.path` | `SONDA_DB_PATH` | `sonda.db` | SQLite file (driver `sqlite`). | | `database.dsn` | `SONDA_DB_DSN` | | Connection string (driver `postgres`). | @@ -298,6 +300,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | admin | `GET`, `POST /libraries`; `PATCH`, `DELETE /libraries/{id}` | Manage libraries. Deleting one removes its index entries, never your files. | | admin | `POST`, `DELETE /library/covers`; `POST`, `DELETE /library/artist-images`; `POST`, `DELETE /library/descriptions` | Start, in the background, a job that builds the cover of every album, looks for the picture of every artist or for the description of every artist (`?refresh=true` for all), or stop it. One of each at a time. | | admin | `POST /library/scan`, `POST /libraries/{id}/scan` | Scan everything, or one library. `?full=true` reads every file again. | +| admin | `GET /library/duplicates`; `POST`, `GET`, `DELETE /library/duplicates/check`; `POST /library/duplicates/clean` | Find the songs that are in more than one file, read their files to tell the identical ones, and delete the extra copies. See **Duplicates**. | | admin | `GET /providers`, `PUT /providers/{name}`, `POST /providers/{name}/test`, `DELETE /providers/{name}/data` | See, set up and check the providers (Last.fm, MusicBrainz, Wikipedia, Commons, the Cover Art Archive, Lidarr), and remove what they gave. The API key is write-only. | | admin | `GET /providers/lidarr/options` | The profiles and root folders that Lidarr offers, to choose the ones artists are sent with. | | admin | `POST /artists/{id}/lidarr` | Send an artist to your own Lidarr, by its MusicBrainz id, which then looks for the music. Never by name, never by itself. | @@ -310,7 +313,7 @@ A library path must be an absolute, existing folder, never the filesystem root, - Sonda listens on localhost only, unless you change `server.listen`. To use it from other machines, put it behind a reverse proxy that terminates HTTPS. - Passwords are hashed with Argon2id. Refresh tokens are stored only as hashes. An access token is checked against the database on every request, so it stops working as soon as its account is deleted, even if the same user id is given to a new account. -- Music folders are read-only to Sonda. Streaming opens files through `os.Root`, so a path can never leave its library, even if a symlink is swapped in later. +- Music folders are read-only to Sonda, except for one thing that is off until you turn it on: deleting the extra copy of a song (`library.allow_delete`). Streaming and that deletion open files through `os.Root`, so a path can never leave its library, even if a symlink is swapped in later. - File paths are never returned by the catalog, and audio file paths are not logged. - Provider API keys are write-only through the API and never logged. They sit in the database, so protect your backups. diff --git a/compose.yaml b/compose.yaml index b8a96c2..041bc88 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,7 +21,7 @@ services: # TZ: Europe/Rome volumes: - sonda-data:/data # the database and the files Sonda makes: back up the database - - ${MUSIC_DIR:?set MUSIC_DIR in .env to the folder of your music}:/music:ro # your music, read-only: a library is added with the path /music + - ${MUSIC_DIR:?set MUSIC_DIR in .env to the folder of your music}:/music:ro # your music, read-only: a library is added with the path /music. To let Sonda delete duplicate files, drop ":ro" and set SONDA_ALLOW_DELETE: "true" above volumes: sonda-data: diff --git a/sonda.example.toml b/sonda.example.toml index 823ba48..57399e6 100644 --- a/sonda.example.toml +++ b/sonda.example.toml @@ -9,6 +9,9 @@ listen = "127.0.0.1:8080" # localhost by default # Libraries (name + directory) are managed through the API: POST /api/v1/libraries. [library] scan_workers = 4 # parallel tag readers during a scan +# allow_delete = true # let an admin delete the extra copy of a song from the music folders. + # Off by default: it is the one thing that makes Sonda write there, and it + # needs the folders mounted writable. Or set SONDA_ALLOW_DELETE=true. [database] driver = "sqlite" # "sqlite" or "postgres" From 05f4e33470ece649e6430128f46b637a27f1523d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 16:52:38 +0100 Subject: [PATCH 7/8] fix(duplicates): finish updating the index even if the client leaves after the file is deleted Found reviewing the branch: a request cancelled between deleting the file and updating the index left a track whose file does not exist. The index step no longer depends on the request's context. Test first: it cancels the request at exactly that moment. --- internal/duplicates/service.go | 7 +++--- internal/duplicates/service_test.go | 34 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/internal/duplicates/service.go b/internal/duplicates/service.go index 185f309..1a9e7bb 100644 --- a/internal/duplicates/service.go +++ b/internal/duplicates/service.go @@ -181,9 +181,10 @@ func (s *Service) Clean(ctx context.Context, keep int64, remove []int64) (Result if err := removeFile(t.root, t.rel); err != nil { return res, err } - // The file is gone. If the index cannot follow, the next scan flags the path as missing, and - // nothing is left broken; the error says so. - if err := s.store.RemoveCopy(ctx, keep, t.id); err != nil { + // The file is gone, so the index must follow whether or not the client is still there: a request + // that was cancelled now would leave a track whose file does not exist. If the index still cannot + // follow, the next scan flags the path as missing, and nothing is left broken; the error says so. + if err := s.store.RemoveCopy(context.WithoutCancel(ctx), keep, t.id); err != nil { s.log.Error("a duplicate file was removed but its track is still in the index; a scan will flag it as missing", zap.Int64("track_id", t.id), zap.Int64("kept_track_id", keep), zap.Error(err)) return res, fmt.Errorf("the file was removed but the index could not be updated: %w", err) diff --git a/internal/duplicates/service_test.go b/internal/duplicates/service_test.go index 576c8d7..56367c0 100644 --- a/internal/duplicates/service_test.go +++ b/internal/duplicates/service_test.go @@ -495,3 +495,37 @@ func TestTheListensOfARemovedCopyMoveToTheOneThatStays(t *testing.T) { t.Errorf("the copy that stays has %d listens, want 3 (err=%v)", n, err) } } + +// cancelsBeforeUpdating ends the request at the worst moment: after the file was deleted, when the +// index is about to follow. The context the service hands to the index is the one that is cancelled, +// unless the service has made sure that what it began is finished. +type cancelsBeforeUpdating struct { + duplicates.Store + cancel context.CancelFunc +} + +func (s cancelsBeforeUpdating) RemoveCopy(ctx context.Context, keep, remove int64) error { + s.cancel() + return s.Store.RemoveCopy(ctx, keep, remove) +} + +// A client that disconnects once the file is deleted must not leave the index behind: the file is gone, +// so the track has to go too, whether or not anyone is still waiting for the answer. +func TestTheIndexIsUpdatedEvenIfTheClientLeavesAfterTheFileIsDeleted(t *testing.T) { + e := newEnv(t) + flac, mp3 := twoCopies(e) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + svc := duplicates.NewService(cancelsBeforeUpdating{Store: e.store, cancel: cancel}, + duplicates.Options{AllowDelete: true, Logger: zap.NewNop()}) + + if _, err := svc.Clean(ctx, flac, []int64{mp3}); err != nil { + t.Fatalf("got %v: the request was cancelled, but the work had begun and has to be finished", err) + } + if e.exists("b/song.mp3") { + t.Error("the copy is still on the disk") + } + if e.tracks() != 1 { + t.Errorf("%d tracks, want 1: the file is gone but its track is still in the index", e.tracks()) + } +} From ce109a14a3321a54a6e3c63b2bd6c5501c7be162 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Thu, 12 Nov 2026 17:01:36 +0100 Subject: [PATCH 8/8] fix(duplicates): move the playlist items of a removed copy to the one that stays playlist_items.track_id is ON DELETE CASCADE (added by the playlists work on main), so deleting a duplicate would have taken the song out of every user's playlists, as it would have done with the listens. RemoveCopy now repoints the items first, in the same transaction. Also merges main (playlists) into the branch. --- README.md | 2 +- internal/duplicates/service.go | 2 +- internal/storage/duplicates.go | 13 ++++++--- internal/storage/duplicates_test.go | 43 +++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9339570..5ce1c8d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Libraries.** A library is a name and a folder, managed through the API. Sonda only reads your music folders: it never writes or renames a file, and it deletes one only when an admin asks to remove the extra copy of a song and you turned on `library.allow_delete` (see **Duplicates**), which is off by default. - **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped. - **Catalog.** Browse and search artists, albums and tracks, with paging, by library, artist or album. Only tracks you can actually play are listed. -- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens of a deleted copy move to the one that stays, so your taste is not lost. Folders are never removed, and nothing is sent anywhere. +- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the tracks are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens and the playlist items of a deleted copy move to the one that stays, so your taste and your playlists are not lost. Folders are never removed, and nothing is sent anywhere. - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. diff --git a/internal/duplicates/service.go b/internal/duplicates/service.go index 1a9e7bb..e203bda 100644 --- a/internal/duplicates/service.go +++ b/internal/duplicates/service.go @@ -91,7 +91,7 @@ type Result struct { // Build, the copy to keep is still the file that was scanned (so it is never the last copy that is // removed), and so is every copy to remove (a file that changed, a link, or a path that leaves the // library is refused). The files are checked all together before the first is removed. Each removed -// file takes its track out of the index, and its listens go to the track that stays. +// file takes its track out of the index, and its listens and playlist items go to the track that stays. // // On an error part way (the folder turned out to be read-only on the second file), the Result says // what was already removed. diff --git a/internal/storage/duplicates.go b/internal/storage/duplicates.go index 93965ec..9df531e 100644 --- a/internal/storage/duplicates.go +++ b/internal/storage/duplicates.go @@ -176,10 +176,10 @@ func (r *DuplicatesRepo) SaveHash(ctx context.Context, fileID, sizeBytes, mtimeU return n > 0, nil } -// RemoveCopy takes a track out of the index in favour of another that stays: the listens of the -// removed track become listens of the one that stays (they would be deleted with it otherwise), and -// the track goes, with its file, credits and genres. The covers of the albums involved are built -// again. It is one transaction. +// RemoveCopy takes a track out of the index in favour of another that stays: the listens and the +// playlist items of the removed track move to the one that stays (they would be deleted with it +// otherwise), and the track goes, with its file, credits and genres. The covers of the albums +// involved are built again. It is one transaction. func (r *DuplicatesRepo) RemoveCopy(ctx context.Context, keepTrackID, removeTrackID int64) error { if keepTrackID == removeTrackID { return duplicates.ErrInvalid @@ -200,6 +200,11 @@ func (r *DuplicatesRepo) RemoveCopy(ctx context.Context, keepTrackID, removeTrac if _, err := tx.ExecContext(ctx, "UPDATE listening_events SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil { return fmt.Errorf("move the listens: %w", err) } + // Playlist items go the same way: an item has its own id and position, so the song may appear + // twice in a playlist and nothing shifts. + if _, err := tx.ExecContext(ctx, "UPDATE playlist_items SET track_id = ? WHERE track_id = ?", keepTrackID, removeTrackID); err != nil { + return fmt.Errorf("move the playlist items: %w", err) + } if _, err := tx.ExecContext(ctx, "DELETE FROM tracks WHERE id = ?", removeTrackID); err != nil { return fmt.Errorf("remove the track: %w", err) } diff --git a/internal/storage/duplicates_test.go b/internal/storage/duplicates_test.go index d303d33..bddb187 100644 --- a/internal/storage/duplicates_test.go +++ b/internal/storage/duplicates_test.go @@ -3,6 +3,7 @@ package storage import ( "context" "errors" + "slices" "testing" "time" @@ -245,6 +246,48 @@ func TestRemovingACopyMovesItsListensToTheOneThatStays(t *testing.T) { }) } +func TestRemovingACopyKeepsItsPlaceInThePlaylists(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + ctx := context.Background() + lib := NewLibraryRepo(db) + l := mkLib(t, lib, "music") + saved(t, lib, l, song("a/1.flac", "Song", "A", 200_000, 1, "flac", 900, "Ann")) + saved(t, lib, l, song("b/2.mp3", "Song", "B", 200_000, 1, "mp3", 192, "Ann")) + saved(t, lib, l, song("c/3.mp3", "Other", "C", 100_000, 1, "mp3", 192, "Ann")) + keep, gone, other := trackOf(t, db, "a/1.flac"), trackOf(t, db, "b/2.mp3"), trackOf(t, db, "c/3.mp3") + + user, err := NewUserRepo(db).CreateUser(ctx, "alice@example.org", "h", false) + if err != nil { + t.Fatal(err) + } + pl := NewPlaylistRepo(db) + now := time.Unix(1_700_000_000, 0) + p, err := pl.Create(ctx, user.ID, "Mix", "", now) + if err != nil { + t.Fatal(err) + } + // The song twice in a row (the removed copy between two others), then another song. + if _, err := pl.AddTracks(ctx, user.ID, p.ID, []int64{keep, gone, other}, nil, now); err != nil { + t.Fatal(err) + } + + if err := NewDuplicatesRepo(db).RemoveCopy(ctx, keep, gone); err != nil { + t.Fatal(err) + } + + items, total, err := pl.Items(ctx, user.ID, p.ID, 50, 0) + if err != nil { + t.Fatal(err) + } + if total != 3 || len(items) != 3 { + t.Fatalf("the playlist has %d items, want 3: the removed copy's place must stay", total) + } + if got := []int64{items[0].Track.ID, items[1].Track.ID, items[2].Track.ID}; !slices.Equal(got, []int64{keep, keep, other}) { + t.Errorf("tracks in the playlist = %v, want %v", got, []int64{keep, keep, other}) + } + }) +} + func TestRemovingACopyNeedsTwoTracksThatExistAndAreDifferent(t *testing.T) { forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { ctx := context.Background()