diff --git a/internal/duplicates/group.go b/internal/duplicates/group.go new file mode 100644 index 0000000..52e7dfd --- /dev/null +++ b/internal/duplicates/group.go @@ -0,0 +1,175 @@ +// Package duplicates finds the same song by the same artist in more than one file, and removes +// the extra copies when an admin says so. Finding only reads the index. Removing is the one thing +// Sonda does to the music folders, so it is off until the operator turns it on, and every +// removal is checked again at the moment it is made. +package duplicates + +import ( + "slices" + "strconv" + "strings" + "time" +) + +// DurationTolerance is how far apart two durations may be for two files to be the same song. It +// keeps a live version that is minutes longer apart from the studio one. +const DurationTolerance = 3 * time.Second + +// ArtistRef names an artist. +type ArtistRef struct { + ID int64 + Name string +} + +// Candidate is one playable file as the grouping sees it: its track, its main artists and the +// facts that tell the copies apart. +type Candidate struct { + TrackID int64 + FileID int64 + Artists []ArtistRef // the main artists of the track + Title string + TitleKey string // the key Sonda matches titles by + DurationMs int64 + AlbumID int64 + Album string + Format string + BitRate int + SizeBytes int64 + Hash string // SHA-256 of the file, empty until it has been read +} + +// Group is a song that is in more than one file. +type Group struct { + Artists []ArtistRef + Title string + Files []Candidate // by track id + // Hashed is true when every file has been read; Identical when, besides, they are byte for byte the same. + Hashed, Identical bool + // KeepTrackID is the copy to keep, by Suggest; FreedBytes is the size of all the others. + KeepTrackID int64 + FreedBytes int64 +} + +// artistsKey identifies a set of artists, whatever the order they are credited in. +func artistsKey(artists []ArtistRef) string { + ids := make([]int64, len(artists)) + for i, a := range artists { + ids[i] = a.ID + } + slices.Sort(ids) + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = strconv.FormatInt(id, 10) + } + return strings.Join(parts, ",") +} + +// Build finds the groups. Two files are the same song when they have the same set of main +// artists, the same title and durations no more than tolerance apart (every pair in a group, not +// a chain of near ones). A file with no known duration or no artist never matches. The groups +// and the files in them come in a fixed order, whatever the order of cands. +func Build(cands []Candidate, tolerance time.Duration) []Group { + buckets := map[string][]Candidate{} + for _, c := range cands { + if c.DurationMs <= 0 || len(c.Artists) == 0 { + continue + } + key := artistsKey(c.Artists) + "\x00" + c.TitleKey + buckets[key] = append(buckets[key], c) + } + + tolMs := tolerance.Milliseconds() + var groups []Group + for _, bucket := range buckets { + if len(bucket) < 2 { + continue + } + slices.SortFunc(bucket, func(a, b Candidate) int { + if a.DurationMs != b.DurationMs { + return int(a.DurationMs - b.DurationMs) + } + return int(a.TrackID - b.TrackID) + }) + for i := 0; i < len(bucket); { + j := i + 1 + for j < len(bucket) && bucket[j].DurationMs-bucket[i].DurationMs <= tolMs { + j++ + } + if j-i >= 2 { + groups = append(groups, newGroup(bucket[i:j])) + } + i = j + } + } + + slices.SortFunc(groups, func(a, b Group) int { + if c := strings.Compare(a.Files[0].TitleKey, b.Files[0].TitleKey); c != 0 { + return c + } + if c := strings.Compare(artistsKey(a.Artists), artistsKey(b.Artists)); c != 0 { + return c + } + return int(a.Files[0].TrackID - b.Files[0].TrackID) + }) + return groups +} + +func newGroup(files []Candidate) Group { + files = slices.Clone(files) + slices.SortFunc(files, func(a, b Candidate) int { return int(a.TrackID - b.TrackID) }) + artists := slices.Clone(files[0].Artists) + slices.SortFunc(artists, func(a, b ArtistRef) int { return int(a.ID - b.ID) }) + + g := Group{Artists: artists, Title: files[0].Title, Files: files, KeepTrackID: Suggest(files)} + g.Hashed = true + for _, f := range files { + if f.Hash == "" { + g.Hashed = false + } + if f.TrackID != g.KeepTrackID { + g.FreedBytes += f.SizeBytes + } + } + g.Identical = g.Hashed + for _, f := range files[1:] { + if f.Hash != files[0].Hash { + g.Identical = false + } + } + return g +} + +// lossless says whether a format keeps everything the recording had. +func lossless(format string) bool { + switch strings.ToLower(format) { + case "flac", "wav", "alac", "ape", "wv", "aiff": + return true + } + return false +} + +// Suggest picks the copy to keep: one without loss before one with loss, then the higher bitrate, +// then the bigger file, and at a tie the oldest track. The answer does not depend on the order. +func Suggest(files []Candidate) int64 { + best := files[0] + for _, f := range files[1:] { + if better(f, best) { + best = f + } + } + return best.TrackID +} + +// better says whether a should be kept over b. +func better(a, b Candidate) bool { + if la, lb := lossless(a.Format), lossless(b.Format); la != lb { + return la + } + if a.BitRate != b.BitRate { + return a.BitRate > b.BitRate + } + if a.SizeBytes != b.SizeBytes { + return a.SizeBytes > b.SizeBytes + } + return a.TrackID < b.TrackID +} diff --git a/internal/duplicates/group_test.go b/internal/duplicates/group_test.go new file mode 100644 index 0000000..8df408d --- /dev/null +++ b/internal/duplicates/group_test.go @@ -0,0 +1,198 @@ +package duplicates + +import ( + "reflect" + "testing" + "time" +) + +const tol = 3 * time.Second + +func cand(track int64, title string, durationMs int64, artists ...int64) Candidate { + refs := make([]ArtistRef, len(artists)) + for i, id := range artists { + refs[i] = ArtistRef{ID: id, Name: "Artist"} + } + return Candidate{ + TrackID: track, FileID: track * 10, Artists: refs, Title: title, TitleKey: title, + DurationMs: durationMs, AlbumID: 1, Album: "Album", Format: "mp3", BitRate: 192, SizeBytes: 5_000_000, + } +} + +func trackIDs(g Group) []int64 { + var out []int64 + for _, f := range g.Files { + out = append(out, f.TrackID) + } + return out +} + +func TestTheSameSongBySameArtistsIsAGroup(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 200_000, 7), + cand(2, "song", 201_000, 7), + }, tol) + if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) { + t.Fatalf("groups = %+v, want one group of tracks 1 and 2", groups) + } +} + +func TestASongThatIsAloneIsNotAGroup(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7)}, tol); len(got) != 0 { + t.Errorf("a single copy made a group: %+v", got) + } +} + +func TestSameTitleByAnotherArtistIsAnotherSong(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 8)}, tol); len(got) != 0 { + t.Errorf("two artists made a group: %+v", got) + } +} + +func TestTheSetOfArtistsMustBeTheSame(t *testing.T) { + // A duet is not the song of one of the two. + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7, 8)}, tol); len(got) != 0 { + t.Errorf("a solo and a duet made a group: %+v", got) + } + // The order in which the artists are credited does not matter. + got := Build([]Candidate{cand(1, "song", 200_000, 7, 8), cand(2, "song", 200_000, 8, 7)}, tol) + if len(got) != 1 { + t.Errorf("the same artists in another order did not make a group: %+v", got) + } +} + +func TestADifferentTitleIsAnotherSong(t *testing.T) { + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song (live)", 200_000, 7)}, tol); len(got) != 0 { + t.Errorf("two titles made a group: %+v", got) + } +} + +func TestDurationsMustBeWithinTheTolerance(t *testing.T) { + // 3 seconds apart is the same song, 3.001 is not. + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_000, 7)}, tol); len(got) != 1 { + t.Errorf("3 s apart: %+v, want a group", got) + } + if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_001, 7)}, tol); len(got) != 0 { + t.Errorf("3.001 s apart: %+v, want none", got) + } +} + +func TestALongLiveVersionIsNotMixedWithTheStudioOnes(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 220_000, 7), + cand(2, "song", 221_000, 7), + cand(3, "song", 370_000, 7), // the same title, but 2.5 minutes longer + }, tol) + if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) { + t.Fatalf("groups = %+v, want only tracks 1 and 2 together", groups) + } +} + +// Every pair in a group is within the tolerance, so a chain of songs each a little longer than the last +// does not become one group. +func TestAChainOfSlightlyLongerCopiesDoesNotGrowOneGroup(t *testing.T) { + groups := Build([]Candidate{ + cand(1, "song", 200_000, 7), + cand(2, "song", 202_000, 7), + cand(3, "song", 204_000, 7), + cand(4, "song", 206_000, 7), + }, tol) + for _, g := range groups { + ids := trackIDs(g) + var lo, hi int64 = 1 << 60, 0 + for _, f := range g.Files { + lo, hi = min(lo, f.DurationMs), max(hi, f.DurationMs) + } + if hi-lo > 3000 { + t.Errorf("group %v spans %d ms", ids, hi-lo) + } + } +} + +func TestATrackWithNoDurationOrNoArtistNeverMatches(t *testing.T) { + noDuration := cand(2, "song", 0, 7) + noArtist := cand(3, "song", 200_000) + if got := Build([]Candidate{cand(1, "song", 200_000, 7), noDuration, noArtist}, tol); len(got) != 0 { + t.Errorf("groups = %+v, want none", got) + } +} + +func TestTheSameSongOnDifferentAlbumsIsAGroup(t *testing.T) { + other := cand(2, "song", 200_000, 7) + other.AlbumID, other.Album = 2, "Greatest hits" + groups := Build([]Candidate{cand(1, "song", 200_000, 7), other}, tol) + if len(groups) != 1 || groups[0].Files[0].Album == groups[0].Files[1].Album { + t.Fatalf("groups = %+v, want one group with two albums", groups) + } +} + +func TestSeveralGroupsComeInAStableOrder(t *testing.T) { + a := []Candidate{cand(1, "b song", 200_000, 7), cand(2, "b song", 200_000, 7), cand(3, "a song", 100_000, 7), cand(4, "a song", 100_000, 7)} + b := []Candidate{a[3], a[2], a[1], a[0]} + first, second := Build(a, tol), Build(b, tol) + if !reflect.DeepEqual(first, second) { + t.Errorf("the order of the input changes the answer:\n%+v\n%+v", first, second) + } + if first[0].Title != "a song" { + t.Errorf("first group is %q, want the titles in order", first[0].Title) + } +} + +func TestAGroupIsIdenticalOnlyWhenEveryFileHasTheSameHash(t *testing.T) { + x, y := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7) + if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical { + t.Errorf("no hashes: hashed=%v identical=%v, want both false", g.Hashed, g.Identical) + } + x.Hash = "aa" + if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical { + t.Errorf("one hash of two: hashed=%v identical=%v, want both false", g.Hashed, g.Identical) + } + y.Hash = "aa" + if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || !g.Identical { + t.Errorf("same hashes: hashed=%v identical=%v, want both true", g.Hashed, g.Identical) + } + y.Hash = "bb" + if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || g.Identical { + t.Errorf("different hashes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical) + } +} + +func TestFreedBytesAreTheSizeOfEverythingButTheCopyToKeep(t *testing.T) { + big, small := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7) + big.Format, big.SizeBytes = "flac", 30_000_000 + small.SizeBytes = 5_000_000 + g := Build([]Candidate{big, small}, tol)[0] + if g.KeepTrackID != 1 || g.FreedBytes != 5_000_000 { + t.Errorf("keep %d, freed %d; want 1 and 5000000", g.KeepTrackID, g.FreedBytes) + } +} + +// --- which copy to keep --------------------------------------------------------------------------- + +func TestSuggestPrefersLosslessThenBitrateThenSizeThenTheOldest(t *testing.T) { + flac := Candidate{TrackID: 5, Format: "flac", BitRate: 800, SizeBytes: 20_000_000} + hiMP3 := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000} + loMP3 := Candidate{TrackID: 1, Format: "mp3", BitRate: 128, SizeBytes: 4_000_000} + if got := Suggest([]Candidate{loMP3, hiMP3, flac}); got != 5 { + t.Errorf("flac among mp3: kept %d, want 5", got) + } + if got := Suggest([]Candidate{loMP3, hiMP3}); got != 2 { + t.Errorf("320 against 128: kept %d, want 2", got) + } + bigger := Candidate{TrackID: 7, Format: "mp3", BitRate: 320, SizeBytes: 9_500_000} + if got := Suggest([]Candidate{hiMP3, bigger}); got != 7 { + t.Errorf("same bitrate, bigger file: kept %d, want 7", got) + } + twin := Candidate{TrackID: 9, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000} + if got := Suggest([]Candidate{twin, hiMP3}); got != 2 { + t.Errorf("all equal: kept %d, want the oldest (2)", got) + } +} + +func TestSuggestIsTheSameWhateverTheOrder(t *testing.T) { + a := Candidate{TrackID: 1, Format: "mp3", BitRate: 320, SizeBytes: 9} + b := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9} + if Suggest([]Candidate{a, b}) != Suggest([]Candidate{b, a}) { + t.Error("the answer depends on the order") + } +}