feat(duplicates): group the copies of a song and propose the one to keep
The same song is the same set of main artists, the same title and durations within 3 seconds, every pair within it; a file with no duration or no artist never matches. A group says whether its files were all read and are byte for byte the same. The copy to keep is the lossless one, then the higher bitrate, then the bigger file, then the oldest, whatever the order of the input.
This commit is contained in:
parent
309160951e
commit
7e1e63bdd2
2 changed files with 373 additions and 0 deletions
175
internal/duplicates/group.go
Normal file
175
internal/duplicates/group.go
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
// Package duplicates finds the same song by the same artist in more than one file, and removes
|
||||
// the extra copies when an admin says so. Finding only reads the index. Removing is the one thing
|
||||
// Sonda does to the music folders, so it is off until the operator turns it on, and every
|
||||
// removal is checked again at the moment it is made.
|
||||
package duplicates
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DurationTolerance is how far apart two durations may be for two files to be the same song. It
|
||||
// keeps a live version that is minutes longer apart from the studio one.
|
||||
const DurationTolerance = 3 * time.Second
|
||||
|
||||
// ArtistRef names an artist.
|
||||
type ArtistRef struct {
|
||||
ID int64
|
||||
Name string
|
||||
}
|
||||
|
||||
// Candidate is one playable file as the grouping sees it: its track, its main artists and the
|
||||
// facts that tell the copies apart.
|
||||
type Candidate struct {
|
||||
TrackID int64
|
||||
FileID int64
|
||||
Artists []ArtistRef // the main artists of the track
|
||||
Title string
|
||||
TitleKey string // the key Sonda matches titles by
|
||||
DurationMs int64
|
||||
AlbumID int64
|
||||
Album string
|
||||
Format string
|
||||
BitRate int
|
||||
SizeBytes int64
|
||||
Hash string // SHA-256 of the file, empty until it has been read
|
||||
}
|
||||
|
||||
// Group is a song that is in more than one file.
|
||||
type Group struct {
|
||||
Artists []ArtistRef
|
||||
Title string
|
||||
Files []Candidate // by track id
|
||||
// Hashed is true when every file has been read; Identical when, besides, they are byte for byte the same.
|
||||
Hashed, Identical bool
|
||||
// KeepTrackID is the copy to keep, by Suggest; FreedBytes is the size of all the others.
|
||||
KeepTrackID int64
|
||||
FreedBytes int64
|
||||
}
|
||||
|
||||
// artistsKey identifies a set of artists, whatever the order they are credited in.
|
||||
func artistsKey(artists []ArtistRef) string {
|
||||
ids := make([]int64, len(artists))
|
||||
for i, a := range artists {
|
||||
ids[i] = a.ID
|
||||
}
|
||||
slices.Sort(ids)
|
||||
parts := make([]string, len(ids))
|
||||
for i, id := range ids {
|
||||
parts[i] = strconv.FormatInt(id, 10)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// Build finds the groups. Two files are the same song when they have the same set of main
|
||||
// artists, the same title and durations no more than tolerance apart (every pair in a group, not
|
||||
// a chain of near ones). A file with no known duration or no artist never matches. The groups
|
||||
// and the files in them come in a fixed order, whatever the order of cands.
|
||||
func Build(cands []Candidate, tolerance time.Duration) []Group {
|
||||
buckets := map[string][]Candidate{}
|
||||
for _, c := range cands {
|
||||
if c.DurationMs <= 0 || len(c.Artists) == 0 {
|
||||
continue
|
||||
}
|
||||
key := artistsKey(c.Artists) + "\x00" + c.TitleKey
|
||||
buckets[key] = append(buckets[key], c)
|
||||
}
|
||||
|
||||
tolMs := tolerance.Milliseconds()
|
||||
var groups []Group
|
||||
for _, bucket := range buckets {
|
||||
if len(bucket) < 2 {
|
||||
continue
|
||||
}
|
||||
slices.SortFunc(bucket, func(a, b Candidate) int {
|
||||
if a.DurationMs != b.DurationMs {
|
||||
return int(a.DurationMs - b.DurationMs)
|
||||
}
|
||||
return int(a.TrackID - b.TrackID)
|
||||
})
|
||||
for i := 0; i < len(bucket); {
|
||||
j := i + 1
|
||||
for j < len(bucket) && bucket[j].DurationMs-bucket[i].DurationMs <= tolMs {
|
||||
j++
|
||||
}
|
||||
if j-i >= 2 {
|
||||
groups = append(groups, newGroup(bucket[i:j]))
|
||||
}
|
||||
i = j
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(groups, func(a, b Group) int {
|
||||
if c := strings.Compare(a.Files[0].TitleKey, b.Files[0].TitleKey); c != 0 {
|
||||
return c
|
||||
}
|
||||
if c := strings.Compare(artistsKey(a.Artists), artistsKey(b.Artists)); c != 0 {
|
||||
return c
|
||||
}
|
||||
return int(a.Files[0].TrackID - b.Files[0].TrackID)
|
||||
})
|
||||
return groups
|
||||
}
|
||||
|
||||
func newGroup(files []Candidate) Group {
|
||||
files = slices.Clone(files)
|
||||
slices.SortFunc(files, func(a, b Candidate) int { return int(a.TrackID - b.TrackID) })
|
||||
artists := slices.Clone(files[0].Artists)
|
||||
slices.SortFunc(artists, func(a, b ArtistRef) int { return int(a.ID - b.ID) })
|
||||
|
||||
g := Group{Artists: artists, Title: files[0].Title, Files: files, KeepTrackID: Suggest(files)}
|
||||
g.Hashed = true
|
||||
for _, f := range files {
|
||||
if f.Hash == "" {
|
||||
g.Hashed = false
|
||||
}
|
||||
if f.TrackID != g.KeepTrackID {
|
||||
g.FreedBytes += f.SizeBytes
|
||||
}
|
||||
}
|
||||
g.Identical = g.Hashed
|
||||
for _, f := range files[1:] {
|
||||
if f.Hash != files[0].Hash {
|
||||
g.Identical = false
|
||||
}
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// lossless says whether a format keeps everything the recording had.
|
||||
func lossless(format string) bool {
|
||||
switch strings.ToLower(format) {
|
||||
case "flac", "wav", "alac", "ape", "wv", "aiff":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Suggest picks the copy to keep: one without loss before one with loss, then the higher bitrate,
|
||||
// then the bigger file, and at a tie the oldest track. The answer does not depend on the order.
|
||||
func Suggest(files []Candidate) int64 {
|
||||
best := files[0]
|
||||
for _, f := range files[1:] {
|
||||
if better(f, best) {
|
||||
best = f
|
||||
}
|
||||
}
|
||||
return best.TrackID
|
||||
}
|
||||
|
||||
// better says whether a should be kept over b.
|
||||
func better(a, b Candidate) bool {
|
||||
if la, lb := lossless(a.Format), lossless(b.Format); la != lb {
|
||||
return la
|
||||
}
|
||||
if a.BitRate != b.BitRate {
|
||||
return a.BitRate > b.BitRate
|
||||
}
|
||||
if a.SizeBytes != b.SizeBytes {
|
||||
return a.SizeBytes > b.SizeBytes
|
||||
}
|
||||
return a.TrackID < b.TrackID
|
||||
}
|
||||
198
internal/duplicates/group_test.go
Normal file
198
internal/duplicates/group_test.go
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
package duplicates
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const tol = 3 * time.Second
|
||||
|
||||
func cand(track int64, title string, durationMs int64, artists ...int64) Candidate {
|
||||
refs := make([]ArtistRef, len(artists))
|
||||
for i, id := range artists {
|
||||
refs[i] = ArtistRef{ID: id, Name: "Artist"}
|
||||
}
|
||||
return Candidate{
|
||||
TrackID: track, FileID: track * 10, Artists: refs, Title: title, TitleKey: title,
|
||||
DurationMs: durationMs, AlbumID: 1, Album: "Album", Format: "mp3", BitRate: 192, SizeBytes: 5_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
func trackIDs(g Group) []int64 {
|
||||
var out []int64
|
||||
for _, f := range g.Files {
|
||||
out = append(out, f.TrackID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheSameSongBySameArtistsIsAGroup(t *testing.T) {
|
||||
groups := Build([]Candidate{
|
||||
cand(1, "song", 200_000, 7),
|
||||
cand(2, "song", 201_000, 7),
|
||||
}, tol)
|
||||
if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) {
|
||||
t.Fatalf("groups = %+v, want one group of tracks 1 and 2", groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASongThatIsAloneIsNotAGroup(t *testing.T) {
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7)}, tol); len(got) != 0 {
|
||||
t.Errorf("a single copy made a group: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameTitleByAnotherArtistIsAnotherSong(t *testing.T) {
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 8)}, tol); len(got) != 0 {
|
||||
t.Errorf("two artists made a group: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSetOfArtistsMustBeTheSame(t *testing.T) {
|
||||
// A duet is not the song of one of the two.
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7, 8)}, tol); len(got) != 0 {
|
||||
t.Errorf("a solo and a duet made a group: %+v", got)
|
||||
}
|
||||
// The order in which the artists are credited does not matter.
|
||||
got := Build([]Candidate{cand(1, "song", 200_000, 7, 8), cand(2, "song", 200_000, 8, 7)}, tol)
|
||||
if len(got) != 1 {
|
||||
t.Errorf("the same artists in another order did not make a group: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADifferentTitleIsAnotherSong(t *testing.T) {
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song (live)", 200_000, 7)}, tol); len(got) != 0 {
|
||||
t.Errorf("two titles made a group: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurationsMustBeWithinTheTolerance(t *testing.T) {
|
||||
// 3 seconds apart is the same song, 3.001 is not.
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_000, 7)}, tol); len(got) != 1 {
|
||||
t.Errorf("3 s apart: %+v, want a group", got)
|
||||
}
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), cand(2, "song", 203_001, 7)}, tol); len(got) != 0 {
|
||||
t.Errorf("3.001 s apart: %+v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALongLiveVersionIsNotMixedWithTheStudioOnes(t *testing.T) {
|
||||
groups := Build([]Candidate{
|
||||
cand(1, "song", 220_000, 7),
|
||||
cand(2, "song", 221_000, 7),
|
||||
cand(3, "song", 370_000, 7), // the same title, but 2.5 minutes longer
|
||||
}, tol)
|
||||
if len(groups) != 1 || !reflect.DeepEqual(trackIDs(groups[0]), []int64{1, 2}) {
|
||||
t.Fatalf("groups = %+v, want only tracks 1 and 2 together", groups)
|
||||
}
|
||||
}
|
||||
|
||||
// Every pair in a group is within the tolerance, so a chain of songs each a little longer than the last
|
||||
// does not become one group.
|
||||
func TestAChainOfSlightlyLongerCopiesDoesNotGrowOneGroup(t *testing.T) {
|
||||
groups := Build([]Candidate{
|
||||
cand(1, "song", 200_000, 7),
|
||||
cand(2, "song", 202_000, 7),
|
||||
cand(3, "song", 204_000, 7),
|
||||
cand(4, "song", 206_000, 7),
|
||||
}, tol)
|
||||
for _, g := range groups {
|
||||
ids := trackIDs(g)
|
||||
var lo, hi int64 = 1 << 60, 0
|
||||
for _, f := range g.Files {
|
||||
lo, hi = min(lo, f.DurationMs), max(hi, f.DurationMs)
|
||||
}
|
||||
if hi-lo > 3000 {
|
||||
t.Errorf("group %v spans %d ms", ids, hi-lo)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrackWithNoDurationOrNoArtistNeverMatches(t *testing.T) {
|
||||
noDuration := cand(2, "song", 0, 7)
|
||||
noArtist := cand(3, "song", 200_000)
|
||||
if got := Build([]Candidate{cand(1, "song", 200_000, 7), noDuration, noArtist}, tol); len(got) != 0 {
|
||||
t.Errorf("groups = %+v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSameSongOnDifferentAlbumsIsAGroup(t *testing.T) {
|
||||
other := cand(2, "song", 200_000, 7)
|
||||
other.AlbumID, other.Album = 2, "Greatest hits"
|
||||
groups := Build([]Candidate{cand(1, "song", 200_000, 7), other}, tol)
|
||||
if len(groups) != 1 || groups[0].Files[0].Album == groups[0].Files[1].Album {
|
||||
t.Fatalf("groups = %+v, want one group with two albums", groups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeveralGroupsComeInAStableOrder(t *testing.T) {
|
||||
a := []Candidate{cand(1, "b song", 200_000, 7), cand(2, "b song", 200_000, 7), cand(3, "a song", 100_000, 7), cand(4, "a song", 100_000, 7)}
|
||||
b := []Candidate{a[3], a[2], a[1], a[0]}
|
||||
first, second := Build(a, tol), Build(b, tol)
|
||||
if !reflect.DeepEqual(first, second) {
|
||||
t.Errorf("the order of the input changes the answer:\n%+v\n%+v", first, second)
|
||||
}
|
||||
if first[0].Title != "a song" {
|
||||
t.Errorf("first group is %q, want the titles in order", first[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGroupIsIdenticalOnlyWhenEveryFileHasTheSameHash(t *testing.T) {
|
||||
x, y := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7)
|
||||
if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical {
|
||||
t.Errorf("no hashes: hashed=%v identical=%v, want both false", g.Hashed, g.Identical)
|
||||
}
|
||||
x.Hash = "aa"
|
||||
if g := Build([]Candidate{x, y}, tol)[0]; g.Hashed || g.Identical {
|
||||
t.Errorf("one hash of two: hashed=%v identical=%v, want both false", g.Hashed, g.Identical)
|
||||
}
|
||||
y.Hash = "aa"
|
||||
if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || !g.Identical {
|
||||
t.Errorf("same hashes: hashed=%v identical=%v, want both true", g.Hashed, g.Identical)
|
||||
}
|
||||
y.Hash = "bb"
|
||||
if g := Build([]Candidate{x, y}, tol)[0]; !g.Hashed || g.Identical {
|
||||
t.Errorf("different hashes: hashed=%v identical=%v, want hashed and not identical", g.Hashed, g.Identical)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreedBytesAreTheSizeOfEverythingButTheCopyToKeep(t *testing.T) {
|
||||
big, small := cand(1, "song", 200_000, 7), cand(2, "song", 200_000, 7)
|
||||
big.Format, big.SizeBytes = "flac", 30_000_000
|
||||
small.SizeBytes = 5_000_000
|
||||
g := Build([]Candidate{big, small}, tol)[0]
|
||||
if g.KeepTrackID != 1 || g.FreedBytes != 5_000_000 {
|
||||
t.Errorf("keep %d, freed %d; want 1 and 5000000", g.KeepTrackID, g.FreedBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// --- which copy to keep ---------------------------------------------------------------------------
|
||||
|
||||
func TestSuggestPrefersLosslessThenBitrateThenSizeThenTheOldest(t *testing.T) {
|
||||
flac := Candidate{TrackID: 5, Format: "flac", BitRate: 800, SizeBytes: 20_000_000}
|
||||
hiMP3 := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000}
|
||||
loMP3 := Candidate{TrackID: 1, Format: "mp3", BitRate: 128, SizeBytes: 4_000_000}
|
||||
if got := Suggest([]Candidate{loMP3, hiMP3, flac}); got != 5 {
|
||||
t.Errorf("flac among mp3: kept %d, want 5", got)
|
||||
}
|
||||
if got := Suggest([]Candidate{loMP3, hiMP3}); got != 2 {
|
||||
t.Errorf("320 against 128: kept %d, want 2", got)
|
||||
}
|
||||
bigger := Candidate{TrackID: 7, Format: "mp3", BitRate: 320, SizeBytes: 9_500_000}
|
||||
if got := Suggest([]Candidate{hiMP3, bigger}); got != 7 {
|
||||
t.Errorf("same bitrate, bigger file: kept %d, want 7", got)
|
||||
}
|
||||
twin := Candidate{TrackID: 9, Format: "mp3", BitRate: 320, SizeBytes: 9_000_000}
|
||||
if got := Suggest([]Candidate{twin, hiMP3}); got != 2 {
|
||||
t.Errorf("all equal: kept %d, want the oldest (2)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestIsTheSameWhateverTheOrder(t *testing.T) {
|
||||
a := Candidate{TrackID: 1, Format: "mp3", BitRate: 320, SizeBytes: 9}
|
||||
b := Candidate{TrackID: 2, Format: "mp3", BitRate: 320, SizeBytes: 9}
|
||||
if Suggest([]Candidate{a, b}) != Suggest([]Candidate{b, a}) {
|
||||
t.Error("the answer depends on the order")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue