// Package duplicates finds the same song by the same artist in more than one file, and removes // the extra copies when an admin says so. Finding only reads the index. Removing is the one thing // Sonda does to the music folders, so it is off until the operator turns it on, and every // removal is checked again at the moment it is made. package duplicates import ( "slices" "strconv" "strings" "time" ) // DurationTolerance is how far apart two durations may be for two files to be the same song. It // keeps a live version that is minutes longer apart from the studio one. const DurationTolerance = 3 * time.Second // ArtistRef names an artist. type ArtistRef struct { ID int64 Name string } // Candidate is one playable file as the grouping sees it: its track, its main artists and the // facts that tell the copies apart. type Candidate struct { TrackID int64 FileID int64 Artists []ArtistRef // the main artists of the track Title string TitleKey string // the key Sonda matches titles by DurationMs int64 AlbumID int64 Album string Format string BitRate int SizeBytes int64 Hash string // SHA-256 of the file, empty until it has been read } // Group is a song that is in more than one file. type Group struct { Artists []ArtistRef Title string Files []Candidate // by track id // Hashed is true when every file has been read; Identical when, besides, they are byte for byte the same. Hashed, Identical bool // KeepTrackID is the copy to keep, by Suggest; FreedBytes is the size of all the others. KeepTrackID int64 FreedBytes int64 } // artistsKey identifies a set of artists, whatever the order they are credited in. func artistsKey(artists []ArtistRef) string { ids := make([]int64, len(artists)) for i, a := range artists { ids[i] = a.ID } slices.Sort(ids) parts := make([]string, len(ids)) for i, id := range ids { parts[i] = strconv.FormatInt(id, 10) } return strings.Join(parts, ",") } // Build finds the groups. Two files are the same song when they have the same set of main // artists, the same title and durations no more than tolerance apart (every pair in a group, not // a chain of near ones). A file with no known duration or no artist never matches. The groups // and the files in them come in a fixed order, whatever the order of cands. func Build(cands []Candidate, tolerance time.Duration) []Group { buckets := map[string][]Candidate{} for _, c := range cands { if c.DurationMs <= 0 || len(c.Artists) == 0 { continue } key := artistsKey(c.Artists) + "\x00" + c.TitleKey buckets[key] = append(buckets[key], c) } tolMs := tolerance.Milliseconds() var groups []Group for _, bucket := range buckets { if len(bucket) < 2 { continue } slices.SortFunc(bucket, func(a, b Candidate) int { if a.DurationMs != b.DurationMs { return int(a.DurationMs - b.DurationMs) } return int(a.TrackID - b.TrackID) }) for i := 0; i < len(bucket); { j := i + 1 for j < len(bucket) && bucket[j].DurationMs-bucket[i].DurationMs <= tolMs { j++ } if j-i >= 2 { groups = append(groups, newGroup(bucket[i:j])) } i = j } } slices.SortFunc(groups, func(a, b Group) int { if c := strings.Compare(a.Files[0].TitleKey, b.Files[0].TitleKey); c != 0 { return c } if c := strings.Compare(artistsKey(a.Artists), artistsKey(b.Artists)); c != 0 { return c } return int(a.Files[0].TrackID - b.Files[0].TrackID) }) return groups } func newGroup(files []Candidate) Group { files = slices.Clone(files) slices.SortFunc(files, func(a, b Candidate) int { return int(a.TrackID - b.TrackID) }) artists := slices.Clone(files[0].Artists) slices.SortFunc(artists, func(a, b ArtistRef) int { return int(a.ID - b.ID) }) g := Group{Artists: artists, Title: files[0].Title, Files: files, KeepTrackID: Suggest(files)} g.Hashed = true for _, f := range files { if f.Hash == "" { g.Hashed = false } if f.TrackID != g.KeepTrackID { g.FreedBytes += f.SizeBytes } } g.Identical = g.Hashed for _, f := range files[1:] { if f.Hash != files[0].Hash { g.Identical = false } } return g } // lossless says whether a format keeps everything the recording had. func lossless(format string) bool { switch strings.ToLower(format) { case "flac", "wav", "alac", "ape", "wv", "aiff": return true } return false } // Suggest picks the copy to keep: one without loss before one with loss, then the higher bitrate, // then the bigger file, and at a tie the oldest track. The answer does not depend on the order. func Suggest(files []Candidate) int64 { best := files[0] for _, f := range files[1:] { if better(f, best) { best = f } } return best.TrackID } // better says whether a should be kept over b. func better(a, b Candidate) bool { if la, lb := lossless(a.Format), lossless(b.Format); la != lb { return la } if a.BitRate != b.BitRate { return a.BitRate > b.BitRate } if a.SizeBytes != b.SizeBytes { return a.SizeBytes > b.SizeBytes } return a.TrackID < b.TrackID }