feat(api): list the duplicate songs, check their files and delete the extra copies

Admin only. GET /library/duplicates lists the groups with what is needed to choose and never a path; POST|GET|DELETE /library/duplicates/check is the job that fingerprints the files; POST /library/duplicates/clean deletes the chosen copies of one song, each refusal with its own status and code (deletion_disabled, scan_running, invalid_input, track_not_found, duplicate_changed, library_read_only). In the spec with the contract test, and wired in main, where an info line says when library.allow_delete is on.
This commit is contained in:
tgiachi 2026-11-12 16:46:50 +01:00
commit 30f45bb383
7 changed files with 977 additions and 0 deletions

View file

@ -21,6 +21,7 @@ import (
"codeberg.org/squidd/sonda/internal/config"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -261,6 +262,17 @@ func run() error {
Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery,
}
// Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders:
// off unless the operator turned it on, and said in the log when it is on.
duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: cfg.Library.AllowDelete,
ScanRunning: func() bool { return scans.Status().Running },
Logger: logger,
})
if cfg.Library.AllowDelete {
logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders")
}
router := httpapi.NewRouter(httpapi.Deps{
Logger: logger,
Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()),
@ -281,6 +293,8 @@ func run() error {
CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)},
ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)},
DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)},
Duplicates: duplicateSvc,
DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)},
ArtistImages: pictures,
About: descriptions,
Settings: userSettings,

View file

@ -20,6 +20,7 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -115,6 +116,14 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun
if deps.DescriptionsJob == nil {
deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))}
}
// Deleting is off unless a test turns it on.
if deps.Duplicates == nil {
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{Logger: zap.NewNop()})
deps.Duplicates = svc
if deps.DuplicatesJob == nil {
deps.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
}
}
// Built from the final services, so a test that swaps one gets a Discover that uses it.
if deps.Discover == nil {
deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar),

View file

@ -26,7 +26,9 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/providers/lastfm"
@ -140,6 +142,11 @@ func newContract(t *testing.T) *contract {
libs := library.NewLibraries(repo)
libs.SetBusy(func() bool { return mgr.Status().Running })
d.Scan = scanAdapter{mgr}
dupSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: true, ScanRunning: func() bool { return mgr.Status().Running }, Logger: zap.NewNop(),
})
d.Duplicates = dupSvc
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(dupSvc)}
d.Libraries = libs
d.Catalog = catalog.NewService(storage.NewCatalogRepo(db))
d.Streams = streaming.NewService(repo)
@ -767,6 +774,9 @@ func TestResponsesMatchTheSpec(t *testing.T) {
c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"nope": true}), 400, "a setting that does not exist", "invalid_input")
c.expect(c.call("GET", "/me/settings", "", nil), 401, "settings without a token", "unauthorized")
// ---- duplicate songs ----
c.duplicates(t, token, libID)
// ---- every operation in the spec was exercised ----
var missing []string
for _, op := range specOperations(loadSpec(t)) {
@ -779,6 +789,132 @@ func TestResponsesMatchTheSpec(t *testing.T) {
}
}
// duplicates exercises the duplicate songs: the same song as a.mp3 is added to the library, as real
// files, so that what is checked against the spec is what the real services answer.
func (c *contract) duplicates(t *testing.T, token, libID string) {
t.Helper()
repo := storage.NewLibraryRepo(c.db)
libInt, err := strconv.ParseInt(libID, 10, 64)
if err != nil {
t.Fatal(err)
}
// addCopy puts a second file with the tags of Song A in the library, as the scanner would.
addCopy := func(name string) {
t.Helper()
path := filepath.Join(c.music, name)
if err := os.WriteFile(path, bytes.Repeat([]byte{0xAB}, 400), 0o644); err != nil {
t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: name, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: "Song A", Album: "Compilation", AlbumArtist: "Artist X", Artists: []string{"Artist X"}, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: "mpeg", DurationMs: 180000, SampleRate: 44100},
},
}
if _, err := repo.SaveFile(context.Background(), libInt, f, 0); err != nil {
t.Fatal(err)
}
}
trackOf := func(name string) int64 {
t.Helper()
var id int64
if err := c.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", name).Scan(&id); err != nil {
t.Fatal(err)
}
return id
}
exists := func(name string) bool { _, err := os.Lstat(filepath.Join(c.music, name)); return err == nil }
body := func(keep int64, remove ...int64) map[string]any {
return map[string]any{"keep_track_id": keep, "remove_track_ids": remove}
}
none := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates when there are none")
if none["total"] != float64(0) || none["groups"] == nil || none["deletion_allowed"] != true {
t.Fatalf("duplicates = %v, want no group, an empty list and deleting allowed", none)
}
addCopy("dup.mp3")
keep, copyID := trackOf("a.mp3"), trackOf("dup.mp3")
list := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates")
groups, _ := list["groups"].([]any)
if list["total"] != float64(1) || len(groups) != 1 {
t.Fatalf("duplicates = %v, want the one group of Song A", list)
}
if g := groups[0].(map[string]any); g["title"] != "Song A" || g["suggested_keep_track_id"] != float64(keep) || g["hashed"] != false {
t.Fatalf("group = %v, want Song A with a.mp3 to keep and nothing read yet", g)
}
c.expect(c.call("GET", "/library/duplicates?limit=-1", token, nil), 400, "duplicates with a bad limit", "invalid_input")
c.expect(c.call("GET", "/library/duplicates", "", nil), 401, "duplicates without a token", "unauthorized")
// Not an admin: nothing of this is theirs.
c.expect(c.call("POST", "/users", token, map[string]any{"email": "carol@example.org", "password": "carols long pass"}), 201, "create a plain user")
carol := c.expect(c.call("POST", "/auth/login", "", map[string]string{"email": "carol@example.org", "password": "carols long pass"}), 200, "carol logs in")["access_token"].(string)
c.expect(c.call("GET", "/library/duplicates", carol, nil), 403, "duplicates as a plain user", "admin_required")
c.expect(c.call("POST", "/library/duplicates/clean", carol, body(keep, copyID)), 403, "cleaning as a plain user", "admin_required")
// The check reads the files, and the group is then identical.
if idle := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check before it ran"); idle["running"] != false {
t.Fatalf("check = %v", idle)
}
c.expect(c.call("POST", "/library/duplicates/check?refresh=maybe", token, nil), 400, "a check with a bad refresh", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/check", "", nil), 401, "a check without a token", "unauthorized")
c.expect(c.call("POST", "/library/duplicates/check", token, nil), 202, "start the check")
for i := 0; ; i++ {
st := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check running")
if st["running"] == false {
if last, _ := st["last"].(map[string]any); last == nil || last["checked"] != float64(2) || last["hashed"] != float64(2) {
t.Fatalf("check = %v, want 2 files checked and hashed", st)
}
break
}
if i > 400 {
t.Fatal("the check did not finish")
}
time.Sleep(25 * time.Millisecond)
}
c.expect(c.call("DELETE", "/library/duplicates/check", token, nil), 204, "stop the check when none runs")
after := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates after the check")
if g := after["groups"].([]any)[0].(map[string]any); g["hashed"] != true || g["identical"] != true {
t.Fatalf("group = %v, want hashed and identical", g)
}
// Refusals first: nothing is removed by any of them.
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, keep)), 400, "cleaning the copy to keep", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/clean", token, map[string]any{"keep_track_id": keep}), 400, "cleaning with nothing to remove", "invalid_input")
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, 99999)), 404, "cleaning a track that does not exist", "track_not_found")
c.expect(c.call("POST", "/library/duplicates/clean", "", body(keep, copyID)), 401, "cleaning without a token", "unauthorized")
other := trackOf("b.flac")
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, other)), 400, "cleaning something that is not a copy", "invalid_input")
if !exists("dup.mp3") || !exists("b.flac") {
t.Fatal("a refused request removed a file")
}
// A copy that changed since the scan is not removed.
addCopy("dup2.mp3")
dup2 := trackOf("dup2.mp3")
if err := os.WriteFile(filepath.Join(c.music, "dup2.mp3"), bytes.Repeat([]byte{0xCD}, 500), 0o644); err != nil {
t.Fatal(err)
}
c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, dup2)), 409, "cleaning a copy that changed", "duplicate_changed")
if !exists("dup2.mp3") {
t.Fatal("a copy that changed was removed")
}
// The real thing: the extra copy leaves the disk and the index, the other stays.
done := c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, copyID)), 200, "clean the duplicate")
if done["removed"] != float64(1) || done["freed_bytes"] != float64(400) {
t.Fatalf("clean = %v, want 1 file and 400 bytes", done)
}
if exists("dup.mp3") || !exists("a.mp3") {
t.Fatalf("after the clean dup.mp3 exists=%v a.mp3 exists=%v", exists("dup.mp3"), exists("a.mp3"))
}
}
type orphanIDs struct{ album, track string }
// addOrphan inserts an album with no album artist and a track in it whose file

View file

@ -0,0 +1,375 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/storage"
)
// dupAPI is the API over a real library on a real folder.
type dupAPI struct {
t *testing.T
h http.Handler
db *bun.DB
dir string
token string
scanning bool
}
func newDupAPI(t *testing.T, allow bool) *dupAPI {
t.Helper()
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
})
d.Duplicates = svc
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
})
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
a.dir = dir
a.token = registerAndLogin(t, a.h)
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
t.Fatal(err)
}
return a
}
// seed writes a file and indexes it as it is, and returns its track.
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
a.t.Helper()
path := filepath.Join(a.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
a.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
a.t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
a.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
a.t.Fatal(err)
}
var id int64
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
a.t.Fatal(err)
}
return id
}
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
func (a *dupAPI) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
return err == nil
}
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
}
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
a := newDupAPI(t, true)
_, bob := addUser(t, a.h, a.token, "bob", false)
calls := []struct{ method, path string }{
{"GET", "/api/v1/library/duplicates"},
{"POST", "/api/v1/library/duplicates/clean"},
{"GET", "/api/v1/library/duplicates/check"},
{"POST", "/api/v1/library/duplicates/check"},
{"DELETE", "/api/v1/library/duplicates/check"},
}
for _, c := range calls {
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
}
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
}
}
}
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
req.Header.Set("Authorization", "Bearer "+a.token)
a.h.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body)
}
raw := rec.Body.String()
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
if strings.Contains(raw, secret) {
t.Errorf("the answer contains %q: file paths are private", secret)
}
}
var out struct {
DeletionAllowed bool `json:"deletion_allowed"`
Total, Limit int
Offset int
Groups []struct {
Title string `json:"title"`
Artists []struct {
ID int64 `json:"id"`
Name string `json:"name"`
} `json:"artists"`
Files []struct {
TrackID int64 `json:"track_id"`
Album *struct {
ID int64 `json:"id"`
Title string `json:"title"`
} `json:"album"`
Format string `json:"format"`
Bitrate int `json:"bitrate"`
SizeBytes int64 `json:"size_bytes"`
DurationMs int64 `json:"duration_ms"`
Hashed bool `json:"hashed"`
} `json:"files"`
Hashed bool `json:"hashed"`
Identical bool `json:"identical"`
Keep int64 `json:"suggested_keep_track_id"`
FreedBytes int64 `json:"freed_bytes"`
} `json:"groups"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
t.Fatalf("answer = %+v", out)
}
g := out.Groups[0]
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
t.Errorf("group = %+v", g)
}
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
t.Errorf("files = %+v", g.Files)
}
}
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
a := newDupAPI(t, false)
a.twoCopies()
r := a.get("/api/v1/library/duplicates")
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
t.Errorf("answer = %d %v", r.Code, r.Body)
}
}
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
a := newDupAPI(t, true)
for _, title := range []string{"A song", "B song"} {
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
first := a.get("/api/v1/library/duplicates?limit=1")
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
t.Fatalf("pages = %v and %v", first.Body, second.Body)
}
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
}
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
}
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
}
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
}
}
}
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
r := a.clean(flac, mp3)
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
t.Fatalf("answer = %d %v", r.Code, r.Body)
}
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("the wrong file was removed")
}
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
t.Errorf("the list still shows %v groups", list.Body["total"])
}
}
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
t.Run("deleting is off", func(t *testing.T) {
a := newDupAPI(t, false)
flac, mp3 := a.twoCopies()
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a scan is running", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.scanning = true
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("not copies of one song", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a track that does not exist", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a file that changed since the scan", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
t.Fatal(err)
}
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a read-only folder", func(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
dir := filepath.Join(a.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
}
func TestACleanRequestMustBeWellFormed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
bodies := map[string]any{
"malformed JSON": `{"keep_track_id":`,
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
}
for name, body := range bodies {
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
}
}
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("a file was removed by a bad request")
}
}
// --- the check job ---------------------------------------------------------------------------------
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
a := newDupAPI(t, true)
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
t.Fatalf("start: %d %v", r.Code, r.Body)
}
var st resp
for i := 0; ; i++ {
st = a.get("/api/v1/library/duplicates/check")
if st.Body["running"] == false {
break
}
if i > 400 {
t.Fatal("the job did not finish")
}
time.Sleep(25 * time.Millisecond)
}
last, _ := st.Body["last"].(map[string]any)
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
}
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
t.Errorf("the group is not called identical after the check: %v", list.Body)
}
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
t.Errorf("stop: %d", r.Code)
}
}
type alreadyRunning struct{}
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
func (alreadyRunning) Stop() bool { return false }
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
d.DuplicatesJob = alreadyRunning{}
})
a.token = registerAndLogin(t, a.h)
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
}
}

View file

@ -0,0 +1,169 @@
package httpapi
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
)
// duplicateHandlers serve the admin's view of the songs that are in more than one file, and the
// removal of the extra copies. File paths never appear in any answer.
type duplicateHandlers struct{ svc duplicateService }
const (
defaultDuplicatesLimit = 50
maxDuplicatesLimit = 200
)
type duplicateFileJSON struct {
TrackID int64 `json:"track_id"`
Album *albumRefJSON `json:"album"`
Format string `json:"format"`
Bitrate int `json:"bitrate"`
SizeBytes int64 `json:"size_bytes"`
DurationMs int64 `json:"duration_ms"`
Hashed bool `json:"hashed"`
}
type duplicateGroupJSON struct {
Title string `json:"title"`
Artists []artistRefJSON `json:"artists"`
Files []duplicateFileJSON `json:"files"`
Hashed bool `json:"hashed"`
Identical bool `json:"identical"`
SuggestedID int64 `json:"suggested_keep_track_id"`
FreedBytes int64 `json:"freed_bytes"`
}
type duplicatePageJSON struct {
DeletionAllowed bool `json:"deletion_allowed"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Groups []duplicateGroupJSON `json:"groups"`
}
// list gives the groups of copies, a page at a time.
func (h *duplicateHandlers) list(c *gin.Context) {
limit, ok := queryInt(c, "limit")
if !ok {
return
}
offset, ok := queryInt(c, "offset")
if !ok {
return
}
if limit < 0 || offset < 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "limit and offset cannot be negative")
return
}
size := clampInt(limit)
if size == 0 {
size = defaultDuplicatesLimit
}
size = min(size, maxDuplicatesLimit)
page, err := h.svc.Groups(c.Request.Context(), size, clampInt(offset))
if err != nil {
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
return
}
out := duplicatePageJSON{
DeletionAllowed: page.DeletionAllowed, Total: page.Total, Limit: size, Offset: clampInt(offset),
Groups: make([]duplicateGroupJSON, len(page.Groups)),
}
for i, g := range page.Groups {
out.Groups[i] = toDuplicateGroupJSON(g)
}
c.JSON(http.StatusOK, out)
}
func toDuplicateGroupJSON(g duplicates.Group) duplicateGroupJSON {
out := duplicateGroupJSON{
Title: g.Title, Artists: make([]artistRefJSON, len(g.Artists)), Files: make([]duplicateFileJSON, len(g.Files)),
Hashed: g.Hashed, Identical: g.Identical, SuggestedID: g.KeepTrackID, FreedBytes: g.FreedBytes,
}
for i, a := range g.Artists {
out.Artists[i] = artistRefJSON{ID: a.ID, Name: a.Name}
}
for i, f := range g.Files {
file := duplicateFileJSON{
TrackID: f.TrackID, Format: f.Format, Bitrate: f.BitRate, SizeBytes: f.SizeBytes, DurationMs: f.DurationMs, Hashed: f.Hash != "",
}
if f.AlbumID != 0 {
file.Album = &albumRefJSON{ID: f.AlbumID, Title: f.Album}
}
out.Files[i] = file
}
return out
}
type cleanRequest struct {
KeepTrackID int64 `json:"keep_track_id"`
RemoveTrackIDs []int64 `json:"remove_track_ids"`
}
// clean removes the copies named of one song, keeping the one named.
func (h *duplicateHandlers) clean(c *gin.Context) {
var req cleanRequest
if !bindJSON(c, &req) {
return
}
if req.KeepTrackID <= 0 || len(req.RemoveTrackIDs) == 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "name the copy to keep and at least one copy to remove")
return
}
for _, id := range req.RemoveTrackIDs {
if id <= 0 {
writeError(c, http.StatusBadRequest, "invalid_input", "track ids must be positive numbers")
return
}
}
res, err := h.svc.Clean(c.Request.Context(), req.KeepTrackID, req.RemoveTrackIDs)
if err != nil {
writeDuplicatesError(c, err)
return
}
c.JSON(http.StatusOK, gin.H{"removed": res.Removed, "freed_bytes": res.FreedBytes})
}
// writeDuplicatesError gives each way a removal is refused its own status and code, so a client can tell them apart.
func writeDuplicatesError(c *gin.Context, err error) {
switch {
case errors.Is(err, duplicates.ErrDisabled):
writeError(c, http.StatusConflict, "deletion_disabled", "deleting files is not turned on: set library.allow_delete")
case errors.Is(err, duplicates.ErrScanRunning):
writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish")
case errors.Is(err, duplicates.ErrInvalid):
writeError(c, http.StatusBadRequest, "invalid_input", err.Error())
case errors.Is(err, duplicates.ErrTrackNotFound):
writeError(c, http.StatusNotFound, "track_not_found", "no such track")
case errors.Is(err, duplicates.ErrChanged):
writeError(c, http.StatusConflict, "duplicate_changed", "a file is not what the last scan saw: scan again")
case errors.Is(err, duplicates.ErrReadOnly):
writeError(c, http.StatusConflict, "library_read_only", "the music folder cannot be written to")
default:
_ = c.Error(err)
writeError(c, http.StatusInternalServerError, "internal", "internal error")
}
}
type duplicatesCheckSummaryJSON struct {
Checked int `json:"checked"`
Hashed int `json:"hashed"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Stopped bool `json:"stopped"`
}
// duplicatesCheckSummary is the summary of the job that fingerprints the files of the groups.
func duplicatesCheckSummary(s *jobs.Summary) any {
return duplicatesCheckSummaryJSON{
Checked: s.Checked, Hashed: s.Outcomes[duplicates.OutcomeHashed], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped,
}
}

View file

@ -18,6 +18,7 @@ import (
"codeberg.org/squidd/sonda/internal/catalog"
"codeberg.org/squidd/sonda/internal/discover"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/images"
"codeberg.org/squidd/sonda/internal/jobs"
@ -105,6 +106,11 @@ type jobService interface {
Status() jobs.Status
}
type duplicateService interface {
Groups(ctx context.Context, limit, offset int) (duplicates.Page, error)
Clean(ctx context.Context, keep int64, remove []int64) (duplicates.Result, error)
}
type coverService interface {
AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error)
}
@ -155,6 +161,8 @@ type Deps struct {
CoversJob jobService
ArtistImagesJob jobService
DescriptionsJob jobService
Duplicates duplicateService
DuplicatesJob jobService
About aboutService
ArtistImages artistImageService
Settings userSettingsService
@ -274,6 +282,14 @@ func NewRouter(d Deps) *gin.Engine {
admin.DELETE("/library/artist-images", aj.stop)
admin.POST("/library/descriptions", dj.start)
admin.DELETE("/library/descriptions", dj.stop)
dup := &duplicateHandlers{svc: d.Duplicates}
dupJob := &jobHandlers{job: d.DuplicatesJob, runningCode: "duplicates_check_running", summary: duplicatesCheckSummary}
admin.GET("/library/duplicates", dup.list)
admin.POST("/library/duplicates/clean", dup.clean)
admin.GET("/library/duplicates/check", dupJob.status)
admin.POST("/library/duplicates/check", dupJob.start)
admin.DELETE("/library/duplicates/check", dupJob.stop)
libs := &librariesHandlers{svc: d.Libraries}
admin.GET("/libraries", libs.list)
admin.POST("/libraries", libs.create)

View file

@ -715,6 +715,162 @@ paths:
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates:
get:
tags: [Scan]
operationId: listDuplicates
summary: The songs that are in more than one file
description: |
Admin only. Groups of files that are the same song by the same artists, worked out from the index and
never from the disk, so it is quick and touches no file. Two files are the same song when they have the
same set of main artists, the same title (the key Sonda matches titles by) and durations no more than
3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches,
a file flagged as missing does not count, and a copy may be on a different album: `files` says which.
Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC)
before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when
every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when,
besides, they are byte for byte the same. No file path is ever returned.
`deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete`
off (the default) the groups can still be seen.
security: [{bearerAuth: []}]
parameters:
- {$ref: '#/components/parameters/Limit'}
- {$ref: '#/components/parameters/Offset'}
responses:
'200':
description: A page of groups, in a fixed order (by title).
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesPage'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates/check:
get:
tags: [Scan]
operationId: getDuplicatesCheck
summary: Progress of the check that reads the files of the duplicates
description: |
Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended.
The check is described at `POST /library/duplicates/check`.
security: [{bearerAuth: []}]
responses:
'200':
description: The job.
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesCheckStatus'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
post:
tags: [Scan]
operationId: startDuplicatesCheck
summary: Read the files of the duplicates, to tell which are identical
description: |
Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in
a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be
called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a
network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the
fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file
again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files
cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it.
security: [{bearerAuth: []}]
parameters:
- name: refresh
in: query
description: "`true` reads every file again, also those that have a fingerprint. `false` when absent."
schema: {type: boolean, default: false}
responses:
'202':
description: The check was started in the background.
content:
application/json:
schema:
type: object
additionalProperties: false
required: [status]
properties:
status: {type: string, enum: [started]}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
'409':
description: The check is already running (`duplicates_check_running`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'500':
description: The groups could not be worked out (`internal`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
delete:
tags: [Scan]
operationId: stopDuplicatesCheck
summary: Stop the check of the duplicates
description: |
Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It
answers the same when no check is running.
security: [{bearerAuth: []}]
responses:
'204': {description: "Done, also when no check was running."}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
/library/duplicates/clean:
post:
tags: [Scan]
operationId: cleanDuplicates
summary: Delete the extra copies of one song
description: |
Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it
is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder
has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies
`remove_track_ids` of one song and keeps `keep_track_id`.
The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is
running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of
`GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to
remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to
remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed`
for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of
a song is never removed. Every file is checked before the first is removed.
Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a
link) and its track leaves the index; **the listens of the removed track become listens of the track that
stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for
example the folder turns out to be read-only for the second file), what was removed stays removed.
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: '#/components/schemas/CleanDuplicates'}
responses:
'200':
description: The extra copies were removed.
content:
application/json:
schema: {$ref: '#/components/schemas/DuplicatesCleaned'}
'400': {$ref: '#/components/responses/InvalidInput'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/AdminRequired'}
'404':
description: A track that was named does not exist (`track_not_found`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
'409':
description: |
Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last
scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`).
content:
application/json:
schema: {$ref: '#/components/schemas/Error'}
/library/scan:
get:
tags: [Scan]
@ -2800,6 +2956,108 @@ components:
stopped: {type: boolean, description: "It was stopped, or the application was."}
last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."}
DuplicatesPage:
type: object
additionalProperties: false
required: [deletion_allowed, total, limit, offset, groups]
properties:
deletion_allowed: {type: boolean, description: "Whether `POST /library/duplicates/clean` can work (`library.allow_delete`)."}
total: {type: integer, description: "Groups in all, not in this page."}
limit: {type: integer}
offset: {type: integer}
groups:
type: array
items: {$ref: '#/components/schemas/DuplicateGroup'}
DuplicateGroup:
type: object
additionalProperties: false
required: [title, artists, files, hashed, identical, suggested_keep_track_id, freed_bytes]
properties:
title: {type: string}
artists:
type: array
description: The main artists the copies share.
items: {$ref: '#/components/schemas/ArtistRef'}
files:
type: array
description: The copies, by track id; at least two.
items: {$ref: '#/components/schemas/DuplicateFile'}
hashed: {type: boolean, description: "Every file of the group has been read by the check."}
identical: {type: boolean, description: "Hashed, and byte for byte the same."}
suggested_keep_track_id: {type: integer, format: int64, description: "The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest."}
freed_bytes: {type: integer, format: int64, description: "The size of every copy but the one to keep."}
DuplicateFile:
type: object
additionalProperties: false
required: [track_id, album, format, bitrate, size_bytes, duration_ms, hashed]
properties:
track_id: {type: integer, format: int64}
album:
type: object
nullable: true
additionalProperties: false
required: [id, title]
properties:
id: {type: integer, format: int64}
title: {type: string}
format: {type: string, description: "`mp3`, `flac`..."}
bitrate: {type: integer, description: "0 when unknown."}
size_bytes: {type: integer, format: int64}
duration_ms: {type: integer, format: int64}
hashed: {type: boolean, description: "This file has been read by the check."}
CleanDuplicates:
type: object
additionalProperties: false
required: [keep_track_id, remove_track_ids]
properties:
keep_track_id: {type: integer, format: int64, minimum: 1, description: "The copy that stays."}
remove_track_ids:
type: array
minItems: 1
maxItems: 100
description: "The copies to delete, of the same song as the one that stays, none twice."
items: {type: integer, format: int64, minimum: 1}
DuplicatesCleaned:
type: object
additionalProperties: false
required: [removed, freed_bytes]
properties:
removed: {type: integer, description: "Files deleted."}
freed_bytes: {type: integer, format: int64}
DuplicatesCheckStatus:
type: object
additionalProperties: false
required: [running]
properties:
running: {type: boolean}
started_at: {type: string, format: date-time, description: "When the last check started."}
last_finished_at: {type: string, format: date-time}
progress:
type: object
description: Only while the check runs.
additionalProperties: false
required: [done, total]
properties:
done: {type: integer}
total: {type: integer, description: "Files the check set out to read."}
last:
type: object
description: How the last finished check ended.
additionalProperties: false
required: [checked, hashed, skipped, failed, stopped]
properties:
checked: {type: integer, description: "Files looked at."}
hashed: {type: integer, description: "Files read and fingerprinted."}
skipped: {type: integer, description: "Files that changed since the scan, or are gone."}
failed: {type: integer, description: "Files that could not be read just then."}
stopped: {type: boolean, description: "It was stopped, or the application was."}
last_error: {type: string, description: "Why the last check ended before the end, for example that the disk is not mounted."}
UserSettings:
type: object
additionalProperties: false