diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 0007097..854e2d5 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -21,6 +21,7 @@ import ( "codeberg.org/squidd/sonda/internal/config" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -261,6 +262,17 @@ func run() error { Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery, } + // Deleting the extra copy of a song is the one thing that makes Sonda write in the music folders: + // off unless the operator turned it on, and said in the log when it is on. + duplicateSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: cfg.Library.AllowDelete, + ScanRunning: func() bool { return scans.Status().Running }, + Logger: logger, + }) + if cfg.Library.AllowDelete { + logger.Warn("library.allow_delete is on: an admin can delete duplicate files from the music folders") + } + router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()), @@ -281,6 +293,8 @@ func run() error { CoversJob: jobs.Runner{Ctx: ctx, Job: images.NewCoversJob(pictures, imageRepo)}, ArtistImagesJob: jobs.Runner{Ctx: ctx, Job: images.NewArtistImagesJob(pictures, imageRepo)}, DescriptionsJob: jobs.Runner{Ctx: ctx, Job: about.NewJob(descriptions, discoveryRepo)}, + Duplicates: duplicateSvc, + DuplicatesJob: jobs.Runner{Ctx: ctx, Job: duplicates.NewHashJob(duplicateSvc)}, ArtistImages: pictures, About: descriptions, Settings: userSettings, diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 82a9eb2..9a8d6ab 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -20,6 +20,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -115,6 +116,14 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun if deps.DescriptionsJob == nil { deps.DescriptionsJob = jobs.Runner{Ctx: context.Background(), Job: about.NewJob(about.NewService(storage.NewDiscoveryRepo(db), noWikipedia{}, prov, []string{"en"}, zap.NewNop()), storage.NewDiscoveryRepo(db))} } + // Deleting is off unless a test turns it on. + if deps.Duplicates == nil { + svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{Logger: zap.NewNop()}) + deps.Duplicates = svc + if deps.DuplicatesJob == nil { + deps.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)} + } + } // Built from the final services, so a test that swaps one gets a Discover that uses it. if deps.Discover == nil { deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar), diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 90bd447..3b79aff 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -26,7 +26,9 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/images" + "codeberg.org/squidd/sonda/internal/jobs" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/lastfm" @@ -140,6 +142,11 @@ func newContract(t *testing.T) *contract { libs := library.NewLibraries(repo) libs.SetBusy(func() bool { return mgr.Status().Running }) d.Scan = scanAdapter{mgr} + dupSvc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: true, ScanRunning: func() bool { return mgr.Status().Running }, Logger: zap.NewNop(), + }) + d.Duplicates = dupSvc + d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(dupSvc)} d.Libraries = libs d.Catalog = catalog.NewService(storage.NewCatalogRepo(db)) d.Streams = streaming.NewService(repo) @@ -767,6 +774,9 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"nope": true}), 400, "a setting that does not exist", "invalid_input") c.expect(c.call("GET", "/me/settings", "", nil), 401, "settings without a token", "unauthorized") + // ---- duplicate songs ---- + c.duplicates(t, token, libID) + // ---- every operation in the spec was exercised ---- var missing []string for _, op := range specOperations(loadSpec(t)) { @@ -779,6 +789,132 @@ func TestResponsesMatchTheSpec(t *testing.T) { } } +// duplicates exercises the duplicate songs: the same song as a.mp3 is added to the library, as real +// files, so that what is checked against the spec is what the real services answer. +func (c *contract) duplicates(t *testing.T, token, libID string) { + t.Helper() + repo := storage.NewLibraryRepo(c.db) + libInt, err := strconv.ParseInt(libID, 10, 64) + if err != nil { + t.Fatal(err) + } + // addCopy puts a second file with the tags of Song A in the library, as the scanner would. + addCopy := func(name string) { + t.Helper() + path := filepath.Join(c.music, name) + if err := os.WriteFile(path, bytes.Repeat([]byte{0xAB}, 400), 0o644); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: name, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: "Song A", Album: "Compilation", AlbumArtist: "Artist X", Artists: []string{"Artist X"}, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: "mpeg", DurationMs: 180000, SampleRate: 44100}, + }, + } + if _, err := repo.SaveFile(context.Background(), libInt, f, 0); err != nil { + t.Fatal(err) + } + } + trackOf := func(name string) int64 { + t.Helper() + var id int64 + if err := c.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", name).Scan(&id); err != nil { + t.Fatal(err) + } + return id + } + exists := func(name string) bool { _, err := os.Lstat(filepath.Join(c.music, name)); return err == nil } + body := func(keep int64, remove ...int64) map[string]any { + return map[string]any{"keep_track_id": keep, "remove_track_ids": remove} + } + + none := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates when there are none") + if none["total"] != float64(0) || none["groups"] == nil || none["deletion_allowed"] != true { + t.Fatalf("duplicates = %v, want no group, an empty list and deleting allowed", none) + } + + addCopy("dup.mp3") + keep, copyID := trackOf("a.mp3"), trackOf("dup.mp3") + list := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates") + groups, _ := list["groups"].([]any) + if list["total"] != float64(1) || len(groups) != 1 { + t.Fatalf("duplicates = %v, want the one group of Song A", list) + } + if g := groups[0].(map[string]any); g["title"] != "Song A" || g["suggested_keep_track_id"] != float64(keep) || g["hashed"] != false { + t.Fatalf("group = %v, want Song A with a.mp3 to keep and nothing read yet", g) + } + c.expect(c.call("GET", "/library/duplicates?limit=-1", token, nil), 400, "duplicates with a bad limit", "invalid_input") + c.expect(c.call("GET", "/library/duplicates", "", nil), 401, "duplicates without a token", "unauthorized") + + // Not an admin: nothing of this is theirs. + c.expect(c.call("POST", "/users", token, map[string]any{"email": "carol@example.org", "password": "carols long pass"}), 201, "create a plain user") + carol := c.expect(c.call("POST", "/auth/login", "", map[string]string{"email": "carol@example.org", "password": "carols long pass"}), 200, "carol logs in")["access_token"].(string) + c.expect(c.call("GET", "/library/duplicates", carol, nil), 403, "duplicates as a plain user", "admin_required") + c.expect(c.call("POST", "/library/duplicates/clean", carol, body(keep, copyID)), 403, "cleaning as a plain user", "admin_required") + + // The check reads the files, and the group is then identical. + if idle := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check before it ran"); idle["running"] != false { + t.Fatalf("check = %v", idle) + } + c.expect(c.call("POST", "/library/duplicates/check?refresh=maybe", token, nil), 400, "a check with a bad refresh", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/check", "", nil), 401, "a check without a token", "unauthorized") + c.expect(c.call("POST", "/library/duplicates/check", token, nil), 202, "start the check") + for i := 0; ; i++ { + st := c.expect(c.call("GET", "/library/duplicates/check", token, nil), 200, "the check running") + if st["running"] == false { + if last, _ := st["last"].(map[string]any); last == nil || last["checked"] != float64(2) || last["hashed"] != float64(2) { + t.Fatalf("check = %v, want 2 files checked and hashed", st) + } + break + } + if i > 400 { + t.Fatal("the check did not finish") + } + time.Sleep(25 * time.Millisecond) + } + c.expect(c.call("DELETE", "/library/duplicates/check", token, nil), 204, "stop the check when none runs") + after := c.expect(c.call("GET", "/library/duplicates", token, nil), 200, "duplicates after the check") + if g := after["groups"].([]any)[0].(map[string]any); g["hashed"] != true || g["identical"] != true { + t.Fatalf("group = %v, want hashed and identical", g) + } + + // Refusals first: nothing is removed by any of them. + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, keep)), 400, "cleaning the copy to keep", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/clean", token, map[string]any{"keep_track_id": keep}), 400, "cleaning with nothing to remove", "invalid_input") + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, 99999)), 404, "cleaning a track that does not exist", "track_not_found") + c.expect(c.call("POST", "/library/duplicates/clean", "", body(keep, copyID)), 401, "cleaning without a token", "unauthorized") + other := trackOf("b.flac") + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, other)), 400, "cleaning something that is not a copy", "invalid_input") + if !exists("dup.mp3") || !exists("b.flac") { + t.Fatal("a refused request removed a file") + } + + // A copy that changed since the scan is not removed. + addCopy("dup2.mp3") + dup2 := trackOf("dup2.mp3") + if err := os.WriteFile(filepath.Join(c.music, "dup2.mp3"), bytes.Repeat([]byte{0xCD}, 500), 0o644); err != nil { + t.Fatal(err) + } + c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, dup2)), 409, "cleaning a copy that changed", "duplicate_changed") + if !exists("dup2.mp3") { + t.Fatal("a copy that changed was removed") + } + + // The real thing: the extra copy leaves the disk and the index, the other stays. + done := c.expect(c.call("POST", "/library/duplicates/clean", token, body(keep, copyID)), 200, "clean the duplicate") + if done["removed"] != float64(1) || done["freed_bytes"] != float64(400) { + t.Fatalf("clean = %v, want 1 file and 400 bytes", done) + } + if exists("dup.mp3") || !exists("a.mp3") { + t.Fatalf("after the clean dup.mp3 exists=%v a.mp3 exists=%v", exists("dup.mp3"), exists("a.mp3")) + } +} + type orphanIDs struct{ album, track string } // addOrphan inserts an album with no album artist and a track in it whose file diff --git a/internal/api/http/duplicates_test.go b/internal/api/http/duplicates_test.go new file mode 100644 index 0000000..ac38afd --- /dev/null +++ b/internal/api/http/duplicates_test.go @@ -0,0 +1,375 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" + "codeberg.org/squidd/sonda/internal/library" + "codeberg.org/squidd/sonda/internal/storage" +) + +// dupAPI is the API over a real library on a real folder. +type dupAPI struct { + t *testing.T + h http.Handler + db *bun.DB + dir string + token string + scanning bool +} + +func newDupAPI(t *testing.T, allow bool) *dupAPI { + t.Helper() + a := &dupAPI{t: t} + a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { + svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{ + AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(), + }) + d.Duplicates = svc + d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)} + }) + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + a.dir = dir + a.token = registerAndLogin(t, a.h) + if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil { + t.Fatal(err) + } + return a +} + +// seed writes a file and indexes it as it is, and returns its track. +func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 { + a.t.Helper() + path := filepath.Join(a.dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + a.t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + a.t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + a.t.Fatal(err) + } + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()}, + Metadata: library.Metadata{ + Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1}, + Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate}, + }, + } + if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil { + a.t.Fatal(err) + } + var id int64 + if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil { + a.t.Fatal(err) + } + return id +} + +func (a *dupAPI) twoCopies() (flac, mp3 int64) { + flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann") + mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann") + return flac, mp3 +} + +func (a *dupAPI) exists(rel string) bool { + _, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel))) + return err == nil +} + +func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) } + +func (a *dupAPI) clean(keep int64, remove ...int64) resp { + return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove}) +} + +func TestDuplicateRoutesAreAdminOnly(t *testing.T) { + a := newDupAPI(t, true) + _, bob := addUser(t, a.h, a.token, "bob", false) + calls := []struct{ method, path string }{ + {"GET", "/api/v1/library/duplicates"}, + {"POST", "/api/v1/library/duplicates/clean"}, + {"GET", "/api/v1/library/duplicates/check"}, + {"POST", "/api/v1/library/duplicates/check"}, + {"DELETE", "/api/v1/library/duplicates/check"}, + } + for _, c := range calls { + if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code) + } + r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}}) + if r.Code != http.StatusForbidden || errCode(r) != "admin_required" { + t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body) + } + } +} + +func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann") + + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil) + req.Header.Set("Authorization", "Bearer "+a.token) + a.h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body) + } + raw := rec.Body.String() + for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} { + if strings.Contains(raw, secret) { + t.Errorf("the answer contains %q: file paths are private", secret) + } + } + + var out struct { + DeletionAllowed bool `json:"deletion_allowed"` + Total, Limit int + Offset int + Groups []struct { + Title string `json:"title"` + Artists []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"artists"` + Files []struct { + TrackID int64 `json:"track_id"` + Album *struct { + ID int64 `json:"id"` + Title string `json:"title"` + } `json:"album"` + Format string `json:"format"` + Bitrate int `json:"bitrate"` + SizeBytes int64 `json:"size_bytes"` + DurationMs int64 `json:"duration_ms"` + Hashed bool `json:"hashed"` + } `json:"files"` + Hashed bool `json:"hashed"` + Identical bool `json:"identical"` + Keep int64 `json:"suggested_keep_track_id"` + FreedBytes int64 `json:"freed_bytes"` + } `json:"groups"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 { + t.Fatalf("answer = %+v", out) + } + g := out.Groups[0] + if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical { + t.Errorf("group = %+v", g) + } + if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" || + g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed { + t.Errorf("files = %+v", g.Files) + } +} + +func TestTheListSaysWhenDeletingIsOff(t *testing.T) { + a := newDupAPI(t, false) + a.twoCopies() + r := a.get("/api/v1/library/duplicates") + if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) { + t.Errorf("answer = %d %v", r.Code, r.Body) + } +} + +func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) { + a := newDupAPI(t, true) + for _, title := range []string{"A song", "B song"} { + a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann") + a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann") + } + first := a.get("/api/v1/library/duplicates?limit=1") + second := a.get("/api/v1/library/duplicates?limit=1&offset=1") + if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 { + t.Fatalf("pages = %v and %v", first.Body, second.Body) + } + if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) { + t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body) + } + if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil { + t.Errorf("past the end: %v, want an empty list and not null", past.Body) + } + if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) { + t.Errorf("a huge limit became %v, want 200", big.Body["limit"]) + } + for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} { + if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body) + } + } +} + +func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + r := a.clean(flac, mp3) + if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) { + t.Fatalf("answer = %d %v", r.Code, r.Body) + } + if a.exists("b/song.mp3") || !a.exists("a/song.flac") { + t.Error("the wrong file was removed") + } + if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) { + t.Errorf("the list still shows %v groups", list.Body["total"]) + } +} + +func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) { + t.Run("deleting is off", func(t *testing.T) { + a := newDupAPI(t, false) + flac, mp3 := a.twoCopies() + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a scan is running", func(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + a.scanning = true + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("not copies of one song", func(t *testing.T) { + a := newDupAPI(t, true) + flac, _ := a.twoCopies() + other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann") + if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a track that does not exist", func(t *testing.T) { + a := newDupAPI(t, true) + flac, _ := a.twoCopies() + if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a file that changed since the scan", func(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil { + t.Fatal(err) + } + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) + t.Run("a read-only folder", func(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("permissions are not enforced here") + } + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + dir := filepath.Join(a.dir, "b") + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") { + t.Errorf("%d %v", r.Code, r.Body) + } + }) +} + +func TestACleanRequestMustBeWellFormed(t *testing.T) { + a := newDupAPI(t, true) + flac, mp3 := a.twoCopies() + bodies := map[string]any{ + "malformed JSON": `{"keep_track_id":`, + "no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}}, + "nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}}, + "the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}}, + "a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}}, + "ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}}, + } + for name, body := range bodies { + r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body) + if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body) + } + } + if !a.exists("b/song.mp3") || !a.exists("a/song.flac") { + t.Error("a file was removed by a bad request") + } +} + +// --- the check job --------------------------------------------------------------------------------- + +func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) { + a := newDupAPI(t, true) + a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann") + a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann") + + if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false { + t.Fatalf("before it ran: %d %v", idle.Code, idle.Body) + } + if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" { + t.Errorf("a bad refresh: %d %v", r.Code, r.Body) + } + if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" { + t.Fatalf("start: %d %v", r.Code, r.Body) + } + var st resp + for i := 0; ; i++ { + st = a.get("/api/v1/library/duplicates/check") + if st.Body["running"] == false { + break + } + if i > 400 { + t.Fatal("the job did not finish") + } + time.Sleep(25 * time.Millisecond) + } + last, _ := st.Body["last"].(map[string]any) + if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false { + t.Errorf("last = %v, want 2 checked and 2 hashed", last) + } + if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true { + t.Errorf("the group is not called identical after the check: %v", list.Body) + } + if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent { + t.Errorf("stop: %d", r.Code) + } +} + +type alreadyRunning struct{} + +func (alreadyRunning) Start(bool) error { return jobs.ErrRunning } +func (alreadyRunning) Stop() bool { return false } +func (alreadyRunning) Status() jobs.Status { return jobs.Status{} } + +func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) { + a := &dupAPI{t: t} + a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) { + d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{}) + d.DuplicatesJob = alreadyRunning{} + }) + a.token = registerAndLogin(t, a.h) + r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil) + if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" { + t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body) + } +} diff --git a/internal/api/http/handlers_duplicates.go b/internal/api/http/handlers_duplicates.go new file mode 100644 index 0000000..969f048 --- /dev/null +++ b/internal/api/http/handlers_duplicates.go @@ -0,0 +1,169 @@ +package httpapi + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/duplicates" + "codeberg.org/squidd/sonda/internal/jobs" +) + +// duplicateHandlers serve the admin's view of the songs that are in more than one file, and the +// removal of the extra copies. File paths never appear in any answer. +type duplicateHandlers struct{ svc duplicateService } + +const ( + defaultDuplicatesLimit = 50 + maxDuplicatesLimit = 200 +) + +type duplicateFileJSON struct { + TrackID int64 `json:"track_id"` + Album *albumRefJSON `json:"album"` + Format string `json:"format"` + Bitrate int `json:"bitrate"` + SizeBytes int64 `json:"size_bytes"` + DurationMs int64 `json:"duration_ms"` + Hashed bool `json:"hashed"` +} + +type duplicateGroupJSON struct { + Title string `json:"title"` + Artists []artistRefJSON `json:"artists"` + Files []duplicateFileJSON `json:"files"` + Hashed bool `json:"hashed"` + Identical bool `json:"identical"` + SuggestedID int64 `json:"suggested_keep_track_id"` + FreedBytes int64 `json:"freed_bytes"` +} + +type duplicatePageJSON struct { + DeletionAllowed bool `json:"deletion_allowed"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Groups []duplicateGroupJSON `json:"groups"` +} + +// list gives the groups of copies, a page at a time. +func (h *duplicateHandlers) list(c *gin.Context) { + limit, ok := queryInt(c, "limit") + if !ok { + return + } + offset, ok := queryInt(c, "offset") + if !ok { + return + } + if limit < 0 || offset < 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "limit and offset cannot be negative") + return + } + size := clampInt(limit) + if size == 0 { + size = defaultDuplicatesLimit + } + size = min(size, maxDuplicatesLimit) + + page, err := h.svc.Groups(c.Request.Context(), size, clampInt(offset)) + if err != nil { + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + out := duplicatePageJSON{ + DeletionAllowed: page.DeletionAllowed, Total: page.Total, Limit: size, Offset: clampInt(offset), + Groups: make([]duplicateGroupJSON, len(page.Groups)), + } + for i, g := range page.Groups { + out.Groups[i] = toDuplicateGroupJSON(g) + } + c.JSON(http.StatusOK, out) +} + +func toDuplicateGroupJSON(g duplicates.Group) duplicateGroupJSON { + out := duplicateGroupJSON{ + Title: g.Title, Artists: make([]artistRefJSON, len(g.Artists)), Files: make([]duplicateFileJSON, len(g.Files)), + Hashed: g.Hashed, Identical: g.Identical, SuggestedID: g.KeepTrackID, FreedBytes: g.FreedBytes, + } + for i, a := range g.Artists { + out.Artists[i] = artistRefJSON{ID: a.ID, Name: a.Name} + } + for i, f := range g.Files { + file := duplicateFileJSON{ + TrackID: f.TrackID, Format: f.Format, Bitrate: f.BitRate, SizeBytes: f.SizeBytes, DurationMs: f.DurationMs, Hashed: f.Hash != "", + } + if f.AlbumID != 0 { + file.Album = &albumRefJSON{ID: f.AlbumID, Title: f.Album} + } + out.Files[i] = file + } + return out +} + +type cleanRequest struct { + KeepTrackID int64 `json:"keep_track_id"` + RemoveTrackIDs []int64 `json:"remove_track_ids"` +} + +// clean removes the copies named of one song, keeping the one named. +func (h *duplicateHandlers) clean(c *gin.Context) { + var req cleanRequest + if !bindJSON(c, &req) { + return + } + if req.KeepTrackID <= 0 || len(req.RemoveTrackIDs) == 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "name the copy to keep and at least one copy to remove") + return + } + for _, id := range req.RemoveTrackIDs { + if id <= 0 { + writeError(c, http.StatusBadRequest, "invalid_input", "track ids must be positive numbers") + return + } + } + res, err := h.svc.Clean(c.Request.Context(), req.KeepTrackID, req.RemoveTrackIDs) + if err != nil { + writeDuplicatesError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"removed": res.Removed, "freed_bytes": res.FreedBytes}) +} + +// writeDuplicatesError gives each way a removal is refused its own status and code, so a client can tell them apart. +func writeDuplicatesError(c *gin.Context, err error) { + switch { + case errors.Is(err, duplicates.ErrDisabled): + writeError(c, http.StatusConflict, "deletion_disabled", "deleting files is not turned on: set library.allow_delete") + case errors.Is(err, duplicates.ErrScanRunning): + writeError(c, http.StatusConflict, "scan_running", "wait for the running scan to finish") + case errors.Is(err, duplicates.ErrInvalid): + writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + case errors.Is(err, duplicates.ErrTrackNotFound): + writeError(c, http.StatusNotFound, "track_not_found", "no such track") + case errors.Is(err, duplicates.ErrChanged): + writeError(c, http.StatusConflict, "duplicate_changed", "a file is not what the last scan saw: scan again") + case errors.Is(err, duplicates.ErrReadOnly): + writeError(c, http.StatusConflict, "library_read_only", "the music folder cannot be written to") + default: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + } +} + +type duplicatesCheckSummaryJSON struct { + Checked int `json:"checked"` + Hashed int `json:"hashed"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + Stopped bool `json:"stopped"` +} + +// duplicatesCheckSummary is the summary of the job that fingerprints the files of the groups. +func duplicatesCheckSummary(s *jobs.Summary) any { + return duplicatesCheckSummaryJSON{ + Checked: s.Checked, Hashed: s.Outcomes[duplicates.OutcomeHashed], Skipped: s.Skipped, Failed: s.Failed, Stopped: s.Stopped, + } +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 41e517c..f611a83 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -18,6 +18,7 @@ import ( "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/duplicates" "codeberg.org/squidd/sonda/internal/feedback" "codeberg.org/squidd/sonda/internal/images" "codeberg.org/squidd/sonda/internal/jobs" @@ -105,6 +106,11 @@ type jobService interface { Status() jobs.Status } +type duplicateService interface { + Groups(ctx context.Context, limit, offset int) (duplicates.Page, error) + Clean(ctx context.Context, keep int64, remove []int64) (duplicates.Result, error) +} + type coverService interface { AlbumCover(ctx context.Context, albumID int64, size images.Size) (*images.Cover, error) } @@ -155,6 +161,8 @@ type Deps struct { CoversJob jobService ArtistImagesJob jobService DescriptionsJob jobService + Duplicates duplicateService + DuplicatesJob jobService About aboutService ArtistImages artistImageService Settings userSettingsService @@ -274,6 +282,14 @@ func NewRouter(d Deps) *gin.Engine { admin.DELETE("/library/artist-images", aj.stop) admin.POST("/library/descriptions", dj.start) admin.DELETE("/library/descriptions", dj.stop) + + dup := &duplicateHandlers{svc: d.Duplicates} + dupJob := &jobHandlers{job: d.DuplicatesJob, runningCode: "duplicates_check_running", summary: duplicatesCheckSummary} + admin.GET("/library/duplicates", dup.list) + admin.POST("/library/duplicates/clean", dup.clean) + admin.GET("/library/duplicates/check", dupJob.status) + admin.POST("/library/duplicates/check", dupJob.start) + admin.DELETE("/library/duplicates/check", dupJob.stop) libs := &librariesHandlers{svc: d.Libraries} admin.GET("/libraries", libs.list) admin.POST("/libraries", libs.create) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 2b15625..8d9b8ea 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -715,6 +715,162 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/AdminRequired'} + /library/duplicates: + get: + tags: [Scan] + operationId: listDuplicates + summary: The songs that are in more than one file + description: | + Admin only. Groups of files that are the same song by the same artists, worked out from the index and + never from the disk, so it is quick and touches no file. Two files are the same song when they have the + same set of main artists, the same title (the key Sonda matches titles by) and durations no more than + 3 seconds apart, every pair in a group within that. A file with no duration or no artist never matches, + a file flagged as missing does not count, and a copy may be on a different album: `files` says which. + + Each group says which copy Sonda would keep (`suggested_keep_track_id`): the one without loss (FLAC) + before one with loss, then the higher bitrate, then the bigger file, then the oldest. `hashed` is true when + every file of the group has been read by the check (`POST /library/duplicates/check`) and `identical` when, + besides, they are byte for byte the same. No file path is ever returned. + + `deletion_allowed` says whether `POST /library/duplicates/clean` can work: with `library.allow_delete` + off (the default) the groups can still be seen. + security: [{bearerAuth: []}] + parameters: + - {$ref: '#/components/parameters/Limit'} + - {$ref: '#/components/parameters/Offset'} + responses: + '200': + description: A page of groups, in a fixed order (by title). + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesPage'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + + /library/duplicates/check: + get: + tags: [Scan] + operationId: getDuplicatesCheck + summary: Progress of the check that reads the files of the duplicates + description: | + Whether the check is running, how far it is (`progress`, only while it runs), and how the last one ended. + The check is described at `POST /library/duplicates/check`. + security: [{bearerAuth: []}] + responses: + '200': + description: The job. + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesCheckStatus'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + post: + tags: [Scan] + operationId: startDuplicatesCheck + summary: Read the files of the duplicates, to tell which are identical + description: | + Admin only. Starts, in the background and one at a time like a scan, a job that reads every file that is in + a group of copies and keeps its SHA-256, so that a group of files that are byte for byte the same can be + called `identical`. It only reads: nothing is removed or sent anywhere. **It reads whole files**, which on a + network drive takes a while, so it is started on purpose. A file that has a fingerprint is skipped, and the + fingerprint of a file is forgotten when a scan finds the file changed; send `refresh=true` to read every file + again. A file that changed since the scan is skipped. It gives up, and says why in `last_error`, when files + cannot be read five times in a row (a disk that is not mounted). `DELETE /library/duplicates/check` stops it. + security: [{bearerAuth: []}] + parameters: + - name: refresh + in: query + description: "`true` reads every file again, also those that have a fingerprint. `false` when absent." + schema: {type: boolean, default: false} + responses: + '202': + description: The check was started in the background. + content: + application/json: + schema: + type: object + additionalProperties: false + required: [status] + properties: + status: {type: string, enum: [started]} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '409': + description: The check is already running (`duplicates_check_running`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '500': + description: The groups could not be worked out (`internal`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + delete: + tags: [Scan] + operationId: stopDuplicatesCheck + summary: Stop the check of the duplicates + description: | + Admin only. Asks a running check to end after the file it is on; the fingerprints already kept stay. It + answers the same when no check is running. + security: [{bearerAuth: []}] + responses: + '204': {description: "Done, also when no check was running."} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + + /library/duplicates/clean: + post: + tags: [Scan] + operationId: cleanDuplicates + summary: Delete the extra copies of one song + description: | + Admin only. **Deletes files from the music folder.** It is the one thing Sonda does that writes there, so it + is off until the operator turns on `library.allow_delete` (answer `deletion_disabled` otherwise), the folder + has to be writable (`library_read_only`), and it is never done by itself: one request removes the copies + `remove_track_ids` of one song and keeps `keep_track_id`. + + The server decides again, whatever the client saw, and removes nothing unless every check passes: no scan is + running (`scan_running`); the tracks exist (`track_not_found`); they are copies of one song by the rule of + `GET /library/duplicates` (`invalid_input` otherwise, and also when the copy to keep is among those to + remove, a copy is named twice or there are none or more than 100); and the copy to keep and every copy to + remove are still the file the last scan saw, a regular file with the same size and date (`duplicate_changed` + for a file that changed, is gone, became a link or is outside the library: scan again). So the last copy of + a song is never removed. Every file is checked before the first is removed. + + Each copy is removed from the disk through the library folder (a path cannot leave it, not even through a + link) and its track leaves the index; **the listens of the removed track become listens of the track that + stays**, so the taste is not lost. Folders are never removed, not even empty ones. If it fails part way (for + example the folder turns out to be read-only for the second file), what was removed stays removed. + security: [{bearerAuth: []}] + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/CleanDuplicates'} + responses: + '200': + description: The extra copies were removed. + content: + application/json: + schema: {$ref: '#/components/schemas/DuplicatesCleaned'} + '400': {$ref: '#/components/responses/InvalidInput'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/AdminRequired'} + '404': + description: A track that was named does not exist (`track_not_found`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '409': + description: | + Deleting is not turned on (`deletion_disabled`), a scan is running (`scan_running`), a file is not what the last + scan saw (`duplicate_changed`), or the music folder cannot be written to (`library_read_only`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + /library/scan: get: tags: [Scan] @@ -2800,6 +2956,108 @@ components: stopped: {type: boolean, description: "It was stopped, or the application was."} last_error: {type: string, description: "Why the last job ended before the end, for example that a provider is off or kept failing."} + DuplicatesPage: + type: object + additionalProperties: false + required: [deletion_allowed, total, limit, offset, groups] + properties: + deletion_allowed: {type: boolean, description: "Whether `POST /library/duplicates/clean` can work (`library.allow_delete`)."} + total: {type: integer, description: "Groups in all, not in this page."} + limit: {type: integer} + offset: {type: integer} + groups: + type: array + items: {$ref: '#/components/schemas/DuplicateGroup'} + + DuplicateGroup: + type: object + additionalProperties: false + required: [title, artists, files, hashed, identical, suggested_keep_track_id, freed_bytes] + properties: + title: {type: string} + artists: + type: array + description: The main artists the copies share. + items: {$ref: '#/components/schemas/ArtistRef'} + files: + type: array + description: The copies, by track id; at least two. + items: {$ref: '#/components/schemas/DuplicateFile'} + hashed: {type: boolean, description: "Every file of the group has been read by the check."} + identical: {type: boolean, description: "Hashed, and byte for byte the same."} + suggested_keep_track_id: {type: integer, format: int64, description: "The copy Sonda would keep: without loss first, then the higher bitrate, then the bigger file, then the oldest."} + freed_bytes: {type: integer, format: int64, description: "The size of every copy but the one to keep."} + + DuplicateFile: + type: object + additionalProperties: false + required: [track_id, album, format, bitrate, size_bytes, duration_ms, hashed] + properties: + track_id: {type: integer, format: int64} + album: + type: object + nullable: true + additionalProperties: false + required: [id, title] + properties: + id: {type: integer, format: int64} + title: {type: string} + format: {type: string, description: "`mp3`, `flac`..."} + bitrate: {type: integer, description: "0 when unknown."} + size_bytes: {type: integer, format: int64} + duration_ms: {type: integer, format: int64} + hashed: {type: boolean, description: "This file has been read by the check."} + + CleanDuplicates: + type: object + additionalProperties: false + required: [keep_track_id, remove_track_ids] + properties: + keep_track_id: {type: integer, format: int64, minimum: 1, description: "The copy that stays."} + remove_track_ids: + type: array + minItems: 1 + maxItems: 100 + description: "The copies to delete, of the same song as the one that stays, none twice." + items: {type: integer, format: int64, minimum: 1} + + DuplicatesCleaned: + type: object + additionalProperties: false + required: [removed, freed_bytes] + properties: + removed: {type: integer, description: "Files deleted."} + freed_bytes: {type: integer, format: int64} + + DuplicatesCheckStatus: + type: object + additionalProperties: false + required: [running] + properties: + running: {type: boolean} + started_at: {type: string, format: date-time, description: "When the last check started."} + last_finished_at: {type: string, format: date-time} + progress: + type: object + description: Only while the check runs. + additionalProperties: false + required: [done, total] + properties: + done: {type: integer} + total: {type: integer, description: "Files the check set out to read."} + last: + type: object + description: How the last finished check ended. + additionalProperties: false + required: [checked, hashed, skipped, failed, stopped] + properties: + checked: {type: integer, description: "Files looked at."} + hashed: {type: integer, description: "Files read and fingerprinted."} + skipped: {type: integer, description: "Files that changed since the scan, or are gone."} + failed: {type: integer, description: "Files that could not be read just then."} + stopped: {type: boolean, description: "It was stopped, or the application was."} + last_error: {type: string, description: "Why the last check ended before the end, for example that the disk is not mounted."} + UserSettings: type: object additionalProperties: false