sonda/internal/api/http/duplicates_test.go
tgiachi 30f45bb383 feat(api): list the duplicate songs, check their files and delete the extra copies
Admin only. GET /library/duplicates lists the groups with what is needed to choose and never a path; POST|GET|DELETE /library/duplicates/check is the job that fingerprints the files; POST /library/duplicates/clean deletes the chosen copies of one song, each refusal with its own status and code (deletion_disabled, scan_running, invalid_input, track_not_found, duplicate_changed, library_read_only). In the spec with the contract test, and wired in main, where an info line says when library.allow_delete is on.
2026-11-12 16:46:50 +01:00

375 lines
14 KiB
Go

package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/uptrace/bun"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/duplicates"
"codeberg.org/squidd/sonda/internal/jobs"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/storage"
)
// dupAPI is the API over a real library on a real folder.
type dupAPI struct {
t *testing.T
h http.Handler
db *bun.DB
dir string
token string
scanning bool
}
func newDupAPI(t *testing.T, allow bool) *dupAPI {
t.Helper()
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
})
d.Duplicates = svc
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
})
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
a.dir = dir
a.token = registerAndLogin(t, a.h)
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
t.Fatal(err)
}
return a
}
// seed writes a file and indexes it as it is, and returns its track.
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
a.t.Helper()
path := filepath.Join(a.dir, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
a.t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
a.t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
a.t.Fatal(err)
}
f := library.ParsedFile{
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
Metadata: library.Metadata{
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
},
}
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
a.t.Fatal(err)
}
var id int64
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
a.t.Fatal(err)
}
return id
}
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
return flac, mp3
}
func (a *dupAPI) exists(rel string) bool {
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
return err == nil
}
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
}
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
a := newDupAPI(t, true)
_, bob := addUser(t, a.h, a.token, "bob", false)
calls := []struct{ method, path string }{
{"GET", "/api/v1/library/duplicates"},
{"POST", "/api/v1/library/duplicates/clean"},
{"GET", "/api/v1/library/duplicates/check"},
{"POST", "/api/v1/library/duplicates/check"},
{"DELETE", "/api/v1/library/duplicates/check"},
}
for _, c := range calls {
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
}
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
}
}
}
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
req.Header.Set("Authorization", "Bearer "+a.token)
a.h.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body)
}
raw := rec.Body.String()
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
if strings.Contains(raw, secret) {
t.Errorf("the answer contains %q: file paths are private", secret)
}
}
var out struct {
DeletionAllowed bool `json:"deletion_allowed"`
Total, Limit int
Offset int
Groups []struct {
Title string `json:"title"`
Artists []struct {
ID int64 `json:"id"`
Name string `json:"name"`
} `json:"artists"`
Files []struct {
TrackID int64 `json:"track_id"`
Album *struct {
ID int64 `json:"id"`
Title string `json:"title"`
} `json:"album"`
Format string `json:"format"`
Bitrate int `json:"bitrate"`
SizeBytes int64 `json:"size_bytes"`
DurationMs int64 `json:"duration_ms"`
Hashed bool `json:"hashed"`
} `json:"files"`
Hashed bool `json:"hashed"`
Identical bool `json:"identical"`
Keep int64 `json:"suggested_keep_track_id"`
FreedBytes int64 `json:"freed_bytes"`
} `json:"groups"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
t.Fatalf("answer = %+v", out)
}
g := out.Groups[0]
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
t.Errorf("group = %+v", g)
}
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
t.Errorf("files = %+v", g.Files)
}
}
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
a := newDupAPI(t, false)
a.twoCopies()
r := a.get("/api/v1/library/duplicates")
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
t.Errorf("answer = %d %v", r.Code, r.Body)
}
}
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
a := newDupAPI(t, true)
for _, title := range []string{"A song", "B song"} {
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
}
first := a.get("/api/v1/library/duplicates?limit=1")
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
t.Fatalf("pages = %v and %v", first.Body, second.Body)
}
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
}
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
}
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
}
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
}
}
}
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
r := a.clean(flac, mp3)
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
t.Fatalf("answer = %d %v", r.Code, r.Body)
}
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("the wrong file was removed")
}
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
t.Errorf("the list still shows %v groups", list.Body["total"])
}
}
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
t.Run("deleting is off", func(t *testing.T) {
a := newDupAPI(t, false)
flac, mp3 := a.twoCopies()
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a scan is running", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
a.scanning = true
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("not copies of one song", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a track that does not exist", func(t *testing.T) {
a := newDupAPI(t, true)
flac, _ := a.twoCopies()
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a file that changed since the scan", func(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
t.Fatal(err)
}
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
t.Run("a read-only folder", func(t *testing.T) {
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
t.Skip("permissions are not enforced here")
}
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
dir := filepath.Join(a.dir, "b")
if err := os.Chmod(dir, 0o555); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
t.Errorf("%d %v", r.Code, r.Body)
}
})
}
func TestACleanRequestMustBeWellFormed(t *testing.T) {
a := newDupAPI(t, true)
flac, mp3 := a.twoCopies()
bodies := map[string]any{
"malformed JSON": `{"keep_track_id":`,
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
}
for name, body := range bodies {
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
}
}
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
t.Error("a file was removed by a bad request")
}
}
// --- the check job ---------------------------------------------------------------------------------
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
a := newDupAPI(t, true)
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
}
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
t.Fatalf("start: %d %v", r.Code, r.Body)
}
var st resp
for i := 0; ; i++ {
st = a.get("/api/v1/library/duplicates/check")
if st.Body["running"] == false {
break
}
if i > 400 {
t.Fatal("the job did not finish")
}
time.Sleep(25 * time.Millisecond)
}
last, _ := st.Body["last"].(map[string]any)
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
}
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
t.Errorf("the group is not called identical after the check: %v", list.Body)
}
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
t.Errorf("stop: %d", r.Code)
}
}
type alreadyRunning struct{}
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
func (alreadyRunning) Stop() bool { return false }
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
a := &dupAPI{t: t}
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
d.DuplicatesJob = alreadyRunning{}
})
a.token = registerAndLogin(t, a.h)
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
}
}