Admin only. GET /library/duplicates lists the groups with what is needed to choose and never a path; POST|GET|DELETE /library/duplicates/check is the job that fingerprints the files; POST /library/duplicates/clean deletes the chosen copies of one song, each refusal with its own status and code (deletion_disabled, scan_running, invalid_input, track_not_found, duplicate_changed, library_read_only). In the spec with the contract test, and wired in main, where an info line says when library.allow_delete is on.
375 lines
14 KiB
Go
375 lines
14 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
"go.uber.org/zap"
|
|
|
|
"codeberg.org/squidd/sonda/internal/duplicates"
|
|
"codeberg.org/squidd/sonda/internal/jobs"
|
|
"codeberg.org/squidd/sonda/internal/library"
|
|
"codeberg.org/squidd/sonda/internal/storage"
|
|
)
|
|
|
|
// dupAPI is the API over a real library on a real folder.
|
|
type dupAPI struct {
|
|
t *testing.T
|
|
h http.Handler
|
|
db *bun.DB
|
|
dir string
|
|
token string
|
|
scanning bool
|
|
}
|
|
|
|
func newDupAPI(t *testing.T, allow bool) *dupAPI {
|
|
t.Helper()
|
|
a := &dupAPI{t: t}
|
|
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
|
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
|
|
})
|
|
d.Duplicates = svc
|
|
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
|
|
})
|
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.dir = dir
|
|
a.token = registerAndLogin(t, a.h)
|
|
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return a
|
|
}
|
|
|
|
// seed writes a file and indexes it as it is, and returns its track.
|
|
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
|
|
a.t.Helper()
|
|
path := filepath.Join(a.dir, filepath.FromSlash(rel))
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
f := library.ParsedFile{
|
|
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
|
|
Metadata: library.Metadata{
|
|
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
|
|
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
|
|
},
|
|
}
|
|
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
var id int64
|
|
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
|
|
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
|
|
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
|
|
return flac, mp3
|
|
}
|
|
|
|
func (a *dupAPI) exists(rel string) bool {
|
|
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
|
|
return err == nil
|
|
}
|
|
|
|
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
|
|
|
|
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
|
|
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
|
|
}
|
|
|
|
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
_, bob := addUser(t, a.h, a.token, "bob", false)
|
|
calls := []struct{ method, path string }{
|
|
{"GET", "/api/v1/library/duplicates"},
|
|
{"POST", "/api/v1/library/duplicates/clean"},
|
|
{"GET", "/api/v1/library/duplicates/check"},
|
|
{"POST", "/api/v1/library/duplicates/check"},
|
|
{"DELETE", "/api/v1/library/duplicates/check"},
|
|
}
|
|
for _, c := range calls {
|
|
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
|
|
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
|
|
}
|
|
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
|
|
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
|
|
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
|
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
|
|
req.Header.Set("Authorization", "Bearer "+a.token)
|
|
a.h.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status %d: %s", rec.Code, rec.Body)
|
|
}
|
|
raw := rec.Body.String()
|
|
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
|
|
if strings.Contains(raw, secret) {
|
|
t.Errorf("the answer contains %q: file paths are private", secret)
|
|
}
|
|
}
|
|
|
|
var out struct {
|
|
DeletionAllowed bool `json:"deletion_allowed"`
|
|
Total, Limit int
|
|
Offset int
|
|
Groups []struct {
|
|
Title string `json:"title"`
|
|
Artists []struct {
|
|
ID int64 `json:"id"`
|
|
Name string `json:"name"`
|
|
} `json:"artists"`
|
|
Files []struct {
|
|
TrackID int64 `json:"track_id"`
|
|
Album *struct {
|
|
ID int64 `json:"id"`
|
|
Title string `json:"title"`
|
|
} `json:"album"`
|
|
Format string `json:"format"`
|
|
Bitrate int `json:"bitrate"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
DurationMs int64 `json:"duration_ms"`
|
|
Hashed bool `json:"hashed"`
|
|
} `json:"files"`
|
|
Hashed bool `json:"hashed"`
|
|
Identical bool `json:"identical"`
|
|
Keep int64 `json:"suggested_keep_track_id"`
|
|
FreedBytes int64 `json:"freed_bytes"`
|
|
} `json:"groups"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
|
|
t.Fatalf("answer = %+v", out)
|
|
}
|
|
g := out.Groups[0]
|
|
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
|
|
t.Errorf("group = %+v", g)
|
|
}
|
|
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
|
|
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
|
|
t.Errorf("files = %+v", g.Files)
|
|
}
|
|
}
|
|
|
|
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
|
|
a := newDupAPI(t, false)
|
|
a.twoCopies()
|
|
r := a.get("/api/v1/library/duplicates")
|
|
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
|
|
t.Errorf("answer = %d %v", r.Code, r.Body)
|
|
}
|
|
}
|
|
|
|
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
for _, title := range []string{"A song", "B song"} {
|
|
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
|
|
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
|
|
}
|
|
first := a.get("/api/v1/library/duplicates?limit=1")
|
|
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
|
|
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
|
|
t.Fatalf("pages = %v and %v", first.Body, second.Body)
|
|
}
|
|
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
|
|
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
|
|
}
|
|
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
|
|
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
|
|
}
|
|
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
|
|
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
|
|
}
|
|
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
|
|
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
|
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
r := a.clean(flac, mp3)
|
|
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
|
|
t.Fatalf("answer = %d %v", r.Code, r.Body)
|
|
}
|
|
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
|
t.Error("the wrong file was removed")
|
|
}
|
|
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
|
|
t.Errorf("the list still shows %v groups", list.Body["total"])
|
|
}
|
|
}
|
|
|
|
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
|
|
t.Run("deleting is off", func(t *testing.T) {
|
|
a := newDupAPI(t, false)
|
|
flac, mp3 := a.twoCopies()
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a scan is running", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
a.scanning = true
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("not copies of one song", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, _ := a.twoCopies()
|
|
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
|
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a track that does not exist", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, _ := a.twoCopies()
|
|
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a file that changed since the scan", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a read-only folder", func(t *testing.T) {
|
|
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
|
|
t.Skip("permissions are not enforced here")
|
|
}
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
dir := filepath.Join(a.dir, "b")
|
|
if err := os.Chmod(dir, 0o555); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestACleanRequestMustBeWellFormed(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
bodies := map[string]any{
|
|
"malformed JSON": `{"keep_track_id":`,
|
|
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
|
|
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
|
|
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
|
|
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
|
|
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
|
|
}
|
|
for name, body := range bodies {
|
|
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
|
|
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
|
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
|
|
}
|
|
}
|
|
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
|
t.Error("a file was removed by a bad request")
|
|
}
|
|
}
|
|
|
|
// --- the check job ---------------------------------------------------------------------------------
|
|
|
|
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
|
|
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
|
|
|
|
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
|
|
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
|
|
}
|
|
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
|
|
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
|
|
}
|
|
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
|
|
t.Fatalf("start: %d %v", r.Code, r.Body)
|
|
}
|
|
var st resp
|
|
for i := 0; ; i++ {
|
|
st = a.get("/api/v1/library/duplicates/check")
|
|
if st.Body["running"] == false {
|
|
break
|
|
}
|
|
if i > 400 {
|
|
t.Fatal("the job did not finish")
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
last, _ := st.Body["last"].(map[string]any)
|
|
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
|
|
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
|
|
}
|
|
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
|
|
t.Errorf("the group is not called identical after the check: %v", list.Body)
|
|
}
|
|
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
|
|
t.Errorf("stop: %d", r.Code)
|
|
}
|
|
}
|
|
|
|
type alreadyRunning struct{}
|
|
|
|
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
|
|
func (alreadyRunning) Stop() bool { return false }
|
|
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
|
|
|
|
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
|
|
a := &dupAPI{t: t}
|
|
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
|
|
d.DuplicatesJob = alreadyRunning{}
|
|
})
|
|
a.token = registerAndLogin(t, a.h)
|
|
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
|
|
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
|
|
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
|
|
}
|
|
}
|