Migrating hosting from Codeberg to git.kmalloc.org. Renames the module path in go.mod, every internal import, and the OCI image source label in the Dockerfile. Build and full test suite verified green after the rename.
375 lines
14 KiB
Go
375 lines
14 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
"go.uber.org/zap"
|
|
|
|
"git.kmalloc.org/squid/sonda/internal/duplicates"
|
|
"git.kmalloc.org/squid/sonda/internal/jobs"
|
|
"git.kmalloc.org/squid/sonda/internal/library"
|
|
"git.kmalloc.org/squid/sonda/internal/storage"
|
|
)
|
|
|
|
// dupAPI is the API over a real library on a real folder.
|
|
type dupAPI struct {
|
|
t *testing.T
|
|
h http.Handler
|
|
db *bun.DB
|
|
dir string
|
|
token string
|
|
scanning bool
|
|
}
|
|
|
|
func newDupAPI(t *testing.T, allow bool) *dupAPI {
|
|
t.Helper()
|
|
a := &dupAPI{t: t}
|
|
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
svc := duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{
|
|
AllowDelete: allow, ScanRunning: func() bool { return a.scanning }, Logger: zap.NewNop(),
|
|
})
|
|
d.Duplicates = svc
|
|
d.DuplicatesJob = jobs.Runner{Ctx: context.Background(), Job: duplicates.NewHashJob(svc)}
|
|
})
|
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.dir = dir
|
|
a.token = registerAndLogin(t, a.h)
|
|
if _, err := storage.NewLibraryRepo(a.db).CreateLibrary(context.Background(), "music", dir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return a
|
|
}
|
|
|
|
// seed writes a file and indexes it as it is, and returns its track.
|
|
func (a *dupAPI) seed(rel, content, title, album string, durationMs int64, format string, bitrate int, artists ...string) int64 {
|
|
a.t.Helper()
|
|
path := filepath.Join(a.dir, filepath.FromSlash(rel))
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
f := library.ParsedFile{
|
|
FileInfo: library.FileInfo{RelPath: rel, Size: info.Size(), MTimeUnix: info.ModTime().Unix()},
|
|
Metadata: library.Metadata{
|
|
Tags: library.Tags{Title: title, Album: album, AlbumArtist: artists[0], Artists: artists, TrackNo: 1, DiscNo: 1},
|
|
Properties: library.Properties{Format: format, DurationMs: durationMs, SampleRate: 44100, BitRate: bitrate},
|
|
},
|
|
}
|
|
if _, err := storage.NewLibraryRepo(a.db).SaveFile(context.Background(), 1, f, 0); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
var id int64
|
|
if err := a.db.QueryRow("SELECT track_id FROM audio_files WHERE rel_path = ?", rel).Scan(&id); err != nil {
|
|
a.t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
func (a *dupAPI) twoCopies() (flac, mp3 int64) {
|
|
flac = a.seed("a/song.flac", "FLAC-CONTENT-OF-THE-SONG", "Song", "Album", 200_000, "flac", 900, "Ann")
|
|
mp3 = a.seed("b/song.mp3", "mp3 content", "Song", "Greatest hits", 200_500, "mp3", 192, "Ann")
|
|
return flac, mp3
|
|
}
|
|
|
|
func (a *dupAPI) exists(rel string) bool {
|
|
_, err := os.Lstat(filepath.Join(a.dir, filepath.FromSlash(rel)))
|
|
return err == nil
|
|
}
|
|
|
|
func (a *dupAPI) get(path string) resp { return do(a.t, a.h, "GET", path, a.token, nil) }
|
|
|
|
func (a *dupAPI) clean(keep int64, remove ...int64) resp {
|
|
return do(a.t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, map[string]any{"keep_track_id": keep, "remove_track_ids": remove})
|
|
}
|
|
|
|
func TestDuplicateRoutesAreAdminOnly(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
_, bob := addUser(t, a.h, a.token, "bob", false)
|
|
calls := []struct{ method, path string }{
|
|
{"GET", "/api/v1/library/duplicates"},
|
|
{"POST", "/api/v1/library/duplicates/clean"},
|
|
{"GET", "/api/v1/library/duplicates/check"},
|
|
{"POST", "/api/v1/library/duplicates/check"},
|
|
{"DELETE", "/api/v1/library/duplicates/check"},
|
|
}
|
|
for _, c := range calls {
|
|
if r := do(t, a.h, c.method, c.path, "", nil); r.Code != http.StatusUnauthorized {
|
|
t.Errorf("%s %s without a token: %d", c.method, c.path, r.Code)
|
|
}
|
|
r := do(t, a.h, c.method, c.path, bob, map[string]any{"keep_track_id": 1, "remove_track_ids": []int{2}})
|
|
if r.Code != http.StatusForbidden || errCode(r) != "admin_required" {
|
|
t.Errorf("%s %s as a plain user: %d %v", c.method, c.path, r.Code, r.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheListShowsTheGroupsAndNeverAPath(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
|
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest("GET", "/api/v1/library/duplicates", nil)
|
|
req.Header.Set("Authorization", "Bearer "+a.token)
|
|
a.h.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status %d: %s", rec.Code, rec.Body)
|
|
}
|
|
raw := rec.Body.String()
|
|
for _, secret := range []string{a.dir, "song.flac", "song.mp3", "rel_path", "path"} {
|
|
if strings.Contains(raw, secret) {
|
|
t.Errorf("the answer contains %q: file paths are private", secret)
|
|
}
|
|
}
|
|
|
|
var out struct {
|
|
DeletionAllowed bool `json:"deletion_allowed"`
|
|
Total, Limit int
|
|
Offset int
|
|
Groups []struct {
|
|
Title string `json:"title"`
|
|
Artists []struct {
|
|
ID int64 `json:"id"`
|
|
Name string `json:"name"`
|
|
} `json:"artists"`
|
|
Files []struct {
|
|
TrackID int64 `json:"track_id"`
|
|
Album *struct {
|
|
ID int64 `json:"id"`
|
|
Title string `json:"title"`
|
|
} `json:"album"`
|
|
Format string `json:"format"`
|
|
Bitrate int `json:"bitrate"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
DurationMs int64 `json:"duration_ms"`
|
|
Hashed bool `json:"hashed"`
|
|
} `json:"files"`
|
|
Hashed bool `json:"hashed"`
|
|
Identical bool `json:"identical"`
|
|
Keep int64 `json:"suggested_keep_track_id"`
|
|
FreedBytes int64 `json:"freed_bytes"`
|
|
} `json:"groups"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !out.DeletionAllowed || out.Total != 1 || out.Limit != 50 || out.Offset != 0 || len(out.Groups) != 1 {
|
|
t.Fatalf("answer = %+v", out)
|
|
}
|
|
g := out.Groups[0]
|
|
if g.Title != "Song" || len(g.Artists) != 1 || g.Artists[0].Name != "Ann" || g.Keep != flac || g.FreedBytes != int64(len("mp3 content")) || g.Hashed || g.Identical {
|
|
t.Errorf("group = %+v", g)
|
|
}
|
|
if len(g.Files) != 2 || g.Files[1].TrackID != mp3 || g.Files[1].Album == nil || g.Files[1].Album.Title != "Greatest hits" ||
|
|
g.Files[1].Format != "mp3" || g.Files[1].Bitrate != 192 || g.Files[1].DurationMs != 200_500 || g.Files[1].Hashed {
|
|
t.Errorf("files = %+v", g.Files)
|
|
}
|
|
}
|
|
|
|
func TestTheListSaysWhenDeletingIsOff(t *testing.T) {
|
|
a := newDupAPI(t, false)
|
|
a.twoCopies()
|
|
r := a.get("/api/v1/library/duplicates")
|
|
if r.Code != 200 || r.Body["deletion_allowed"] != false || r.Body["total"] != float64(1) {
|
|
t.Errorf("answer = %d %v", r.Code, r.Body)
|
|
}
|
|
}
|
|
|
|
func TestTheListIsPagedAndCheckedForGoodParameters(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
for _, title := range []string{"A song", "B song"} {
|
|
a.seed("x/"+title+".mp3", title+"1", title, "Album", 100_000, "mp3", 192, "Ann")
|
|
a.seed("y/"+title+".mp3", title+"22", title, "Album", 100_000, "mp3", 192, "Ann")
|
|
}
|
|
first := a.get("/api/v1/library/duplicates?limit=1")
|
|
second := a.get("/api/v1/library/duplicates?limit=1&offset=1")
|
|
if first.Body["total"] != float64(2) || len(first.Body["groups"].([]any)) != 1 || len(second.Body["groups"].([]any)) != 1 {
|
|
t.Fatalf("pages = %v and %v", first.Body, second.Body)
|
|
}
|
|
if first.Body["limit"] != float64(1) || second.Body["offset"] != float64(1) {
|
|
t.Errorf("limit and offset are not echoed: %v %v", first.Body, second.Body)
|
|
}
|
|
if past := a.get("/api/v1/library/duplicates?offset=10"); len(past.Body["groups"].([]any)) != 0 || past.Body["groups"] == nil {
|
|
t.Errorf("past the end: %v, want an empty list and not null", past.Body)
|
|
}
|
|
if big := a.get("/api/v1/library/duplicates?limit=100000"); big.Body["limit"] != float64(200) {
|
|
t.Errorf("a huge limit became %v, want 200", big.Body["limit"])
|
|
}
|
|
for _, q := range []string{"limit=-1", "offset=-1", "limit=x", "offset=1.5"} {
|
|
if r := a.get("/api/v1/library/duplicates?" + q); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
|
t.Errorf("?%s: %d %v, want 400 invalid_input", q, r.Code, r.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCleaningRemovesTheCopyAndSaysHowMuchWasFreed(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
r := a.clean(flac, mp3)
|
|
if r.Code != http.StatusOK || r.Body["removed"] != float64(1) || r.Body["freed_bytes"] != float64(len("mp3 content")) {
|
|
t.Fatalf("answer = %d %v", r.Code, r.Body)
|
|
}
|
|
if a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
|
t.Error("the wrong file was removed")
|
|
}
|
|
if list := a.get("/api/v1/library/duplicates"); list.Body["total"] != float64(0) {
|
|
t.Errorf("the list still shows %v groups", list.Body["total"])
|
|
}
|
|
}
|
|
|
|
func TestEveryRefusalHasItsOwnStatusAndCode(t *testing.T) {
|
|
t.Run("deleting is off", func(t *testing.T) {
|
|
a := newDupAPI(t, false)
|
|
flac, mp3 := a.twoCopies()
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "deletion_disabled" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a scan is running", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
a.scanning = true
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "scan_running" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("not copies of one song", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, _ := a.twoCopies()
|
|
other := a.seed("c/other.flac", "else", "Other", "Album", 100_000, "flac", 900, "Ann")
|
|
if r := a.clean(flac, other); r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" || !a.exists("c/other.flac") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a track that does not exist", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, _ := a.twoCopies()
|
|
if r := a.clean(flac, 9999); r.Code != http.StatusNotFound || errCode(r) != "track_not_found" {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a file that changed since the scan", func(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
if err := os.WriteFile(filepath.Join(a.dir, "b/song.mp3"), []byte("edited, and longer than it was"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "duplicate_changed" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
t.Run("a read-only folder", func(t *testing.T) {
|
|
if runtime.GOOS == "windows" || os.Geteuid() == 0 {
|
|
t.Skip("permissions are not enforced here")
|
|
}
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
dir := filepath.Join(a.dir, "b")
|
|
if err := os.Chmod(dir, 0o555); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
|
|
if r := a.clean(flac, mp3); r.Code != http.StatusConflict || errCode(r) != "library_read_only" || !a.exists("b/song.mp3") {
|
|
t.Errorf("%d %v", r.Code, r.Body)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestACleanRequestMustBeWellFormed(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
flac, mp3 := a.twoCopies()
|
|
bodies := map[string]any{
|
|
"malformed JSON": `{"keep_track_id":`,
|
|
"no copy to keep": map[string]any{"remove_track_ids": []int64{mp3}},
|
|
"nothing to remove": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{}},
|
|
"the one to keep too": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{flac, mp3}},
|
|
"a negative id": map[string]any{"keep_track_id": flac, "remove_track_ids": []int64{-1}},
|
|
"ids that are strings": map[string]any{"keep_track_id": "1", "remove_track_ids": []string{"2"}},
|
|
}
|
|
for name, body := range bodies {
|
|
r := do(t, a.h, "POST", "/api/v1/library/duplicates/clean", a.token, body)
|
|
if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" {
|
|
t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body)
|
|
}
|
|
}
|
|
if !a.exists("b/song.mp3") || !a.exists("a/song.flac") {
|
|
t.Error("a file was removed by a bad request")
|
|
}
|
|
}
|
|
|
|
// --- the check job ---------------------------------------------------------------------------------
|
|
|
|
func TestTheCheckJobReadsTheFilesAndReportsWhatItDid(t *testing.T) {
|
|
a := newDupAPI(t, true)
|
|
a.seed("a/1.mp3", "the same bytes", "Same", "A", 100_000, "mp3", 192, "Ann")
|
|
a.seed("b/2.mp3", "the same bytes", "Same", "B", 100_000, "mp3", 192, "Ann")
|
|
|
|
if idle := a.get("/api/v1/library/duplicates/check"); idle.Code != 200 || idle.Body["running"] != false {
|
|
t.Fatalf("before it ran: %d %v", idle.Code, idle.Body)
|
|
}
|
|
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check?refresh=maybe", a.token, nil); r.Code != 400 || errCode(r) != "invalid_input" {
|
|
t.Errorf("a bad refresh: %d %v", r.Code, r.Body)
|
|
}
|
|
if r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusAccepted || r.Body["status"] != "started" {
|
|
t.Fatalf("start: %d %v", r.Code, r.Body)
|
|
}
|
|
var st resp
|
|
for i := 0; ; i++ {
|
|
st = a.get("/api/v1/library/duplicates/check")
|
|
if st.Body["running"] == false {
|
|
break
|
|
}
|
|
if i > 400 {
|
|
t.Fatal("the job did not finish")
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
last, _ := st.Body["last"].(map[string]any)
|
|
if last["checked"] != float64(2) || last["hashed"] != float64(2) || last["skipped"] != float64(0) || last["failed"] != float64(0) || last["stopped"] != false {
|
|
t.Errorf("last = %v, want 2 checked and 2 hashed", last)
|
|
}
|
|
if list := a.get("/api/v1/library/duplicates"); list.Body["groups"].([]any)[0].(map[string]any)["identical"] != true {
|
|
t.Errorf("the group is not called identical after the check: %v", list.Body)
|
|
}
|
|
if r := do(t, a.h, "DELETE", "/api/v1/library/duplicates/check", a.token, nil); r.Code != http.StatusNoContent {
|
|
t.Errorf("stop: %d", r.Code)
|
|
}
|
|
}
|
|
|
|
type alreadyRunning struct{}
|
|
|
|
func (alreadyRunning) Start(bool) error { return jobs.ErrRunning }
|
|
func (alreadyRunning) Stop() bool { return false }
|
|
func (alreadyRunning) Status() jobs.Status { return jobs.Status{} }
|
|
|
|
func TestStartingTheCheckWhileItRunsIsExplained(t *testing.T) {
|
|
a := &dupAPI{t: t}
|
|
a.h, a.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
d.Duplicates = duplicates.NewService(storage.NewDuplicatesRepo(db), duplicates.Options{})
|
|
d.DuplicatesJob = alreadyRunning{}
|
|
})
|
|
a.token = registerAndLogin(t, a.h)
|
|
r := do(t, a.h, "POST", "/api/v1/library/duplicates/check", a.token, nil)
|
|
if r.Code != http.StatusConflict || errCode(r) != "duplicates_check_running" {
|
|
t.Errorf("%d %v, want 409 duplicates_check_running", r.Code, r.Body)
|
|
}
|
|
}
|