- feat: DELETE /api/v1/providers/{name}/data (admin) removes everything a
provider gave: its lists of similar artists, its tags and the records of
when it was asked, then the artists that only that data named and that
nobody has files for. Last.fm's terms ask for this when an agreement ends.
The user's own artists, music, listens and settings stay, the provider is not
switched off, and a second call removes nothing
- feat: a discovered artist that no list names any more is removed when the
list that named it is replaced, together with its tags, so they do not pile
up. Real artists, and any artist that another list still names, are kept
- feat: the scan reads the GENRE tag of each file, offline (several genres
separated by ";", lower case, at most five per file), and
GET /artists/{id}/tags gets a third entry, files: the genres of the artist's
own playable tracks, each weighted by how many tracks have it. It asks nobody
and is always there
- feat: POST /library/scan and POST /libraries/{id}/scan take full=true to read
every file again, not only the new or changed ones, so MusicBrainz ids and
genres reach music that was indexed before. Files keep their identity, so
listening history is kept. Anything but true or false is a 400
- test: the plan of a full scan, the scanner and the manager; the purge on both
databases, checked by removing the rule that spares real artists; the
cleanup of replaced lists; genres per file and per artist
- docs: README, OpenAPI and the todo list updated
417 lines
15 KiB
Go
417 lines
15 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest/observer"
|
|
|
|
"codeberg.org/squidd/sonda/internal/catalog"
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/library"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
"codeberg.org/squidd/sonda/internal/providers/lastfm"
|
|
"codeberg.org/squidd/sonda/internal/providers/outbound"
|
|
"codeberg.org/squidd/sonda/internal/storage"
|
|
)
|
|
|
|
// fakeLastfm answers for Last.fm and counts the questions.
|
|
type fakeLastfm struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
key string
|
|
result []lastfm.Similar
|
|
err error
|
|
release chan struct{} // when set, a call waits for it to be closed
|
|
}
|
|
|
|
func (f *fakeLastfm) SimilarArtists(_ context.Context, key string, _ lastfm.Artist, _ int) ([]lastfm.Similar, error) {
|
|
f.mu.Lock()
|
|
f.calls++
|
|
f.key = key
|
|
release, result, err := f.release, f.result, f.err
|
|
f.mu.Unlock()
|
|
if release != nil {
|
|
<-release
|
|
}
|
|
return result, err
|
|
}
|
|
|
|
func (f *fakeLastfm) count() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.calls
|
|
}
|
|
|
|
type similarEnv struct {
|
|
h http.Handler
|
|
db *bun.DB
|
|
admin string
|
|
source *fakeLastfm
|
|
alpha int64 // a local artist
|
|
beta int64 // another local artist
|
|
}
|
|
|
|
// newSimilarEnv wires the real catalog, discovery and provider services on a
|
|
// database that holds two local artists, Alpha and Beta, with Last.fm switched
|
|
// off and without a key.
|
|
func newSimilarEnv(t *testing.T) *similarEnv {
|
|
t.Helper()
|
|
return newSimilarEnvWith(t, func(*Deps, *bun.DB, *providers.Service) {})
|
|
}
|
|
|
|
// newSimilarEnvWith is newSimilarEnv that lets a test add more to Deps.
|
|
func newSimilarEnvWith(t *testing.T, more func(*Deps, *bun.DB, *providers.Service)) *similarEnv {
|
|
t.Helper()
|
|
src := &fakeLastfm{result: []lastfm.Similar{
|
|
{Name: "Beta", Match: 1},
|
|
{Name: "Gamma", MBID: "99999999-0000-0000-0000-000000000001", Match: 0.5},
|
|
}}
|
|
e := &similarEnv{source: src}
|
|
e.h, e.db = newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
prov := newProviders(db)
|
|
d.Providers = prov
|
|
d.Catalog = catalog.NewService(storage.NewCatalogRepo(db))
|
|
d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), src, prov, zap.NewNop())
|
|
more(d, db, prov)
|
|
})
|
|
e.admin = registerAndLogin(t, e.h)
|
|
e.seedLibrary(t)
|
|
return e
|
|
}
|
|
|
|
// seedLibrary adds a library with the local artists Alpha and Beta.
|
|
func (e *similarEnv) seedLibrary(t *testing.T) {
|
|
t.Helper()
|
|
lib := storage.NewLibraryRepo(e.db)
|
|
l, err := lib.CreateLibrary(context.Background(), "music", "/lib")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i, name := range []string{"Alpha", "Beta"} {
|
|
f := library.ParsedFile{
|
|
FileInfo: library.FileInfo{RelPath: name + ".flac", Size: 1, MTimeUnix: 1},
|
|
Metadata: library.Metadata{
|
|
Tags: library.Tags{Title: name + " song", Artists: []string{name}},
|
|
Properties: library.Properties{Format: "flac", DurationMs: 1000},
|
|
},
|
|
}
|
|
if _, err := lib.SaveFile(context.Background(), l.ID, f, 0); err != nil {
|
|
t.Fatalf("seed %d: %v", i, err)
|
|
}
|
|
}
|
|
e.alpha, e.beta = e.artist(t, "Alpha"), e.artist(t, "Beta")
|
|
}
|
|
|
|
func (e *similarEnv) artist(t *testing.T, name string) int64 {
|
|
t.Helper()
|
|
var id int64
|
|
if err := e.db.QueryRow("SELECT id FROM artists WHERE name = ?", name).Scan(&id); err != nil {
|
|
t.Fatalf("artist %q: %v", name, err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
func (e *similarEnv) enableLastfm(t *testing.T) {
|
|
t.Helper()
|
|
for _, body := range []map[string]any{{"api_key": lastfmKey}, {"enabled": true}} {
|
|
if r := do(t, e.h, "PUT", "/api/v1/providers/lastfm", e.admin, body); r.Code != http.StatusOK {
|
|
t.Fatalf("set up lastfm: %d %v", r.Code, r.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *similarEnv) path(id int64) string {
|
|
return "/api/v1/artists/" + strconv.FormatInt(id, 10) + "/similar"
|
|
}
|
|
|
|
func TestSimilarNeedsASignedInUser(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
if r := do(t, e.h, "GET", e.path(e.alpha), "", nil); r.Code != http.StatusUnauthorized {
|
|
t.Errorf("no token: %d", r.Code)
|
|
}
|
|
}
|
|
|
|
func TestSimilarRejectsWhatIsNotAnArtistOrALimit(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
e.enableLastfm(t)
|
|
for name, tt := range map[string]struct {
|
|
path string
|
|
code int
|
|
err string
|
|
}{
|
|
"unknown artist": {e.path(99999), 404, "artist_not_found"},
|
|
"not a number": {"/api/v1/artists/abc/similar", 400, "invalid_input"},
|
|
"artist with no files and no discovery": {e.path(e.artistWithoutFiles(t)), 404, "artist_not_found"},
|
|
"limit not a number": {e.path(e.alpha) + "?limit=lots", 400, "invalid_input"},
|
|
"negative limit": {e.path(e.alpha) + "?limit=-1", 400, "invalid_input"},
|
|
} {
|
|
r := do(t, e.h, "GET", tt.path, e.admin, nil)
|
|
if r.Code != tt.code || errCode(r) != tt.err {
|
|
t.Errorf("%s: %d %v, want %d %s", name, r.Code, r.Body, tt.code, tt.err)
|
|
}
|
|
}
|
|
if e.source.count() != 0 {
|
|
t.Error("Last.fm was asked about something that is not an artist")
|
|
}
|
|
}
|
|
|
|
func (e *similarEnv) artistWithoutFiles(t *testing.T) int64 {
|
|
t.Helper()
|
|
var id int64
|
|
if err := e.db.QueryRow("INSERT INTO artists (name, name_key) VALUES ('Nobody', 'nobody') RETURNING id").Scan(&id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
func TestSimilarWithTheProviderOffAsksNobody(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
sameKeys(t, "similar", r.Body, "provider", "provider_status", "stale", "retrieved_at", "attribution", "items")
|
|
items, ok := r.Body["items"].([]any)
|
|
if r.Body["provider"] != "lastfm" || r.Body["provider_status"] != "disabled" || r.Body["stale"] != false ||
|
|
!ok || len(items) != 0 {
|
|
t.Errorf("answer = %v", r.Body)
|
|
}
|
|
if v, present := r.Body["retrieved_at"]; !present || v != nil {
|
|
t.Errorf("retrieved_at = %#v, want an explicit null when nothing is known", v)
|
|
}
|
|
if e.source.count() != 0 {
|
|
t.Error("a provider that is off was contacted")
|
|
}
|
|
}
|
|
|
|
func TestSimilarArtists(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
e.enableLastfm(t)
|
|
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
sameKeys(t, "similar", r.Body, "provider", "provider_status", "stale", "retrieved_at", "attribution", "items")
|
|
if r.Body["provider_status"] != "ok" || r.Body["stale"] != false || r.Body["retrieved_at"] == nil {
|
|
t.Errorf("answer = %v", r.Body)
|
|
}
|
|
if !strings.Contains(r.Body["attribution"].(string), "Last.fm") {
|
|
t.Errorf("attribution = %v", r.Body["attribution"])
|
|
}
|
|
items := r.Body["items"].([]any)
|
|
if len(items) != 2 {
|
|
t.Fatalf("items = %v", items)
|
|
}
|
|
first, second := items[0].(map[string]any), items[1].(map[string]any)
|
|
sameKeys(t, "item", first, "artist", "in_library", "score", "rank")
|
|
sameSet := first["artist"].(map[string]any)
|
|
if sameSet["name"] != "Beta" || sameSet["id"] != float64(e.beta) || first["in_library"] != true || first["rank"] != float64(1) || first["score"] != float64(1) {
|
|
t.Errorf("first = %v", first)
|
|
}
|
|
if second["artist"].(map[string]any)["name"] != "Gamma" || second["in_library"] != false || second["rank"] != float64(2) || second["score"] != 0.5 {
|
|
t.Errorf("second = %v", second)
|
|
}
|
|
if e.source.key != lastfmKey {
|
|
t.Error("the stored key was not the one used")
|
|
}
|
|
raw, _ := json.Marshal(r.Body)
|
|
if strings.Contains(string(raw), lastfmKey) {
|
|
t.Fatal("the answer contains the API key")
|
|
}
|
|
|
|
// A second look comes from the database; the page size trims it.
|
|
again := do(t, e.h, "GET", e.path(e.alpha)+"?limit=1", e.admin, nil)
|
|
if again.Code != http.StatusOK || len(again.Body["items"].([]any)) != 1 || e.source.count() != 1 {
|
|
t.Errorf("second look: %d items=%v calls=%d", again.Code, again.Body["items"], e.source.count())
|
|
}
|
|
}
|
|
|
|
func TestSimilarWhenLastfmFailsIsAnAnswerNotAnError(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
e.enableLastfm(t)
|
|
e.source.err = lastfm.ErrUnavailable
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK || r.Body["provider_status"] != "error" || len(r.Body["items"].([]any)) != 0 {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
}
|
|
|
|
// Artists that only a provider named can be shown, and never appear in the
|
|
// lists of the library.
|
|
func TestADiscoveredArtistCanBeOpenedButIsNotListed(t *testing.T) {
|
|
e := newSimilarEnv(t)
|
|
e.enableLastfm(t)
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
gamma := int64(r.Body["items"].([]any)[1].(map[string]any)["artist"].(map[string]any)["id"].(float64))
|
|
|
|
a := do(t, e.h, "GET", "/api/v1/artists/"+strconv.FormatInt(gamma, 10), e.admin, nil)
|
|
if a.Code != http.StatusOK || a.Body["name"] != "Gamma" || a.Body["track_count"] != float64(0) || a.Body["album_count"] != float64(0) {
|
|
t.Fatalf("GetArtist: %d %v", a.Code, a.Body)
|
|
}
|
|
list := do(t, e.h, "GET", "/api/v1/artists?q=gamma", e.admin, nil)
|
|
if list.Body["total"] != float64(0) {
|
|
t.Errorf("a discovered artist is listed: %v", list.Body)
|
|
}
|
|
|
|
// And it can be asked about in turn.
|
|
e.source.result = []lastfm.Similar{{Name: "Delta", Match: 1}}
|
|
next := do(t, e.h, "GET", e.path(gamma), e.admin, nil)
|
|
if next.Code != http.StatusOK || len(next.Body["items"].([]any)) != 1 {
|
|
t.Errorf("similar to a discovered artist: %d %v", next.Code, next.Body)
|
|
}
|
|
}
|
|
|
|
// What leaves the server, seen from the other end: a real Last.fm client and
|
|
// outbound client talk to a local server that records the whole request. The
|
|
// database holds a user, a listen, a track title and a library path; none of
|
|
// them may be in it.
|
|
func TestWhatLeavesTheServerIsTheArtistAndTheKeyOnly(t *testing.T) {
|
|
var (
|
|
mu sync.Mutex
|
|
seen []string
|
|
hdr http.Header
|
|
)
|
|
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
seen = append(seen, r.Method+" "+r.URL.String())
|
|
hdr = r.Header.Clone()
|
|
mu.Unlock()
|
|
_, _ = w.Write([]byte(`{"similarartists":{"artist":[{"name":"Gamma","mbid":"","match":"1"}]}}`))
|
|
}))
|
|
defer remote.Close()
|
|
|
|
var alpha int64
|
|
h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
prov := newProviders(db)
|
|
d.Providers = prov
|
|
client := lastfm.New(outbound.New(outbound.Options{UserAgent: providers.UserAgent("1.2.3", "")}), remote.URL)
|
|
d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), client, prov, zap.NewNop())
|
|
})
|
|
admin := registerAndLogin(t, h)
|
|
l, err := storage.NewLibraryRepo(db).CreateLibrary(context.Background(), "Private Collection", "/home/alice/secret-music")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f := library.ParsedFile{
|
|
FileInfo: library.FileInfo{RelPath: "private/embarrassing-song.flac", Size: 1, MTimeUnix: 1},
|
|
Metadata: library.Metadata{
|
|
Tags: library.Tags{Title: "Embarrassing Song", Album: "Guilty Pleasures", Artists: []string{"Alpha"}},
|
|
Properties: library.Properties{Format: "flac", DurationMs: 1000},
|
|
},
|
|
}
|
|
if _, err := storage.NewLibraryRepo(db).SaveFile(context.Background(), l.ID, f, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var track int64
|
|
_ = db.QueryRow("SELECT id FROM tracks WHERE title = 'Embarrassing Song'").Scan(&track)
|
|
do(t, h, "POST", "/api/v1/listens", admin, map[string]any{
|
|
"track_id": track, "event_type": "complete", "idempotency_key": "k1"})
|
|
_ = db.QueryRow("SELECT id FROM artists WHERE name = 'Alpha'").Scan(&alpha)
|
|
|
|
for _, body := range []map[string]any{{"api_key": lastfmKey}, {"enabled": true}} {
|
|
if r := do(t, h, "PUT", "/api/v1/providers/lastfm", admin, body); r.Code != http.StatusOK {
|
|
t.Fatal(r.Body)
|
|
}
|
|
}
|
|
r := do(t, h, "GET", "/api/v1/artists/"+strconv.FormatInt(alpha, 10)+"/similar", admin, nil)
|
|
if r.Code != http.StatusOK || r.Body["provider_status"] != "ok" {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if len(seen) != 1 {
|
|
t.Fatalf("requests = %v, want exactly one", seen)
|
|
}
|
|
u, _ := url.Parse(strings.SplitN(seen[0], " ", 2)[1])
|
|
want := map[string]string{
|
|
"method": "artist.getsimilar", "artist": "Alpha", "autocorrect": "0", "limit": "50",
|
|
"api_key": lastfmKey, "format": "json",
|
|
}
|
|
got := map[string]string{}
|
|
for k, v := range u.Query() {
|
|
got[k] = v[0]
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("query = %v, want %v", got, want)
|
|
}
|
|
// Only a User-Agent that says who Sonda is, and nothing that says who uses it.
|
|
if hdr.Get("User-Agent") != "Sonda/1.2.3 ( "+providers.ProjectURL+" )" {
|
|
t.Errorf("User-Agent = %q", hdr.Get("User-Agent"))
|
|
}
|
|
for name := range hdr {
|
|
switch http.CanonicalHeaderKey(name) {
|
|
case "User-Agent", "Accept", "Accept-Encoding":
|
|
default:
|
|
t.Errorf("unexpected header %s: %v", name, hdr[name])
|
|
}
|
|
}
|
|
everything := seen[0] + " " + hdr.Get("User-Agent")
|
|
for _, secret := range []string{"alice", "Embarrassing", "Guilty", "Private Collection", "secret-music", "flac"} {
|
|
if strings.Contains(strings.ToLower(everything), strings.ToLower(secret)) {
|
|
t.Errorf("the request mentions %q: %s", secret, everything)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Someone closing the page while Last.fm is being asked is not a server error:
|
|
// nothing is logged as one, and no 500 is sent to nobody.
|
|
func TestALeavingClientIsNotAServerError(t *testing.T) {
|
|
core, logs := observer.New(zap.DebugLevel)
|
|
src := &fakeLastfm{release: make(chan struct{}), result: []lastfm.Similar{{Name: "Gamma", Match: 1}}}
|
|
h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) {
|
|
prov := newProviders(db)
|
|
d.Providers = prov
|
|
d.Logger = zap.New(core)
|
|
d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), src, prov, zap.NewNop())
|
|
})
|
|
admin := registerAndLogin(t, h)
|
|
l, _ := storage.NewLibraryRepo(db).CreateLibrary(context.Background(), "m", "/lib")
|
|
f := library.ParsedFile{
|
|
FileInfo: library.FileInfo{RelPath: "a.flac", Size: 1, MTimeUnix: 1},
|
|
Metadata: library.Metadata{Tags: library.Tags{Title: "s", Artists: []string{"Alpha"}}, Properties: library.Properties{Format: "flac", DurationMs: 1}},
|
|
}
|
|
if _, err := storage.NewLibraryRepo(db).SaveFile(context.Background(), l.ID, f, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var alpha int64
|
|
_ = db.QueryRow("SELECT id FROM artists WHERE name = 'Alpha'").Scan(&alpha)
|
|
for _, body := range []map[string]any{{"api_key": lastfmKey}, {"enabled": true}} {
|
|
do(t, h, "PUT", "/api/v1/providers/lastfm", admin, body)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
req := httptest.NewRequest("GET", "/api/v1/artists/"+strconv.FormatInt(alpha, 10)+"/similar", nil).WithContext(ctx)
|
|
req.Header.Set("Authorization", "Bearer "+admin)
|
|
rec := httptest.NewRecorder()
|
|
done := make(chan struct{})
|
|
go func() { h.ServeHTTP(rec, req); close(done) }()
|
|
for src.count() == 0 {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
cancel()
|
|
<-done
|
|
close(src.release)
|
|
|
|
if rec.Code == http.StatusInternalServerError {
|
|
t.Errorf("status = %d: a client that left is not an internal error", rec.Code)
|
|
}
|
|
for _, e := range logs.All() {
|
|
if v, ok := e.ContextMap()["error"]; ok {
|
|
t.Errorf("a log entry carries an error: %v", v)
|
|
}
|
|
}
|
|
}
|