From 28c249c2682d93faa4992ce1c3ce8eccb63e7174 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 19:42:09 +0100 Subject: [PATCH 1/2] feat(discover): artists I may like, with a score I can explain - GET /discover suggests artists, in my library or not, with a score from 0 to 1 and its parts: affinity (similar to the artists I like), genre (tags close to mine), preference (what my listening says) and discovery (a bonus for what I have not played). Default weights 0.40, 0.25, 0.25, 0.10, set in [scoring]. - A part with no data (no tags) is left out and the others rescaled; the answer says which parts were there, and which seeds led to each suggestion. - Seeds are the artists I name with ?seed=, or, only when I turned on discover_from_listening, the artists I play most. Without either the request is refused and nothing is sent to anyone. - The tags of the best candidates are read from Last.fm only, only for those with none stored, a few at a time. - taste.Service.Learn gives the whole picture (every preference, the liked artists, the tag profile); TagService.ProviderTags asks one provider. - Spec, contract test, README and example config updated. Closes #33 --- README.md | 8 +- cmd/sondad/main.go | 14 +- internal/api/http/api_test.go | 4 + internal/api/http/contract_test.go | 21 + internal/api/http/discover_test.go | 229 ++++++++++ internal/api/http/handlers_discover.go | 133 ++++++ internal/api/http/router.go | 8 + internal/api/openapi/openapi.yaml | 128 +++++- internal/config/config.go | 30 ++ internal/config/config_test.go | 23 + internal/discover/score.go | 134 ++++++ internal/discover/score_test.go | 152 +++++++ internal/discover/service.go | 461 +++++++++++++++++++ internal/discover/service_test.go | 607 +++++++++++++++++++++++++ internal/discovery/tags.go | 27 ++ internal/discovery/tags_test.go | 42 ++ internal/taste/service.go | 81 ++-- internal/taste/service_test.go | 35 ++ sonda.example.toml | 6 + 19 files changed, 2105 insertions(+), 38 deletions(-) create mode 100644 internal/api/http/discover_test.go create mode 100644 internal/api/http/handlers_discover.go create mode 100644 internal/discover/score.go create mode 100644 internal/discover/score_test.go create mode 100644 internal/discover/service.go create mode 100644 internal/discover/service_test.go diff --git a/README.md b/README.md index a71d0bb..a83e13b 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,14 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. - **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. +- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on; without it the answer says `disabled`. See [External providers](#external-providers). - **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, the Discover feature (coming next) may start from the artists you listen to most, and their names would be sent to Last.fm (only if an admin switched it on). - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet -Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Recommendations from all this data (scoring) are next: your taste is already learned, the scoring of what to discover is what is left. +Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Radio queues from all this data are next. ## Requirements @@ -63,6 +64,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An | `auth.refresh_ttl_hours` | | `720` | Lifetime of a refresh token (30 days). | | `listening.min_listen_seconds` | | `20` | A play counts as a listen after this long (1 to 3600), or when it is `complete`. | | `taste.half_life_days` | | `90` | After this many days a listen weighs half as much when learning your taste (1 to 3650). | +| `scoring.weight_affinity`, `weight_genre`, `weight_preference`, `weight_discovery` | | `0.40`, `0.25`, `0.25`, `0.10` | The weights of the parts of a Discover score (0 to 10 each, not all zero). Only their proportions matter. | Libraries and the external providers are not in the config: you set them up through the API (see the demo and [External providers](#external-providers)). @@ -197,7 +199,7 @@ curl -s -X PUT $BASE/providers/musicbrainz -H "$AUTH" -d '{"enabled":true,"conta The key is stored in the database, in clear text, and is never returned by the API or written to the log: the list only says `has_api_key`. Because the database holds it, treat database backups like secrets. Removing the key turns the provider off, and each provider describes in the list exactly what it sends. -**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key. +**What leaves your server, when a provider is on:** to Last.fm, the artist name (or MusicBrainz id) being looked up, your API key, your IP address and the `User-Agent`; to MusicBrainz, only the MusicBrainz id of the artist, your IP address and a `User-Agent` that carries your contact. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. **Discover** asks Last.fm for the similar artists of each seed (the seed's name or MusicBrainz id) and, for the best suggestions that have no tags yet, for the tags of that suggestion (its name); never MusicBrainz. Seeds you name are sent when you ask. Seeds taken from your listening, which are the names of the artists you play most, are sent only if you turned on `discover_from_listening` yourself; otherwise Discover refuses to run without a `seed`. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key. **Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer. @@ -229,6 +231,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | signed in | `GET`, `HEAD /tracks/{id}/stream` | Stream the original file. Also accepts `?access_token=`. | | signed in | `POST /listens` | Record a `play`, `complete` or `skip`; the answer says whether it counts as a listen. | | signed in | `GET /listens` | Your own history, newest first, with `counted` per event. Query: `limit`, `offset`, `counted=true`. | +| signed in | `GET /discover` | Artists you may like, with a score and its parts. Query: `seed` (an artist id, up to 5 times), `limit` (default 20, max 30), `in_library=true\|false`. | | signed in | `GET /me/taste` | The artists and tags Sonda learned you like, from your own listening. Query: `limit` (default 20, max 50). | | signed in | `GET /me/settings`, `PATCH /me/settings` | Your own settings (today: `discover_from_listening`, off by default). | | signed in | `GET /library/scan` | Scan status and the last result. | @@ -285,6 +288,7 @@ internal/discovery/ similar artists: what the providers said, kept and refres internal/providers/ external providers: settings, the outbound HTTP client, the Last.fm and MusicBrainz clients internal/listening/ listen events internal/taste/ what the listener likes, learned from the listening history +internal/discover/ the score of an artist you may like, and the suggestions internal/usersettings/ the settings that belong to one user internal/storage/ SQLite and PostgreSQL through bun, the repositories internal/streaming/ opening a track's file safely diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go index 61d1bdb..637925d 100644 --- a/cmd/sondad/main.go +++ b/cmd/sondad/main.go @@ -18,6 +18,7 @@ import ( "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/config" + "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" @@ -139,6 +140,14 @@ func run() error { providers.MusicBrainz: discovery.MusicBrainzTags{Client: musicbrainzClient}, }, providerSettings, logger) + tasteRepo := storage.NewTasteRepo(db) + tasteService := taste.NewService(tasteRepo, taste.Options{MinListen: cfg.Listening.MinListen(), HalfLife: cfg.Taste.HalfLife()}) + userSettings := usersettings.NewService(storage.NewUserSettingsRepo(db)) + weights := discover.Weights{ + Affinity: cfg.Scoring.WeightAffinity, Genre: cfg.Scoring.WeightGenre, + Preference: cfg.Scoring.WeightPreference, Discovery: cfg.Scoring.WeightDiscovery, + } + router := httpapi.NewRouter(httpapi.Deps{ Logger: logger, Auth: auth.NewService(storage.NewUserRepo(db), storage.NewRefreshRepo(db), tokens, cfg.Auth.RefreshTTL()), @@ -150,8 +159,9 @@ func run() error { Providers: providerSettings, Discovery: discoveries, Tags: tags, - Taste: taste.NewService(storage.NewTasteRepo(db), taste.Options{MinListen: cfg.Listening.MinListen(), HalfLife: cfg.Taste.HalfLife()}), - Settings: usersettings.NewService(storage.NewUserSettingsRepo(db)), + Taste: tasteService, + Discover: discover.NewService(tasteService, discoveries, tags, tasteRepo, userSettings, weights), + Settings: userSettings, Streams: streaming.NewService(libraryRepo), }) diff --git a/internal/api/http/api_test.go b/internal/api/http/api_test.go index 943573d..4238c25 100644 --- a/internal/api/http/api_test.go +++ b/internal/api/http/api_test.go @@ -16,6 +16,7 @@ import ( "go.uber.org/zap" "codeberg.org/squidd/sonda/internal/auth" + "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" @@ -92,6 +93,9 @@ func newTestEnvDB(t *testing.T, mutate func(*Deps, *bun.DB)) (http.Handler, *bun Settings: usersettings.NewService(storage.NewUserSettingsRepo(db)), Streams: streaming.NewService(noFiles{}), } + deps.Discover = discover.NewService(deps.Taste.(discover.Taster), deps.Discovery.(discover.Similar), + discovery.NewTagService(storage.NewDiscoveryRepo(db), map[string]discovery.TagSource{}, prov, zap.NewNop()), + storage.NewTasteRepo(db), usersettings.NewService(storage.NewUserSettingsRepo(db)), discover.DefaultWeights()) mutate(&deps, db) return NewRouter(deps), db } diff --git a/internal/api/http/contract_test.go b/internal/api/http/contract_test.go index 4e390b4..2c65401 100644 --- a/internal/api/http/contract_test.go +++ b/internal/api/http/contract_test.go @@ -526,6 +526,27 @@ func TestResponsesMatchTheSpec(t *testing.T) { c.expect(c.call("GET", "/me/taste?limit=-1", token, nil), 400, "taste with a bad limit", "invalid_input") c.expect(c.call("GET", "/me/taste?limit=abc", token, nil), 400, "taste with a limit that is not a number", "invalid_input") + // ---- discover ---- + c.expect(c.call("GET", "/discover", token, nil), 400, "discover with no seed and no permission", "seed_required") + found := c.expect(c.call("GET", "/discover?seed="+artistID+"&limit=5", token, nil), 200, "discover from a named seed") + if found["provider_status"] != "ok" || found["from_listening"] != false || len(found["seeds"].([]any)) != 1 { + t.Fatalf("discover = %v", found) + } + for _, it := range found["items"].([]any) { + if _, present := it.(map[string]any)["components"].(map[string]any)["g"]; !present { + t.Fatalf("an item has no g, not even null: %v", it) + } + } + c.expect(c.call("GET", "/discover?seed=999", token, nil), 400, "discover from an unknown seed", "invalid_input") + c.expect(c.call("GET", "/discover?seed="+artistID+"&in_library=maybe", token, nil), 400, "discover with a bad filter", "invalid_input") + c.expect(c.call("GET", "/discover?seed="+artistID+"&in_library=false", token, nil), 200, "discover outside the library") + c.expect(c.call("GET", "/discover", "", nil), 401, "discover without a token", "unauthorized") + c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"discover_from_listening": true}), 200, "allow discover from my listening") + if fromListening := c.expect(c.call("GET", "/discover", token, nil), 200, "discover from my listening"); fromListening["from_listening"] != true { + t.Fatalf("discover = %v", fromListening) + } + c.expect(c.call("PATCH", "/me/settings", token, map[string]any{"discover_from_listening": false}), 200, "forbid it again") + // ---- my settings ---- if got := c.expect(c.call("GET", "/me/settings", token, nil), 200, "settings"); got["discover_from_listening"] != false { t.Fatalf("settings = %v, want everything off", got) diff --git a/internal/api/http/discover_test.go b/internal/api/http/discover_test.go new file mode 100644 index 0000000..e6aa423 --- /dev/null +++ b/internal/api/http/discover_test.go @@ -0,0 +1,229 @@ +package httpapi + +import ( + "math" + "net/http" + "strconv" + "testing" + "time" + + "github.com/uptrace/bun" + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/discover" + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/storage" + "codeberg.org/squidd/sonda/internal/taste" + "codeberg.org/squidd/sonda/internal/usersettings" +) + +type discoverEnv struct { + *similarEnv + lf *fakeTags // what Last.fm says about tags, for any artist +} + +// newDiscoverEnv wires the real Discover on the similar-artists environment: Last.fm +// is off until a test turns it on. The fake answers say Beta (match 1, in the +// library) and Gamma (match 0.5, not in it) are similar to anyone. +func newDiscoverEnv(t *testing.T) *discoverEnv { + t.Helper() + e := &discoverEnv{lf: &fakeTags{tags: []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "grunge", Weight: 40}}}} + e.similarEnv = newSimilarEnvWith(t, func(d *Deps, db *bun.DB, prov *providers.Service) { + tasteSvc := taste.NewService(storage.NewTasteRepo(db), taste.Options{MinListen: 20 * time.Second, HalfLife: 90 * 24 * time.Hour}) + settings := usersettings.NewService(storage.NewUserSettingsRepo(db)) + tags := discovery.NewTagService(storage.NewDiscoveryRepo(db), map[string]discovery.TagSource{providers.LastFM: e.lf}, prov, zap.NewNop()) + d.Taste, d.Settings = tasteSvc, settings + d.Discover = discover.NewService(tasteSvc, d.Discovery.(discover.Similar), tags, storage.NewTasteRepo(db), settings, discover.DefaultWeights()) + }) + return e +} + +func (e *discoverEnv) get(t *testing.T, query string) resp { + t.Helper() + return do(t, e.h, "GET", "/api/v1/discover"+query, e.admin, nil) +} + +func (e *discoverEnv) allowListening(t *testing.T) { + t.Helper() + if r := do(t, e.h, "PATCH", "/api/v1/me/settings", e.admin, map[string]any{"discover_from_listening": true}); r.Code != http.StatusOK { + t.Fatalf("turn the setting on: %d %v", r.Code, r.Body) + } +} + +func (e *discoverEnv) seed(id int64) string { return "seed=" + strconv.FormatInt(id, 10) } + +func itemFor(t *testing.T, r resp, name string) map[string]any { + t.Helper() + items, _ := r.Body["items"].([]any) + for _, it := range items { + m := it.(map[string]any) + if m["artist"].(map[string]any)["name"] == name { + return m + } + } + t.Fatalf("%s is not in %v", name, r.Body["items"]) + return nil +} + +func TestDiscoverNeedsASignedInUser(t *testing.T) { + e := newDiscoverEnv(t) + if r := do(t, e.h, "GET", "/api/v1/discover", "", nil); r.Code != http.StatusUnauthorized { + t.Errorf("no token: %d", r.Code) + } +} + +func TestDiscoverWithoutASeedAndTheSettingOffAsksNobody(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + r := e.get(t, "") + if r.Code != http.StatusBadRequest || errCode(r) != "seed_required" { + t.Errorf("%d %v, want 400 seed_required", r.Code, r.Body) + } + if e.source.count() != 0 || e.lf.calls != 0 { + t.Errorf("Last.fm was asked (%d similar, %d tags) without the listener's permission", e.source.count(), e.lf.calls) + } +} + +func TestDiscoverFromANamedSeed(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + r := e.get(t, "?"+e.seed(e.alpha)) + if r.Code != http.StatusOK { + t.Fatalf("%d %v", r.Code, r.Body) + } + sameKeys(t, "discover", r.Body, "provider_status", "attribution", "from_listening", "seeds", "items") + if r.Body["provider_status"] != "ok" || r.Body["from_listening"] != false || r.Body["attribution"] == "" { + t.Errorf("answer = %v", r.Body) + } + seeds := r.Body["seeds"].([]any) + if len(seeds) != 1 { + t.Fatalf("seeds = %v", seeds) + } + sameKeys(t, "seed", seeds[0].(map[string]any), "id", "name", "weight") + if s := seeds[0].(map[string]any); s["name"] != "Alpha" || s["weight"] != float64(1) { + t.Errorf("seed = %v", s) + } + + beta, gamma := itemFor(t, r, "Beta"), itemFor(t, r, "Gamma") + sameKeys(t, "item", beta, "artist", "in_library", "score", "components", "because") + sameKeys(t, "components", beta["components"].(map[string]any), "a", "g", "p", "d") + if beta["in_library"] != true || gamma["in_library"] != false { + t.Errorf("in_library: Beta %v, Gamma %v", beta["in_library"], gamma["in_library"]) + } + // Nothing was ever listened to: no profile, so no genre, a neutral preference, full discovery. + c := beta["components"].(map[string]any) + if v, present := c["g"]; !present || v != nil { + t.Errorf("g = %#v, want an explicit null", v) + } + if c["p"] != 0.5 || c["d"] != float64(1) || c["a"] != float64(1) { + t.Errorf("components = %v, want a=1 p=0.5 d=1", c) + } + because := beta["because"].([]any) + if len(because) != 1 { + t.Fatalf("because = %v", because) + } + sameKeys(t, "because", because[0].(map[string]any), "artist", "match") + if because[0].(map[string]any)["artist"].(map[string]any)["name"] != "Alpha" { + t.Errorf("because = %v", because) + } + if e.lf.calls != 0 { + t.Error("tags were read although there is no taste to compare them with") + } +} + +func TestDiscoverFromMyListeningNeedsAndUsesTheSetting(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + alpha := e.trackOf(t, "Alpha") + e.listen(t, e.admin, alpha, "complete", 1000, "a1") + e.listen(t, e.admin, alpha, "complete", 1000, "a2") + if _, err := e.db.Exec("INSERT INTO track_genres (track_id, genre) VALUES (?, 'rock')", alpha); err != nil { + t.Fatal(err) + } + + if r := e.get(t, ""); r.Code != http.StatusBadRequest || errCode(r) != "seed_required" { + t.Fatalf("setting off: %d %v", r.Code, r.Body) + } + if e.source.count() != 0 { + t.Fatal("Last.fm was asked about the listening although the setting is off") + } + + e.allowListening(t) + r := e.get(t, "") + if r.Code != http.StatusOK || r.Body["from_listening"] != true { + t.Fatalf("%d %v", r.Code, r.Body) + } + seeds := r.Body["seeds"].([]any) + if len(seeds) != 1 || seeds[0].(map[string]any)["name"] != "Alpha" { + t.Fatalf("seeds = %v, want Alpha", seeds) + } + if w := seeds[0].(map[string]any)["weight"].(float64); math.Abs(w-math.Tanh(2.0/3)) > 0.001 { + t.Errorf("seed weight = %v, want the preference %v", w, math.Tanh(2.0/3)) + } + gamma := itemFor(t, r, "Gamma") + comp := gamma["components"].(map[string]any) + // The profile is {rock: 1}; Gamma's Last.fm tags are rock 1 and grunge 0.4. + if g, _ := comp["g"].(float64); math.Abs(g-1/math.Sqrt(1.16)) > 0.001 { + t.Errorf("g = %v, want %v", comp["g"], 1/math.Sqrt(1.16)) + } + if e.lf.calls == 0 { + t.Error("the tags of the candidates were never read") + } +} + +func TestDiscoverFilterAndLimit(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + base := "?" + e.seed(e.alpha) + r := e.get(t, base+"&in_library=false") + if r.Code != http.StatusOK || len(r.Body["items"].([]any)) != 1 { + t.Fatalf("outside: %d %v", r.Code, r.Body) + } + itemFor(t, r, "Gamma") + r = e.get(t, base+"&in_library=true") + if r.Code != http.StatusOK || len(r.Body["items"].([]any)) != 1 { + t.Fatalf("inside: %d %v", r.Code, r.Body) + } + itemFor(t, r, "Beta") + if r = e.get(t, base+"&limit=1"); len(r.Body["items"].([]any)) != 1 { + t.Errorf("limit 1: %v", r.Body["items"]) + } + if r = e.get(t, base+"&limit=1000"); r.Code != http.StatusOK { + t.Errorf("a large limit is lowered, not refused: %d", r.Code) + } +} + +func TestDiscoverRefusesWhatItCannotUnderstand(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + for name, q := range map[string]string{ + "seed not a number": "?seed=abc", + "unknown seed": "?seed=99999", + "zero seed": "?seed=0", + "too many seeds": "?seed=1&seed=2&seed=3&seed=4&seed=5&seed=6", + "limit not number": "?" + e.seed(e.alpha) + "&limit=lots", + "negative limit": "?" + e.seed(e.alpha) + "&limit=-1", + "bad in_library": "?" + e.seed(e.alpha) + "&in_library=maybe", + "in_library twice": "?" + e.seed(e.alpha) + "&in_library=true&in_library=false", + } { + r := e.get(t, q) + if r.Code != http.StatusBadRequest || errCode(r) != "invalid_input" { + t.Errorf("%s: %d %v, want 400 invalid_input", name, r.Code, r.Body) + } + } + if e.source.count() != 0 { + t.Error("Last.fm was asked about a request that was refused") + } +} + +func TestDiscoverWithLastfmOffSaysSoAndAsksNobody(t *testing.T) { + e := newDiscoverEnv(t) // Last.fm is off + r := e.get(t, "?"+e.seed(e.alpha)) + if r.Code != http.StatusOK || r.Body["provider_status"] != "disabled" { + t.Fatalf("%d %v, want 200 and status disabled", r.Code, r.Body) + } + if len(r.Body["items"].([]any)) != 0 || e.source.count() != 0 || e.lf.calls != 0 { + t.Errorf("items %v, similar calls %d, tag calls %d", r.Body["items"], e.source.count(), e.lf.calls) + } +} diff --git a/internal/api/http/handlers_discover.go b/internal/api/http/handlers_discover.go new file mode 100644 index 0000000..3d8a941 --- /dev/null +++ b/internal/api/http/handlers_discover.go @@ -0,0 +1,133 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + + "codeberg.org/squidd/sonda/internal/discover" + "codeberg.org/squidd/sonda/internal/discovery" +) + +type discoverHandlers struct{ svc discoverService } + +type seedJSON struct { + ID int64 `json:"id"` + Name string `json:"name"` + Weight float64 `json:"weight"` +} + +type componentsJSON struct { + A float64 `json:"a"` + G *float64 `json:"g"` // null when the artist has no tags to compare + P float64 `json:"p"` + D float64 `json:"d"` +} + +type reasonJSON struct { + Artist artistRefJSON `json:"artist"` + Match float64 `json:"match"` +} + +type suggestionJSON struct { + Artist artistRefJSON `json:"artist"` + InLibrary bool `json:"in_library"` + Score float64 `json:"score"` + Components componentsJSON `json:"components"` + Because []reasonJSON `json:"because"` +} + +type discoverJSON struct { + ProviderStatus string `json:"provider_status"` + Attribution string `json:"attribution"` + FromListening bool `json:"from_listening"` + Seeds []seedJSON `json:"seeds"` + Items []suggestionJSON `json:"items"` +} + +// seedParams reads every ?seed=, each a whole number. It answers 400 itself. +func seedParams(c *gin.Context) ([]int64, bool) { + var out []int64 + for _, raw := range c.QueryArray("seed") { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + writeError(c, http.StatusBadRequest, "invalid_input", "seed must be an artist id") + return nil, false + } + out = append(out, id) + } + return out, true +} + +// inLibraryParam reads ?in_library=: absent means both, otherwise exactly true or false once. +func inLibraryParam(c *gin.Context) (*bool, bool) { + values := c.QueryArray("in_library") + switch { + case len(values) == 0: + return nil, true + case len(values) == 1 && (values[0] == "true" || values[0] == "false"): + v := values[0] == "true" + return &v, true + } + writeError(c, http.StatusBadRequest, "invalid_input", "in_library must be true or false") + return nil, false +} + +// discover suggests artists the signed-in user may like. +func (h *discoverHandlers) discover(c *gin.Context) { + seeds, ok := seedParams(c) + if !ok { + return + } + limit, ok := queryInt(c, "limit") + if !ok { + return + } + inLibrary, ok := inLibraryParam(c) + if !ok { + return + } + res, err := h.svc.Discover(c.Request.Context(), currentUserID(c), + discover.Query{Seeds: seeds, Limit: clampInt(limit), InLibrary: inLibrary}) + switch { + case errors.Is(err, discover.ErrSeedRequired): + writeError(c, http.StatusBadRequest, "seed_required", err.Error()) + return + case errors.Is(err, discover.ErrInvalidQuery): + writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + return + case errors.Is(err, context.Canceled): + c.AbortWithStatus(statusClientClosed) // the client left; nobody is waiting + return + case err != nil: + _ = c.Error(err) + writeError(c, http.StatusInternalServerError, "internal", "internal error") + return + } + c.JSON(http.StatusOK, toDiscoverJSON(res)) +} + +func toDiscoverJSON(r discover.Result) discoverJSON { + out := discoverJSON{ + ProviderStatus: r.ProviderStatus, Attribution: discovery.Attribution, FromListening: r.FromListening, + Seeds: make([]seedJSON, len(r.Seeds)), Items: make([]suggestionJSON, len(r.Items)), + } + for i, s := range r.Seeds { + out.Seeds[i] = seedJSON{ID: s.ID, Name: s.Name, Weight: s.Weight} + } + for i, it := range r.Items { + s := suggestionJSON{ + Artist: artistRefJSON{ID: it.Artist.ID, Name: it.Artist.Name}, InLibrary: it.InLibrary, Score: it.Score, + Components: componentsJSON{A: it.Components.A, G: it.Components.G, P: it.Components.P, D: it.Components.D}, + Because: make([]reasonJSON, len(it.Because)), + } + for j, b := range it.Because { + s.Because[j] = reasonJSON{Artist: artistRefJSON{ID: b.Artist.ID, Name: b.Artist.Name}, Match: b.Match} + } + out.Items[i] = s + } + return out +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 18e5f45..feadb6b 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -12,6 +12,7 @@ import ( "codeberg.org/squidd/sonda/internal/auth" "codeberg.org/squidd/sonda/internal/catalog" + "codeberg.org/squidd/sonda/internal/discover" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/listening" @@ -72,6 +73,10 @@ type discoveryService interface { Similar(ctx context.Context, artistID int64, limit int) (discovery.Result, error) } +type discoverService interface { + Discover(ctx context.Context, userID int64, q discover.Query) (discover.Result, error) +} + type tasteService interface { Taste(ctx context.Context, userID int64, limit int) (taste.Profile, error) } @@ -110,6 +115,7 @@ type Deps struct { Discovery discoveryService Tags tagService Taste tasteService + Discover discoverService Settings userSettingsService Streams streamService } @@ -141,6 +147,8 @@ func NewRouter(d Deps) *gin.Engine { ts := &tasteHandlers{svc: d.Taste} protected.GET("/me/taste", ts.taste) + dc := &discoverHandlers{svc: d.Discover} + protected.GET("/discover", dc.discover) us := &userSettingsHandlers{svc: d.Settings} protected.GET("/me/settings", us.get) protected.PATCH("/me/settings", us.update) diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 62baf99..65ec3e9 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -36,7 +36,7 @@ tags: - name: Taste description: What Sonda has learned from your listening, and the settings that are yours. - name: Discovery - description: Music outside the library, from external providers an admin switched on. + description: Music outside the library, from external providers an admin switched on, and suggestions scored for you. - name: Providers description: External services Sonda can ask about artists, set up by an admin. @@ -650,6 +650,76 @@ paths: application/json: schema: {$ref: '#/components/schemas/Error'} + /discover: + get: + tags: [Discovery] + operationId: discover + summary: Artists you may like + description: | + Suggests artists, in your library or not, each with a `score` from 0 to 1 and the parts + it is made of, so every number can be explained. + + **Where it starts.** From the artists you name with `seed` (one to five, repeat the + parameter), or, when you named none and turned on `discover_from_listening` in + `PATCH /me/settings`, from the up to 20 artists you play most. With neither the request + is refused (`seed_required`) and nothing is sent to anyone. For each seed, the similar + artists are the stored ones, or are asked from Last.fm when they are missing or old and + an admin switched Last.fm on (see `GET /artists/{id}/similar`). **What is sent**: + the name or MusicBrainz id of each seed, and, for the best candidates that have no tags + yet, the name of the candidate (tags are asked from Last.fm only, never from MusicBrainz). + The seeds from your listening are therefore sent only when you allowed it. A seed is never + suggested to itself, and nobody else's listening is read. + + **The score** is `S = wA·A + wG·G + wP·P + wD·D` with the weights of the server's + `[scoring]` config (default 0.40, 0.25, 0.25, 0.10; only their proportions matter). + `a` is the affinity: 1 − Π(1 − weight·match) over the seeds that point at the artist, where + `match` is Last.fm's similarity (relative to each seed's own list) and `weight` is 1 for a + seed you named or your `preference` for one taken from your listening (see + `GET /me/taste`). `g` is how close the artist's tags are to the tags of the artists you like (cosine + of the tag vectors); it is `null` when there is nothing to compare, and then the score is + worked out without it, the other weights rescaled, so a missing part never counts as a bad + one. `p` is what your listening says about the artist, from `preference` + mapped to 0 to 1, and 0.5 for an artist you never played. `d` is a bonus for what you have + not played: `1 / (1 + listens)`. + + Only the best candidates by affinity (at least 20) are scored, then the best `limit` by + score are returned; ties go to the higher affinity, then to the lower id. `because` names + up to three seeds that count most for the suggestion. The first request after a seed's data + gets old can take several seconds, because the provider is asked politely, a few at a time; + later ones are served from the database. + + `provider_status` is `disabled` when Last.fm is off or has no key (anything stored earlier + is still used, so the list can be empty), `error` when Last.fm failed for some seed (the + rest is still returned) and `ok` otherwise. Show `attribution` where you show the data. + security: [{bearerAuth: []}] + parameters: + - name: seed + in: query + description: An artist id to start from. Repeat up to 5 times. Without it, your listening is used if you allowed that. + schema: {type: array, items: {type: integer, format: int64}, maxItems: 5} + style: form + explode: true + - name: limit + in: query + description: How many suggestions to return. 20 when absent or 0; values above 30 are lowered to 30. + schema: {type: integer, minimum: 0, default: 20} + - name: in_library + in: query + description: "`true` keeps only artists you have, `false` only artists you do not. Absent keeps both." + schema: {type: boolean} + responses: + '200': + description: The suggestions, best first. + content: + application/json: + schema: {$ref: '#/components/schemas/Discover'} + '400': + description: A seed that is not an artist, too many seeds, a bad `limit` or `in_library` (`invalid_input`), or no seed and no permission to use your listening (`seed_required`). + content: + application/json: + schema: {$ref: '#/components/schemas/Error'} + '401': {$ref: '#/components/responses/Unauthorized'} + /me/taste: get: tags: [Taste] @@ -1655,6 +1725,62 @@ components: type: array items: {$ref: '#/components/schemas/TasteTag'} + Discover: + type: object + additionalProperties: false + required: [provider_status, attribution, from_listening, seeds, items] + properties: + provider_status: {type: string, enum: [ok, disabled, error]} + attribution: {type: string, description: What to show where the data is shown.} + from_listening: {type: boolean, description: "True when the seeds were taken from your listening, false when you named them."} + seeds: + type: array + items: {$ref: '#/components/schemas/DiscoverSeed'} + items: + type: array + items: {$ref: '#/components/schemas/Suggestion'} + + DiscoverSeed: + type: object + additionalProperties: false + required: [id, name, weight] + properties: + id: {type: integer, format: int64} + name: {type: string} + weight: {type: number, minimum: 0, maximum: 1, description: "1 for a seed you named, your preference for one taken from your listening."} + + Suggestion: + type: object + additionalProperties: false + required: [artist, in_library, score, components, because] + properties: + artist: {$ref: '#/components/schemas/ArtistRef'} + in_library: {type: boolean} + score: {type: number, minimum: 0, maximum: 1} + components: {$ref: '#/components/schemas/ScoreParts'} + because: + type: array + description: Up to three seeds that count most for this suggestion, strongest first. + items: {$ref: '#/components/schemas/SuggestionReason'} + + ScoreParts: + type: object + additionalProperties: false + required: [a, g, p, d] + properties: + a: {type: number, minimum: 0, maximum: 1, description: Affinity.} + g: {type: number, minimum: 0, maximum: 1, nullable: true, description: "Genre. Null when the artist has no tags or you have no taste to compare with; the score is then worked out without it."} + p: {type: number, minimum: 0, maximum: 1, description: Preference. 0.5 for an artist you never played.} + d: {type: number, minimum: 0, maximum: 1, description: Discovery bonus.} + + SuggestionReason: + type: object + additionalProperties: false + required: [artist, match] + properties: + artist: {$ref: '#/components/schemas/ArtistRef'} + match: {type: number, minimum: 0, maximum: 1, description: "Last.fm's similarity of the suggestion to this seed, relative to the seed's own list."} + UserSettings: type: object additionalProperties: false diff --git a/internal/config/config.go b/internal/config/config.go index 3a5db8b..0b6c28e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ import ( "bytes" "errors" "fmt" + "math" "os" "time" @@ -24,6 +25,7 @@ type Config struct { Auth Auth `toml:"auth"` Listening Listening `toml:"listening"` Taste Taste `toml:"taste"` + Scoring Scoring `toml:"scoring"` } // Server holds HTTP listener settings. @@ -92,6 +94,15 @@ func (t Taste) HalfLife() time.Duration { return time.Duration(t.HalfLifeDays) * 24 * time.Hour } +// Scoring holds the weights of the parts of a Discover score. Only their +// proportions matter: 0.4, 0.25, 0.25 and 0.1 are the same as 4, 2.5, 2.5 and 1. +type Scoring struct { + WeightAffinity float64 `toml:"weight_affinity"` // similar to the artists I like + WeightGenre float64 `toml:"weight_genre"` // tags close to the tags I like + WeightPreference float64 `toml:"weight_preference"` // what my listening says about the artist + WeightDiscovery float64 `toml:"weight_discovery"` // bonus for what I have not played +} + // Default returns the configuration used when nothing else is set. func Default() Config { return Config{ @@ -102,6 +113,7 @@ func Default() Config { Auth: Auth{AccessTTLMinutes: 15, RefreshTTLHours: 720}, // 15 minutes, 30 days Listening: Listening{MinListenSeconds: 20}, Taste: Taste{HalfLifeDays: 90}, + Scoring: Scoring{WeightAffinity: 0.40, WeightGenre: 0.25, WeightPreference: 0.25, WeightDiscovery: 0.10}, } } @@ -190,6 +202,24 @@ func (c Config) Validate() error { if c.Taste.HalfLifeDays < 1 || c.Taste.HalfLifeDays > 3650 { errs = append(errs, errors.New("taste.half_life_days must be between 1 and 3650")) } + sum := 0.0 + for _, w := range []struct { + key string + v float64 + }{ + {"scoring.weight_affinity", c.Scoring.WeightAffinity}, + {"scoring.weight_genre", c.Scoring.WeightGenre}, + {"scoring.weight_preference", c.Scoring.WeightPreference}, + {"scoring.weight_discovery", c.Scoring.WeightDiscovery}, + } { + if math.IsNaN(w.v) || w.v < 0 || w.v > 10 { + errs = append(errs, fmt.Errorf("%s must be a number from 0 to 10", w.key)) + } + sum += w.v + } + if sum == 0 { + errs = append(errs, errors.New("scoring.weight_*: at least one weight must be above 0")) + } if c.Library.ScanWorkers < 1 { errs = append(errs, errors.New("library.scan_workers must be at least 1")) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b8fc240..89cadfb 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -134,6 +134,10 @@ func TestValidate(t *testing.T) { {"zero half life", func(c *Config) { c.Taste.HalfLifeDays = 0 }, "taste.half_life_days"}, {"negative half life", func(c *Config) { c.Taste.HalfLifeDays = -1 }, "taste.half_life_days"}, {"absurd half life", func(c *Config) { c.Taste.HalfLifeDays = 3651 }, "taste.half_life_days"}, + {"negative weight", func(c *Config) { c.Scoring.WeightGenre = -0.1 }, "scoring.weight_genre"}, + {"huge weight", func(c *Config) { c.Scoring.WeightAffinity = 11 }, "scoring.weight_affinity"}, + {"all weights zero", func(c *Config) { c.Scoring = Scoring{} }, "scoring.weight"}, + {"one weight is enough", func(c *Config) { c.Scoring = Scoring{WeightPreference: 1} }, ""}, {"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"}, {"bad access ttl", func(c *Config) { c.Auth.AccessTTLMinutes = 0 }, "auth.access_ttl_minutes"}, {"bad refresh ttl", func(c *Config) { c.Auth.RefreshTTLHours = 0 }, "auth.refresh_ttl_hours"}, @@ -197,6 +201,25 @@ func TestTasteSettingIsReadFromTheFile(t *testing.T) { } } +func TestScoringDefaultsAndFile(t *testing.T) { + def := Default().Scoring + if def.WeightAffinity != 0.40 || def.WeightGenre != 0.25 || def.WeightPreference != 0.25 || def.WeightDiscovery != 0.10 { + t.Errorf("default weights = %+v", def) + } + cfg, err := Load(writeFile(t, t.TempDir(), "[scoring]\nweight_affinity = 1\nweight_genre = 0\nweight_preference = 0\nweight_discovery = 0\n")) + if err != nil { + t.Fatal(err) + } + if cfg.Scoring.WeightAffinity != 1 || cfg.Scoring.WeightGenre != 0 { + t.Errorf("weights = %+v", cfg.Scoring) + } + // A weight left out of the file keeps its default. + cfg, err = Load(writeFile(t, t.TempDir(), "[scoring]\nweight_genre = 0.5\n")) + if err != nil || cfg.Scoring.WeightAffinity != 0.40 || cfg.Scoring.WeightGenre != 0.5 { + t.Errorf("weights = %+v err=%v", cfg.Scoring, err) + } +} + // Provider settings moved into the database. An old config must fail with a // hint, never be ignored in silence. func TestLoadExplainsRemovedProviderSettings(t *testing.T) { diff --git a/internal/discover/score.go b/internal/discover/score.go new file mode 100644 index 0000000..31efb86 --- /dev/null +++ b/internal/discover/score.go @@ -0,0 +1,134 @@ +// Package discover scores artists the listener may like. +// +// The score is S = wA·A + wG·G + wP·P + wD·D, each part from 0 to 1: +// +// A affinity: how close the artist is to the artists the listener likes +// G genre: how close its tags are to the tags of those artists +// P preference: what the listening history says about the artist +// D discovery: a bonus for what the listener has not played yet +// +// A part with no data (an artist whose tags are unknown) is left out and the +// others are rescaled, so a missing part never counts as a bad one. The parts are +// returned with the score, so every number can be explained. +package discover + +import ( + "errors" + "math" +) + +// Weights say how much each part counts. Only their proportions matter. +type Weights struct { + Affinity, Genre, Preference, Discovery float64 +} + +// maxWeight bounds one weight; they are only proportions, so large ones mean nothing. +const maxWeight = 10 + +// DefaultWeights are the weights used when the config does not set any. +func DefaultWeights() Weights { + return Weights{Affinity: 0.40, Genre: 0.25, Preference: 0.25, Discovery: 0.10} +} + +// Validate refuses weights that cannot score: negative, not a number, too big, or all zero. +func (w Weights) Validate() error { + sum := 0.0 + for _, v := range []float64{w.Affinity, w.Genre, w.Preference, w.Discovery} { + if math.IsNaN(v) || v < 0 || v > maxWeight { + return errors.New("each weight must be a number from 0 to 10") + } + sum += v + } + if sum == 0 { + return errors.New("at least one weight must be above 0") + } + return nil +} + +// Link says one seed artist considers the candidate similar. Weight is how much the +// listener cares about the seed (1 for a seed they named). Match is the provider's +// similarity from 0 to 1; it is relative to the seed's own list, which is why the +// absolute scale comes from the weight. +type Link struct { + Seed int64 + Weight float64 + Match float64 +} + +func clamp01(v float64) float64 { + switch { + case math.IsNaN(v) || v < 0: + return 0 + case v > 1: + return 1 + default: + return v + } +} + +// Affinity is 1 − Π(1 − weight·match) over the links: each seed that points at the +// candidate adds to the others, and the result stays below 1. +func Affinity(links []Link) float64 { + miss := 1.0 + for _, l := range links { + miss *= 1 - clamp01(l.Weight)*clamp01(l.Match) + } + return clamp01(1 - miss) +} + +// Cosine compares two tag vectors. ok is false when either has nothing to compare +// (no tags, or all weights zero), which is different from "nothing in common". +func Cosine(a, b map[string]float64) (float64, bool) { + var dot, na, nb float64 + for tag, wa := range a { + na += wa * wa + dot += wa * b[tag] + } + for _, wb := range b { + nb += wb * wb + } + if na <= 0 || nb <= 0 { + return 0, false + } + return clamp01(dot / (math.Sqrt(na) * math.Sqrt(nb))), true +} + +// PreferenceScore maps the preference from -1..1 to 0..1. An artist the listener +// never played (known is false) is neutral: 0.5. +func PreferenceScore(pref float64, known bool) float64 { + if !known { + return 0.5 + } + return clamp01((pref + 1) / 2) +} + +// DiscoveryBonus is 1 for what has never been listened to and falls with every listen. +func DiscoveryBonus(listens int) float64 { + if listens < 0 { + listens = 0 + } + return 1 / float64(1+listens) +} + +// Components are the parts of a score. G is nil when it is not known. +type Components struct { + A float64 + G *float64 + P float64 + D float64 +} + +// Combine adds the parts up with the weights, leaving out G when it is nil and +// rescaling the others. It returns 0 when no weight is left to score with. +func Combine(c Components, w Weights) float64 { + sum := w.Affinity*clamp01(c.A) + w.Preference*clamp01(c.P) + w.Discovery*clamp01(c.D) + total := w.Affinity + w.Preference + w.Discovery + if c.G != nil { + sum += w.Genre * clamp01(*c.G) + total += w.Genre + } + if total <= 0 { + return 0 + } + return clamp01(sum / total) +} diff --git a/internal/discover/score_test.go b/internal/discover/score_test.go new file mode 100644 index 0000000..20fc0f5 --- /dev/null +++ b/internal/discover/score_test.go @@ -0,0 +1,152 @@ +package discover + +import ( + "math" + "testing" +) + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-9 } + +func ptr(v float64) *float64 { return &v } + +func TestAffinityOfOneSeedIsItsWeightTimesTheMatch(t *testing.T) { + got := Affinity([]Link{{Seed: 1, Weight: 0.8, Match: 0.5}}) + if !near(got, 0.4) { + t.Errorf("affinity = %v, want 0.4", got) + } +} + +func TestSeveralSeedsReinforceEachOtherAndStayBelowOne(t *testing.T) { + one := Affinity([]Link{{Weight: 0.8, Match: 0.5}}) + two := Affinity([]Link{{Weight: 0.8, Match: 0.5}, {Weight: 0.6, Match: 0.5}}) + // 1 - (1-0.4)(1-0.3) + if !near(two, 1-0.6*0.7) || two <= one { + t.Errorf("two seeds = %v (one seed %v), want %v", two, one, 1-0.6*0.7) + } + many := make([]Link, 100) + for i := range many { + many[i] = Link{Weight: 1, Match: 0.9} + } + if got := Affinity(many); got > 1 || got < 0.999 { + t.Errorf("a hundred strong seeds = %v, want just under 1", got) + } +} + +func TestAffinityIgnoresNonsense(t *testing.T) { + if got := Affinity(nil); got != 0 { + t.Errorf("no links = %v", got) + } + // Out of range numbers are pulled into 0..1: a bad provider number cannot make a score above 1 or below 0. + got := Affinity([]Link{{Weight: 2, Match: 3}, {Weight: -1, Match: 1}, {Weight: 1, Match: math.NaN()}}) + if got < 0 || got > 1 || math.IsNaN(got) { + t.Errorf("affinity = %v, want a number from 0 to 1", got) + } + if !near(got, 1) { // the first link is clamped to 1*1 + t.Errorf("affinity = %v, want 1", got) + } +} + +func TestCosine(t *testing.T) { + a := map[string]float64{"rock": 1, "grunge": 1} + if got, ok := Cosine(a, a); !ok || !near(got, 1) { + t.Errorf("same tags = %v %v, want 1", got, ok) + } + if got, ok := Cosine(a, map[string]float64{"jazz": 1}); !ok || got != 0 { + t.Errorf("nothing in common = %v %v, want 0", got, ok) + } + if got, ok := Cosine(map[string]float64{"rock": 1}, map[string]float64{"rock": 1, "pop": 1}); !ok || !near(got, 1/math.Sqrt2) { + t.Errorf("half in common = %v %v, want %v", got, ok, 1/math.Sqrt2) + } +} + +func TestCosineSaysWhenThereIsNothingToCompare(t *testing.T) { + for name, pair := range map[string][2]map[string]float64{ + "empty left": {nil, {"rock": 1}}, + "empty right": {{"rock": 1}, {}}, + "zero weight": {{"rock": 0}, {"rock": 1}}, + } { + if got, ok := Cosine(pair[0], pair[1]); ok || got != 0 { + t.Errorf("%s: %v %v, want no answer", name, got, ok) + } + } +} + +func TestPreferenceScore(t *testing.T) { + for _, tt := range []struct { + pref float64 + known bool + want float64 + }{{0.8, true, 0.9}, {-1, true, 0}, {0, true, 0.5}, {0.8, false, 0.5}} { + if got := PreferenceScore(tt.pref, tt.known); !near(got, tt.want) { + t.Errorf("PreferenceScore(%v, %v) = %v, want %v", tt.pref, tt.known, got, tt.want) + } + } +} + +func TestDiscoveryBonusFallsWithEveryListen(t *testing.T) { + if DiscoveryBonus(0) != 1 || !near(DiscoveryBonus(1), 0.5) || !near(DiscoveryBonus(3), 0.25) { + t.Errorf("bonus = %v %v %v", DiscoveryBonus(0), DiscoveryBonus(1), DiscoveryBonus(3)) + } + if DiscoveryBonus(-5) != 1 { + t.Errorf("a negative count = %v, want 1", DiscoveryBonus(-5)) + } +} + +func TestCombineWithEverythingKnown(t *testing.T) { + w := Weights{Affinity: 0.40, Genre: 0.25, Preference: 0.25, Discovery: 0.10} + got := Combine(Components{A: 1, G: ptr(0), P: 0.5, D: 1}, w) + if want := 0.40 + 0 + 0.125 + 0.10; !near(got, want) { + t.Errorf("score = %v, want %v", got, want) + } +} + +func TestCombineDropsWhatIsMissingAndRescalesTheRest(t *testing.T) { + w := Weights{Affinity: 0.40, Genre: 0.25, Preference: 0.25, Discovery: 0.10} + got := Combine(Components{A: 1, G: nil, P: 1, D: 1}, w) + // Without G everything is 1, so the score must be 1, not 0.75. + if !near(got, 1) { + t.Errorf("score = %v, want 1", got) + } + got = Combine(Components{A: 0.5, G: nil, P: 0.5, D: 0.5}, w) + if !near(got, 0.5) { + t.Errorf("score = %v, want 0.5", got) + } +} + +func TestCombineStaysInRangeWhateverComesIn(t *testing.T) { + w := Weights{Affinity: 0.40, Genre: 0.25, Preference: 0.25, Discovery: 0.10} + got := Combine(Components{A: 7, G: ptr(-3), P: math.NaN(), D: 2}, w) + if got < 0 || got > 1 || math.IsNaN(got) { + t.Errorf("score = %v, want a number from 0 to 1", got) + } +} + +func TestCombineWithNoWeightsIsZero(t *testing.T) { + if got := Combine(Components{A: 1, P: 1, D: 1}, Weights{}); got != 0 { + t.Errorf("score = %v, want 0", got) + } + // Only G has weight, and G is missing: nothing is left to score with. + if got := Combine(Components{A: 1, P: 1, D: 1}, Weights{Genre: 1}); got != 0 { + t.Errorf("score = %v, want 0", got) + } +} + +func TestWeightsValidation(t *testing.T) { + good := Weights{Affinity: 0.4, Genre: 0.25, Preference: 0.25, Discovery: 0.1} + if err := good.Validate(); err != nil { + t.Errorf("default-like weights refused: %v", err) + } + for name, w := range map[string]Weights{ + "all zero": {}, + "negative": {Affinity: -0.1, Genre: 1}, + "nan": {Affinity: math.NaN(), Genre: 1}, + "too big": {Affinity: 11, Genre: 1}, + } { + if err := w.Validate(); err == nil { + t.Errorf("%s: accepted", name) + } + } + if d := DefaultWeights(); d != good { + t.Errorf("default weights = %+v, want %+v", d, good) + } +} diff --git a/internal/discover/service.go b/internal/discover/service.go new file mode 100644 index 0000000..dad5b8e --- /dev/null +++ b/internal/discover/service.go @@ -0,0 +1,461 @@ +package discover + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/taste" + "codeberg.org/squidd/sonda/internal/usersettings" +) + +// Limits. +const ( + DefaultLimit = 20 + MaxLimit = 30 + + // MaxSeeds is how many artists a listener may name. + MaxSeeds = 5 + // MaxListeningSeeds is how many of the most played artists seed a request when + // the listener allowed it. + MaxListeningSeeds = 20 + // Parallelism is how many provider requests are in flight at once. The provider + // client spaces them out anyway; this only keeps a request from queueing dozens. + Parallelism = 4 + // minPool is how many candidates, best affinity first, are scored whatever the + // limit: the genre can reorder them, and each one without tags costs a request. + minPool = 20 + // MaxBecause is how many seeds are named as the reason for a suggestion. + MaxBecause = 3 +) + +// Errors returned by the service. +var ( + ErrInvalidQuery = errors.New("invalid query") + // ErrSeedRequired is returned when no seed was named and the listener did not + // allow Discover to start from their listening. + ErrSeedRequired = errors.New("name a seed artist, or allow Discover to start from your listening") +) + +// Taster says what the listener likes. +type Taster interface { + Learn(ctx context.Context, userID int64) (taste.Learned, error) +} + +// Similar gives the artists similar to one. +type Similar interface { + Similar(ctx context.Context, artistID int64, limit int) (discovery.Result, error) +} + +// TagFetcher reads the tags of one artist from one provider into the database. +type TagFetcher interface { + ProviderTags(ctx context.Context, artistID int64, provider string) (discovery.TagSourceResult, error) +} + +// Store reads what is already stored. +type Store interface { + // ArtistNames returns the names of the artists that exist. + ArtistNames(ctx context.Context, ids []int64) (map[int64]string, error) + // ArtistTags returns the stored tags per artist with weights from 0 to 1; + // artists without tags are left out. + ArtistTags(ctx context.Context, ids []int64) (map[int64]map[string]float64, error) +} + +// Settings reads the listener's own settings. +type Settings interface { + Get(ctx context.Context, userID int64) (usersettings.Settings, error) +} + +// Query is a request for suggestions. +type Query struct { + Seeds []int64 // artists the listener names; none means "from my listening" + Limit int // zero means DefaultLimit + InLibrary *bool // keep only artists the listener has (true) or has not (false) +} + +// Seed is an artist the suggestions start from. +type Seed struct { + ID int64 + Name string + Weight float64 // 1 for a named seed, the preference for one taken from the listening +} + +// Reason says a seed led to a suggestion, and how strongly the provider links them. +type Reason struct { + Artist discovery.ArtistRef + Match float64 +} + +// Item is one suggestion with the parts of its score. +type Item struct { + Artist discovery.ArtistRef + InLibrary bool + Score float64 + Components Components + Because []Reason +} + +// Result is the answer to Discover. +type Result struct { + // ProviderStatus is "disabled" when the provider is off or has no key (what was + // stored earlier is still used), "error" when it failed for some seed, else "ok". + ProviderStatus string + FromListening bool // the seeds were taken from the listening, not named + Seeds []Seed + Items []Item +} + +// Service scores suggestions. +type Service struct { + taste Taster + similar Similar + tags TagFetcher + store Store + settings Settings + weights Weights +} + +// NewService builds a Service. +func NewService(t Taster, sim Similar, tags TagFetcher, store Store, settings Settings, w Weights) *Service { + return &Service{taste: t, similar: sim, tags: tags, store: store, settings: settings, weights: w} +} + +// Discover suggests artists. Seeds are the artists the listener names, or, when none +// is named and the listener allowed it, the ones they play most. For each seed the +// stored similar artists are used, and asked from the provider only when the stored +// ones are missing or old (see discovery.Service). The tags of the best candidates +// that have none are read from Last.fm only. +func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, error) { + limit, err := checkLimit(q.Limit) + if err != nil { + return Result{}, err + } + named, err := s.checkSeeds(ctx, q.Seeds) + if err != nil { + return Result{}, err + } + + learned, err := s.taste.Learn(ctx, userID) + if err != nil { + return Result{}, err + } + + res := Result{ProviderStatus: discovery.StatusOK, Seeds: []Seed{}, Items: []Item{}} + seeds := named + if len(named) == 0 { + set, err := s.settings.Get(ctx, userID) + if err != nil { + return Result{}, err + } + if !set.DiscoverFromListening { + return Result{}, ErrSeedRequired + } + res.FromListening = true + for i, p := range learned.Liked { + if i == MaxListeningSeeds { + break + } + seeds = append(seeds, Seed{ID: p.ArtistID, Weight: p.Preference}) + } + if err := s.nameSeeds(ctx, seeds); err != nil { + return Result{}, err + } + } + res.Seeds = append(res.Seeds, seeds...) + if len(seeds) == 0 { + return res, nil + } + + lists, status, err := s.readSimilar(ctx, seeds) + if err != nil { + return Result{}, err + } + res.ProviderStatus = status + + cands := gather(seeds, lists, q.InLibrary) + pool := rankByAffinity(cands, max(limit, minPool)) + tags, err := s.candidateTags(ctx, pool, len(learned.Tags) > 0) + if err != nil { + return Result{}, err + } + profile := make(map[string]float64, len(learned.Tags)) + for _, t := range learned.Tags { + profile[t.Tag] = t.Weight + } + + names := map[int64]string{} + for _, sd := range seeds { + names[sd.ID] = sd.Name + } + for _, c := range pool { + pref, known := learned.Prefs[c.id] + comp := Components{A: Affinity(c.links), P: PreferenceScore(pref.Preference, known), D: DiscoveryBonus(pref.Listens)} + if g, ok := Cosine(tags[c.id], profile); ok { + comp.G = &g + } + res.Items = append(res.Items, Item{ + Artist: discovery.ArtistRef{ID: c.id, Name: c.name}, InLibrary: c.inLibrary, + Score: Combine(comp, s.weights), Components: comp, Because: because(c.links, names), + }) + } + sort.SliceStable(res.Items, func(i, j int) bool { + a, b := res.Items[i], res.Items[j] + switch { + case a.Score != b.Score: + return a.Score > b.Score + case a.Components.A != b.Components.A: + return a.Components.A > b.Components.A + default: + return a.Artist.ID < b.Artist.ID + } + }) + if len(res.Items) > limit { + res.Items = res.Items[:limit] + } + return res, nil +} + +func checkLimit(limit int) (int, error) { + switch { + case limit < 0: + return 0, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery) + case limit == 0: + return DefaultLimit, nil + case limit > MaxLimit: + return MaxLimit, nil + } + return limit, nil +} + +// checkSeeds refuses too many seeds and ids that are not artists, and drops repeats. +func (s *Service) checkSeeds(ctx context.Context, in []int64) ([]Seed, error) { + if len(in) > MaxSeeds { + return nil, fmt.Errorf("%w: at most %d seeds", ErrInvalidQuery, MaxSeeds) + } + var out []Seed + seen := map[int64]bool{} + for _, id := range in { + if id <= 0 { + return nil, fmt.Errorf("%w: %d is not an artist id", ErrInvalidQuery, id) + } + if !seen[id] { + seen[id] = true + out = append(out, Seed{ID: id, Weight: 1}) + } + } + if len(out) == 0 { + return nil, nil + } + if err := s.nameSeeds(ctx, out); err != nil { + return nil, err + } + for _, sd := range out { + if sd.Name == "" { + return nil, fmt.Errorf("%w: %d is not an artist", ErrInvalidQuery, sd.ID) + } + } + return out, nil +} + +func (s *Service) nameSeeds(ctx context.Context, seeds []Seed) error { + ids := make([]int64, len(seeds)) + for i, sd := range seeds { + ids[i] = sd.ID + } + names, err := s.store.ArtistNames(ctx, ids) + if err != nil { + return fmt.Errorf("read names: %w", err) + } + for i := range seeds { + seeds[i].Name = names[seeds[i].ID] + } + return nil +} + +// readSimilar reads the similar artists of every seed, Parallelism at a time. A seed +// that was removed in the meantime is skipped; any other error stops the request. +func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]discovery.Result, string, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + results := make([]discovery.Result, len(seeds)) + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + gate = make(chan struct{}, Parallelism) + ) + for i, sd := range seeds { + wg.Add(1) + go func() { + defer wg.Done() + select { + case gate <- struct{}{}: + defer func() { <-gate }() + case <-ctx.Done(): + return + } + r, err := s.similar.Similar(ctx, sd.ID, discovery.FetchLimit) + if errors.Is(err, discovery.ErrNotFound) { + return + } + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + cancel() + } + mu.Unlock() + return + } + results[i] = r + }() + } + wg.Wait() + if firstErr != nil { + return nil, "", firstErr + } + if err := ctx.Err(); err != nil { + return nil, "", err + } + status := discovery.StatusOK + for _, r := range results { + switch r.ProviderStatus { + case discovery.StatusDisabled: + status = discovery.StatusDisabled + case discovery.StatusError: + if status != discovery.StatusDisabled { + status = discovery.StatusError + } + } + } + return results, status, nil +} + +type candidate struct { + id int64 + name string + inLibrary bool + links []Link +} + +// gather turns the similar lists into candidates. A seed is never suggested, and +// the library filter is applied here. +func gather(seeds []Seed, lists []discovery.Result, inLibrary *bool) []*candidate { + isSeed := map[int64]bool{} + for _, sd := range seeds { + isSeed[sd.ID] = true + } + byID := map[int64]*candidate{} + var order []*candidate + for i, sd := range seeds { + for _, it := range lists[i].Items { + if isSeed[it.Artist.ID] || (inLibrary != nil && it.InLibrary != *inLibrary) { + continue + } + c := byID[it.Artist.ID] + if c == nil { + c = &candidate{id: it.Artist.ID, name: it.Artist.Name, inLibrary: it.InLibrary} + byID[c.id] = c + order = append(order, c) + } + c.links = append(c.links, Link{Seed: sd.ID, Weight: sd.Weight, Match: it.Score}) + } + } + return order +} + +// rankByAffinity keeps the n candidates with the best affinity (ties by id). +func rankByAffinity(cands []*candidate, n int) []*candidate { + aff := make(map[int64]float64, len(cands)) + for _, c := range cands { + aff[c.id] = Affinity(c.links) + } + sort.SliceStable(cands, func(i, j int) bool { + if aff[cands[i].id] != aff[cands[j].id] { + return aff[cands[i].id] > aff[cands[j].id] + } + return cands[i].id < cands[j].id + }) + if len(cands) > n { + cands = cands[:n] + } + return cands +} + +// candidateTags returns the stored tags of the pool. A candidate with none gets its +// Last.fm tags read first (only when there is a profile to compare them with), and +// a provider that is off or fails just leaves its genre unknown. +func (s *Service) candidateTags(ctx context.Context, pool []*candidate, haveProfile bool) (map[int64]map[string]float64, error) { + ids := make([]int64, len(pool)) + for i, c := range pool { + ids[i] = c.id + } + tags, err := s.store.ArtistTags(ctx, ids) + if err != nil { + return nil, fmt.Errorf("read tags: %w", err) + } + if !haveProfile { + return tags, nil + } + var missing []int64 + for _, id := range ids { + if len(tags[id]) == 0 { + missing = append(missing, id) + } + } + if len(missing) == 0 { + return tags, nil + } + + var wg sync.WaitGroup + gate := make(chan struct{}, Parallelism) + for _, id := range missing { + wg.Add(1) + go func() { + defer wg.Done() + select { + case gate <- struct{}{}: + defer func() { <-gate }() + case <-ctx.Done(): + return + } + _, _ = s.tags.ProviderTags(ctx, id, providers.LastFM) // a failure leaves the genre out + }() + } + wg.Wait() + if err := ctx.Err(); err != nil { + return nil, err + } + again, err := s.store.ArtistTags(ctx, missing) + if err != nil { + return nil, fmt.Errorf("read tags: %w", err) + } + for id, t := range again { + tags[id] = t + } + return tags, nil +} + +// because names the seeds that count most for a candidate, strongest first. +func because(links []Link, names map[int64]string) []Reason { + sorted := append([]Link(nil), links...) + sort.SliceStable(sorted, func(i, j int) bool { + a, b := sorted[i].Weight*sorted[i].Match, sorted[j].Weight*sorted[j].Match + if a != b { + return a > b + } + return sorted[i].Seed < sorted[j].Seed + }) + if len(sorted) > MaxBecause { + sorted = sorted[:MaxBecause] + } + out := make([]Reason, len(sorted)) + for i, l := range sorted { + out[i] = Reason{Artist: discovery.ArtistRef{ID: l.Seed, Name: names[l.Seed]}, Match: l.Match} + } + return out +} diff --git a/internal/discover/service_test.go b/internal/discover/service_test.go new file mode 100644 index 0000000..7bcc8ac --- /dev/null +++ b/internal/discover/service_test.go @@ -0,0 +1,607 @@ +package discover + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/taste" + "codeberg.org/squidd/sonda/internal/usersettings" +) + +type fakeTaste struct { + learned taste.Learned + err error + user int64 +} + +func (f *fakeTaste) Learn(_ context.Context, userID int64) (taste.Learned, error) { + f.user = userID + return f.learned, f.err +} + +type fakeSimilar struct { + mu sync.Mutex + lists map[int64]discovery.Result + errs map[int64]error + calls []int64 + inFlight int32 + maxSeen int32 + delay time.Duration +} + +func (f *fakeSimilar) Similar(ctx context.Context, id int64, limit int) (discovery.Result, error) { + n := atomic.AddInt32(&f.inFlight, 1) + defer atomic.AddInt32(&f.inFlight, -1) + for { + m := atomic.LoadInt32(&f.maxSeen) + if n <= m || atomic.CompareAndSwapInt32(&f.maxSeen, m, n) { + break + } + } + f.mu.Lock() + f.calls = append(f.calls, id) + res, err := f.lists[id], f.errs[id] + f.mu.Unlock() + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return discovery.Result{}, ctx.Err() + } + } + if res.ProviderStatus == "" { + res.ProviderStatus = discovery.StatusOK + } + return res, err +} + +func (f *fakeSimilar) called() []int64 { + f.mu.Lock() + defer f.mu.Unlock() + return append([]int64(nil), f.calls...) +} + +// fakeStore holds names and stored tags; the tag fetcher writes into it, like the +// real one saves into the database. +type fakeStore struct { + mu sync.Mutex + names map[int64]string + tags map[int64]map[string]float64 +} + +func (f *fakeStore) ArtistNames(_ context.Context, ids []int64) (map[int64]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + out := map[int64]string{} + for _, id := range ids { + if n, ok := f.names[id]; ok { + out[id] = n + } + } + return out, nil +} + +func (f *fakeStore) ArtistTags(_ context.Context, ids []int64) (map[int64]map[string]float64, error) { + f.mu.Lock() + defer f.mu.Unlock() + out := map[int64]map[string]float64{} + for _, id := range ids { + if t := f.tags[id]; len(t) > 0 { + out[id] = t + } + } + return out, nil +} + +type fakeFetcher struct { + store *fakeStore + gives map[int64]map[string]float64 // what Last.fm would say, by artist + status string + err error + mu sync.Mutex + calls []int64 + prov []string +} + +func (f *fakeFetcher) ProviderTags(_ context.Context, id int64, provider string) (discovery.TagSourceResult, error) { + f.mu.Lock() + f.calls = append(f.calls, id) + f.prov = append(f.prov, provider) + f.mu.Unlock() + if f.err != nil { + return discovery.TagSourceResult{}, f.err + } + if t := f.gives[id]; len(t) > 0 { + f.store.mu.Lock() + f.store.tags[id] = t + f.store.mu.Unlock() + } + st := f.status + if st == "" { + st = discovery.StatusOK + } + return discovery.TagSourceResult{Provider: provider, Status: st}, nil +} + +func (f *fakeFetcher) called() []int64 { + f.mu.Lock() + defer f.mu.Unlock() + return append([]int64(nil), f.calls...) +} + +type fakeSettings struct { + on bool + err error +} + +func (f fakeSettings) Get(context.Context, int64) (usersettings.Settings, error) { + return usersettings.Settings{DiscoverFromListening: f.on}, f.err +} + +type rig struct { + svc *Service + taste *fakeTaste + similar *fakeSimilar + store *fakeStore + fetch *fakeFetcher + set *fakeSettings +} + +func item(id int64, name string, rank int, score float64, inLib bool) discovery.Item { + return discovery.Item{Artist: discovery.ArtistRef{ID: id, Name: name}, InLibrary: inLib, Score: score, Rank: rank} +} + +// newRig has seeds 1 (Seed One) and 2 (Seed Two), candidates 10 to 13, and a +// listener who likes "rock". Artist 1 is liked (0.8), 2 is not heard at all. +func newRig() *rig { + r := &rig{ + taste: &fakeTaste{learned: taste.Learned{ + Prefs: map[int64]taste.ArtistPref{1: {ArtistID: 1, Preference: 0.8, Score: 2, Listens: 5}, 12: {ArtistID: 12, Preference: 0.5, Score: 1, Listens: 3}}, + Liked: []taste.ArtistPref{{ArtistID: 1, Preference: 0.8, Score: 2, Listens: 5}, {ArtistID: 12, Preference: 0.5, Score: 1, Listens: 3}}, + Tags: []taste.TagWeight{{Tag: "rock", Weight: 1}}, + }}, + similar: &fakeSimilar{lists: map[int64]discovery.Result{ + 1: {Items: []discovery.Item{item(10, "Ten", 1, 1, false), item(11, "Eleven", 2, 0.5, false), item(12, "Twelve", 3, 0.4, true), item(2, "Seed Two", 4, 0.3, true)}}, + 2: {Items: []discovery.Item{item(10, "Ten", 1, 1, false), item(13, "Thirteen", 2, 0.8, false)}}, + }, errs: map[int64]error{}}, + store: &fakeStore{ + names: map[int64]string{1: "Seed One", 2: "Seed Two", 10: "Ten", 11: "Eleven", 12: "Twelve", 13: "Thirteen"}, + tags: map[int64]map[string]float64{10: {"rock": 1}, 11: {"jazz": 1}}, + }, + set: &fakeSettings{}, + } + r.fetch = &fakeFetcher{store: r.store, gives: map[int64]map[string]float64{13: {"rock": 0.5, "pop": 1}}} + r.svc = NewService(r.taste, r.similar, r.fetch, r.store, r.set, DefaultWeights()) + return r +} + +func ids(items []Item) []int64 { + var out []int64 + for _, it := range items { + out = append(out, it.Artist.ID) + } + return out +} + +func find(t *testing.T, res Result, id int64) Item { + t.Helper() + for _, it := range res.Items { + if it.Artist.ID == id { + return it + } + } + t.Fatalf("artist %d is not in the answer: %v", id, ids(res.Items)) + return Item{} +} + +func TestDiscoverFromNamedSeedsNeedsNoSetting(t *testing.T) { + r := newRig() // the setting is off + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}) + if err != nil { + t.Fatal(err) + } + if r.taste.user != 7 { + t.Errorf("learned user %d", r.taste.user) + } + if res.FromListening || len(res.Seeds) != 1 || res.Seeds[0].ID != 1 || res.Seeds[0].Name != "Seed One" || res.Seeds[0].Weight != 1 { + t.Errorf("seeds = %+v from listening %v, want Seed One at weight 1", res.Seeds, res.FromListening) + } + if got := r.similar.called(); len(got) != 1 || got[0] != 1 { + t.Errorf("similar lists asked for %v, want only the seed", got) + } + if res.ProviderStatus != discovery.StatusOK { + t.Errorf("status = %q", res.ProviderStatus) + } + // The seed itself is never a suggestion, and Seed Two (similar to One) is a candidate when it is not a seed. + for _, it := range res.Items { + if it.Artist.ID == 1 { + t.Error("a seed was suggested to itself") + } + } + if len(res.Items) != 4 { + t.Errorf("items = %v, want 10, 11, 12 and 2", ids(res.Items)) + } +} + +func TestDiscoverWithoutSeedsAndTheSettingOffIsRefusedAndAsksNobody(t *testing.T) { + r := newRig() + _, err := r.svc.Discover(context.Background(), 7, Query{}) + if !errors.Is(err, ErrSeedRequired) { + t.Fatalf("err = %v, want ErrSeedRequired", err) + } + if len(r.similar.called())+len(r.fetch.called()) != 0 { + t.Error("a provider was asked although the listener did not allow it") + } +} + +func TestDiscoverFromListeningUsesTheArtistsILikeWeightedByHowMuch(t *testing.T) { + r := newRig() + r.set.on = true + res, err := r.svc.Discover(context.Background(), 7, Query{}) + if err != nil { + t.Fatal(err) + } + if !res.FromListening || len(res.Seeds) != 2 || res.Seeds[0].ID != 1 || res.Seeds[0].Weight != 0.8 || res.Seeds[1].ID != 12 || res.Seeds[1].Weight != 0.5 { + t.Errorf("seeds = %+v", res.Seeds) + } + if res.Seeds[0].Name != "Seed One" || res.Seeds[1].Name != "Twelve" { + t.Errorf("seed names = %+v", res.Seeds) + } + // Artist 10 is similar to seed 1 with match 1: A = 0.8 * 1. + if got := find(t, res, 10).Components.A; !near(got, 0.8) { + t.Errorf("A of artist 10 = %v, want 0.8", got) + } + // 12 is a seed, so it is not a candidate. + for _, it := range res.Items { + if it.Artist.ID == 12 || it.Artist.ID == 1 { + t.Errorf("seed %d came back as a candidate", it.Artist.ID) + } + } +} + +func TestDiscoverFromListeningWithNothingListenedToIsEmptyAndAsksNobody(t *testing.T) { + r := newRig() + r.set.on = true + r.taste.learned = taste.Learned{Prefs: map[int64]taste.ArtistPref{}, Liked: []taste.ArtistPref{}, Tags: []taste.TagWeight{}} + res, err := r.svc.Discover(context.Background(), 7, Query{}) + if err != nil || len(res.Items) != 0 || len(res.Seeds) != 0 || !res.FromListening { + t.Errorf("result = %+v err=%v", res, err) + } + if res.Items == nil || res.Seeds == nil { + t.Error("lists must be empty, not nil") + } + if len(r.similar.called()) != 0 { + t.Error("a provider was asked about nothing") + } +} + +func TestNamedSeedsWinOverTheListening(t *testing.T) { + r := newRig() + r.set.on = true + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{2}}) + if err != nil || res.FromListening || len(res.Seeds) != 1 || res.Seeds[0].ID != 2 { + t.Errorf("result = %+v err=%v", res, err) + } +} + +func TestDiscoverUsesAtMostTwentySeedsFromListening(t *testing.T) { + r := newRig() + r.set.on = true + var liked []taste.ArtistPref + for id := int64(100); id < 140; id++ { + liked = append(liked, taste.ArtistPref{ArtistID: id, Preference: 0.5, Score: 1}) + r.store.names[id] = "x" + } + r.taste.learned.Liked = liked + res, err := r.svc.Discover(context.Background(), 7, Query{}) + if err != nil || len(res.Seeds) != MaxListeningSeeds || len(r.similar.called()) != MaxListeningSeeds { + t.Errorf("seeds %d, similar calls %d, want %d each (err %v)", len(res.Seeds), len(r.similar.called()), MaxListeningSeeds, err) + } +} + +func TestSeveralSeedsReinforceACandidateAndSayWhy(t *testing.T) { + r := newRig() + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + ten := find(t, res, 10) + // Both seeds weigh 1 here and both match 1: A = 1. + if !near(ten.Components.A, 1) { + t.Errorf("A = %v, want 1", ten.Components.A) + } + if len(ten.Because) != 2 || ten.Because[0].Match != 1 { + t.Errorf("because = %+v", ten.Because) + } + if got := find(t, res, 11).Because; len(got) != 1 || got[0].Artist.ID != 1 || got[0].Artist.Name != "Seed One" || got[0].Match != 0.5 { + t.Errorf("because for 11 = %+v", got) + } + // Seed One and Seed Two are seeds: neither is suggested. + for _, it := range res.Items { + if it.Artist.ID == 1 || it.Artist.ID == 2 { + t.Errorf("seed %d suggested", it.Artist.ID) + } + } +} + +func TestBecauseKeepsTheThreeStrongestSeeds(t *testing.T) { + r := newRig() + r.similar.lists = map[int64]discovery.Result{} + var seeds []int64 + for i := int64(1); i <= 5; i++ { + seeds = append(seeds, i) + r.store.names[i] = "s" + r.similar.lists[i] = discovery.Result{Items: []discovery.Item{item(10, "Ten", 1, float64(i)/10, false)}} + } + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: seeds}) + if err != nil { + t.Fatal(err) + } + got := find(t, res, 10).Because + if len(got) != 3 || got[0].Match != 0.5 || got[1].Match != 0.4 || got[2].Match != 0.3 { + t.Errorf("because = %+v, want the matches 0.5, 0.4, 0.3", got) + } +} + +func TestPreferenceAndDiscoveryComeFromTheListening(t *testing.T) { + r := newRig() + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}) + if err != nil { + t.Fatal(err) + } + // Artist 12 was played 3 times with preference 0.5; 11 never. + twelve, eleven := find(t, res, 12), find(t, res, 11) + if !near(twelve.Components.P, 0.75) || !near(twelve.Components.D, 0.25) { + t.Errorf("12: P=%v D=%v, want 0.75 and 0.25", twelve.Components.P, twelve.Components.D) + } + if !near(eleven.Components.P, 0.5) || !near(eleven.Components.D, 1) { + t.Errorf("11: P=%v D=%v, want 0.5 and 1", eleven.Components.P, eleven.Components.D) + } +} + +func TestGenreComesFromTheTagsOfTheCandidate(t *testing.T) { + r := newRig() + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + if g := find(t, res, 10).Components.G; g == nil || !near(*g, 1) { + t.Errorf("G of 10 (rock) = %v, want 1", g) + } + if g := find(t, res, 11).Components.G; g == nil || *g != 0 { + t.Errorf("G of 11 (jazz) = %v, want 0", g) + } +} + +func TestTagsAreFetchedOnlyForCandidatesWithNone_AndOnlyFromLastfm(t *testing.T) { + r := newRig() + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + // 10 and 11 have stored tags; 12 (in library, none stored), 13 (none stored) get asked. + got := map[int64]bool{} + for _, id := range r.fetch.called() { + got[id] = true + } + if got[10] || got[11] || !got[13] { + t.Errorf("fetched for %v, want only artists without tags", got) + } + for _, p := range r.fetch.prov { + if p != providers.LastFM { + t.Errorf("asked %q, want only Last.fm", p) + } + } + // What was fetched is used. + if g := find(t, res, 13).Components.G; g == nil || *g <= 0 { + t.Errorf("G of 13 = %v, want a value from the tags just read", g) + } +} + +func TestAnArtistWithoutAnyTagsHasNoGenreAndIsScoredWithoutIt(t *testing.T) { + r := newRig() + r.fetch.gives = nil // Last.fm knows nothing about 12 and 13 + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + thirteen := find(t, res, 13) + if thirteen.Components.G != nil { + t.Errorf("G = %v, want nil", *thirteen.Components.G) + } + want := Combine(thirteen.Components, DefaultWeights()) + if !near(thirteen.Score, want) { + t.Errorf("score = %v, want %v (G left out)", thirteen.Score, want) + } +} + +func TestWithoutAListenerProfileThereIsNoGenre(t *testing.T) { + r := newRig() + r.taste.learned.Tags = nil + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}) + if err != nil { + t.Fatal(err) + } + for _, it := range res.Items { + if it.Components.G != nil { + t.Errorf("artist %d has a genre score without a profile to compare with", it.Artist.ID) + } + } + if len(r.fetch.called()) != 0 { + t.Error("tags were fetched although there is no profile to compare them with") + } +} + +func TestAFailingTagFetchLeavesTheGenreOutAndNothingBreaks(t *testing.T) { + r := newRig() + r.fetch.err = errors.New("boom") + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatalf("err = %v, want the answer without those tags", err) + } + if find(t, res, 13).Components.G != nil { + t.Error("G of 13 should be unknown") + } +} + +func TestScoreOrdersTheAnswerAndTiesGoToAffinityThenId(t *testing.T) { + r := newRig() + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + for i := 1; i < len(res.Items); i++ { + if res.Items[i-1].Score < res.Items[i].Score { + t.Errorf("not ordered by score: %v then %v", res.Items[i-1].Score, res.Items[i].Score) + } + } + // 10 is similar to both seeds with match 1 and likes rock: it must lead. + if res.Items[0].Artist.ID != 10 { + t.Errorf("first = %d, want 10", res.Items[0].Artist.ID) + } + for _, it := range res.Items { + if it.Score < 0 || it.Score > 1 { + t.Errorf("score %v out of range", it.Score) + } + } +} + +func TestLimitsAndTheLibraryFilter(t *testing.T) { + r := newRig() + // Only in-library candidates of seed 1: 12 (and Seed Two, id 2). + yes, no := true, false + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, InLibrary: &yes}) + if err != nil || len(res.Items) != 2 { + t.Fatalf("in library: %v err=%v", ids(res.Items), err) + } + for _, it := range res.Items { + if !it.InLibrary { + t.Errorf("artist %d is not in the library", it.Artist.ID) + } + } + if res, err = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, InLibrary: &no}); err != nil || len(res.Items) != 2 { + t.Fatalf("outside: %v err=%v", ids(res.Items), err) + } + if res, err = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, Limit: 1}); err != nil || len(res.Items) != 1 { + t.Fatalf("limit 1: %v err=%v", ids(res.Items), err) + } + if _, err = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, Limit: -1}); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("negative limit: err = %v", err) + } + if res, err = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, Limit: 1000}); err != nil || len(res.Items) > MaxLimit { + t.Errorf("large limit: %d items err=%v", len(res.Items), err) + } +} + +func TestSeedsMustBeFewDistinctAndKnown(t *testing.T) { + r := newRig() + for name, q := range map[string]Query{ + "too many": {Seeds: []int64{1, 2, 3, 4, 5, 6}}, + "unknown": {Seeds: []int64{999}}, + "zero": {Seeds: []int64{0}}, + "negative": {Seeds: []int64{-1}}, + "one unknown": {Seeds: []int64{1, 999}}, + } { + if _, err := r.svc.Discover(context.Background(), 7, q); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("%s: err = %v, want ErrInvalidQuery", name, err) + } + } + if len(r.similar.called()) != 0 { + t.Error("a provider was asked although the request was refused") + } + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 1, 1}}) + if err != nil || len(res.Seeds) != 1 || len(r.similar.called()) != 1 { + t.Errorf("the same seed three times: %d seeds, %d calls, err %v", len(res.Seeds), len(r.similar.called()), err) + } +} + +func TestProviderStatusSaysWhatHappened(t *testing.T) { + r := newRig() + r.similar.lists[1] = discovery.Result{ProviderStatus: discovery.StatusDisabled, Items: r.similar.lists[1].Items} + res, _ := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}) + if res.ProviderStatus != discovery.StatusDisabled || len(res.Items) == 0 { + t.Errorf("status %q with %d items: stored data must still be used", res.ProviderStatus, len(res.Items)) + } + + r = newRig() + r.similar.lists[2] = discovery.Result{ProviderStatus: discovery.StatusError, Items: r.similar.lists[2].Items} + if res, _ = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}); res.ProviderStatus != discovery.StatusError { + t.Errorf("status = %q, want error when any seed failed", res.ProviderStatus) + } + + r = newRig() + r.similar.lists[1] = discovery.Result{ProviderStatus: discovery.StatusDisabled, Items: r.similar.lists[1].Items} + r.similar.lists[2] = discovery.Result{ProviderStatus: discovery.StatusError, Items: r.similar.lists[2].Items} + if res, _ = r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}); res.ProviderStatus != discovery.StatusDisabled { + t.Errorf("status = %q, want disabled to win over error", res.ProviderStatus) + } +} + +func TestOneSeedThatCannotBeReadDoesNotSpoilTheOthers(t *testing.T) { + r := newRig() + r.similar.errs[1] = discovery.ErrNotFound // removed in the meantime + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + if len(res.Items) == 0 || find(t, res, 13).Artist.ID != 13 { + t.Errorf("items = %v", ids(res.Items)) + } + // Other errors are real. + r = newRig() + boom := errors.New("boom") + r.similar.errs[1] = boom + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}); !errors.Is(err, boom) { + t.Errorf("err = %v, want the store error", err) + } +} + +func TestProvidersAreAskedFourAtATime(t *testing.T) { + r := newRig() + r.similar.delay = 20 * time.Millisecond + r.set.on = true + var liked []taste.ArtistPref + for id := int64(100); id < 112; id++ { + liked = append(liked, taste.ArtistPref{ArtistID: id, Preference: 0.5, Score: 1}) + r.store.names[id] = "x" + } + r.taste.learned.Liked = liked + if _, err := r.svc.Discover(context.Background(), 7, Query{}); err != nil { + t.Fatal(err) + } + if m := atomic.LoadInt32(&r.similar.maxSeen); m < 2 || m > Parallelism { + t.Errorf("at most %d calls at once, saw %d (want several, up to %d)", Parallelism, m, Parallelism) + } +} + +func TestAListenerWhoLeavesStopsTheWork(t *testing.T) { + r := newRig() + r.similar.delay = time.Second + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + start := time.Now() + _, err := r.svc.Discover(ctx, 7, Query{Seeds: []int64{1, 2}}) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 500*time.Millisecond { + t.Errorf("err = %v after %v, want a prompt deadline error", err, time.Since(start)) + } +} + +func TestSettingsAndTasteErrorsPass(t *testing.T) { + boom := errors.New("boom") + r := newRig() + r.taste.err = boom + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); !errors.Is(err, boom) { + t.Errorf("taste: err = %v", err) + } + r = newRig() + r.set.err = boom + if _, err := r.svc.Discover(context.Background(), 7, Query{}); !errors.Is(err, boom) { + t.Errorf("settings: err = %v", err) + } +} diff --git a/internal/discovery/tags.go b/internal/discovery/tags.go index b74afcf..55c7281 100644 --- a/internal/discovery/tags.go +++ b/internal/discovery/tags.go @@ -39,6 +39,9 @@ const StatusNoID = "no_id" // the artist. var ErrSourceNotFound = errors.New("the provider does not know this artist") +// ErrUnknownProvider is returned by ProviderTags for a name that is not a tag provider. +var ErrUnknownProvider = errors.New("unknown tag provider") + // Tag is one tag of an artist. Weight is the provider's own number, kept as it // came: Last.fm's is relative to the artist (0 to 100), MusicBrainz's is an // absolute count of votes, so weights of different providers are never compared. @@ -157,6 +160,30 @@ func (s *TagService) Tags(ctx context.Context, artistID int64) (TagResult, error return TagResult{Sources: out}, nil } +// ProviderTags returns what one provider says about an artist, with the same rules +// as Tags (stored answer first, asked only when it is on and the answer is missing or +// old) but without asking the others. ErrUnknownProvider is returned for a name that +// is not a provider (the files are not one). +func (s *TagService) ProviderTags(ctx context.Context, artistID int64, provider string) (TagSourceResult, error) { + var p *tagProvider + for i := range tagProviders { + if tagProviders[i].name == provider { + p = &tagProviders[i] + } + } + if p == nil { + return TagSourceResult{}, ErrUnknownProvider + } + if artistID <= 0 { + return TagSourceResult{}, ErrNotFound + } + artist, err := s.store.Artist(ctx, artistID) + if err != nil { + return TagSourceResult{}, err + } + return s.one(ctx, artist, *p) +} + func (s *TagService) one(ctx context.Context, a Artist, p tagProvider) (TagSourceResult, error) { snap, err := s.store.Tags(ctx, a.ID, p.name) if err != nil { diff --git a/internal/discovery/tags_test.go b/internal/discovery/tags_test.go index e7bc667..771ef1b 100644 --- a/internal/discovery/tags_test.go +++ b/internal/discovery/tags_test.go @@ -341,3 +341,45 @@ func TestAnArtistRemovedWhileItsTagsWereReadIsNotFound(t *testing.T) { t.Fatalf("err = %v, want ErrNotFound", err) } } + +func TestProviderTagsAsksOnlyThatProvider(t *testing.T) { + r := newTagRig(true, true) + res, err := r.svc.ProviderTags(context.Background(), 1, providers.LastFM) + if err != nil { + t.Fatal(err) + } + if res.Provider != providers.LastFM || res.Status != StatusOK || len(res.Tags) != 2 { + t.Errorf("result = %+v", res) + } + if r.lf.count() != 1 || r.mb.count() != 0 { + t.Errorf("calls: lastfm %d, musicbrainz %d, want 1 and 0", r.lf.count(), r.mb.count()) + } + // Stored now: asking again asks nobody. + if _, err := r.svc.ProviderTags(context.Background(), 1, providers.LastFM); err != nil || r.lf.count() != 1 { + t.Errorf("second ask: err=%v calls=%d", err, r.lf.count()) + } +} + +func TestProviderTagsWithTheProviderOffAsksNobody(t *testing.T) { + r := newTagRig(false, true) + res, err := r.svc.ProviderTags(context.Background(), 1, providers.LastFM) + if err != nil || res.Status != StatusDisabled || r.lf.count() != 0 || len(res.Tags) != 0 { + t.Errorf("result = %+v err=%v calls=%d", res, err, r.lf.count()) + } +} + +func TestProviderTagsRefusesWhatIsNotAProviderOrAnArtist(t *testing.T) { + r := newTagRig(true, true) + if _, err := r.svc.ProviderTags(context.Background(), 1, "spotify"); !errors.Is(err, ErrUnknownProvider) { + t.Errorf("unknown provider: err = %v", err) + } + if _, err := r.svc.ProviderTags(context.Background(), 1, SourceFiles); !errors.Is(err, ErrUnknownProvider) { + t.Errorf("the files are not a provider: err = %v", err) + } + if _, err := r.svc.ProviderTags(context.Background(), 99, providers.LastFM); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown artist: err = %v", err) + } + if r.lf.count() != 0 { + t.Error("a provider was asked for something that is not valid") + } +} diff --git a/internal/taste/service.go b/internal/taste/service.go index 3798e00..a09ce94 100644 --- a/internal/taste/service.go +++ b/internal/taste/service.go @@ -73,48 +73,41 @@ func NewService(store Store, o Options) *Service { return &Service{store: store, opts: o, now: time.Now} } -// Taste returns the listener's profile: up to limit liked artists (zero means -// DefaultLimit) and the tag profile built from the liked artists (the best -// MaxTags). Artists are ordered by how much the listener played them, recent -// plays counting more; ties go to the lower id. -func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, error) { - if limit < 0 { - return Profile{}, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery) - } - switch { - case limit == 0: - limit = DefaultLimit - case limit > MaxLimit: - limit = MaxLimit - } +// Learned is everything the model knows about a listener, for other parts of +// Sonda to build on. +type Learned struct { + Prefs map[int64]ArtistPref // every artist with events, the avoided ones too + Liked []ArtistPref // the ones with a positive score, the most played first + Tags []TagWeight // the tag profile of the liked artists, best first, not cut +} +// Learn works out what the listener's history says. Artists are ordered by how much +// the listener played them, recent plays counting more; ties go to the lower id. The +// tag profile is built from the MaxProfileArtists most played liked artists. +func (s *Service) Learn(ctx context.Context, userID int64) (Learned, error) { now := s.now() events, err := s.store.ArtistEvents(ctx, userID, now.Add(-lookbackHalfLives*s.opts.HalfLife)) if err != nil { - return Profile{}, fmt.Errorf("read events: %w", err) + return Learned{}, fmt.Errorf("read events: %w", err) } - var liked []ArtistPref - for _, p := range Preferences(events, now, s.opts) { + out := Learned{Prefs: Preferences(events, now, s.opts), Liked: []ArtistPref{}, Tags: []TagWeight{}} + for _, p := range out.Prefs { if p.Score > 0 { - liked = append(liked, p) + out.Liked = append(out.Liked, p) } } - sort.Slice(liked, func(i, j int) bool { - a, b := liked[i], liked[j] - switch { - case a.Score != b.Score: + sort.Slice(out.Liked, func(i, j int) bool { + a, b := out.Liked[i], out.Liked[j] + if a.Score != b.Score { return a.Score > b.Score - default: - return a.ArtistID < b.ArtistID } + return a.ArtistID < b.ArtistID }) - - profile := Profile{Artists: []ArtistTaste{}, Tags: []TagWeight{}} - if len(liked) == 0 { - return profile, nil + if len(out.Liked) == 0 { + return out, nil } - top := liked + top := out.Liked if len(top) > MaxProfileArtists { top = top[:MaxProfileArtists] } @@ -125,14 +118,36 @@ func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, } tags, err := s.store.ArtistTags(ctx, ids) if err != nil { - return Profile{}, fmt.Errorf("read tags: %w", err) + return Learned{}, fmt.Errorf("read tags: %w", err) } - profile.Tags = TagProfile(prefs, tags) + out.Tags = TagProfile(prefs, tags) + return out, nil +} + +// Taste returns the listener's profile: up to limit liked artists (zero means +// DefaultLimit) and the best MaxTags tags of the profile built from the liked artists. +func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, error) { + if limit < 0 { + return Profile{}, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery) + } + switch { + case limit == 0: + limit = DefaultLimit + case limit > MaxLimit: + limit = MaxLimit + } + l, err := s.Learn(ctx, userID) + if err != nil { + return Profile{}, err + } + profile := Profile{Artists: []ArtistTaste{}, Tags: l.Tags} if len(profile.Tags) > MaxTags { profile.Tags = profile.Tags[:MaxTags] } - - shown := liked + if len(l.Liked) == 0 { + return profile, nil + } + shown := l.Liked if len(shown) > limit { shown = shown[:limit] } diff --git a/internal/taste/service_test.go b/internal/taste/service_test.go index a88e00e..6626ff7 100644 --- a/internal/taste/service_test.go +++ b/internal/taste/service_test.go @@ -200,3 +200,38 @@ func TestTasteReturnsStoreErrors(t *testing.T) { t.Errorf("err = %v", err) } } + +func TestLearnGivesEveryPreferenceTheLikedOnesInOrderAndTheWholeTagProfile(t *testing.T) { + tags := map[string]float64{} + for i := 0; i < MaxTags+10; i++ { + tags[string(rune('a'+i%26))+string(rune('a'+i/26))] = 1 + } + f := &fakeStore{ + events: append(append(complete(1, 5), complete(2, 1)...), + Event{ArtistID: 3, Type: listening.Skip, PlayedMs: 100, At: now}), + tags: map[int64]map[string]float64{1: tags}, + } + l, err := newService(f).Learn(context.Background(), 4) + if err != nil { + t.Fatal(err) + } + if f.user != 4 { + t.Errorf("asked for user %d", f.user) + } + if len(l.Prefs) != 3 || l.Prefs[3].Preference >= 0 { + t.Errorf("prefs = %+v, want all three, the skipped one negative", l.Prefs) + } + if len(l.Liked) != 2 || l.Liked[0].ArtistID != 1 || l.Liked[1].ArtistID != 2 { + t.Errorf("liked = %+v, want 1 then 2", l.Liked) + } + if len(l.Tags) != MaxTags+10 { + t.Errorf("%d tags, want the whole profile (%d): only the display is cut", len(l.Tags), MaxTags+10) + } +} + +func TestLearnOfSomeoneWhoHasListenedToNothing(t *testing.T) { + l, err := newService(&fakeStore{}).Learn(context.Background(), 1) + if err != nil || l.Prefs == nil || l.Liked == nil || l.Tags == nil || len(l.Prefs)+len(l.Liked)+len(l.Tags) != 0 { + t.Errorf("learned = %+v err=%v, want empty and not nil", l, err) + } +} diff --git a/sonda.example.toml b/sonda.example.toml index 7dfbceb..cbd7bd1 100644 --- a/sonda.example.toml +++ b/sonda.example.toml @@ -29,4 +29,10 @@ min_listen_seconds = 20 # a play counts as a listen after this long, or when [taste] half_life_days = 90 # an old listen weighs half as much after this many days +[scoring] # weights of the parts of a Discover score (only their proportions matter) +weight_affinity = 0.40 # similar to the artists I like +weight_genre = 0.25 # tags close to the tags I like +weight_preference = 0.25 # what my listening says about the artist +weight_discovery = 0.10 # bonus for what I have not played yet + # Last.fm and MusicBrainz are set up through the API, not here: PUT /api/v1/providers/{name}. From f548423f73fcebdd4a115a4571f88f0f29fe9edf Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 19:48:51 +0100 Subject: [PATCH 2/2] fix(discover): ask permission first, stable order, honest docs - Check the discover_from_listening setting before reading the listening history, so a refused request does no work and a database failure cannot turn it into a 500. - Cosine adds up in tag order: the sum in map order changed in the last digit between runs, and equal artists could swap places between requests. - A seed that could not be read is no longer listed as a seed. - README and spec: Discover exists now; "disabled" means something had to be asked and Last.fm is off; the stored lookups are shared, so another local user can see from retrieved_at that an artist was looked up, but not who. - Tests: the history is not read for a refused request, a stable cosine, tags fetched only for the pool and four at a time, a cancelled request stops, MusicBrainz is never asked, the setting is per user. --- README.md | 8 +-- internal/api/http/discover_test.go | 34 ++++++++++- internal/api/openapi/openapi.yaml | 10 +++- internal/discover/score.go | 22 +++++-- internal/discover/score_test.go | 15 +++++ internal/discover/service.go | 40 +++++++++---- internal/discover/service_test.go | 92 +++++++++++++++++++++++++++++- 7 files changed, 196 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a83e13b..66372df 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Sonda is a privacy-first, self-hosted music server written in Go. It streams the It is a single executable, one database (SQLite by default, PostgreSQL if you prefer) and the folders that hold your music. There is no Redis, no message broker and no cloud account. -> **Status: early.** The local-playback part works end to end: accounts, libraries, scanning, search, and streaming with HTTP Range. Recommendations, a web client and OpenSubsonic compatibility are not built yet (see [What is not there yet](#what-is-not-there-yet)). The API may still change. +> **Status: early.** The local-playback part works end to end: accounts, libraries, scanning, search, and streaming with HTTP Range. Suggestions of new artists work too (Discover). Radio queues, a web client and OpenSubsonic compatibility are not built yet (see [What is not there yet](#what-is-not-there-yet)). The API may still change. ## What works today @@ -16,14 +16,14 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. - **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. -- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on; without it the answer says `disabled`. See [External providers](#external-providers). -- **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, the Discover feature (coming next) may start from the artists you listen to most, and their names would be sent to Last.fm (only if an admin switched it on). +- **Discover.** `GET /discover` suggests artists you may like, in your library or not, each with a score from 0 to 1 and the parts it is made of: how close it is to the artists you like (affinity), how close its tags are to yours (genre), what your listening says about it (preference) and a bonus for what you have not played (discovery). The weights are in `[scoring]` in the config. A part with no data is left out and the rest rescaled, and every answer says which seeds led to each suggestion. It starts from the artists you name with `?seed=`, or, only if you turned on `discover_from_listening`, from the artists you play most. It needs Last.fm to be switched on to ask for anything new: when Last.fm is off and a seed's similar artists are not stored yet, the answer says `disabled` (what was stored earlier is still used). See [External providers](#external-providers). +- **Your own settings.** `GET` and `PATCH /me/settings` hold choices that belong to you, not to the server. Everyone starts private: `discover_from_listening` is off, so this setting never sends your listening to a provider. When you turn it on, Discover may start from the artists you listen to most, and their names are sent to Last.fm (only if an admin switched it on). What Last.fm answers is kept in the shared database like any other lookup, so another local user can see, from `retrieved_at` of `GET /artists/{id}/similar`, that someone looked up an artist, but not who. - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet -Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Radio queues from all this data are next. +Radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. ## Requirements diff --git a/internal/api/http/discover_test.go b/internal/api/http/discover_test.go index e6aa423..512a98f 100644 --- a/internal/api/http/discover_test.go +++ b/internal/api/http/discover_test.go @@ -21,6 +21,7 @@ import ( type discoverEnv struct { *similarEnv lf *fakeTags // what Last.fm says about tags, for any artist + mb *fakeTags // MusicBrainz, which Discover must never ask } // newDiscoverEnv wires the real Discover on the similar-artists environment: Last.fm @@ -28,11 +29,14 @@ type discoverEnv struct { // library) and Gamma (match 0.5, not in it) are similar to anyone. func newDiscoverEnv(t *testing.T) *discoverEnv { t.Helper() - e := &discoverEnv{lf: &fakeTags{tags: []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "grunge", Weight: 40}}}} + e := &discoverEnv{ + lf: &fakeTags{tags: []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "grunge", Weight: 40}}}, + mb: &fakeTags{tags: []discovery.Tag{{Name: "alternative rock", Weight: 43, Genre: true}}}, + } e.similarEnv = newSimilarEnvWith(t, func(d *Deps, db *bun.DB, prov *providers.Service) { tasteSvc := taste.NewService(storage.NewTasteRepo(db), taste.Options{MinListen: 20 * time.Second, HalfLife: 90 * 24 * time.Hour}) settings := usersettings.NewService(storage.NewUserSettingsRepo(db)) - tags := discovery.NewTagService(storage.NewDiscoveryRepo(db), map[string]discovery.TagSource{providers.LastFM: e.lf}, prov, zap.NewNop()) + tags := discovery.NewTagService(storage.NewDiscoveryRepo(db), map[string]discovery.TagSource{providers.LastFM: e.lf, providers.MusicBrainz: e.mb}, prov, zap.NewNop()) d.Taste, d.Settings = tasteSvc, settings d.Discover = discover.NewService(tasteSvc, d.Discovery.(discover.Similar), tags, storage.NewTasteRepo(db), settings, discover.DefaultWeights()) }) @@ -170,6 +174,32 @@ func TestDiscoverFromMyListeningNeedsAndUsesTheSetting(t *testing.T) { if e.lf.calls == 0 { t.Error("the tags of the candidates were never read") } + if e.mb.calls != 0 { + t.Errorf("MusicBrainz was asked %d times: Discover reads tags from Last.fm only", e.mb.calls) + } +} + +func TestTheSettingBelongsToEachUser(t *testing.T) { + e := newDiscoverEnv(t) + e.enableLastfm(t) + e.listen(t, e.admin, e.trackOf(t, "Alpha"), "complete", 1000, "a1") + e.allowListening(t) // alice allows it + + body := map[string]any{"username": "bob", "password": "longenough"} + if r := do(t, e.h, "POST", "/api/v1/users", e.admin, body); r.Code != http.StatusCreated { + t.Fatalf("create bob: %d %v", r.Code, r.Body) + } + bob, _ := do(t, e.h, "POST", "/api/v1/auth/login", "", body).Body["access_token"].(string) + r := do(t, e.h, "GET", "/api/v1/discover", bob, nil) + if r.Code != http.StatusBadRequest || errCode(r) != "seed_required" { + t.Errorf("bob did not allow it: %d %v, want 400 seed_required", r.Code, r.Body) + } + if e.source.count() != 0 { + t.Error("Last.fm was asked on behalf of bob, who never allowed it") + } + if r := e.get(t, ""); r.Code != http.StatusOK { + t.Errorf("alice did: %d %v", r.Code, r.Body) + } } func TestDiscoverFilterAndLimit(t *testing.T) { diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 65ec3e9..15cce76 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -688,8 +688,10 @@ paths: gets old can take several seconds, because the provider is asked politely, a few at a time; later ones are served from the database. - `provider_status` is `disabled` when Last.fm is off or has no key (anything stored earlier - is still used, so the list can be empty), `error` when Last.fm failed for some seed (the + `provider_status` is `disabled` when something had to be asked from Last.fm (a seed's + similar artists are missing or old) and Last.fm is off or has no key; anything stored earlier + is still used, so the list can be empty. With every seed's data stored and fresh the status is `ok` + whether Last.fm is on or not. It is `error` when Last.fm failed for some seed (the rest is still returned) and `ok` otherwise. Show `attribution` where you show the data. security: [{bearerAuth: []}] parameters: @@ -1792,7 +1794,9 @@ components: When true, Discover may start from the artists you listen to most. That sends their names to Last.fm, which is asked for artists similar to each one (only when an admin switched Last.fm on). When false, Discover starts only from artists you name yourself. - False until you turn it on. + False until you turn it on. What Last.fm answers is stored like any other lookup and is + shared: another user can see from `retrieved_at` in `GET /artists/{id}/similar` that + somebody looked an artist up, but not who. UpdateUserSettings: type: object diff --git a/internal/discover/score.go b/internal/discover/score.go index 31efb86..2be0c92 100644 --- a/internal/discover/score.go +++ b/internal/discover/score.go @@ -15,6 +15,7 @@ package discover import ( "errors" "math" + "sort" ) // Weights say how much each part counts. Only their proportions matter. @@ -79,13 +80,15 @@ func Affinity(links []Link) float64 { // Cosine compares two tag vectors. ok is false when either has nothing to compare // (no tags, or all weights zero), which is different from "nothing in common". func Cosine(a, b map[string]float64) (float64, bool) { + // Added up in tag order: a sum in map order changes in the last digit from one + // run to the next, and two equal artists could swap places between requests. var dot, na, nb float64 - for tag, wa := range a { - na += wa * wa - dot += wa * b[tag] + for _, tag := range sortedKeys(a) { + na += a[tag] * a[tag] + dot += a[tag] * b[tag] } - for _, wb := range b { - nb += wb * wb + for _, tag := range sortedKeys(b) { + nb += b[tag] * b[tag] } if na <= 0 || nb <= 0 { return 0, false @@ -93,6 +96,15 @@ func Cosine(a, b map[string]float64) (float64, bool) { return clamp01(dot / (math.Sqrt(na) * math.Sqrt(nb))), true } +func sortedKeys(m map[string]float64) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + // PreferenceScore maps the preference from -1..1 to 0..1. An artist the listener // never played (known is false) is neutral: 0.5. func PreferenceScore(pref float64, known bool) float64 { diff --git a/internal/discover/score_test.go b/internal/discover/score_test.go index 20fc0f5..4e8d2ff 100644 --- a/internal/discover/score_test.go +++ b/internal/discover/score_test.go @@ -150,3 +150,18 @@ func TestWeightsValidation(t *testing.T) { t.Errorf("default weights = %+v, want %+v", d, good) } } + +func TestCosineGivesTheSameNumberEveryTime(t *testing.T) { + a, b := map[string]float64{}, map[string]float64{} + for i := 0; i < 60; i++ { + tag := string(rune('a'+i%26)) + string(rune('a'+i/26)) + a[tag] = 1 / float64(i+3) + b[tag] = 1 / float64(2*i+7) + } + first, _ := Cosine(a, b) + for i := 0; i < 300; i++ { + if got, _ := Cosine(a, b); got != first { + t.Fatalf("run %d: %v, first was %v: the sums depend on map order, and equal artists can swap places", i, got, first) + } + } +} diff --git a/internal/discover/service.go b/internal/discover/service.go index dad5b8e..324786e 100644 --- a/internal/discover/service.go +++ b/internal/discover/service.go @@ -139,14 +139,10 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, return Result{}, err } - learned, err := s.taste.Learn(ctx, userID) - if err != nil { - return Result{}, err - } - res := Result{ProviderStatus: discovery.StatusOK, Seeds: []Seed{}, Items: []Item{}} seeds := named if len(named) == 0 { + // Ask permission before doing any work for a request that may be refused. set, err := s.settings.Get(ctx, userID) if err != nil { return Result{}, err @@ -155,6 +151,13 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, return Result{}, ErrSeedRequired } res.FromListening = true + } + + learned, err := s.taste.Learn(ctx, userID) + if err != nil { + return Result{}, err + } + if res.FromListening { for i, p := range learned.Liked { if i == MaxListeningSeeds { break @@ -165,7 +168,6 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, return Result{}, err } } - res.Seeds = append(res.Seeds, seeds...) if len(seeds) == 0 { return res, nil } @@ -175,6 +177,8 @@ func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, return Result{}, err } res.ProviderStatus = status + seeds, lists = withoutUnread(seeds, lists) + res.Seeds = append(res.Seeds, seeds...) cands := gather(seeds, lists, q.InLibrary) pool := rankByAffinity(cands, max(limit, minPool)) @@ -278,10 +282,10 @@ func (s *Service) nameSeeds(ctx context.Context, seeds []Seed) error { // readSimilar reads the similar artists of every seed, Parallelism at a time. A seed // that was removed in the meantime is skipped; any other error stops the request. -func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]discovery.Result, string, error) { +func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]*discovery.Result, string, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() - results := make([]discovery.Result, len(seeds)) + results := make([]*discovery.Result, len(seeds)) // nil for a seed that could not be read var ( wg sync.WaitGroup mu sync.Mutex @@ -311,7 +315,7 @@ func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]discovery.Re mu.Unlock() return } - results[i] = r + results[i] = &r }() } wg.Wait() @@ -323,6 +327,9 @@ func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]discovery.Re } status := discovery.StatusOK for _, r := range results { + if r == nil { + continue + } switch r.ProviderStatus { case discovery.StatusDisabled: status = discovery.StatusDisabled @@ -335,6 +342,19 @@ func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]discovery.Re return results, status, nil } +// withoutUnread drops the seeds that could not be read, with their lists, so that +// they are not reported as the reason for anything. +func withoutUnread(seeds []Seed, lists []*discovery.Result) ([]Seed, []*discovery.Result) { + var keptSeeds []Seed + var keptLists []*discovery.Result + for i, sd := range seeds { + if lists[i] != nil { + keptSeeds, keptLists = append(keptSeeds, sd), append(keptLists, lists[i]) + } + } + return keptSeeds, keptLists +} + type candidate struct { id int64 name string @@ -344,7 +364,7 @@ type candidate struct { // gather turns the similar lists into candidates. A seed is never suggested, and // the library filter is applied here. -func gather(seeds []Seed, lists []discovery.Result, inLibrary *bool) []*candidate { +func gather(seeds []Seed, lists []*discovery.Result, inLibrary *bool) []*candidate { isSeed := map[int64]bool{} for _, sd := range seeds { isSeed[sd.ID] = true diff --git a/internal/discover/service_test.go b/internal/discover/service_test.go index 7bcc8ac..c59a8cf 100644 --- a/internal/discover/service_test.go +++ b/internal/discover/service_test.go @@ -18,9 +18,11 @@ type fakeTaste struct { learned taste.Learned err error user int64 + calls int } func (f *fakeTaste) Learn(_ context.Context, userID int64) (taste.Learned, error) { + f.calls++ f.user = userID return f.learned, f.err } @@ -238,6 +240,9 @@ func TestDiscoverWithoutSeedsAndTheSettingOffIsRefusedAndAsksNobody(t *testing.T if len(r.similar.called())+len(r.fetch.called()) != 0 { t.Error("a provider was asked although the listener did not allow it") } + if r.taste.calls != 0 { + t.Error("the history was read for a request that was refused") + } } func TestDiscoverFromListeningUsesTheArtistsILikeWeightedByHowMuch(t *testing.T) { @@ -390,7 +395,7 @@ func TestTagsAreFetchedOnlyForCandidatesWithNone_AndOnlyFromLastfm(t *testing.T) for _, id := range r.fetch.called() { got[id] = true } - if got[10] || got[11] || !got[13] { + if got[10] || got[11] || !got[12] || !got[13] { t.Errorf("fetched for %v, want only artists without tags", got) } for _, p := range r.fetch.prov { @@ -605,3 +610,88 @@ func TestSettingsAndTasteErrorsPass(t *testing.T) { t.Errorf("settings: err = %v", err) } } + +func TestASeedThatCannotBeReadIsNotListedAsASeed(t *testing.T) { + r := newRig() + r.similar.errs[1] = discovery.ErrNotFound + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1, 2}}) + if err != nil { + t.Fatal(err) + } + if len(res.Seeds) != 1 || res.Seeds[0].ID != 2 { + t.Errorf("seeds = %+v, want only the one that gave something", res.Seeds) + } +} + +func TestOnlyThePoolOfBestCandidatesIsScoredAndTaggedWhateverTheLimit(t *testing.T) { + r := newRig() + var items []discovery.Item + for i := 0; i < 50; i++ { + id := int64(200 + i) + items = append(items, item(id, "c", i+1, 1-float64(i)/100, false)) + r.store.names[id] = "c" + } + r.similar.lists[1] = discovery.Result{Items: items} + res, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}, Limit: 3}) + if err != nil || len(res.Items) != 3 { + t.Fatalf("items %d err %v", len(res.Items), err) + } + fetched := r.fetch.called() + if len(fetched) != minPool { + t.Errorf("tags fetched for %d candidates, want the pool of %d", len(fetched), minPool) + } + for _, id := range fetched { + if id >= 200+minPool { + t.Errorf("tags fetched for candidate %d, which is below the pool", id) + } + } +} + +func TestTagsAreFetchedFourAtATimeAndStopWhenTheListenerLeaves(t *testing.T) { + r := newRig() + var items []discovery.Item + for i := 0; i < 20; i++ { + items = append(items, item(int64(300+i), "c", i+1, 0.5, false)) + } + r.similar.lists[1] = discovery.Result{Items: items} + slow := &slowFetcher{delay: 20 * time.Millisecond} + r.svc = NewService(r.taste, r.similar, slow, r.store, r.set, DefaultWeights()) + if _, err := r.svc.Discover(context.Background(), 7, Query{Seeds: []int64{1}}); err != nil { + t.Fatal(err) + } + if m := atomic.LoadInt32(&slow.maxSeen); m < 2 || m > Parallelism { + t.Errorf("at most %d tag requests at once, saw %d", Parallelism, m) + } + + slow = &slowFetcher{delay: time.Second} + r.svc = NewService(r.taste, r.similar, slow, r.store, r.set, DefaultWeights()) + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := r.svc.Discover(ctx, 7, Query{Seeds: []int64{1}}); !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 500*time.Millisecond { + t.Errorf("err = %v after %v, want a prompt deadline error", err, time.Since(start)) + } +} + +type slowFetcher struct { + delay time.Duration + inFlight int32 + maxSeen int32 +} + +func (f *slowFetcher) ProviderTags(ctx context.Context, id int64, provider string) (discovery.TagSourceResult, error) { + n := atomic.AddInt32(&f.inFlight, 1) + defer atomic.AddInt32(&f.inFlight, -1) + for { + m := atomic.LoadInt32(&f.maxSeen) + if n <= m || atomic.CompareAndSwapInt32(&f.maxSeen, m, n) { + break + } + } + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return discovery.TagSourceResult{}, ctx.Err() + } + return discovery.TagSourceResult{Provider: provider, Status: discovery.StatusOK}, nil +}