diff --git a/README.md b/README.md index f990b5b..1b9eb56 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. -- **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`). It is worked out on every request from data already in your database; nothing is sent to any provider. +- **Your taste.** `GET /me/taste` shows what Sonda has learned from your own listening: the artists you like and the kinds of music (tags) they have in common. A complete listen counts for more than a short one, a skip counts against the artist, and older listens weigh less (they halve every 90 days, `taste.half_life_days`; only the last ten half-lives are read). It is worked out on every request from data already in your database; nothing is sent to any provider. - **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights, and the genres of the artist's own files, read offline. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 2809085..d51ac9f 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -661,22 +661,26 @@ paths: Each event has a value: a `complete` listen +1, any other listen that counts +0.5 (see `POST /listens`), a `skip` before that -1, and a play too short to mean anything 0. An - event weighs half as much for every 90 days (`taste.half_life_days`) that has passed. For - each artist the values add up, and `preference` is `tanh(sum / 3)`: from -1 to 1, never - reaching either, so one skip does not sink an artist and no amount of listening makes - one perfect. A track with several artists credits each of them. + event weighs half as much for every 90 days (`taste.half_life_days`) that has passed, and + one dated after the moment of the request weighs as if it happened then. Only the last + ten half-lives are read (900 days by default); older events weigh less than a thousandth + and are left out, also from `listens`. For each artist the values add up, and `preference` + is `tanh(sum / 3)`: from -1 to 1, approaching both (a large sum rounds to exactly 1), so + one skip does not sink an artist and listening makes one loved at most. A track with + several artists credits each of them. - `artists` are the ones you like (a `preference` above 0), the most liked first, ties going - to the most recent listen. `tags` are the kinds of music you like: the tags of those - artists (the genres of your files, and what Last.fm and MusicBrainz said earlier, if - they were ever asked) added up, each artist weighing as much as its `preference`, and - scaled so the best tag is 1. Tags are read from what is already stored; this request never - asks a provider. The tag profile is built from your 200 most liked artists. + `artists` are the ones you like (a `preference` above 0), the most played first. They are + ordered by the sum itself, so artists whose `preference` rounds to 1 keep their order; + ties go to the lower id. `tags` are the kinds of music you like, at most 50: the tags of + those artists (the genres of your files that still exist, and what Last.fm and MusicBrainz + said earlier, if they were ever asked) added up, each artist weighing as much as its + `preference`, and scaled so the best tag is 1, best first. Tags are read from what is already stored; this + request never asks a provider. The tag profile is built from your 200 most liked artists. security: [{bearerAuth: []}] parameters: - name: limit in: query - description: How many artists to return. 20 when absent or 0; values above 50 are lowered to 50. Tags are not limited by it. + description: How many artists to return. 20 when absent or 0; values above 50 are lowered to 50. It does not limit `tags`. schema: {type: integer, minimum: 0, default: 20} responses: '200': @@ -1619,7 +1623,7 @@ components: id: {type: integer, format: int64} name: {type: string} preference: {type: number, minimum: 0, exclusiveMinimum: true, maximum: 1, description: How much you like the artist. Only artists above 0 are listed.} - listens: {type: integer, description: "Events that count as a listen, with no decay."} + listens: {type: integer, description: "Events that count as a listen, with no decay, within the period that is read (ten half-lives)."} last_listened_at: {type: string, format: date-time, description: The most recent listen that counted.} TasteTag: diff --git a/internal/storage/taste.go b/internal/storage/taste.go index 8834ce6..9f0374e 100644 --- a/internal/storage/taste.go +++ b/internal/storage/taste.go @@ -95,12 +95,15 @@ func (r *TasteRepo) ArtistTags(ctx context.Context, ids []int64) (map[int64]map[ } } - // Files: track counts, scaled to the artist's most common genre. + // Files: track counts, scaled to the artist's most common genre. Only tracks + // that still have a file: the genre of a file that is gone says nothing about + // the music that is there (the same rule as the genres of /artists/{id}/tags). + vis, visArgs := visible("t", 0) files, err := r.db.QueryContext(ctx, `SELECT ta.artist_id, g.genre, COUNT(DISTINCT ta.track_id) - FROM track_artists ta JOIN track_genres g ON g.track_id = ta.track_id - WHERE ta.artist_id IN (?) AND ta.role = 'main' GROUP BY ta.artist_id, g.genre`, - bun.In(ids)) + FROM track_artists ta JOIN track_genres g ON g.track_id = ta.track_id JOIN tracks t ON t.id = ta.track_id + WHERE ta.artist_id IN (?) AND ta.role = 'main' AND `+vis+` GROUP BY ta.artist_id, g.genre`, + append([]any{bun.In(ids)}, visArgs...)...) if err != nil { return nil, fmt.Errorf("read genres: %w", err) } diff --git a/internal/storage/taste_test.go b/internal/storage/taste_test.go index 7349bf9..65b43d5 100644 --- a/internal/storage/taste_test.go +++ b/internal/storage/taste_test.go @@ -2,6 +2,7 @@ package storage import ( "context" + "math" "testing" "time" @@ -41,6 +42,10 @@ func TestTasteArtistEvents(t *testing.T) { record(alice.ID, "Army of Me", "a3", listening.Play, 30_000, recent) record(bob.ID, "Airbag", "b1", listening.Complete, 1, recent) // somebody else's // Two artists credited on one track: both are credited with the event. + if _, err := db.Exec("INSERT INTO track_artists (track_id, artist_id, role) VALUES (?, ?, 'main')", + s.track["Song A"], artistID(t, db, "Artist B")); err != nil { + t.Fatal(err) + } record(alice.ID, "Song A", "a4", listening.Complete, 100_000, recent) got, err := NewTasteRepo(db).ArtistEvents(ctx, alice.ID, time.Unix(1_600_000_000, 0)) @@ -62,15 +67,18 @@ func TestTasteArtistEvents(t *testing.T) { if len(by[artistID(t, db, "Artist A")]) != 1 { t.Errorf("the artist credited on Song A has %d events, want 1", len(by[artistID(t, db, "Artist A")])) } - if len(got) != 3 { - t.Errorf("got %d events, want 3: %+v", len(got), got) + if len(by[artistID(t, db, "Artist B")]) != 1 || by[artistID(t, db, "Artist B")][0].PlayedMs != 100_000 { + t.Errorf("the second artist credited on Song A has %+v, want the same event", by[artistID(t, db, "Artist B")]) + } + if len(got) != 4 { + t.Errorf("got %d events, want 4: %+v", len(got), got) } // A track whose files are gone still counts: the history is the listener's own. record(alice.ID, "Ghost", "a5", listening.Complete, 100_000, recent) got, _ = NewTasteRepo(db).ArtistEvents(ctx, alice.ID, time.Unix(1_600_000_000, 0)) - if len(got) != 4 { - t.Errorf("with a track that has no file: %d events, want 4", len(got)) + if len(got) != 5 { + t.Errorf("with a track that has no file: %d events, want 5", len(got)) } }) } @@ -96,7 +104,8 @@ func TestTasteArtistTags(t *testing.T) { ctx := context.Background() radiohead, bjork := artistID(t, db, "Radiohead"), artistID(t, db, "Björk") - // Genres of the files: Radiohead has 4 tracks, 4 rock and 2 art rock. + // Genres of the files: Radiohead has 4 tracks. Ghost's file is gone, so its + // genre says nothing about the music that is there: 3 rock and 2 art rock. for _, g := range []struct{ track, genre string }{ {"Airbag", "rock"}, {"Paranoid Android", "rock"}, {"Karma Police", "rock"}, {"Ghost", "rock"}, {"Airbag", "art rock"}, {"Karma Police", "art rock"}, @@ -105,6 +114,9 @@ func TestTasteArtistTags(t *testing.T) { t.Fatal(err) } } + if _, err := db.Exec("INSERT INTO track_genres (track_id, genre) VALUES (?, 'trip hop')", s.track["Ghost"]); err != nil { + t.Fatal(err) + } // Provider tags: Last.fm is 0-100, MusicBrainz is votes. for _, row := range []struct { artist int64 @@ -128,12 +140,12 @@ func TestTasteArtistTags(t *testing.T) { if err != nil { t.Fatal(err) } - want := map[string]float64{"rock": 1, "art rock": 0.5, "alternative": 1, "electronic": 0.5} + want := map[string]float64{"rock": 1, "art rock": 2.0 / 3, "alternative": 1, "electronic": 0.5} if len(got[radiohead]) != len(want) { t.Errorf("Radiohead tags = %v, want %v", got[radiohead], want) } for tag, w := range want { - if got[radiohead][tag] != w { + if math.Abs(got[radiohead][tag]-w) > 1e-9 { t.Errorf("Radiohead %q = %v, want %v", tag, got[radiohead][tag], w) } } diff --git a/internal/taste/service.go b/internal/taste/service.go index 5fa01d6..3798e00 100644 --- a/internal/taste/service.go +++ b/internal/taste/service.go @@ -16,6 +16,9 @@ const ( DefaultLimit = 20 MaxLimit = 50 + // MaxTags is how many tags the profile keeps, the best first. + MaxTags = 50 + // MaxProfileArtists is how many of the liked artists, the most liked first, // shape the tag profile. The rest weigh too little to change it and cost a read each. MaxProfileArtists = 200 @@ -23,6 +26,8 @@ const ( // lookbackHalfLives is how far back the history is read: an event ten // half-lives old weighs less than a thousandth of a new one. lookbackHalfLives = 10 + + defaultHalfLife = 90 * 24 * time.Hour ) // Store reads what the model needs. It never writes. @@ -62,11 +67,16 @@ type Service struct { // NewService builds a Service. func NewService(store Store, o Options) *Service { + if o.HalfLife <= 0 { + o.HalfLife = defaultHalfLife // a zero would make every weight NaN + } return &Service{store: store, opts: o, now: time.Now} } // Taste returns the listener's profile: up to limit liked artists (zero means -// DefaultLimit) and the tag profile built from the liked artists. +// DefaultLimit) and the tag profile built from the liked artists (the best +// MaxTags). Artists are ordered by how much the listener played them, recent +// plays counting more; ties go to the lower id. func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, error) { if limit < 0 { return Profile{}, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery) @@ -85,17 +95,15 @@ func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, } var liked []ArtistPref for _, p := range Preferences(events, now, s.opts) { - if p.Preference > 0 { + if p.Score > 0 { liked = append(liked, p) } } sort.Slice(liked, func(i, j int) bool { a, b := liked[i], liked[j] switch { - case a.Preference != b.Preference: - return a.Preference > b.Preference - case !a.LastListened.Equal(b.LastListened): - return a.LastListened.After(b.LastListened) + case a.Score != b.Score: + return a.Score > b.Score default: return a.ArtistID < b.ArtistID } @@ -120,6 +128,9 @@ func (s *Service) Taste(ctx context.Context, userID int64, limit int) (Profile, return Profile{}, fmt.Errorf("read tags: %w", err) } profile.Tags = TagProfile(prefs, tags) + if len(profile.Tags) > MaxTags { + profile.Tags = profile.Tags[:MaxTags] + } shown := liked if len(shown) > limit { diff --git a/internal/taste/service_test.go b/internal/taste/service_test.go index fcc0308..a88e00e 100644 --- a/internal/taste/service_test.go +++ b/internal/taste/service_test.go @@ -3,6 +3,7 @@ package taste import ( "context" "errors" + "math" "testing" "time" @@ -129,8 +130,8 @@ func TestTasteLimit(t *testing.T) { } func TestTasteTiesAreStable(t *testing.T) { - // Same preference, so the one heard more recently goes first, then the lower id. - f := &fakeStore{ + // Same listens at the same time: the lower id goes first. + same := &fakeStore{ events: []Event{ {ArtistID: 5, Type: listening.Complete, At: now.Add(-time.Hour)}, {ArtistID: 4, Type: listening.Complete, At: now.Add(-time.Hour)}, @@ -138,9 +139,44 @@ func TestTasteTiesAreStable(t *testing.T) { }, names: map[int64]string{}, } - p, _ := newService(f).Taste(context.Background(), 1, 0) + p, _ := newService(same).Taste(context.Background(), 1, 0) if len(p.Artists) != 3 || p.Artists[0].ID != 4 || p.Artists[1].ID != 5 || p.Artists[2].ID != 6 { - t.Errorf("order = %+v", p.Artists) + t.Errorf("order by id = %+v", p.Artists) + } +} + +func TestTasteOrdersByTheScoreNotByTheRoundedPreference(t *testing.T) { + var events []Event + for i := 0; i < 500; i++ { + events = append(events, Event{ArtistID: 1, Type: listening.Complete, At: now.Add(-time.Hour)}) + } + for i := 0; i < 60; i++ { + events = append(events, Event{ArtistID: 2, Type: listening.Complete, At: now}) + } + p, _ := newService(&fakeStore{events: events, names: map[int64]string{}}).Taste(context.Background(), 1, 0) + if len(p.Artists) != 2 || p.Artists[0].ID != 1 { + t.Errorf("order = %+v, want the artist heard 500 times first", p.Artists) + } +} + +func TestTasteKeepsTheTagListShort(t *testing.T) { + tags := map[string]float64{} + for i := 0; i < MaxTags+30; i++ { + tags[string(rune('a'+i%26))+string(rune('a'+i/26))] = 1 - float64(i)/1000 + } + f := &fakeStore{events: complete(1, 1), tags: map[int64]map[string]float64{1: tags}, names: map[int64]string{1: "One"}} + p, _ := newService(f).Taste(context.Background(), 1, 0) + if len(p.Tags) != MaxTags || p.Tags[0].Weight != 1 { + t.Errorf("%d tags, first %+v; want %d, best first", len(p.Tags), p.Tags[0], MaxTags) + } +} + +func TestAZeroHalfLifeDoesNotBreakTheModel(t *testing.T) { + s := NewService(&fakeStore{events: complete(1, 3), names: map[int64]string{1: "One"}}, Options{MinListen: 20 * time.Second}) + s.now = func() time.Time { return now } + p, err := s.Taste(context.Background(), 1, 0) + if err != nil || len(p.Artists) != 1 || math.IsNaN(p.Artists[0].Preference) || p.Artists[0].Preference <= 0 { + t.Errorf("profile = %+v err=%v, want a normal preference", p, err) } } @@ -154,7 +190,7 @@ func TestTasteBoundsTheArtistsBehindTheTagProfile(t *testing.T) { t.Fatal(err) } if len(f.tagCalls) != 1 || len(f.tagCalls[0]) != MaxProfileArtists { - t.Errorf("tags were read for %d artists in %d calls, want %d in one", len(f.tagCalls[len(f.tagCalls)-1]), len(f.tagCalls), MaxProfileArtists) + t.Errorf("tags were read in %d calls (%v), want one call for %d artists", len(f.tagCalls), f.tagCalls, MaxProfileArtists) } } diff --git a/internal/taste/taste.go b/internal/taste/taste.go index 587aa3a..1e7dd18 100644 --- a/internal/taste/taste.go +++ b/internal/taste/taste.go @@ -8,7 +8,9 @@ // Each listening event has a value (see EventValue). Older events weigh less // (the weight halves every Options.HalfLife). The values of one artist add up // and are squeezed into (-1, 1) by tanh, so a few listens already say something -// but no amount of them reaches 1, and one skip does not sink an artist. +// but they only approach 1, and one skip does not sink an artist. tanh is for +// showing: it rounds to exactly 1 for a very large sum, so artists are ordered by +// the sum itself (Score). package taste import ( @@ -41,6 +43,7 @@ type Event struct { type ArtistPref struct { ArtistID int64 Preference float64 // from -1 (avoided) to 1 (loved); 0 is neutral + Score float64 // the weighted sum behind Preference; it keeps order where tanh rounds to 1 Listens int // events that count as a listen, with no decay LastListened time.Time // the latest counted event; zero when there is none } @@ -76,10 +79,10 @@ func Preferences(events []Event, now time.Time, o Options) map[int64]ArtistPref sums := map[int64]float64{} out := map[int64]ArtistPref{} for _, e := range events { - age := now.Sub(e.At) - if age < 0 { - age = 0 // a fast clock does not make an event more than new + if e.At.After(now) { + e.At = now // a wrong clock does not make an event newer than now } + age := now.Sub(e.At) weight := math.Exp2(-float64(age) / float64(o.HalfLife)) sums[e.ArtistID] += weight * EventValue(e.Type, e.PlayedMs, o.MinListen) @@ -94,6 +97,7 @@ func Preferences(events []Event, now time.Time, o Options) map[int64]ArtistPref out[e.ArtistID] = p } for id, p := range out { + p.Score = sums[id] p.Preference = math.Tanh(sums[id] / sumScale) out[id] = p } diff --git a/internal/taste/taste_test.go b/internal/taste/taste_test.go index d90c343..5b418bb 100644 --- a/internal/taste/taste_test.go +++ b/internal/taste/taste_test.go @@ -58,6 +58,31 @@ func TestPreferenceGrowsWithListensAndStaysBelowOne(t *testing.T) { } } +func TestHeavilyPlayedArtistsKeepTheirOrder(t *testing.T) { + var events []Event + for i := 0; i < 500; i++ { + events = append(events, Event{ArtistID: 1, Type: listening.Complete, At: now.Add(-time.Hour)}) + } + for i := 0; i < 60; i++ { + events = append(events, Event{ArtistID: 2, Type: listening.Complete, At: now}) + } + got := Preferences(events, now, options) + if got[1].Score <= got[2].Score { + t.Errorf("500 listens score %v, 60 listens score %v: the one heard more must score more", got[1].Score, got[2].Score) + } + if got[1].Preference > 1 || got[1].Preference < 0.999 { + t.Errorf("preference = %v, want at most 1 and close to it", got[1].Preference) + } +} + +func TestAnEventFromTheFutureIsNewAndNothingMore(t *testing.T) { + future := now.Add(365 * 24 * time.Hour) + got := Preferences([]Event{{ArtistID: 1, Type: listening.Complete, At: future}}, now, options)[1] + if !got.LastListened.Equal(now) { + t.Errorf("last listened = %v, want now (%v): a clock that is wrong cannot make a listen later than now", got.LastListened, now) + } +} + func TestOneSkipDoesNotSinkAnArtist(t *testing.T) { got := Preferences([]Event{{ArtistID: 1, Type: listening.Skip, PlayedMs: 2_000, At: now}}, now, options)[1] if want := math.Tanh(-1.0 / 3); !near(got.Preference, want) {