- PUT/DELETE /wishlist/{artist} and GET /wishlist: the artists I want.
- POST/DELETE /discover/{artist}/dismiss and GET /discover/dismissed: the
artists I am not interested in. An artist is wished or dismissed, never both,
and both lists are mine alone.
- Discover never suggests a dismissed artist, does not use one as a seed taken
from my listening, and marks the wished ones it shows (wishlisted).
- New artist_feedback table (migration 0012, SQLite and PostgreSQL), removed
with the account.
- An artist I marked is not cleaned up when the list that named it is replaced,
and goes when its last mark is taken off and nobody names it. Taking a
provider's data back still removes what only it named, marks included.
- Spec, contract test and README updated.
Closes #35
497 lines
14 KiB
Go
497 lines
14 KiB
Go
package discover
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/feedback"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
"codeberg.org/squidd/sonda/internal/taste"
|
|
"codeberg.org/squidd/sonda/internal/usersettings"
|
|
)
|
|
|
|
// Limits.
|
|
const (
|
|
DefaultLimit = 20
|
|
MaxLimit = 30
|
|
|
|
// MaxSeeds is how many artists a listener may name.
|
|
MaxSeeds = 5
|
|
// MaxListeningSeeds is how many of the most played artists seed a request when
|
|
// the listener allowed it.
|
|
MaxListeningSeeds = 20
|
|
// Parallelism is how many provider requests are in flight at once. The provider
|
|
// client spaces them out anyway; this only keeps a request from queueing dozens.
|
|
Parallelism = 4
|
|
// minPool is how many candidates, best affinity first, are scored whatever the
|
|
// limit: the genre can reorder them, and each one without tags costs a request.
|
|
minPool = 20
|
|
// MaxBecause is how many seeds are named as the reason for a suggestion.
|
|
MaxBecause = 3
|
|
)
|
|
|
|
// Errors returned by the service.
|
|
var (
|
|
ErrInvalidQuery = errors.New("invalid query")
|
|
// ErrSeedRequired is returned when no seed was named and the listener did not
|
|
// allow Discover to start from their listening.
|
|
ErrSeedRequired = errors.New("name a seed artist, or allow Discover to start from your listening")
|
|
)
|
|
|
|
// Taster says what the listener likes.
|
|
type Taster interface {
|
|
Learn(ctx context.Context, userID int64) (taste.Learned, error)
|
|
}
|
|
|
|
// Similar gives the artists similar to one.
|
|
type Similar interface {
|
|
Similar(ctx context.Context, artistID int64, limit int) (discovery.Result, error)
|
|
}
|
|
|
|
// TagFetcher reads the tags of one artist from one provider into the database.
|
|
type TagFetcher interface {
|
|
ProviderTags(ctx context.Context, artistID int64, provider string) (discovery.TagSourceResult, error)
|
|
}
|
|
|
|
// Store reads what is already stored.
|
|
type Store interface {
|
|
// ArtistNames returns the names of the artists that exist.
|
|
ArtistNames(ctx context.Context, ids []int64) (map[int64]string, error)
|
|
// ArtistTags returns the stored tags per artist with weights from 0 to 1;
|
|
// artists without tags are left out.
|
|
ArtistTags(ctx context.Context, ids []int64) (map[int64]map[string]float64, error)
|
|
}
|
|
|
|
// Settings reads the listener's own settings.
|
|
type Settings interface {
|
|
Get(ctx context.Context, userID int64) (usersettings.Settings, error)
|
|
}
|
|
|
|
// Marks reads what the listener decided about artists.
|
|
type Marks interface {
|
|
Marks(ctx context.Context, userID int64) (map[int64]feedback.Kind, error)
|
|
}
|
|
|
|
// Query is a request for suggestions.
|
|
type Query struct {
|
|
Seeds []int64 // artists the listener names; none means "from my listening"
|
|
Limit int // zero means DefaultLimit
|
|
InLibrary *bool // keep only artists the listener has (true) or has not (false)
|
|
}
|
|
|
|
// Seed is an artist the suggestions start from.
|
|
type Seed struct {
|
|
ID int64
|
|
Name string
|
|
Weight float64 // 1 for a named seed, the preference for one taken from the listening
|
|
}
|
|
|
|
// Reason says a seed led to a suggestion, and how strongly the provider links them.
|
|
type Reason struct {
|
|
Artist discovery.ArtistRef
|
|
Match float64
|
|
}
|
|
|
|
// Item is one suggestion with the parts of its score.
|
|
type Item struct {
|
|
Artist discovery.ArtistRef
|
|
InLibrary bool
|
|
Wishlisted bool // the listener put it on their wishlist
|
|
Score float64
|
|
Components Components
|
|
Because []Reason
|
|
}
|
|
|
|
// Result is the answer to Discover.
|
|
type Result struct {
|
|
// ProviderStatus is "disabled" when the provider is off or has no key (what was
|
|
// stored earlier is still used), "error" when it failed for some seed, else "ok".
|
|
ProviderStatus string
|
|
FromListening bool // the seeds were taken from the listening, not named
|
|
Seeds []Seed
|
|
Items []Item
|
|
}
|
|
|
|
// Service scores suggestions.
|
|
type Service struct {
|
|
taste Taster
|
|
similar Similar
|
|
tags TagFetcher
|
|
store Store
|
|
settings Settings
|
|
marks Marks
|
|
weights Weights
|
|
}
|
|
|
|
// NewService builds a Service.
|
|
func NewService(t Taster, sim Similar, tags TagFetcher, store Store, settings Settings, marks Marks, w Weights) *Service {
|
|
return &Service{taste: t, similar: sim, tags: tags, store: store, settings: settings, marks: marks, weights: w}
|
|
}
|
|
|
|
// Discover suggests artists. Seeds are the artists the listener names, or, when none
|
|
// is named and the listener allowed it, the ones they play most. For each seed the
|
|
// stored similar artists are used, and asked from the provider only when the stored
|
|
// ones are missing or old (see discovery.Service). The tags of the best candidates
|
|
// that have none are read from Last.fm only.
|
|
func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, error) {
|
|
limit, err := checkLimit(q.Limit)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
named, err := s.checkSeeds(ctx, q.Seeds)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
res := Result{ProviderStatus: discovery.StatusOK, Seeds: []Seed{}, Items: []Item{}}
|
|
seeds := named
|
|
if len(named) == 0 {
|
|
// Ask permission before doing any work for a request that may be refused.
|
|
set, err := s.settings.Get(ctx, userID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if !set.DiscoverFromListening {
|
|
return Result{}, ErrSeedRequired
|
|
}
|
|
res.FromListening = true
|
|
}
|
|
|
|
marks, err := s.marks.Marks(ctx, userID)
|
|
if err != nil {
|
|
return Result{}, fmt.Errorf("read marks: %w", err)
|
|
}
|
|
learned, err := s.taste.Learn(ctx, userID)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if res.FromListening {
|
|
for _, p := range learned.Liked {
|
|
if len(seeds) == MaxListeningSeeds {
|
|
break
|
|
}
|
|
if marks[p.ArtistID] == feedback.Dismissed {
|
|
continue // the listener said they are not interested
|
|
}
|
|
seeds = append(seeds, Seed{ID: p.ArtistID, Weight: p.Preference})
|
|
}
|
|
if err := s.nameSeeds(ctx, seeds); err != nil {
|
|
return Result{}, err
|
|
}
|
|
}
|
|
if len(seeds) == 0 {
|
|
return res, nil
|
|
}
|
|
|
|
lists, status, err := s.readSimilar(ctx, seeds)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
res.ProviderStatus = status
|
|
seeds, lists = withoutUnread(seeds, lists)
|
|
res.Seeds = append(res.Seeds, seeds...)
|
|
|
|
cands := gather(seeds, lists, q.InLibrary, marks)
|
|
pool := rankByAffinity(cands, max(limit, minPool))
|
|
tags, err := s.candidateTags(ctx, pool, len(learned.Tags) > 0)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
profile := make(map[string]float64, len(learned.Tags))
|
|
for _, t := range learned.Tags {
|
|
profile[t.Tag] = t.Weight
|
|
}
|
|
|
|
names := map[int64]string{}
|
|
for _, sd := range seeds {
|
|
names[sd.ID] = sd.Name
|
|
}
|
|
for _, c := range pool {
|
|
pref, known := learned.Prefs[c.id]
|
|
comp := Components{A: Affinity(c.links), P: PreferenceScore(pref.Preference, known), D: DiscoveryBonus(pref.Listens)}
|
|
if g, ok := Cosine(tags[c.id], profile); ok {
|
|
comp.G = &g
|
|
}
|
|
res.Items = append(res.Items, Item{
|
|
Artist: discovery.ArtistRef{ID: c.id, Name: c.name}, InLibrary: c.inLibrary,
|
|
Wishlisted: marks[c.id] == feedback.Wishlist,
|
|
Score: Combine(comp, s.weights), Components: comp, Because: because(c.links, names),
|
|
})
|
|
}
|
|
sort.SliceStable(res.Items, func(i, j int) bool {
|
|
a, b := res.Items[i], res.Items[j]
|
|
switch {
|
|
case a.Score != b.Score:
|
|
return a.Score > b.Score
|
|
case a.Components.A != b.Components.A:
|
|
return a.Components.A > b.Components.A
|
|
default:
|
|
return a.Artist.ID < b.Artist.ID
|
|
}
|
|
})
|
|
if len(res.Items) > limit {
|
|
res.Items = res.Items[:limit]
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
func checkLimit(limit int) (int, error) {
|
|
switch {
|
|
case limit < 0:
|
|
return 0, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery)
|
|
case limit == 0:
|
|
return DefaultLimit, nil
|
|
case limit > MaxLimit:
|
|
return MaxLimit, nil
|
|
}
|
|
return limit, nil
|
|
}
|
|
|
|
// checkSeeds refuses too many seeds and ids that are not artists, and drops repeats.
|
|
func (s *Service) checkSeeds(ctx context.Context, in []int64) ([]Seed, error) {
|
|
if len(in) > MaxSeeds {
|
|
return nil, fmt.Errorf("%w: at most %d seeds", ErrInvalidQuery, MaxSeeds)
|
|
}
|
|
var out []Seed
|
|
seen := map[int64]bool{}
|
|
for _, id := range in {
|
|
if id <= 0 {
|
|
return nil, fmt.Errorf("%w: %d is not an artist id", ErrInvalidQuery, id)
|
|
}
|
|
if !seen[id] {
|
|
seen[id] = true
|
|
out = append(out, Seed{ID: id, Weight: 1})
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil, nil
|
|
}
|
|
if err := s.nameSeeds(ctx, out); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, sd := range out {
|
|
if sd.Name == "" {
|
|
return nil, fmt.Errorf("%w: %d is not an artist", ErrInvalidQuery, sd.ID)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Service) nameSeeds(ctx context.Context, seeds []Seed) error {
|
|
ids := make([]int64, len(seeds))
|
|
for i, sd := range seeds {
|
|
ids[i] = sd.ID
|
|
}
|
|
names, err := s.store.ArtistNames(ctx, ids)
|
|
if err != nil {
|
|
return fmt.Errorf("read names: %w", err)
|
|
}
|
|
for i := range seeds {
|
|
seeds[i].Name = names[seeds[i].ID]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readSimilar reads the similar artists of every seed, Parallelism at a time. A seed
|
|
// that was removed in the meantime is skipped; any other error stops the request.
|
|
func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]*discovery.Result, string, error) {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
results := make([]*discovery.Result, len(seeds)) // nil for a seed that could not be read
|
|
var (
|
|
wg sync.WaitGroup
|
|
mu sync.Mutex
|
|
firstErr error
|
|
gate = make(chan struct{}, Parallelism)
|
|
)
|
|
for i, sd := range seeds {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
select {
|
|
case gate <- struct{}{}:
|
|
defer func() { <-gate }()
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
r, err := s.similar.Similar(ctx, sd.ID, discovery.FetchLimit)
|
|
if errors.Is(err, discovery.ErrNotFound) {
|
|
return
|
|
}
|
|
if err != nil {
|
|
mu.Lock()
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
cancel()
|
|
}
|
|
mu.Unlock()
|
|
return
|
|
}
|
|
results[i] = &r
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if firstErr != nil {
|
|
return nil, "", firstErr
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, "", err
|
|
}
|
|
status := discovery.StatusOK
|
|
for _, r := range results {
|
|
if r == nil {
|
|
continue
|
|
}
|
|
switch r.ProviderStatus {
|
|
case discovery.StatusDisabled:
|
|
status = discovery.StatusDisabled
|
|
case discovery.StatusError:
|
|
if status != discovery.StatusDisabled {
|
|
status = discovery.StatusError
|
|
}
|
|
}
|
|
}
|
|
return results, status, nil
|
|
}
|
|
|
|
// withoutUnread drops the seeds that could not be read, with their lists, so that
|
|
// they are not reported as the reason for anything.
|
|
func withoutUnread(seeds []Seed, lists []*discovery.Result) ([]Seed, []*discovery.Result) {
|
|
var keptSeeds []Seed
|
|
var keptLists []*discovery.Result
|
|
for i, sd := range seeds {
|
|
if lists[i] != nil {
|
|
keptSeeds, keptLists = append(keptSeeds, sd), append(keptLists, lists[i])
|
|
}
|
|
}
|
|
return keptSeeds, keptLists
|
|
}
|
|
|
|
type candidate struct {
|
|
id int64
|
|
name string
|
|
inLibrary bool
|
|
links []Link
|
|
}
|
|
|
|
// gather turns the similar lists into candidates. A seed is never suggested, nor
|
|
// an artist the listener dismissed, and the library filter is applied here.
|
|
func gather(seeds []Seed, lists []*discovery.Result, inLibrary *bool, marks map[int64]feedback.Kind) []*candidate {
|
|
isSeed := map[int64]bool{}
|
|
for _, sd := range seeds {
|
|
isSeed[sd.ID] = true
|
|
}
|
|
byID := map[int64]*candidate{}
|
|
var order []*candidate
|
|
for i, sd := range seeds {
|
|
for _, it := range lists[i].Items {
|
|
if isSeed[it.Artist.ID] || marks[it.Artist.ID] == feedback.Dismissed || (inLibrary != nil && it.InLibrary != *inLibrary) {
|
|
continue
|
|
}
|
|
c := byID[it.Artist.ID]
|
|
if c == nil {
|
|
c = &candidate{id: it.Artist.ID, name: it.Artist.Name, inLibrary: it.InLibrary}
|
|
byID[c.id] = c
|
|
order = append(order, c)
|
|
}
|
|
c.links = append(c.links, Link{Seed: sd.ID, Weight: sd.Weight, Match: it.Score})
|
|
}
|
|
}
|
|
return order
|
|
}
|
|
|
|
// rankByAffinity keeps the n candidates with the best affinity (ties by id).
|
|
func rankByAffinity(cands []*candidate, n int) []*candidate {
|
|
aff := make(map[int64]float64, len(cands))
|
|
for _, c := range cands {
|
|
aff[c.id] = Affinity(c.links)
|
|
}
|
|
sort.SliceStable(cands, func(i, j int) bool {
|
|
if aff[cands[i].id] != aff[cands[j].id] {
|
|
return aff[cands[i].id] > aff[cands[j].id]
|
|
}
|
|
return cands[i].id < cands[j].id
|
|
})
|
|
if len(cands) > n {
|
|
cands = cands[:n]
|
|
}
|
|
return cands
|
|
}
|
|
|
|
// candidateTags returns the stored tags of the pool. A candidate with none gets its
|
|
// Last.fm tags read first (only when there is a profile to compare them with), and
|
|
// a provider that is off or fails just leaves its genre unknown.
|
|
func (s *Service) candidateTags(ctx context.Context, pool []*candidate, haveProfile bool) (map[int64]map[string]float64, error) {
|
|
ids := make([]int64, len(pool))
|
|
for i, c := range pool {
|
|
ids[i] = c.id
|
|
}
|
|
tags, err := s.store.ArtistTags(ctx, ids)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read tags: %w", err)
|
|
}
|
|
if !haveProfile {
|
|
return tags, nil
|
|
}
|
|
var missing []int64
|
|
for _, id := range ids {
|
|
if len(tags[id]) == 0 {
|
|
missing = append(missing, id)
|
|
}
|
|
}
|
|
if len(missing) == 0 {
|
|
return tags, nil
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
gate := make(chan struct{}, Parallelism)
|
|
for _, id := range missing {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
select {
|
|
case gate <- struct{}{}:
|
|
defer func() { <-gate }()
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
_, _ = s.tags.ProviderTags(ctx, id, providers.LastFM) // a failure leaves the genre out
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
again, err := s.store.ArtistTags(ctx, missing)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read tags: %w", err)
|
|
}
|
|
for id, t := range again {
|
|
tags[id] = t
|
|
}
|
|
return tags, nil
|
|
}
|
|
|
|
// because names the seeds that count most for a candidate, strongest first.
|
|
func because(links []Link, names map[int64]string) []Reason {
|
|
sorted := append([]Link(nil), links...)
|
|
sort.SliceStable(sorted, func(i, j int) bool {
|
|
a, b := sorted[i].Weight*sorted[i].Match, sorted[j].Weight*sorted[j].Match
|
|
if a != b {
|
|
return a > b
|
|
}
|
|
return sorted[i].Seed < sorted[j].Seed
|
|
})
|
|
if len(sorted) > MaxBecause {
|
|
sorted = sorted[:MaxBecause]
|
|
}
|
|
out := make([]Reason, len(sorted))
|
|
for i, l := range sorted {
|
|
out[i] = Reason{Artist: discovery.ArtistRef{ID: l.Seed, Name: names[l.Seed]}, Match: l.Match}
|
|
}
|
|
return out
|
|
}
|