sonda/internal/discover/service.go
tgiachi 69da67065d fix(discover): refusals do not wait for a turn, and waiting for one has an end
- The gate now comes after what only the database decides (a seed that is not an
  artist, no seed and no permission), so those answers are not held up by three
  slow Discovers and do not make the listener "busy".
- A request that waits more than 30 seconds for a turn answers 429 instead of
  hanging for as long as the providers are slow.
- The spec and the README say that a waiting request counts as running.
2026-11-12 12:00:18 +01:00

552 lines
16 KiB
Go

package discover
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"time"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/feedback"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/taste"
"codeberg.org/squidd/sonda/internal/usersettings"
)
// Limits.
const (
DefaultLimit = 20
MaxLimit = 30
// MaxSeeds is how many artists a listener may name.
MaxSeeds = 5
// MaxListeningSeeds is how many of the most played artists seed a request when
// the listener allowed it.
MaxListeningSeeds = 20
// Parallelism is how many provider requests are in flight at once. The provider
// client spaces them out anyway; this only keeps a request from queueing dozens.
Parallelism = 4
// minPool is how many candidates, best affinity first, are scored whatever the
// limit: the genre can reorder them, and each one without tags costs a request.
minPool = 20
// MaxConcurrent is how many Discovers run at the same time, whoever asks. A request that finds them
// all busy waits for its turn; one that stops waiting does no work.
MaxConcurrent = 3
// TurnWait is the longest a request waits for a turn: after that it is told it is busy, so that
// slow providers cannot hold a queue of requests open for as long as they like.
TurnWait = 30 * time.Second
// MaxBecause is how many seeds are named as the reason for a suggestion.
MaxBecause = 3
)
// Errors returned by the service.
var (
ErrInvalidQuery = errors.New("invalid query")
// ErrSeedRequired is returned when no seed was named and the listener did not
// allow Discover to start from their listening.
ErrSeedRequired = errors.New("name a seed artist, or allow Discover to start from your listening")
// ErrBusy is returned when the same listener already has a Discover running: one at a time, so that
// a client that keeps asking cannot line up work for the providers.
ErrBusy = errors.New("a Discover of yours is already running")
)
// Taster says what the listener likes.
type Taster interface {
Learn(ctx context.Context, userID int64) (taste.Learned, error)
}
// Similar gives the artists similar to one.
type Similar interface {
Similar(ctx context.Context, artistID int64, limit int) (discovery.Result, error)
}
// TagFetcher reads the tags of one artist from one provider into the database.
type TagFetcher interface {
ProviderTags(ctx context.Context, artistID int64, provider string) (discovery.TagSourceResult, error)
}
// Store reads what is already stored.
type Store interface {
// ArtistNames returns the names of the artists that exist.
ArtistNames(ctx context.Context, ids []int64) (map[int64]string, error)
// ArtistTags returns the stored tags per artist with weights from 0 to 1;
// artists without tags are left out.
ArtistTags(ctx context.Context, ids []int64) (map[int64]map[string]float64, error)
}
// Settings reads the listener's own settings.
type Settings interface {
Get(ctx context.Context, userID int64) (usersettings.Settings, error)
}
// Marks reads what the listener decided about artists.
type Marks interface {
Marks(ctx context.Context, userID int64) (map[int64]feedback.Kind, error)
}
// Query is a request for suggestions.
type Query struct {
Seeds []int64 // artists the listener names; none means "from my listening"
Limit int // zero means DefaultLimit
InLibrary *bool // keep only artists the listener has (true) or has not (false)
}
// Seed is an artist the suggestions start from.
type Seed struct {
ID int64
Name string
Weight float64 // 1 for a named seed, the preference for one taken from the listening
}
// Reason says a seed led to a suggestion, and how strongly the provider links them.
type Reason struct {
Artist discovery.ArtistRef
Match float64
}
// Item is one suggestion with the parts of its score.
type Item struct {
Artist discovery.ArtistRef
InLibrary bool
Wishlisted bool // the listener put it on their wishlist
Score float64
Components Components
Because []Reason
}
// Result is the answer to Discover.
type Result struct {
// ProviderStatus is "disabled" when the provider is off or has no key (what was
// stored earlier is still used), "error" when it failed for some seed, else "ok".
ProviderStatus string
FromListening bool // the seeds were taken from the listening, not named
Seeds []Seed
Items []Item
}
// Service scores suggestions.
type Service struct {
taste Taster
similar Similar
tags TagFetcher
store Store
settings Settings
marks Marks
weights Weights
mu sync.Mutex
running map[int64]bool // the listeners who have a Discover running or waiting for its turn
turns chan struct{} // MaxConcurrent places
turnWait time.Duration // how long a request waits for a turn before it is told it is busy
}
// NewService builds a Service.
func NewService(t Taster, sim Similar, tags TagFetcher, store Store, settings Settings, marks Marks, w Weights) *Service {
return &Service{
taste: t, similar: sim, tags: tags, store: store, settings: settings, marks: marks, weights: w,
running: map[int64]bool{}, turns: make(chan struct{}, MaxConcurrent), turnWait: TurnWait,
}
}
// enter takes the place of the listener and a turn, and returns what gives them back. A listener who
// has a Discover already is refused at once; one who leaves while waiting for a turn, or waits for
// TurnWait without getting one, gives up the place.
func (s *Service) enter(ctx context.Context, userID int64) (func(), error) {
s.mu.Lock()
if s.running[userID] {
s.mu.Unlock()
return nil, ErrBusy
}
s.running[userID] = true
s.mu.Unlock()
free := func() {
s.mu.Lock()
delete(s.running, userID)
s.mu.Unlock()
}
wait := time.NewTimer(s.turnWait)
defer wait.Stop()
select {
case s.turns <- struct{}{}:
return func() { <-s.turns; free() }, nil
case <-ctx.Done():
free()
return nil, ctx.Err()
case <-wait.C:
free()
return nil, ErrBusy
}
}
// Discover suggests artists. Seeds are the artists the listener names, or, when none
// is named and the listener allowed it, the ones they play most. For each seed the
// stored similar artists are used, and asked from the provider only when the stored
// ones are missing or old (see discovery.Service). The tags of the best candidates
// that have none are read from Last.fm only.
func (s *Service) Discover(ctx context.Context, userID int64, q Query) (Result, error) {
limit, err := checkLimit(q.Limit)
if err != nil {
return Result{}, err
}
named, err := s.checkSeeds(ctx, q.Seeds)
if err != nil {
return Result{}, err
}
res := Result{ProviderStatus: discovery.StatusOK, Seeds: []Seed{}, Items: []Item{}}
seeds := named
if len(named) == 0 {
// Ask permission before doing any work for a request that may be refused.
set, err := s.settings.Get(ctx, userID)
if err != nil {
return Result{}, err
}
if !set.DiscoverFromListening {
return Result{}, ErrSeedRequired
}
res.FromListening = true
}
// What is refused above costs the database and nothing else, so it does not wait for a turn. From
// here on the providers may be asked.
leave, err := s.enter(ctx, userID)
if err != nil {
return Result{}, err
}
defer leave()
marks, err := s.marks.Marks(ctx, userID)
if err != nil {
return Result{}, fmt.Errorf("read marks: %w", err)
}
learned, err := s.taste.Learn(ctx, userID)
if err != nil {
return Result{}, err
}
if res.FromListening {
for _, p := range learned.Liked {
if len(seeds) == MaxListeningSeeds {
break
}
if marks[p.ArtistID] == feedback.Dismissed {
continue // the listener said they are not interested
}
seeds = append(seeds, Seed{ID: p.ArtistID, Weight: p.Preference})
}
if err := s.nameSeeds(ctx, seeds); err != nil {
return Result{}, err
}
}
if len(seeds) == 0 {
return res, nil
}
lists, status, err := s.readSimilar(ctx, seeds)
if err != nil {
return Result{}, err
}
res.ProviderStatus = status
seeds, lists = withoutUnread(seeds, lists)
res.Seeds = append(res.Seeds, seeds...)
cands := gather(seeds, lists, q.InLibrary, marks)
pool := rankByAffinity(cands, max(limit, minPool))
tags, err := s.candidateTags(ctx, pool, len(learned.Tags) > 0)
if err != nil {
return Result{}, err
}
profile := make(map[string]float64, len(learned.Tags))
for _, t := range learned.Tags {
profile[t.Tag] = t.Weight
}
names := map[int64]string{}
for _, sd := range seeds {
names[sd.ID] = sd.Name
}
for _, c := range pool {
pref, known := learned.Prefs[c.id]
comp := Components{A: Affinity(c.links), P: PreferenceScore(pref.Preference, known), D: DiscoveryBonus(pref.Listens)}
if g, ok := Cosine(tags[c.id], profile); ok {
comp.G = &g
}
res.Items = append(res.Items, Item{
Artist: discovery.ArtistRef{ID: c.id, Name: c.name}, InLibrary: c.inLibrary,
Wishlisted: marks[c.id] == feedback.Wishlist,
Score: Combine(comp, s.weights), Components: comp, Because: because(c.links, names),
})
}
sort.SliceStable(res.Items, func(i, j int) bool {
a, b := res.Items[i], res.Items[j]
switch {
case a.Score != b.Score:
return a.Score > b.Score
case a.Components.A != b.Components.A:
return a.Components.A > b.Components.A
default:
return a.Artist.ID < b.Artist.ID
}
})
if len(res.Items) > limit {
res.Items = res.Items[:limit]
}
return res, nil
}
func checkLimit(limit int) (int, error) {
switch {
case limit < 0:
return 0, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery)
case limit == 0:
return DefaultLimit, nil
case limit > MaxLimit:
return MaxLimit, nil
}
return limit, nil
}
// checkSeeds refuses too many seeds and ids that are not artists, and drops repeats.
func (s *Service) checkSeeds(ctx context.Context, in []int64) ([]Seed, error) {
if len(in) > MaxSeeds {
return nil, fmt.Errorf("%w: at most %d seeds", ErrInvalidQuery, MaxSeeds)
}
var out []Seed
seen := map[int64]bool{}
for _, id := range in {
if id <= 0 {
return nil, fmt.Errorf("%w: %d is not an artist id", ErrInvalidQuery, id)
}
if !seen[id] {
seen[id] = true
out = append(out, Seed{ID: id, Weight: 1})
}
}
if len(out) == 0 {
return nil, nil
}
if err := s.nameSeeds(ctx, out); err != nil {
return nil, err
}
for _, sd := range out {
if sd.Name == "" {
return nil, fmt.Errorf("%w: %d is not an artist", ErrInvalidQuery, sd.ID)
}
}
return out, nil
}
func (s *Service) nameSeeds(ctx context.Context, seeds []Seed) error {
ids := make([]int64, len(seeds))
for i, sd := range seeds {
ids[i] = sd.ID
}
names, err := s.store.ArtistNames(ctx, ids)
if err != nil {
return fmt.Errorf("read names: %w", err)
}
for i := range seeds {
seeds[i].Name = names[seeds[i].ID]
}
return nil
}
// readSimilar reads the similar artists of every seed, Parallelism at a time. A seed
// that was removed in the meantime is skipped; any other error stops the request.
func (s *Service) readSimilar(ctx context.Context, seeds []Seed) ([]*discovery.Result, string, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
results := make([]*discovery.Result, len(seeds)) // nil for a seed that could not be read
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
gate = make(chan struct{}, Parallelism)
)
for i, sd := range seeds {
wg.Add(1)
go func() {
defer wg.Done()
select {
case gate <- struct{}{}:
defer func() { <-gate }()
case <-ctx.Done():
return
}
r, err := s.similar.Similar(ctx, sd.ID, discovery.FetchLimit)
if errors.Is(err, discovery.ErrNotFound) {
return
}
if err != nil {
mu.Lock()
if firstErr == nil {
firstErr = err
cancel()
}
mu.Unlock()
return
}
results[i] = &r
}()
}
wg.Wait()
if firstErr != nil {
return nil, "", firstErr
}
if err := ctx.Err(); err != nil {
return nil, "", err
}
status := discovery.StatusOK
for _, r := range results {
if r == nil {
continue
}
switch r.ProviderStatus {
case discovery.StatusDisabled:
status = discovery.StatusDisabled
case discovery.StatusError:
if status != discovery.StatusDisabled {
status = discovery.StatusError
}
}
}
return results, status, nil
}
// withoutUnread drops the seeds that could not be read, with their lists, so that
// they are not reported as the reason for anything.
func withoutUnread(seeds []Seed, lists []*discovery.Result) ([]Seed, []*discovery.Result) {
var keptSeeds []Seed
var keptLists []*discovery.Result
for i, sd := range seeds {
if lists[i] != nil {
keptSeeds, keptLists = append(keptSeeds, sd), append(keptLists, lists[i])
}
}
return keptSeeds, keptLists
}
type candidate struct {
id int64
name string
inLibrary bool
links []Link
}
// gather turns the similar lists into candidates. A seed is never suggested, nor
// an artist the listener dismissed, and the library filter is applied here.
func gather(seeds []Seed, lists []*discovery.Result, inLibrary *bool, marks map[int64]feedback.Kind) []*candidate {
isSeed := map[int64]bool{}
for _, sd := range seeds {
isSeed[sd.ID] = true
}
byID := map[int64]*candidate{}
var order []*candidate
for i, sd := range seeds {
for _, it := range lists[i].Items {
if isSeed[it.Artist.ID] || marks[it.Artist.ID] == feedback.Dismissed || (inLibrary != nil && it.InLibrary != *inLibrary) {
continue
}
c := byID[it.Artist.ID]
if c == nil {
c = &candidate{id: it.Artist.ID, name: it.Artist.Name, inLibrary: it.InLibrary}
byID[c.id] = c
order = append(order, c)
}
c.links = append(c.links, Link{Seed: sd.ID, Weight: sd.Weight, Match: it.Score})
}
}
return order
}
// rankByAffinity keeps the n candidates with the best affinity (ties by id).
func rankByAffinity(cands []*candidate, n int) []*candidate {
aff := make(map[int64]float64, len(cands))
for _, c := range cands {
aff[c.id] = Affinity(c.links)
}
sort.SliceStable(cands, func(i, j int) bool {
if aff[cands[i].id] != aff[cands[j].id] {
return aff[cands[i].id] > aff[cands[j].id]
}
return cands[i].id < cands[j].id
})
if len(cands) > n {
cands = cands[:n]
}
return cands
}
// candidateTags returns the stored tags of the pool. A candidate with none gets its
// Last.fm tags read first (only when there is a profile to compare them with), and
// a provider that is off or fails just leaves its genre unknown.
func (s *Service) candidateTags(ctx context.Context, pool []*candidate, haveProfile bool) (map[int64]map[string]float64, error) {
ids := make([]int64, len(pool))
for i, c := range pool {
ids[i] = c.id
}
tags, err := s.store.ArtistTags(ctx, ids)
if err != nil {
return nil, fmt.Errorf("read tags: %w", err)
}
if !haveProfile {
return tags, nil
}
var missing []int64
for _, id := range ids {
if len(tags[id]) == 0 {
missing = append(missing, id)
}
}
if len(missing) == 0 {
return tags, nil
}
var wg sync.WaitGroup
gate := make(chan struct{}, Parallelism)
for _, id := range missing {
wg.Add(1)
go func() {
defer wg.Done()
select {
case gate <- struct{}{}:
defer func() { <-gate }()
case <-ctx.Done():
return
}
_, _ = s.tags.ProviderTags(ctx, id, providers.LastFM) // a failure leaves the genre out
}()
}
wg.Wait()
if err := ctx.Err(); err != nil {
return nil, err
}
again, err := s.store.ArtistTags(ctx, missing)
if err != nil {
return nil, fmt.Errorf("read tags: %w", err)
}
for id, t := range again {
tags[id] = t
}
return tags, nil
}
// because names the seeds that count most for a candidate, strongest first.
func because(links []Link, names map[int64]string) []Reason {
sorted := append([]Link(nil), links...)
sort.SliceStable(sorted, func(i, j int) bool {
a, b := sorted[i].Weight*sorted[i].Match, sorted[j].Weight*sorted[j].Match
if a != b {
return a > b
}
return sorted[i].Seed < sorted[j].Seed
})
if len(sorted) > MaxBecause {
sorted = sorted[:MaxBecause]
}
out := make([]Reason, len(sorted))
for i, l := range sorted {
out[i] = Reason{Artist: discovery.ArtistRef{ID: l.Seed, Name: names[l.Seed]}, Match: l.Match}
}
return out
}