sonda/internal/discovery/discovery.go
tgiachi b0a7a562a6 fix(discovery): purge lock, cleanup chains and rescans that cannot read
- fix: on PostgreSQL a purge could delete a discovered artist while a scan was
  adopting it. The scan then failed on the missing artist for the whole file,
  and the two could also deadlock. Purge now takes the same artist lock as the
  scanner and discovery before it removes anything. Saving a list or tags does
  too, and an artist that is gone by then is reported as not found instead of a
  foreign key error, so a user who opened an artist that was removed meanwhile
  gets a 404
- fix: a discovered artist that was only the source of a list stayed forever and
  kept the artists of its own list alive, so chains built up. Cleanup now removes
  such an artist and then follows the chain until nothing more goes
- fix: in a full rescan a file that came back but could not be read stayed
  marked missing. It is restored first, as in a normal scan, and read as well
- fix: ?full given twice is refused, so a typo cannot start a scan of another
  kind
- fix: genre cleaning lives in one place, used by the reader and the store
- docs: the files genres count the tracks the artist is credited on, and the spec
  says every files tag is a genre
- test: a purge colliding with a scan on both databases (it fails without the
  lock), a chain of discovered artists, saving for an artist that is gone, a
  file that returns unreadable during a full scan
2026-11-11 17:36:14 +01:00

272 lines
8.4 KiB
Go

// Package discovery finds music outside the library: artists similar to one the
// user already has.
//
// Data comes from an external provider (Last.fm today), only when the admin
// switched that provider on, and only when someone asks. What was read is kept
// in the database, so a second look costs nothing and nothing leaves the server
// again until the data is old. The raw numbers of the provider are stored as
// they are: turning them into a score is a later step.
package discovery
import (
"context"
"errors"
"fmt"
"strings"
"time"
"go.uber.org/zap"
"golang.org/x/sync/singleflight"
"codeberg.org/squidd/sonda/internal/providers"
"codeberg.org/squidd/sonda/internal/providers/lastfm"
)
// Provider is the provider similar artists come from.
const Provider = providers.LastFM
// Attribution is what has to be shown with the data: Last.fm asks for credit and
// allows non-commercial use only.
const Attribution = "Similar artists from Last.fm (https://www.last.fm), for non-commercial use."
// How long what was read is believed. Last.fm sends no cache headers, so these
// are our own choice: similarity moves slowly, and a "no such artist" answer is
// remembered shortly in case it was a typo that was since fixed.
const (
PositiveTTL = 7 * 24 * time.Hour
NegativeTTL = 24 * time.Hour
)
// FetchLimit is how many similar artists are read from the provider each time,
// whatever size of page was asked for, so that one answer serves every page.
const FetchLimit = 50
// Bounds of the number of items returned. Only FetchLimit are ever stored, so no
// page can ask for more.
const (
DefaultLimit = 20
MaxLimit = FetchLimit
)
// fetchTimeout bounds a shared provider call, which does not depend on whoever
// started it still waiting.
const fetchTimeout = time.Minute
// Errors returned by the service.
var (
ErrNotFound = errors.New("artist not found")
ErrInvalidQuery = errors.New("invalid query")
)
// Provider states reported with an answer.
const (
StatusOK = "ok" // the data is as fresh as it needs to be
StatusDisabled = "disabled" // the provider is off or has no key, so it was not asked
StatusError = "error" // the provider was asked and failed
)
// Artist is who is being looked up.
type Artist struct {
ID int64
Name string
MBID string // empty when unknown
}
// ArtistRef is the short form of an artist.
type ArtistRef struct {
ID int64
Name string
}
// Item is one similar artist. Score is the provider's own number, relative to
// the list it came in, and Rank is the position in that list, starting at 1.
type Item struct {
Artist ArtistRef
InLibrary bool // has something playable in this library
Score float64
Rank int
}
// Snapshot is what is stored for one artist and one provider. Fetched is false
// when the provider was never asked; NotFound when it said it does not know the
// artist (or knew nothing similar).
type Snapshot struct {
Fetched bool
NotFound bool
RetrievedAt time.Time
Items []Item
}
// Result is the answer to Similar. RetrievedAt is zero when nothing is known.
// Stale means the items are older than they should be because the provider could
// not be asked.
type Result struct {
Provider string
ProviderStatus string
Stale bool
RetrievedAt time.Time
Items []Item
}
// Store keeps what the providers said.
type Store interface {
// Artist returns ErrNotFound for an id the catalog does not know.
Artist(ctx context.Context, id int64) (Artist, error)
// Similar reads the stored items (at most limit, best first) and when they were read.
Similar(ctx context.Context, id int64, provider string, limit int) (Snapshot, error)
// SaveSimilar replaces what was stored for the artist. The candidates become
// artists without files unless one is already known.
SaveSimilar(ctx context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error
}
// Source asks the provider.
type Source interface {
SimilarArtists(ctx context.Context, key string, a lastfm.Artist, limit int) ([]lastfm.Similar, error)
}
// Settings says whether a provider may be used right now.
type Settings interface {
Active(ctx context.Context, name string) (providers.Settings, bool, error)
}
// Service answers "who is similar to this artist".
type Service struct {
store Store
source Source
settings Settings
log *zap.Logger
now func() time.Time
flight singleflight.Group
}
// NewService builds a Service.
func NewService(store Store, source Source, settings Settings, log *zap.Logger) *Service {
if log == nil {
log = zap.NewNop()
}
return &Service{store: store, source: source, settings: settings, log: log, now: time.Now}
}
// Similar returns the artists similar to artistID, at most limit of them
// (DefaultLimit for 0). It asks the provider only when the stored data is
// missing or old and the provider is on. A provider that fails is not an error
// for the caller: the answer says so, and carries what was stored, if anything.
func (s *Service) Similar(ctx context.Context, artistID int64, limit int) (Result, error) {
switch {
case limit < 0:
return Result{}, fmt.Errorf("%w: limit cannot be negative", ErrInvalidQuery)
case limit == 0:
limit = DefaultLimit
case limit > MaxLimit:
limit = MaxLimit
}
if artistID <= 0 {
return Result{}, ErrNotFound
}
artist, err := s.store.Artist(ctx, artistID)
if err != nil {
return Result{}, err
}
snap, err := s.store.Similar(ctx, artistID, Provider, limit)
if err != nil {
return Result{}, err
}
if snap.Fetched && s.fresh(snap) {
return result(snap, StatusOK, false), nil
}
cfg, active, err := s.settings.Active(ctx, Provider)
if err != nil {
return Result{}, err
}
if !active {
return result(snap, StatusDisabled, snap.Fetched), nil
}
// One call per artist, however many ask at once. It runs on its own context,
// so the first caller leaving does not fail the others; each caller still
// waits only as long as its own context allows.
done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) {
// singleflight re-raises a panic of the shared call in a goroutine nobody
// can recover, which would end the server: turn it into a failure.
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("reading similar artists panicked: %v", r)
}
}()
return nil, s.refresh(artist, cfg.APIKey)
})
select {
case <-ctx.Done():
return Result{}, ctx.Err()
case r := <-done:
if errors.Is(r.Err, ErrNotFound) {
return Result{}, ErrNotFound // removed while it was being read
}
if r.Err != nil {
s.log.Warn("similar artists could not be read", zap.String("provider", Provider), zap.Error(r.Err))
return result(snap, StatusError, snap.Fetched), nil
}
}
snap, err = s.store.Similar(ctx, artistID, Provider, limit)
if err != nil {
return Result{}, err
}
return result(snap, StatusOK, false), nil
}
func (s *Service) fresh(snap Snapshot) bool {
return fresh(s.now(), snap.RetrievedAt, snap.NotFound)
}
// fresh says whether what was read at retrievedAt is still believed.
func fresh(now, retrievedAt time.Time, notFound bool) bool {
ttl := PositiveTTL
if notFound {
ttl = NegativeTTL
}
return now.Sub(retrievedAt) < ttl
}
// refresh reads the provider and stores what it said.
func (s *Service) refresh(a Artist, key string) error {
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
defer cancel()
got, err := s.source.SimilarArtists(ctx, key, lastfm.Artist{Name: a.Name, MBID: a.MBID}, FetchLimit)
switch {
case errors.Is(err, lastfm.ErrNotFound):
got = nil
case err != nil:
return err
}
cands := withoutArtist(got, a)
return s.store.SaveSimilar(ctx, a.ID, Provider, s.now(), len(cands) > 0, cands)
}
// withoutArtist drops the artist itself, which providers sometimes list under
// another spelling or the same id.
func withoutArtist(in []lastfm.Similar, a Artist) []lastfm.Similar {
name := strings.ToLower(strings.TrimSpace(a.Name))
out := make([]lastfm.Similar, 0, len(in))
for _, c := range in {
if strings.ToLower(strings.TrimSpace(c.Name)) == name ||
(a.MBID != "" && strings.EqualFold(c.MBID, a.MBID)) {
continue
}
out = append(out, c)
}
return out
}
func result(snap Snapshot, status string, stale bool) Result {
items := snap.Items
if items == nil {
items = []Item{}
}
r := Result{Provider: Provider, ProviderStatus: status, Stale: stale && snap.Fetched, Items: items}
if snap.Fetched {
r.RetrievedAt = snap.RetrievedAt
}
return r
}