sonda/internal/storage/discovery.go
tgiachi b769ad7ac5 fix(discovery): concurrent writes, adopted artists and a client that leaves
- fix: on SQLite a transaction that read and then wrote failed at once with
  "database is locked" when another writer committed in between, because the
  busy timeout does not apply there. Transactions now start with BEGIN
  IMMEDIATE. Before this change a scan running while someone opened similar
  artists made files fail, and parallel requests stored nothing
- fix: on PostgreSQL two requests naming the same new artist could fail on the
  unique MusicBrainz id or leave two rows with one name. Creating artists, in
  the discovery store and in the scanner, now takes turns through an advisory
  lock and looks again once it has the turn
- fix: music arriving for an artist that a provider only named used to keep the
  provider's spelling and its unchecked MusicBrainz id, which later lookups
  would then send instead of the name. Such artists are now marked
  (artists.discovered); the first scan that finds them renames them to the
  spelling of the file tags, drops the id and clears the mark
- fix: a panic in the shared provider call is turned into a failure; the
  singleflight group would have re-raised it where nothing can recover
- fix: a client that leaves while Last.fm is being asked no longer gets a 500
  and an error in the log
- fix: no page can ask for more than is stored (limit tops out at 50), and the
  docs no longer say the best score is always 1
- test: concurrent saves plus a scan on both databases, checked by removing each
  fix; adoption keeps neither the name nor the id; the panic; the client that
  leaves
2026-11-11 16:47:33 +01:00

271 lines
8.9 KiB
Go

package storage
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/providers/lastfm"
)
// kindSimilar is what provider_fetches records for a "similar artists" lookup.
const kindSimilar = "similar"
// Answers recorded in provider_fetches.
const (
fetchOK = "ok"
fetchNotFound = "not_found"
)
// DiscoveryRepo keeps what the discovery providers said. It implements
// discovery.Store.
type DiscoveryRepo struct{ db *bun.DB }
// NewDiscoveryRepo builds a DiscoveryRepo.
func NewDiscoveryRepo(db *bun.DB) *DiscoveryRepo { return &DiscoveryRepo{db: db} }
// playableArtist is the condition "this artist can be played": credited on a
// track with a playable file, or the album artist of an album with one
// (compilations). expr is the artist id in the surrounding query.
func playableArtist(expr string) string {
vis, _ := visible("t", 0)
return `(EXISTS (SELECT 1 FROM track_artists ta JOIN tracks t ON t.id = ta.track_id WHERE ta.artist_id = ` + expr + ` AND ` + vis + `)
OR EXISTS (SELECT 1 FROM albums al JOIN tracks t ON t.album_id = al.id WHERE al.artist_id = ` + expr + ` AND ` + vis + `))`
}
// discoveredArtist is the condition "a provider named this artist as similar to
// another". Such an artist can have no files at all.
func discoveredArtist(expr string) string {
return "EXISTS (SELECT 1 FROM artist_similar s WHERE s.similar_id = " + expr + ")"
}
// Artist returns an artist that can be looked up: one with something playable,
// or one a provider already named. Anything else is discovery.ErrNotFound.
func (r *DiscoveryRepo) Artist(ctx context.Context, id int64) (discovery.Artist, error) {
var (
a discovery.Artist
mbid sql.NullString
)
err := r.db.QueryRowContext(ctx,
"SELECT a.id, a.name, a.mbid FROM artists a WHERE a.id = ? AND ("+playableArtist("a.id")+" OR "+discoveredArtist("a.id")+")", id).
Scan(&a.ID, &a.Name, &mbid)
if errors.Is(err, sql.ErrNoRows) {
return discovery.Artist{}, discovery.ErrNotFound
}
if err != nil {
return discovery.Artist{}, fmt.Errorf("find artist: %w", err)
}
a.MBID = mbid.String
return a, nil
}
// Similar reads what was stored about an artist, best first, at most limit items.
func (r *DiscoveryRepo) Similar(ctx context.Context, id int64, provider string, limit int) (discovery.Snapshot, error) {
var (
snap discovery.Snapshot
status string
when int64
)
err := r.db.QueryRowContext(ctx,
"SELECT status, retrieved_at FROM provider_fetches WHERE provider = ? AND kind = ? AND artist_id = ?",
provider, kindSimilar, id).Scan(&status, &when)
switch {
case errors.Is(err, sql.ErrNoRows):
return snap, nil
case err != nil:
return snap, fmt.Errorf("read fetch ledger: %w", err)
}
snap.Fetched, snap.NotFound, snap.RetrievedAt = true, status == fetchNotFound, time.Unix(when, 0)
rows, err := r.db.QueryContext(ctx,
`SELECT a.id, a.name, s.rank, s.score, `+playableArtist("a.id")+`
FROM artist_similar s JOIN artists a ON a.id = s.similar_id
WHERE s.artist_id = ? AND s.provider = ? ORDER BY s.rank, a.id LIMIT ?`,
id, provider, limit)
if err != nil {
return snap, fmt.Errorf("read similar artists: %w", err)
}
defer rows.Close()
snap.Items = []discovery.Item{}
for rows.Next() {
var it discovery.Item
if err := rows.Scan(&it.Artist.ID, &it.Artist.Name, &it.Rank, &it.Score, &it.InLibrary); err != nil {
return snap, err
}
snap.Items = append(snap.Items, it)
}
return snap, rows.Err()
}
// SaveSimilar replaces what is stored about an artist and records the answer.
//
// Each candidate is matched to an artist that is already known, or becomes a new
// artist without files. Two artists are the same when their MusicBrainz ids are
// equal, or when their names match and at least one side has no id: an id that
// is missing never says "different", and one that differs always does. An id
// from a provider is never written onto an existing artist, because nobody
// checked it. Artists created here are kept when the list is replaced later.
func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error {
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// Two requests can name the same new artist at once; nothing in the schema
// forbids two rows with one name, so they take turns.
if err := lockArtistWrites(ctx, tx); err != nil {
return err
}
ids, err := resolveArtists(ctx, tx, id, cands)
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM artist_similar WHERE artist_id = ? AND provider = ?", id, provider); err != nil {
return fmt.Errorf("clear similar artists: %w", err)
}
rank := 0
for i, c := range cands {
if ids[i] == 0 {
continue // the artist itself, or already listed above
}
rank++
if _, err := tx.ExecContext(ctx,
"INSERT INTO artist_similar (artist_id, provider, similar_id, rank, score) VALUES (?, ?, ?, ?, ?)",
id, provider, ids[i], rank, c.Match); err != nil {
return fmt.Errorf("insert similar artist: %w", err)
}
}
status := fetchOK
if !found {
status = fetchNotFound
}
if _, err := tx.ExecContext(ctx,
`INSERT INTO provider_fetches (provider, kind, artist_id, status, retrieved_at) VALUES (?, ?, ?, ?, ?)
ON CONFLICT (provider, kind, artist_id) DO UPDATE SET status = excluded.status, retrieved_at = excluded.retrieved_at`,
provider, kindSimilar, id, status, at.Unix()); err != nil {
return fmt.Errorf("record fetch: %w", err)
}
return nil
})
}
// resolveArtists returns the artist id of each candidate, creating the ones that
// are not known. A candidate that is the artist itself, or an artist already
// seen earlier in the list, gets 0.
func resolveArtists(ctx context.Context, tx bun.Tx, self int64, cands []lastfm.Similar) ([]int64, error) {
type known struct {
id int64
mbid string
}
var mbids, keys []string
for _, c := range cands {
if m := cleanMBID(c.MBID); m != "" {
mbids = append(mbids, m)
}
keys = append(keys, matchKey(c.Name))
}
byMBID := map[string]int64{}
byKey := map[string][]known{}
if len(cands) > 0 {
// Two conditions with an empty list would match everything or nothing,
// depending on the database: always send at least one value.
if len(mbids) == 0 {
mbids = []string{""}
}
rows, err := tx.QueryContext(ctx,
"SELECT id, mbid, name_key FROM artists WHERE mbid IN (?) OR name_key IN (?) ORDER BY id",
bun.In(mbids), bun.In(keys))
if err != nil {
return nil, fmt.Errorf("find artists: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id int64
mbid sql.NullString
nameKey string
)
if err := rows.Scan(&id, &mbid, &nameKey); err != nil {
return nil, err
}
if mbid.String != "" {
byMBID[strings.ToLower(mbid.String)] = id
}
byKey[nameKey] = append(byKey[nameKey], known{id, strings.ToLower(mbid.String)})
}
if err := rows.Err(); err != nil {
return nil, err
}
}
out := make([]int64, len(cands))
used := map[int64]bool{self: true}
for i, c := range cands {
mbid, key := cleanMBID(c.MBID), matchKey(c.Name)
var id int64
if mbid != "" {
id = byMBID[mbid]
}
if id == 0 {
for _, k := range byKey[key] {
if mbid == "" || k.mbid == "" {
id = k.id
break
}
}
}
if id == 0 {
var err error
if id, err = insertDiscovered(ctx, tx, c.Name, mbid); err != nil {
return nil, err
}
if mbid != "" {
byMBID[mbid] = id
}
byKey[key] = append(byKey[key], known{id, mbid})
}
if used[id] {
continue
}
used[id] = true
out[i] = id
}
return out, nil
}
// insertDiscovered creates an artist that exists only because a provider named it.
func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64, error) {
var m sql.NullString
if mbid != "" {
m = sql.NullString{String: mbid, Valid: true}
}
var id int64
err := tx.QueryRowContext(ctx,
"INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", m, name, matchKey(name), true).Scan(&id)
if err != nil {
return 0, fmt.Errorf("insert artist: %w", err)
}
return id, nil
}
// artistWritesLock is the key of the PostgreSQL advisory lock that serializes the
// creation of artists. SQLite needs none: its transactions already run one at a time.
const artistWritesLock = 0x534f4e4441 // "SONDA"
// lockArtistWrites waits for any other transaction that is creating artists and
// holds the turn until this one ends.
func lockArtistWrites(ctx context.Context, tx bun.Tx) error {
if tx.Dialect().Name() != dialect.PG {
return nil
}
if _, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(?)", artistWritesLock); err != nil {
return fmt.Errorf("lock artist writes: %w", err)
}
return nil
}
// cleanMBID lower-cases an id the way it is stored.
func cleanMBID(s string) string { return strings.ToLower(strings.TrimSpace(s)) }