- fix: on SQLite a transaction that read and then wrote failed at once with "database is locked" when another writer committed in between, because the busy timeout does not apply there. Transactions now start with BEGIN IMMEDIATE. Before this change a scan running while someone opened similar artists made files fail, and parallel requests stored nothing - fix: on PostgreSQL two requests naming the same new artist could fail on the unique MusicBrainz id or leave two rows with one name. Creating artists, in the discovery store and in the scanner, now takes turns through an advisory lock and looks again once it has the turn - fix: music arriving for an artist that a provider only named used to keep the provider's spelling and its unchecked MusicBrainz id, which later lookups would then send instead of the name. Such artists are now marked (artists.discovered); the first scan that finds them renames them to the spelling of the file tags, drops the id and clears the mark - fix: a panic in the shared provider call is turned into a failure; the singleflight group would have re-raised it where nothing can recover - fix: a client that leaves while Last.fm is being asked no longer gets a 500 and an error in the log - fix: no page can ask for more than is stored (limit tops out at 50), and the docs no longer say the best score is always 1 - test: concurrent saves plus a scan on both databases, checked by removing each fix; adoption keeps neither the name nor the id; the panic; the client that leaves
271 lines
8.9 KiB
Go
271 lines
8.9 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
"github.com/uptrace/bun/dialect"
|
|
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/providers/lastfm"
|
|
)
|
|
|
|
// kindSimilar is what provider_fetches records for a "similar artists" lookup.
|
|
const kindSimilar = "similar"
|
|
|
|
// Answers recorded in provider_fetches.
|
|
const (
|
|
fetchOK = "ok"
|
|
fetchNotFound = "not_found"
|
|
)
|
|
|
|
// DiscoveryRepo keeps what the discovery providers said. It implements
|
|
// discovery.Store.
|
|
type DiscoveryRepo struct{ db *bun.DB }
|
|
|
|
// NewDiscoveryRepo builds a DiscoveryRepo.
|
|
func NewDiscoveryRepo(db *bun.DB) *DiscoveryRepo { return &DiscoveryRepo{db: db} }
|
|
|
|
// playableArtist is the condition "this artist can be played": credited on a
|
|
// track with a playable file, or the album artist of an album with one
|
|
// (compilations). expr is the artist id in the surrounding query.
|
|
func playableArtist(expr string) string {
|
|
vis, _ := visible("t", 0)
|
|
return `(EXISTS (SELECT 1 FROM track_artists ta JOIN tracks t ON t.id = ta.track_id WHERE ta.artist_id = ` + expr + ` AND ` + vis + `)
|
|
OR EXISTS (SELECT 1 FROM albums al JOIN tracks t ON t.album_id = al.id WHERE al.artist_id = ` + expr + ` AND ` + vis + `))`
|
|
}
|
|
|
|
// discoveredArtist is the condition "a provider named this artist as similar to
|
|
// another". Such an artist can have no files at all.
|
|
func discoveredArtist(expr string) string {
|
|
return "EXISTS (SELECT 1 FROM artist_similar s WHERE s.similar_id = " + expr + ")"
|
|
}
|
|
|
|
// Artist returns an artist that can be looked up: one with something playable,
|
|
// or one a provider already named. Anything else is discovery.ErrNotFound.
|
|
func (r *DiscoveryRepo) Artist(ctx context.Context, id int64) (discovery.Artist, error) {
|
|
var (
|
|
a discovery.Artist
|
|
mbid sql.NullString
|
|
)
|
|
err := r.db.QueryRowContext(ctx,
|
|
"SELECT a.id, a.name, a.mbid FROM artists a WHERE a.id = ? AND ("+playableArtist("a.id")+" OR "+discoveredArtist("a.id")+")", id).
|
|
Scan(&a.ID, &a.Name, &mbid)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return discovery.Artist{}, discovery.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return discovery.Artist{}, fmt.Errorf("find artist: %w", err)
|
|
}
|
|
a.MBID = mbid.String
|
|
return a, nil
|
|
}
|
|
|
|
// Similar reads what was stored about an artist, best first, at most limit items.
|
|
func (r *DiscoveryRepo) Similar(ctx context.Context, id int64, provider string, limit int) (discovery.Snapshot, error) {
|
|
var (
|
|
snap discovery.Snapshot
|
|
status string
|
|
when int64
|
|
)
|
|
err := r.db.QueryRowContext(ctx,
|
|
"SELECT status, retrieved_at FROM provider_fetches WHERE provider = ? AND kind = ? AND artist_id = ?",
|
|
provider, kindSimilar, id).Scan(&status, &when)
|
|
switch {
|
|
case errors.Is(err, sql.ErrNoRows):
|
|
return snap, nil
|
|
case err != nil:
|
|
return snap, fmt.Errorf("read fetch ledger: %w", err)
|
|
}
|
|
snap.Fetched, snap.NotFound, snap.RetrievedAt = true, status == fetchNotFound, time.Unix(when, 0)
|
|
|
|
rows, err := r.db.QueryContext(ctx,
|
|
`SELECT a.id, a.name, s.rank, s.score, `+playableArtist("a.id")+`
|
|
FROM artist_similar s JOIN artists a ON a.id = s.similar_id
|
|
WHERE s.artist_id = ? AND s.provider = ? ORDER BY s.rank, a.id LIMIT ?`,
|
|
id, provider, limit)
|
|
if err != nil {
|
|
return snap, fmt.Errorf("read similar artists: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
snap.Items = []discovery.Item{}
|
|
for rows.Next() {
|
|
var it discovery.Item
|
|
if err := rows.Scan(&it.Artist.ID, &it.Artist.Name, &it.Rank, &it.Score, &it.InLibrary); err != nil {
|
|
return snap, err
|
|
}
|
|
snap.Items = append(snap.Items, it)
|
|
}
|
|
return snap, rows.Err()
|
|
}
|
|
|
|
// SaveSimilar replaces what is stored about an artist and records the answer.
|
|
//
|
|
// Each candidate is matched to an artist that is already known, or becomes a new
|
|
// artist without files. Two artists are the same when their MusicBrainz ids are
|
|
// equal, or when their names match and at least one side has no id: an id that
|
|
// is missing never says "different", and one that differs always does. An id
|
|
// from a provider is never written onto an existing artist, because nobody
|
|
// checked it. Artists created here are kept when the list is replaced later.
|
|
func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error {
|
|
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
|
// Two requests can name the same new artist at once; nothing in the schema
|
|
// forbids two rows with one name, so they take turns.
|
|
if err := lockArtistWrites(ctx, tx); err != nil {
|
|
return err
|
|
}
|
|
ids, err := resolveArtists(ctx, tx, id, cands)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "DELETE FROM artist_similar WHERE artist_id = ? AND provider = ?", id, provider); err != nil {
|
|
return fmt.Errorf("clear similar artists: %w", err)
|
|
}
|
|
rank := 0
|
|
for i, c := range cands {
|
|
if ids[i] == 0 {
|
|
continue // the artist itself, or already listed above
|
|
}
|
|
rank++
|
|
if _, err := tx.ExecContext(ctx,
|
|
"INSERT INTO artist_similar (artist_id, provider, similar_id, rank, score) VALUES (?, ?, ?, ?, ?)",
|
|
id, provider, ids[i], rank, c.Match); err != nil {
|
|
return fmt.Errorf("insert similar artist: %w", err)
|
|
}
|
|
}
|
|
status := fetchOK
|
|
if !found {
|
|
status = fetchNotFound
|
|
}
|
|
if _, err := tx.ExecContext(ctx,
|
|
`INSERT INTO provider_fetches (provider, kind, artist_id, status, retrieved_at) VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT (provider, kind, artist_id) DO UPDATE SET status = excluded.status, retrieved_at = excluded.retrieved_at`,
|
|
provider, kindSimilar, id, status, at.Unix()); err != nil {
|
|
return fmt.Errorf("record fetch: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// resolveArtists returns the artist id of each candidate, creating the ones that
|
|
// are not known. A candidate that is the artist itself, or an artist already
|
|
// seen earlier in the list, gets 0.
|
|
func resolveArtists(ctx context.Context, tx bun.Tx, self int64, cands []lastfm.Similar) ([]int64, error) {
|
|
type known struct {
|
|
id int64
|
|
mbid string
|
|
}
|
|
var mbids, keys []string
|
|
for _, c := range cands {
|
|
if m := cleanMBID(c.MBID); m != "" {
|
|
mbids = append(mbids, m)
|
|
}
|
|
keys = append(keys, matchKey(c.Name))
|
|
}
|
|
byMBID := map[string]int64{}
|
|
byKey := map[string][]known{}
|
|
if len(cands) > 0 {
|
|
// Two conditions with an empty list would match everything or nothing,
|
|
// depending on the database: always send at least one value.
|
|
if len(mbids) == 0 {
|
|
mbids = []string{""}
|
|
}
|
|
rows, err := tx.QueryContext(ctx,
|
|
"SELECT id, mbid, name_key FROM artists WHERE mbid IN (?) OR name_key IN (?) ORDER BY id",
|
|
bun.In(mbids), bun.In(keys))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("find artists: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var (
|
|
id int64
|
|
mbid sql.NullString
|
|
nameKey string
|
|
)
|
|
if err := rows.Scan(&id, &mbid, &nameKey); err != nil {
|
|
return nil, err
|
|
}
|
|
if mbid.String != "" {
|
|
byMBID[strings.ToLower(mbid.String)] = id
|
|
}
|
|
byKey[nameKey] = append(byKey[nameKey], known{id, strings.ToLower(mbid.String)})
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
out := make([]int64, len(cands))
|
|
used := map[int64]bool{self: true}
|
|
for i, c := range cands {
|
|
mbid, key := cleanMBID(c.MBID), matchKey(c.Name)
|
|
var id int64
|
|
if mbid != "" {
|
|
id = byMBID[mbid]
|
|
}
|
|
if id == 0 {
|
|
for _, k := range byKey[key] {
|
|
if mbid == "" || k.mbid == "" {
|
|
id = k.id
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if id == 0 {
|
|
var err error
|
|
if id, err = insertDiscovered(ctx, tx, c.Name, mbid); err != nil {
|
|
return nil, err
|
|
}
|
|
if mbid != "" {
|
|
byMBID[mbid] = id
|
|
}
|
|
byKey[key] = append(byKey[key], known{id, mbid})
|
|
}
|
|
if used[id] {
|
|
continue
|
|
}
|
|
used[id] = true
|
|
out[i] = id
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// insertDiscovered creates an artist that exists only because a provider named it.
|
|
func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64, error) {
|
|
var m sql.NullString
|
|
if mbid != "" {
|
|
m = sql.NullString{String: mbid, Valid: true}
|
|
}
|
|
var id int64
|
|
err := tx.QueryRowContext(ctx,
|
|
"INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", m, name, matchKey(name), true).Scan(&id)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("insert artist: %w", err)
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// artistWritesLock is the key of the PostgreSQL advisory lock that serializes the
|
|
// creation of artists. SQLite needs none: its transactions already run one at a time.
|
|
const artistWritesLock = 0x534f4e4441 // "SONDA"
|
|
|
|
// lockArtistWrites waits for any other transaction that is creating artists and
|
|
// holds the turn until this one ends.
|
|
func lockArtistWrites(ctx context.Context, tx bun.Tx) error {
|
|
if tx.Dialect().Name() != dialect.PG {
|
|
return nil
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(?)", artistWritesLock); err != nil {
|
|
return fmt.Errorf("lock artist writes: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// cleanMBID lower-cases an id the way it is stored.
|
|
func cleanMBID(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
|