- feat: GET /api/v1/artists/{id}/similar lists the artists Last.fm finds similar
to one in the library, best first, each with the provider's own score and
rank and a flag saying whether it is in the library. It answers 200 when the
provider is off or failing, with provider_status, stale and retrieved_at
- feat: a thin Last.fm client on top of the outbound client. It always sends
autocorrect=0 so a misspelled local name cannot turn into another artist,
prefers the MusicBrainz id when known, always sets a limit, and reads the
error field even when the status is 200. Errors never carry the URL or the
provider's own message, because the key travels in the query string
- feat: results are stored in artist_similar with a freshness ledger in
provider_fetches: 7 days for an answer, 1 day for "unknown artist". Old data
is served when the provider fails, and concurrent requests for the same
artist share one call that does not depend on the first caller staying
- feat: artists named by a provider but without files become artists without
tracks. GET /artists/{id} can show them (track_count 0), the lists of the
library never include them, and a later scan adopts them by name. A provider
id is never written onto an existing artist; names match only when at least
one side has no id, and different ids always mean different artists
- feat: POST /providers/{name}/test (admin) checks a stored key with one cheap
request, whether the provider is on or off, and reports ok, no_api_key,
key_refused, rate_limited, unreachable or error
- test: the whole request is checked from the receiving end, with a user, a
listen, a title and a library path in the database: only the artist and the
key leave, with a User-Agent that names the project
- docs: README and OpenAPI updated; the contract test calls the new operations
503 lines
15 KiB
Go
503 lines
15 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/uptrace/bun"
|
|
"github.com/uptrace/bun/dialect"
|
|
|
|
"codeberg.org/squidd/sonda/internal/catalog"
|
|
)
|
|
|
|
// CatalogRepo reads the catalog. It implements catalog.Store.
|
|
//
|
|
// Everything listed is a track with at least one audio file that is not flagged
|
|
// missing ("visible"). Lists run a fixed number of queries: one count, one page
|
|
// and a few batched lookups for counts and related items, never one per row.
|
|
// The SQL is written to behave the same on SQLite and PostgreSQL.
|
|
type CatalogRepo struct{ db *bun.DB }
|
|
|
|
// NewCatalogRepo builds a CatalogRepo.
|
|
func NewCatalogRepo(db *bun.DB) *CatalogRepo { return &CatalogRepo{db: db} }
|
|
|
|
// ---- query helpers ----
|
|
|
|
// where collects AND-ed conditions together with their arguments, in order.
|
|
type where struct {
|
|
parts []string
|
|
args []any
|
|
}
|
|
|
|
func (w *where) add(sql string, args ...any) {
|
|
w.parts = append(w.parts, sql)
|
|
w.args = append(w.args, args...)
|
|
}
|
|
|
|
func (w *where) sql() string {
|
|
if len(w.parts) == 0 {
|
|
return "1=1"
|
|
}
|
|
return strings.Join(w.parts, " AND ")
|
|
}
|
|
|
|
// visible is the condition "this track has a playable file", optionally inside
|
|
// one library. alias is the name of the tracks table in the surrounding query.
|
|
func visible(alias string, libraryID int64) (string, []any) {
|
|
q := "EXISTS (SELECT 1 FROM audio_files f WHERE f.track_id = " + alias + ".id AND f.missing_at IS NULL"
|
|
var args []any
|
|
if libraryID != 0 {
|
|
q += " AND f.library_id = ?"
|
|
args = append(args, libraryID)
|
|
}
|
|
return q + ")", args
|
|
}
|
|
|
|
// likePattern turns a search text into a LIKE pattern for the match keys. The
|
|
// wildcard characters of the text are escaped with '!', so "100%" only matches
|
|
// a literal percent sign.
|
|
func likePattern(q string) string {
|
|
esc := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(matchKey(q))
|
|
return "%" + esc + "%"
|
|
}
|
|
|
|
// binary orders by plain byte value. PostgreSQL would otherwise follow the
|
|
// database locale while SQLite compares bytes, and listings would differ.
|
|
func (r *CatalogRepo) binary(expr string) string {
|
|
if r.db.Dialect().Name() == dialect.PG {
|
|
return expr + ` COLLATE "C"`
|
|
}
|
|
return expr
|
|
}
|
|
|
|
func (r *CatalogRepo) count(ctx context.Context, query string, args ...any) (int, error) {
|
|
var n int
|
|
if err := r.db.QueryRowContext(ctx, query, args...).Scan(&n); err != nil {
|
|
return 0, fmt.Errorf("count: %w", err)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// pairs runs a query that returns (id, number) rows and collects them in a map.
|
|
func (r *CatalogRepo) pairs(ctx context.Context, query string, args ...any) (map[int64]int64, error) {
|
|
rows, err := r.db.QueryContext(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := map[int64]int64{}
|
|
for rows.Next() {
|
|
var id, n int64
|
|
if err := rows.Scan(&id, &n); err != nil {
|
|
return nil, err
|
|
}
|
|
out[id] = n
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// ---- artists ----
|
|
|
|
// ListArtists lists artists with a playable credited track or a playable album.
|
|
func (r *CatalogRepo) ListArtists(ctx context.Context, f catalog.Filter) (catalog.Page[catalog.Artist], error) {
|
|
return r.listArtists(ctx, f, 0)
|
|
}
|
|
|
|
// GetArtist returns one artist, or catalog.ErrNotFound.
|
|
func (r *CatalogRepo) GetArtist(ctx context.Context, id int64) (catalog.Artist, error) {
|
|
p, err := r.listArtists(ctx, catalog.Filter{Limit: 1}, id)
|
|
if err != nil {
|
|
return catalog.Artist{}, err
|
|
}
|
|
if len(p.Items) > 0 {
|
|
return p.Items[0], nil
|
|
}
|
|
// Not in the library, but maybe named by a discovery provider: such an artist
|
|
// can be shown, with nothing to play. The lists never include it.
|
|
a := catalog.Artist{}
|
|
err = r.db.QueryRowContext(ctx,
|
|
"SELECT a.id, a.name FROM artists a WHERE a.id = ? AND "+discoveredArtist("a.id"), id).Scan(&a.ID, &a.Name)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return catalog.Artist{}, catalog.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return catalog.Artist{}, fmt.Errorf("find discovered artist: %w", err)
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
func (r *CatalogRepo) listArtists(ctx context.Context, f catalog.Filter, id int64) (catalog.Page[catalog.Artist], error) {
|
|
page := catalog.Page[catalog.Artist]{Items: []catalog.Artist{}, Limit: f.Limit, Offset: f.Offset}
|
|
vis, visArgs := visible("t", f.LibraryID)
|
|
|
|
var w where
|
|
// An artist counts when credited on a playable track, or when it is the
|
|
// album artist of an album with a playable track (compilations).
|
|
w.add(`(EXISTS (SELECT 1 FROM track_artists ta JOIN tracks t ON t.id = ta.track_id WHERE ta.artist_id = a.id AND `+vis+`)
|
|
OR EXISTS (SELECT 1 FROM albums al JOIN tracks t ON t.album_id = al.id WHERE al.artist_id = a.id AND `+vis+`))`,
|
|
append(append([]any{}, visArgs...), visArgs...)...)
|
|
if id != 0 {
|
|
w.add("a.id = ?", id)
|
|
}
|
|
if f.Query != "" {
|
|
w.add("a.name_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
|
}
|
|
|
|
total, err := r.count(ctx, "SELECT COUNT(*) FROM artists a WHERE "+w.sql(), w.args...)
|
|
if err != nil {
|
|
return page, err
|
|
}
|
|
page.Total = total
|
|
if f.Offset >= total {
|
|
return page, nil
|
|
}
|
|
|
|
rows, err := r.db.QueryContext(ctx,
|
|
"SELECT a.id, a.name FROM artists a WHERE "+w.sql()+" ORDER BY "+r.binary("a.name_key")+", a.id LIMIT ? OFFSET ?",
|
|
append(append([]any{}, w.args...), f.Limit, f.Offset)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("list artists: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
var ids []int64
|
|
for rows.Next() {
|
|
var a catalog.Artist
|
|
if err := rows.Scan(&a.ID, &a.Name); err != nil {
|
|
return page, err
|
|
}
|
|
page.Items = append(page.Items, a)
|
|
ids = append(ids, a.ID)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return page, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return page, nil
|
|
}
|
|
|
|
tracks, err := r.pairs(ctx,
|
|
`SELECT ta.artist_id, COUNT(DISTINCT t.id) FROM track_artists ta JOIN tracks t ON t.id = ta.track_id
|
|
WHERE ta.artist_id IN (?) AND `+vis+` GROUP BY ta.artist_id`,
|
|
append([]any{bun.In(ids)}, visArgs...)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("count artist tracks: %w", err)
|
|
}
|
|
albums, err := r.pairs(ctx,
|
|
`SELECT x.artist_id, COUNT(DISTINCT x.album_id) FROM (
|
|
SELECT ta.artist_id AS artist_id, t.album_id AS album_id
|
|
FROM track_artists ta JOIN tracks t ON t.id = ta.track_id
|
|
WHERE ta.artist_id IN (?) AND t.album_id IS NOT NULL AND `+vis+`
|
|
UNION
|
|
SELECT al.artist_id AS artist_id, al.id AS album_id FROM albums al
|
|
WHERE al.artist_id IN (?) AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = al.id AND `+vis+`)
|
|
) x GROUP BY x.artist_id`,
|
|
append(append(append([]any{bun.In(ids)}, visArgs...), bun.In(ids)), visArgs...)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("count artist albums: %w", err)
|
|
}
|
|
for i := range page.Items {
|
|
page.Items[i].TrackCount = int(tracks[page.Items[i].ID])
|
|
page.Items[i].AlbumCount = int(albums[page.Items[i].ID])
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
// ---- albums ----
|
|
|
|
// ListAlbums lists albums that have a playable track.
|
|
func (r *CatalogRepo) ListAlbums(ctx context.Context, f catalog.Filter) (catalog.Page[catalog.Album], error) {
|
|
return r.listAlbums(ctx, f, 0)
|
|
}
|
|
|
|
// GetAlbum returns one album, or catalog.ErrNotFound.
|
|
func (r *CatalogRepo) GetAlbum(ctx context.Context, id int64) (catalog.Album, error) {
|
|
p, err := r.listAlbums(ctx, catalog.Filter{Limit: 1}, id)
|
|
if err != nil {
|
|
return catalog.Album{}, err
|
|
}
|
|
if len(p.Items) == 0 {
|
|
return catalog.Album{}, catalog.ErrNotFound
|
|
}
|
|
return p.Items[0], nil
|
|
}
|
|
|
|
func (r *CatalogRepo) listAlbums(ctx context.Context, f catalog.Filter, id int64) (catalog.Page[catalog.Album], error) {
|
|
page := catalog.Page[catalog.Album]{Items: []catalog.Album{}, Limit: f.Limit, Offset: f.Offset}
|
|
vis, visArgs := visible("t", f.LibraryID)
|
|
|
|
var w where
|
|
w.add("EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = al.id AND "+vis+")", visArgs...)
|
|
if id != 0 {
|
|
w.add("al.id = ?", id)
|
|
}
|
|
if f.ArtistID != 0 {
|
|
// Same meaning as an artist's album_count: albums the artist owns, plus
|
|
// albums where the artist is credited on a playable track.
|
|
w.add(`(al.artist_id = ? OR EXISTS (SELECT 1 FROM tracks t JOIN track_artists ta ON ta.track_id = t.id
|
|
WHERE t.album_id = al.id AND ta.artist_id = ? AND `+vis+`))`,
|
|
append([]any{f.ArtistID, f.ArtistID}, visArgs...)...)
|
|
}
|
|
if f.Query != "" {
|
|
w.add("al.title_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
|
}
|
|
|
|
total, err := r.count(ctx, "SELECT COUNT(*) FROM albums al WHERE "+w.sql(), w.args...)
|
|
if err != nil {
|
|
return page, err
|
|
}
|
|
page.Total = total
|
|
if f.Offset >= total {
|
|
return page, nil
|
|
}
|
|
|
|
rows, err := r.db.QueryContext(ctx,
|
|
`SELECT al.id, al.title, al.artist_id, ar.name FROM albums al
|
|
LEFT JOIN artists ar ON ar.id = al.artist_id
|
|
WHERE `+w.sql()+` ORDER BY `+r.binary("al.title_key")+`, al.id LIMIT ? OFFSET ?`,
|
|
append(append([]any{}, w.args...), f.Limit, f.Offset)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("list albums: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
var ids []int64
|
|
for rows.Next() {
|
|
var (
|
|
a catalog.Album
|
|
artistID sql.NullInt64
|
|
artistName sql.NullString
|
|
)
|
|
if err := rows.Scan(&a.ID, &a.Title, &artistID, &artistName); err != nil {
|
|
return page, err
|
|
}
|
|
if artistID.Valid {
|
|
a.Artist = &catalog.ArtistRef{ID: artistID.Int64, Name: artistName.String}
|
|
}
|
|
page.Items = append(page.Items, a)
|
|
ids = append(ids, a.ID)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return page, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return page, nil
|
|
}
|
|
|
|
type totals struct {
|
|
tracks int
|
|
ms int64
|
|
}
|
|
sums := map[int64]totals{}
|
|
cr, err := r.db.QueryContext(ctx,
|
|
`SELECT t.album_id, COUNT(*), CAST(COALESCE(SUM(t.duration_ms), 0) AS BIGINT) FROM tracks t
|
|
WHERE t.album_id IN (?) AND `+vis+` GROUP BY t.album_id`,
|
|
append([]any{bun.In(ids)}, visArgs...)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("count album tracks: %w", err)
|
|
}
|
|
defer cr.Close()
|
|
for cr.Next() {
|
|
var albumID, n, ms int64
|
|
if err := cr.Scan(&albumID, &n, &ms); err != nil {
|
|
return page, err
|
|
}
|
|
sums[albumID] = totals{tracks: int(n), ms: ms}
|
|
}
|
|
if err := cr.Err(); err != nil {
|
|
return page, err
|
|
}
|
|
for i := range page.Items {
|
|
page.Items[i].TrackCount = sums[page.Items[i].ID].tracks
|
|
page.Items[i].DurationMs = sums[page.Items[i].ID].ms
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
// ---- tracks ----
|
|
|
|
// ListTracks lists playable tracks, grouped by album (two albums with the same
|
|
// title stay apart because the album id is part of the order).
|
|
func (r *CatalogRepo) ListTracks(ctx context.Context, f catalog.Filter) (catalog.Page[catalog.Track], error) {
|
|
return r.listTracks(ctx, f, 0)
|
|
}
|
|
|
|
// GetTrack returns one track, or catalog.ErrNotFound.
|
|
func (r *CatalogRepo) GetTrack(ctx context.Context, id int64) (catalog.Track, error) {
|
|
p, err := r.listTracks(ctx, catalog.Filter{Limit: 1}, id)
|
|
if err != nil {
|
|
return catalog.Track{}, err
|
|
}
|
|
if len(p.Items) == 0 {
|
|
return catalog.Track{}, catalog.ErrNotFound
|
|
}
|
|
return p.Items[0], nil
|
|
}
|
|
|
|
func (r *CatalogRepo) listTracks(ctx context.Context, f catalog.Filter, id int64) (catalog.Page[catalog.Track], error) {
|
|
page := catalog.Page[catalog.Track]{Items: []catalog.Track{}, Limit: f.Limit, Offset: f.Offset}
|
|
vis, visArgs := visible("t", f.LibraryID)
|
|
|
|
var w where
|
|
w.add(vis, visArgs...)
|
|
if id != 0 {
|
|
w.add("t.id = ?", id)
|
|
}
|
|
if f.AlbumID != 0 {
|
|
w.add("t.album_id = ?", f.AlbumID)
|
|
}
|
|
if f.ArtistID != 0 {
|
|
w.add("EXISTS (SELECT 1 FROM track_artists ta WHERE ta.track_id = t.id AND ta.artist_id = ?)", f.ArtistID)
|
|
}
|
|
if f.Query != "" {
|
|
w.add("t.title_key LIKE ? ESCAPE '!'", likePattern(f.Query))
|
|
}
|
|
|
|
total, err := r.count(ctx, "SELECT COUNT(*) FROM tracks t WHERE "+w.sql(), w.args...)
|
|
if err != nil {
|
|
return page, err
|
|
}
|
|
page.Total = total
|
|
if f.Offset >= total {
|
|
return page, nil
|
|
}
|
|
|
|
rows, err := r.db.QueryContext(ctx,
|
|
`SELECT t.id, t.title, COALESCE(t.duration_ms, 0), COALESCE(t.track_no, 0), COALESCE(t.disc_no, 0), t.album_id
|
|
FROM tracks t LEFT JOIN albums al ON al.id = t.album_id
|
|
WHERE `+w.sql()+`
|
|
ORDER BY `+r.binary("COALESCE(al.title_key, '')")+`, COALESCE(t.album_id, 0), COALESCE(t.disc_no, 0), COALESCE(t.track_no, 0),
|
|
`+r.binary("t.title_key")+`, t.id
|
|
LIMIT ? OFFSET ?`,
|
|
append(append([]any{}, w.args...), f.Limit, f.Offset)...)
|
|
if err != nil {
|
|
return page, fmt.Errorf("list tracks: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
albumOf := map[int64]int64{} // track id -> album id
|
|
var ids []int64
|
|
for rows.Next() {
|
|
var (
|
|
t catalog.Track
|
|
albumID sql.NullInt64
|
|
)
|
|
if err := rows.Scan(&t.ID, &t.Title, &t.DurationMs, &t.TrackNo, &t.DiscNo, &albumID); err != nil {
|
|
return page, err
|
|
}
|
|
t.Artists = []catalog.ArtistRef{}
|
|
if albumID.Valid {
|
|
albumOf[t.ID] = albumID.Int64
|
|
}
|
|
page.Items = append(page.Items, t)
|
|
ids = append(ids, t.ID)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return page, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return page, nil
|
|
}
|
|
return page, r.attachTrackDetails(ctx, &page, ids, albumOf, f.LibraryID)
|
|
}
|
|
|
|
// attachTrackDetails fills album, credits and format for a page of tracks with
|
|
// three queries, whatever the page size.
|
|
func (r *CatalogRepo) attachTrackDetails(ctx context.Context, page *catalog.Page[catalog.Track], ids []int64, albumOf map[int64]int64, libraryID int64) error {
|
|
albums := map[int64]catalog.AlbumRef{}
|
|
if len(albumOf) > 0 {
|
|
var albumIDs []int64
|
|
seen := map[int64]bool{}
|
|
for _, a := range albumOf {
|
|
if !seen[a] {
|
|
seen[a] = true
|
|
albumIDs = append(albumIDs, a)
|
|
}
|
|
}
|
|
rows, err := r.db.QueryContext(ctx, "SELECT id, title FROM albums WHERE id IN (?)", bun.In(albumIDs))
|
|
if err != nil {
|
|
return fmt.Errorf("load albums: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var a catalog.AlbumRef
|
|
if err := rows.Scan(&a.ID, &a.Title); err != nil {
|
|
return err
|
|
}
|
|
albums[a.ID] = a
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
credits := map[int64][]catalog.ArtistRef{}
|
|
crows, err := r.db.QueryContext(ctx,
|
|
`SELECT ta.track_id, a.id, a.name FROM track_artists ta JOIN artists a ON a.id = ta.artist_id
|
|
WHERE ta.track_id IN (?) ORDER BY ta.track_id, a.id`, bun.In(ids))
|
|
if err != nil {
|
|
return fmt.Errorf("load credits: %w", err)
|
|
}
|
|
defer crows.Close()
|
|
for crows.Next() {
|
|
var trackID int64
|
|
var a catalog.ArtistRef
|
|
if err := crows.Scan(&trackID, &a.ID, &a.Name); err != nil {
|
|
return err
|
|
}
|
|
credits[trackID] = append(credits[trackID], a)
|
|
}
|
|
if err := crows.Err(); err != nil {
|
|
return err
|
|
}
|
|
|
|
formats := map[int64]string{}
|
|
fq := "SELECT f.track_id, f.format FROM audio_files f WHERE f.track_id IN (?) AND f.missing_at IS NULL"
|
|
fargs := []any{bun.In(ids)}
|
|
if libraryID != 0 {
|
|
fq += " AND f.library_id = ?"
|
|
fargs = append(fargs, libraryID)
|
|
}
|
|
frows, err := r.db.QueryContext(ctx, fq+" ORDER BY f.id", fargs...)
|
|
if err != nil {
|
|
return fmt.Errorf("load formats: %w", err)
|
|
}
|
|
defer frows.Close()
|
|
for frows.Next() {
|
|
var trackID int64
|
|
var format string
|
|
if err := frows.Scan(&trackID, &format); err != nil {
|
|
return err
|
|
}
|
|
if _, ok := formats[trackID]; !ok { // the first playable file wins
|
|
formats[trackID] = normalizeFormat(format)
|
|
}
|
|
}
|
|
if err := frows.Err(); err != nil {
|
|
return err
|
|
}
|
|
|
|
for i := range page.Items {
|
|
t := &page.Items[i]
|
|
if albumID, ok := albumOf[t.ID]; ok {
|
|
if ref, ok := albums[albumID]; ok {
|
|
a := ref
|
|
t.Album = &a
|
|
}
|
|
}
|
|
if c := credits[t.ID]; c != nil {
|
|
t.Artists = c
|
|
}
|
|
t.Format = formats[t.ID]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// normalizeFormat shows TagLib's "mpeg" as the familiar "mp3".
|
|
func normalizeFormat(f string) string {
|
|
if f == "mpeg" {
|
|
return "mp3"
|
|
}
|
|
return f
|
|
}
|