sonda/internal/storage/library.go
tgiachi b769ad7ac5 fix(discovery): concurrent writes, adopted artists and a client that leaves
- fix: on SQLite a transaction that read and then wrote failed at once with
  "database is locked" when another writer committed in between, because the
  busy timeout does not apply there. Transactions now start with BEGIN
  IMMEDIATE. Before this change a scan running while someone opened similar
  artists made files fail, and parallel requests stored nothing
- fix: on PostgreSQL two requests naming the same new artist could fail on the
  unique MusicBrainz id or leave two rows with one name. Creating artists, in
  the discovery store and in the scanner, now takes turns through an advisory
  lock and looks again once it has the turn
- fix: music arriving for an artist that a provider only named used to keep the
  provider's spelling and its unchecked MusicBrainz id, which later lookups
  would then send instead of the name. Such artists are now marked
  (artists.discovered); the first scan that finds them renames them to the
  spelling of the file tags, drops the id and clears the mark
- fix: a panic in the shared provider call is turned into a failure; the
  singleflight group would have re-raised it where nothing can recover
- fix: a client that leaves while Last.fm is being asked no longer gets a 500
  and an error in the log
- fix: no page can ask for more than is stored (limit tops out at 50), and the
  docs no longer say the best score is always 1
- test: concurrent saves plus a scan on both databases, checked by removing each
  fix; adoption keeps neither the name nor the id; the panic; the client that
  leaves
2026-11-11 16:47:33 +01:00

459 lines
15 KiB
Go

package storage
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"github.com/uptrace/bun"
"codeberg.org/squidd/sonda/internal/library"
"codeberg.org/squidd/sonda/internal/streaming"
)
type artistModel struct {
bun.BaseModel `bun:"table:artists"`
ID int64 `bun:"id,pk,autoincrement"`
Name string `bun:"name,notnull"`
NameKey string `bun:"name_key,notnull"`
Discovered bool `bun:"discovered,notnull"` // named by a provider, never seen in a file
}
type albumModel struct {
bun.BaseModel `bun:"table:albums"`
ID int64 `bun:"id,pk,autoincrement"`
Title string `bun:"title,notnull"`
TitleKey string `bun:"title_key,notnull"`
ArtistID int64 `bun:"artist_id,nullzero"`
}
type trackModel struct {
bun.BaseModel `bun:"table:tracks"`
ID int64 `bun:"id,pk,autoincrement"`
Title string `bun:"title,notnull"`
TitleKey string `bun:"title_key,notnull"`
DurationMs int64 `bun:"duration_ms,nullzero"`
TrackNo int `bun:"track_no,nullzero"`
DiscNo int `bun:"disc_no,nullzero"`
AlbumID int64 `bun:"album_id,nullzero"`
}
type trackArtistModel struct {
bun.BaseModel `bun:"table:track_artists"`
TrackID int64 `bun:"track_id,pk"`
ArtistID int64 `bun:"artist_id,pk"`
Role string `bun:"role,pk"`
}
type audioFileModel struct {
bun.BaseModel `bun:"table:audio_files"`
ID int64 `bun:"id,pk,autoincrement"`
TrackID int64 `bun:"track_id,notnull"`
LibraryID int64 `bun:"library_id,notnull"`
RelPath string `bun:"rel_path,notnull"`
Format string `bun:"format,notnull"`
SizeBytes int64 `bun:"size_bytes,notnull"`
MTimeUnix int64 `bun:"mtime_unix,notnull"`
SampleRate int `bun:"sample_rate,nullzero"`
BitRate int `bun:"bitrate,nullzero"`
MissingAt sql.NullInt64 `bun:"missing_at"`
}
type libraryModel struct {
bun.BaseModel `bun:"table:libraries"`
ID int64 `bun:"id,pk,autoincrement"`
Name string `bun:"name,notnull"`
Path string `bun:"path,notnull"`
CreatedAt int64 `bun:"created_at,notnull"`
}
func (m libraryModel) toLibrary() library.Library {
return library.Library{ID: m.ID, Name: m.Name, Path: m.Path, CreatedAt: time.Unix(m.CreatedAt, 0)}
}
// LibraryRepo stores the scanned catalog and the libraries themselves. It
// implements library.Store, library.LibraryStore and streaming.Source.
type LibraryRepo struct{ db *bun.DB }
// NewLibraryRepo builds a LibraryRepo.
func NewLibraryRepo(db *bun.DB) *LibraryRepo { return &LibraryRepo{db: db} }
// ExistingFiles lists the files indexed in a library, including missing ones.
func (r *LibraryRepo) ExistingFiles(ctx context.Context, libraryID int64) ([]library.ExistingFile, error) {
var rows []audioFileModel
if err := r.db.NewSelect().Model(&rows).Where("library_id = ?", libraryID).Scan(ctx); err != nil {
return nil, fmt.Errorf("select audio files: %w", err)
}
out := make([]library.ExistingFile, len(rows))
for i, row := range rows {
out[i] = library.ExistingFile{
ID: row.ID,
FileInfo: library.FileInfo{RelPath: row.RelPath, Size: row.SizeBytes, MTimeUnix: row.MTimeUnix},
Missing: row.MissingAt.Valid,
}
}
return out, nil
}
// SaveFile creates or updates one file with its track, album and artists in a
// single transaction. MusicBrainz ids are never invented: they stay NULL.
func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.ParsedFile, existingID int64) (bool, error) {
var added bool
err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
artistIDs := make([]int64, 0, len(f.Artists))
seen := map[int64]bool{}
for _, name := range f.Artists {
id, err := findOrCreateArtist(ctx, tx, name)
if err != nil {
return err
}
if !seen[id] {
seen[id] = true
artistIDs = append(artistIDs, id)
}
}
var albumID int64
if f.Album != "" {
albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist))
if err != nil {
return err
}
if albumID, err = findOrCreateAlbum(ctx, tx, f.Album, albumArtistID); err != nil {
return err
}
}
track := &trackModel{
Title: f.Title, TitleKey: matchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID,
}
if existingID == 0 {
if _, err := tx.NewInsert().Model(track).Exec(ctx); err != nil {
return fmt.Errorf("insert track: %w", err)
}
file := newAudioFile(libraryID, f, track.ID)
if _, err := tx.NewInsert().Model(file).Exec(ctx); err != nil {
return fmt.Errorf("insert audio file: %w", err)
}
added = true
return replaceCredits(ctx, tx, track.ID, artistIDs)
}
var file audioFileModel
if err := tx.NewSelect().Model(&file).Where("id = ?", existingID).Scan(ctx); err != nil {
return fmt.Errorf("load audio file %d: %w", existingID, err)
}
track.ID = file.TrackID
if _, err := tx.NewUpdate().Model(track).
Column("title", "title_key", "duration_ms", "track_no", "disc_no", "album_id").
WherePK().Exec(ctx); err != nil {
return fmt.Errorf("update track: %w", err)
}
file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix
file.SampleRate, file.BitRate = f.SampleRate, f.BitRate
file.MissingAt = sql.NullInt64{}
if _, err := tx.NewUpdate().Model(&file).
Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "missing_at").
WherePK().Exec(ctx); err != nil {
return fmt.Errorf("update audio file: %w", err)
}
return replaceCredits(ctx, tx, track.ID, artistIDs)
})
if err != nil {
return false, err
}
return added, nil
}
func newAudioFile(libraryID int64, f library.ParsedFile, trackID int64) *audioFileModel {
return &audioFileModel{
TrackID: trackID, LibraryID: libraryID, RelPath: f.RelPath, Format: f.Format,
SizeBytes: f.Size, MTimeUnix: f.MTimeUnix, SampleRate: f.SampleRate, BitRate: f.BitRate,
}
}
// RenameFile points an indexed file at a new relative path and clears its missing flag.
func (r *LibraryRepo) RenameFile(ctx context.Context, id int64, newRelPath string) error {
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
Set("rel_path = ?", newRelPath).
Set("missing_at = NULL").
Where("id = ?", id).Exec(ctx)
if err != nil {
return fmt.Errorf("rename audio file: %w", err)
}
return nil
}
// MarkMissing flags files that were not found. Rows and tracks are kept.
func (r *LibraryRepo) MarkMissing(ctx context.Context, ids []int64, at time.Time) error {
if len(ids) == 0 {
return nil
}
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
Set("missing_at = ?", at.Unix()).
Where("id IN (?)", bun.In(ids)).Exec(ctx)
if err != nil {
return fmt.Errorf("mark missing: %w", err)
}
return nil
}
// Restore clears the missing flag of files that reappeared.
func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error {
if len(ids) == 0 {
return nil
}
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
Set("missing_at = NULL").
Where("id IN (?)", bun.In(ids)).Exec(ctx)
if err != nil {
return fmt.Errorf("restore files: %w", err)
}
return nil
}
// PlayableFile returns the file to play for a track: the oldest one that was
// not flagged missing, with the path of its library. It returns
// streaming.ErrNotFound when the track does not exist or has no such file. It
// implements streaming.Source.
func (r *LibraryRepo) PlayableFile(ctx context.Context, trackID int64) (streaming.File, error) {
var row struct {
Path string `bun:"path"`
RelPath string `bun:"rel_path"`
}
err := r.db.NewSelect().
TableExpr("audio_files AS f").
ColumnExpr("l.path AS path, f.rel_path AS rel_path").
Join("JOIN libraries AS l ON l.id = f.library_id").
Where("f.track_id = ?", trackID).
Where("f.missing_at IS NULL").
OrderExpr("f.id").Limit(1).
Scan(ctx, &row)
if errors.Is(err, sql.ErrNoRows) {
return streaming.File{}, streaming.ErrNotFound
}
if err != nil {
return streaming.File{}, fmt.Errorf("select playable file: %w", err)
}
return streaming.File{Root: row.Path, RelPath: row.RelPath}, nil
}
// ---- libraries ----
// ListLibraries returns every library, oldest first.
func (r *LibraryRepo) ListLibraries(ctx context.Context) ([]library.Library, error) {
var rows []libraryModel
if err := r.db.NewSelect().Model(&rows).OrderExpr("id").Scan(ctx); err != nil {
return nil, fmt.Errorf("list libraries: %w", err)
}
out := make([]library.Library, len(rows))
for i, row := range rows {
out[i] = row.toLibrary()
}
return out, nil
}
// GetLibrary returns one library, or library.ErrLibraryNotFound.
func (r *LibraryRepo) GetLibrary(ctx context.Context, id int64) (library.Library, error) {
var row libraryModel
err := r.db.NewSelect().Model(&row).Where("id = ?", id).Limit(1).Scan(ctx)
if errors.Is(err, sql.ErrNoRows) {
return library.Library{}, library.ErrLibraryNotFound
}
if err != nil {
return library.Library{}, fmt.Errorf("select library: %w", err)
}
return row.toLibrary(), nil
}
// CreateLibrary adds a library. Names are unique case-insensitively, paths exactly.
func (r *LibraryRepo) CreateLibrary(ctx context.Context, name, path string) (library.Library, error) {
row := &libraryModel{Name: name, Path: path, CreatedAt: time.Now().Unix()}
if _, err := r.db.NewInsert().Model(row).Exec(ctx); err != nil {
if isUniqueViolation(err) {
return library.Library{}, library.ErrLibraryExists
}
return library.Library{}, fmt.Errorf("insert library: %w", err)
}
return row.toLibrary(), nil
}
// UpdateLibrary changes the name and path of a library. Its files keep their
// identity because they point at the library by id.
func (r *LibraryRepo) UpdateLibrary(ctx context.Context, id int64, name, path string) (library.Library, error) {
res, err := r.db.NewUpdate().Model((*libraryModel)(nil)).
Set("name = ?", name).Set("path = ?", path).Where("id = ?", id).Exec(ctx)
if err != nil {
if isUniqueViolation(err) {
return library.Library{}, library.ErrLibraryExists
}
return library.Library{}, fmt.Errorf("update library: %w", err)
}
if n, _ := res.RowsAffected(); n == 0 {
return library.Library{}, library.ErrLibraryNotFound
}
return r.GetLibrary(ctx, id)
}
// DeleteLibrary removes a library and, through the foreign key, its indexed
// files. Tracks stay, so listening history is kept. Music on disk is untouched.
func (r *LibraryRepo) DeleteLibrary(ctx context.Context, id int64) error {
res, err := r.db.NewDelete().Model((*libraryModel)(nil)).Where("id = ?", id).Exec(ctx)
if err != nil {
return fmt.Errorf("delete library: %w", err)
}
if n, _ := res.RowsAffected(); n == 0 {
return library.ErrLibraryNotFound
}
return nil
}
// RefreshMatchKeys recomputes the match keys of every artist, album and track
// whose stored key differs from the current rule. It is idempotent and cheap
// when nothing changed, so it can run at every start: rows created before
// migration 0003 only have an approximate key from SQL lower().
func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error {
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
var artists []artistModel
if err := tx.NewSelect().Model(&artists).Column("id", "name", "name_key").Scan(ctx); err != nil {
return fmt.Errorf("list artists: %w", err)
}
for _, a := range artists {
key := matchKey(a.Name)
if key == a.NameKey {
continue
}
if _, err := tx.NewUpdate().Model((*artistModel)(nil)).
Set("name_key = ?", key).Where("id = ?", a.ID).Exec(ctx); err != nil {
return fmt.Errorf("update artist key: %w", err)
}
}
var tracks []trackModel
if err := tx.NewSelect().Model(&tracks).Column("id", "title", "title_key").Scan(ctx); err != nil {
return fmt.Errorf("list tracks: %w", err)
}
for _, t := range tracks {
key := matchKey(t.Title)
if key == t.TitleKey {
continue
}
if _, err := tx.NewUpdate().Model((*trackModel)(nil)).
Set("title_key = ?", key).Where("id = ?", t.ID).Exec(ctx); err != nil {
return fmt.Errorf("update track key: %w", err)
}
}
var albums []albumModel
if err := tx.NewSelect().Model(&albums).Column("id", "title", "title_key").Scan(ctx); err != nil {
return fmt.Errorf("list albums: %w", err)
}
for _, al := range albums {
key := matchKey(al.Title)
if key == al.TitleKey {
continue
}
if _, err := tx.NewUpdate().Model((*albumModel)(nil)).
Set("title_key = ?", key).Where("id = ?", al.ID).Exec(ctx); err != nil {
return fmt.Errorf("update album key: %w", err)
}
}
return nil
})
}
// ---- helpers (run inside a transaction) ----
func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) {
key := matchKey(name)
find := func() (artistModel, error) {
var a artistModel
err := tx.NewSelect().Model(&a).Column("id", "name", "discovered").
Where("name_key = ?", key).OrderExpr("id").Limit(1).Scan(ctx)
return a, err
}
a, err := find()
if errors.Is(err, sql.ErrNoRows) {
// About to create one: wait for anyone else who is, then look again.
if err := lockArtistWrites(ctx, tx); err != nil {
return 0, err
}
a, err = find()
}
switch {
case err == nil:
if a.Discovered {
// First music for an artist a provider only named: from now on it is the
// user's, with their spelling and without the provider's unchecked id.
if _, err := tx.NewUpdate().Model((*artistModel)(nil)).
Set("name = ?", name).Set("mbid = NULL").Set("discovered = ?", false).
Where("id = ?", a.ID).Exec(ctx); err != nil {
return 0, fmt.Errorf("adopt discovered artist: %w", err)
}
}
return a.ID, nil
case !errors.Is(err, sql.ErrNoRows):
return 0, fmt.Errorf("find artist: %w", err)
}
a = artistModel{Name: name, NameKey: key}
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
return 0, fmt.Errorf("insert artist: %w", err)
}
return a.ID, nil
}
func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) {
var a albumModel
err := tx.NewSelect().Model(&a).
Where("title_key = ?", matchKey(title)).Where("artist_id = ?", artistID).
OrderExpr("id").Limit(1).Scan(ctx)
if err == nil {
return a.ID, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, fmt.Errorf("find album: %w", err)
}
a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID}
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
return 0, fmt.Errorf("insert album: %w", err)
}
return a.ID, nil
}
func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []int64) error {
if _, err := tx.NewDelete().Model((*trackArtistModel)(nil)).Where("track_id = ?", trackID).Exec(ctx); err != nil {
return fmt.Errorf("clear credits: %w", err)
}
for _, id := range artistIDs {
row := &trackArtistModel{TrackID: trackID, ArtistID: id, Role: "main"}
if _, err := tx.NewInsert().Model(row).Exec(ctx); err != nil {
return fmt.Errorf("insert credit: %w", err)
}
}
return nil
}
// matchKey is the case-insensitive key used to match artist and album names.
// It is computed here, not in SQL, so SQLite and PostgreSQL agree on Unicode.
func matchKey(s string) string {
return strings.ToLower(strings.TrimSpace(s))
}
func firstNonEmpty(values ...string) string {
for _, v := range values {
if v != "" {
return v
}
}
return ""
}