- fix: on SQLite a transaction that read and then wrote failed at once with "database is locked" when another writer committed in between, because the busy timeout does not apply there. Transactions now start with BEGIN IMMEDIATE. Before this change a scan running while someone opened similar artists made files fail, and parallel requests stored nothing - fix: on PostgreSQL two requests naming the same new artist could fail on the unique MusicBrainz id or leave two rows with one name. Creating artists, in the discovery store and in the scanner, now takes turns through an advisory lock and looks again once it has the turn - fix: music arriving for an artist that a provider only named used to keep the provider's spelling and its unchecked MusicBrainz id, which later lookups would then send instead of the name. Such artists are now marked (artists.discovered); the first scan that finds them renames them to the spelling of the file tags, drops the id and clears the mark - fix: a panic in the shared provider call is turned into a failure; the singleflight group would have re-raised it where nothing can recover - fix: a client that leaves while Last.fm is being asked no longer gets a 500 and an error in the log - fix: no page can ask for more than is stored (limit tops out at 50), and the docs no longer say the best score is always 1 - test: concurrent saves plus a scan on both databases, checked by removing each fix; adoption keeps neither the name nor the id; the panic; the client that leaves
459 lines
15 KiB
Go
459 lines
15 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
|
|
"codeberg.org/squidd/sonda/internal/library"
|
|
"codeberg.org/squidd/sonda/internal/streaming"
|
|
)
|
|
|
|
type artistModel struct {
|
|
bun.BaseModel `bun:"table:artists"`
|
|
|
|
ID int64 `bun:"id,pk,autoincrement"`
|
|
Name string `bun:"name,notnull"`
|
|
NameKey string `bun:"name_key,notnull"`
|
|
Discovered bool `bun:"discovered,notnull"` // named by a provider, never seen in a file
|
|
}
|
|
|
|
type albumModel struct {
|
|
bun.BaseModel `bun:"table:albums"`
|
|
|
|
ID int64 `bun:"id,pk,autoincrement"`
|
|
Title string `bun:"title,notnull"`
|
|
TitleKey string `bun:"title_key,notnull"`
|
|
ArtistID int64 `bun:"artist_id,nullzero"`
|
|
}
|
|
|
|
type trackModel struct {
|
|
bun.BaseModel `bun:"table:tracks"`
|
|
|
|
ID int64 `bun:"id,pk,autoincrement"`
|
|
Title string `bun:"title,notnull"`
|
|
TitleKey string `bun:"title_key,notnull"`
|
|
DurationMs int64 `bun:"duration_ms,nullzero"`
|
|
TrackNo int `bun:"track_no,nullzero"`
|
|
DiscNo int `bun:"disc_no,nullzero"`
|
|
AlbumID int64 `bun:"album_id,nullzero"`
|
|
}
|
|
|
|
type trackArtistModel struct {
|
|
bun.BaseModel `bun:"table:track_artists"`
|
|
|
|
TrackID int64 `bun:"track_id,pk"`
|
|
ArtistID int64 `bun:"artist_id,pk"`
|
|
Role string `bun:"role,pk"`
|
|
}
|
|
|
|
type audioFileModel struct {
|
|
bun.BaseModel `bun:"table:audio_files"`
|
|
|
|
ID int64 `bun:"id,pk,autoincrement"`
|
|
TrackID int64 `bun:"track_id,notnull"`
|
|
LibraryID int64 `bun:"library_id,notnull"`
|
|
RelPath string `bun:"rel_path,notnull"`
|
|
Format string `bun:"format,notnull"`
|
|
SizeBytes int64 `bun:"size_bytes,notnull"`
|
|
MTimeUnix int64 `bun:"mtime_unix,notnull"`
|
|
SampleRate int `bun:"sample_rate,nullzero"`
|
|
BitRate int `bun:"bitrate,nullzero"`
|
|
MissingAt sql.NullInt64 `bun:"missing_at"`
|
|
}
|
|
|
|
type libraryModel struct {
|
|
bun.BaseModel `bun:"table:libraries"`
|
|
|
|
ID int64 `bun:"id,pk,autoincrement"`
|
|
Name string `bun:"name,notnull"`
|
|
Path string `bun:"path,notnull"`
|
|
CreatedAt int64 `bun:"created_at,notnull"`
|
|
}
|
|
|
|
func (m libraryModel) toLibrary() library.Library {
|
|
return library.Library{ID: m.ID, Name: m.Name, Path: m.Path, CreatedAt: time.Unix(m.CreatedAt, 0)}
|
|
}
|
|
|
|
// LibraryRepo stores the scanned catalog and the libraries themselves. It
|
|
// implements library.Store, library.LibraryStore and streaming.Source.
|
|
type LibraryRepo struct{ db *bun.DB }
|
|
|
|
// NewLibraryRepo builds a LibraryRepo.
|
|
func NewLibraryRepo(db *bun.DB) *LibraryRepo { return &LibraryRepo{db: db} }
|
|
|
|
// ExistingFiles lists the files indexed in a library, including missing ones.
|
|
func (r *LibraryRepo) ExistingFiles(ctx context.Context, libraryID int64) ([]library.ExistingFile, error) {
|
|
var rows []audioFileModel
|
|
if err := r.db.NewSelect().Model(&rows).Where("library_id = ?", libraryID).Scan(ctx); err != nil {
|
|
return nil, fmt.Errorf("select audio files: %w", err)
|
|
}
|
|
out := make([]library.ExistingFile, len(rows))
|
|
for i, row := range rows {
|
|
out[i] = library.ExistingFile{
|
|
ID: row.ID,
|
|
FileInfo: library.FileInfo{RelPath: row.RelPath, Size: row.SizeBytes, MTimeUnix: row.MTimeUnix},
|
|
Missing: row.MissingAt.Valid,
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SaveFile creates or updates one file with its track, album and artists in a
|
|
// single transaction. MusicBrainz ids are never invented: they stay NULL.
|
|
func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.ParsedFile, existingID int64) (bool, error) {
|
|
var added bool
|
|
err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
|
artistIDs := make([]int64, 0, len(f.Artists))
|
|
seen := map[int64]bool{}
|
|
for _, name := range f.Artists {
|
|
id, err := findOrCreateArtist(ctx, tx, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !seen[id] {
|
|
seen[id] = true
|
|
artistIDs = append(artistIDs, id)
|
|
}
|
|
}
|
|
|
|
var albumID int64
|
|
if f.Album != "" {
|
|
albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if albumID, err = findOrCreateAlbum(ctx, tx, f.Album, albumArtistID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
track := &trackModel{
|
|
Title: f.Title, TitleKey: matchKey(f.Title), DurationMs: f.DurationMs, TrackNo: f.TrackNo, DiscNo: f.DiscNo, AlbumID: albumID,
|
|
}
|
|
|
|
if existingID == 0 {
|
|
if _, err := tx.NewInsert().Model(track).Exec(ctx); err != nil {
|
|
return fmt.Errorf("insert track: %w", err)
|
|
}
|
|
file := newAudioFile(libraryID, f, track.ID)
|
|
if _, err := tx.NewInsert().Model(file).Exec(ctx); err != nil {
|
|
return fmt.Errorf("insert audio file: %w", err)
|
|
}
|
|
added = true
|
|
return replaceCredits(ctx, tx, track.ID, artistIDs)
|
|
}
|
|
|
|
var file audioFileModel
|
|
if err := tx.NewSelect().Model(&file).Where("id = ?", existingID).Scan(ctx); err != nil {
|
|
return fmt.Errorf("load audio file %d: %w", existingID, err)
|
|
}
|
|
track.ID = file.TrackID
|
|
if _, err := tx.NewUpdate().Model(track).
|
|
Column("title", "title_key", "duration_ms", "track_no", "disc_no", "album_id").
|
|
WherePK().Exec(ctx); err != nil {
|
|
return fmt.Errorf("update track: %w", err)
|
|
}
|
|
file.Format, file.SizeBytes, file.MTimeUnix = f.Format, f.Size, f.MTimeUnix
|
|
file.SampleRate, file.BitRate = f.SampleRate, f.BitRate
|
|
file.MissingAt = sql.NullInt64{}
|
|
if _, err := tx.NewUpdate().Model(&file).
|
|
Column("format", "size_bytes", "mtime_unix", "sample_rate", "bitrate", "missing_at").
|
|
WherePK().Exec(ctx); err != nil {
|
|
return fmt.Errorf("update audio file: %w", err)
|
|
}
|
|
return replaceCredits(ctx, tx, track.ID, artistIDs)
|
|
})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return added, nil
|
|
}
|
|
|
|
func newAudioFile(libraryID int64, f library.ParsedFile, trackID int64) *audioFileModel {
|
|
return &audioFileModel{
|
|
TrackID: trackID, LibraryID: libraryID, RelPath: f.RelPath, Format: f.Format,
|
|
SizeBytes: f.Size, MTimeUnix: f.MTimeUnix, SampleRate: f.SampleRate, BitRate: f.BitRate,
|
|
}
|
|
}
|
|
|
|
// RenameFile points an indexed file at a new relative path and clears its missing flag.
|
|
func (r *LibraryRepo) RenameFile(ctx context.Context, id int64, newRelPath string) error {
|
|
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
|
|
Set("rel_path = ?", newRelPath).
|
|
Set("missing_at = NULL").
|
|
Where("id = ?", id).Exec(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("rename audio file: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MarkMissing flags files that were not found. Rows and tracks are kept.
|
|
func (r *LibraryRepo) MarkMissing(ctx context.Context, ids []int64, at time.Time) error {
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
|
|
Set("missing_at = ?", at.Unix()).
|
|
Where("id IN (?)", bun.In(ids)).Exec(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("mark missing: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Restore clears the missing flag of files that reappeared.
|
|
func (r *LibraryRepo) Restore(ctx context.Context, ids []int64) error {
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
_, err := r.db.NewUpdate().Model((*audioFileModel)(nil)).
|
|
Set("missing_at = NULL").
|
|
Where("id IN (?)", bun.In(ids)).Exec(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("restore files: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PlayableFile returns the file to play for a track: the oldest one that was
|
|
// not flagged missing, with the path of its library. It returns
|
|
// streaming.ErrNotFound when the track does not exist or has no such file. It
|
|
// implements streaming.Source.
|
|
func (r *LibraryRepo) PlayableFile(ctx context.Context, trackID int64) (streaming.File, error) {
|
|
var row struct {
|
|
Path string `bun:"path"`
|
|
RelPath string `bun:"rel_path"`
|
|
}
|
|
err := r.db.NewSelect().
|
|
TableExpr("audio_files AS f").
|
|
ColumnExpr("l.path AS path, f.rel_path AS rel_path").
|
|
Join("JOIN libraries AS l ON l.id = f.library_id").
|
|
Where("f.track_id = ?", trackID).
|
|
Where("f.missing_at IS NULL").
|
|
OrderExpr("f.id").Limit(1).
|
|
Scan(ctx, &row)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return streaming.File{}, streaming.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return streaming.File{}, fmt.Errorf("select playable file: %w", err)
|
|
}
|
|
return streaming.File{Root: row.Path, RelPath: row.RelPath}, nil
|
|
}
|
|
|
|
// ---- libraries ----
|
|
|
|
// ListLibraries returns every library, oldest first.
|
|
func (r *LibraryRepo) ListLibraries(ctx context.Context) ([]library.Library, error) {
|
|
var rows []libraryModel
|
|
if err := r.db.NewSelect().Model(&rows).OrderExpr("id").Scan(ctx); err != nil {
|
|
return nil, fmt.Errorf("list libraries: %w", err)
|
|
}
|
|
out := make([]library.Library, len(rows))
|
|
for i, row := range rows {
|
|
out[i] = row.toLibrary()
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// GetLibrary returns one library, or library.ErrLibraryNotFound.
|
|
func (r *LibraryRepo) GetLibrary(ctx context.Context, id int64) (library.Library, error) {
|
|
var row libraryModel
|
|
err := r.db.NewSelect().Model(&row).Where("id = ?", id).Limit(1).Scan(ctx)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return library.Library{}, library.ErrLibraryNotFound
|
|
}
|
|
if err != nil {
|
|
return library.Library{}, fmt.Errorf("select library: %w", err)
|
|
}
|
|
return row.toLibrary(), nil
|
|
}
|
|
|
|
// CreateLibrary adds a library. Names are unique case-insensitively, paths exactly.
|
|
func (r *LibraryRepo) CreateLibrary(ctx context.Context, name, path string) (library.Library, error) {
|
|
row := &libraryModel{Name: name, Path: path, CreatedAt: time.Now().Unix()}
|
|
if _, err := r.db.NewInsert().Model(row).Exec(ctx); err != nil {
|
|
if isUniqueViolation(err) {
|
|
return library.Library{}, library.ErrLibraryExists
|
|
}
|
|
return library.Library{}, fmt.Errorf("insert library: %w", err)
|
|
}
|
|
return row.toLibrary(), nil
|
|
}
|
|
|
|
// UpdateLibrary changes the name and path of a library. Its files keep their
|
|
// identity because they point at the library by id.
|
|
func (r *LibraryRepo) UpdateLibrary(ctx context.Context, id int64, name, path string) (library.Library, error) {
|
|
res, err := r.db.NewUpdate().Model((*libraryModel)(nil)).
|
|
Set("name = ?", name).Set("path = ?", path).Where("id = ?", id).Exec(ctx)
|
|
if err != nil {
|
|
if isUniqueViolation(err) {
|
|
return library.Library{}, library.ErrLibraryExists
|
|
}
|
|
return library.Library{}, fmt.Errorf("update library: %w", err)
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return library.Library{}, library.ErrLibraryNotFound
|
|
}
|
|
return r.GetLibrary(ctx, id)
|
|
}
|
|
|
|
// DeleteLibrary removes a library and, through the foreign key, its indexed
|
|
// files. Tracks stay, so listening history is kept. Music on disk is untouched.
|
|
func (r *LibraryRepo) DeleteLibrary(ctx context.Context, id int64) error {
|
|
res, err := r.db.NewDelete().Model((*libraryModel)(nil)).Where("id = ?", id).Exec(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("delete library: %w", err)
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return library.ErrLibraryNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RefreshMatchKeys recomputes the match keys of every artist, album and track
|
|
// whose stored key differs from the current rule. It is idempotent and cheap
|
|
// when nothing changed, so it can run at every start: rows created before
|
|
// migration 0003 only have an approximate key from SQL lower().
|
|
func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error {
|
|
return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
|
var artists []artistModel
|
|
if err := tx.NewSelect().Model(&artists).Column("id", "name", "name_key").Scan(ctx); err != nil {
|
|
return fmt.Errorf("list artists: %w", err)
|
|
}
|
|
for _, a := range artists {
|
|
key := matchKey(a.Name)
|
|
if key == a.NameKey {
|
|
continue
|
|
}
|
|
if _, err := tx.NewUpdate().Model((*artistModel)(nil)).
|
|
Set("name_key = ?", key).Where("id = ?", a.ID).Exec(ctx); err != nil {
|
|
return fmt.Errorf("update artist key: %w", err)
|
|
}
|
|
}
|
|
|
|
var tracks []trackModel
|
|
if err := tx.NewSelect().Model(&tracks).Column("id", "title", "title_key").Scan(ctx); err != nil {
|
|
return fmt.Errorf("list tracks: %w", err)
|
|
}
|
|
for _, t := range tracks {
|
|
key := matchKey(t.Title)
|
|
if key == t.TitleKey {
|
|
continue
|
|
}
|
|
if _, err := tx.NewUpdate().Model((*trackModel)(nil)).
|
|
Set("title_key = ?", key).Where("id = ?", t.ID).Exec(ctx); err != nil {
|
|
return fmt.Errorf("update track key: %w", err)
|
|
}
|
|
}
|
|
|
|
var albums []albumModel
|
|
if err := tx.NewSelect().Model(&albums).Column("id", "title", "title_key").Scan(ctx); err != nil {
|
|
return fmt.Errorf("list albums: %w", err)
|
|
}
|
|
for _, al := range albums {
|
|
key := matchKey(al.Title)
|
|
if key == al.TitleKey {
|
|
continue
|
|
}
|
|
if _, err := tx.NewUpdate().Model((*albumModel)(nil)).
|
|
Set("title_key = ?", key).Where("id = ?", al.ID).Exec(ctx); err != nil {
|
|
return fmt.Errorf("update album key: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ---- helpers (run inside a transaction) ----
|
|
|
|
func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) {
|
|
key := matchKey(name)
|
|
find := func() (artistModel, error) {
|
|
var a artistModel
|
|
err := tx.NewSelect().Model(&a).Column("id", "name", "discovered").
|
|
Where("name_key = ?", key).OrderExpr("id").Limit(1).Scan(ctx)
|
|
return a, err
|
|
}
|
|
a, err := find()
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
// About to create one: wait for anyone else who is, then look again.
|
|
if err := lockArtistWrites(ctx, tx); err != nil {
|
|
return 0, err
|
|
}
|
|
a, err = find()
|
|
}
|
|
switch {
|
|
case err == nil:
|
|
if a.Discovered {
|
|
// First music for an artist a provider only named: from now on it is the
|
|
// user's, with their spelling and without the provider's unchecked id.
|
|
if _, err := tx.NewUpdate().Model((*artistModel)(nil)).
|
|
Set("name = ?", name).Set("mbid = NULL").Set("discovered = ?", false).
|
|
Where("id = ?", a.ID).Exec(ctx); err != nil {
|
|
return 0, fmt.Errorf("adopt discovered artist: %w", err)
|
|
}
|
|
}
|
|
return a.ID, nil
|
|
case !errors.Is(err, sql.ErrNoRows):
|
|
return 0, fmt.Errorf("find artist: %w", err)
|
|
}
|
|
a = artistModel{Name: name, NameKey: key}
|
|
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
|
|
return 0, fmt.Errorf("insert artist: %w", err)
|
|
}
|
|
return a.ID, nil
|
|
}
|
|
|
|
func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) {
|
|
var a albumModel
|
|
err := tx.NewSelect().Model(&a).
|
|
Where("title_key = ?", matchKey(title)).Where("artist_id = ?", artistID).
|
|
OrderExpr("id").Limit(1).Scan(ctx)
|
|
if err == nil {
|
|
return a.ID, nil
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return 0, fmt.Errorf("find album: %w", err)
|
|
}
|
|
a = albumModel{Title: title, TitleKey: matchKey(title), ArtistID: artistID}
|
|
if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil {
|
|
return 0, fmt.Errorf("insert album: %w", err)
|
|
}
|
|
return a.ID, nil
|
|
}
|
|
|
|
func replaceCredits(ctx context.Context, tx bun.Tx, trackID int64, artistIDs []int64) error {
|
|
if _, err := tx.NewDelete().Model((*trackArtistModel)(nil)).Where("track_id = ?", trackID).Exec(ctx); err != nil {
|
|
return fmt.Errorf("clear credits: %w", err)
|
|
}
|
|
for _, id := range artistIDs {
|
|
row := &trackArtistModel{TrackID: trackID, ArtistID: id, Role: "main"}
|
|
if _, err := tx.NewInsert().Model(row).Exec(ctx); err != nil {
|
|
return fmt.Errorf("insert credit: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// matchKey is the case-insensitive key used to match artist and album names.
|
|
// It is computed here, not in SQL, so SQLite and PostgreSQL agree on Unicode.
|
|
func matchKey(s string) string {
|
|
return strings.ToLower(strings.TrimSpace(s))
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, v := range values {
|
|
if v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|