From b769ad7ac57b26457c6f1bf21b80e34e7586c811 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 16:47:33 +0100 Subject: [PATCH] fix(discovery): concurrent writes, adopted artists and a client that leaves - fix: on SQLite a transaction that read and then wrote failed at once with "database is locked" when another writer committed in between, because the busy timeout does not apply there. Transactions now start with BEGIN IMMEDIATE. Before this change a scan running while someone opened similar artists made files fail, and parallel requests stored nothing - fix: on PostgreSQL two requests naming the same new artist could fail on the unique MusicBrainz id or leave two rows with one name. Creating artists, in the discovery store and in the scanner, now takes turns through an advisory lock and looks again once it has the turn - fix: music arriving for an artist that a provider only named used to keep the provider's spelling and its unchecked MusicBrainz id, which later lookups would then send instead of the name. Such artists are now marked (artists.discovered); the first scan that finds them renames them to the spelling of the file tags, drops the id and clears the mark - fix: a panic in the shared provider call is turned into a failure; the singleflight group would have re-raised it where nothing can recover - fix: a client that leaves while Last.fm is being asked no longer gets a 500 and an error in the log - fix: no page can ask for more than is stored (limit tops out at 50), and the docs no longer say the best score is always 1 - test: concurrent saves plus a scan on both databases, checked by removing each fix; adoption keeps neither the name nor the id; the panic; the client that leaves --- README.md | 6 +- internal/api/http/handlers_similar.go | 8 +++ internal/api/http/similar_test.go | 70 +++++++++++++++++-- internal/api/openapi/openapi.yaml | 4 +- internal/discovery/discovery.go | 14 +++- internal/discovery/discovery_test.go | 30 +++++++- internal/providers/lastfm/lastfm.go | 2 +- internal/storage/discovery.go | 24 ++++++- internal/storage/discovery_test.go | 76 ++++++++++++++++++++- internal/storage/library.go | 41 ++++++++--- internal/storage/storage.go | 5 +- migrations/postgres/0008_artist_similar.sql | 6 ++ migrations/sqlite/0008_artist_similar.sql | 6 ++ 13 files changed, 262 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f931491..0335128 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. -- **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to one of yours, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. +- **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet @@ -194,7 +194,7 @@ The key is stored in the database, in clear text, and is never returned by the A **What leaves your server, when a provider is on:** the artist name (or MusicBrainz id) being looked up, your API key for Last.fm, your IP address and the `User-Agent`. Never your users, your listens, track titles or file paths. Requests only happen when someone asks for them, never as a bulk pass over your library, and they are spaced out, retried with backoff, and stopped by a circuit breaker if a provider is down. The one exception to "only when someone asks" is `POST /providers/{name}/test`, which an admin sends to check a key; it sends nothing but the key. -**Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to one of yours and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the first is always 1), so compare artists only inside one answer. +**Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer. The name of the artist is sent exactly as it is written in your library, and Sonda never lets Last.fm correct it into another artist. Names and ids from Last.fm are never written onto your own artists. @@ -211,7 +211,7 @@ Everything lives under `/api/v1`. Bodies are JSON. Errors always look like `{"er | anyone | `POST /auth/login`, `/auth/refresh`, `/auth/logout` | Open, renew and end a session. | | signed in | `GET /auth/me`, `POST /auth/password` | Your account; change your password (signs you out everywhere). | | signed in | `GET /artists`, `/albums`, `/tracks` and `/…/{id}` | Browse and search. Query: `q`, `limit` (default 50, max 200), `offset`, `library_id`, `artist_id`, `album_id`. | -| signed in | `GET /artists/{id}/similar` | Artists similar to one, from Last.fm when an admin switched it on. Query: `limit` (default 20, max 100). | +| signed in | `GET /artists/{id}/similar` | Artists similar to one, from Last.fm when an admin switched it on. Query: `limit` (default 20, max 50). | | signed in | `GET`, `HEAD /tracks/{id}/stream` | Stream the original file. Also accepts `?access_token=`. | | signed in | `POST /listens` | Record a `play`, `complete` or `skip`; the answer says whether it counts as a listen. | | signed in | `GET /listens` | Your own history, newest first, with `counted` per event. Query: `limit`, `offset`, `counted=true`. | diff --git a/internal/api/http/handlers_similar.go b/internal/api/http/handlers_similar.go index b1f1873..d06468e 100644 --- a/internal/api/http/handlers_similar.go +++ b/internal/api/http/handlers_similar.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "errors" "net/http" @@ -9,6 +10,9 @@ import ( "codeberg.org/squidd/sonda/internal/discovery" ) +// statusClientClosed is the status nginx logs for a request the client gave up on. +const statusClientClosed = 499 + // discoveryHandlers serve what providers say about artists outside the library. type discoveryHandlers struct{ svc discoveryService } @@ -60,6 +64,10 @@ func (h *discoveryHandlers) similar(c *gin.Context) { writeError(c, http.StatusNotFound, "artist_not_found", "not found") case errors.Is(err, discovery.ErrInvalidQuery): writeError(c, http.StatusBadRequest, "invalid_input", err.Error()) + case errors.Is(err, context.Canceled): + // The client left while the provider was being asked. There is nobody to + // answer and nothing went wrong on our side: 499 is only for the access log. + c.AbortWithStatus(statusClientClosed) case err != nil: _ = c.Error(err) writeError(c, http.StatusInternalServerError, "internal", "internal error") diff --git a/internal/api/http/similar_test.go b/internal/api/http/similar_test.go index 6f046a5..77aeeb6 100644 --- a/internal/api/http/similar_test.go +++ b/internal/api/http/similar_test.go @@ -11,9 +11,11 @@ import ( "strings" "sync" "testing" + "time" "github.com/uptrace/bun" "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" @@ -26,19 +28,24 @@ import ( // fakeLastfm answers for Last.fm and counts the questions. type fakeLastfm struct { - mu sync.Mutex - calls int - key string - result []lastfm.Similar - err error + mu sync.Mutex + calls int + key string + result []lastfm.Similar + err error + release chan struct{} // when set, a call waits for it to be closed } func (f *fakeLastfm) SimilarArtists(_ context.Context, key string, _ lastfm.Artist, _ int) ([]lastfm.Similar, error) { f.mu.Lock() - defer f.mu.Unlock() f.calls++ f.key = key - return f.result, f.err + release, result, err := f.release, f.result, f.err + f.mu.Unlock() + if release != nil { + <-release + } + return result, err } func (f *fakeLastfm) count() int { @@ -347,3 +354,52 @@ func TestWhatLeavesTheServerIsTheArtistAndTheKeyOnly(t *testing.T) { } } } + +// Someone closing the page while Last.fm is being asked is not a server error: +// nothing is logged as one, and no 500 is sent to nobody. +func TestALeavingClientIsNotAServerError(t *testing.T) { + core, logs := observer.New(zap.DebugLevel) + src := &fakeLastfm{release: make(chan struct{}), result: []lastfm.Similar{{Name: "Gamma", Match: 1}}} + h, db := newTestEnvDB(t, func(d *Deps, db *bun.DB) { + prov := providers.NewService(storage.NewProviderRepo(db)) + d.Providers = prov + d.Logger = zap.New(core) + d.Discovery = discovery.NewService(storage.NewDiscoveryRepo(db), src, prov, zap.NewNop()) + }) + admin := registerAndLogin(t, h) + l, _ := storage.NewLibraryRepo(db).CreateLibrary(context.Background(), "m", "/lib") + f := library.ParsedFile{ + FileInfo: library.FileInfo{RelPath: "a.flac", Size: 1, MTimeUnix: 1}, + Metadata: library.Metadata{Tags: library.Tags{Title: "s", Artists: []string{"Alpha"}}, Properties: library.Properties{Format: "flac", DurationMs: 1}}, + } + if _, err := storage.NewLibraryRepo(db).SaveFile(context.Background(), l.ID, f, 0); err != nil { + t.Fatal(err) + } + var alpha int64 + _ = db.QueryRow("SELECT id FROM artists WHERE name = 'Alpha'").Scan(&alpha) + for _, body := range []map[string]any{{"api_key": lastfmKey}, {"enabled": true}} { + do(t, h, "PUT", "/api/v1/providers/lastfm", admin, body) + } + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest("GET", "/api/v1/artists/"+strconv.FormatInt(alpha, 10)+"/similar", nil).WithContext(ctx) + req.Header.Set("Authorization", "Bearer "+admin) + rec := httptest.NewRecorder() + done := make(chan struct{}) + go func() { h.ServeHTTP(rec, req); close(done) }() + for src.count() == 0 { + time.Sleep(time.Millisecond) + } + cancel() + <-done + close(src.release) + + if rec.Code == http.StatusInternalServerError { + t.Errorf("status = %d: a client that left is not an internal error", rec.Code) + } + for _, e := range logs.All() { + if v, ok := e.ContextMap()["error"]; ok { + t.Errorf("a log entry carries an error: %v", v) + } + } +} diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index 63511a5..aca6090 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -575,13 +575,13 @@ paths: when it is older than it should be. `score` is the provider's own number between 0 and 1. It is relative to the list it came - in (the first is always 1), so it only compares artists inside one answer. `rank` is the + in (the best one is usually 1), so it only compares artists inside one answer. `rank` is the position. Show `attribution` where you show the data. security: [{bearerAuth: []}] parameters: - name: limit in: query - description: How many to return. 20 when absent or 0; values above 100 are lowered to 100. + description: How many to return. 20 when absent or 0; values above 50 are lowered to 50, the most that is kept. schema: {type: integer, minimum: 0, default: 20} responses: '200': diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 8745ca3..35561d5 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -41,10 +41,11 @@ const ( // whatever size of page was asked for, so that one answer serves every page. const FetchLimit = 50 -// Bounds of the number of items returned. +// Bounds of the number of items returned. Only FetchLimit are ever stored, so no +// page can ask for more. const ( DefaultLimit = 20 - MaxLimit = 100 + MaxLimit = FetchLimit ) // fetchTimeout bounds a shared provider call, which does not depend on whoever @@ -185,7 +186,14 @@ func (s *Service) Similar(ctx context.Context, artistID int64, limit int) (Resul // One call per artist, however many ask at once. It runs on its own context, // so the first caller leaving does not fail the others; each caller still // waits only as long as its own context allows. - done := s.flight.DoChan(fmt.Sprint(artistID), func() (any, error) { + done := s.flight.DoChan(fmt.Sprint(artistID), func() (_ any, err error) { + // singleflight re-raises a panic of the shared call in a goroutine nobody + // can recover, which would end the server: turn it into a failure. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("reading similar artists panicked: %v", r) + } + }() return nil, s.refresh(artist, cfg.APIKey) }) select { diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index 2537e97..39166a7 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -92,6 +92,7 @@ type fakeSource struct { started chan struct{} // closed on the first call, when set release chan struct{} // the call waits for it, when set ctxErr error + boom any // when set, the call panics with it } func (s *fakeSource) SimilarArtists(ctx context.Context, k string, a lastfm.Artist, limit int) ([]lastfm.Similar, error) { @@ -107,6 +108,9 @@ func (s *fakeSource) SimilarArtists(ctx context.Context, k string, a lastfm.Arti if release != nil { <-release } + if s.boom != nil { + panic(s.boom) + } s.mu.Lock() s.ctxErr = ctx.Err() s.mu.Unlock() @@ -349,9 +353,9 @@ func TestConcurrentLooksShareOneCall(t *testing.T) { wg.Add(1) go run(context.Background()) } - time.Sleep(50 * time.Millisecond) // let the others join the flight + time.Sleep(300 * time.Millisecond) // let the others join the flight cancel() - time.Sleep(20 * time.Millisecond) + time.Sleep(100 * time.Millisecond) close(r.src.release) wg.Wait() close(results) @@ -387,3 +391,25 @@ func TestAnEmptyListIsRememberedLikeAnUnknownArtist(t *testing.T) { t.Fatalf("result = %+v saves = %+v", res, r.store.saves) } } + +// A panic in the shared call would be re-raised in a goroutine nobody can +// recover, taking the server down: it must come back as a failure instead. +func TestAPanicInTheProviderCallIsAFailureNotACrash(t *testing.T) { + r := newRig(true) + r.src.boom = "the client has a bug" + res, err := r.svc.Similar(context.Background(), 1, 10) + if err != nil || res.ProviderStatus != StatusError || len(res.Items) != 0 { + t.Fatalf("result = %+v err = %v", res, err) + } + r.src.boom = nil + if res = r.similar(t, 10); res.ProviderStatus != StatusOK { + t.Errorf("after the panic the service did not recover: %+v", res) + } +} + +// Only FetchLimit items are ever stored, so no page may promise more. +func TestNoPageCanAskForMoreThanIsStored(t *testing.T) { + if MaxLimit > FetchLimit { + t.Errorf("MaxLimit = %d is above FetchLimit = %d", MaxLimit, FetchLimit) + } +} diff --git a/internal/providers/lastfm/lastfm.go b/internal/providers/lastfm/lastfm.go index 8a1d83a..be3ca1c 100644 --- a/internal/providers/lastfm/lastfm.go +++ b/internal/providers/lastfm/lastfm.go @@ -69,7 +69,7 @@ type Artist struct { } // Similar is one similar artist. Match is what Last.fm calls "match", between -// 0 and 1. It is relative to the list it came in (the first is always 1), so it +// 0 and 1. It is relative to the list it came in (the best one is usually 1), so it // says "how high in this list", not "how alike in general". type Similar struct { Name string diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index 84fdf9f..168d26d 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -9,6 +9,7 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/discovery" "codeberg.org/squidd/sonda/internal/providers/lastfm" @@ -113,6 +114,11 @@ func (r *DiscoveryRepo) Similar(ctx context.Context, id int64, provider string, // checked it. Artists created here are kept when the list is replaced later. func (r *DiscoveryRepo) SaveSimilar(ctx context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error { return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Two requests can name the same new artist at once; nothing in the schema + // forbids two rows with one name, so they take turns. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } ids, err := resolveArtists(ctx, tx, id, cands) if err != nil { return err @@ -238,12 +244,28 @@ func insertDiscovered(ctx context.Context, tx bun.Tx, name, mbid string) (int64, } var id int64 err := tx.QueryRowContext(ctx, - "INSERT INTO artists (mbid, name, name_key) VALUES (?, ?, ?) RETURNING id", m, name, matchKey(name)).Scan(&id) + "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", m, name, matchKey(name), true).Scan(&id) if err != nil { return 0, fmt.Errorf("insert artist: %w", err) } return id, nil } +// artistWritesLock is the key of the PostgreSQL advisory lock that serializes the +// creation of artists. SQLite needs none: its transactions already run one at a time. +const artistWritesLock = 0x534f4e4441 // "SONDA" + +// lockArtistWrites waits for any other transaction that is creating artists and +// holds the turn until this one ends. +func lockArtistWrites(ctx context.Context, tx bun.Tx) error { + if tx.Dialect().Name() != dialect.PG { + return nil + } + if _, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(?)", artistWritesLock); err != nil { + return fmt.Errorf("lock artist writes: %w", err) + } + return nil +} + // cleanMBID lower-cases an id the way it is stored. func cleanMBID(s string) string { return strings.ToLower(strings.TrimSpace(s)) } diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index 3f19507..48e02b8 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -3,6 +3,8 @@ package storage import ( "context" "errors" + "fmt" + "sync" "testing" "time" @@ -268,7 +270,7 @@ func TestAScannedFileAdoptsADiscoveredArtist(t *testing.T) { ctx := context.Background() repo := NewDiscoveryRepo(db) radiohead := artistID(t, db, "Radiohead") - _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", MBID: "ebebebeb-0000-0000-0000-000000000001", Match: 1}}) elbow := artistID(t, db, "Elbow") lib := NewLibraryRepo(db) @@ -285,6 +287,22 @@ func TestAScannedFileAdoptsADiscoveredArtist(t *testing.T) { if got, err := s.repo.GetArtist(ctx, elbow); err != nil || got.TrackCount != 1 { t.Errorf("GetArtist = %+v err=%v", got, err) } + // The artist is now the user's: it carries the name from their tags, and + // nothing a provider said about it (nobody checked that id). + a, err := repo.Artist(ctx, elbow) + if err != nil || a.Name != "ELBOW" || a.MBID != "" { + t.Errorf("after the scan the artist is %+v err=%v, want name ELBOW and no id", a, err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE id = "+fmt.Sprint(elbow)+" AND mbid IS NOT NULL"); n != 0 { + t.Error("the provider's id stayed on the artist") + } + // A second scan of the same artist changes nothing more. + if _, err := lib.SaveFile(ctx, s.rock, pf("e/2.flac", "Another", "", "", 0, 0, "flac", "elbow"), 0); err != nil { + t.Fatal(err) + } + if a, _ = repo.Artist(ctx, elbow); a.Name != "ELBOW" { + t.Errorf("name = %q: only the first scan renames a discovered artist", a.Name) + } }) } @@ -303,3 +321,59 @@ func TestDeletingAnArtistRemovesWhatWasKnownAboutIt(t *testing.T) { } }) } + +// Several people opening artists at the same time, and a scan running meanwhile, +// must not make each other fail, and must not leave two rows for one artist. +func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + lib := NewLibraryRepo(db) + sources := []int64{artistID(t, db, "Radiohead"), artistID(t, db, "Björk"), artistID(t, db, "Miles Davis"), artistID(t, db, "Wild Cards")} + + const rounds = 15 + errs := make(chan error, 1000) + for round := 0; round < rounds; round++ { + var wg sync.WaitGroup // one round at a time: five writers at once, not seventy-five + // Every source names the same two new artists: one with an id, one without. + shared := []lastfm.Similar{ + {Name: fmt.Sprintf("Shared With Id %d", round), MBID: fmt.Sprintf("00000000-0000-0000-0000-%012d", round), Match: 1}, + {Name: fmt.Sprintf("Shared No Id %d", round), Match: 0.5}, + } + for _, src := range sources { + wg.Add(1) + go func() { + defer wg.Done() + if err := repo.SaveSimilar(ctx, src, "lastfm", at, true, shared); err != nil { + errs <- fmt.Errorf("SaveSimilar: %w", err) + } + }() + } + // A scan adding files at the same time. + wg.Add(1) + go func() { + defer wg.Done() + f := pf(fmt.Sprintf("scan/%d.flac", round), fmt.Sprintf("Scanned %d", round), "", "", 0, 0, "flac", fmt.Sprintf("Scanned Artist %d", round)) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + errs <- fmt.Errorf("SaveFile: %w", err) + } + }() + wg.Wait() + } + close(errs) + for err := range errs { + t.Error(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM (SELECT name_key FROM artists WHERE name LIKE 'Shared %' GROUP BY name_key HAVING COUNT(*) > 1) d"); n != 0 { + t.Errorf("%d artists exist more than once", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name LIKE 'Shared %'"); n != 2*rounds { + t.Errorf("shared artists = %d, want %d", n, 2*rounds) + } + // Each save replaces the source's edges, so what is left is the last round. + if n := count(t, db, "SELECT COUNT(*) FROM artist_similar WHERE provider = 'lastfm'"); n != len(sources)*2 { + t.Errorf("edges = %d, want %d", n, len(sources)*2) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 55e42c6..23937a9 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -17,9 +17,10 @@ import ( type artistModel struct { bun.BaseModel `bun:"table:artists"` - ID int64 `bun:"id,pk,autoincrement"` - Name string `bun:"name,notnull"` - NameKey string `bun:"name_key,notnull"` + ID int64 `bun:"id,pk,autoincrement"` + Name string `bun:"name,notnull"` + NameKey string `bun:"name_key,notnull"` + Discovered bool `bun:"discovered,notnull"` // named by a provider, never seen in a file } type albumModel struct { @@ -374,15 +375,37 @@ func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error { // ---- helpers (run inside a transaction) ---- func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) { - var a artistModel - err := tx.NewSelect().Model(&a).Where("name_key = ?", matchKey(name)).OrderExpr("id").Limit(1).Scan(ctx) - if err == nil { - return a.ID, nil + key := matchKey(name) + find := func() (artistModel, error) { + var a artistModel + err := tx.NewSelect().Model(&a).Column("id", "name", "discovered"). + Where("name_key = ?", key).OrderExpr("id").Limit(1).Scan(ctx) + return a, err } - if !errors.Is(err, sql.ErrNoRows) { + a, err := find() + if errors.Is(err, sql.ErrNoRows) { + // About to create one: wait for anyone else who is, then look again. + if err := lockArtistWrites(ctx, tx); err != nil { + return 0, err + } + a, err = find() + } + switch { + case err == nil: + if a.Discovered { + // First music for an artist a provider only named: from now on it is the + // user's, with their spelling and without the provider's unchecked id. + if _, err := tx.NewUpdate().Model((*artistModel)(nil)). + Set("name = ?", name).Set("mbid = NULL").Set("discovered = ?", false). + Where("id = ?", a.ID).Exec(ctx); err != nil { + return 0, fmt.Errorf("adopt discovered artist: %w", err) + } + } + return a.ID, nil + case !errors.Is(err, sql.ErrNoRows): return 0, fmt.Errorf("find artist: %w", err) } - a = artistModel{Name: name, NameKey: matchKey(name)} + a = artistModel{Name: name, NameKey: key} if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { return 0, fmt.Errorf("insert artist: %w", err) } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 9686f87..67d0c16 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -43,9 +43,12 @@ func Open(driver, target string) (*bun.DB, error) { } // openSQLite enables WAL, foreign keys and a busy timeout on every connection. +// Transactions start with BEGIN IMMEDIATE: one that reads and then writes would +// otherwise fail at once with "database is locked" when another writer commits in +// between, and the busy timeout does not apply to that case. func openSQLite(path string) (*bun.DB, error) { dsn := "file:" + pathEscaper.Replace(path) + - "?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)" + "?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_txlock=immediate" sqldb, err := sql.Open("sqlite", dsn) if err != nil { diff --git a/migrations/postgres/0008_artist_similar.sql b/migrations/postgres/0008_artist_similar.sql index ca66ef7..e49a029 100644 --- a/migrations/postgres/0008_artist_similar.sql +++ b/migrations/postgres/0008_artist_similar.sql @@ -9,6 +9,12 @@ -- what it answered ('ok' or 'not_found'), so a fresh answer is not asked again -- and an unknown artist is not asked again at once. +-- Artists that exist only because a provider named them, with no files. The first +-- scan that finds music by the same name makes the artist the user's own: it +-- takes the spelling from the file tags, loses the provider's id (nobody checked +-- it) and stops being "discovered". +ALTER TABLE artists ADD COLUMN discovered BOOLEAN NOT NULL DEFAULT FALSE; + CREATE TABLE artist_similar ( artist_id BIGINT NOT NULL REFERENCES artists(id) ON DELETE CASCADE, provider TEXT NOT NULL, diff --git a/migrations/sqlite/0008_artist_similar.sql b/migrations/sqlite/0008_artist_similar.sql index 25b4382..44d34c7 100644 --- a/migrations/sqlite/0008_artist_similar.sql +++ b/migrations/sqlite/0008_artist_similar.sql @@ -9,6 +9,12 @@ -- what it answered ('ok' or 'not_found'), so a fresh answer is not asked again -- and an unknown artist is not asked again at once. +-- Artists that exist only because a provider named them, with no files. The first +-- scan that finds music by the same name makes the artist the user's own: it +-- takes the spelling from the file tags, loses the provider's id (nobody checked +-- it) and stops being "discovered". +ALTER TABLE artists ADD COLUMN discovered INTEGER NOT NULL DEFAULT 0; + CREATE TABLE artist_similar ( artist_id INTEGER NOT NULL REFERENCES artists(id) ON DELETE CASCADE, provider TEXT NOT NULL,