diff --git a/README.md b/README.md index f34ef88..fe75cdc 100644 --- a/README.md +++ b/README.md @@ -198,11 +198,11 @@ The key is stored in the database, in clear text, and is never returned by the A **Similar artists.** `GET /api/v1/artists/{id}/similar` asks Last.fm for the artists similar to an artist (one of yours, or one Last.fm named earlier) and keeps the answer in your database: 7 days, or 1 day when Last.fm does not know the artist. Until then a second look costs nothing and sends nothing, and if Last.fm is down you still get the last answer, marked `stale`. If the provider is off the answer says `disabled` instead of failing. Each similar artist says whether you have it (`in_library`); the ones you do not have are stored as artists without files. They can be opened with `GET /artists/{id}` but are never part of your artist lists, and if you later add their music, the scan finds them there. Last.fm's `score` is relative to each answer (the best one is usually 1), so compare artists only inside one answer. -**Tags.** `GET /api/v1/artists/{id}/tags` asks each provider that is on, at the same time, and keeps the answers with the same rules (7 days, 1 for an unknown artist, old data if a provider fails). Each provider has its own entry with its own weights: Last.fm's are relative to the artist (0 to 100), MusicBrainz's are counts of votes, and they are never mixed. MusicBrainz also marks the tags it lists as genres. Its tags and genres are CC BY-NC-SA, so show the `attribution` that comes with them. +**Tags.** `GET /api/v1/artists/{id}/tags` asks each provider that is on, at the same time, and keeps the answers with the same rules (7 days, 1 for an unknown artist, old data if a provider fails). Each provider has its own entry with its own weights: Last.fm's are relative to the artist (1 to 100), MusicBrainz's are counts of votes, and they are never mixed. MusicBrainz also marks the tags it lists as genres. Its tags and genres are CC BY-NC-SA, so show the `attribution` that comes with them. **MusicBrainz ids come from your files, never from a guess.** If your files carry MusicBrainz artist ids in their tags (taggers like Picard write them), the scan reads them, offline, and keeps them on your artists, but only when each id can be paired with its artist for certain. MusicBrainz is asked only about artists whose id is known; for the others its entry says `no_id`. Sonda never searches MusicBrainz by name, because a search returns look-alikes and picking one would be a guess. The scan reads a file's tags when the file is new or changed, so to pick up ids for music that is already indexed, the files have to be touched or re-added. -The name of the artist is sent exactly as it is written in your library, and Sonda never lets Last.fm correct it into another artist. Names and ids from Last.fm are never written onto your own artists. +The artist is sent to Last.fm by its MusicBrainz id when your files gave one (if Last.fm does not know the id, by name), otherwise by its name exactly as it is written in your library, and Sonda never lets Last.fm correct it into another artist. Names and ids from Last.fm are never written onto your own artists. ## The API at a glance diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml index e7850cf..e50d767 100644 --- a/internal/api/openapi/openapi.yaml +++ b/internal/api/openapi/openapi.yaml @@ -608,15 +608,16 @@ paths: What each provider says kind of music this artist plays: Last.fm's top tags and MusicBrainz's tags and genres. There is always one entry for each, Last.fm first. The weights are the providers' own numbers, kept as they came and never mixed: Last.fm's is - relative to the artist (0 to 100, the top tag is usually 100), MusicBrainz's is a count of - votes. `genre` is true for a MusicBrainz tag that it also lists among its curated genres. + relative to the artist (1 to 100, the top tag is usually 100; a tag with no votes is dropped), + MusicBrainz's is a count of votes. `genre` is true for a MusicBrainz tag that it also lists among its curated genres. A provider is only asked when an admin switched it on and its stored answer is missing or old (7 days; a "no such artist" answer is kept 1 day), with the same rules as `GET /artists/{id}/similar`. What is sent is the artist's name or MusicBrainz id to - Last.fm, and only the MusicBrainz id to MusicBrainz: Sonda never looks an id up by name, - so for an artist whose id is not known (it comes from the tags in the music files) the - MusicBrainz entry says `no_id`. Show each `attribution` where you show the tags. + Last.fm (its id when known), and only the MusicBrainz id to MusicBrainz: Sonda never looks + an id up by name. An artist's id comes from the tags in the music files, or from Last.fm + for an artist it named that is not in the library; for any other artist the MusicBrainz + entry says `no_id`. Show each `attribution` where you show the tags. security: [{bearerAuth: []}] responses: '200': diff --git a/internal/discovery/tags.go b/internal/discovery/tags.go index f0a8873..4367e1a 100644 --- a/internal/discovery/tags.go +++ b/internal/discovery/tags.go @@ -20,6 +20,11 @@ import ( // weighty ones. Beyond that they are noise. const MaxTags = 50 +// tagFetchTimeout bounds the wait for one provider's tags, queueing behind its +// rate limit included: a provider limited to one request a second must not keep +// a page waiting for a minute because a client asked about fifty artists. +const tagFetchTimeout = 20 * time.Second + // StatusNoID says a provider that needs the MusicBrainz id of the artist cannot // be asked, because the id is not known. Sonda never looks one up by name. const StatusNoID = "no_id" @@ -194,7 +199,7 @@ func (s *TagService) one(ctx context.Context, a Artist, p tagProvider) (TagSourc // refresh reads one provider and stores what it said. func (s *TagService) refresh(src TagSource, key string, a Artist, provider string) error { - ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) + ctx, cancel := context.WithTimeout(context.Background(), tagFetchTimeout) defer cancel() tags, err := src.ArtistTags(ctx, key, a) switch { diff --git a/internal/library/taglib.go b/internal/library/taglib.go index ac6f085..98dcee7 100644 --- a/internal/library/taglib.go +++ b/internal/library/taglib.go @@ -33,14 +33,21 @@ func (TaglibReader) Read(path string) (Metadata, error) { } artists := nonEmpty(tags["ARTIST"]) + albumArtist := first(tags, "ALBUMARTIST") + albumArtistID := "" + if albumArtist != "" { + // The id belongs to the album artist tag: when the tag is missing the name + // below is only a fallback, and an id would land on the wrong artist. + albumArtistID = singleID(tags["MUSICBRAINZ_ALBUMARTISTID"]) + } m := Metadata{ Tags: Tags{ Title: first(tags, "TITLE"), Album: first(tags, "ALBUM"), - AlbumArtist: first(tags, "ALBUMARTIST"), + AlbumArtist: albumArtist, Artists: artists, ArtistMBIDs: pairArtistIDs(artists, tags["MUSICBRAINZ_ARTISTID"]), - AlbumArtistMBID: singleID(tags["MUSICBRAINZ_ALBUMARTISTID"]), + AlbumArtistMBID: albumArtistID, TrackNo: leadingInt(first(tags, "TRACKNUMBER")), DiscNo: leadingInt(first(tags, "DISCNUMBER")), }, @@ -95,9 +102,31 @@ func pairArtistIDs(artists, values []string) []string { if !ok || len(ids) == 0 || len(ids) != len(artists) { return nil } + // One credit with one id ("X feat. Y" with the id of X) is the usual way a + // tagger writes the id of the first artist only. Without a way to tell, no id. + if len(artists) == 1 && looksLikeCredit(artists[0]) { + return nil + } return ids } +// creditWords are the separators of a credit that names several artists. A slash +// is not one: AC/DC is a band. +var creditWords = []string{" feat.", " feat ", " featuring ", " ft.", " ft ", " with ", " & ", " and ", " vs ", " vs. ", " x ", ",", ";"} + +// looksLikeCredit says whether a name seems to credit more than one artist. It +// errs on the side of "yes": a missing id costs a lookup, a wrong one a wrong artist. +func looksLikeCredit(name string) bool { + n := " " + strings.ToLower(name) + " " + for _, w := range creditWords { + // Text must stand on both sides: a band called "X" is not a credit. + if i := strings.Index(n, w); i > 1 && i+len(w) < len(n)-1 { + return true + } + } + return false +} + // singleID returns the id when the tag holds exactly one valid id. func singleID(values []string) string { ids, ok := splitIDs(values) diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go index f4baa70..8f5aaf9 100644 --- a/internal/library/taglib_test.go +++ b/internal/library/taglib_test.go @@ -143,6 +143,14 @@ func TestArtistIDsAreOnlyKeptWhenTheyCanBePairedWithTheirArtists(t *testing.T) { {"no ids", []string{"X"}, nil, nil}, {"one bad id spoils the pairing", []string{"X", "Y"}, []string{a, "not-an-id"}, nil}, {"empty values are ignored", []string{"X"}, []string{" ", a}, []string{a}}, + // A credit with one id is usually the id of the first artist only. + {"a featuring credit with one id", []string{"X feat. Y"}, []string{a}, nil}, + {"ft. credit", []string{"X ft. Y"}, []string{a}, nil}, + {"with an ampersand", []string{"Simon & Garfunkel"}, []string{a}, nil}, + {"with a comma", []string{"X, Y"}, []string{a}, nil}, + {"a slash is part of a name", []string{"AC/DC"}, []string{a}, []string{a}}, + {"a plain name that contains 'and' inside a word", []string{"Sandra Bland"}, []string{a}, []string{a}}, + {"credits are fine when each artist has an id", []string{"X", "Y"}, []string{a, b}, []string{a, b}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -190,3 +198,40 @@ func TestTaglibReaderReadsMusicBrainzArtistIDs(t *testing.T) { t.Errorf("an id for an artist nobody named: %v", m.ArtistMBIDs) } } + +// The id of the album artist belongs to the name of the album artist tag. When +// the tag is missing the name is only a fallback, and an id would land on the +// wrong artist. +func TestAlbumArtistIDNeedsAnAlbumArtistTag(t *testing.T) { + const variousArtists = "89ad4ac3-39f7-470e-963a-56509c546377" + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "T", "artist": "Foo", "MUSICBRAINZ_ALBUMARTISTID": variousArtists, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if m.AlbumArtist != "Foo" || m.AlbumArtistMBID != "" { + t.Errorf("album artist = %q with id %q: the id must not follow a fallback name", m.AlbumArtist, m.AlbumArtistMBID) + } + + p = makeAudio(t, t.TempDir(), "nothing.flac", map[string]string{"title": "T", "MUSICBRAINZ_ALBUMARTISTID": variousArtists}) + m, _ = TaglibReader{}.Read(p) + if m.AlbumArtistMBID != "" { + t.Errorf("an id for an album artist nobody named: %q", m.AlbumArtistMBID) + } +} + +func TestTaglibReaderReadsMusicBrainzIDsFromMP3(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + p := makeAudio(t, t.TempDir(), "song.mp3", map[string]string{ + "title": "T", "artist": "Synthetic Band", "MusicBrainz Artist Id": a, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if len(m.ArtistMBIDs) != 1 || m.ArtistMBIDs[0] != a { + t.Errorf("ids = %v: an ID3 'MusicBrainz Artist Id' frame should be read", m.ArtistMBIDs) + } +} diff --git a/internal/providers/lastfm/lastfm.go b/internal/providers/lastfm/lastfm.go index 1799007..cfac7ae 100644 --- a/internal/providers/lastfm/lastfm.go +++ b/internal/providers/lastfm/lastfm.go @@ -94,14 +94,7 @@ func (c *Client) SimilarArtists(ctx context.Context, key string, a Artist, limit case limit > MaxLimit: limit = MaxLimit } - q := url.Values{"method": {"artist.getsimilar"}, "autocorrect": {"0"}, "limit": {strconv.Itoa(limit)}} - if mbid != "" { - q.Set("mbid", mbid) - } else { - q.Set("artist", name) - } - - body, err := c.call(ctx, key, q) + body, err := c.callFor(ctx, key, name, mbid, url.Values{"method": {"artist.getsimilar"}, "limit": {strconv.Itoa(limit)}}) if err != nil { return nil, err } @@ -133,13 +126,7 @@ func (c *Client) TopTags(ctx context.Context, key string, a Artist) ([]Tag, erro if name == "" && mbid == "" { return nil, ErrBadRequest } - q := url.Values{"method": {"artist.gettoptags"}, "autocorrect": {"0"}} - if mbid != "" { - q.Set("mbid", mbid) - } else { - q.Set("artist", name) - } - body, err := c.call(ctx, key, q) + body, err := c.callFor(ctx, key, name, mbid, url.Values{"method": {"artist.gettoptags"}}) if err != nil { return nil, err } @@ -215,6 +202,35 @@ func (c *Client) Check(ctx context.Context, key string) error { return nil } +// callFor asks about one artist. The MusicBrainz id goes first, since it cannot be +// confused with a namesake. Last.fm does not know every id: when it says it does +// not know this one and there is a name, the name is tried, never autocorrected, +// which is what was done before an id was known. Any other failure is final. +func (c *Client) callFor(ctx context.Context, key, name, mbid string, q url.Values) ([]byte, error) { + q.Set("autocorrect", "0") + if mbid != "" { + byID := cloneValues(q) + byID.Set("mbid", mbid) + body, err := c.call(ctx, key, byID) + if !errors.Is(err, ErrNotFound) || name == "" { + return body, err + } + } + if name == "" { + return nil, ErrNotFound + } + q.Set("artist", name) + return c.call(ctx, key, q) +} + +func cloneValues(q url.Values) url.Values { + out := make(url.Values, len(q)+1) + for k, v := range q { + out[k] = append([]string(nil), v...) + } + return out +} + // Probe is Check as a providers.Checker: it answers with the errors the provider // settings understand, so that a test does not depend on this client. func (c *Client) Probe(ctx context.Context, key string) error { diff --git a/internal/providers/lastfm/lastfm_test.go b/internal/providers/lastfm/lastfm_test.go index f2bd46f..275063c 100644 --- a/internal/providers/lastfm/lastfm_test.go +++ b/internal/providers/lastfm/lastfm_test.go @@ -351,3 +351,65 @@ func TestTopTagsErrors(t *testing.T) { } } } + +// Last.fm does not know every MusicBrainz id. When it does not know this one, +// the name is tried, which is what was done before the file gave an id. A name +// is only tried then: never when the id worked or failed for another reason. +func TestAnIDLastFMDoesNotKnowFallsBackToTheName(t *testing.T) { + const id = "22222222-2222-2222-2222-222222222222" + unknownThenKnown := func(t *testing.T) (*httptest.Server, *[]url.Values) { + var mu sync.Mutex + var seen []url.Values + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen = append(seen, r.URL.Query()) + n := len(seen) + mu.Unlock() + if r.URL.Query().Has("mbid") && n == 1 { + _, _ = w.Write([]byte(`{"error":6,"message":"The artist you supplied could not be found"}`)) + return + } + if r.URL.Query().Get("method") == "artist.gettoptags" { + _, _ = w.Write([]byte(`{"toptags":{"tag":[{"name":"rock","count":9}]}}`)) + return + } + _, _ = w.Write([]byte(similarBody)) + })) + t.Cleanup(srv.Close) + return srv, &seen + } + client := func(srv *httptest.Server) *Client { + return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( test )", Retries: outbound.NoRetries}), srv.URL) + } + + srv, seen := unknownThenKnown(t) + got, err := client(srv).SimilarArtists(context.Background(), testKey, Artist{Name: "Source", MBID: id}, 10) + if err != nil || len(got) != 2 || len(*seen) != 2 { + t.Fatalf("similar: %v err=%v requests=%d", got, err, len(*seen)) + } + if (*seen)[0].Get("mbid") != id || (*seen)[1].Get("artist") != "Source" || (*seen)[1].Has("mbid") || (*seen)[1].Get("autocorrect") != "0" { + t.Errorf("requests = %v", *seen) + } + + srv, seen = unknownThenKnown(t) + tags, err := client(srv).TopTags(context.Background(), testKey, Artist{Name: "Source", MBID: id}) + if err != nil || len(tags) != 1 || len(*seen) != 2 || (*seen)[1].Get("artist") != "Source" { + t.Fatalf("tags: %v err=%v requests=%v", tags, err, *seen) + } + + // Without a name there is nothing to try. + srv, seen = unknownThenKnown(t) + if _, err := client(srv).SimilarArtists(context.Background(), testKey, Artist{MBID: id}, 10); !errors.Is(err, ErrNotFound) || len(*seen) != 1 { + t.Errorf("no name: err=%v requests=%d", err, len(*seen)) + } + // Another failure is not an unknown artist. + bad := newServer(t, 403, `{"error":10,"message":"Invalid API key"}`) + if _, err := bad.client().SimilarArtists(context.Background(), testKey, Artist{Name: "x", MBID: id}, 10); !errors.Is(err, ErrInvalidKey) || len(bad.queries) != 1 { + t.Errorf("invalid key: err=%v requests=%d", err, len(bad.queries)) + } + // An id that works is the only request. + ok := newServer(t, 200, similarBody) + if _, err := ok.client().SimilarArtists(context.Background(), testKey, Artist{Name: "x", MBID: id}, 10); err != nil || len(ok.queries) != 1 { + t.Errorf("working id: err=%v requests=%d", err, len(ok.queries)) + } +} diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index fa2efe6..2563058 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -357,7 +357,13 @@ func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - f := pf(fmt.Sprintf("scan/%d.flac", round), fmt.Sprintf("Scanned %d", round), "", "", 0, 0, "flac", fmt.Sprintf("Scanned Artist %d", round)) + // A new artist, and one that an earlier round only discovered (adopted by + // name, which drops its id), as the second of the file's artists. + artists := []string{fmt.Sprintf("Scanned Artist %d", round)} + if round > 0 { + artists = append(artists, fmt.Sprintf("Shared With Id %d", round-1)) + } + f := pf(fmt.Sprintf("scan/%d.flac", round), fmt.Sprintf("Scanned %d", round), "", "", 0, 0, "flac", artists...) if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { errs <- fmt.Errorf("SaveFile: %w", err) } @@ -592,3 +598,132 @@ func TestDeletingAnArtistRemovesItsTags(t *testing.T) { } }) } + +// ---- review: adopting by id ---- + +func artistsNamed(t *testing.T, db *bun.DB, key string) int { + t.Helper() + return count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = '"+key+"'") +} + +// The key follows the name: an artist adopted under another spelling must be +// found by that spelling afterwards. +func TestAnAdoptedArtistIsFoundByItsNewSpelling(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000001" + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beyonce", MBID: m, Match: 1}}) + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("y/1.flac", "Y1", "", "", 0, 0, "flac", "Beyoncé"), "", m), 0); err != nil { + t.Fatal(err) + } + if _, err := lib.SaveFile(ctx, s.rock, pf("y/2.flac", "Y2", "", "", 0, 0, "flac", "Beyoncé"), 0); err != nil { + t.Fatal(err) + } + if n := artistsNamed(t, db, "beyoncé"); n != 1 { + t.Errorf("artists with the key of Beyoncé = %d, want 1", n) + } + if n := artistsNamed(t, db, "beyonce"); n != 0 { + t.Errorf("the old key is still in use by %d artists", n) + } + }) +} + +// A real artist that has no id yet must receive it, not be split from the +// discovered artist that already holds it. +func TestAFileIDGoesToTheRealArtistNotToTheDiscoveredNamesake(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000002" + + // The user already has The Beatles, without an id. + if _, err := lib.SaveFile(ctx, s.rock, pf("bt/1.flac", "B1", "Abbey Road", "The Beatles", 1, 1, "flac", "The Beatles"), 0); err != nil { + t.Fatal(err) + } + real := artistID(t, db, "The Beatles") + // Last.fm names "Beatles" with the id the real band has. + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beatles", MBID: m, Match: 1}}) + discovered := artistID(t, db, "Beatles") + // A new file of the real band carries that id. + f := withIDs(pf("bt/2.flac", "B2", "Abbey Road", "The Beatles", 2, 1, "flac", "The Beatles"), m, m) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + + if n := artistsNamed(t, db, "the beatles"); n != 1 { + t.Fatalf("artists named The Beatles = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM albums WHERE title_key = 'abbey road'"); n != 1 { + t.Errorf("albums = %d, want 1", n) + } + if mbidOf(t, db, real) != m { + t.Errorf("the real artist's id = %q, want %q", mbidOf(t, db, real), m) + } + if mbidOf(t, db, discovered) != "" { + t.Error("the discovered namesake still holds the id") + } + // The next look at Last.fm's answer lands on the real artist. + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beatles", MBID: m, Match: 1}}) + snap, _ := repo.Similar(ctx, artistID(t, db, "Radiohead"), "lastfm", 5) + if len(snap.Items) != 1 || snap.Items[0].Artist.ID != real || !snap.Items[0].InLibrary { + t.Errorf("after the next refresh = %+v, want the real artist", snap.Items) + } + }) +} + +// Whatever was read about an artist under the provider's id is not known to be +// about the user's artist once that id is dropped. +func TestDroppingAProviderIDAlsoDropsWhatWasReadWithIt(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + radiohead := artistID(t, db, "Radiohead") + const m = "cccccccc-0000-4000-8000-000000000003" + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Namesake", MBID: m, Match: 1}}) + ns := artistID(t, db, "Namesake") + _ = repo.SaveTags(ctx, ns, "musicbrainz", at, true, []discovery.Tag{{Name: "folk", Weight: 5}}) + _ = repo.SaveSimilar(ctx, ns, "lastfm", at, true, []lastfm.Similar{{Name: "Someone Else", Match: 1}}) + + // A file with no id adopts it by name: the id is dropped. + if _, err := lib.SaveFile(ctx, s.rock, pf("n/1.flac", "N1", "", "", 0, 0, "flac", "Namesake"), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, ns) != "" { + t.Fatal("the id stayed") + } + if tg, _ := repo.Tags(ctx, ns, "musicbrainz"); tg.Fetched || len(tg.Tags) != 0 { + t.Errorf("tags read with the dropped id are still served: %+v", tg) + } + if sm, _ := repo.Similar(ctx, ns, "lastfm", 5); sm.Fetched || len(sm.Items) != 0 { + t.Errorf("similar artists read with the dropped id are still served: %+v", sm) + } + // What others said about it stays: Radiohead's list still names it. + if sm, _ := repo.Similar(ctx, radiohead, "lastfm", 5); len(sm.Items) != 1 || sm.Items[0].Artist.ID != ns || !sm.Items[0].InLibrary { + t.Errorf("the link from Radiohead = %+v", sm.Items) + } + }) +} + +// When the file confirms the very same id, what was read is kept. +func TestConfirmingTheIDKeepsWhatWasRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000004" + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Confirmed Two", MBID: m, Match: 1}}) + c := artistID(t, db, "Confirmed Two") + _ = repo.SaveTags(ctx, c, "musicbrainz", at, true, []discovery.Tag{{Name: "folk", Weight: 5}}) + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("n/2.flac", "N2", "", "", 0, 0, "flac", "Confirmed Two"), "", m), 0); err != nil { + t.Fatal(err) + } + if tg, _ := repo.Tags(ctx, c, "musicbrainz"); !tg.Fetched || len(tg.Tags) != 1 { + t.Errorf("tags = %+v", tg) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 4a85582..c9b6516 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -105,10 +105,16 @@ func (r *LibraryRepo) ExistingFiles(ctx context.Context, libraryID int64) ([]lib } // SaveFile creates or updates one file with its track, album and artists in a -// single transaction. MusicBrainz ids are never invented: they stay NULL. +// single transaction. A MusicBrainz id is only stored when the file's own tags +// say so; it is never invented. func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.ParsedFile, existingID int64) (bool, error) { var added bool err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Artists are created and matched here and by discovery: take the turn once, + // before any write, so two transactions cannot wait for each other. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } artistIDs := make([]int64, 0, len(f.Artists)) seen := map[int64]bool{} for i, name := range f.Artists { @@ -379,71 +385,79 @@ func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error { // ---- helpers (run inside a transaction) ---- // findOrCreateArtist returns the artist a file belongs to, creating it if needed. +// The caller holds the artist lock (lockArtistWrites) for the whole transaction: +// taking it here, after other writes, could deadlock with discovery. // -// mbid is what the file's own tags say ("" when they say nothing). With one, two -// artists are the same when the ids are equal, or when the names match and the -// artist we have has no id yet (it then gets the one from the file); another id -// under the same name is another artist. Without one, the name decides, as it -// always did, and a real artist is preferred to one that a provider only named. +// mbid is what the file's own tags say ("" when they say nothing). // -// An artist that a provider only named becomes the user's when a file finds it: -// it takes the spelling of the tags and, unless the file confirms the very same -// id, loses the provider's id, which nobody checked. +// With an id, two artists are the same when the ids are equal; another id under +// the same name is another artist. A real artist of that name that has no id yet +// receives the id, even when a discovered artist already holds it: that one +// lets go of an id nobody checked, and the next answer from the provider lands +// on the real artist. A discovered artist holding exactly this id is confirmed +// by the file and becomes the user's with its data kept. +// +// Without an id the name decides, and a real artist is preferred to a +// discovered one. A discovered artist adopted this way loses the provider's id, +// and with it everything that was read under that id, since it is not known to +// be about the user's artist. func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int64, error) { key := matchKey(name) mbid = strings.ToLower(strings.TrimSpace(mbid)) - if mbid != "" { - // Ids are written here, and discovery writes them too: take turns. - if err := lockArtistWrites(ctx, tx); err != nil { - return 0, err - } - } type row struct { ID int64 Discovered bool + MBID string } - var found row - lookup := func(where string, args ...any) (bool, error) { + lookup := func(where string, args ...any) (row, bool, error) { + var ( + r row + m sql.NullString + ) err := tx.QueryRowContext(ctx, - "SELECT id, discovered FROM artists WHERE "+where+" ORDER BY discovered, id LIMIT 1", args...). - Scan(&found.ID, &found.Discovered) + "SELECT id, discovered, mbid FROM artists WHERE "+where+" ORDER BY discovered, id LIMIT 1", args...). + Scan(&r.ID, &r.Discovered, &m) + r.MBID = m.String if errors.Is(err, sql.ErrNoRows) { - return false, nil + return r, false, nil } - return err == nil, err + return r, err == nil, err + } + exec := func(what, query string, args ...any) error { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("%s: %w", what, err) + } + return nil } - var ( - ok bool - err error - ) if mbid != "" { - if ok, err = lookup("mbid = ?", mbid); err == nil && ok { - return adopt(ctx, tx, found.ID, found.Discovered, name, "", false) + byID, haveID, err := lookup("mbid = ?", mbid) + if err != nil { + return 0, fmt.Errorf("find artist: %w", err) } - if err == nil { - if ok, err = lookup("name_key = ? AND (mbid IS NULL OR mbid = '')", key); err == nil && ok { - return adopt(ctx, tx, found.ID, found.Discovered, name, mbid, true) + if haveID && !byID.Discovered { + return byID.ID, nil // a real artist already has this id, whatever its spelling + } + byName, haveName, err := lookup("name_key = ? AND (mbid IS NULL OR mbid = '')", key) + if err != nil { + return 0, fmt.Errorf("find artist: %w", err) + } + switch { + case haveName && !byName.Discovered: + if haveID { + if err := exec("release discovered id", "UPDATE artists SET mbid = NULL WHERE id = ?", byID.ID); err != nil { + return 0, err + } } + return byName.ID, exec("set artist id", "UPDATE artists SET mbid = ? WHERE id = ?", mbid, byName.ID) + case haveID: + return byID.ID, exec("confirm discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ? WHERE id = ?", name, key, false, byID.ID) + case haveName: + return byName.ID, exec("adopt discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, false, mbid, byName.ID) } - } else { - if ok, err = lookup("name_key = ?", key); err == nil && !ok { - // About to create one: wait for anyone else who is, then look again. - if err = lockArtistWrites(ctx, tx); err == nil { - ok, err = lookup("name_key = ?", key) - } - } - if err == nil && ok { - return adopt(ctx, tx, found.ID, found.Discovered, name, "", found.Discovered) - } - } - if err != nil { - return 0, fmt.Errorf("find artist: %w", err) - } - - a := artistModel{Name: name, NameKey: key} - if mbid != "" { var id int64 if err := tx.QueryRowContext(ctx, "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", @@ -452,36 +466,37 @@ func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int6 } return id, nil } - if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { - return 0, fmt.Errorf("insert artist: %w", err) - } - return a.ID, nil -} -// adopt settles an artist that a file just matched. setID is the id the file -// gives it ("" for none); drop removes the id a provider had given a discovered -// artist. Only a discovered artist changes its name and loses its mark. -func adopt(ctx context.Context, tx bun.Tx, id int64, discovered bool, name, setID string, drop bool) (int64, error) { - q := tx.NewUpdate().Model((*artistModel)(nil)).Where("id = ?", id) - changed := false - if discovered { - q = q.Set("name = ?", name).Set("discovered = ?", false) - changed = true + found, ok, err := lookup("name_key = ?", key) + if err != nil { + return 0, fmt.Errorf("find artist: %w", err) } - switch { - case setID != "": - q = q.Set("mbid = ?", setID) - changed = true - case drop && discovered: - q = q.Set("mbid = NULL") + if !ok { + a := artistModel{Name: name, NameKey: key} + if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { + return 0, fmt.Errorf("insert artist: %w", err) + } + return a.ID, nil } - if !changed { - return id, nil + if found.Discovered { + if err := exec("adopt discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, false, found.ID); err != nil { + return 0, err + } + if found.MBID != "" { + // What was read under the provider's id is not known to be about this artist. + for _, q := range []string{ + "DELETE FROM artist_tags WHERE artist_id = ?", + "DELETE FROM provider_fetches WHERE artist_id = ?", + "DELETE FROM artist_similar WHERE artist_id = ?", + } { + if err := exec("forget what was read with the dropped id", q, found.ID); err != nil { + return 0, err + } + } + } } - if _, err := q.Exec(ctx); err != nil { - return 0, fmt.Errorf("update artist: %w", err) - } - return id, nil + return found.ID, nil } func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) {