diff --git a/README.md b/README.md index 0335128..fe75cdc 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. +- **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet -Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. MusicBrainz can be set up (see [External providers](#external-providers)) but nothing asks it anything yet. +Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Genres from your own files, and the removal of provider data on request. ## Requirements @@ -176,7 +177,7 @@ curl -s -X POST $BASE/users -H "$AUTH" -d '{"username":"bob","password":"another ## External providers -Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today only Last.fm is used, to find similar artists. +Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, and MusicBrainz for tags. - **Last.fm** needs an API key. Create one for your own use at `last.fm/api` (it needs a Last.fm account). No user name or password is involved and the "shared secret" is not needed. Last.fm data is for non-commercial use only, and you should credit Last.fm when you show it. - **MusicBrainz** needs no key, but it asks every client to identify itself with a contact (a URL or an e-mail address) in the `User-Agent`. By default Sonda uses the address of the project, so no personal address is sent unless you choose to. @@ -188,15 +189,20 @@ curl -s -X PUT $BASE/providers/lastfm -H "$AUTH" -d '{"api_key":" tags[j].Weight }) + if len(tags) > MaxTags { + tags = tags[:MaxTags] + } + return s.store.SaveTags(ctx, a.ID, provider, s.now(), len(tags) > 0, tags) +} + +// LastfmTags adapts the Last.fm client to a TagSource. +type LastfmTags struct{ Client *lastfm.Client } + +// ArtistTags implements TagSource. +func (l LastfmTags) ArtistTags(ctx context.Context, key string, a Artist) ([]Tag, error) { + got, err := l.Client.TopTags(ctx, key, lastfm.Artist{Name: a.Name, MBID: a.MBID}) + if errors.Is(err, lastfm.ErrNotFound) { + return nil, ErrSourceNotFound + } + if err != nil { + return nil, err + } + out := make([]Tag, len(got)) + for i, t := range got { + out[i] = Tag{Name: t.Name, Weight: float64(t.Count)} + } + return out, nil +} + +// MusicBrainzTags adapts the MusicBrainz client to a TagSource. It only ever +// asks by the id of the artist. +type MusicBrainzTags struct{ Client *musicbrainz.Client } + +// ArtistTags implements TagSource. +func (m MusicBrainzTags) ArtistTags(ctx context.Context, _ string, a Artist) ([]Tag, error) { + got, err := m.Client.ArtistTags(ctx, a.MBID) + if errors.Is(err, musicbrainz.ErrNotFound) { + return nil, ErrSourceNotFound + } + if err != nil { + return nil, err + } + out := make([]Tag, len(got)) + for i, t := range got { + out[i] = Tag{Name: t.Name, Weight: float64(t.Count), Genre: t.Genre} + } + return out, nil +} diff --git a/internal/discovery/tags_test.go b/internal/discovery/tags_test.go new file mode 100644 index 0000000..1f814a1 --- /dev/null +++ b/internal/discovery/tags_test.go @@ -0,0 +1,295 @@ +package discovery + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/providers" +) + +type savedTags struct { + artistID int64 + provider string + at time.Time + found bool + tags []Tag +} + +// tagStore keeps one snapshot per artist and provider. +type tagStore struct { + mu sync.Mutex + artists map[int64]Artist + snaps map[string]TagSnapshot + saves []savedTags +} + +func newTagStore() *tagStore { + return &tagStore{ + artists: map[int64]Artist{ + 1: {ID: 1, Name: "Source"}, + 2: {ID: 2, Name: "With Id", MBID: "aaaaaaaa-0000-4000-8000-000000000001"}, + }, + snaps: map[string]TagSnapshot{}, + } +} + +func tkey(id int64, provider string) string { return provider + "/" + string(rune('0'+id)) } + +func (f *tagStore) Artist(_ context.Context, id int64) (Artist, error) { + f.mu.Lock() + defer f.mu.Unlock() + a, ok := f.artists[id] + if !ok { + return Artist{}, ErrNotFound + } + return a, nil +} + +func (f *tagStore) Tags(_ context.Context, id int64, provider string) (TagSnapshot, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.snaps[tkey(id, provider)], nil +} + +func (f *tagStore) SaveTags(_ context.Context, id int64, provider string, at time.Time, found bool, tags []Tag) error { + f.mu.Lock() + defer f.mu.Unlock() + f.saves = append(f.saves, savedTags{id, provider, at, found, tags}) + f.snaps[tkey(id, provider)] = TagSnapshot{Fetched: true, NotFound: !found, RetrievedAt: at, Tags: tags} + return nil +} + +type fakeTagSource struct { + mu sync.Mutex + calls int + key string + artist Artist + tags []Tag + err error + release chan struct{} + boom any +} + +func (s *fakeTagSource) ArtistTags(_ context.Context, key string, a Artist) ([]Tag, error) { + s.mu.Lock() + s.calls++ + s.key, s.artist = key, a + release := s.release + s.mu.Unlock() + if release != nil { + <-release + } + if s.boom != nil { + panic(s.boom) + } + return s.tags, s.err +} + +func (s *fakeTagSource) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +// perProvider answers Active for each provider on its own. +type perProvider map[string]bool + +func (p perProvider) Active(_ context.Context, name string) (providers.Settings, bool, error) { + return providers.Settings{Provider: name, Enabled: p[name], APIKey: key}, p[name], nil +} + +type tagRig struct { + svc *TagService + store *tagStore + lf *fakeTagSource + mb *fakeTagSource + on perProvider + now time.Time +} + +func newTagRig(lastfm, musicbrainz bool) *tagRig { + r := &tagRig{ + store: newTagStore(), lf: &fakeTagSource{}, mb: &fakeTagSource{}, + on: perProvider{providers.LastFM: lastfm, providers.MusicBrainz: musicbrainz}, + now: t0, + } + r.lf.tags = []Tag{{Name: "rock", Weight: 100}, {Name: "grunge", Weight: 40}} + r.mb.tags = []Tag{{Name: "alternative rock", Weight: 43, Genre: true}, {Name: "british", Weight: 2}} + r.svc = NewTagService(r.store, map[string]TagSource{providers.LastFM: r.lf, providers.MusicBrainz: r.mb}, r.on, zap.NewNop()) + r.svc.now = func() time.Time { return r.now } + return r +} + +func (r *tagRig) tags(t *testing.T, id int64) TagResult { + t.Helper() + res, err := r.svc.Tags(context.Background(), id) + if err != nil { + t.Fatalf("Tags: %v", err) + } + return res +} + +func TestTagsOfAnUnknownArtistAreNotFoundAndNobodyIsAsked(t *testing.T) { + r := newTagRig(true, true) + for _, id := range []int64{0, -1, 99} { + if _, err := r.svc.Tags(context.Background(), id); !errors.Is(err, ErrNotFound) { + t.Errorf("id %d: err = %v", id, err) + } + } + if r.lf.count()+r.mb.count() != 0 { + t.Error("a provider was asked about an artist that does not exist") + } +} + +func TestTagsAlwaysListBothProvidersInTheSameOrder(t *testing.T) { + r := newTagRig(false, false) + res := r.tags(t, 1) + if len(res.Sources) != 2 || res.Sources[0].Provider != providers.LastFM || res.Sources[1].Provider != providers.MusicBrainz { + t.Fatalf("sources = %+v", res.Sources) + } + for _, s := range res.Sources { + if s.Status != StatusDisabled || len(s.Tags) != 0 || !s.RetrievedAt.IsZero() || s.Stale || s.Attribution == "" { + t.Errorf("%s = %+v", s.Provider, s) + } + if s.Tags == nil { + t.Errorf("%s: tags is nil, want an empty list", s.Provider) + } + } + if r.lf.count()+r.mb.count() != 0 { + t.Error("a provider that is off was contacted") + } +} + +// MusicBrainz is only asked for an artist whose id is known; Last.fm can be +// asked by name, and by id when there is one. +func TestMusicBrainzIsNeverAskedWithoutAnID(t *testing.T) { + r := newTagRig(true, true) + res := r.tags(t, 1) + if res.Sources[0].Status != StatusOK || len(res.Sources[0].Tags) != 2 || r.lf.artist.Name != "Source" || r.lf.artist.MBID != "" || r.lf.key != key { + t.Errorf("lastfm = %+v asked about %+v with %q", res.Sources[0], r.lf.artist, r.lf.key) + } + if res.Sources[1].Status != StatusNoID || len(res.Sources[1].Tags) != 0 || r.mb.count() != 0 { + t.Errorf("musicbrainz = %+v calls = %d", res.Sources[1], r.mb.count()) + } + + res = r.tags(t, 2) + if r.mb.count() != 1 || r.mb.artist.MBID != "aaaaaaaa-0000-4000-8000-000000000001" { + t.Fatalf("musicbrainz was asked about %+v (%d calls)", r.mb.artist, r.mb.count()) + } + if r.lf.artist.MBID != "aaaaaaaa-0000-4000-8000-000000000001" { + t.Errorf("lastfm was not given the id: %+v", r.lf.artist) + } + mb := res.Sources[1] + if mb.Status != StatusOK || len(mb.Tags) != 2 || !mb.Tags[0].Genre || mb.Tags[0].Name != "alternative rock" || mb.Tags[0].Weight != 43 || !mb.RetrievedAt.Equal(t0) { + t.Errorf("musicbrainz = %+v", mb) + } +} + +func TestTagsOfOneProviderDoNotDependOnTheOther(t *testing.T) { + r := newTagRig(true, true) + r.lf.err = errors.New("last.fm is down") + res := r.tags(t, 2) + if res.Sources[0].Status != StatusError || len(res.Sources[0].Tags) != 0 { + t.Errorf("lastfm = %+v", res.Sources[0]) + } + if res.Sources[1].Status != StatusOK || len(res.Sources[1].Tags) != 2 { + t.Errorf("musicbrainz = %+v: one provider failing must not hide the other", res.Sources[1]) + } +} + +func TestFreshTagsAreNotAskedAgainAndOldOnesAre(t *testing.T) { + r := newTagRig(true, false) + r.tags(t, 1) + r.now = t0.Add(PositiveTTL - time.Second) + res := r.tags(t, 1) + if r.lf.count() != 1 || res.Sources[0].Stale || res.Sources[0].Status != StatusOK { + t.Errorf("calls = %d result = %+v", r.lf.count(), res.Sources[0]) + } + r.now = t0.Add(PositiveTTL) + r.tags(t, 1) + if r.lf.count() != 2 { + t.Errorf("calls = %d, want 2", r.lf.count()) + } +} + +func TestUnknownAtTheProviderIsRememberedForADayAndOldDataSurvivesFailure(t *testing.T) { + r := newTagRig(true, false) + r.lf.err = ErrSourceNotFound + res := r.tags(t, 1) + if res.Sources[0].Status != StatusOK || len(res.Sources[0].Tags) != 0 || len(r.store.saves) != 1 || r.store.saves[0].found { + t.Fatalf("result = %+v saves = %+v", res.Sources[0], r.store.saves) + } + r.now = t0.Add(NegativeTTL - time.Second) + r.tags(t, 1) + if r.lf.count() != 1 { + t.Errorf("calls = %d: a negative answer is kept for %v", r.lf.count(), NegativeTTL) + } + + // Old data is served when the provider fails. + r2 := newTagRig(true, false) + r2.tags(t, 1) + r2.now = t0.Add(PositiveTTL + time.Hour) + r2.lf.err = errors.New("down") + res = r2.tags(t, 1) + s := res.Sources[0] + if !s.Stale || s.Status != StatusError || len(s.Tags) != 2 || !s.RetrievedAt.Equal(t0) || len(r2.store.saves) != 1 { + t.Errorf("result = %+v saves = %d", s, len(r2.store.saves)) + } +} + +func TestAnEmptyTagListIsRememberedLikeAnUnknownArtist(t *testing.T) { + r := newTagRig(true, false) + r.lf.tags = nil + r.tags(t, 1) + if len(r.store.saves) != 1 || r.store.saves[0].found { + t.Errorf("saves = %+v", r.store.saves) + } +} + +func TestOnlyTheBestTagsAreKeptAndNamesAreClean(t *testing.T) { + r := newTagRig(true, false) + var many []Tag + for i := 0; i < MaxTags+20; i++ { + many = append(many, Tag{Name: "tag" + string(rune('a'+i%26)) + string(rune('a'+i/26)), Weight: float64(1000 - i)}) + } + r.lf.tags = many + res := r.tags(t, 1) + if got := len(res.Sources[0].Tags); got != MaxTags || res.Sources[0].Tags[0].Weight != 1000 { + t.Errorf("kept %d tags, first weight %v; want %d, best first", got, res.Sources[0].Tags[0].Weight, MaxTags) + } +} + +func TestAPanicWhileReadingTagsIsAFailureNotACrash(t *testing.T) { + r := newTagRig(true, false) + r.lf.boom = "bug" + res, err := r.svc.Tags(context.Background(), 1) + if err != nil || res.Sources[0].Status != StatusError { + t.Fatalf("result = %+v err = %v", res.Sources[0], err) + } +} + +func TestConcurrentTagLooksShareOneCall(t *testing.T) { + r := newTagRig(true, false) + r.lf.release = make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if res, err := r.svc.Tags(context.Background(), 1); err != nil || len(res.Sources[0].Tags) != 2 { + t.Errorf("result = %+v err = %v", res, err) + } + }() + } + time.Sleep(300 * time.Millisecond) + close(r.lf.release) + wg.Wait() + if r.lf.count() != 1 { + t.Errorf("calls = %d, want 1", r.lf.count()) + } +} diff --git a/internal/library/taglib.go b/internal/library/taglib.go index 620d270..98dcee7 100644 --- a/internal/library/taglib.go +++ b/internal/library/taglib.go @@ -3,6 +3,7 @@ package library import ( "fmt" "path/filepath" + "regexp" "strconv" "strings" @@ -31,14 +32,24 @@ func (TaglibReader) Read(path string) (Metadata, error) { return Metadata{}, fmt.Errorf("%w: no audio stream found", ErrUnreadable) } + artists := nonEmpty(tags["ARTIST"]) + albumArtist := first(tags, "ALBUMARTIST") + albumArtistID := "" + if albumArtist != "" { + // The id belongs to the album artist tag: when the tag is missing the name + // below is only a fallback, and an id would land on the wrong artist. + albumArtistID = singleID(tags["MUSICBRAINZ_ALBUMARTISTID"]) + } m := Metadata{ Tags: Tags{ - Title: first(tags, "TITLE"), - Album: first(tags, "ALBUM"), - AlbumArtist: first(tags, "ALBUMARTIST"), - Artists: nonEmpty(tags["ARTIST"]), - TrackNo: leadingInt(first(tags, "TRACKNUMBER")), - DiscNo: leadingInt(first(tags, "DISCNUMBER")), + Title: first(tags, "TITLE"), + Album: first(tags, "ALBUM"), + AlbumArtist: albumArtist, + Artists: artists, + ArtistMBIDs: pairArtistIDs(artists, tags["MUSICBRAINZ_ARTISTID"]), + AlbumArtistMBID: albumArtistID, + TrackNo: leadingInt(first(tags, "TRACKNUMBER")), + DiscNo: leadingInt(first(tags, "DISCNUMBER")), }, Properties: Properties{ Format: strings.ToLower(props.Format), @@ -54,6 +65,7 @@ func (TaglibReader) Read(path string) (Metadata, error) { } if len(m.Artists) == 0 { m.Artists = []string{UnknownArtist} + m.ArtistMBIDs = nil // an id for an artist the file never named is not ours to use } if m.AlbumArtist == "" { m.AlbumArtist = m.Artists[0] @@ -61,6 +73,69 @@ func (TaglibReader) Read(path string) (Metadata, error) { return m, nil } +var mbidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// splitIDs reads the MusicBrainz ids out of the values of one tag. Taggers write +// several artists either as several values or as one value joined with "/" or ";". +// ok is false when any piece is not a MusicBrainz id. +func splitIDs(values []string) (ids []string, ok bool) { + for _, v := range values { + for _, piece := range strings.FieldsFunc(v, func(r rune) bool { return r == '/' || r == ';' || r == ',' }) { + piece = strings.TrimSpace(piece) + if piece == "" { + continue + } + if !mbidRe.MatchString(piece) { + return nil, false + } + ids = append(ids, strings.ToLower(piece)) + } + } + return ids, true +} + +// pairArtistIDs returns one id per artist, in order, or nothing. Fewer or more +// ids than artists, or an id that is not one, means the pairing would be a guess, +// and a wrong id on an artist is worse than none. +func pairArtistIDs(artists, values []string) []string { + ids, ok := splitIDs(values) + if !ok || len(ids) == 0 || len(ids) != len(artists) { + return nil + } + // One credit with one id ("X feat. Y" with the id of X) is the usual way a + // tagger writes the id of the first artist only. Without a way to tell, no id. + if len(artists) == 1 && looksLikeCredit(artists[0]) { + return nil + } + return ids +} + +// creditWords are the separators of a credit that names several artists. A slash +// is not one: AC/DC is a band. +var creditWords = []string{" feat.", " feat ", " featuring ", " ft.", " ft ", " with ", " & ", " and ", " vs ", " vs. ", " x ", ",", ";"} + +// looksLikeCredit says whether a name seems to credit more than one artist. It +// errs on the side of "yes": a missing id costs a lookup, a wrong one a wrong artist. +func looksLikeCredit(name string) bool { + n := " " + strings.ToLower(name) + " " + for _, w := range creditWords { + // Text must stand on both sides: a band called "X" is not a credit. + if i := strings.Index(n, w); i > 1 && i+len(w) < len(n)-1 { + return true + } + } + return false +} + +// singleID returns the id when the tag holds exactly one valid id. +func singleID(values []string) string { + ids, ok := splitIDs(values) + if !ok || len(ids) != 1 { + return "" + } + return ids[0] +} + func first(tags map[string][]string, key string) string { for _, v := range tags[key] { if v = strings.TrimSpace(v); v != "" { diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go index 092387f..8f5aaf9 100644 --- a/internal/library/taglib_test.go +++ b/internal/library/taglib_test.go @@ -121,3 +121,117 @@ func TestScanWithRealFiles(t *testing.T) { t.Errorf("a.mp3 row = %+v", r) } } + +// ---- MusicBrainz ids in the file tags ---- + +func TestArtistIDsAreOnlyKeptWhenTheyCanBePairedWithTheirArtists(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + const b = "66666666-7777-4888-8999-aaaaaaaaaaaa" + tests := []struct { + name string + artists []string + ids []string + want []string + }{ + {"one and one", []string{"X"}, []string{a}, []string{a}}, + {"two and two, in order", []string{"X", "Y"}, []string{a, b}, []string{a, b}}, + {"upper case is lowered", []string{"X"}, []string{"11111111-2222-4333-8444-55555555555A"}, []string{"11111111-2222-4333-8444-55555555555a"}}, + {"ids joined in one value", []string{"X", "Y"}, []string{a + "/" + b}, []string{a, b}}, + {"ids joined with a semicolon", []string{"X", "Y"}, []string{a + "; " + b}, []string{a, b}}, + {"fewer ids than artists is a guess", []string{"X", "Y"}, []string{a}, nil}, + {"more ids than artists is a guess", []string{"X"}, []string{a, b}, nil}, + {"no ids", []string{"X"}, nil, nil}, + {"one bad id spoils the pairing", []string{"X", "Y"}, []string{a, "not-an-id"}, nil}, + {"empty values are ignored", []string{"X"}, []string{" ", a}, []string{a}}, + // A credit with one id is usually the id of the first artist only. + {"a featuring credit with one id", []string{"X feat. Y"}, []string{a}, nil}, + {"ft. credit", []string{"X ft. Y"}, []string{a}, nil}, + {"with an ampersand", []string{"Simon & Garfunkel"}, []string{a}, nil}, + {"with a comma", []string{"X, Y"}, []string{a}, nil}, + {"a slash is part of a name", []string{"AC/DC"}, []string{a}, []string{a}}, + {"a plain name that contains 'and' inside a word", []string{"Sandra Bland"}, []string{a}, []string{a}}, + {"credits are fine when each artist has an id", []string{"X", "Y"}, []string{a, b}, []string{a, b}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := pairArtistIDs(tt.artists, tt.ids) + if len(got) != len(tt.want) { + t.Fatalf("ids = %v, want %v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("ids = %v, want %v", got, tt.want) + } + } + }) + } + if id := singleID([]string{a}); id != a { + t.Errorf("singleID = %q", id) + } + if id := singleID([]string{a, b}); id != "" { + t.Errorf("two ids for the album artist: %q, want none", id) + } + if id := singleID([]string{"nope"}); id != "" { + t.Errorf("a bad id: %q, want none", id) + } +} + +func TestTaglibReaderReadsMusicBrainzArtistIDs(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + const aa = "66666666-7777-4888-8999-aaaaaaaaaaaa" + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "T", "artist": "Synthetic Band", "album_artist": "Synthetic Band", + "MUSICBRAINZ_ARTISTID": a, "MUSICBRAINZ_ALBUMARTISTID": aa, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if len(m.ArtistMBIDs) != 1 || m.ArtistMBIDs[0] != a || m.AlbumArtistMBID != aa { + t.Errorf("ids = %v / %q", m.ArtistMBIDs, m.AlbumArtistMBID) + } + + // A file without them has none, and a file with no artist tag has no pairing. + p = makeAudio(t, t.TempDir(), "plain.flac", map[string]string{"title": "T", "MUSICBRAINZ_ARTISTID": a}) + m, _ = TaglibReader{}.Read(p) + if len(m.ArtistMBIDs) != 0 { + t.Errorf("an id for an artist nobody named: %v", m.ArtistMBIDs) + } +} + +// The id of the album artist belongs to the name of the album artist tag. When +// the tag is missing the name is only a fallback, and an id would land on the +// wrong artist. +func TestAlbumArtistIDNeedsAnAlbumArtistTag(t *testing.T) { + const variousArtists = "89ad4ac3-39f7-470e-963a-56509c546377" + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "T", "artist": "Foo", "MUSICBRAINZ_ALBUMARTISTID": variousArtists, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if m.AlbumArtist != "Foo" || m.AlbumArtistMBID != "" { + t.Errorf("album artist = %q with id %q: the id must not follow a fallback name", m.AlbumArtist, m.AlbumArtistMBID) + } + + p = makeAudio(t, t.TempDir(), "nothing.flac", map[string]string{"title": "T", "MUSICBRAINZ_ALBUMARTISTID": variousArtists}) + m, _ = TaglibReader{}.Read(p) + if m.AlbumArtistMBID != "" { + t.Errorf("an id for an album artist nobody named: %q", m.AlbumArtistMBID) + } +} + +func TestTaglibReaderReadsMusicBrainzIDsFromMP3(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + p := makeAudio(t, t.TempDir(), "song.mp3", map[string]string{ + "title": "T", "artist": "Synthetic Band", "MusicBrainz Artist Id": a, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if len(m.ArtistMBIDs) != 1 || m.ArtistMBIDs[0] != a { + t.Errorf("ids = %v: an ID3 'MusicBrainz Artist Id' frame should be read", m.ArtistMBIDs) + } +} diff --git a/internal/library/types.go b/internal/library/types.go index 9fc50df..13b1d6b 100644 --- a/internal/library/types.go +++ b/internal/library/types.go @@ -13,13 +13,19 @@ import ( var ErrUnreadable = errors.New("unreadable audio file") // Tags are the descriptive tags of a track. Fields the file lacks are empty. +// +// The MusicBrainz ids are what the file itself says, read offline. ArtistMBIDs is +// either empty or has one id per entry of Artists, in the same order: ids that +// cannot be paired with their artists with certainty are not kept. type Tags struct { - Title string - Album string - AlbumArtist string - Artists []string - TrackNo int - DiscNo int + Title string + Album string + AlbumArtist string + Artists []string + ArtistMBIDs []string + AlbumArtistMBID string + TrackNo int + DiscNo int } // Properties are the technical properties of an audio file. diff --git a/internal/providers/lastfm/lastfm.go b/internal/providers/lastfm/lastfm.go index be3ca1c..cfac7ae 100644 --- a/internal/providers/lastfm/lastfm.go +++ b/internal/providers/lastfm/lastfm.go @@ -17,8 +17,10 @@ import ( "errors" "fmt" "net/url" + "sort" "strconv" "strings" + "unicode/utf8" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/outbound" @@ -92,14 +94,7 @@ func (c *Client) SimilarArtists(ctx context.Context, key string, a Artist, limit case limit > MaxLimit: limit = MaxLimit } - q := url.Values{"method": {"artist.getsimilar"}, "autocorrect": {"0"}, "limit": {strconv.Itoa(limit)}} - if mbid != "" { - q.Set("mbid", mbid) - } else { - q.Set("artist", name) - } - - body, err := c.call(ctx, key, q) + body, err := c.callFor(ctx, key, name, mbid, url.Values{"method": {"artist.getsimilar"}, "limit": {strconv.Itoa(limit)}}) if err != nil { return nil, err } @@ -114,6 +109,84 @@ func (c *Client) SimilarArtists(ctx context.Context, key string, a Artist, limit return decodeSimilar(doc.Similar.Artist) } +// Tag is one tag of an artist. Count is Last.fm's own number between 0 and 100, +// relative to the artist (the top tag is usually 100). +type Tag struct { + Name string // lower case + Count int +} + +// maxTagLen bounds a tag name; longer ones are not tags, they are noise. +const maxTagLen = 100 + +// TopTags returns the top tags of an artist, most voted first. The name is never +// autocorrected, and the MusicBrainz id wins over the name when known. +func (c *Client) TopTags(ctx context.Context, key string, a Artist) ([]Tag, error) { + name, mbid := strings.TrimSpace(a.Name), strings.TrimSpace(a.MBID) + if name == "" && mbid == "" { + return nil, ErrBadRequest + } + body, err := c.callFor(ctx, key, name, mbid, url.Values{"method": {"artist.gettoptags"}}) + if err != nil { + return nil, err + } + var doc struct { + Top *struct { + Tag json.RawMessage `json:"tag"` + } `json:"toptags"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.Top == nil { + return nil, ErrBadResponse + } + return decodeTags(doc.Top.Tag) +} + +// decodeTags reads the tag list, which Last.fm sends as an array or, with one +// tag, as an object. Names are lower-cased and kept once with their highest +// count; empty names, long names and counts of 0 or less are dropped. +func decodeTags(raw json.RawMessage) ([]Tag, error) { + type item struct { + Name string `json:"name"` + Count json.RawMessage `json:"count"` + } + var items []item + switch { + case len(raw) == 0 || string(raw) == "null": + case raw[0] == '[': + if err := json.Unmarshal(raw, &items); err != nil { + return nil, ErrBadResponse + } + default: + var one item + if err := json.Unmarshal(raw, &one); err != nil { + return nil, ErrBadResponse + } + items = []item{one} + } + best := map[string]int{} + for _, it := range items { + name := strings.ToLower(strings.TrimSpace(it.Name)) + count, err := strconv.ParseFloat(strings.Trim(string(it.Count), `"`), 64) + if name == "" || utf8.RuneCountInString(name) > maxTagLen || err != nil || count < 1 || count > 100 { + continue + } + if int(count) > best[name] { + best[name] = int(count) + } + } + out := make([]Tag, 0, len(best)) + for name, count := range best { + out = append(out, Tag{Name: name, Count: count}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out, nil +} + // Check asks one cheap question to find out whether the key works. func (c *Client) Check(ctx context.Context, key string) error { body, err := c.call(ctx, key, url.Values{"method": {"chart.gettopartists"}, "limit": {"1"}}) @@ -129,6 +202,35 @@ func (c *Client) Check(ctx context.Context, key string) error { return nil } +// callFor asks about one artist. The MusicBrainz id goes first, since it cannot be +// confused with a namesake. Last.fm does not know every id: when it says it does +// not know this one and there is a name, the name is tried, never autocorrected, +// which is what was done before an id was known. Any other failure is final. +func (c *Client) callFor(ctx context.Context, key, name, mbid string, q url.Values) ([]byte, error) { + q.Set("autocorrect", "0") + if mbid != "" { + byID := cloneValues(q) + byID.Set("mbid", mbid) + body, err := c.call(ctx, key, byID) + if !errors.Is(err, ErrNotFound) || name == "" { + return body, err + } + } + if name == "" { + return nil, ErrNotFound + } + q.Set("artist", name) + return c.call(ctx, key, q) +} + +func cloneValues(q url.Values) url.Values { + out := make(url.Values, len(q)+1) + for k, v := range q { + out[k] = append([]string(nil), v...) + } + return out +} + // Probe is Check as a providers.Checker: it answers with the errors the provider // settings understand, so that a test does not depend on this client. func (c *Client) Probe(ctx context.Context, key string) error { diff --git a/internal/providers/lastfm/lastfm_test.go b/internal/providers/lastfm/lastfm_test.go index 4da95a5..275063c 100644 --- a/internal/providers/lastfm/lastfm_test.go +++ b/internal/providers/lastfm/lastfm_test.go @@ -282,3 +282,134 @@ func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { t.Errorf("an unreadable answer: err = %v", err) } } + +// ---- tags ---- + +const tagsBody = `{"toptags":{"tag":[ + {"name":"Rock","count":100,"url":"https://example.invalid/tag/rock"}, + {"name":"alternative rock","count":"54","url":"https://example.invalid/x"}, + {"name":"rock","count":10}, + {"name":"","count":9}, + {"name":"zero","count":0} +],"@attr":{"artist":"Source"}}}` + +func TestTopTagsDecodesTheAnswer(t *testing.T) { + s := newServer(t, 200, tagsBody) + got, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "Source"}) + if err != nil { + t.Fatal(err) + } + // Names are lower-cased and kept once (the higher count wins); empty names and + // a count of 0 are dropped; the count may come as a string. + if len(got) != 2 || got[0] != (Tag{Name: "rock", Count: 100}) || got[1] != (Tag{Name: "alternative rock", Count: 54}) { + t.Fatalf("tags = %+v", got) + } + q := s.last(t) + want := map[string]string{"method": "artist.gettoptags", "artist": "Source", "autocorrect": "0", "api_key": testKey, "format": "json"} + for k, v := range want { + if q.Get(k) != v { + t.Errorf("%s = %q, want %q", k, q.Get(k), v) + } + } + if len(q) != len(want) { + t.Errorf("query = %v", q) + } +} + +func TestTopTagsPrefersTheMBIDAndToleratesShapes(t *testing.T) { + s := newServer(t, 200, `{"toptags":{"tag":{"name":"solo","count":7}}}`) + got, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "x", MBID: "22222222-2222-2222-2222-222222222222"}) + if err != nil || len(got) != 1 || got[0].Name != "solo" { + t.Fatalf("tags = %+v err=%v", got, err) + } + if q := s.last(t); q.Get("mbid") != "22222222-2222-2222-2222-222222222222" || q.Has("artist") || q.Get("autocorrect") != "0" { + t.Errorf("query = %v", q) + } + empty := newServer(t, 200, `{"toptags":{"tag":[],"@attr":{"artist":"x"}}}`) + if got, err := empty.client().TopTags(context.Background(), testKey, Artist{Name: "x"}); err != nil || len(got) != 0 { + t.Errorf("no tags: %+v err=%v", got, err) + } +} + +func TestTopTagsErrors(t *testing.T) { + if _, err := newServer(t, 200, tagsBody).client().TopTags(context.Background(), testKey, Artist{}); !errors.Is(err, ErrBadRequest) { + t.Errorf("nothing to look up: %v", err) + } + for _, tt := range []struct { + status int + body string + want error + }{ + {200, `{"error":6,"message":"The artist you supplied could not be found"}`, ErrNotFound}, + {403, `{"error":10,"message":"Invalid API key"}`, ErrInvalidKey}, + {200, `{"hello":"world"}`, ErrBadResponse}, + {500, `x`, ErrUnavailable}, + } { + s := newServer(t, tt.status, tt.body) + if _, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "x"}); !errors.Is(err, tt.want) { + t.Errorf("%d %s: err = %v, want %v", tt.status, tt.body, err, tt.want) + } + } +} + +// Last.fm does not know every MusicBrainz id. When it does not know this one, +// the name is tried, which is what was done before the file gave an id. A name +// is only tried then: never when the id worked or failed for another reason. +func TestAnIDLastFMDoesNotKnowFallsBackToTheName(t *testing.T) { + const id = "22222222-2222-2222-2222-222222222222" + unknownThenKnown := func(t *testing.T) (*httptest.Server, *[]url.Values) { + var mu sync.Mutex + var seen []url.Values + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen = append(seen, r.URL.Query()) + n := len(seen) + mu.Unlock() + if r.URL.Query().Has("mbid") && n == 1 { + _, _ = w.Write([]byte(`{"error":6,"message":"The artist you supplied could not be found"}`)) + return + } + if r.URL.Query().Get("method") == "artist.gettoptags" { + _, _ = w.Write([]byte(`{"toptags":{"tag":[{"name":"rock","count":9}]}}`)) + return + } + _, _ = w.Write([]byte(similarBody)) + })) + t.Cleanup(srv.Close) + return srv, &seen + } + client := func(srv *httptest.Server) *Client { + return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( test )", Retries: outbound.NoRetries}), srv.URL) + } + + srv, seen := unknownThenKnown(t) + got, err := client(srv).SimilarArtists(context.Background(), testKey, Artist{Name: "Source", MBID: id}, 10) + if err != nil || len(got) != 2 || len(*seen) != 2 { + t.Fatalf("similar: %v err=%v requests=%d", got, err, len(*seen)) + } + if (*seen)[0].Get("mbid") != id || (*seen)[1].Get("artist") != "Source" || (*seen)[1].Has("mbid") || (*seen)[1].Get("autocorrect") != "0" { + t.Errorf("requests = %v", *seen) + } + + srv, seen = unknownThenKnown(t) + tags, err := client(srv).TopTags(context.Background(), testKey, Artist{Name: "Source", MBID: id}) + if err != nil || len(tags) != 1 || len(*seen) != 2 || (*seen)[1].Get("artist") != "Source" { + t.Fatalf("tags: %v err=%v requests=%v", tags, err, *seen) + } + + // Without a name there is nothing to try. + srv, seen = unknownThenKnown(t) + if _, err := client(srv).SimilarArtists(context.Background(), testKey, Artist{MBID: id}, 10); !errors.Is(err, ErrNotFound) || len(*seen) != 1 { + t.Errorf("no name: err=%v requests=%d", err, len(*seen)) + } + // Another failure is not an unknown artist. + bad := newServer(t, 403, `{"error":10,"message":"Invalid API key"}`) + if _, err := bad.client().SimilarArtists(context.Background(), testKey, Artist{Name: "x", MBID: id}, 10); !errors.Is(err, ErrInvalidKey) || len(bad.queries) != 1 { + t.Errorf("invalid key: err=%v requests=%d", err, len(bad.queries)) + } + // An id that works is the only request. + ok := newServer(t, 200, similarBody) + if _, err := ok.client().SimilarArtists(context.Background(), testKey, Artist{Name: "x", MBID: id}, 10); err != nil || len(ok.queries) != 1 { + t.Errorf("working id: err=%v requests=%d", err, len(ok.queries)) + } +} diff --git a/internal/providers/musicbrainz/musicbrainz.go b/internal/providers/musicbrainz/musicbrainz.go new file mode 100644 index 0000000..b17afad --- /dev/null +++ b/internal/providers/musicbrainz/musicbrainz.go @@ -0,0 +1,178 @@ +// Package musicbrainz is a small read-only client for the MusicBrainz web +// service. +// +// Sonda only reads an artist whose MusicBrainz id it already knows. It never +// searches by name: a search returns look-alikes (a conductor named like a rock +// band) and picking one would be a guess. Only the id leaves the server. +// +// MusicBrainz asks every client to identify itself with a contact in the +// User-Agent and to stay at one request per second: both are the business of the +// outbound client this one sends its requests through. Errors never repeat the +// provider's own text. +package musicbrainz + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "unicode/utf8" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +// BaseURL is the MusicBrainz web service, version 2. +const BaseURL = "https://musicbrainz.org/ws/2/" + +// probeID is a public artist used to check that the service answers. +const probeID = "a74b1b7f-71a5-4011-9441-d0b5e4122711" + +// maxTagLen bounds a tag name; longer ones are not tags, they are noise. +const maxTagLen = 100 + +// Errors a call can return, to be matched with errors.Is. +var ( + ErrBadRequest = errors.New("musicbrainz: not a MusicBrainz id") + ErrNotFound = errors.New("musicbrainz: unknown artist") + ErrRateLimited = errors.New("musicbrainz: too many requests") + ErrUnavailable = errors.New("musicbrainz: service unavailable") + ErrBadResponse = errors.New("musicbrainz: unreadable answer") +) + +var idRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// Getter performs a GET. *outbound.Client is the one used in production. +type Getter interface { + Get(ctx context.Context, rawURL string) (*outbound.Response, error) +} + +// Client talks to MusicBrainz. It is safe for concurrent use. +type Client struct { + http Getter + base string +} + +// New builds a Client that sends its requests through g to base (BaseURL in +// production). +func New(g Getter, base string) *Client { return &Client{http: g, base: base} } + +// Tag is one tag of an artist. Count is the number of people who voted for it, an +// absolute number (unlike Last.fm's, which is relative). Genre says that +// MusicBrainz also lists it among its curated genres. +type Tag struct { + Name string // lower case + Count int + Genre bool +} + +// ArtistTags returns the tags of the artist with this MusicBrainz id, most voted +// first. An artist nobody tagged is an empty list, not an error. +func (c *Client) ArtistTags(ctx context.Context, id string) ([]Tag, error) { + if !idRe.MatchString(id) { + return nil, ErrBadRequest + } + body, err := c.get(ctx, c.base+"artist/"+strings.ToLower(id)+"?inc=tags+genres&fmt=json") + if err != nil { + return nil, err + } + var doc struct { + ID string `json:"id"` + Tags []struct { + Name string `json:"name"` + Count int `json:"count"` + } `json:"tags"` + Genres []struct { + Name string `json:"name"` + Count int `json:"count"` + } `json:"genres"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.ID == "" { + return nil, ErrBadResponse + } + + byName := map[string]*Tag{} + add := func(name string, count int, genre bool) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" || utf8.RuneCountInString(name) > maxTagLen || count <= 0 { + return + } + if t, ok := byName[name]; ok { + t.Genre = t.Genre || genre + if count > t.Count { + t.Count = count + } + return + } + byName[name] = &Tag{Name: name, Count: count, Genre: genre} + } + for _, t := range doc.Tags { + add(t.Name, t.Count, false) + } + for _, g := range doc.Genres { + add(g.Name, g.Count, true) + } + out := make([]Tag, 0, len(byName)) + for _, t := range byName { + out = append(out, *t) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out, nil +} + +// Probe asks for one public artist, to find out whether the service answers. It +// is a providers.Checker, and answers with the errors the settings understand. +func (c *Client) Probe(ctx context.Context) error { + body, err := c.get(ctx, c.base+"artist/"+probeID+"?fmt=json") + switch { + case errors.Is(err, ErrRateLimited): + return providers.ErrRateLimited + case errors.Is(err, ErrUnavailable): + return providers.ErrUnreachable + case err != nil: + return err + } + var doc struct { + ID string `json:"id"` + } + if json.Unmarshal(body, &doc) != nil || doc.ID == "" { + return ErrBadResponse + } + return nil +} + +// get sends one request and returns the body of a successful answer. +func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { + resp, err := c.http.Get(ctx, rawURL) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + var limited *outbound.RateLimitedError + if errors.As(err, &limited) { + return nil, ErrRateLimited + } + return nil, fmt.Errorf("%w: %w", ErrUnavailable, err) + } + switch { + case resp.Status == 200: + return resp.Body, nil + case resp.Status == 404: + return nil, ErrNotFound + case resp.Status == 400: + return nil, ErrBadRequest + case resp.Status == 429: + return nil, ErrRateLimited + case resp.Status >= 500: + return nil, fmt.Errorf("%w: status %d", ErrUnavailable, resp.Status) + } + return nil, fmt.Errorf("%w: status %d", ErrBadResponse, resp.Status) +} diff --git a/internal/providers/musicbrainz/musicbrainz_test.go b/internal/providers/musicbrainz/musicbrainz_test.go new file mode 100644 index 0000000..3c6a592 --- /dev/null +++ b/internal/providers/musicbrainz/musicbrainz_test.go @@ -0,0 +1,161 @@ +package musicbrainz + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +const mbid = "11111111-2222-4333-8444-555555555555" + +type server struct { + *httptest.Server + mu sync.Mutex + paths []string + hdr http.Header +} + +func newServer(t *testing.T, status int, body string) *server { + t.Helper() + s := &server{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.paths = append(s.paths, r.URL.RequestURI()) + s.hdr = r.Header.Clone() + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(s.Close) + return s +} + +func (s *server) client() *Client { + return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}), s.URL+"/ws/2/") +} + +func (s *server) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.paths...) +} + +const artistBody = `{"id":"11111111-2222-4333-8444-555555555555","name":"Synthetic Band","type":"Group", + "tags":[{"name":"Alternative Rock","count":43},{"name":"british","count":2},{"name":"ambient pop","count":1},{"name":"zero","count":0},{"name":"","count":9}], + "genres":[{"id":"x","name":"alternative rock","count":43,"disambiguation":""},{"id":"y","name":"art pop","count":3,"disambiguation":""}]}` + +func TestArtistTagsReadsTagsAndMarksTheGenres(t *testing.T) { + s := newServer(t, 200, artistBody) + got, err := s.client().ArtistTags(context.Background(), mbid) + if err != nil { + t.Fatal(err) + } + want := []Tag{ + {Name: "alternative rock", Count: 43, Genre: true}, // names are compared in lower case + {Name: "art pop", Count: 3, Genre: true}, // a genre that is not among the tags is kept + {Name: "british", Count: 2}, + {Name: "ambient pop", Count: 1}, + } + if len(got) != len(want) { + t.Fatalf("tags = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("tag %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func TestArtistTagsAsksOnlyForTheIdOfTheArtist(t *testing.T) { + s := newServer(t, 200, artistBody) + if _, err := s.client().ArtistTags(context.Background(), strings.ToUpper(mbid)); err != nil { + t.Fatal(err) + } + reqs := s.requests() + if len(reqs) != 1 || reqs[0] != "/ws/2/artist/"+mbid+"?inc=tags+genres&fmt=json" { + t.Errorf("requests = %v", reqs) + } + if s.hdr.Get("User-Agent") != "Sonda/test ( https://example.org )" { + t.Errorf("User-Agent = %q", s.hdr.Get("User-Agent")) + } +} + +// Anything that is not a plain id never reaches the network: it would end up +// inside the path of the request. +func TestArtistTagsRefusesWhatIsNotAnID(t *testing.T) { + for _, bad := range []string{"", "not-a-uuid", mbid + "/../x", mbid + "?x=1", " " + mbid + "\n", "00000000-0000-0000-0000-00000000000g"} { + s := newServer(t, 200, artistBody) + if _, err := s.client().ArtistTags(context.Background(), bad); !errors.Is(err, ErrBadRequest) { + t.Errorf("%q: err = %v, want ErrBadRequest", bad, err) + } + if len(s.requests()) != 0 { + t.Errorf("%q: a request left", bad) + } + } +} + +func TestArtistTagsErrors(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + want error + }{ + {"unknown id", 404, `{"error":"Not Found"}`, ErrNotFound}, + {"invalid id", 400, `{"error":"Invalid mbid."}`, ErrBadRequest}, + {"too many requests", 429, `{"error":"slow down"}`, ErrRateLimited}, + {"over the limit", 503, `{"error":"Your requests are being throttled"}`, ErrUnavailable}, + {"server error", 500, `oops`, ErrUnavailable}, + {"not json", 200, ``, ErrBadResponse}, + {"empty", 200, ``, ErrBadResponse}, + {"json of another kind", 200, `{"hello":"world"}`, ErrBadResponse}, + {"forbidden", 403, `{"error":"banned"}`, ErrBadResponse}, + } { + t.Run(tt.name, func(t *testing.T) { + s := newServer(t, tt.status, tt.body) + if _, err := s.client().ArtistTags(context.Background(), mbid); !errors.Is(err, tt.want) { + t.Fatalf("err = %v, want %v", err, tt.want) + } + }) + } + // An artist with no tags at all is an answer, not an error. + s := newServer(t, 200, `{"id":"`+mbid+`","name":"Quiet"}`) + got, err := s.client().ArtistTags(context.Background(), mbid) + if err != nil || len(got) != 0 { + t.Errorf("no tags: %+v err=%v", got, err) + } +} + +func TestErrorsDoNotCopyTheProvidersText(t *testing.T) { + s := newServer(t, 400, `{"error":"secret-text-from-the-provider"}`) + _, err := s.client().ArtistTags(context.Background(), mbid) + if err == nil || strings.Contains(err.Error(), "secret-text") { + t.Errorf("err = %v", err) + } +} + +func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { + ok := newServer(t, 200, `{"id":"a74b1b7f-71a5-4011-9441-d0b5e4122711","name":"Anyone"}`) + if err := ok.client().Probe(context.Background()); err != nil { + t.Fatal(err) + } + if reqs := ok.requests(); len(reqs) != 1 || strings.Contains(reqs[0], "inc=") { + t.Errorf("a test asks for one plain artist: %v", reqs) + } + down := newServer(t, 503, `x`) + if err := down.client().Probe(context.Background()); !errors.Is(err, providers.ErrUnreachable) { + t.Errorf("down: %v", err) + } + slow := newServer(t, 429, `x`) + if err := slow.client().Probe(context.Background()); !errors.Is(err, providers.ErrRateLimited) { + t.Errorf("rate limited: %v", err) + } +} diff --git a/internal/providers/outbound/outbound.go b/internal/providers/outbound/outbound.go index 200a32d..aba0b34 100644 --- a/internal/providers/outbound/outbound.go +++ b/internal/providers/outbound/outbound.go @@ -70,8 +70,9 @@ func (realClock) Sleep(ctx context.Context, d time.Duration) error { // Options configure a Client. Zero values mean sensible defaults. type Options struct { - UserAgent string - Timeout time.Duration // per attempt; default 10s + UserAgent string + UserAgentFunc func() string // when set, read for every request instead of UserAgent + Timeout time.Duration // per attempt; default 10s MinInterval time.Duration // minimum time between requests; 0 = no limit MaxBody int64 // largest accepted answer; default 4 MiB @@ -99,7 +100,7 @@ type Response struct { // Client talks to one provider. It is safe for concurrent use. type Client struct { - userAgent string + userAgent func() string timeout time.Duration minInterval time.Duration maxBody int64 @@ -130,7 +131,7 @@ func New(o Options) *Client { return v } c := &Client{ - userAgent: o.UserAgent, + userAgent: func() string { return o.UserAgent }, timeout: def(o.Timeout, 10*time.Second), minInterval: o.MinInterval, maxBody: o.MaxBody, @@ -143,6 +144,9 @@ func New(o Options) *Client { clock: o.Clock, jitter: o.Jitter, } + if o.UserAgentFunc != nil { + c.userAgent = o.UserAgentFunc + } if c.maxBody <= 0 { c.maxBody = 4 << 20 } @@ -268,7 +272,7 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) { if err != nil { return nil, ErrBadURL } - req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("User-Agent", c.userAgent()) req.Header.Set("Accept", "application/json") resp, err := c.hc.Do(req) diff --git a/internal/providers/outbound/outbound_test.go b/internal/providers/outbound/outbound_test.go index b74d3df..e405b47 100644 --- a/internal/providers/outbound/outbound_test.go +++ b/internal/providers/outbound/outbound_test.go @@ -492,8 +492,8 @@ func TestDefaultsAreSane(t *testing.T) { if c.timeout <= 0 || c.maxBody <= 0 || c.retries != 2 || c.breakerThreshold <= 0 { t.Errorf("defaults: timeout=%v maxBody=%d retries=%d threshold=%d", c.timeout, c.maxBody, c.retries, c.breakerThreshold) } - if !strings.Contains(strings.ToLower(c.userAgent), "x") { - t.Errorf("user agent = %q", c.userAgent) + if !strings.Contains(strings.ToLower(c.userAgent()), "x") { + t.Errorf("user agent = %q", c.userAgent()) } } @@ -698,3 +698,24 @@ func TestRedirectsAndOversizedAnswersCountAsFailures(t *testing.T) { }) } } + +// A User-Agent that can change while the server runs (MusicBrainz asks for a +// contact an admin may edit) is read again for every request. +func TestUserAgentFuncIsReadForEveryRequest(t *testing.T) { + var seen []string + srv, _ := server(t, func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Get("User-Agent")) + _, _ = w.Write([]byte("{}")) + }) + current := "Sonda/test ( one )" + c := newClient(newClock(), func(o *Options) { + o.UserAgent = "fixed" + o.UserAgentFunc = func() string { return current } + }) + _, _ = c.Get(context.Background(), srv.URL) + current = "Sonda/test ( two )" + _, _ = c.Get(context.Background(), srv.URL) + if len(seen) != 2 || seen[0] != "Sonda/test ( one )" || seen[1] != "Sonda/test ( two )" { + t.Errorf("User-Agents = %q", seen) + } +} diff --git a/internal/providers/settings.go b/internal/providers/settings.go index 2105b2f..5275dce 100644 --- a/internal/providers/settings.go +++ b/internal/providers/settings.go @@ -99,8 +99,9 @@ var known = []meta{ }, { name: MusicBrainz, - sends: "The artist name or MusicBrainz id you look up, your IP address and the User-Agent, " + - "which carries the contact below. Never your users, your listens, track titles or file paths.", + sends: "The MusicBrainz id of the artist you look up (never a name: Sonda does not search), your " + + "IP address and the User-Agent, which carries the contact below. Never your users, your listens, " + + "track titles or file paths.", note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " + "non-commercial, with attribution to MusicBrainz.", }, @@ -337,3 +338,19 @@ func (s *Service) Test(ctx context.Context, name string) (TestResult, error) { return TestResult{Status: TestFailed}, nil } } + +// UserAgentFor returns a function that builds the User-Agent for a provider from +// the contact stored now, so a change by an admin applies to the next request. +// If the settings cannot be read it uses the project address: a request is never +// held up or sent with a broken header because of it. +func (s *Service) UserAgentFor(name, version string) func() string { + return func() string { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cur, _, err := s.store.Get(ctx, name) + if err != nil { + return UserAgent(version, "") + } + return UserAgent(version, cur.Contact) + } +} diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go index c810806..12d59fd 100644 --- a/internal/providers/settings_test.go +++ b/internal/providers/settings_test.go @@ -346,3 +346,23 @@ func TestTestResultNeverCarriesTheKey(t *testing.T) { t.Errorf("the result repeats the key: %+v", res) } } + +// MusicBrainz asks for a contact, which an admin can change: the User-Agent +// follows the stored value, falls back to the project address, and never fails. +func TestUserAgentForFollowsTheStoredContact(t *testing.T) { + s, st := newSvc() + ua := s.UserAgentFor(MusicBrainz, "1.2.3") + if got := ua(); got != "Sonda/1.2.3 ( "+ProjectURL+" )" { + t.Errorf("default = %q", got) + } + if _, err := s.Update(context.Background(), MusicBrainz, Patch{Contact: strp("https://example.org/me")}); err != nil { + t.Fatal(err) + } + if got := ua(); got != "Sonda/1.2.3 ( https://example.org/me )" { + t.Errorf("after the change = %q", got) + } + st.err = errors.New("db down") + if got := ua(); got != "Sonda/1.2.3 ( "+ProjectURL+" )" { + t.Errorf("with the store failing = %q, want the project address", got) + } +} diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index 168d26d..b8dde5d 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -15,8 +15,11 @@ import ( "codeberg.org/squidd/sonda/internal/providers/lastfm" ) -// kindSimilar is what provider_fetches records for a "similar artists" lookup. -const kindSimilar = "similar" +// What provider_fetches records a lookup as. +const ( + kindSimilar = "similar" + kindTags = "tags" +) // Answers recorded in provider_fetches. const ( @@ -269,3 +272,70 @@ func lockArtistWrites(ctx context.Context, tx bun.Tx) error { // cleanMBID lower-cases an id the way it is stored. func cleanMBID(s string) string { return strings.ToLower(strings.TrimSpace(s)) } + +// Tags reads the tags a provider gave to an artist, heaviest first. +func (r *DiscoveryRepo) Tags(ctx context.Context, id int64, provider string) (discovery.TagSnapshot, error) { + var ( + snap discovery.TagSnapshot + status string + when int64 + ) + err := r.db.QueryRowContext(ctx, + "SELECT status, retrieved_at FROM provider_fetches WHERE provider = ? AND kind = ? AND artist_id = ?", + provider, kindTags, id).Scan(&status, &when) + switch { + case errors.Is(err, sql.ErrNoRows): + return snap, nil + case err != nil: + return snap, fmt.Errorf("read fetch ledger: %w", err) + } + snap.Fetched, snap.NotFound, snap.RetrievedAt = true, status == fetchNotFound, time.Unix(when, 0) + + rows, err := r.db.QueryContext(ctx, + "SELECT tag, weight, genre FROM artist_tags WHERE artist_id = ? AND provider = ? ORDER BY weight DESC, tag", id, provider) + if err != nil { + return snap, fmt.Errorf("read tags: %w", err) + } + defer rows.Close() + snap.Tags = []discovery.Tag{} + for rows.Next() { + var t discovery.Tag + if err := rows.Scan(&t.Name, &t.Weight, &t.Genre); err != nil { + return snap, err + } + snap.Tags = append(snap.Tags, t) + } + return snap, rows.Err() +} + +// SaveTags replaces the tags a provider gave to an artist and records the answer. +func (r *DiscoveryRepo) SaveTags(ctx context.Context, id int64, provider string, at time.Time, found bool, tags []discovery.Tag) error { + return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM artist_tags WHERE artist_id = ? AND provider = ?", id, provider); err != nil { + return fmt.Errorf("clear tags: %w", err) + } + seen := map[string]bool{} + for _, t := range tags { + if t.Name == "" || seen[t.Name] { + continue + } + seen[t.Name] = true + if _, err := tx.ExecContext(ctx, + "INSERT INTO artist_tags (artist_id, provider, tag, weight, genre) VALUES (?, ?, ?, ?, ?)", + id, provider, t.Name, t.Weight, t.Genre); err != nil { + return fmt.Errorf("insert tag: %w", err) + } + } + status := fetchOK + if !found { + status = fetchNotFound + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO provider_fetches (provider, kind, artist_id, status, retrieved_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (provider, kind, artist_id) DO UPDATE SET status = excluded.status, retrieved_at = excluded.retrieved_at`, + provider, kindTags, id, status, at.Unix()); err != nil { + return fmt.Errorf("record fetch: %w", err) + } + return nil + }) +} diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index 48e02b8..2563058 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -2,6 +2,7 @@ package storage import ( "context" + "database/sql" "errors" "fmt" "sync" @@ -9,9 +10,11 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/providers/lastfm" ) @@ -354,7 +357,13 @@ func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - f := pf(fmt.Sprintf("scan/%d.flac", round), fmt.Sprintf("Scanned %d", round), "", "", 0, 0, "flac", fmt.Sprintf("Scanned Artist %d", round)) + // A new artist, and one that an earlier round only discovered (adopted by + // name, which drops its id), as the second of the file's artists. + artists := []string{fmt.Sprintf("Scanned Artist %d", round)} + if round > 0 { + artists = append(artists, fmt.Sprintf("Shared With Id %d", round-1)) + } + f := pf(fmt.Sprintf("scan/%d.flac", round), fmt.Sprintf("Scanned %d", round), "", "", 0, 0, "flac", artists...) if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { errs <- fmt.Errorf("SaveFile: %w", err) } @@ -377,3 +386,344 @@ func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { } }) } + +// ---- MusicBrainz ids that come from the files ---- + +// withIDs is pf with the ids a tagger wrote: one per artist, and the album artist's. +func withIDs(f library.ParsedFile, albumArtistID string, ids ...string) library.ParsedFile { + f.ArtistMBIDs, f.AlbumArtistMBID = ids, albumArtistID + return f +} + +func mbidOf(t *testing.T, db *bun.DB, id int64) string { + t.Helper() + var m sql.NullString + if err := db.QueryRow("SELECT mbid FROM artists WHERE id = ?", id).Scan(&m); err != nil { + t.Fatal(err) + } + return m.String +} + +func TestFileIDsAreKeptOnTheArtistsOfTheFiles(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + lib := NewLibraryRepo(db) + const idNew = "aaaaaaaa-0000-4000-8000-000000000001" + const idAlbum = "aaaaaaaa-0000-4000-8000-000000000002" + + // A new artist is created with its id, and so is the album artist. + f := withIDs(pf("n/1.flac", "N1", "Album", "Album Artist", 1, 1, "flac", "New Artist"), idAlbum, idNew) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + newID, albumID := artistID(t, db, "New Artist"), artistID(t, db, "Album Artist") + if mbidOf(t, db, newID) != idNew || mbidOf(t, db, albumID) != idAlbum { + t.Errorf("ids = %q / %q", mbidOf(t, db, newID), mbidOf(t, db, albumID)) + } + + // The same id under another spelling is the same artist. + f = withIDs(pf("n/2.flac", "N2", "", "", 0, 0, "flac", "NEW ARTIST (feat. nobody)"), "", idNew) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM track_artists WHERE artist_id = "+fmt.Sprint(newID)); n != 2 { + t.Errorf("tracks credited to the artist = %d, want 2: the id says it is the same artist", n) + } + + // A later file without the id keeps what is known. + if _, err := lib.SaveFile(ctx, s.rock, pf("n/3.flac", "N3", "", "", 0, 0, "flac", "new artist"), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, newID) != idNew { + t.Error("a file without an id erased the id") + } + + // An artist that was there without an id gets the one the file says. + const idBjork = "aaaaaaaa-0000-4000-8000-000000000003" + bjork := artistID(t, db, "Björk") + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("b/3.flac", "B3", "", "", 0, 0, "flac", "Björk"), "", idBjork), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, bjork) != idBjork { + t.Errorf("Björk's id = %q", mbidOf(t, db, bjork)) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'björk'"); n != 1 { + t.Errorf("artists named Björk = %d, want 1", n) + } + + // Another id under the same name is another artist. + const idOther = "aaaaaaaa-0000-4000-8000-000000000004" + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("b/4.flac", "B4", "", "", 0, 0, "flac", "Björk"), "", idOther), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'björk'"); n != 2 { + t.Errorf("artists named Björk = %d, want 2: the ids differ", n) + } + if mbidOf(t, db, bjork) != idBjork { + t.Error("the first Björk lost her id") + } + }) +} + +// A file whose tags name the same id as a provider did confirms that artist, and +// the id stays. A file with another id under the same name is somebody else. +func TestFileIDsAndDiscoveredArtists(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + lib := NewLibraryRepo(db) + radiohead := artistID(t, db, "Radiohead") + const same = "bbbbbbbb-0000-4000-8000-000000000001" + const other = "bbbbbbbb-0000-4000-8000-000000000002" + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{ + {Name: "Confirmed", MBID: same, Match: 1}, {Name: "Namesake", MBID: same[:35] + "3", Match: 0.5}, + }) + confirmed, namesake := artistID(t, db, "Confirmed"), artistID(t, db, "Namesake") + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("c/9.flac", "C9", "", "", 0, 0, "flac", "CONFIRMED"), "", same), 0); err != nil { + t.Fatal(err) + } + a, _ := repo.Artist(ctx, confirmed) + if a.Name != "CONFIRMED" || a.MBID != same { + t.Errorf("confirmed artist = %+v, want the file's spelling and the id that both sides agree on", a) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'confirmed'"); n != 1 { + t.Errorf("artists named Confirmed = %d", n) + } + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("c/10.flac", "C10", "", "", 0, 0, "flac", "Namesake"), "", other), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'namesake'"); n != 2 { + t.Errorf("artists named Namesake = %d, want 2: a file with another id is another artist", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE id = "+fmt.Sprint(namesake)+" AND discovered = "+boolLit(db, true)); n != 1 { + t.Error("the discovered Namesake was taken by a file that is not it") + } + }) +} + +func boolLit(db *bun.DB, v bool) string { + if db.Dialect().Name() == dialect.PG { + return fmt.Sprint(v) + } + if v { + return "1" + } + return "0" +} + +// ---- tags ---- + +func TestTagsSaveAndRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead := artistID(t, db, "Radiohead") + + snap, err := repo.Tags(ctx, radiohead, "lastfm") + if err != nil || snap.Fetched || len(snap.Tags) != 0 { + t.Fatalf("never asked: %+v err=%v", snap, err) + } + + tags := []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "alternative rock", Weight: 54.5, Genre: true}, {Name: "british", Weight: 54.5}} + if err := repo.SaveTags(ctx, radiohead, "lastfm", at, true, tags); err != nil { + t.Fatal(err) + } + snap, err = repo.Tags(ctx, radiohead, "lastfm") + if err != nil || !snap.Fetched || snap.NotFound || !snap.RetrievedAt.Equal(at) { + t.Fatalf("snapshot = %+v err=%v", snap, err) + } + // Heaviest first; equal weights in name order, so the order is stable. + want := []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "alternative rock", Weight: 54.5, Genre: true}, {Name: "british", Weight: 54.5}} + if len(snap.Tags) != 3 { + t.Fatalf("tags = %+v", snap.Tags) + } + for i := range want { + if snap.Tags[i] != want[i] { + t.Errorf("tag %d = %+v, want %+v", i, snap.Tags[i], want[i]) + } + } + + // Saving again replaces; another provider is kept apart; "not found" clears. + _ = repo.SaveTags(ctx, radiohead, "musicbrainz", at, true, []discovery.Tag{{Name: "art rock", Weight: 30, Genre: true}}) + later := at.Add(8 * 24 * time.Hour) + if err := repo.SaveTags(ctx, radiohead, "lastfm", later, true, []discovery.Tag{{Name: "grunge", Weight: 10}}); err != nil { + t.Fatal(err) + } + snap, _ = repo.Tags(ctx, radiohead, "lastfm") + if len(snap.Tags) != 1 || snap.Tags[0].Name != "grunge" || !snap.RetrievedAt.Equal(later) { + t.Errorf("after replacing: %+v", snap) + } + if other, _ := repo.Tags(ctx, radiohead, "musicbrainz"); len(other.Tags) != 1 || other.Tags[0].Name != "art rock" { + t.Errorf("another provider's tags = %+v", other) + } + if err := repo.SaveTags(ctx, radiohead, "lastfm", later, false, nil); err != nil { + t.Fatal(err) + } + snap, _ = repo.Tags(ctx, radiohead, "lastfm") + if !snap.Fetched || !snap.NotFound || len(snap.Tags) != 0 { + t.Errorf("not found: %+v", snap) + } + // The ledger for tags and for similar artists do not touch each other. + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + if s, _ := repo.Similar(ctx, radiohead, "lastfm", 10); !s.Fetched || s.NotFound || !s.RetrievedAt.Equal(at) { + t.Errorf("similar artists were disturbed by tags: %+v", s) + } + if n := count(t, db, "SELECT COUNT(*) FROM provider_fetches"); n != 3 { + t.Errorf("ledger rows = %d, want 3 (tags lastfm, tags musicbrainz, similar lastfm)", n) + } + }) +} + +func TestDeletingAnArtistRemovesItsTags(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + elbow := artistID(t, db, "Elbow") // an artist with no albums, so it can be deleted + _ = repo.SaveTags(ctx, elbow, "lastfm", at, true, []discovery.Tag{{Name: "rock", Weight: 1}}) + if _, err := db.Exec("DELETE FROM artists WHERE id = ?", elbow); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_tags"); n != 0 { + t.Errorf("tags left = %d", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM provider_fetches WHERE kind = 'tags'"); n != 0 { + t.Errorf("ledger rows left = %d", n) + } + }) +} + +// ---- review: adopting by id ---- + +func artistsNamed(t *testing.T, db *bun.DB, key string) int { + t.Helper() + return count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = '"+key+"'") +} + +// The key follows the name: an artist adopted under another spelling must be +// found by that spelling afterwards. +func TestAnAdoptedArtistIsFoundByItsNewSpelling(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000001" + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beyonce", MBID: m, Match: 1}}) + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("y/1.flac", "Y1", "", "", 0, 0, "flac", "Beyoncé"), "", m), 0); err != nil { + t.Fatal(err) + } + if _, err := lib.SaveFile(ctx, s.rock, pf("y/2.flac", "Y2", "", "", 0, 0, "flac", "Beyoncé"), 0); err != nil { + t.Fatal(err) + } + if n := artistsNamed(t, db, "beyoncé"); n != 1 { + t.Errorf("artists with the key of Beyoncé = %d, want 1", n) + } + if n := artistsNamed(t, db, "beyonce"); n != 0 { + t.Errorf("the old key is still in use by %d artists", n) + } + }) +} + +// A real artist that has no id yet must receive it, not be split from the +// discovered artist that already holds it. +func TestAFileIDGoesToTheRealArtistNotToTheDiscoveredNamesake(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000002" + + // The user already has The Beatles, without an id. + if _, err := lib.SaveFile(ctx, s.rock, pf("bt/1.flac", "B1", "Abbey Road", "The Beatles", 1, 1, "flac", "The Beatles"), 0); err != nil { + t.Fatal(err) + } + real := artistID(t, db, "The Beatles") + // Last.fm names "Beatles" with the id the real band has. + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beatles", MBID: m, Match: 1}}) + discovered := artistID(t, db, "Beatles") + // A new file of the real band carries that id. + f := withIDs(pf("bt/2.flac", "B2", "Abbey Road", "The Beatles", 2, 1, "flac", "The Beatles"), m, m) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + + if n := artistsNamed(t, db, "the beatles"); n != 1 { + t.Fatalf("artists named The Beatles = %d, want 1", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM albums WHERE title_key = 'abbey road'"); n != 1 { + t.Errorf("albums = %d, want 1", n) + } + if mbidOf(t, db, real) != m { + t.Errorf("the real artist's id = %q, want %q", mbidOf(t, db, real), m) + } + if mbidOf(t, db, discovered) != "" { + t.Error("the discovered namesake still holds the id") + } + // The next look at Last.fm's answer lands on the real artist. + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Beatles", MBID: m, Match: 1}}) + snap, _ := repo.Similar(ctx, artistID(t, db, "Radiohead"), "lastfm", 5) + if len(snap.Items) != 1 || snap.Items[0].Artist.ID != real || !snap.Items[0].InLibrary { + t.Errorf("after the next refresh = %+v, want the real artist", snap.Items) + } + }) +} + +// Whatever was read about an artist under the provider's id is not known to be +// about the user's artist once that id is dropped. +func TestDroppingAProviderIDAlsoDropsWhatWasReadWithIt(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + radiohead := artistID(t, db, "Radiohead") + const m = "cccccccc-0000-4000-8000-000000000003" + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Namesake", MBID: m, Match: 1}}) + ns := artistID(t, db, "Namesake") + _ = repo.SaveTags(ctx, ns, "musicbrainz", at, true, []discovery.Tag{{Name: "folk", Weight: 5}}) + _ = repo.SaveSimilar(ctx, ns, "lastfm", at, true, []lastfm.Similar{{Name: "Someone Else", Match: 1}}) + + // A file with no id adopts it by name: the id is dropped. + if _, err := lib.SaveFile(ctx, s.rock, pf("n/1.flac", "N1", "", "", 0, 0, "flac", "Namesake"), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, ns) != "" { + t.Fatal("the id stayed") + } + if tg, _ := repo.Tags(ctx, ns, "musicbrainz"); tg.Fetched || len(tg.Tags) != 0 { + t.Errorf("tags read with the dropped id are still served: %+v", tg) + } + if sm, _ := repo.Similar(ctx, ns, "lastfm", 5); sm.Fetched || len(sm.Items) != 0 { + t.Errorf("similar artists read with the dropped id are still served: %+v", sm) + } + // What others said about it stays: Radiohead's list still names it. + if sm, _ := repo.Similar(ctx, radiohead, "lastfm", 5); len(sm.Items) != 1 || sm.Items[0].Artist.ID != ns || !sm.Items[0].InLibrary { + t.Errorf("the link from Radiohead = %+v", sm.Items) + } + }) +} + +// When the file confirms the very same id, what was read is kept. +func TestConfirmingTheIDKeepsWhatWasRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo, lib := NewDiscoveryRepo(db), NewLibraryRepo(db) + const m = "cccccccc-0000-4000-8000-000000000004" + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Confirmed Two", MBID: m, Match: 1}}) + c := artistID(t, db, "Confirmed Two") + _ = repo.SaveTags(ctx, c, "musicbrainz", at, true, []discovery.Tag{{Name: "folk", Weight: 5}}) + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("n/2.flac", "N2", "", "", 0, 0, "flac", "Confirmed Two"), "", m), 0); err != nil { + t.Fatal(err) + } + if tg, _ := repo.Tags(ctx, c, "musicbrainz"); !tg.Fetched || len(tg.Tags) != 1 { + t.Errorf("tags = %+v", tg) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 23937a9..c9b6516 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -105,14 +105,24 @@ func (r *LibraryRepo) ExistingFiles(ctx context.Context, libraryID int64) ([]lib } // SaveFile creates or updates one file with its track, album and artists in a -// single transaction. MusicBrainz ids are never invented: they stay NULL. +// single transaction. A MusicBrainz id is only stored when the file's own tags +// say so; it is never invented. func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.ParsedFile, existingID int64) (bool, error) { var added bool err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Artists are created and matched here and by discovery: take the turn once, + // before any write, so two transactions cannot wait for each other. + if err := lockArtistWrites(ctx, tx); err != nil { + return err + } artistIDs := make([]int64, 0, len(f.Artists)) seen := map[int64]bool{} - for _, name := range f.Artists { - id, err := findOrCreateArtist(ctx, tx, name) + for i, name := range f.Artists { + mbid := "" + if len(f.ArtistMBIDs) == len(f.Artists) { + mbid = f.ArtistMBIDs[i] + } + id, err := findOrCreateArtist(ctx, tx, name, mbid) if err != nil { return err } @@ -124,7 +134,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P var albumID int64 if f.Album != "" { - albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist)) + albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist), f.AlbumArtistMBID) if err != nil { return err } @@ -374,42 +384,119 @@ func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error { // ---- helpers (run inside a transaction) ---- -func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) { +// findOrCreateArtist returns the artist a file belongs to, creating it if needed. +// The caller holds the artist lock (lockArtistWrites) for the whole transaction: +// taking it here, after other writes, could deadlock with discovery. +// +// mbid is what the file's own tags say ("" when they say nothing). +// +// With an id, two artists are the same when the ids are equal; another id under +// the same name is another artist. A real artist of that name that has no id yet +// receives the id, even when a discovered artist already holds it: that one +// lets go of an id nobody checked, and the next answer from the provider lands +// on the real artist. A discovered artist holding exactly this id is confirmed +// by the file and becomes the user's with its data kept. +// +// Without an id the name decides, and a real artist is preferred to a +// discovered one. A discovered artist adopted this way loses the provider's id, +// and with it everything that was read under that id, since it is not known to +// be about the user's artist. +func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int64, error) { key := matchKey(name) - find := func() (artistModel, error) { - var a artistModel - err := tx.NewSelect().Model(&a).Column("id", "name", "discovered"). - Where("name_key = ?", key).OrderExpr("id").Limit(1).Scan(ctx) - return a, err + mbid = strings.ToLower(strings.TrimSpace(mbid)) + + type row struct { + ID int64 + Discovered bool + MBID string } - a, err := find() - if errors.Is(err, sql.ErrNoRows) { - // About to create one: wait for anyone else who is, then look again. - if err := lockArtistWrites(ctx, tx); err != nil { - return 0, err + lookup := func(where string, args ...any) (row, bool, error) { + var ( + r row + m sql.NullString + ) + err := tx.QueryRowContext(ctx, + "SELECT id, discovered, mbid FROM artists WHERE "+where+" ORDER BY discovered, id LIMIT 1", args...). + Scan(&r.ID, &r.Discovered, &m) + r.MBID = m.String + if errors.Is(err, sql.ErrNoRows) { + return r, false, nil } - a, err = find() + return r, err == nil, err } - switch { - case err == nil: - if a.Discovered { - // First music for an artist a provider only named: from now on it is the - // user's, with their spelling and without the provider's unchecked id. - if _, err := tx.NewUpdate().Model((*artistModel)(nil)). - Set("name = ?", name).Set("mbid = NULL").Set("discovered = ?", false). - Where("id = ?", a.ID).Exec(ctx); err != nil { - return 0, fmt.Errorf("adopt discovered artist: %w", err) + exec := func(what, query string, args ...any) error { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("%s: %w", what, err) + } + return nil + } + + if mbid != "" { + byID, haveID, err := lookup("mbid = ?", mbid) + if err != nil { + return 0, fmt.Errorf("find artist: %w", err) + } + if haveID && !byID.Discovered { + return byID.ID, nil // a real artist already has this id, whatever its spelling + } + byName, haveName, err := lookup("name_key = ? AND (mbid IS NULL OR mbid = '')", key) + if err != nil { + return 0, fmt.Errorf("find artist: %w", err) + } + switch { + case haveName && !byName.Discovered: + if haveID { + if err := exec("release discovered id", "UPDATE artists SET mbid = NULL WHERE id = ?", byID.ID); err != nil { + return 0, err + } } + return byName.ID, exec("set artist id", "UPDATE artists SET mbid = ? WHERE id = ?", mbid, byName.ID) + case haveID: + return byID.ID, exec("confirm discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ? WHERE id = ?", name, key, false, byID.ID) + case haveName: + return byName.ID, exec("adopt discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = ? WHERE id = ?", name, key, false, mbid, byName.ID) } - return a.ID, nil - case !errors.Is(err, sql.ErrNoRows): + var id int64 + if err := tx.QueryRowContext(ctx, + "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", + mbid, name, key, false).Scan(&id); err != nil { + return 0, fmt.Errorf("insert artist: %w", err) + } + return id, nil + } + + found, ok, err := lookup("name_key = ?", key) + if err != nil { return 0, fmt.Errorf("find artist: %w", err) } - a = artistModel{Name: name, NameKey: key} - if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { - return 0, fmt.Errorf("insert artist: %w", err) + if !ok { + a := artistModel{Name: name, NameKey: key} + if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { + return 0, fmt.Errorf("insert artist: %w", err) + } + return a.ID, nil } - return a.ID, nil + if found.Discovered { + if err := exec("adopt discovered artist", + "UPDATE artists SET name = ?, name_key = ?, discovered = ?, mbid = NULL WHERE id = ?", name, key, false, found.ID); err != nil { + return 0, err + } + if found.MBID != "" { + // What was read under the provider's id is not known to be about this artist. + for _, q := range []string{ + "DELETE FROM artist_tags WHERE artist_id = ?", + "DELETE FROM provider_fetches WHERE artist_id = ?", + "DELETE FROM artist_similar WHERE artist_id = ?", + } { + if err := exec("forget what was read with the dropped id", q, found.ID); err != nil { + return 0, err + } + } + } + } + return found.ID, nil } func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) { diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 073aed2..6212e82 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -147,7 +147,7 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := Migrate(context.Background(), db, fsys); err != nil { t.Fatalf("second Migrate: %v", err) } - for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches"} { + for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags"} { if !tableExists(t, db, driver, table) { t.Errorf("table %s missing", table) } @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 8 { - t.Errorf("applied migrations = %d, want 8", applied) + if applied != 9 { + t.Errorf("applied migrations = %d, want 9", applied) } }) } diff --git a/migrations/postgres/0009_artist_tags.sql b/migrations/postgres/0009_artist_tags.sql new file mode 100644 index 0000000..8c4dc47 --- /dev/null +++ b/migrations/postgres/0009_artist_tags.sql @@ -0,0 +1,17 @@ +-- Tags that the providers gave to artists, kept as they gave them. +-- +-- weight is the provider's own number and means different things: Last.fm's is +-- relative to the artist (0 to 100, the top tag is usually 100), MusicBrainz's is +-- an absolute number of votes. They are never mixed here. genre is set when +-- MusicBrainz also lists the tag among its curated genres. +-- +-- Freshness is recorded in provider_fetches (kind 'tags'), as for similar artists. + +CREATE TABLE artist_tags ( + artist_id BIGINT NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + tag TEXT NOT NULL, -- lower case + weight DOUBLE PRECISION NOT NULL, + genre BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (artist_id, provider, tag) +); diff --git a/migrations/sqlite/0009_artist_tags.sql b/migrations/sqlite/0009_artist_tags.sql new file mode 100644 index 0000000..543be68 --- /dev/null +++ b/migrations/sqlite/0009_artist_tags.sql @@ -0,0 +1,17 @@ +-- Tags that the providers gave to artists, kept as they gave them. +-- +-- weight is the provider's own number and means different things: Last.fm's is +-- relative to the artist (0 to 100, the top tag is usually 100), MusicBrainz's is +-- an absolute number of votes. They are never mixed here. genre is set when +-- MusicBrainz also lists the tag among its curated genres. +-- +-- Freshness is recorded in provider_fetches (kind 'tags'), as for similar artists. + +CREATE TABLE artist_tags ( + artist_id INTEGER NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + tag TEXT NOT NULL, -- lower case + weight REAL NOT NULL, + genre INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (artist_id, provider, tag) +);