From 07ea997cd03b1ecb3163347a08afb943157104b7 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Wed, 11 Nov 2026 17:01:43 +0100 Subject: [PATCH] feat(discovery): artist tags from Last.fm and MusicBrainz, and ids from my files - feat: GET /api/v1/artists/{id}/tags lists, for each provider that is on, the tags it gives to an artist with its own weights, kept as they came: Last.fm's are relative to the artist, MusicBrainz's are counts of votes, and they are never mixed. MusicBrainz also marks the tags it lists as genres. The same rules as similar artists apply: asked only on demand, kept 7 days (1 for an unknown artist), old data served when a provider fails, and one call for simultaneous requests. A provider that is off, failing or lacking an id does not hide the other - feat: a thin MusicBrainz client that reads an artist's tags and genres by its id. It never searches by name, because a search returns look-alikes and picking one would be a guess. The id is checked before it reaches the path, and errors never repeat the provider's text - feat: the User-Agent can change while the server runs, so the contact an admin sets for MusicBrainz applies to the next request; if the settings cannot be read it falls back to the project address - feat: Last.fm top tags, by id when known and by name otherwise, never autocorrected - feat: the scan reads MusicBrainz artist ids from the file tags, offline, and keeps them on the artists. An id is only used when it can be paired with its artist for certain: the same number of ids as artists, all valid. The same id is the same artist whatever the spelling, an artist without an id takes the one its file gives, and another id under the same name is another artist. A file that confirms the id a provider gave to a discovered artist keeps it - feat: POST /providers/musicbrainz/test checks that the service answers - test: what leaves the server for MusicBrainz is checked from the other end: the id of the artist and the admin's contact, nothing from the library; removing the safeguards on ids and pairing makes tests fail - docs: README, OpenAPI and the provider text updated --- README.md | 15 +- cmd/sondad/main.go | 21 +- internal/api/http/api_test.go | 1 + internal/api/http/contract_test.go | 15 +- internal/api/http/handlers_similar.go | 51 +++ internal/api/http/router.go | 7 + internal/api/http/similar_test.go | 14 +- internal/api/http/tags_test.go | 234 ++++++++++++++ internal/api/openapi/openapi.yaml | 77 ++++- internal/discovery/discovery.go | 9 +- internal/discovery/tags.go | 250 +++++++++++++++ internal/discovery/tags_test.go | 295 ++++++++++++++++++ internal/library/taglib.go | 58 +++- internal/library/taglib_test.go | 69 ++++ internal/library/types.go | 18 +- internal/providers/lastfm/lastfm.go | 86 +++++ internal/providers/lastfm/lastfm_test.go | 69 ++++ internal/providers/musicbrainz/musicbrainz.go | 178 +++++++++++ .../providers/musicbrainz/musicbrainz_test.go | 161 ++++++++++ internal/providers/outbound/outbound.go | 14 +- internal/providers/outbound/outbound_test.go | 25 +- internal/providers/settings.go | 21 +- internal/providers/settings_test.go | 20 ++ internal/storage/discovery.go | 74 ++++- internal/storage/discovery_test.go | 215 +++++++++++++ internal/storage/library.go | 124 ++++++-- internal/storage/storage_test.go | 6 +- migrations/postgres/0009_artist_tags.sql | 17 + migrations/sqlite/0009_artist_tags.sql | 17 + 29 files changed, 2099 insertions(+), 62 deletions(-) create mode 100644 internal/api/http/tags_test.go create mode 100644 internal/discovery/tags.go create mode 100644 internal/discovery/tags_test.go create mode 100644 internal/providers/musicbrainz/musicbrainz.go create mode 100644 internal/providers/musicbrainz/musicbrainz_test.go create mode 100644 migrations/postgres/0009_artist_tags.sql create mode 100644 migrations/sqlite/0009_artist_tags.sql diff --git a/README.md b/README.md index 0335128..f34ef88 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr - **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. - **API documentation.** The full API is described in OpenAPI and browsable with Swagger UI, served by Sonda itself. The UI is embedded in the binary, so opening it contacts no other server. - **Listening history.** Clients report explicit `play`, `complete` and `skip` events. An event counts as a listen when it is `complete`, or when it was played for at least 20 seconds (`listening.min_listen_seconds`); the rule is applied when the history is read, so changing it re-reads the past. You can read your own history, with a `counted` flag on every event. It stays in your database and is never sent anywhere. +- **Tags.** `GET /artists/{id}/tags` shows what Last.fm and MusicBrainz say kind of music an artist plays, each with its own weights. See [External providers](#external-providers). - **Similar artists.** With Last.fm switched on, `GET /artists/{id}/similar` lists artists similar to an artist, and says which of them you already have. See [External providers](#external-providers). Without it, Sonda makes no outbound network requests at all. ## What is not there yet -Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. MusicBrainz can be set up (see [External providers](#external-providers)) but nothing asks it anything yet. +Recommendations and radio queues, a web client, OpenSubsonic compatibility, transcoding, and the ListenBrainz integration. Genres from your own files, and the removal of provider data on request. ## Requirements @@ -176,7 +177,7 @@ curl -s -X POST $BASE/users -H "$AUTH" -d '{"username":"bob","password":"another ## External providers -Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today only Last.fm is used, to find similar artists. +Sonda can ask Last.fm and MusicBrainz about artists. This is **off by default**: until an admin turns a provider on, nothing leaves your server. Today Last.fm is used for similar artists and tags, and MusicBrainz for tags. - **Last.fm** needs an API key. Create one for your own use at `last.fm/api` (it needs a Last.fm account). No user name or password is involved and the "shared secret" is not needed. Last.fm data is for non-commercial use only, and you should credit Last.fm when you show it. - **MusicBrainz** needs no key, but it asks every client to identify itself with a contact (a URL or an e-mail address) in the `User-Agent`. By default Sonda uses the address of the project, so no personal address is sent unless you choose to. @@ -188,14 +189,19 @@ curl -s -X PUT $BASE/providers/lastfm -H "$AUTH" -d '{"api_key":" tags[j].Weight }) + if len(tags) > MaxTags { + tags = tags[:MaxTags] + } + return s.store.SaveTags(ctx, a.ID, provider, s.now(), len(tags) > 0, tags) +} + +// LastfmTags adapts the Last.fm client to a TagSource. +type LastfmTags struct{ Client *lastfm.Client } + +// ArtistTags implements TagSource. +func (l LastfmTags) ArtistTags(ctx context.Context, key string, a Artist) ([]Tag, error) { + got, err := l.Client.TopTags(ctx, key, lastfm.Artist{Name: a.Name, MBID: a.MBID}) + if errors.Is(err, lastfm.ErrNotFound) { + return nil, ErrSourceNotFound + } + if err != nil { + return nil, err + } + out := make([]Tag, len(got)) + for i, t := range got { + out[i] = Tag{Name: t.Name, Weight: float64(t.Count)} + } + return out, nil +} + +// MusicBrainzTags adapts the MusicBrainz client to a TagSource. It only ever +// asks by the id of the artist. +type MusicBrainzTags struct{ Client *musicbrainz.Client } + +// ArtistTags implements TagSource. +func (m MusicBrainzTags) ArtistTags(ctx context.Context, _ string, a Artist) ([]Tag, error) { + got, err := m.Client.ArtistTags(ctx, a.MBID) + if errors.Is(err, musicbrainz.ErrNotFound) { + return nil, ErrSourceNotFound + } + if err != nil { + return nil, err + } + out := make([]Tag, len(got)) + for i, t := range got { + out[i] = Tag{Name: t.Name, Weight: float64(t.Count), Genre: t.Genre} + } + return out, nil +} diff --git a/internal/discovery/tags_test.go b/internal/discovery/tags_test.go new file mode 100644 index 0000000..1f814a1 --- /dev/null +++ b/internal/discovery/tags_test.go @@ -0,0 +1,295 @@ +package discovery + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "go.uber.org/zap" + + "codeberg.org/squidd/sonda/internal/providers" +) + +type savedTags struct { + artistID int64 + provider string + at time.Time + found bool + tags []Tag +} + +// tagStore keeps one snapshot per artist and provider. +type tagStore struct { + mu sync.Mutex + artists map[int64]Artist + snaps map[string]TagSnapshot + saves []savedTags +} + +func newTagStore() *tagStore { + return &tagStore{ + artists: map[int64]Artist{ + 1: {ID: 1, Name: "Source"}, + 2: {ID: 2, Name: "With Id", MBID: "aaaaaaaa-0000-4000-8000-000000000001"}, + }, + snaps: map[string]TagSnapshot{}, + } +} + +func tkey(id int64, provider string) string { return provider + "/" + string(rune('0'+id)) } + +func (f *tagStore) Artist(_ context.Context, id int64) (Artist, error) { + f.mu.Lock() + defer f.mu.Unlock() + a, ok := f.artists[id] + if !ok { + return Artist{}, ErrNotFound + } + return a, nil +} + +func (f *tagStore) Tags(_ context.Context, id int64, provider string) (TagSnapshot, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.snaps[tkey(id, provider)], nil +} + +func (f *tagStore) SaveTags(_ context.Context, id int64, provider string, at time.Time, found bool, tags []Tag) error { + f.mu.Lock() + defer f.mu.Unlock() + f.saves = append(f.saves, savedTags{id, provider, at, found, tags}) + f.snaps[tkey(id, provider)] = TagSnapshot{Fetched: true, NotFound: !found, RetrievedAt: at, Tags: tags} + return nil +} + +type fakeTagSource struct { + mu sync.Mutex + calls int + key string + artist Artist + tags []Tag + err error + release chan struct{} + boom any +} + +func (s *fakeTagSource) ArtistTags(_ context.Context, key string, a Artist) ([]Tag, error) { + s.mu.Lock() + s.calls++ + s.key, s.artist = key, a + release := s.release + s.mu.Unlock() + if release != nil { + <-release + } + if s.boom != nil { + panic(s.boom) + } + return s.tags, s.err +} + +func (s *fakeTagSource) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +// perProvider answers Active for each provider on its own. +type perProvider map[string]bool + +func (p perProvider) Active(_ context.Context, name string) (providers.Settings, bool, error) { + return providers.Settings{Provider: name, Enabled: p[name], APIKey: key}, p[name], nil +} + +type tagRig struct { + svc *TagService + store *tagStore + lf *fakeTagSource + mb *fakeTagSource + on perProvider + now time.Time +} + +func newTagRig(lastfm, musicbrainz bool) *tagRig { + r := &tagRig{ + store: newTagStore(), lf: &fakeTagSource{}, mb: &fakeTagSource{}, + on: perProvider{providers.LastFM: lastfm, providers.MusicBrainz: musicbrainz}, + now: t0, + } + r.lf.tags = []Tag{{Name: "rock", Weight: 100}, {Name: "grunge", Weight: 40}} + r.mb.tags = []Tag{{Name: "alternative rock", Weight: 43, Genre: true}, {Name: "british", Weight: 2}} + r.svc = NewTagService(r.store, map[string]TagSource{providers.LastFM: r.lf, providers.MusicBrainz: r.mb}, r.on, zap.NewNop()) + r.svc.now = func() time.Time { return r.now } + return r +} + +func (r *tagRig) tags(t *testing.T, id int64) TagResult { + t.Helper() + res, err := r.svc.Tags(context.Background(), id) + if err != nil { + t.Fatalf("Tags: %v", err) + } + return res +} + +func TestTagsOfAnUnknownArtistAreNotFoundAndNobodyIsAsked(t *testing.T) { + r := newTagRig(true, true) + for _, id := range []int64{0, -1, 99} { + if _, err := r.svc.Tags(context.Background(), id); !errors.Is(err, ErrNotFound) { + t.Errorf("id %d: err = %v", id, err) + } + } + if r.lf.count()+r.mb.count() != 0 { + t.Error("a provider was asked about an artist that does not exist") + } +} + +func TestTagsAlwaysListBothProvidersInTheSameOrder(t *testing.T) { + r := newTagRig(false, false) + res := r.tags(t, 1) + if len(res.Sources) != 2 || res.Sources[0].Provider != providers.LastFM || res.Sources[1].Provider != providers.MusicBrainz { + t.Fatalf("sources = %+v", res.Sources) + } + for _, s := range res.Sources { + if s.Status != StatusDisabled || len(s.Tags) != 0 || !s.RetrievedAt.IsZero() || s.Stale || s.Attribution == "" { + t.Errorf("%s = %+v", s.Provider, s) + } + if s.Tags == nil { + t.Errorf("%s: tags is nil, want an empty list", s.Provider) + } + } + if r.lf.count()+r.mb.count() != 0 { + t.Error("a provider that is off was contacted") + } +} + +// MusicBrainz is only asked for an artist whose id is known; Last.fm can be +// asked by name, and by id when there is one. +func TestMusicBrainzIsNeverAskedWithoutAnID(t *testing.T) { + r := newTagRig(true, true) + res := r.tags(t, 1) + if res.Sources[0].Status != StatusOK || len(res.Sources[0].Tags) != 2 || r.lf.artist.Name != "Source" || r.lf.artist.MBID != "" || r.lf.key != key { + t.Errorf("lastfm = %+v asked about %+v with %q", res.Sources[0], r.lf.artist, r.lf.key) + } + if res.Sources[1].Status != StatusNoID || len(res.Sources[1].Tags) != 0 || r.mb.count() != 0 { + t.Errorf("musicbrainz = %+v calls = %d", res.Sources[1], r.mb.count()) + } + + res = r.tags(t, 2) + if r.mb.count() != 1 || r.mb.artist.MBID != "aaaaaaaa-0000-4000-8000-000000000001" { + t.Fatalf("musicbrainz was asked about %+v (%d calls)", r.mb.artist, r.mb.count()) + } + if r.lf.artist.MBID != "aaaaaaaa-0000-4000-8000-000000000001" { + t.Errorf("lastfm was not given the id: %+v", r.lf.artist) + } + mb := res.Sources[1] + if mb.Status != StatusOK || len(mb.Tags) != 2 || !mb.Tags[0].Genre || mb.Tags[0].Name != "alternative rock" || mb.Tags[0].Weight != 43 || !mb.RetrievedAt.Equal(t0) { + t.Errorf("musicbrainz = %+v", mb) + } +} + +func TestTagsOfOneProviderDoNotDependOnTheOther(t *testing.T) { + r := newTagRig(true, true) + r.lf.err = errors.New("last.fm is down") + res := r.tags(t, 2) + if res.Sources[0].Status != StatusError || len(res.Sources[0].Tags) != 0 { + t.Errorf("lastfm = %+v", res.Sources[0]) + } + if res.Sources[1].Status != StatusOK || len(res.Sources[1].Tags) != 2 { + t.Errorf("musicbrainz = %+v: one provider failing must not hide the other", res.Sources[1]) + } +} + +func TestFreshTagsAreNotAskedAgainAndOldOnesAre(t *testing.T) { + r := newTagRig(true, false) + r.tags(t, 1) + r.now = t0.Add(PositiveTTL - time.Second) + res := r.tags(t, 1) + if r.lf.count() != 1 || res.Sources[0].Stale || res.Sources[0].Status != StatusOK { + t.Errorf("calls = %d result = %+v", r.lf.count(), res.Sources[0]) + } + r.now = t0.Add(PositiveTTL) + r.tags(t, 1) + if r.lf.count() != 2 { + t.Errorf("calls = %d, want 2", r.lf.count()) + } +} + +func TestUnknownAtTheProviderIsRememberedForADayAndOldDataSurvivesFailure(t *testing.T) { + r := newTagRig(true, false) + r.lf.err = ErrSourceNotFound + res := r.tags(t, 1) + if res.Sources[0].Status != StatusOK || len(res.Sources[0].Tags) != 0 || len(r.store.saves) != 1 || r.store.saves[0].found { + t.Fatalf("result = %+v saves = %+v", res.Sources[0], r.store.saves) + } + r.now = t0.Add(NegativeTTL - time.Second) + r.tags(t, 1) + if r.lf.count() != 1 { + t.Errorf("calls = %d: a negative answer is kept for %v", r.lf.count(), NegativeTTL) + } + + // Old data is served when the provider fails. + r2 := newTagRig(true, false) + r2.tags(t, 1) + r2.now = t0.Add(PositiveTTL + time.Hour) + r2.lf.err = errors.New("down") + res = r2.tags(t, 1) + s := res.Sources[0] + if !s.Stale || s.Status != StatusError || len(s.Tags) != 2 || !s.RetrievedAt.Equal(t0) || len(r2.store.saves) != 1 { + t.Errorf("result = %+v saves = %d", s, len(r2.store.saves)) + } +} + +func TestAnEmptyTagListIsRememberedLikeAnUnknownArtist(t *testing.T) { + r := newTagRig(true, false) + r.lf.tags = nil + r.tags(t, 1) + if len(r.store.saves) != 1 || r.store.saves[0].found { + t.Errorf("saves = %+v", r.store.saves) + } +} + +func TestOnlyTheBestTagsAreKeptAndNamesAreClean(t *testing.T) { + r := newTagRig(true, false) + var many []Tag + for i := 0; i < MaxTags+20; i++ { + many = append(many, Tag{Name: "tag" + string(rune('a'+i%26)) + string(rune('a'+i/26)), Weight: float64(1000 - i)}) + } + r.lf.tags = many + res := r.tags(t, 1) + if got := len(res.Sources[0].Tags); got != MaxTags || res.Sources[0].Tags[0].Weight != 1000 { + t.Errorf("kept %d tags, first weight %v; want %d, best first", got, res.Sources[0].Tags[0].Weight, MaxTags) + } +} + +func TestAPanicWhileReadingTagsIsAFailureNotACrash(t *testing.T) { + r := newTagRig(true, false) + r.lf.boom = "bug" + res, err := r.svc.Tags(context.Background(), 1) + if err != nil || res.Sources[0].Status != StatusError { + t.Fatalf("result = %+v err = %v", res.Sources[0], err) + } +} + +func TestConcurrentTagLooksShareOneCall(t *testing.T) { + r := newTagRig(true, false) + r.lf.release = make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if res, err := r.svc.Tags(context.Background(), 1); err != nil || len(res.Sources[0].Tags) != 2 { + t.Errorf("result = %+v err = %v", res, err) + } + }() + } + time.Sleep(300 * time.Millisecond) + close(r.lf.release) + wg.Wait() + if r.lf.count() != 1 { + t.Errorf("calls = %d, want 1", r.lf.count()) + } +} diff --git a/internal/library/taglib.go b/internal/library/taglib.go index 620d270..ac6f085 100644 --- a/internal/library/taglib.go +++ b/internal/library/taglib.go @@ -3,6 +3,7 @@ package library import ( "fmt" "path/filepath" + "regexp" "strconv" "strings" @@ -31,14 +32,17 @@ func (TaglibReader) Read(path string) (Metadata, error) { return Metadata{}, fmt.Errorf("%w: no audio stream found", ErrUnreadable) } + artists := nonEmpty(tags["ARTIST"]) m := Metadata{ Tags: Tags{ - Title: first(tags, "TITLE"), - Album: first(tags, "ALBUM"), - AlbumArtist: first(tags, "ALBUMARTIST"), - Artists: nonEmpty(tags["ARTIST"]), - TrackNo: leadingInt(first(tags, "TRACKNUMBER")), - DiscNo: leadingInt(first(tags, "DISCNUMBER")), + Title: first(tags, "TITLE"), + Album: first(tags, "ALBUM"), + AlbumArtist: first(tags, "ALBUMARTIST"), + Artists: artists, + ArtistMBIDs: pairArtistIDs(artists, tags["MUSICBRAINZ_ARTISTID"]), + AlbumArtistMBID: singleID(tags["MUSICBRAINZ_ALBUMARTISTID"]), + TrackNo: leadingInt(first(tags, "TRACKNUMBER")), + DiscNo: leadingInt(first(tags, "DISCNUMBER")), }, Properties: Properties{ Format: strings.ToLower(props.Format), @@ -54,6 +58,7 @@ func (TaglibReader) Read(path string) (Metadata, error) { } if len(m.Artists) == 0 { m.Artists = []string{UnknownArtist} + m.ArtistMBIDs = nil // an id for an artist the file never named is not ours to use } if m.AlbumArtist == "" { m.AlbumArtist = m.Artists[0] @@ -61,6 +66,47 @@ func (TaglibReader) Read(path string) (Metadata, error) { return m, nil } +var mbidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// splitIDs reads the MusicBrainz ids out of the values of one tag. Taggers write +// several artists either as several values or as one value joined with "/" or ";". +// ok is false when any piece is not a MusicBrainz id. +func splitIDs(values []string) (ids []string, ok bool) { + for _, v := range values { + for _, piece := range strings.FieldsFunc(v, func(r rune) bool { return r == '/' || r == ';' || r == ',' }) { + piece = strings.TrimSpace(piece) + if piece == "" { + continue + } + if !mbidRe.MatchString(piece) { + return nil, false + } + ids = append(ids, strings.ToLower(piece)) + } + } + return ids, true +} + +// pairArtistIDs returns one id per artist, in order, or nothing. Fewer or more +// ids than artists, or an id that is not one, means the pairing would be a guess, +// and a wrong id on an artist is worse than none. +func pairArtistIDs(artists, values []string) []string { + ids, ok := splitIDs(values) + if !ok || len(ids) == 0 || len(ids) != len(artists) { + return nil + } + return ids +} + +// singleID returns the id when the tag holds exactly one valid id. +func singleID(values []string) string { + ids, ok := splitIDs(values) + if !ok || len(ids) != 1 { + return "" + } + return ids[0] +} + func first(tags map[string][]string, key string) string { for _, v := range tags[key] { if v = strings.TrimSpace(v); v != "" { diff --git a/internal/library/taglib_test.go b/internal/library/taglib_test.go index 092387f..f4baa70 100644 --- a/internal/library/taglib_test.go +++ b/internal/library/taglib_test.go @@ -121,3 +121,72 @@ func TestScanWithRealFiles(t *testing.T) { t.Errorf("a.mp3 row = %+v", r) } } + +// ---- MusicBrainz ids in the file tags ---- + +func TestArtistIDsAreOnlyKeptWhenTheyCanBePairedWithTheirArtists(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + const b = "66666666-7777-4888-8999-aaaaaaaaaaaa" + tests := []struct { + name string + artists []string + ids []string + want []string + }{ + {"one and one", []string{"X"}, []string{a}, []string{a}}, + {"two and two, in order", []string{"X", "Y"}, []string{a, b}, []string{a, b}}, + {"upper case is lowered", []string{"X"}, []string{"11111111-2222-4333-8444-55555555555A"}, []string{"11111111-2222-4333-8444-55555555555a"}}, + {"ids joined in one value", []string{"X", "Y"}, []string{a + "/" + b}, []string{a, b}}, + {"ids joined with a semicolon", []string{"X", "Y"}, []string{a + "; " + b}, []string{a, b}}, + {"fewer ids than artists is a guess", []string{"X", "Y"}, []string{a}, nil}, + {"more ids than artists is a guess", []string{"X"}, []string{a, b}, nil}, + {"no ids", []string{"X"}, nil, nil}, + {"one bad id spoils the pairing", []string{"X", "Y"}, []string{a, "not-an-id"}, nil}, + {"empty values are ignored", []string{"X"}, []string{" ", a}, []string{a}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := pairArtistIDs(tt.artists, tt.ids) + if len(got) != len(tt.want) { + t.Fatalf("ids = %v, want %v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("ids = %v, want %v", got, tt.want) + } + } + }) + } + if id := singleID([]string{a}); id != a { + t.Errorf("singleID = %q", id) + } + if id := singleID([]string{a, b}); id != "" { + t.Errorf("two ids for the album artist: %q, want none", id) + } + if id := singleID([]string{"nope"}); id != "" { + t.Errorf("a bad id: %q, want none", id) + } +} + +func TestTaglibReaderReadsMusicBrainzArtistIDs(t *testing.T) { + const a = "11111111-2222-4333-8444-555555555555" + const aa = "66666666-7777-4888-8999-aaaaaaaaaaaa" + p := makeAudio(t, t.TempDir(), "song.flac", map[string]string{ + "title": "T", "artist": "Synthetic Band", "album_artist": "Synthetic Band", + "MUSICBRAINZ_ARTISTID": a, "MUSICBRAINZ_ALBUMARTISTID": aa, + }) + m, err := TaglibReader{}.Read(p) + if err != nil { + t.Fatal(err) + } + if len(m.ArtistMBIDs) != 1 || m.ArtistMBIDs[0] != a || m.AlbumArtistMBID != aa { + t.Errorf("ids = %v / %q", m.ArtistMBIDs, m.AlbumArtistMBID) + } + + // A file without them has none, and a file with no artist tag has no pairing. + p = makeAudio(t, t.TempDir(), "plain.flac", map[string]string{"title": "T", "MUSICBRAINZ_ARTISTID": a}) + m, _ = TaglibReader{}.Read(p) + if len(m.ArtistMBIDs) != 0 { + t.Errorf("an id for an artist nobody named: %v", m.ArtistMBIDs) + } +} diff --git a/internal/library/types.go b/internal/library/types.go index 9fc50df..13b1d6b 100644 --- a/internal/library/types.go +++ b/internal/library/types.go @@ -13,13 +13,19 @@ import ( var ErrUnreadable = errors.New("unreadable audio file") // Tags are the descriptive tags of a track. Fields the file lacks are empty. +// +// The MusicBrainz ids are what the file itself says, read offline. ArtistMBIDs is +// either empty or has one id per entry of Artists, in the same order: ids that +// cannot be paired with their artists with certainty are not kept. type Tags struct { - Title string - Album string - AlbumArtist string - Artists []string - TrackNo int - DiscNo int + Title string + Album string + AlbumArtist string + Artists []string + ArtistMBIDs []string + AlbumArtistMBID string + TrackNo int + DiscNo int } // Properties are the technical properties of an audio file. diff --git a/internal/providers/lastfm/lastfm.go b/internal/providers/lastfm/lastfm.go index be3ca1c..1799007 100644 --- a/internal/providers/lastfm/lastfm.go +++ b/internal/providers/lastfm/lastfm.go @@ -17,8 +17,10 @@ import ( "errors" "fmt" "net/url" + "sort" "strconv" "strings" + "unicode/utf8" "codeberg.org/squidd/sonda/internal/providers" "codeberg.org/squidd/sonda/internal/providers/outbound" @@ -114,6 +116,90 @@ func (c *Client) SimilarArtists(ctx context.Context, key string, a Artist, limit return decodeSimilar(doc.Similar.Artist) } +// Tag is one tag of an artist. Count is Last.fm's own number between 0 and 100, +// relative to the artist (the top tag is usually 100). +type Tag struct { + Name string // lower case + Count int +} + +// maxTagLen bounds a tag name; longer ones are not tags, they are noise. +const maxTagLen = 100 + +// TopTags returns the top tags of an artist, most voted first. The name is never +// autocorrected, and the MusicBrainz id wins over the name when known. +func (c *Client) TopTags(ctx context.Context, key string, a Artist) ([]Tag, error) { + name, mbid := strings.TrimSpace(a.Name), strings.TrimSpace(a.MBID) + if name == "" && mbid == "" { + return nil, ErrBadRequest + } + q := url.Values{"method": {"artist.gettoptags"}, "autocorrect": {"0"}} + if mbid != "" { + q.Set("mbid", mbid) + } else { + q.Set("artist", name) + } + body, err := c.call(ctx, key, q) + if err != nil { + return nil, err + } + var doc struct { + Top *struct { + Tag json.RawMessage `json:"tag"` + } `json:"toptags"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.Top == nil { + return nil, ErrBadResponse + } + return decodeTags(doc.Top.Tag) +} + +// decodeTags reads the tag list, which Last.fm sends as an array or, with one +// tag, as an object. Names are lower-cased and kept once with their highest +// count; empty names, long names and counts of 0 or less are dropped. +func decodeTags(raw json.RawMessage) ([]Tag, error) { + type item struct { + Name string `json:"name"` + Count json.RawMessage `json:"count"` + } + var items []item + switch { + case len(raw) == 0 || string(raw) == "null": + case raw[0] == '[': + if err := json.Unmarshal(raw, &items); err != nil { + return nil, ErrBadResponse + } + default: + var one item + if err := json.Unmarshal(raw, &one); err != nil { + return nil, ErrBadResponse + } + items = []item{one} + } + best := map[string]int{} + for _, it := range items { + name := strings.ToLower(strings.TrimSpace(it.Name)) + count, err := strconv.ParseFloat(strings.Trim(string(it.Count), `"`), 64) + if name == "" || utf8.RuneCountInString(name) > maxTagLen || err != nil || count < 1 || count > 100 { + continue + } + if int(count) > best[name] { + best[name] = int(count) + } + } + out := make([]Tag, 0, len(best)) + for name, count := range best { + out = append(out, Tag{Name: name, Count: count}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out, nil +} + // Check asks one cheap question to find out whether the key works. func (c *Client) Check(ctx context.Context, key string) error { body, err := c.call(ctx, key, url.Values{"method": {"chart.gettopartists"}, "limit": {"1"}}) diff --git a/internal/providers/lastfm/lastfm_test.go b/internal/providers/lastfm/lastfm_test.go index 4da95a5..f2bd46f 100644 --- a/internal/providers/lastfm/lastfm_test.go +++ b/internal/providers/lastfm/lastfm_test.go @@ -282,3 +282,72 @@ func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { t.Errorf("an unreadable answer: err = %v", err) } } + +// ---- tags ---- + +const tagsBody = `{"toptags":{"tag":[ + {"name":"Rock","count":100,"url":"https://example.invalid/tag/rock"}, + {"name":"alternative rock","count":"54","url":"https://example.invalid/x"}, + {"name":"rock","count":10}, + {"name":"","count":9}, + {"name":"zero","count":0} +],"@attr":{"artist":"Source"}}}` + +func TestTopTagsDecodesTheAnswer(t *testing.T) { + s := newServer(t, 200, tagsBody) + got, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "Source"}) + if err != nil { + t.Fatal(err) + } + // Names are lower-cased and kept once (the higher count wins); empty names and + // a count of 0 are dropped; the count may come as a string. + if len(got) != 2 || got[0] != (Tag{Name: "rock", Count: 100}) || got[1] != (Tag{Name: "alternative rock", Count: 54}) { + t.Fatalf("tags = %+v", got) + } + q := s.last(t) + want := map[string]string{"method": "artist.gettoptags", "artist": "Source", "autocorrect": "0", "api_key": testKey, "format": "json"} + for k, v := range want { + if q.Get(k) != v { + t.Errorf("%s = %q, want %q", k, q.Get(k), v) + } + } + if len(q) != len(want) { + t.Errorf("query = %v", q) + } +} + +func TestTopTagsPrefersTheMBIDAndToleratesShapes(t *testing.T) { + s := newServer(t, 200, `{"toptags":{"tag":{"name":"solo","count":7}}}`) + got, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "x", MBID: "22222222-2222-2222-2222-222222222222"}) + if err != nil || len(got) != 1 || got[0].Name != "solo" { + t.Fatalf("tags = %+v err=%v", got, err) + } + if q := s.last(t); q.Get("mbid") != "22222222-2222-2222-2222-222222222222" || q.Has("artist") || q.Get("autocorrect") != "0" { + t.Errorf("query = %v", q) + } + empty := newServer(t, 200, `{"toptags":{"tag":[],"@attr":{"artist":"x"}}}`) + if got, err := empty.client().TopTags(context.Background(), testKey, Artist{Name: "x"}); err != nil || len(got) != 0 { + t.Errorf("no tags: %+v err=%v", got, err) + } +} + +func TestTopTagsErrors(t *testing.T) { + if _, err := newServer(t, 200, tagsBody).client().TopTags(context.Background(), testKey, Artist{}); !errors.Is(err, ErrBadRequest) { + t.Errorf("nothing to look up: %v", err) + } + for _, tt := range []struct { + status int + body string + want error + }{ + {200, `{"error":6,"message":"The artist you supplied could not be found"}`, ErrNotFound}, + {403, `{"error":10,"message":"Invalid API key"}`, ErrInvalidKey}, + {200, `{"hello":"world"}`, ErrBadResponse}, + {500, `x`, ErrUnavailable}, + } { + s := newServer(t, tt.status, tt.body) + if _, err := s.client().TopTags(context.Background(), testKey, Artist{Name: "x"}); !errors.Is(err, tt.want) { + t.Errorf("%d %s: err = %v, want %v", tt.status, tt.body, err, tt.want) + } + } +} diff --git a/internal/providers/musicbrainz/musicbrainz.go b/internal/providers/musicbrainz/musicbrainz.go new file mode 100644 index 0000000..b17afad --- /dev/null +++ b/internal/providers/musicbrainz/musicbrainz.go @@ -0,0 +1,178 @@ +// Package musicbrainz is a small read-only client for the MusicBrainz web +// service. +// +// Sonda only reads an artist whose MusicBrainz id it already knows. It never +// searches by name: a search returns look-alikes (a conductor named like a rock +// band) and picking one would be a guess. Only the id leaves the server. +// +// MusicBrainz asks every client to identify itself with a contact in the +// User-Agent and to stay at one request per second: both are the business of the +// outbound client this one sends its requests through. Errors never repeat the +// provider's own text. +package musicbrainz + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "unicode/utf8" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +// BaseURL is the MusicBrainz web service, version 2. +const BaseURL = "https://musicbrainz.org/ws/2/" + +// probeID is a public artist used to check that the service answers. +const probeID = "a74b1b7f-71a5-4011-9441-d0b5e4122711" + +// maxTagLen bounds a tag name; longer ones are not tags, they are noise. +const maxTagLen = 100 + +// Errors a call can return, to be matched with errors.Is. +var ( + ErrBadRequest = errors.New("musicbrainz: not a MusicBrainz id") + ErrNotFound = errors.New("musicbrainz: unknown artist") + ErrRateLimited = errors.New("musicbrainz: too many requests") + ErrUnavailable = errors.New("musicbrainz: service unavailable") + ErrBadResponse = errors.New("musicbrainz: unreadable answer") +) + +var idRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// Getter performs a GET. *outbound.Client is the one used in production. +type Getter interface { + Get(ctx context.Context, rawURL string) (*outbound.Response, error) +} + +// Client talks to MusicBrainz. It is safe for concurrent use. +type Client struct { + http Getter + base string +} + +// New builds a Client that sends its requests through g to base (BaseURL in +// production). +func New(g Getter, base string) *Client { return &Client{http: g, base: base} } + +// Tag is one tag of an artist. Count is the number of people who voted for it, an +// absolute number (unlike Last.fm's, which is relative). Genre says that +// MusicBrainz also lists it among its curated genres. +type Tag struct { + Name string // lower case + Count int + Genre bool +} + +// ArtistTags returns the tags of the artist with this MusicBrainz id, most voted +// first. An artist nobody tagged is an empty list, not an error. +func (c *Client) ArtistTags(ctx context.Context, id string) ([]Tag, error) { + if !idRe.MatchString(id) { + return nil, ErrBadRequest + } + body, err := c.get(ctx, c.base+"artist/"+strings.ToLower(id)+"?inc=tags+genres&fmt=json") + if err != nil { + return nil, err + } + var doc struct { + ID string `json:"id"` + Tags []struct { + Name string `json:"name"` + Count int `json:"count"` + } `json:"tags"` + Genres []struct { + Name string `json:"name"` + Count int `json:"count"` + } `json:"genres"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.ID == "" { + return nil, ErrBadResponse + } + + byName := map[string]*Tag{} + add := func(name string, count int, genre bool) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" || utf8.RuneCountInString(name) > maxTagLen || count <= 0 { + return + } + if t, ok := byName[name]; ok { + t.Genre = t.Genre || genre + if count > t.Count { + t.Count = count + } + return + } + byName[name] = &Tag{Name: name, Count: count, Genre: genre} + } + for _, t := range doc.Tags { + add(t.Name, t.Count, false) + } + for _, g := range doc.Genres { + add(g.Name, g.Count, true) + } + out := make([]Tag, 0, len(byName)) + for _, t := range byName { + out = append(out, *t) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out, nil +} + +// Probe asks for one public artist, to find out whether the service answers. It +// is a providers.Checker, and answers with the errors the settings understand. +func (c *Client) Probe(ctx context.Context) error { + body, err := c.get(ctx, c.base+"artist/"+probeID+"?fmt=json") + switch { + case errors.Is(err, ErrRateLimited): + return providers.ErrRateLimited + case errors.Is(err, ErrUnavailable): + return providers.ErrUnreachable + case err != nil: + return err + } + var doc struct { + ID string `json:"id"` + } + if json.Unmarshal(body, &doc) != nil || doc.ID == "" { + return ErrBadResponse + } + return nil +} + +// get sends one request and returns the body of a successful answer. +func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { + resp, err := c.http.Get(ctx, rawURL) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + var limited *outbound.RateLimitedError + if errors.As(err, &limited) { + return nil, ErrRateLimited + } + return nil, fmt.Errorf("%w: %w", ErrUnavailable, err) + } + switch { + case resp.Status == 200: + return resp.Body, nil + case resp.Status == 404: + return nil, ErrNotFound + case resp.Status == 400: + return nil, ErrBadRequest + case resp.Status == 429: + return nil, ErrRateLimited + case resp.Status >= 500: + return nil, fmt.Errorf("%w: status %d", ErrUnavailable, resp.Status) + } + return nil, fmt.Errorf("%w: status %d", ErrBadResponse, resp.Status) +} diff --git a/internal/providers/musicbrainz/musicbrainz_test.go b/internal/providers/musicbrainz/musicbrainz_test.go new file mode 100644 index 0000000..3c6a592 --- /dev/null +++ b/internal/providers/musicbrainz/musicbrainz_test.go @@ -0,0 +1,161 @@ +package musicbrainz + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "codeberg.org/squidd/sonda/internal/providers" + "codeberg.org/squidd/sonda/internal/providers/outbound" +) + +const mbid = "11111111-2222-4333-8444-555555555555" + +type server struct { + *httptest.Server + mu sync.Mutex + paths []string + hdr http.Header +} + +func newServer(t *testing.T, status int, body string) *server { + t.Helper() + s := &server{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.paths = append(s.paths, r.URL.RequestURI()) + s.hdr = r.Header.Clone() + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(s.Close) + return s +} + +func (s *server) client() *Client { + return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}), s.URL+"/ws/2/") +} + +func (s *server) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.paths...) +} + +const artistBody = `{"id":"11111111-2222-4333-8444-555555555555","name":"Synthetic Band","type":"Group", + "tags":[{"name":"Alternative Rock","count":43},{"name":"british","count":2},{"name":"ambient pop","count":1},{"name":"zero","count":0},{"name":"","count":9}], + "genres":[{"id":"x","name":"alternative rock","count":43,"disambiguation":""},{"id":"y","name":"art pop","count":3,"disambiguation":""}]}` + +func TestArtistTagsReadsTagsAndMarksTheGenres(t *testing.T) { + s := newServer(t, 200, artistBody) + got, err := s.client().ArtistTags(context.Background(), mbid) + if err != nil { + t.Fatal(err) + } + want := []Tag{ + {Name: "alternative rock", Count: 43, Genre: true}, // names are compared in lower case + {Name: "art pop", Count: 3, Genre: true}, // a genre that is not among the tags is kept + {Name: "british", Count: 2}, + {Name: "ambient pop", Count: 1}, + } + if len(got) != len(want) { + t.Fatalf("tags = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("tag %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func TestArtistTagsAsksOnlyForTheIdOfTheArtist(t *testing.T) { + s := newServer(t, 200, artistBody) + if _, err := s.client().ArtistTags(context.Background(), strings.ToUpper(mbid)); err != nil { + t.Fatal(err) + } + reqs := s.requests() + if len(reqs) != 1 || reqs[0] != "/ws/2/artist/"+mbid+"?inc=tags+genres&fmt=json" { + t.Errorf("requests = %v", reqs) + } + if s.hdr.Get("User-Agent") != "Sonda/test ( https://example.org )" { + t.Errorf("User-Agent = %q", s.hdr.Get("User-Agent")) + } +} + +// Anything that is not a plain id never reaches the network: it would end up +// inside the path of the request. +func TestArtistTagsRefusesWhatIsNotAnID(t *testing.T) { + for _, bad := range []string{"", "not-a-uuid", mbid + "/../x", mbid + "?x=1", " " + mbid + "\n", "00000000-0000-0000-0000-00000000000g"} { + s := newServer(t, 200, artistBody) + if _, err := s.client().ArtistTags(context.Background(), bad); !errors.Is(err, ErrBadRequest) { + t.Errorf("%q: err = %v, want ErrBadRequest", bad, err) + } + if len(s.requests()) != 0 { + t.Errorf("%q: a request left", bad) + } + } +} + +func TestArtistTagsErrors(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + want error + }{ + {"unknown id", 404, `{"error":"Not Found"}`, ErrNotFound}, + {"invalid id", 400, `{"error":"Invalid mbid."}`, ErrBadRequest}, + {"too many requests", 429, `{"error":"slow down"}`, ErrRateLimited}, + {"over the limit", 503, `{"error":"Your requests are being throttled"}`, ErrUnavailable}, + {"server error", 500, `oops`, ErrUnavailable}, + {"not json", 200, ``, ErrBadResponse}, + {"empty", 200, ``, ErrBadResponse}, + {"json of another kind", 200, `{"hello":"world"}`, ErrBadResponse}, + {"forbidden", 403, `{"error":"banned"}`, ErrBadResponse}, + } { + t.Run(tt.name, func(t *testing.T) { + s := newServer(t, tt.status, tt.body) + if _, err := s.client().ArtistTags(context.Background(), mbid); !errors.Is(err, tt.want) { + t.Fatalf("err = %v, want %v", err, tt.want) + } + }) + } + // An artist with no tags at all is an answer, not an error. + s := newServer(t, 200, `{"id":"`+mbid+`","name":"Quiet"}`) + got, err := s.client().ArtistTags(context.Background(), mbid) + if err != nil || len(got) != 0 { + t.Errorf("no tags: %+v err=%v", got, err) + } +} + +func TestErrorsDoNotCopyTheProvidersText(t *testing.T) { + s := newServer(t, 400, `{"error":"secret-text-from-the-provider"}`) + _, err := s.client().ArtistTags(context.Background(), mbid) + if err == nil || strings.Contains(err.Error(), "secret-text") { + t.Errorf("err = %v", err) + } +} + +func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) { + ok := newServer(t, 200, `{"id":"a74b1b7f-71a5-4011-9441-d0b5e4122711","name":"Anyone"}`) + if err := ok.client().Probe(context.Background()); err != nil { + t.Fatal(err) + } + if reqs := ok.requests(); len(reqs) != 1 || strings.Contains(reqs[0], "inc=") { + t.Errorf("a test asks for one plain artist: %v", reqs) + } + down := newServer(t, 503, `x`) + if err := down.client().Probe(context.Background()); !errors.Is(err, providers.ErrUnreachable) { + t.Errorf("down: %v", err) + } + slow := newServer(t, 429, `x`) + if err := slow.client().Probe(context.Background()); !errors.Is(err, providers.ErrRateLimited) { + t.Errorf("rate limited: %v", err) + } +} diff --git a/internal/providers/outbound/outbound.go b/internal/providers/outbound/outbound.go index 200a32d..aba0b34 100644 --- a/internal/providers/outbound/outbound.go +++ b/internal/providers/outbound/outbound.go @@ -70,8 +70,9 @@ func (realClock) Sleep(ctx context.Context, d time.Duration) error { // Options configure a Client. Zero values mean sensible defaults. type Options struct { - UserAgent string - Timeout time.Duration // per attempt; default 10s + UserAgent string + UserAgentFunc func() string // when set, read for every request instead of UserAgent + Timeout time.Duration // per attempt; default 10s MinInterval time.Duration // minimum time between requests; 0 = no limit MaxBody int64 // largest accepted answer; default 4 MiB @@ -99,7 +100,7 @@ type Response struct { // Client talks to one provider. It is safe for concurrent use. type Client struct { - userAgent string + userAgent func() string timeout time.Duration minInterval time.Duration maxBody int64 @@ -130,7 +131,7 @@ func New(o Options) *Client { return v } c := &Client{ - userAgent: o.UserAgent, + userAgent: func() string { return o.UserAgent }, timeout: def(o.Timeout, 10*time.Second), minInterval: o.MinInterval, maxBody: o.MaxBody, @@ -143,6 +144,9 @@ func New(o Options) *Client { clock: o.Clock, jitter: o.Jitter, } + if o.UserAgentFunc != nil { + c.userAgent = o.UserAgentFunc + } if c.maxBody <= 0 { c.maxBody = 4 << 20 } @@ -268,7 +272,7 @@ func (c *Client) once(ctx context.Context, rawURL string) (*Response, error) { if err != nil { return nil, ErrBadURL } - req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("User-Agent", c.userAgent()) req.Header.Set("Accept", "application/json") resp, err := c.hc.Do(req) diff --git a/internal/providers/outbound/outbound_test.go b/internal/providers/outbound/outbound_test.go index b74d3df..e405b47 100644 --- a/internal/providers/outbound/outbound_test.go +++ b/internal/providers/outbound/outbound_test.go @@ -492,8 +492,8 @@ func TestDefaultsAreSane(t *testing.T) { if c.timeout <= 0 || c.maxBody <= 0 || c.retries != 2 || c.breakerThreshold <= 0 { t.Errorf("defaults: timeout=%v maxBody=%d retries=%d threshold=%d", c.timeout, c.maxBody, c.retries, c.breakerThreshold) } - if !strings.Contains(strings.ToLower(c.userAgent), "x") { - t.Errorf("user agent = %q", c.userAgent) + if !strings.Contains(strings.ToLower(c.userAgent()), "x") { + t.Errorf("user agent = %q", c.userAgent()) } } @@ -698,3 +698,24 @@ func TestRedirectsAndOversizedAnswersCountAsFailures(t *testing.T) { }) } } + +// A User-Agent that can change while the server runs (MusicBrainz asks for a +// contact an admin may edit) is read again for every request. +func TestUserAgentFuncIsReadForEveryRequest(t *testing.T) { + var seen []string + srv, _ := server(t, func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Get("User-Agent")) + _, _ = w.Write([]byte("{}")) + }) + current := "Sonda/test ( one )" + c := newClient(newClock(), func(o *Options) { + o.UserAgent = "fixed" + o.UserAgentFunc = func() string { return current } + }) + _, _ = c.Get(context.Background(), srv.URL) + current = "Sonda/test ( two )" + _, _ = c.Get(context.Background(), srv.URL) + if len(seen) != 2 || seen[0] != "Sonda/test ( one )" || seen[1] != "Sonda/test ( two )" { + t.Errorf("User-Agents = %q", seen) + } +} diff --git a/internal/providers/settings.go b/internal/providers/settings.go index 2105b2f..5275dce 100644 --- a/internal/providers/settings.go +++ b/internal/providers/settings.go @@ -99,8 +99,9 @@ var known = []meta{ }, { name: MusicBrainz, - sends: "The artist name or MusicBrainz id you look up, your IP address and the User-Agent, " + - "which carries the contact below. Never your users, your listens, track titles or file paths.", + sends: "The MusicBrainz id of the artist you look up (never a name: Sonda does not search), your " + + "IP address and the User-Agent, which carries the contact below. Never your users, your listens, " + + "track titles or file paths.", note: "Core data (artist names and ids) is public domain. Tags and genres are CC BY-NC-SA 3.0: " + "non-commercial, with attribution to MusicBrainz.", }, @@ -337,3 +338,19 @@ func (s *Service) Test(ctx context.Context, name string) (TestResult, error) { return TestResult{Status: TestFailed}, nil } } + +// UserAgentFor returns a function that builds the User-Agent for a provider from +// the contact stored now, so a change by an admin applies to the next request. +// If the settings cannot be read it uses the project address: a request is never +// held up or sent with a broken header because of it. +func (s *Service) UserAgentFor(name, version string) func() string { + return func() string { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cur, _, err := s.store.Get(ctx, name) + if err != nil { + return UserAgent(version, "") + } + return UserAgent(version, cur.Contact) + } +} diff --git a/internal/providers/settings_test.go b/internal/providers/settings_test.go index c810806..12d59fd 100644 --- a/internal/providers/settings_test.go +++ b/internal/providers/settings_test.go @@ -346,3 +346,23 @@ func TestTestResultNeverCarriesTheKey(t *testing.T) { t.Errorf("the result repeats the key: %+v", res) } } + +// MusicBrainz asks for a contact, which an admin can change: the User-Agent +// follows the stored value, falls back to the project address, and never fails. +func TestUserAgentForFollowsTheStoredContact(t *testing.T) { + s, st := newSvc() + ua := s.UserAgentFor(MusicBrainz, "1.2.3") + if got := ua(); got != "Sonda/1.2.3 ( "+ProjectURL+" )" { + t.Errorf("default = %q", got) + } + if _, err := s.Update(context.Background(), MusicBrainz, Patch{Contact: strp("https://example.org/me")}); err != nil { + t.Fatal(err) + } + if got := ua(); got != "Sonda/1.2.3 ( https://example.org/me )" { + t.Errorf("after the change = %q", got) + } + st.err = errors.New("db down") + if got := ua(); got != "Sonda/1.2.3 ( "+ProjectURL+" )" { + t.Errorf("with the store failing = %q, want the project address", got) + } +} diff --git a/internal/storage/discovery.go b/internal/storage/discovery.go index 168d26d..b8dde5d 100644 --- a/internal/storage/discovery.go +++ b/internal/storage/discovery.go @@ -15,8 +15,11 @@ import ( "codeberg.org/squidd/sonda/internal/providers/lastfm" ) -// kindSimilar is what provider_fetches records for a "similar artists" lookup. -const kindSimilar = "similar" +// What provider_fetches records a lookup as. +const ( + kindSimilar = "similar" + kindTags = "tags" +) // Answers recorded in provider_fetches. const ( @@ -269,3 +272,70 @@ func lockArtistWrites(ctx context.Context, tx bun.Tx) error { // cleanMBID lower-cases an id the way it is stored. func cleanMBID(s string) string { return strings.ToLower(strings.TrimSpace(s)) } + +// Tags reads the tags a provider gave to an artist, heaviest first. +func (r *DiscoveryRepo) Tags(ctx context.Context, id int64, provider string) (discovery.TagSnapshot, error) { + var ( + snap discovery.TagSnapshot + status string + when int64 + ) + err := r.db.QueryRowContext(ctx, + "SELECT status, retrieved_at FROM provider_fetches WHERE provider = ? AND kind = ? AND artist_id = ?", + provider, kindTags, id).Scan(&status, &when) + switch { + case errors.Is(err, sql.ErrNoRows): + return snap, nil + case err != nil: + return snap, fmt.Errorf("read fetch ledger: %w", err) + } + snap.Fetched, snap.NotFound, snap.RetrievedAt = true, status == fetchNotFound, time.Unix(when, 0) + + rows, err := r.db.QueryContext(ctx, + "SELECT tag, weight, genre FROM artist_tags WHERE artist_id = ? AND provider = ? ORDER BY weight DESC, tag", id, provider) + if err != nil { + return snap, fmt.Errorf("read tags: %w", err) + } + defer rows.Close() + snap.Tags = []discovery.Tag{} + for rows.Next() { + var t discovery.Tag + if err := rows.Scan(&t.Name, &t.Weight, &t.Genre); err != nil { + return snap, err + } + snap.Tags = append(snap.Tags, t) + } + return snap, rows.Err() +} + +// SaveTags replaces the tags a provider gave to an artist and records the answer. +func (r *DiscoveryRepo) SaveTags(ctx context.Context, id int64, provider string, at time.Time, found bool, tags []discovery.Tag) error { + return r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM artist_tags WHERE artist_id = ? AND provider = ?", id, provider); err != nil { + return fmt.Errorf("clear tags: %w", err) + } + seen := map[string]bool{} + for _, t := range tags { + if t.Name == "" || seen[t.Name] { + continue + } + seen[t.Name] = true + if _, err := tx.ExecContext(ctx, + "INSERT INTO artist_tags (artist_id, provider, tag, weight, genre) VALUES (?, ?, ?, ?, ?)", + id, provider, t.Name, t.Weight, t.Genre); err != nil { + return fmt.Errorf("insert tag: %w", err) + } + } + status := fetchOK + if !found { + status = fetchNotFound + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO provider_fetches (provider, kind, artist_id, status, retrieved_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (provider, kind, artist_id) DO UPDATE SET status = excluded.status, retrieved_at = excluded.retrieved_at`, + provider, kindTags, id, status, at.Unix()); err != nil { + return fmt.Errorf("record fetch: %w", err) + } + return nil + }) +} diff --git a/internal/storage/discovery_test.go b/internal/storage/discovery_test.go index 48e02b8..fa2efe6 100644 --- a/internal/storage/discovery_test.go +++ b/internal/storage/discovery_test.go @@ -2,6 +2,7 @@ package storage import ( "context" + "database/sql" "errors" "fmt" "sync" @@ -9,9 +10,11 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" "codeberg.org/squidd/sonda/internal/catalog" "codeberg.org/squidd/sonda/internal/discovery" + "codeberg.org/squidd/sonda/internal/library" "codeberg.org/squidd/sonda/internal/providers/lastfm" ) @@ -377,3 +380,215 @@ func TestConcurrentSavesDoNotFailOrDuplicate(t *testing.T) { } }) } + +// ---- MusicBrainz ids that come from the files ---- + +// withIDs is pf with the ids a tagger wrote: one per artist, and the album artist's. +func withIDs(f library.ParsedFile, albumArtistID string, ids ...string) library.ParsedFile { + f.ArtistMBIDs, f.AlbumArtistMBID = ids, albumArtistID + return f +} + +func mbidOf(t *testing.T, db *bun.DB, id int64) string { + t.Helper() + var m sql.NullString + if err := db.QueryRow("SELECT mbid FROM artists WHERE id = ?", id).Scan(&m); err != nil { + t.Fatal(err) + } + return m.String +} + +func TestFileIDsAreKeptOnTheArtistsOfTheFiles(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + lib := NewLibraryRepo(db) + const idNew = "aaaaaaaa-0000-4000-8000-000000000001" + const idAlbum = "aaaaaaaa-0000-4000-8000-000000000002" + + // A new artist is created with its id, and so is the album artist. + f := withIDs(pf("n/1.flac", "N1", "Album", "Album Artist", 1, 1, "flac", "New Artist"), idAlbum, idNew) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + newID, albumID := artistID(t, db, "New Artist"), artistID(t, db, "Album Artist") + if mbidOf(t, db, newID) != idNew || mbidOf(t, db, albumID) != idAlbum { + t.Errorf("ids = %q / %q", mbidOf(t, db, newID), mbidOf(t, db, albumID)) + } + + // The same id under another spelling is the same artist. + f = withIDs(pf("n/2.flac", "N2", "", "", 0, 0, "flac", "NEW ARTIST (feat. nobody)"), "", idNew) + if _, err := lib.SaveFile(ctx, s.rock, f, 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM track_artists WHERE artist_id = "+fmt.Sprint(newID)); n != 2 { + t.Errorf("tracks credited to the artist = %d, want 2: the id says it is the same artist", n) + } + + // A later file without the id keeps what is known. + if _, err := lib.SaveFile(ctx, s.rock, pf("n/3.flac", "N3", "", "", 0, 0, "flac", "new artist"), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, newID) != idNew { + t.Error("a file without an id erased the id") + } + + // An artist that was there without an id gets the one the file says. + const idBjork = "aaaaaaaa-0000-4000-8000-000000000003" + bjork := artistID(t, db, "Björk") + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("b/3.flac", "B3", "", "", 0, 0, "flac", "Björk"), "", idBjork), 0); err != nil { + t.Fatal(err) + } + if mbidOf(t, db, bjork) != idBjork { + t.Errorf("Björk's id = %q", mbidOf(t, db, bjork)) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'björk'"); n != 1 { + t.Errorf("artists named Björk = %d, want 1", n) + } + + // Another id under the same name is another artist. + const idOther = "aaaaaaaa-0000-4000-8000-000000000004" + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("b/4.flac", "B4", "", "", 0, 0, "flac", "Björk"), "", idOther), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'björk'"); n != 2 { + t.Errorf("artists named Björk = %d, want 2: the ids differ", n) + } + if mbidOf(t, db, bjork) != idBjork { + t.Error("the first Björk lost her id") + } + }) +} + +// A file whose tags name the same id as a provider did confirms that artist, and +// the id stays. A file with another id under the same name is somebody else. +func TestFileIDsAndDiscoveredArtists(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + s := seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + lib := NewLibraryRepo(db) + radiohead := artistID(t, db, "Radiohead") + const same = "bbbbbbbb-0000-4000-8000-000000000001" + const other = "bbbbbbbb-0000-4000-8000-000000000002" + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{ + {Name: "Confirmed", MBID: same, Match: 1}, {Name: "Namesake", MBID: same[:35] + "3", Match: 0.5}, + }) + confirmed, namesake := artistID(t, db, "Confirmed"), artistID(t, db, "Namesake") + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("c/9.flac", "C9", "", "", 0, 0, "flac", "CONFIRMED"), "", same), 0); err != nil { + t.Fatal(err) + } + a, _ := repo.Artist(ctx, confirmed) + if a.Name != "CONFIRMED" || a.MBID != same { + t.Errorf("confirmed artist = %+v, want the file's spelling and the id that both sides agree on", a) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'confirmed'"); n != 1 { + t.Errorf("artists named Confirmed = %d", n) + } + + if _, err := lib.SaveFile(ctx, s.rock, withIDs(pf("c/10.flac", "C10", "", "", 0, 0, "flac", "Namesake"), "", other), 0); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE name_key = 'namesake'"); n != 2 { + t.Errorf("artists named Namesake = %d, want 2: a file with another id is another artist", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM artists WHERE id = "+fmt.Sprint(namesake)+" AND discovered = "+boolLit(db, true)); n != 1 { + t.Error("the discovered Namesake was taken by a file that is not it") + } + }) +} + +func boolLit(db *bun.DB, v bool) string { + if db.Dialect().Name() == dialect.PG { + return fmt.Sprint(v) + } + if v { + return "1" + } + return "0" +} + +// ---- tags ---- + +func TestTagsSaveAndRead(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + radiohead := artistID(t, db, "Radiohead") + + snap, err := repo.Tags(ctx, radiohead, "lastfm") + if err != nil || snap.Fetched || len(snap.Tags) != 0 { + t.Fatalf("never asked: %+v err=%v", snap, err) + } + + tags := []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "alternative rock", Weight: 54.5, Genre: true}, {Name: "british", Weight: 54.5}} + if err := repo.SaveTags(ctx, radiohead, "lastfm", at, true, tags); err != nil { + t.Fatal(err) + } + snap, err = repo.Tags(ctx, radiohead, "lastfm") + if err != nil || !snap.Fetched || snap.NotFound || !snap.RetrievedAt.Equal(at) { + t.Fatalf("snapshot = %+v err=%v", snap, err) + } + // Heaviest first; equal weights in name order, so the order is stable. + want := []discovery.Tag{{Name: "rock", Weight: 100}, {Name: "alternative rock", Weight: 54.5, Genre: true}, {Name: "british", Weight: 54.5}} + if len(snap.Tags) != 3 { + t.Fatalf("tags = %+v", snap.Tags) + } + for i := range want { + if snap.Tags[i] != want[i] { + t.Errorf("tag %d = %+v, want %+v", i, snap.Tags[i], want[i]) + } + } + + // Saving again replaces; another provider is kept apart; "not found" clears. + _ = repo.SaveTags(ctx, radiohead, "musicbrainz", at, true, []discovery.Tag{{Name: "art rock", Weight: 30, Genre: true}}) + later := at.Add(8 * 24 * time.Hour) + if err := repo.SaveTags(ctx, radiohead, "lastfm", later, true, []discovery.Tag{{Name: "grunge", Weight: 10}}); err != nil { + t.Fatal(err) + } + snap, _ = repo.Tags(ctx, radiohead, "lastfm") + if len(snap.Tags) != 1 || snap.Tags[0].Name != "grunge" || !snap.RetrievedAt.Equal(later) { + t.Errorf("after replacing: %+v", snap) + } + if other, _ := repo.Tags(ctx, radiohead, "musicbrainz"); len(other.Tags) != 1 || other.Tags[0].Name != "art rock" { + t.Errorf("another provider's tags = %+v", other) + } + if err := repo.SaveTags(ctx, radiohead, "lastfm", later, false, nil); err != nil { + t.Fatal(err) + } + snap, _ = repo.Tags(ctx, radiohead, "lastfm") + if !snap.Fetched || !snap.NotFound || len(snap.Tags) != 0 { + t.Errorf("not found: %+v", snap) + } + // The ledger for tags and for similar artists do not touch each other. + _ = repo.SaveSimilar(ctx, radiohead, "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + if s, _ := repo.Similar(ctx, radiohead, "lastfm", 10); !s.Fetched || s.NotFound || !s.RetrievedAt.Equal(at) { + t.Errorf("similar artists were disturbed by tags: %+v", s) + } + if n := count(t, db, "SELECT COUNT(*) FROM provider_fetches"); n != 3 { + t.Errorf("ledger rows = %d, want 3 (tags lastfm, tags musicbrainz, similar lastfm)", n) + } + }) +} + +func TestDeletingAnArtistRemovesItsTags(t *testing.T) { + forEachBackend(t, func(t *testing.T, db *bun.DB, _ string) { + seedCatalog(t, db) + ctx := context.Background() + repo := NewDiscoveryRepo(db) + _ = repo.SaveSimilar(ctx, artistID(t, db, "Radiohead"), "lastfm", at, true, []lastfm.Similar{{Name: "Elbow", Match: 1}}) + elbow := artistID(t, db, "Elbow") // an artist with no albums, so it can be deleted + _ = repo.SaveTags(ctx, elbow, "lastfm", at, true, []discovery.Tag{{Name: "rock", Weight: 1}}) + if _, err := db.Exec("DELETE FROM artists WHERE id = ?", elbow); err != nil { + t.Fatal(err) + } + if n := count(t, db, "SELECT COUNT(*) FROM artist_tags"); n != 0 { + t.Errorf("tags left = %d", n) + } + if n := count(t, db, "SELECT COUNT(*) FROM provider_fetches WHERE kind = 'tags'"); n != 0 { + t.Errorf("ledger rows left = %d", n) + } + }) +} diff --git a/internal/storage/library.go b/internal/storage/library.go index 23937a9..4a85582 100644 --- a/internal/storage/library.go +++ b/internal/storage/library.go @@ -111,8 +111,12 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P err := r.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { artistIDs := make([]int64, 0, len(f.Artists)) seen := map[int64]bool{} - for _, name := range f.Artists { - id, err := findOrCreateArtist(ctx, tx, name) + for i, name := range f.Artists { + mbid := "" + if len(f.ArtistMBIDs) == len(f.Artists) { + mbid = f.ArtistMBIDs[i] + } + id, err := findOrCreateArtist(ctx, tx, name, mbid) if err != nil { return err } @@ -124,7 +128,7 @@ func (r *LibraryRepo) SaveFile(ctx context.Context, libraryID int64, f library.P var albumID int64 if f.Album != "" { - albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist)) + albumArtistID, err := findOrCreateArtist(ctx, tx, firstNonEmpty(f.AlbumArtist, library.UnknownArtist), f.AlbumArtistMBID) if err != nil { return err } @@ -374,44 +378,112 @@ func (r *LibraryRepo) RefreshMatchKeys(ctx context.Context) error { // ---- helpers (run inside a transaction) ---- -func findOrCreateArtist(ctx context.Context, tx bun.Tx, name string) (int64, error) { +// findOrCreateArtist returns the artist a file belongs to, creating it if needed. +// +// mbid is what the file's own tags say ("" when they say nothing). With one, two +// artists are the same when the ids are equal, or when the names match and the +// artist we have has no id yet (it then gets the one from the file); another id +// under the same name is another artist. Without one, the name decides, as it +// always did, and a real artist is preferred to one that a provider only named. +// +// An artist that a provider only named becomes the user's when a file finds it: +// it takes the spelling of the tags and, unless the file confirms the very same +// id, loses the provider's id, which nobody checked. +func findOrCreateArtist(ctx context.Context, tx bun.Tx, name, mbid string) (int64, error) { key := matchKey(name) - find := func() (artistModel, error) { - var a artistModel - err := tx.NewSelect().Model(&a).Column("id", "name", "discovered"). - Where("name_key = ?", key).OrderExpr("id").Limit(1).Scan(ctx) - return a, err - } - a, err := find() - if errors.Is(err, sql.ErrNoRows) { - // About to create one: wait for anyone else who is, then look again. + mbid = strings.ToLower(strings.TrimSpace(mbid)) + if mbid != "" { + // Ids are written here, and discovery writes them too: take turns. if err := lockArtistWrites(ctx, tx); err != nil { return 0, err } - a, err = find() } - switch { - case err == nil: - if a.Discovered { - // First music for an artist a provider only named: from now on it is the - // user's, with their spelling and without the provider's unchecked id. - if _, err := tx.NewUpdate().Model((*artistModel)(nil)). - Set("name = ?", name).Set("mbid = NULL").Set("discovered = ?", false). - Where("id = ?", a.ID).Exec(ctx); err != nil { - return 0, fmt.Errorf("adopt discovered artist: %w", err) + + type row struct { + ID int64 + Discovered bool + } + var found row + lookup := func(where string, args ...any) (bool, error) { + err := tx.QueryRowContext(ctx, + "SELECT id, discovered FROM artists WHERE "+where+" ORDER BY discovered, id LIMIT 1", args...). + Scan(&found.ID, &found.Discovered) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err + } + + var ( + ok bool + err error + ) + if mbid != "" { + if ok, err = lookup("mbid = ?", mbid); err == nil && ok { + return adopt(ctx, tx, found.ID, found.Discovered, name, "", false) + } + if err == nil { + if ok, err = lookup("name_key = ? AND (mbid IS NULL OR mbid = '')", key); err == nil && ok { + return adopt(ctx, tx, found.ID, found.Discovered, name, mbid, true) } } - return a.ID, nil - case !errors.Is(err, sql.ErrNoRows): + } else { + if ok, err = lookup("name_key = ?", key); err == nil && !ok { + // About to create one: wait for anyone else who is, then look again. + if err = lockArtistWrites(ctx, tx); err == nil { + ok, err = lookup("name_key = ?", key) + } + } + if err == nil && ok { + return adopt(ctx, tx, found.ID, found.Discovered, name, "", found.Discovered) + } + } + if err != nil { return 0, fmt.Errorf("find artist: %w", err) } - a = artistModel{Name: name, NameKey: key} + + a := artistModel{Name: name, NameKey: key} + if mbid != "" { + var id int64 + if err := tx.QueryRowContext(ctx, + "INSERT INTO artists (mbid, name, name_key, discovered) VALUES (?, ?, ?, ?) RETURNING id", + mbid, name, key, false).Scan(&id); err != nil { + return 0, fmt.Errorf("insert artist: %w", err) + } + return id, nil + } if _, err := tx.NewInsert().Model(&a).Exec(ctx); err != nil { return 0, fmt.Errorf("insert artist: %w", err) } return a.ID, nil } +// adopt settles an artist that a file just matched. setID is the id the file +// gives it ("" for none); drop removes the id a provider had given a discovered +// artist. Only a discovered artist changes its name and loses its mark. +func adopt(ctx context.Context, tx bun.Tx, id int64, discovered bool, name, setID string, drop bool) (int64, error) { + q := tx.NewUpdate().Model((*artistModel)(nil)).Where("id = ?", id) + changed := false + if discovered { + q = q.Set("name = ?", name).Set("discovered = ?", false) + changed = true + } + switch { + case setID != "": + q = q.Set("mbid = ?", setID) + changed = true + case drop && discovered: + q = q.Set("mbid = NULL") + } + if !changed { + return id, nil + } + if _, err := q.Exec(ctx); err != nil { + return 0, fmt.Errorf("update artist: %w", err) + } + return id, nil +} + func findOrCreateAlbum(ctx context.Context, tx bun.Tx, title string, artistID int64) (int64, error) { var a albumModel err := tx.NewSelect().Model(&a). diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 073aed2..6212e82 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -147,7 +147,7 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := Migrate(context.Background(), db, fsys); err != nil { t.Fatalf("second Migrate: %v", err) } - for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches"} { + for _, table := range []string{"artists", "albums", "tracks", "track_artists", "audio_files", "users", "listening_events", "refresh_tokens", "libraries", "provider_settings", "artist_similar", "provider_fetches", "artist_tags"} { if !tableExists(t, db, driver, table) { t.Errorf("table %s missing", table) } @@ -156,8 +156,8 @@ func TestMigrateCreatesSchemaAndIsIdempotent(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { t.Fatal(err) } - if applied != 8 { - t.Errorf("applied migrations = %d, want 8", applied) + if applied != 9 { + t.Errorf("applied migrations = %d, want 9", applied) } }) } diff --git a/migrations/postgres/0009_artist_tags.sql b/migrations/postgres/0009_artist_tags.sql new file mode 100644 index 0000000..8c4dc47 --- /dev/null +++ b/migrations/postgres/0009_artist_tags.sql @@ -0,0 +1,17 @@ +-- Tags that the providers gave to artists, kept as they gave them. +-- +-- weight is the provider's own number and means different things: Last.fm's is +-- relative to the artist (0 to 100, the top tag is usually 100), MusicBrainz's is +-- an absolute number of votes. They are never mixed here. genre is set when +-- MusicBrainz also lists the tag among its curated genres. +-- +-- Freshness is recorded in provider_fetches (kind 'tags'), as for similar artists. + +CREATE TABLE artist_tags ( + artist_id BIGINT NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + tag TEXT NOT NULL, -- lower case + weight DOUBLE PRECISION NOT NULL, + genre BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (artist_id, provider, tag) +); diff --git a/migrations/sqlite/0009_artist_tags.sql b/migrations/sqlite/0009_artist_tags.sql new file mode 100644 index 0000000..543be68 --- /dev/null +++ b/migrations/sqlite/0009_artist_tags.sql @@ -0,0 +1,17 @@ +-- Tags that the providers gave to artists, kept as they gave them. +-- +-- weight is the provider's own number and means different things: Last.fm's is +-- relative to the artist (0 to 100, the top tag is usually 100), MusicBrainz's is +-- an absolute number of votes. They are never mixed here. genre is set when +-- MusicBrainz also lists the tag among its curated genres. +-- +-- Freshness is recorded in provider_fetches (kind 'tags'), as for similar artists. + +CREATE TABLE artist_tags ( + artist_id INTEGER NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + tag TEXT NOT NULL, -- lower case + weight REAL NOT NULL, + genre INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (artist_id, provider, tag) +);