- GET /artists/{id}/description shows what Wikipedia says about an artist: a short
text with its page, a link and the licence (CC BY-SA 4.0).
- New provider "wikipedia" (off by default, no key, contact in the User-Agent, which
Wikimedia requires). The way to the text is a chain of ids, never a name: the
artist's MusicBrainz id, the Wikidata item MusicBrainz links to it, the Wikipedia
page Wikidata names in the first configured language that has one. A page is used
only if it is an article about that same item.
- [wikipedia] languages in the config (one to five codes of two or three lower case
letters, checked before they are sent: an unknown site makes Wikidata answer with
every language).
- Stored 30 days (a day when there was nothing to find), served stale if Wikipedia
fails, removed with the provider's data (the purge now reports descriptions).
- MusicBrainz client: WikidataID. Wikipedia client for Wikidata and the summary,
which also reads Wikidata's errors that come with HTTP 200.
- artist_descriptions table (migration 0014, SQLite and PostgreSQL).
- Spec, contract test, README and example config updated.
Closes #39
200 lines
6.4 KiB
Go
200 lines
6.4 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"strconv"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/uptrace/bun"
|
|
"go.uber.org/zap"
|
|
|
|
"codeberg.org/squidd/sonda/internal/about"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
"codeberg.org/squidd/sonda/internal/storage"
|
|
)
|
|
|
|
// fakeWikipedia is the whole chain, in one place: it counts what was asked.
|
|
type fakeWikipedia struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
ids []string
|
|
item string
|
|
pages map[string]string
|
|
sums map[string]about.Summary
|
|
}
|
|
|
|
func (f *fakeWikipedia) WikidataID(_ context.Context, id string) (string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls++
|
|
f.ids = append(f.ids, id)
|
|
return f.item, nil
|
|
}
|
|
|
|
func (f *fakeWikipedia) Pages(context.Context, string, []string) (map[string]string, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls++
|
|
return f.pages, nil
|
|
}
|
|
|
|
func (f *fakeWikipedia) Summary(_ context.Context, lang, _ string) (about.Summary, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls++
|
|
return f.sums[lang], nil
|
|
}
|
|
|
|
func (f *fakeWikipedia) count() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.calls
|
|
}
|
|
|
|
type descEnv struct {
|
|
*similarEnv
|
|
wp *fakeWikipedia
|
|
}
|
|
|
|
func newDescEnv(t *testing.T) *descEnv {
|
|
t.Helper()
|
|
e := &descEnv{wp: &fakeWikipedia{
|
|
item: "Q44190",
|
|
pages: map[string]string{"it": "Alpha"},
|
|
sums: map[string]about.Summary{"it": {Type: "standard", Title: "Alpha", Lang: "it", Extract: "Alpha è un gruppo.",
|
|
Description: "gruppo musicale", URL: "https://it.wikipedia.org/wiki/Alpha", WikibaseItem: "Q44190"}},
|
|
}}
|
|
e.similarEnv = newSimilarEnvWith(t, func(d *Deps, db *bun.DB, prov *providers.Service) {
|
|
d.About = about.NewService(storage.NewDiscoveryRepo(db), e.wp, prov, []string{"it", "en"}, zap.NewNop())
|
|
})
|
|
return e
|
|
}
|
|
|
|
func (e *descEnv) path(id int64) string {
|
|
return "/api/v1/artists/" + strconv.FormatInt(id, 10) + "/description"
|
|
}
|
|
|
|
func (e *descEnv) enable(t *testing.T) {
|
|
t.Helper()
|
|
if r := do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": true}); r.Code != http.StatusOK {
|
|
t.Fatalf("enable wikipedia: %d %v", r.Code, r.Body)
|
|
}
|
|
}
|
|
|
|
func (e *descEnv) giveAlphaAnID(t *testing.T) {
|
|
t.Helper()
|
|
if _, err := e.db.Exec("UPDATE artists SET mbid = ? WHERE id = ?", alphaMBID, e.alpha); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDescriptionNeedsASignedInUser(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
if r := do(t, e.h, "GET", e.path(e.alpha), "", nil); r.Code != http.StatusUnauthorized {
|
|
t.Errorf("no token: %d", r.Code)
|
|
}
|
|
}
|
|
|
|
func TestDescriptionRefusesWhatIsNotAnArtist(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
e.enable(t)
|
|
for name, tt := range map[string]struct {
|
|
path string
|
|
code int
|
|
err string
|
|
}{
|
|
"unknown artist": {e.path(99999), 404, "artist_not_found"},
|
|
"not a number": {"/api/v1/artists/abc/description", 400, "invalid_input"},
|
|
"zero": {"/api/v1/artists/0/description", 400, "invalid_input"},
|
|
} {
|
|
r := do(t, e.h, "GET", tt.path, e.admin, nil)
|
|
if r.Code != tt.code || errCode(r) != tt.err {
|
|
t.Errorf("%s: %d %v, want %d %s", name, r.Code, r.Body, tt.code, tt.err)
|
|
}
|
|
}
|
|
if e.wp.count() != 0 {
|
|
t.Error("Wikipedia was asked about something that is not an artist")
|
|
}
|
|
}
|
|
|
|
func TestDescriptionWithTheProviderOffAsksNobody(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
e.giveAlphaAnID(t)
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
sameKeys(t, "description", r.Body, "provider", "provider_status", "stale", "retrieved_at", "attribution", "description")
|
|
if r.Body["provider"] != "wikipedia" || r.Body["provider_status"] != "disabled" || r.Body["stale"] != false || r.Body["attribution"] == "" {
|
|
t.Errorf("answer = %v", r.Body)
|
|
}
|
|
if v, present := r.Body["description"]; !present || v != nil {
|
|
t.Errorf("description = %#v, want an explicit null", v)
|
|
}
|
|
if v, present := r.Body["retrieved_at"]; !present || v != nil {
|
|
t.Errorf("retrieved_at = %#v, want an explicit null", v)
|
|
}
|
|
if e.wp.count() != 0 {
|
|
t.Error("a provider that is off was contacted")
|
|
}
|
|
}
|
|
|
|
func TestDescriptionOfAnArtistWithoutAnIdSaysSo(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
e.enable(t)
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK || r.Body["provider_status"] != "no_id" || r.Body["description"] != nil || e.wp.count() != 0 {
|
|
t.Errorf("%d %v, calls %d: Wikipedia must never be asked about an artist by name", r.Code, r.Body, e.wp.count())
|
|
}
|
|
}
|
|
|
|
func TestDescriptionIsReadStoredAndShownWithItsLicence(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
e.enable(t)
|
|
e.giveAlphaAnID(t)
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Code != http.StatusOK || r.Body["provider_status"] != "ok" {
|
|
t.Fatalf("%d %v", r.Code, r.Body)
|
|
}
|
|
d, _ := r.Body["description"].(map[string]any)
|
|
sameKeys(t, "description body", d, "text", "short", "language", "title", "url", "license", "license_url")
|
|
if d["text"] != "Alpha è un gruppo." || d["short"] != "gruppo musicale" || d["language"] != "it" || d["title"] != "Alpha" ||
|
|
d["url"] != "https://it.wikipedia.org/wiki/Alpha" || d["license"] != "CC BY-SA 4.0" || d["license_url"] != "https://creativecommons.org/licenses/by-sa/4.0/" {
|
|
t.Errorf("description = %v", d)
|
|
}
|
|
if _, err := time.Parse(time.RFC3339, r.Body["retrieved_at"].(string)); err != nil {
|
|
t.Errorf("retrieved_at = %v", r.Body["retrieved_at"])
|
|
}
|
|
if len(e.wp.ids) != 1 || e.wp.ids[0] != alphaMBID {
|
|
t.Errorf("asked about %v, want only the MusicBrainz id", e.wp.ids)
|
|
}
|
|
// Stored: a second look asks nobody.
|
|
calls := e.wp.count()
|
|
if r = do(t, e.h, "GET", e.path(e.alpha), e.admin, nil); r.Code != http.StatusOK || e.wp.count() != calls {
|
|
t.Errorf("second look: %d, calls %d -> %d", r.Code, calls, e.wp.count())
|
|
}
|
|
}
|
|
|
|
func TestPurgingWikipediaTakesTheDescriptionsBack(t *testing.T) {
|
|
e := newDescEnv(t)
|
|
e.enable(t)
|
|
e.giveAlphaAnID(t)
|
|
do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
p := do(t, e.h, "DELETE", "/api/v1/providers/wikipedia/data", e.admin, nil)
|
|
if p.Code != http.StatusOK {
|
|
t.Fatalf("purge: %d %v", p.Code, p.Body)
|
|
}
|
|
sameKeys(t, "purge", p.Body, "similar_artists", "tags", "descriptions", "artists")
|
|
if p.Body["descriptions"] != float64(1) {
|
|
t.Errorf("purge = %v, want one description", p.Body)
|
|
}
|
|
// Switched off, nothing is read again, and nothing is left to show.
|
|
do(t, e.h, "PUT", "/api/v1/providers/wikipedia", e.admin, map[string]any{"enabled": false})
|
|
r := do(t, e.h, "GET", e.path(e.alpha), e.admin, nil)
|
|
if r.Body["description"] != nil || r.Body["provider_status"] != "disabled" {
|
|
t.Errorf("after the purge: %v", r.Body)
|
|
}
|
|
}
|