- A page that cannot be used (no text, an address off Wikipedia, a redirect that is never followed) is now an answer: the chain goes on to the next language and the outcome is stored. Before, it was a failure asked about again at every view, and redirects counted against the provider's breaker. - The chain also needs the MusicBrainz provider to be on, since the id goes to it with its own contact: nothing is sent to MusicBrainz that the admin did not allow. The disclosure in the provider settings, the spec and the README say so. - A Wikidata item that was merged into another is understood through the target: the entity's own id is returned, compared with the page's item and stored. Verified on the real API. - The title shown is the one Wikidata names (with spaces), control characters are dropped from the text (PostgreSQL refuses NUL), the text is cut to at most 4000 characters with the mark included, and a stored description is reported disabled when the provider is off. - A read that started before the provider was switched off does not save its result. - Tests: the real clients with a redirect and a page without text, merged items, MusicBrainz off, switching off during a read.
513 lines
16 KiB
Go
513 lines
16 KiB
Go
package about
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
)
|
|
|
|
const (
|
|
mbid = "11111111-2222-4333-8444-555555555555"
|
|
key = "unused"
|
|
)
|
|
|
|
var t0 = time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
|
|
|
type saved struct {
|
|
id int64
|
|
at time.Time
|
|
qid string
|
|
desc *Description
|
|
}
|
|
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
artists map[int64]discovery.Artist
|
|
snaps map[int64]Snapshot
|
|
saves []saved
|
|
saveErr error
|
|
}
|
|
|
|
func newStore() *fakeStore {
|
|
return &fakeStore{
|
|
artists: map[int64]discovery.Artist{
|
|
1: {ID: 1, Name: "With Id", MBID: mbid},
|
|
2: {ID: 2, Name: "No Id"},
|
|
},
|
|
snaps: map[int64]Snapshot{},
|
|
}
|
|
}
|
|
|
|
func (f *fakeStore) Artist(_ context.Context, id int64) (discovery.Artist, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
a, ok := f.artists[id]
|
|
if !ok {
|
|
return discovery.Artist{}, discovery.ErrNotFound
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
func (f *fakeStore) Description(_ context.Context, id int64) (Snapshot, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.snaps[id], nil
|
|
}
|
|
|
|
func (f *fakeStore) SaveDescription(_ context.Context, id int64, at time.Time, qid string, d *Description) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.saveErr != nil {
|
|
return f.saveErr
|
|
}
|
|
f.saves = append(f.saves, saved{id, at, qid, d})
|
|
snap := Snapshot{Fetched: true, None: d == nil, RetrievedAt: at}
|
|
if d != nil {
|
|
snap.Description = *d
|
|
}
|
|
f.snaps[id] = snap
|
|
return nil
|
|
}
|
|
|
|
type fakeSource struct {
|
|
mu sync.Mutex
|
|
qid string
|
|
resolved string // the item the pages belong to; empty means the same as qid
|
|
qidErr error
|
|
pages map[string]string
|
|
pagesErr error
|
|
summaries map[string]Summary // by language
|
|
sumErrs map[string]error
|
|
calls struct{ qid, pages, summary int }
|
|
asked []string // languages asked for a summary, in order
|
|
langs []string // what Pages was asked for
|
|
mbid string
|
|
release chan struct{}
|
|
boom any
|
|
}
|
|
|
|
func (s *fakeSource) WikidataID(_ context.Context, id string) (string, error) {
|
|
s.mu.Lock()
|
|
s.calls.qid++
|
|
s.mbid = id
|
|
release, boom := s.release, s.boom
|
|
s.mu.Unlock()
|
|
if release != nil {
|
|
<-release
|
|
}
|
|
if boom != nil {
|
|
panic(boom)
|
|
}
|
|
return s.qid, s.qidErr
|
|
}
|
|
|
|
func (s *fakeSource) Pages(_ context.Context, qid string, langs []string) (string, map[string]string, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.calls.pages++
|
|
s.langs = append([]string(nil), langs...)
|
|
resolved := s.resolved
|
|
if resolved == "" {
|
|
resolved = qid
|
|
}
|
|
return resolved, s.pages, s.pagesErr
|
|
}
|
|
|
|
func (s *fakeSource) Summary(_ context.Context, lang, title string) (Summary, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.calls.summary++
|
|
s.asked = append(s.asked, lang)
|
|
if err := s.sumErrs[lang]; err != nil {
|
|
return Summary{}, err
|
|
}
|
|
return s.summaries[lang], nil
|
|
}
|
|
|
|
func (s *fakeSource) total() int {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.calls.qid + s.calls.pages + s.calls.summary
|
|
}
|
|
|
|
// switches says which providers are on, and can be changed while a read is running.
|
|
type switches struct {
|
|
mu sync.Mutex
|
|
m map[string]bool
|
|
}
|
|
|
|
func (p *switches) Active(_ context.Context, name string) (providers.Settings, bool, error) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return providers.Settings{Provider: name, Enabled: p.m[name]}, p.m[name], nil
|
|
}
|
|
|
|
func (p *switches) set(name string, on bool) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.m[name] = on
|
|
}
|
|
|
|
type rig struct {
|
|
sw *switches
|
|
svc *Service
|
|
store *fakeStore
|
|
src *fakeSource
|
|
now time.Time
|
|
}
|
|
|
|
func standard(lang, qid, text string) Summary {
|
|
return Summary{Type: "standard", Title: "Radiohead", Lang: lang, Extract: text, Description: "una band",
|
|
URL: "https://" + lang + ".wikipedia.org/wiki/Radiohead", WikibaseItem: qid}
|
|
}
|
|
|
|
func newRig(enabled bool) *rig {
|
|
r := &rig{store: newStore(), now: t0}
|
|
r.src = &fakeSource{
|
|
qid: "Q44190",
|
|
pages: map[string]string{"it": "Radiohead", "en": "Radiohead"},
|
|
summaries: map[string]Summary{
|
|
"it": standard("it", "Q44190", "I Radiohead sono un gruppo musicale."),
|
|
"en": standard("en", "Q44190", "Radiohead are an English rock band."),
|
|
},
|
|
sumErrs: map[string]error{},
|
|
}
|
|
r.sw = &switches{m: map[string]bool{providers.Wikipedia: enabled, providers.MusicBrainz: enabled}}
|
|
r.svc = NewService(r.store, r.src, r.sw, []string{"it", "en"}, zap.NewNop())
|
|
r.svc.now = func() time.Time { return r.now }
|
|
return r
|
|
}
|
|
|
|
func (r *rig) get(t *testing.T, id int64) Result {
|
|
t.Helper()
|
|
res, err := r.svc.Description(context.Background(), id)
|
|
if err != nil {
|
|
t.Fatalf("Description: %v", err)
|
|
}
|
|
return res
|
|
}
|
|
|
|
func TestTheDescriptionComesInTheFirstLanguageThatHasAPage(t *testing.T) {
|
|
r := newRig(true)
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusOK || res.Description == nil {
|
|
t.Fatalf("result = %+v", res)
|
|
}
|
|
d := res.Description
|
|
if d.Language != "it" || d.Text != "I Radiohead sono un gruppo musicale." || d.Short != "una band" || d.Title != "Radiohead" ||
|
|
d.URL != "https://it.wikipedia.org/wiki/Radiohead" {
|
|
t.Errorf("description = %+v", d)
|
|
}
|
|
if !res.RetrievedAt.Equal(t0) || res.Stale {
|
|
t.Errorf("retrieved %v stale %v", res.RetrievedAt, res.Stale)
|
|
}
|
|
if r.src.mbid != mbid {
|
|
t.Errorf("asked about %q, want the MusicBrainz id", r.src.mbid)
|
|
}
|
|
if got := strings.Join(r.src.langs, ","); got != "it,en" {
|
|
t.Errorf("languages = %s, want it,en in the order of the settings", got)
|
|
}
|
|
if len(r.src.asked) != 1 {
|
|
t.Errorf("summaries read: %v, want only the first language that works", r.src.asked)
|
|
}
|
|
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q44190" {
|
|
t.Errorf("saves = %+v", r.store.saves)
|
|
}
|
|
}
|
|
|
|
func TestTheNextLanguageIsUsedWhenThereIsNoPageOrNoUsableOne(t *testing.T) {
|
|
for name, tt := range map[string]func(*rig){
|
|
"no page in Italian": func(r *rig) { r.src.pages = map[string]string{"en": "Radiohead"} },
|
|
"the page is gone": func(r *rig) { r.src.sumErrs["it"] = ErrSourceNotFound },
|
|
"a disambiguation": func(r *rig) {
|
|
s := r.src.summaries["it"]
|
|
s.Type = "disambiguation"
|
|
r.src.summaries["it"] = s
|
|
},
|
|
"another item": func(r *rig) {
|
|
s := r.src.summaries["it"]
|
|
s.WikibaseItem = "Q1"
|
|
r.src.summaries["it"] = s
|
|
},
|
|
"no text": func(r *rig) {
|
|
s := r.src.summaries["it"]
|
|
s.Extract = " "
|
|
r.src.summaries["it"] = s
|
|
},
|
|
} {
|
|
r := newRig(true)
|
|
tt(r)
|
|
res := r.get(t, 1)
|
|
if res.Description == nil || res.Description.Language != "en" {
|
|
t.Errorf("%s: description = %+v, want the English one", name, res.Description)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNothingFoundIsRememberedAsSuchForADay(t *testing.T) {
|
|
for name, tt := range map[string]func(*rig){
|
|
"no wikidata link": func(r *rig) { r.src.qid = "" },
|
|
"no unknown item": func(r *rig) { r.src.pagesErr = ErrSourceNotFound },
|
|
"no page at all": func(r *rig) { r.src.pages = map[string]string{} },
|
|
"musicbrainz does not know the artist": func(r *rig) { r.src.qidErr = ErrSourceNotFound },
|
|
} {
|
|
r := newRig(true)
|
|
tt(r)
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusOK || res.Description != nil || res.RetrievedAt.IsZero() {
|
|
t.Errorf("%s: result = %+v, want ok with no description", name, res)
|
|
continue
|
|
}
|
|
if len(r.store.saves) != 1 || r.store.saves[0].desc != nil {
|
|
t.Errorf("%s: saves = %+v, want one 'nothing' record", name, r.store.saves)
|
|
}
|
|
calls := r.src.total()
|
|
r.now = t0.Add(NegativeTTL - time.Minute)
|
|
r.get(t, 1)
|
|
if r.src.total() != calls {
|
|
t.Errorf("%s: asked again before the day was over", name)
|
|
}
|
|
r.now = t0.Add(NegativeTTL + time.Minute)
|
|
r.get(t, 1)
|
|
if r.src.total() == calls {
|
|
t.Errorf("%s: never asked again", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestADescriptionIsKeptThirtyDaysAndStaleWhenWikipediaFails(t *testing.T) {
|
|
r := newRig(true)
|
|
r.get(t, 1)
|
|
calls := r.src.total()
|
|
r.now = t0.Add(PositiveTTL - time.Hour)
|
|
if res := r.get(t, 1); res.Stale || r.src.total() != calls {
|
|
t.Fatalf("within the time: stale %v, calls %d -> %d", res.Stale, calls, r.src.total())
|
|
}
|
|
r.now = t0.Add(PositiveTTL + time.Hour)
|
|
r.src.pagesErr = ErrSourceUnavailable
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusError || !res.Stale || res.Description == nil || res.Description.Language != "it" {
|
|
t.Errorf("when Wikipedia fails: %+v, want the old description, stale, with status error", res)
|
|
}
|
|
r.src.pagesErr = nil
|
|
if res = r.get(t, 1); res.Status != discovery.StatusOK || res.Stale || !res.RetrievedAt.Equal(r.now) {
|
|
t.Errorf("after it came back: %+v", res)
|
|
}
|
|
}
|
|
|
|
func TestAFailureIsNotRememberedAndASlowLanguageDoesNotFallThrough(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.sumErrs["it"] = ErrSourceUnavailable // Italian fails: English must not be saved in its place
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusError || res.Description != nil || len(r.store.saves) != 0 {
|
|
t.Errorf("result = %+v saves = %+v: a failure must not be saved, nor replaced by the next language", res, r.store.saves)
|
|
}
|
|
if len(r.src.asked) != 1 {
|
|
t.Errorf("summaries read: %v", r.src.asked)
|
|
}
|
|
}
|
|
|
|
func TestWithTheProviderOffNobodyIsAskedAndTheOldAnswerIsStillGiven(t *testing.T) {
|
|
r := newRig(false)
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusDisabled || res.Description != nil || !res.RetrievedAt.IsZero() || res.Stale || r.src.total() != 0 {
|
|
t.Errorf("result = %+v calls %d", res, r.src.total())
|
|
}
|
|
r.store.snaps[1] = Snapshot{Fetched: true, RetrievedAt: t0.Add(-100 * 24 * time.Hour), Description: Description{Text: "old", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}}
|
|
res = r.get(t, 1)
|
|
if res.Status != discovery.StatusDisabled || res.Description == nil || res.Description.Text != "old" || !res.Stale || r.src.total() != 0 {
|
|
t.Errorf("with an old answer: %+v", res)
|
|
}
|
|
}
|
|
|
|
func TestAnArtistWithoutAnIdIsNeverLookedUpByName(t *testing.T) {
|
|
r := newRig(true)
|
|
res := r.get(t, 2)
|
|
if res.Status != discovery.StatusNoID || res.Description != nil || r.src.total() != 0 || len(r.store.saves) != 0 {
|
|
t.Errorf("result = %+v calls %d saves %d", res, r.src.total(), len(r.store.saves))
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownArtistIsNotFound(t *testing.T) {
|
|
r := newRig(true)
|
|
for _, id := range []int64{0, -1, 99} {
|
|
if _, err := r.svc.Description(context.Background(), id); !errors.Is(err, discovery.ErrNotFound) {
|
|
t.Errorf("id %d: err = %v", id, err)
|
|
}
|
|
}
|
|
if r.src.total() != 0 {
|
|
t.Error("Wikipedia was asked about an artist that does not exist")
|
|
}
|
|
}
|
|
|
|
func TestTheTextIsCutAtTheLimit(t *testing.T) {
|
|
r := newRig(true)
|
|
long := standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
|
|
r.src.summaries["it"] = long
|
|
d := r.get(t, 1).Description
|
|
if n := len([]rune(d.Text)); n > MaxTextLen+1 || !strings.HasSuffix(d.Text, "…") {
|
|
t.Errorf("text has %d characters, want at most %d and a mark that it was cut", n, MaxTextLen)
|
|
}
|
|
}
|
|
|
|
func TestManyAsksAtOnceAskWikipediaOnce(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.release = make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 10; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if _, err := r.svc.Description(context.Background(), 1); err != nil {
|
|
t.Errorf("err = %v", err)
|
|
}
|
|
}()
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
close(r.src.release)
|
|
wg.Wait()
|
|
if r.src.calls.qid != 1 {
|
|
t.Errorf("MusicBrainz asked %d times, want 1", r.src.calls.qid)
|
|
}
|
|
}
|
|
|
|
func TestAListenerWhoLeavesDoesNotStopTheRead(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.release = make(chan struct{})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
errc := make(chan error, 1)
|
|
go func() { _, err := r.svc.Description(ctx, 1); errc <- err }()
|
|
time.Sleep(30 * time.Millisecond)
|
|
cancel()
|
|
if err := <-errc; !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
close(r.src.release)
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
r.store.mu.Lock()
|
|
n := len(r.store.saves)
|
|
r.store.mu.Unlock()
|
|
if n == 1 {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the read was stopped when the listener left")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestAPanicInTheSourceIsAFailureNotACrash(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.boom = "boom"
|
|
if res := r.get(t, 1); res.Status != discovery.StatusError {
|
|
t.Errorf("status = %q", res.Status)
|
|
}
|
|
}
|
|
|
|
func TestAnArtistRemovedWhileItWasReadIsNotFound(t *testing.T) {
|
|
r := newRig(true)
|
|
r.store.saveErr = discovery.ErrNotFound
|
|
if _, err := r.svc.Description(context.Background(), 1); !errors.Is(err, discovery.ErrNotFound) {
|
|
t.Errorf("err = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestItNeedsMusicBrainzToBeOnToo(t *testing.T) {
|
|
// The link to Wikidata comes from MusicBrainz: with MusicBrainz switched off nothing is sent to it.
|
|
r := newRig(true)
|
|
r.sw.set(providers.MusicBrainz, false)
|
|
res := r.get(t, 1)
|
|
if res.Status != discovery.StatusDisabled || res.Description != nil || r.src.total() != 0 {
|
|
t.Errorf("result = %+v, calls %d: nobody may be asked while MusicBrainz is off", res, r.src.total())
|
|
}
|
|
r.sw.set(providers.MusicBrainz, true)
|
|
r.sw.set(providers.Wikipedia, false)
|
|
if res = r.get(t, 1); res.Status != discovery.StatusDisabled || r.src.total() != 0 {
|
|
t.Errorf("with Wikipedia off: %+v calls %d", res, r.src.total())
|
|
}
|
|
}
|
|
|
|
func TestAMergedItemIsUnderstoodThroughTheItemItWasMergedInto(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.qid, r.src.resolved = "Q1", "Q2" // MusicBrainz knows the old id; the pages say Q2
|
|
for lang, sum := range r.src.summaries {
|
|
sum.WikibaseItem = "Q2"
|
|
r.src.summaries[lang] = sum
|
|
}
|
|
res := r.get(t, 1)
|
|
if res.Description == nil || res.Description.Language != "it" {
|
|
t.Fatalf("result = %+v, want the description of the item it was merged into", res)
|
|
}
|
|
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q2" {
|
|
t.Errorf("saves = %+v, want the resolved item stored", r.store.saves)
|
|
}
|
|
}
|
|
|
|
func TestTheTitleShownIsTheOneWikidataNames(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.pages = map[string]string{"it": "Guns N' Roses"}
|
|
s := r.src.summaries["it"]
|
|
s.Title = "Guns_N'_Roses" // the REST title is the database key, with underscores
|
|
r.src.summaries["it"] = s
|
|
if d := r.get(t, 1).Description; d == nil || d.Title != "Guns N' Roses" {
|
|
t.Errorf("description = %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestAPageWithNoAddressOnWikipediaIsNotUsed(t *testing.T) {
|
|
r := newRig(true)
|
|
s := r.src.summaries["it"]
|
|
s.URL = "" // the client drops an address that is not on that Wikipedia
|
|
r.src.summaries["it"] = s
|
|
if d := r.get(t, 1).Description; d == nil || d.Language != "en" {
|
|
t.Errorf("description = %+v, want the next language: a text without its source cannot be shown", d)
|
|
}
|
|
}
|
|
|
|
func TestTheTextNeverExceedsTheLimit(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.summaries["it"] = standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
|
|
if n := len([]rune(r.get(t, 1).Description.Text)); n != MaxTextLen {
|
|
t.Errorf("text has %d characters, want exactly %d with the mark that it was cut", n, MaxTextLen)
|
|
}
|
|
}
|
|
|
|
func TestAStoredDescriptionIsStillReportedAsDisabledWhenTheProviderIsOff(t *testing.T) {
|
|
r := newRig(true)
|
|
r.get(t, 1)
|
|
r.sw.set(providers.Wikipedia, false)
|
|
res := r.get(t, 1) // fresh, but the switch is off
|
|
if res.Status != discovery.StatusDisabled || res.Stale || res.Description == nil {
|
|
t.Errorf("result = %+v, want the description, not stale, with status disabled", res)
|
|
}
|
|
}
|
|
|
|
func TestSwitchedOffWhileItWasBeingReadNothingIsSaved(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.release = make(chan struct{})
|
|
done := make(chan Result, 1)
|
|
go func() {
|
|
res, _ := r.svc.Description(context.Background(), 1)
|
|
done <- res
|
|
}()
|
|
time.Sleep(50 * time.Millisecond)
|
|
r.sw.set(providers.Wikipedia, false) // the admin took the data back and switched it off
|
|
close(r.src.release)
|
|
res := <-done
|
|
if len(r.store.saves) != 0 {
|
|
t.Errorf("saves = %+v: what a read that started before the switch found must not be kept", r.store.saves)
|
|
}
|
|
if res.Status != discovery.StatusDisabled || res.Description != nil {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
}
|