sonda/internal/about/about_test.go
tgiachi 762d424816 fix(about): unusable pages fall through, MusicBrainz switch, merged items
- A page that cannot be used (no text, an address off Wikipedia, a redirect that is
  never followed) is now an answer: the chain goes on to the next language and the
  outcome is stored. Before, it was a failure asked about again at every view, and
  redirects counted against the provider's breaker.
- The chain also needs the MusicBrainz provider to be on, since the id goes to it
  with its own contact: nothing is sent to MusicBrainz that the admin did not allow.
  The disclosure in the provider settings, the spec and the README say so.
- A Wikidata item that was merged into another is understood through the target: the
  entity's own id is returned, compared with the page's item and stored. Verified on
  the real API.
- The title shown is the one Wikidata names (with spaces), control characters are
  dropped from the text (PostgreSQL refuses NUL), the text is cut to at most 4000
  characters with the mark included, and a stored description is reported disabled
  when the provider is off.
- A read that started before the provider was switched off does not save its result.
- Tests: the real clients with a redirect and a page without text, merged items,
  MusicBrainz off, switching off during a read.
2026-11-11 21:45:20 +01:00

513 lines
16 KiB
Go

package about
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"go.uber.org/zap"
"codeberg.org/squidd/sonda/internal/discovery"
"codeberg.org/squidd/sonda/internal/providers"
)
const (
mbid = "11111111-2222-4333-8444-555555555555"
key = "unused"
)
var t0 = time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
type saved struct {
id int64
at time.Time
qid string
desc *Description
}
type fakeStore struct {
mu sync.Mutex
artists map[int64]discovery.Artist
snaps map[int64]Snapshot
saves []saved
saveErr error
}
func newStore() *fakeStore {
return &fakeStore{
artists: map[int64]discovery.Artist{
1: {ID: 1, Name: "With Id", MBID: mbid},
2: {ID: 2, Name: "No Id"},
},
snaps: map[int64]Snapshot{},
}
}
func (f *fakeStore) Artist(_ context.Context, id int64) (discovery.Artist, error) {
f.mu.Lock()
defer f.mu.Unlock()
a, ok := f.artists[id]
if !ok {
return discovery.Artist{}, discovery.ErrNotFound
}
return a, nil
}
func (f *fakeStore) Description(_ context.Context, id int64) (Snapshot, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.snaps[id], nil
}
func (f *fakeStore) SaveDescription(_ context.Context, id int64, at time.Time, qid string, d *Description) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.saveErr != nil {
return f.saveErr
}
f.saves = append(f.saves, saved{id, at, qid, d})
snap := Snapshot{Fetched: true, None: d == nil, RetrievedAt: at}
if d != nil {
snap.Description = *d
}
f.snaps[id] = snap
return nil
}
type fakeSource struct {
mu sync.Mutex
qid string
resolved string // the item the pages belong to; empty means the same as qid
qidErr error
pages map[string]string
pagesErr error
summaries map[string]Summary // by language
sumErrs map[string]error
calls struct{ qid, pages, summary int }
asked []string // languages asked for a summary, in order
langs []string // what Pages was asked for
mbid string
release chan struct{}
boom any
}
func (s *fakeSource) WikidataID(_ context.Context, id string) (string, error) {
s.mu.Lock()
s.calls.qid++
s.mbid = id
release, boom := s.release, s.boom
s.mu.Unlock()
if release != nil {
<-release
}
if boom != nil {
panic(boom)
}
return s.qid, s.qidErr
}
func (s *fakeSource) Pages(_ context.Context, qid string, langs []string) (string, map[string]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.calls.pages++
s.langs = append([]string(nil), langs...)
resolved := s.resolved
if resolved == "" {
resolved = qid
}
return resolved, s.pages, s.pagesErr
}
func (s *fakeSource) Summary(_ context.Context, lang, title string) (Summary, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.calls.summary++
s.asked = append(s.asked, lang)
if err := s.sumErrs[lang]; err != nil {
return Summary{}, err
}
return s.summaries[lang], nil
}
func (s *fakeSource) total() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls.qid + s.calls.pages + s.calls.summary
}
// switches says which providers are on, and can be changed while a read is running.
type switches struct {
mu sync.Mutex
m map[string]bool
}
func (p *switches) Active(_ context.Context, name string) (providers.Settings, bool, error) {
p.mu.Lock()
defer p.mu.Unlock()
return providers.Settings{Provider: name, Enabled: p.m[name]}, p.m[name], nil
}
func (p *switches) set(name string, on bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.m[name] = on
}
type rig struct {
sw *switches
svc *Service
store *fakeStore
src *fakeSource
now time.Time
}
func standard(lang, qid, text string) Summary {
return Summary{Type: "standard", Title: "Radiohead", Lang: lang, Extract: text, Description: "una band",
URL: "https://" + lang + ".wikipedia.org/wiki/Radiohead", WikibaseItem: qid}
}
func newRig(enabled bool) *rig {
r := &rig{store: newStore(), now: t0}
r.src = &fakeSource{
qid: "Q44190",
pages: map[string]string{"it": "Radiohead", "en": "Radiohead"},
summaries: map[string]Summary{
"it": standard("it", "Q44190", "I Radiohead sono un gruppo musicale."),
"en": standard("en", "Q44190", "Radiohead are an English rock band."),
},
sumErrs: map[string]error{},
}
r.sw = &switches{m: map[string]bool{providers.Wikipedia: enabled, providers.MusicBrainz: enabled}}
r.svc = NewService(r.store, r.src, r.sw, []string{"it", "en"}, zap.NewNop())
r.svc.now = func() time.Time { return r.now }
return r
}
func (r *rig) get(t *testing.T, id int64) Result {
t.Helper()
res, err := r.svc.Description(context.Background(), id)
if err != nil {
t.Fatalf("Description: %v", err)
}
return res
}
func TestTheDescriptionComesInTheFirstLanguageThatHasAPage(t *testing.T) {
r := newRig(true)
res := r.get(t, 1)
if res.Status != discovery.StatusOK || res.Description == nil {
t.Fatalf("result = %+v", res)
}
d := res.Description
if d.Language != "it" || d.Text != "I Radiohead sono un gruppo musicale." || d.Short != "una band" || d.Title != "Radiohead" ||
d.URL != "https://it.wikipedia.org/wiki/Radiohead" {
t.Errorf("description = %+v", d)
}
if !res.RetrievedAt.Equal(t0) || res.Stale {
t.Errorf("retrieved %v stale %v", res.RetrievedAt, res.Stale)
}
if r.src.mbid != mbid {
t.Errorf("asked about %q, want the MusicBrainz id", r.src.mbid)
}
if got := strings.Join(r.src.langs, ","); got != "it,en" {
t.Errorf("languages = %s, want it,en in the order of the settings", got)
}
if len(r.src.asked) != 1 {
t.Errorf("summaries read: %v, want only the first language that works", r.src.asked)
}
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q44190" {
t.Errorf("saves = %+v", r.store.saves)
}
}
func TestTheNextLanguageIsUsedWhenThereIsNoPageOrNoUsableOne(t *testing.T) {
for name, tt := range map[string]func(*rig){
"no page in Italian": func(r *rig) { r.src.pages = map[string]string{"en": "Radiohead"} },
"the page is gone": func(r *rig) { r.src.sumErrs["it"] = ErrSourceNotFound },
"a disambiguation": func(r *rig) {
s := r.src.summaries["it"]
s.Type = "disambiguation"
r.src.summaries["it"] = s
},
"another item": func(r *rig) {
s := r.src.summaries["it"]
s.WikibaseItem = "Q1"
r.src.summaries["it"] = s
},
"no text": func(r *rig) {
s := r.src.summaries["it"]
s.Extract = " "
r.src.summaries["it"] = s
},
} {
r := newRig(true)
tt(r)
res := r.get(t, 1)
if res.Description == nil || res.Description.Language != "en" {
t.Errorf("%s: description = %+v, want the English one", name, res.Description)
}
}
}
func TestNothingFoundIsRememberedAsSuchForADay(t *testing.T) {
for name, tt := range map[string]func(*rig){
"no wikidata link": func(r *rig) { r.src.qid = "" },
"no unknown item": func(r *rig) { r.src.pagesErr = ErrSourceNotFound },
"no page at all": func(r *rig) { r.src.pages = map[string]string{} },
"musicbrainz does not know the artist": func(r *rig) { r.src.qidErr = ErrSourceNotFound },
} {
r := newRig(true)
tt(r)
res := r.get(t, 1)
if res.Status != discovery.StatusOK || res.Description != nil || res.RetrievedAt.IsZero() {
t.Errorf("%s: result = %+v, want ok with no description", name, res)
continue
}
if len(r.store.saves) != 1 || r.store.saves[0].desc != nil {
t.Errorf("%s: saves = %+v, want one 'nothing' record", name, r.store.saves)
}
calls := r.src.total()
r.now = t0.Add(NegativeTTL - time.Minute)
r.get(t, 1)
if r.src.total() != calls {
t.Errorf("%s: asked again before the day was over", name)
}
r.now = t0.Add(NegativeTTL + time.Minute)
r.get(t, 1)
if r.src.total() == calls {
t.Errorf("%s: never asked again", name)
}
}
}
func TestADescriptionIsKeptThirtyDaysAndStaleWhenWikipediaFails(t *testing.T) {
r := newRig(true)
r.get(t, 1)
calls := r.src.total()
r.now = t0.Add(PositiveTTL - time.Hour)
if res := r.get(t, 1); res.Stale || r.src.total() != calls {
t.Fatalf("within the time: stale %v, calls %d -> %d", res.Stale, calls, r.src.total())
}
r.now = t0.Add(PositiveTTL + time.Hour)
r.src.pagesErr = ErrSourceUnavailable
res := r.get(t, 1)
if res.Status != discovery.StatusError || !res.Stale || res.Description == nil || res.Description.Language != "it" {
t.Errorf("when Wikipedia fails: %+v, want the old description, stale, with status error", res)
}
r.src.pagesErr = nil
if res = r.get(t, 1); res.Status != discovery.StatusOK || res.Stale || !res.RetrievedAt.Equal(r.now) {
t.Errorf("after it came back: %+v", res)
}
}
func TestAFailureIsNotRememberedAndASlowLanguageDoesNotFallThrough(t *testing.T) {
r := newRig(true)
r.src.sumErrs["it"] = ErrSourceUnavailable // Italian fails: English must not be saved in its place
res := r.get(t, 1)
if res.Status != discovery.StatusError || res.Description != nil || len(r.store.saves) != 0 {
t.Errorf("result = %+v saves = %+v: a failure must not be saved, nor replaced by the next language", res, r.store.saves)
}
if len(r.src.asked) != 1 {
t.Errorf("summaries read: %v", r.src.asked)
}
}
func TestWithTheProviderOffNobodyIsAskedAndTheOldAnswerIsStillGiven(t *testing.T) {
r := newRig(false)
res := r.get(t, 1)
if res.Status != discovery.StatusDisabled || res.Description != nil || !res.RetrievedAt.IsZero() || res.Stale || r.src.total() != 0 {
t.Errorf("result = %+v calls %d", res, r.src.total())
}
r.store.snaps[1] = Snapshot{Fetched: true, RetrievedAt: t0.Add(-100 * 24 * time.Hour), Description: Description{Text: "old", Language: "en", Title: "T", URL: "https://en.wikipedia.org/wiki/T"}}
res = r.get(t, 1)
if res.Status != discovery.StatusDisabled || res.Description == nil || res.Description.Text != "old" || !res.Stale || r.src.total() != 0 {
t.Errorf("with an old answer: %+v", res)
}
}
func TestAnArtistWithoutAnIdIsNeverLookedUpByName(t *testing.T) {
r := newRig(true)
res := r.get(t, 2)
if res.Status != discovery.StatusNoID || res.Description != nil || r.src.total() != 0 || len(r.store.saves) != 0 {
t.Errorf("result = %+v calls %d saves %d", res, r.src.total(), len(r.store.saves))
}
}
func TestAnUnknownArtistIsNotFound(t *testing.T) {
r := newRig(true)
for _, id := range []int64{0, -1, 99} {
if _, err := r.svc.Description(context.Background(), id); !errors.Is(err, discovery.ErrNotFound) {
t.Errorf("id %d: err = %v", id, err)
}
}
if r.src.total() != 0 {
t.Error("Wikipedia was asked about an artist that does not exist")
}
}
func TestTheTextIsCutAtTheLimit(t *testing.T) {
r := newRig(true)
long := standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
r.src.summaries["it"] = long
d := r.get(t, 1).Description
if n := len([]rune(d.Text)); n > MaxTextLen+1 || !strings.HasSuffix(d.Text, "…") {
t.Errorf("text has %d characters, want at most %d and a mark that it was cut", n, MaxTextLen)
}
}
func TestManyAsksAtOnceAskWikipediaOnce(t *testing.T) {
r := newRig(true)
r.src.release = make(chan struct{})
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := r.svc.Description(context.Background(), 1); err != nil {
t.Errorf("err = %v", err)
}
}()
}
time.Sleep(50 * time.Millisecond)
close(r.src.release)
wg.Wait()
if r.src.calls.qid != 1 {
t.Errorf("MusicBrainz asked %d times, want 1", r.src.calls.qid)
}
}
func TestAListenerWhoLeavesDoesNotStopTheRead(t *testing.T) {
r := newRig(true)
r.src.release = make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
errc := make(chan error, 1)
go func() { _, err := r.svc.Description(ctx, 1); errc <- err }()
time.Sleep(30 * time.Millisecond)
cancel()
if err := <-errc; !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v", err)
}
close(r.src.release)
deadline := time.Now().Add(3 * time.Second)
for {
r.store.mu.Lock()
n := len(r.store.saves)
r.store.mu.Unlock()
if n == 1 {
break
}
if time.Now().After(deadline) {
t.Fatal("the read was stopped when the listener left")
}
time.Sleep(10 * time.Millisecond)
}
}
func TestAPanicInTheSourceIsAFailureNotACrash(t *testing.T) {
r := newRig(true)
r.src.boom = "boom"
if res := r.get(t, 1); res.Status != discovery.StatusError {
t.Errorf("status = %q", res.Status)
}
}
func TestAnArtistRemovedWhileItWasReadIsNotFound(t *testing.T) {
r := newRig(true)
r.store.saveErr = discovery.ErrNotFound
if _, err := r.svc.Description(context.Background(), 1); !errors.Is(err, discovery.ErrNotFound) {
t.Errorf("err = %v", err)
}
}
func TestItNeedsMusicBrainzToBeOnToo(t *testing.T) {
// The link to Wikidata comes from MusicBrainz: with MusicBrainz switched off nothing is sent to it.
r := newRig(true)
r.sw.set(providers.MusicBrainz, false)
res := r.get(t, 1)
if res.Status != discovery.StatusDisabled || res.Description != nil || r.src.total() != 0 {
t.Errorf("result = %+v, calls %d: nobody may be asked while MusicBrainz is off", res, r.src.total())
}
r.sw.set(providers.MusicBrainz, true)
r.sw.set(providers.Wikipedia, false)
if res = r.get(t, 1); res.Status != discovery.StatusDisabled || r.src.total() != 0 {
t.Errorf("with Wikipedia off: %+v calls %d", res, r.src.total())
}
}
func TestAMergedItemIsUnderstoodThroughTheItemItWasMergedInto(t *testing.T) {
r := newRig(true)
r.src.qid, r.src.resolved = "Q1", "Q2" // MusicBrainz knows the old id; the pages say Q2
for lang, sum := range r.src.summaries {
sum.WikibaseItem = "Q2"
r.src.summaries[lang] = sum
}
res := r.get(t, 1)
if res.Description == nil || res.Description.Language != "it" {
t.Fatalf("result = %+v, want the description of the item it was merged into", res)
}
if len(r.store.saves) != 1 || r.store.saves[0].qid != "Q2" {
t.Errorf("saves = %+v, want the resolved item stored", r.store.saves)
}
}
func TestTheTitleShownIsTheOneWikidataNames(t *testing.T) {
r := newRig(true)
r.src.pages = map[string]string{"it": "Guns N' Roses"}
s := r.src.summaries["it"]
s.Title = "Guns_N'_Roses" // the REST title is the database key, with underscores
r.src.summaries["it"] = s
if d := r.get(t, 1).Description; d == nil || d.Title != "Guns N' Roses" {
t.Errorf("description = %+v", d)
}
}
func TestAPageWithNoAddressOnWikipediaIsNotUsed(t *testing.T) {
r := newRig(true)
s := r.src.summaries["it"]
s.URL = "" // the client drops an address that is not on that Wikipedia
r.src.summaries["it"] = s
if d := r.get(t, 1).Description; d == nil || d.Language != "en" {
t.Errorf("description = %+v, want the next language: a text without its source cannot be shown", d)
}
}
func TestTheTextNeverExceedsTheLimit(t *testing.T) {
r := newRig(true)
r.src.summaries["it"] = standard("it", "Q44190", strings.Repeat("è", MaxTextLen+500))
if n := len([]rune(r.get(t, 1).Description.Text)); n != MaxTextLen {
t.Errorf("text has %d characters, want exactly %d with the mark that it was cut", n, MaxTextLen)
}
}
func TestAStoredDescriptionIsStillReportedAsDisabledWhenTheProviderIsOff(t *testing.T) {
r := newRig(true)
r.get(t, 1)
r.sw.set(providers.Wikipedia, false)
res := r.get(t, 1) // fresh, but the switch is off
if res.Status != discovery.StatusDisabled || res.Stale || res.Description == nil {
t.Errorf("result = %+v, want the description, not stale, with status disabled", res)
}
}
func TestSwitchedOffWhileItWasBeingReadNothingIsSaved(t *testing.T) {
r := newRig(true)
r.src.release = make(chan struct{})
done := make(chan Result, 1)
go func() {
res, _ := r.svc.Description(context.Background(), 1)
done <- res
}()
time.Sleep(50 * time.Millisecond)
r.sw.set(providers.Wikipedia, false) // the admin took the data back and switched it off
close(r.src.release)
res := <-done
if len(r.store.saves) != 0 {
t.Errorf("saves = %+v: what a read that started before the switch found must not be kept", r.store.saves)
}
if res.Status != discovery.StatusDisabled || res.Description != nil {
t.Errorf("result = %+v", res)
}
}