Migrating hosting from Codeberg to git.kmalloc.org. Renames the module path in go.mod, every internal import, and the OCI image source label in the Dockerfile. Build and full test suite verified green after the rename.
429 lines
12 KiB
Go
429 lines
12 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"git.kmalloc.org/squid/sonda/internal/providers"
|
|
"git.kmalloc.org/squid/sonda/internal/providers/lastfm"
|
|
)
|
|
|
|
const key = "0123456789abcdef0123456789abcdef"
|
|
|
|
var t0 = time.Unix(1_700_000_000, 0)
|
|
|
|
type saved struct {
|
|
artistID int64
|
|
provider string
|
|
at time.Time
|
|
found bool
|
|
cands []lastfm.Similar
|
|
}
|
|
|
|
// fakeStore keeps one snapshot per artist, replaced by every save.
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
artists map[int64]Artist
|
|
snaps map[int64]Snapshot
|
|
saves []saved
|
|
lastLim int
|
|
saveErr error // returned by SaveSimilar when set
|
|
}
|
|
|
|
func newStore() *fakeStore {
|
|
return &fakeStore{
|
|
artists: map[int64]Artist{1: {ID: 1, Name: "Source", MBID: "aaaaaaaa-0000-0000-0000-000000000001"}},
|
|
snaps: map[int64]Snapshot{},
|
|
}
|
|
}
|
|
|
|
func (f *fakeStore) Artist(_ context.Context, id int64) (Artist, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
a, ok := f.artists[id]
|
|
if !ok {
|
|
return Artist{}, ErrNotFound
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
func (f *fakeStore) Similar(_ context.Context, id int64, _ string, limit int) (Snapshot, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.lastLim = limit
|
|
s := f.snaps[id]
|
|
if len(s.Items) > limit {
|
|
s.Items = s.Items[:limit]
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (f *fakeStore) SaveSimilar(_ context.Context, id int64, provider string, at time.Time, found bool, cands []lastfm.Similar) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.saveErr != nil {
|
|
return f.saveErr
|
|
}
|
|
f.saves = append(f.saves, saved{id, provider, at, found, cands})
|
|
snap := Snapshot{Fetched: true, NotFound: !found, RetrievedAt: at}
|
|
for i, c := range cands {
|
|
snap.Items = append(snap.Items, Item{Artist: ArtistRef{ID: int64(100 + i), Name: c.Name}, Score: c.Match, Rank: i + 1})
|
|
}
|
|
f.snaps[id] = snap
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) put(id int64, s Snapshot) {
|
|
f.mu.Lock()
|
|
f.snaps[id] = s
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
type fakeSource struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
key string
|
|
artist lastfm.Artist
|
|
limit int
|
|
result []lastfm.Similar
|
|
err error
|
|
started chan struct{} // closed on the first call, when set
|
|
release chan struct{} // the call waits for it, when set
|
|
ctxErr error
|
|
boom any // when set, the call panics with it
|
|
}
|
|
|
|
func (s *fakeSource) SimilarArtists(ctx context.Context, k string, a lastfm.Artist, limit int) ([]lastfm.Similar, error) {
|
|
s.mu.Lock()
|
|
s.calls++
|
|
s.key, s.artist, s.limit = k, a, limit
|
|
first := s.calls == 1
|
|
started, release := s.started, s.release
|
|
s.mu.Unlock()
|
|
if started != nil && first {
|
|
close(started)
|
|
}
|
|
if release != nil {
|
|
<-release
|
|
}
|
|
if s.boom != nil {
|
|
panic(s.boom)
|
|
}
|
|
s.mu.Lock()
|
|
s.ctxErr = ctx.Err()
|
|
s.mu.Unlock()
|
|
return s.result, s.err
|
|
}
|
|
|
|
func (s *fakeSource) count() int {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.calls
|
|
}
|
|
|
|
type fakeSettings struct {
|
|
active bool
|
|
err error
|
|
}
|
|
|
|
func (f fakeSettings) Active(_ context.Context, name string) (providers.Settings, bool, error) {
|
|
return providers.Settings{Provider: name, Enabled: f.active, APIKey: key}, f.active, f.err
|
|
}
|
|
|
|
type rig struct {
|
|
svc *Service
|
|
store *fakeStore
|
|
src *fakeSource
|
|
now time.Time
|
|
}
|
|
|
|
func newRig(active bool) *rig {
|
|
r := &rig{store: newStore(), src: &fakeSource{}, now: t0}
|
|
r.src.result = []lastfm.Similar{{Name: "Alpha", Match: 1}, {Name: "Beta", MBID: "bbbbbbbb-0000-0000-0000-000000000002", Match: 0.5}}
|
|
r.svc = NewService(r.store, r.src, fakeSettings{active: active}, zap.NewNop())
|
|
r.svc.now = func() time.Time { return r.now }
|
|
return r
|
|
}
|
|
|
|
func (r *rig) similar(t *testing.T, limit int) Result {
|
|
t.Helper()
|
|
res, err := r.svc.Similar(context.Background(), 1, limit)
|
|
if err != nil {
|
|
t.Fatalf("Similar: %v", err)
|
|
}
|
|
return res
|
|
}
|
|
|
|
func TestAnUnknownArtistIsNotFoundAndNothingIsAsked(t *testing.T) {
|
|
r := newRig(true)
|
|
if _, err := r.svc.Similar(context.Background(), 999, 10); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("err = %v, want ErrNotFound", err)
|
|
}
|
|
if _, err := r.svc.Similar(context.Background(), 0, 10); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("id 0: err = %v, want ErrNotFound", err)
|
|
}
|
|
if r.src.count() != 0 {
|
|
t.Error("the provider was asked about an artist that does not exist")
|
|
}
|
|
}
|
|
|
|
func TestLimitIsCheckedAndDefaulted(t *testing.T) {
|
|
r := newRig(true)
|
|
if _, err := r.svc.Similar(context.Background(), 1, -1); !errors.Is(err, ErrInvalidQuery) {
|
|
t.Errorf("negative limit: err = %v", err)
|
|
}
|
|
for _, tt := range []struct{ in, want int }{{0, DefaultLimit}, {7, 7}, {MaxLimit, MaxLimit}, {MaxLimit + 50, MaxLimit}} {
|
|
r.similar(t, tt.in)
|
|
if r.store.lastLim != tt.want {
|
|
t.Errorf("limit %d reached the store as %d, want %d", tt.in, r.store.lastLim, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAProviderThatIsOffIsNeverAsked(t *testing.T) {
|
|
r := newRig(false)
|
|
res := r.similar(t, 10)
|
|
if res.ProviderStatus != StatusDisabled || len(res.Items) != 0 || !res.RetrievedAt.IsZero() || res.Stale {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
if r.src.count() != 0 {
|
|
t.Error("a disabled provider was contacted")
|
|
}
|
|
}
|
|
|
|
func TestFirstLookAsksTheProviderAndStoresTheAnswer(t *testing.T) {
|
|
r := newRig(true)
|
|
res := r.similar(t, 10)
|
|
if r.src.count() != 1 {
|
|
t.Fatalf("calls = %d, want 1", r.src.count())
|
|
}
|
|
if r.src.key != key || r.src.artist.Name != "Source" || r.src.artist.MBID != "aaaaaaaa-0000-0000-0000-000000000001" {
|
|
t.Errorf("the provider was asked %q about %+v", r.src.key, r.src.artist)
|
|
}
|
|
// Always the same number, whatever the page asked for, so one answer serves any page size.
|
|
if r.src.limit != FetchLimit {
|
|
t.Errorf("fetch limit = %d, want %d", r.src.limit, FetchLimit)
|
|
}
|
|
if len(r.store.saves) != 1 || !r.store.saves[0].found || r.store.saves[0].provider != Provider || !r.store.saves[0].at.Equal(t0) {
|
|
t.Fatalf("saves = %+v", r.store.saves)
|
|
}
|
|
if res.ProviderStatus != StatusOK || res.Stale || !res.RetrievedAt.Equal(t0) || len(res.Items) != 2 || res.Provider != Provider {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
if res.Items[0].Artist.Name != "Alpha" || res.Items[0].Rank != 1 || res.Items[1].Score != 0.5 {
|
|
t.Errorf("items = %+v", res.Items)
|
|
}
|
|
}
|
|
|
|
func TestAnArtistIsNeverSimilarToItself(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.result = []lastfm.Similar{
|
|
{Name: " SOURCE ", Match: 1},
|
|
{Name: "Other Spelling", MBID: "AAAAAAAA-0000-0000-0000-000000000001", Match: 0.9},
|
|
{Name: "Real One", Match: 0.8},
|
|
}
|
|
r.similar(t, 10)
|
|
got := r.store.saves[0].cands
|
|
if len(got) != 1 || got[0].Name != "Real One" {
|
|
t.Errorf("stored candidates = %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestFreshDataIsNotAskedAgain(t *testing.T) {
|
|
r := newRig(true)
|
|
r.similar(t, 10)
|
|
r.now = t0.Add(PositiveTTL - time.Second)
|
|
res := r.similar(t, 10)
|
|
if r.src.count() != 1 {
|
|
t.Errorf("calls = %d, want 1: data younger than %v is fresh", r.src.count(), PositiveTTL)
|
|
}
|
|
if res.Stale || res.ProviderStatus != StatusOK || !res.RetrievedAt.Equal(t0) {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
r.now = t0.Add(PositiveTTL)
|
|
r.similar(t, 10)
|
|
if r.src.count() != 2 {
|
|
t.Errorf("calls = %d, want 2: data of %v is old", r.src.count(), PositiveTTL)
|
|
}
|
|
}
|
|
|
|
func TestFreshDataIsServedEvenIfTheProviderIsSwitchedOff(t *testing.T) {
|
|
r := newRig(true)
|
|
r.similar(t, 10)
|
|
r.svc.settings = fakeSettings{active: false}
|
|
res := r.similar(t, 10)
|
|
if len(res.Items) != 2 || res.Stale {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
r.now = t0.Add(PositiveTTL + time.Hour)
|
|
res = r.similar(t, 10)
|
|
if !res.Stale || res.ProviderStatus != StatusDisabled || len(res.Items) != 2 || r.src.count() != 1 {
|
|
t.Errorf("old data with the provider off: %+v calls=%d", res, r.src.count())
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownArtistAtTheProviderIsRememberedForADay(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.err = lastfm.ErrNotFound
|
|
res := r.similar(t, 10)
|
|
if res.ProviderStatus != StatusOK || len(res.Items) != 0 || res.Stale {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
if len(r.store.saves) != 1 || r.store.saves[0].found {
|
|
t.Fatalf("saves = %+v, want one 'not found'", r.store.saves)
|
|
}
|
|
r.now = t0.Add(NegativeTTL - time.Second)
|
|
r.similar(t, 10)
|
|
if r.src.count() != 1 {
|
|
t.Errorf("calls = %d: a negative answer is kept for %v", r.src.count(), NegativeTTL)
|
|
}
|
|
r.now = t0.Add(NegativeTTL)
|
|
r.similar(t, 10)
|
|
if r.src.count() != 2 {
|
|
t.Errorf("calls = %d: a negative answer expires after %v", r.src.count(), NegativeTTL)
|
|
}
|
|
}
|
|
|
|
func TestOldDataIsServedWhenTheProviderFails(t *testing.T) {
|
|
r := newRig(true)
|
|
r.similar(t, 10)
|
|
r.now = t0.Add(PositiveTTL + time.Hour)
|
|
r.src.err = lastfm.ErrUnavailable
|
|
res := r.similar(t, 10)
|
|
if !res.Stale || res.ProviderStatus != StatusError || len(res.Items) != 2 || !res.RetrievedAt.Equal(t0) {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
if len(r.store.saves) != 1 {
|
|
t.Errorf("a failed call changed the stored data: %+v", r.store.saves)
|
|
}
|
|
}
|
|
|
|
func TestAFailureWithNothingStoredIsAnEmptyAnswerNotAnError(t *testing.T) {
|
|
for _, e := range []error{lastfm.ErrUnavailable, lastfm.ErrInvalidKey, lastfm.ErrRateLimited, lastfm.ErrBadResponse, errors.New("boom")} {
|
|
r := newRig(true)
|
|
r.src.err = e
|
|
res, err := r.svc.Similar(context.Background(), 1, 10)
|
|
if err != nil {
|
|
t.Fatalf("%v: err = %v", e, err)
|
|
}
|
|
if res.ProviderStatus != StatusError || len(res.Items) != 0 || res.Stale || !res.RetrievedAt.IsZero() {
|
|
t.Errorf("%v: result = %+v", e, res)
|
|
}
|
|
if len(r.store.saves) != 0 {
|
|
t.Errorf("%v: a failure was stored", e)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSettingsErrorsAreReal(t *testing.T) {
|
|
r := newRig(true)
|
|
boom := errors.New("db down")
|
|
r.svc.settings = fakeSettings{err: boom}
|
|
if _, err := r.svc.Similar(context.Background(), 1, 10); !errors.Is(err, boom) {
|
|
t.Fatalf("err = %v, want the settings error", err)
|
|
}
|
|
}
|
|
|
|
// Ten people opening the same artist at once cost one call, and one of them
|
|
// walking away does not cancel it for the others.
|
|
func TestConcurrentLooksShareOneCall(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.started = make(chan struct{})
|
|
r.src.release = make(chan struct{})
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
var wg sync.WaitGroup
|
|
var failed atomic.Int32
|
|
results := make(chan Result, 10)
|
|
run := func(c context.Context) {
|
|
defer wg.Done()
|
|
res, err := r.svc.Similar(c, 1, 10)
|
|
if err != nil {
|
|
failed.Add(1)
|
|
return
|
|
}
|
|
results <- res
|
|
}
|
|
wg.Add(1)
|
|
go run(ctx)
|
|
<-r.src.started
|
|
for i := 0; i < 9; i++ {
|
|
wg.Add(1)
|
|
go run(context.Background())
|
|
}
|
|
time.Sleep(300 * time.Millisecond) // let the others join the flight
|
|
cancel()
|
|
time.Sleep(100 * time.Millisecond)
|
|
close(r.src.release)
|
|
wg.Wait()
|
|
close(results)
|
|
|
|
if r.src.count() != 1 {
|
|
t.Errorf("calls = %d, want 1", r.src.count())
|
|
}
|
|
if r.src.ctxErr != nil {
|
|
t.Errorf("the shared call was cancelled with its first caller: %v", r.src.ctxErr)
|
|
}
|
|
if failed.Load() != 1 {
|
|
t.Errorf("failed = %d, want only the caller that gave up", failed.Load())
|
|
}
|
|
n := 0
|
|
for res := range results {
|
|
n++
|
|
if len(res.Items) != 2 {
|
|
t.Errorf("result = %+v", res)
|
|
}
|
|
}
|
|
if n != 9 {
|
|
t.Errorf("%d callers got an answer, want 9", n)
|
|
}
|
|
}
|
|
|
|
// "No such artist" and "an empty list" both mean there is nothing to show, and
|
|
// both are remembered for a short time only.
|
|
func TestAnEmptyListIsRememberedLikeAnUnknownArtist(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.result = nil
|
|
res := r.similar(t, 10)
|
|
if len(res.Items) != 0 || len(r.store.saves) != 1 || r.store.saves[0].found {
|
|
t.Fatalf("result = %+v saves = %+v", res, r.store.saves)
|
|
}
|
|
}
|
|
|
|
// A panic in the shared call would be re-raised in a goroutine nobody can
|
|
// recover, taking the server down: it must come back as a failure instead.
|
|
func TestAPanicInTheProviderCallIsAFailureNotACrash(t *testing.T) {
|
|
r := newRig(true)
|
|
r.src.boom = "the client has a bug"
|
|
res, err := r.svc.Similar(context.Background(), 1, 10)
|
|
if err != nil || res.ProviderStatus != StatusError || len(res.Items) != 0 {
|
|
t.Fatalf("result = %+v err = %v", res, err)
|
|
}
|
|
r.src.boom = nil
|
|
if res = r.similar(t, 10); res.ProviderStatus != StatusOK {
|
|
t.Errorf("after the panic the service did not recover: %+v", res)
|
|
}
|
|
}
|
|
|
|
// Only FetchLimit items are ever stored, so no page may promise more.
|
|
func TestNoPageCanAskForMoreThanIsStored(t *testing.T) {
|
|
if MaxLimit > FetchLimit {
|
|
t.Errorf("MaxLimit = %d is above FetchLimit = %d", MaxLimit, FetchLimit)
|
|
}
|
|
}
|
|
|
|
// An artist removed while its list was being read is not there any more: that is
|
|
// "not found", not a provider failure.
|
|
func TestAnArtistRemovedWhileItsListWasReadIsNotFound(t *testing.T) {
|
|
r := newRig(true)
|
|
r.store.saveErr = ErrNotFound
|
|
if _, err := r.svc.Similar(context.Background(), 1, 10); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("err = %v, want ErrNotFound", err)
|
|
}
|
|
}
|