- A page that cannot be used (no text, an address off Wikipedia, a redirect that is never followed) is now an answer: the chain goes on to the next language and the outcome is stored. Before, it was a failure asked about again at every view, and redirects counted against the provider's breaker. - The chain also needs the MusicBrainz provider to be on, since the id goes to it with its own contact: nothing is sent to MusicBrainz that the admin did not allow. The disclosure in the provider settings, the spec and the README say so. - A Wikidata item that was merged into another is understood through the target: the entity's own id is returned, compared with the page's item and stored. Verified on the real API. - The title shown is the one Wikidata names (with spaces), control characters are dropped from the text (PostgreSQL refuses NUL), the text is cut to at most 4000 characters with the mark included, and a stored description is reported disabled when the provider is off. - A read that started before the provider was switched off does not save its result. - Tests: the real clients with a redirect and a page without text, merged items, MusicBrainz off, switching off during a read.
111 lines
5.1 KiB
Go
111 lines
5.1 KiB
Go
package about
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"codeberg.org/squidd/sonda/internal/discovery"
|
|
"codeberg.org/squidd/sonda/internal/providers"
|
|
"codeberg.org/squidd/sonda/internal/providers/musicbrainz"
|
|
"codeberg.org/squidd/sonda/internal/providers/outbound"
|
|
"codeberg.org/squidd/sonda/internal/providers/wikipedia"
|
|
)
|
|
|
|
// fakeWeb answers for MusicBrainz, Wikidata and Wikipedia on one server.
|
|
func fakeWeb(t *testing.T, wikidataLink string) WikipediaSource {
|
|
t.Helper()
|
|
return fakeWebWith(t, wikidataLink, "ok")
|
|
}
|
|
|
|
// fakeWebWith can make the Italian page a redirect ("redirect") or a page with no text ("noextract").
|
|
func fakeWebWith(t *testing.T, wikidataLink, italian string) WikipediaSource {
|
|
t.Helper()
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"+mbid):
|
|
rels := `[]`
|
|
if wikidataLink != "" {
|
|
rels = `[{"type":"wikidata","url":{"resource":"` + wikidataLink + `"}}]`
|
|
}
|
|
_, _ = w.Write([]byte(`{"id":"` + mbid + `","relations":` + rels + `}`))
|
|
case strings.HasPrefix(r.URL.Path, "/ws/2/artist/"):
|
|
http.NotFound(w, r)
|
|
case r.URL.Path == "/w/api.php":
|
|
_, _ = w.Write([]byte(`{"entities":{"Q44190":{"id":"Q44190","sitelinks":{"itwiki":{"title":"Radiohead"},"enwiki":{"title":"Radiohead"}}}}}`))
|
|
case r.URL.Path == "/it/summary/Radiohead" && italian == "redirect":
|
|
http.Redirect(w, r, "https://evil.example/", http.StatusFound)
|
|
case r.URL.Path == "/it/summary/Radiohead" && italian == "noextract":
|
|
_, _ = w.Write([]byte(`{"type":"no-extract","title":"Radiohead","lang":"it","extract":"","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"}}}`))
|
|
case r.URL.Path == "/en/summary/Radiohead":
|
|
_, _ = w.Write([]byte(`{"type":"standard","title":"Radiohead","lang":"en","extract":"Text.","description":"band","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://en.wikipedia.org/wiki/Radiohead"}}}`))
|
|
case r.URL.Path == "/it/summary/Radiohead":
|
|
_, _ = w.Write([]byte(`{"type":"standard","title":"Radiohead","lang":"it","extract":"Testo.","description":"gruppo","wikibase_item":"Q44190","content_urls":{"desktop":{"page":"https://it.wikipedia.org/wiki/Radiohead"}}}`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
client := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries})
|
|
return WikipediaSource{
|
|
MusicBrainz: musicbrainz.New(client, srv.URL+"/ws/2/"),
|
|
Wikipedia: wikipedia.New(client, srv.URL+"/w/api.php", srv.URL+"/%s/summary/"),
|
|
}
|
|
}
|
|
|
|
func TestTheRealClientsFollowTheChain(t *testing.T) {
|
|
src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190")
|
|
ctx := context.Background()
|
|
item, err := src.WikidataID(ctx, mbid)
|
|
if err != nil || item != "Q44190" {
|
|
t.Fatalf("item = %q err = %v", item, err)
|
|
}
|
|
resolved, pages, err := src.Pages(ctx, item, []string{"it", "en"})
|
|
if err != nil || resolved != "Q44190" || pages["it"] != "Radiohead" || pages["en"] != "Radiohead" {
|
|
t.Fatalf("item %q pages = %v err = %v", resolved, pages, err)
|
|
}
|
|
sum, err := src.Summary(ctx, "it", "Radiohead")
|
|
if err != nil || sum.WikibaseItem != "Q44190" || sum.Extract != "Testo." || sum.Type != "standard" || sum.Description != "gruppo" {
|
|
t.Fatalf("summary = %+v err = %v", sum, err)
|
|
}
|
|
}
|
|
|
|
func TestWhatThereIsNothingToFindIsNotAFailure(t *testing.T) {
|
|
ctx := context.Background()
|
|
if item, err := fakeWeb(t, "").WikidataID(ctx, mbid); err != nil || item != "" {
|
|
t.Errorf("no link: %q %v", item, err)
|
|
}
|
|
src := fakeWeb(t, "https://www.wikidata.org/wiki/Q44190")
|
|
if _, err := src.WikidataID(ctx, "22222222-2222-4333-8444-555555555555"); !errors.Is(err, ErrSourceNotFound) {
|
|
t.Errorf("unknown artist: %v", err)
|
|
}
|
|
if _, err := src.WikidataID(ctx, "not an id"); !errors.Is(err, ErrSourceNotFound) {
|
|
t.Errorf("a bad id: %v", err)
|
|
}
|
|
if _, err := src.Summary(ctx, "it", "No such page"); !errors.Is(err, ErrSourceNotFound) {
|
|
t.Errorf("no page: %v", err)
|
|
}
|
|
}
|
|
|
|
// With the real clients, a page that cannot be used (a redirect that is never followed, a
|
|
// page with no text) is an answer: the chain goes on to the next language, and the outcome
|
|
// is stored, so the same page is not asked about again at every view.
|
|
func TestAPageThatCannotBeUsedFallsThroughToTheNextLanguage(t *testing.T) {
|
|
for _, italian := range []string{"redirect", "noextract"} {
|
|
store := newStore()
|
|
svc := NewService(store, fakeWebWith(t, "https://www.wikidata.org/wiki/Q44190", italian),
|
|
&switches{m: map[string]bool{providers.Wikipedia: true, providers.MusicBrainz: true}}, []string{"it", "en"}, zap.NewNop())
|
|
res, err := svc.Description(context.Background(), 1)
|
|
if err != nil || res.Status != discovery.StatusOK || res.Description == nil || res.Description.Language != "en" {
|
|
t.Errorf("Italian page %s: result = %+v err = %v, want the English description", italian, res, err)
|
|
}
|
|
if len(store.saves) != 1 {
|
|
t.Errorf("Italian page %s: %d saves, want the outcome stored once", italian, len(store.saves))
|
|
}
|
|
}
|
|
}
|