sonda/internal/providers/commons/commons_test.go
tgiachi a880b87a9e chore: move the module path to git.kmalloc.org/squid/sonda
Migrating hosting from Codeberg to git.kmalloc.org. Renames the
module path in go.mod, every internal import, and the OCI image
source label in the Dockerfile. Build and full test suite verified
green after the rename.
2026-11-15 17:30:29 +01:00

354 lines
16 KiB
Go

package commons
import (
"bytes"
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"git.kmalloc.org/squid/sonda/internal/providers"
"git.kmalloc.org/squid/sonda/internal/providers/outbound"
)
type server struct {
*httptest.Server
mu sync.Mutex
paths []string
status int
body string
}
func newServer(t *testing.T, status int, body string) *server {
t.Helper()
s := &server{status: status, body: body}
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
s.paths = append(s.paths, r.URL.RequestURI())
status, body := s.status, s.body
s.mu.Unlock()
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
t.Cleanup(s.Close)
return s
}
func (s *server) client() *Client {
return New(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}),
s.URL+"/wikidata/w/api.php", s.URL+"/commons/w/api.php")
}
func (s *server) requests() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.paths...)
}
func TestFileNameReadsTheImageOfAnItem(t *testing.T) {
body := `{"claims":{"P18":[{"rank":"normal","mainsnak":{"snaktype":"value","datatype":"commonsMedia","datavalue":{"value":"RadioheadO2211125 composite.jpg"}}}]}}`
s := newServer(t, 200, body)
got, err := s.client().FileName(context.Background(), "Q44190")
if err != nil || got != "RadioheadO2211125 composite.jpg" {
t.Fatalf("name = %q err = %v", got, err)
}
reqs := s.requests()
if len(reqs) != 1 || !strings.Contains(reqs[0], "action=wbgetclaims") || !strings.Contains(reqs[0], "entity=Q44190") ||
!strings.Contains(reqs[0], "property=P18") || !strings.Contains(reqs[0], "maxlag=5") {
t.Errorf("requests = %v", reqs)
}
}
func TestFileNameChoosesThePreferredThenTheFirstAndSkipsWhatIsNotAFile(t *testing.T) {
claim := func(rank, snak, value string) string {
return `{"rank":"` + rank + `","mainsnak":{"snaktype":"` + snak + `","datavalue":{"value":"` + value + `"}}}`
}
for name, tt := range map[string]struct{ claims, want string }{
"preferred wins": {claim("normal", "value", "A.jpg") + "," + claim("preferred", "value", "B.jpg"), "B.jpg"},
"first of the normals": {claim("normal", "value", "A.jpg") + "," + claim("normal", "value", "B.jpg"), "A.jpg"},
"deprecated is skipped": {claim("deprecated", "value", "A.jpg") + "," + claim("normal", "value", "B.jpg"), "B.jpg"},
"only deprecated": {claim("deprecated", "value", "A.jpg"), ""},
"unknown value": {claim("normal", "somevalue", "A.jpg"), ""},
"no value": {claim("normal", "novalue", "A.jpg"), ""},
"a name with a pipe": {claim("normal", "value", "A|B.jpg") + "," + claim("normal", "value", "C.jpg"), "C.jpg"},
"a name with a bracket": {claim("normal", "value", "A[x].jpg"), ""},
"a long name": {claim("normal", "value", strings.Repeat("a", 400)), ""},
} {
body := `{"claims":{"P18":[` + tt.claims + `]}}`
got, err := newServer(t, 200, body).client().FileName(context.Background(), "Q1")
if err != nil || got != tt.want {
t.Errorf("%s: name = %q err = %v, want %q", name, got, err, tt.want)
}
}
}
func TestFileNameIsEmptyWhenThereIsNoImage(t *testing.T) {
for name, body := range map[string]string{"no claims": `{"claims":{}}`, "no P18": `{"claims":{"P31":[]}}`} {
if got, err := newServer(t, 200, body).client().FileName(context.Background(), "Q1"); err != nil || got != "" {
t.Errorf("%s: %q %v", name, got, err)
}
}
}
func TestFileNameRefusesABadItemAndReadsWikidataErrors(t *testing.T) {
s := newServer(t, 200, `{"claims":{}}`)
for _, item := range []string{"", "q1", "P18", "Q1&x=y", "Q0", "Q99999999999999999999"} {
if _, err := s.client().FileName(context.Background(), item); !errors.Is(err, ErrBadRequest) {
t.Errorf("item %q: err = %v", item, err)
}
}
if len(s.requests()) != 0 {
t.Errorf("requests for a bad item: %v", s.requests())
}
for name, tt := range map[string]struct {
status int
body string
want error
}{
"no such item (HTTP 200)": {200, `{"error":{"code":"no-such-entity"}}`, ErrNotFound},
"lag": {200, `{"error":{"code":"maxlag"}}`, ErrUnavailable},
"another error": {200, `{"error":{"code":"x"}}`, ErrBadResponse},
"rate limited": {429, ``, ErrRateLimited},
"server error": {503, ``, ErrUnavailable},
"not json": {200, `<html>`, ErrBadResponse},
"forbidden": {403, `no`, ErrBadResponse},
} {
if _, err := newServer(t, tt.status, tt.body).client().FileName(context.Background(), "Q1"); !errors.Is(err, tt.want) {
t.Errorf("%s: err = %v, want %v", name, err, tt.want)
}
}
}
const infoBody = `{"continue":{},"query":{"pages":[{"pageid":1,"ns":6,"title":"File:Band.jpg","imagerepository":"local","imageinfo":[{
"thumburl":"https://thumb.wikimedia.org/wikipedia/commons/thumb/a/a1/Band.jpg/960px-Band.jpg?utm_source=commons.wikimedia.org",
"thumbwidth":960,"thumbheight":349,"width":5940,"height":2160,"mime":"image/jpeg","size":9175547,
"descriptionurl":"https://commons.wikimedia.org/wiki/File:Band.jpg",
"extmetadata":{"LicenseShortName":{"value":"CC BY 4.0"},"LicenseUrl":{"value":"https://creativecommons.org/licenses/by/4.0"},
"Artist":{"value":"<a rel=\"nofollow\" class=\"external text\" href=\"https://www.flickr.com/photos/raph_ph/\">Raph_PH</a>"},
"AttributionRequired":{"value":"true"},"Copyrighted":{"value":"True"},"UsageTerms":{"value":"Creative Commons Attribution 4.0"}}}]}]}}`
func TestInfoReadsTheLicenceTheAuthorAndTheMiniature(t *testing.T) {
s := newServer(t, 200, infoBody)
got, err := s.client().Info(context.Background(), "Band.jpg")
if err != nil {
t.Fatal(err)
}
want := File{
Name: "Band.jpg", PageURL: "https://commons.wikimedia.org/wiki/File:Band.jpg", Mime: "image/jpeg", Width: 5940, Height: 2160,
ThumbURL: "https://thumb.wikimedia.org/wikipedia/commons/thumb/a/a1/Band.jpg/960px-Band.jpg?utm_source=commons.wikimedia.org",
Author: "Raph_PH", License: "CC BY 4.0", LicenseURL: "https://creativecommons.org/licenses/by/4.0",
}
if got != want {
t.Errorf("file = %+v\nwant %+v", got, want)
}
reqs := s.requests()
if len(reqs) != 1 || !strings.Contains(reqs[0], "iiurlwidth=960") || !strings.Contains(reqs[0], "formatversion=2") ||
!strings.Contains(reqs[0], "titles=File%3ABand.jpg") || !strings.Contains(reqs[0], "iiextmetadatafilter=") {
t.Errorf("requests = %v", reqs)
}
}
func withLicense(short, url, extra string) string {
body := strings.Replace(infoBody, `"LicenseShortName":{"value":"CC BY 4.0"}`, `"LicenseShortName":{"value":"`+short+`"}`, 1)
body = strings.Replace(body, `"LicenseUrl":{"value":"https://creativecommons.org/licenses/by/4.0"}`, `"LicenseUrl":{"value":"`+url+`"}`, 1)
if extra != "" {
body = strings.Replace(body, `"AttributionRequired"`, extra+`,"AttributionRequired"`, 1)
}
return body
}
func TestOnlyFreeLicencesAreAccepted(t *testing.T) {
for short, free := range map[string]bool{
"CC BY 4.0": true, "CC BY 2.0": true, "CC BY-SA 4.0": true, "CC BY-SA 3.0": true, "CC BY-SA 2.5": true, "CC BY-SA 3.0 de": true,
"CC0": true, "CC0 1.0": true, "Public domain": true, "PD-old-100": true, "PD-US": true, "PD-self": true,
"CC BY-NC 4.0": false, "CC BY-NC-SA 3.0": false, "CC BY-ND 4.0": false, "CC BY-NC-ND 2.0": false,
"Fair use": false, "GFDL": false, "GFDL 1.2": false, "": false, "All rights reserved": false, "Attribution": false,
"CC BY": false, "Copyrighted free use": false, "Public domain but not really": false, "cc by-nd": false,
} {
_, err := newServer(t, 200, withLicense(short, "", "")).client().Info(context.Background(), "Band.jpg")
if free && err != nil {
t.Errorf("%q refused: %v", short, err)
}
if !free && !errors.Is(err, ErrNotFree) {
t.Errorf("%q: err = %v, want ErrNotFree", short, err)
}
if Free(short) != free {
t.Errorf("Free(%q) = %v, want %v", short, Free(short), free)
}
}
}
func TestANonFreeFlagWinsOverTheName(t *testing.T) {
body := withLicense("CC BY 4.0", "https://creativecommons.org/licenses/by/4.0", `"NonFree":{"value":"true"}`)
if _, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg"); !errors.Is(err, ErrNotFree) {
t.Errorf("err = %v, want ErrNotFree", err)
}
}
func TestTheLicenceAddressIsKeptOnlyOnCreativeCommons(t *testing.T) {
for url, want := range map[string]string{
"https://creativecommons.org/licenses/by/4.0": "https://creativecommons.org/licenses/by/4.0",
"https://evil.example/licenses/by/4.0": "",
"javascript:alert(1)": "",
"http://creativecommons.org/licenses/by/4.0": "",
"": "",
} {
got, err := newServer(t, 200, withLicense("CC BY 4.0", url, "")).client().Info(context.Background(), "Band.jpg")
if err != nil || got.LicenseURL != want {
t.Errorf("%q: LicenseURL = %q err = %v, want %q", url, got.LicenseURL, err, want)
}
}
// The public domain has no address in Commons: the standard mark is given.
got, _ := newServer(t, 200, withLicense("Public domain", "", "")).client().Info(context.Background(), "Band.jpg")
if got.LicenseURL != "https://creativecommons.org/publicdomain/mark/1.0/" {
t.Errorf("public domain address = %q", got.LicenseURL)
}
got, _ = newServer(t, 200, withLicense("CC0 1.0", "", "")).client().Info(context.Background(), "Band.jpg")
if got.LicenseURL != "https://creativecommons.org/publicdomain/zero/1.0/" {
t.Errorf("CC0 address = %q", got.LicenseURL)
}
}
func TestTheAuthorHasNoInvisibleOrReorderingCharacters(t *testing.T) {
// A right-to-left override or a zero-width character can make a name read as another.
body := strings.Replace(infoBody, `<a rel=\"nofollow\" class=\"external text\" href=\"https://www.flickr.com/photos/raph_ph/\">Raph_PH</a>`, `Ra\u202ephH\u200b_\u2066x`, 1)
got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg")
if err != nil || got.Author != "RaphH_x" {
t.Errorf("author = %q err = %v", got.Author, err)
}
}
func TestTheAuthorIsPlainText(t *testing.T) {
for artist, want := range map[string]string{
`<a href=\"https://x\">Raph_PH</a>`: "Raph_PH",
`Jane &amp; John <b>Doe</b>`: "Jane & John Doe",
`<script>alert(1)</script>Eve`: "alert(1)Eve",
"Line one\\nLine two": "Line one Line two",
` <span> spaced </span> `: "spaced",
`&lt;b&gt;not a tag&lt;/b&gt;`: "<b>not a tag</b>",
`<a href=\"https://x\"></a>`: "",
strings.Repeat("a", 500): strings.Repeat("a", maxAuthorLen),
} {
body := strings.Replace(infoBody, `<a rel=\"nofollow\" class=\"external text\" href=\"https://www.flickr.com/photos/raph_ph/\">Raph_PH</a>`, artist, 1)
got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg")
if err != nil || got.Author != want {
t.Errorf("%q: author = %q err = %v, want %q", artist, got.Author, err, want)
}
}
}
func TestInfoRefusesWhatCannotBeUsed(t *testing.T) {
for name, tt := range map[string]struct {
body string
want error
}{
"no such file": {`{"query":{"pages":[{"title":"File:X.jpg","missing":true}]}}`, ErrNotFound},
"not a picture": {strings.Replace(infoBody, `"mime":"image/jpeg"`, `"mime":"audio/ogg"`, 1), ErrUnusable},
"a drawing": {strings.Replace(infoBody, `"mime":"image/jpeg"`, `"mime":"image/svg+xml"`, 1), ErrUnusable},
"no thumbnail": {strings.Replace(infoBody, `"thumburl"`, `"thumbxx"`, 1), ErrUnusable},
"a thumbnail on another host": {strings.Replace(infoBody, "thumb.wikimedia.org", "evil.example", 1), ErrUnusable},
"a thumbnail with a password": {strings.Replace(infoBody, "https://thumb.wikimedia.org", "https://user:pw@thumb.wikimedia.org", 1), ErrUnusable},
"a thumbnail over http": {strings.Replace(infoBody, "https://thumb.wikimedia.org", "http://thumb.wikimedia.org", 1), ErrUnusable},
"no pages": {`{"query":{"pages":[]}}`, ErrBadResponse},
"not json": {`<html>`, ErrBadResponse},
} {
if _, err := newServer(t, 200, tt.body).client().Info(context.Background(), "Band.jpg"); !errors.Is(err, tt.want) {
t.Errorf("%s: err = %v, want %v", name, err, tt.want)
}
}
for _, bad := range []string{"", strings.Repeat("a", 400), "A\nB", "A|B", "A[B]", "A{B}"} {
if _, err := newServer(t, 200, infoBody).client().Info(context.Background(), bad); !errors.Is(err, ErrBadRequest) {
t.Errorf("name %q: err = %v", bad, err)
}
}
}
func TestDownloadOnlyFromWikimediaPictureHosts(t *testing.T) {
for url, ok := range map[string]bool{
"https://thumb.wikimedia.org/wikipedia/commons/thumb/a/a1/B.jpg/960px-B.jpg": true,
"https://upload.wikimedia.org/wikipedia/commons/a/a1/B.jpg": true,
"http://upload.wikimedia.org/wikipedia/commons/a/a1/B.jpg": false,
"https://upload.wikimedia.org.evil.example/a.jpg": false,
"https://evil.example/upload.wikimedia.org/a.jpg": false,
"https://user@upload.wikimedia.org/a.jpg": false,
"https://upload.wikimedia.org:8443/a.jpg": false,
"https://commons.wikimedia.org/w/api.php": false,
"file:///etc/passwd": false,
"": false,
} {
if got := allowedPictureURL(url); got != ok {
t.Errorf("allowedPictureURL(%q) = %v, want %v", url, got, ok)
}
}
// A refused address is never requested.
s := newServer(t, 200, "x")
if _, err := s.client().Download(context.Background(), s.URL+"/a.jpg"); !errors.Is(err, ErrBadRequest) {
t.Errorf("err = %v", err)
}
if len(s.requests()) != 0 {
t.Errorf("requests = %v", s.requests())
}
}
// What the picture host does wrong is a problem of that picture, not of the service: it must
// not be taken for an outage, and it must not count against the breaker of the API client.
func TestDownloadTreatsARedirectOrAHugePictureAsUnusable(t *testing.T) {
redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "https://evil.example/", http.StatusFound)
}))
defer redirect.Close()
huge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(bytes.Repeat([]byte{1}, 5000))
}))
defer huge.Close()
for name, srv := range map[string]*httptest.Server{"redirect": redirect, "huge": huge} {
dl := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries, MaxBody: 1000})
c := New(nil, "", "").WithDownloader(dl)
c.allowAny = true // the test servers are not on a Wikimedia host
if _, err := c.Download(context.Background(), srv.URL+"/a.jpg"); !errors.Is(err, ErrUnusable) {
t.Errorf("%s: err = %v, want ErrUnusable", name, err)
}
}
}
func TestDownloadUsesItsOwnClient(t *testing.T) {
api := newServer(t, 200, `{"claims":{}}`)
pics := newServer(t, 200, "PICTURE")
c := api.client().WithDownloader(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}))
c.allowAny = true
got, err := c.Download(context.Background(), pics.URL+"/a.jpg")
if err != nil || string(got) != "PICTURE" {
t.Fatalf("got %q err %v", got, err)
}
if len(api.requests()) != 0 || len(pics.requests()) != 1 {
t.Errorf("api saw %v, pictures saw %v: the download must go through its own client", api.requests(), pics.requests())
}
}
func TestErrorsDoNotCopyTheProvidersText(t *testing.T) {
secret := "SECRET-FROM-THE-PROVIDER"
_, err := newServer(t, 200, `{"error":{"code":"x","info":"`+secret+`"}}`).client().FileName(context.Background(), "Q1")
if err == nil || strings.Contains(err.Error(), secret) {
t.Errorf("err = %v", err)
}
_, err = newServer(t, 500, secret).client().Info(context.Background(), "A.jpg")
if err == nil || strings.Contains(err.Error(), secret) {
t.Errorf("err = %v", err)
}
}
func TestProbeSpeaksTheLanguageOfTheSettings(t *testing.T) {
if err := newServer(t, 200, `{"claims":{}}`).client().Probe(context.Background()); err != nil {
t.Errorf("ok: %v", err)
}
if err := newServer(t, 429, ``).client().Probe(context.Background()); !errors.Is(err, providers.ErrRateLimited) {
t.Errorf("429: %v", err)
}
if err := newServer(t, 503, ``).client().Probe(context.Background()); !errors.Is(err, providers.ErrUnreachable) {
t.Errorf("503: %v", err)
}
}