sonda/internal/providers/commons/commons.go
tgiachi a880b87a9e chore: move the module path to git.kmalloc.org/squid/sonda
Migrating hosting from Codeberg to git.kmalloc.org. Renames the
module path in go.mod, every internal import, and the OCI image
source label in the Dockerfile. Build and full test suite verified
green after the rename.
2026-11-15 17:30:29 +01:00

380 lines
13 KiB
Go

// Package commons is a small read-only client for the picture of an artist on
// Wikimedia Commons, reached through Wikidata.
//
// Like the Wikipedia client, it follows an identifier and never searches by name: the
// Wikidata item that MusicBrainz links to an artist names a Commons file (property
// P18), and Commons says under which licence the file is. Only a free licence is
// accepted (public domain, CC0, CC BY, CC BY-SA), because Sonda keeps a copy of the
// picture on the user's server: non-commercial, no-derivatives, fair use and unknown
// licences are refused. What leaves the server is the item, the name of the file and the
// contact in the User-Agent, which Wikimedia requires (it answers 403 without one).
//
// Errors never repeat the provider's own text.
package commons
import (
"context"
"encoding/json"
"errors"
"fmt"
"html"
"net/url"
"regexp"
"strings"
"unicode"
"unicode/utf8"
"git.kmalloc.org/squid/sonda/internal/providers"
"git.kmalloc.org/squid/sonda/internal/providers/outbound"
)
// Addresses in production.
const (
WikidataURL = "https://www.wikidata.org/w/api.php"
CommonsURL = "https://commons.wikimedia.org/w/api.php"
)
// ThumbWidth is the width of the copy asked from Commons. It is one of the sizes
// Wikimedia serves: another is rounded up to the next one.
const ThumbWidth = 960
const (
maxNameLen = 300
maxAuthorLen = 300
probeItem = "Q44190" // a public item (Radiohead), to check that the service answers
)
// Errors a call can return, to be matched with errors.Is.
var (
ErrBadRequest = errors.New("commons: not a usable item, file name or address")
ErrNotFound = errors.New("commons: no such item or file")
ErrNotFree = errors.New("commons: the picture is not under a free licence")
ErrUnusable = errors.New("commons: the picture cannot be used")
ErrRateLimited = errors.New("commons: too many requests")
ErrUnavailable = errors.New("commons: service unavailable")
ErrBadResponse = errors.New("commons: unreadable answer")
)
var itemRe = regexp.MustCompile(`^Q[1-9][0-9]{0,15}$`)
// Getter performs a GET. *outbound.Client is the one used in production.
type Getter interface {
Get(ctx context.Context, rawURL string) (*outbound.Response, error)
}
// Client talks to Wikidata and Commons. It is safe for concurrent use.
type Client struct {
http Getter
downloader Getter // for the pictures themselves; the API client when nil
wikidata string
commons string
allowAny bool // tests only: the test servers are not on a Wikimedia host
}
// New builds a Client that sends its requests through g, to wikidata and commons
// (WikidataURL and CommonsURL in production).
func New(g Getter, wikidata, commons string) *Client {
return &Client{http: g, wikidata: wikidata, commons: commons}
}
// WithDownloader makes the pictures themselves be downloaded through g instead of the
// client of the API calls. They have their own limits (a picture is far bigger than an
// answer) and their own breaker: a picture host that misbehaves must not stop the API.
func (c *Client) WithDownloader(g Getter) *Client {
c.downloader = g
return c
}
// File is a Commons picture and what has to be shown with it.
type File struct {
Name string // the file name, without the "File:"
PageURL string // the page of the file on Commons, where author and licence are
ThumbURL string // a copy 960 pixels wide, on a Wikimedia picture host
Mime string
Width int // of the original
Height int
Author string // plain text; may be empty
License string // like "CC BY-SA 4.0"
LicenseURL string // on creativecommons.org; may be empty
}
// FileName returns the name of the Commons file that Wikidata gives as the image of an
// item, or "" when there is none. A preferred statement wins, then the first normal
// one; deprecated ones and values that are not a file name are skipped.
func (c *Client) FileName(ctx context.Context, item string) (string, error) {
if !itemRe.MatchString(item) {
return "", ErrBadRequest
}
q := url.Values{}
q.Set("action", "wbgetclaims")
q.Set("entity", item)
q.Set("property", "P18")
q.Set("maxlag", "5") // Wikimedia asks automated clients to back off when its databases lag
q.Set("format", "json")
body, err := c.get(ctx, c.wikidata+"?"+q.Encode())
if err != nil {
return "", err
}
var doc struct {
Error *struct {
Code string `json:"code"`
} `json:"error"`
Claims map[string][]struct {
Rank string `json:"rank"`
Mainsnak struct {
Snaktype string `json:"snaktype"`
Datavalue struct {
Value json.RawMessage `json:"value"`
} `json:"datavalue"`
} `json:"mainsnak"`
} `json:"claims"`
}
if err := json.Unmarshal(body, &doc); err != nil {
return "", ErrBadResponse
}
if doc.Error != nil { // Wikidata answers 200 to a failed request
switch doc.Error.Code {
case "no-such-entity":
return "", ErrNotFound
case "maxlag":
return "", fmt.Errorf("%w: lagged", ErrUnavailable)
}
return "", ErrBadResponse
}
var first string
for _, cl := range doc.Claims["P18"] {
if cl.Rank == "deprecated" || cl.Mainsnak.Snaktype != "value" {
continue
}
var name string
if json.Unmarshal(cl.Mainsnak.Datavalue.Value, &name) != nil || !validFileName(name) {
continue
}
if cl.Rank == "preferred" {
return name, nil
}
if first == "" {
first = name
}
}
return first, nil
}
// Info reads the licence, the author and a copy of a Commons file. ErrNotFound means
// the file does not exist, ErrNotFree that its licence is not a free one and
// ErrUnusable that it is not a picture Sonda can use.
func (c *Client) Info(ctx context.Context, name string) (File, error) {
if !validFileName(name) {
return File{}, ErrBadRequest
}
q := url.Values{}
q.Set("action", "query")
q.Set("titles", "File:"+name)
q.Set("prop", "imageinfo")
q.Set("iiprop", "url|size|mime|extmetadata")
q.Set("iiurlwidth", fmt.Sprint(ThumbWidth))
q.Set("iiextmetadatafilter", "LicenseShortName|LicenseUrl|Artist|AttributionRequired|Copyrighted|UsageTerms|NonFree")
q.Set("format", "json")
q.Set("formatversion", "2")
body, err := c.get(ctx, c.commons+"?"+q.Encode())
if err != nil {
return File{}, err
}
var doc struct {
Query struct {
Pages []struct {
Missing bool `json:"missing"`
Imageinfo []struct {
ThumbURL string `json:"thumburl"`
Width int `json:"width"`
Height int `json:"height"`
Mime string `json:"mime"`
DescriptionURL string `json:"descriptionurl"`
Extmetadata map[string]struct {
Value json.RawMessage `json:"value"`
} `json:"extmetadata"`
} `json:"imageinfo"`
} `json:"pages"`
} `json:"query"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Query.Pages) != 1 {
return File{}, ErrBadResponse
}
page := doc.Query.Pages[0]
if page.Missing {
return File{}, ErrNotFound
}
if len(page.Imageinfo) != 1 {
return File{}, ErrBadResponse
}
ii := page.Imageinfo[0]
meta := func(key string) string {
var s string
if v, ok := ii.Extmetadata[key]; ok {
_ = json.Unmarshal(v.Value, &s)
}
return s
}
if strings.EqualFold(strings.TrimSpace(meta("NonFree")), "true") || !Free(meta("LicenseShortName")) {
return File{}, ErrNotFree
}
switch ii.Mime {
case "image/jpeg", "image/png", "image/gif":
default:
return File{}, ErrUnusable
}
if !allowedPictureURL(ii.ThumbURL) {
return File{}, ErrUnusable
}
licence := strings.TrimSpace(meta("LicenseShortName"))
return File{
Name: name, PageURL: pageURL(ii.DescriptionURL, name), ThumbURL: ii.ThumbURL, Mime: ii.Mime,
Width: ii.Width, Height: ii.Height, Author: plainText(meta("Artist")),
License: licence, LicenseURL: licenseURL(licence, meta("LicenseUrl")),
}, nil
}
// Download reads a picture from a Wikimedia picture host. Any other address is refused
// without a request.
func (c *Client) Download(ctx context.Context, pictureURL string) ([]byte, error) {
if !c.allowAny && !allowedPictureURL(pictureURL) {
return nil, ErrBadRequest
}
g := c.downloader
if g == nil {
g = c.http
}
body, err := c.getWith(ctx, g, pictureURL)
// A redirect (never followed) or a picture over the limit is a fault of that picture,
// not an outage: there is no usable picture to show.
if errors.Is(err, outbound.ErrRedirect) || errors.Is(err, outbound.ErrResponseTooLarge) {
return nil, fmt.Errorf("%w: the picture host answered in a way that cannot be used", ErrUnusable)
}
return body, err
}
// Probe asks about one public item, to find out whether the service answers. It is a
// providers.Checker, and answers with the errors the settings understand.
func (c *Client) Probe(ctx context.Context) error {
_, err := c.FileName(ctx, probeItem)
switch {
case errors.Is(err, ErrRateLimited):
return providers.ErrRateLimited
case errors.Is(err, ErrUnavailable):
return providers.ErrUnreachable
}
return err
}
var (
pdRe = regexp.MustCompile(`(?i)^(public domain|pd(-[a-z0-9]+)+)$`)
cc0Re = regexp.MustCompile(`(?i)^cc0( 1\.0)?$`)
ccRe = regexp.MustCompile(`(?i)^cc by(-sa)? [0-9](\.[0-9])?( [a-z]{2,3})?$`)
)
// Free says whether a licence short name, as Commons writes it, is one Sonda accepts:
// public domain, CC0, CC BY and CC BY-SA. A copy is kept and scaled, so no-derivatives
// and non-commercial licences, and anything unknown, are refused.
func Free(shortName string) bool {
n := strings.TrimSpace(shortName)
return pdRe.MatchString(n) || cc0Re.MatchString(n) || ccRe.MatchString(n)
}
// licenseURL returns the address of the licence: the one Commons gives if it is on
// creativecommons.org, else the standard one of the public domain and CC0, else "".
func licenseURL(short, given string) string {
if strings.HasPrefix(given, "https://creativecommons.org/") {
return given
}
switch n := strings.TrimSpace(short); {
case cc0Re.MatchString(n):
return "https://creativecommons.org/publicdomain/zero/1.0/"
case pdRe.MatchString(n):
return "https://creativecommons.org/publicdomain/mark/1.0/"
}
return ""
}
// pageURL is the page of the file, shown as the source: it must be on Commons.
func pageURL(given, name string) string {
if strings.HasPrefix(given, "https://commons.wikimedia.org/wiki/") {
return given
}
return "https://commons.wikimedia.org/wiki/File:" + url.PathEscape(strings.ReplaceAll(name, " ", "_"))
}
var tagRe = regexp.MustCompile(`<[^>]*>`)
// plainText turns the HTML Commons gives for an author into text: tags removed,
// entities decoded, control characters dropped, spaces collapsed, at most maxAuthorLen.
func plainText(s string) string {
s = html.UnescapeString(tagRe.ReplaceAllString(s, ""))
// Control characters, and the invisible ones that reorder or hide text (a right-to-left
// override, a zero-width space): they could make an author read as somebody else.
s = strings.Map(func(r rune) rune {
if (unicode.IsControl(r) && !unicode.IsSpace(r)) || unicode.Is(unicode.Cf, r) {
return -1
}
return r
}, s)
s = strings.Join(strings.Fields(s), " ")
if utf8.RuneCountInString(s) > maxAuthorLen {
s = string([]rune(s)[:maxAuthorLen])
}
return s
}
// validFileName accepts what can be a Commons file name: not empty, not long, with no
// control character and none of the characters a title cannot hold.
func validFileName(n string) bool {
if strings.TrimSpace(n) == "" || utf8.RuneCountInString(n) > maxNameLen || !utf8.ValidString(n) {
return false
}
for _, r := range n {
if unicode.IsControl(r) || strings.ContainsRune("|[]{}<>#", r) {
return false
}
}
return true
}
// allowedPictureURL says whether an address is on a Wikimedia picture host, over https,
// with no credentials and no port.
func allowedPictureURL(raw string) bool {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" {
return false
}
return u.Host == "thumb.wikimedia.org" || u.Host == "upload.wikimedia.org"
}
// get sends one request and returns the body of a successful answer.
func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) {
return c.getWith(ctx, c.http, rawURL)
}
func (c *Client) getWith(ctx context.Context, g Getter, rawURL string) ([]byte, error) {
resp, err := g.Get(ctx, rawURL)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
var limited *outbound.RateLimitedError
if errors.As(err, &limited) {
return nil, ErrRateLimited
}
return nil, fmt.Errorf("%w: %w", ErrUnavailable, err)
}
switch {
case resp.Status == 200:
return resp.Body, nil
case resp.Status == 404:
return nil, ErrNotFound
case resp.Status == 429:
return nil, ErrRateLimited
case resp.Status >= 500:
return nil, fmt.Errorf("%w: status %d", ErrUnavailable, resp.Status)
}
return nil, fmt.Errorf("%w: status %d", ErrBadResponse, resp.Status)
}