fix(images): outages are 502, purges are not undone, bad pictures are not outages

- A timeout inside the provider (the outbound client's own deadline, which wraps
  context.DeadlineExceeded) was mistaken for the caller leaving and came out as a 500, so
  an older stored picture was not served. The decision now uses the caller's own context,
  and a build that runs out of its two minutes is an outage (502) too.
- A purge of the Commons data could be undone by a read that was already running: the row
  was deleted, so the generation guard started from zero, and the files were written after
  the files were removed. Pictures are now kept only under a purge counter; BeginArtistPurge
  is called before the purge and a read that started earlier starts again, or is refused if
  the provider is off by then.
- A redirect or an oversized picture from the picture host is a fault of that picture (no
  picture), not an outage, and the pictures are downloaded through a client of their own so
  that one bad file cannot open the breaker of the API calls.
- The author drops the invisible characters that reorder or hide text. Artist pictures are
  decoded only up to 4 million pixels.
- Tests: an outage that ends in a deadline, a build that times out, a purge during a read,
  a redirect and a huge picture from the host, a separate download client.
This commit is contained in:
tgiachi 2026-11-11 22:12:49 +01:00
commit 9e85d4d1ea
10 changed files with 308 additions and 58 deletions

View file

@ -59,6 +59,7 @@ const (
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
const (
commonsMinInterval = 300 * time.Millisecond
commonsAPIMaxBody = 1 << 20
commonsMaxBody = 8 << 20
)
@ -159,11 +160,17 @@ func run() error {
return wikipediaClient.Probe(ctx)
})
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
// The pictures are downloaded through a client of their own: they are far bigger than an
// answer, and a picture host that misbehaves must not open the breaker of the API.
commonsClient := commons.New(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
MinInterval: commonsMinInterval,
MaxBody: commonsAPIMaxBody,
}), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
MinInterval: commonsMinInterval,
MaxBody: commonsMaxBody,
}), commons.WikidataURL, commons.CommonsURL)
}))
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
return commonsClient.Probe(ctx)
})
@ -177,6 +184,9 @@ func run() error {
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
})
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
if name == providers.Commons {
pictures.BeginArtistPurge() // a picture being read right now must not be kept
}
out, err := discoveryRepo.Purge(ctx, name)
if err != nil || name != providers.Commons {
return out, err

View file

@ -1088,7 +1088,8 @@ paths:
Whose the picture is and under which licence, to be shown next to it (the licences ask for
it). The author is plain text and may be empty; `license_url` may be empty when the licence
has no address; `page_url` is the page of the file on Commons, where everything is.
It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a picture.
It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a
picture: the first request for an artist can take as long as the picture does (several seconds).
security: [{bearerAuth: []}]
responses:
'200':

View file

@ -70,13 +70,33 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*
return s.open(kindArtist, artistID, size, built)
}
// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy
// asked from Commons is 960 pixels wide, so this leaves room for a very tall one.
const MaxArtistPixels = 4_000_000
// BeginArtistPurge must be called before the pictures of artists are taken back (their
// records forgotten and their files removed). A read that started before it is not kept:
// it starts again, and is refused if the provider is by then off.
func (s *Service) BeginArtistPurge() {
s.purgeMu.Lock()
s.purges++
s.purgeMu.Unlock()
}
func (s *Service) purgeCount() uint64 {
s.purgeMu.RLock()
defer s.purgeMu.RUnlock()
return s.purges
}
// buildArtist asks the source, stores the picture and records the outcome. If the
// record was cleared while it worked, what it found may be out of date: it starts again.
// record was cleared or the pictures were taken back while it worked, what it found may
// be out of date: it starts again.
func (s *Service) buildArtist(artistID int64) (Record, error) {
if s.artists == nil {
return Record{}, ErrProviderDisabled
}
ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
defer cancel()
var rec Record
for attempt := 0; attempt < 3; attempt++ {
@ -84,14 +104,27 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
if err != nil {
return Record{}, fmt.Errorf("read picture record: %w", err)
}
rec, err = s.searchArtist(ctx, artistID)
epoch := s.purgeCount()
var pic *Processed
rec, pic, err = s.searchArtist(ctx, artistID)
if errors.Is(err, errPurged) {
continue
}
if err != nil {
// Nobody calls this context: when it ends it is the build that ran out of time,
// which is an outage of the provider and not something to hide as an internal error.
if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) {
err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
}
return Record{}, err
}
rec.Gen = cur.Gen
saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
saved, err := s.saveArtist(ctx, artistID, rec, pic, epoch)
if errors.Is(err, errPurged) {
continue
}
if err != nil {
return Record{}, fmt.Errorf("save picture record: %w", err)
return Record{}, err
}
if saved {
return rec, nil
@ -100,27 +133,30 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
return rec, nil
}
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, error) {
// errPurged says the pictures were taken back since the build began.
var errPurged = errors.New("the pictures were taken back while this was being built")
// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
// The files are written by saveArtist, together with the record, so that a purge cannot
// fall between them.
func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) {
select {
case s.artistSl <- struct{}{}:
defer func() { <-s.artistSl }()
case <-ctx.Done():
return Record{}, ctx.Err()
return Record{}, nil, ctx.Err()
}
none := Record{Status: StatusNone, CheckedAt: s.now()}
found, err := s.artists.ArtistImage(ctx, artistID)
switch {
case errors.Is(err, ErrNoImage):
return none, nil
return none, nil, nil
case err != nil:
return Record{}, err
return Record{}, nil, err
}
p, err := Process(found.Data)
p, err := ProcessLimited(found.Data, MaxArtistPixels)
if err != nil {
return none, nil // what the provider sent is not a picture: there is none to show
}
if err := s.write(kindArtist, artistID, p); err != nil {
return Record{}, err
return none, nil, nil // what the provider sent is not a usable picture: there is none to show
}
// The credit is part of the identity: the same picture under another author or licence
// must not be taken for the same one by a client that kept the old credit.
@ -130,7 +166,27 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, err
return Record{
Status: StatusOK, Source: SourceCommons, Credit: found.Credit, ETag: fmt.Sprintf("%x", h.Sum(nil)[:8]),
Width: p.Width, Height: p.Height, CheckedAt: none.CheckedAt,
}, nil
}, &p, nil
}
// saveArtist keeps what was found: the files and the record, unless the pictures were
// taken back since the build began (errPurged) or a change moved the record's generation.
func (s *Service) saveArtist(ctx context.Context, artistID int64, rec Record, pic *Processed, epoch uint64) (bool, error) {
s.purgeMu.RLock()
defer s.purgeMu.RUnlock()
if s.purges != epoch {
return false, errPurged
}
if pic != nil {
if err := s.write(kindArtist, artistID, *pic); err != nil {
return false, err
}
}
saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
if err != nil {
return false, fmt.Errorf("save picture record: %w", err)
}
return saved, nil
}
// RemoveArtistFiles deletes the stored pictures of artists, and only those, and says how

View file

@ -21,15 +21,24 @@ type fakeArtists struct {
err error
calls int32
gate chan struct{}
gate2 bool // the gate is never opened for this call: it waits for the context instead
}
func (f *fakeArtists) ArtistImage(_ context.Context, id int64) (Found, error) {
func (f *fakeArtists) ArtistImage(ctx context.Context, id int64) (Found, error) {
atomic.AddInt32(&f.calls, 1)
f.mu.Lock()
gate, found, err := f.gate, f.found, f.err
f.mu.Unlock()
if gate != nil {
<-gate
if f.gate2 {
select {
case <-gate:
case <-ctx.Done():
return Found{}, ctx.Err()
}
} else {
<-gate
}
}
return found, err
}
@ -205,18 +214,77 @@ func TestManyAsksForAnArtistPictureAskTheProviderOnce(t *testing.T) {
}
}
func TestAChangeDuringTheReadOfAnArtistPictureIsNotUndoneByIt(t *testing.T) {
func TestAPurgeDuringTheReadIsNotUndoneByIt(t *testing.T) {
r := newArtistRig(t)
r.src.gate = make(chan struct{})
done := make(chan error, 1)
go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
time.Sleep(50 * time.Millisecond) // the read has the picture and is about to keep it
r.svc.BeginArtistPurge() // the admin takes the provider's data back...
r.src.set(Found{}, ErrProviderDisabled) // ...and switches it off
close(r.src.gate)
if err := <-done; !errors.Is(err, ErrProviderDisabled) {
t.Fatalf("err = %v, want the read to be refused once the provider is off", err)
}
if len(r.store.recs) != 0 {
t.Errorf("records = %v: what the read found before the purge came back", r.store.recs)
}
if entries, _ := os.ReadDir(filepath.Join(r.data, "images", "artist")); len(entries) != 0 {
t.Errorf("%d files left by a read that started before the purge", len(entries))
}
}
func TestAReadThatStartedBeforeAPurgeStartsAgainIfTheProviderIsStillOn(t *testing.T) {
r := newArtistRig(t)
r.src.gate = make(chan struct{})
done := make(chan error, 1)
go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
time.Sleep(50 * time.Millisecond)
r.store.clear("artist", 7) // for example the purge of the provider
r.svc.BeginArtistPurge()
close(r.src.gate)
<-done
// What the read that started before the change found was not kept as the answer.
if rec := r.store.recs["artist/7"]; rec.Status == StatusOK && rec.Gen == 0 {
t.Errorf("record = %+v: an out of date build was saved over the change", rec)
if err := <-done; err != nil {
t.Fatalf("err = %v", err)
}
if n := atomic.LoadInt32(&r.src.calls); n != 2 {
t.Errorf("the provider was asked %d times, want 2: the first answer was from before the purge", n)
}
if rec := r.store.recs["artist/7"]; rec.Status != StatusOK {
t.Errorf("record = %+v", rec)
}
}
func TestABuildThatTakesTooLongIsAnOutageNotAnInternalError(t *testing.T) {
r := newArtistRig(t)
_, old := r.get(t, 7, Large) // a picture is stored
r.now = r.now.Add(ArtistTTL + time.Hour)
r.svc.buildTimeout = 40 * time.Millisecond
r.src.gate = make(chan struct{}) // never opened during the build
r.src.gate2 = true
c, again := r.get(t, 7, Large)
if !bytes.Equal(old, again) || c.Credit != credit {
t.Error("the old picture was not served when the build timed out")
}
// With nothing stored the caller is told it is an outage (502), not a failure of ours (500).
r2 := newArtistRig(t)
r2.svc.buildTimeout = 40 * time.Millisecond
r2.src.gate = make(chan struct{})
r2.src.gate2 = true
if _, err := r2.svc.ArtistImage(context.Background(), 8, Large); !errors.Is(err, ErrSourceUnavailable) {
t.Errorf("err = %v, want ErrSourceUnavailable", err)
}
close(r.src.gate)
close(r2.src.gate)
}
func TestAHugePictureOfAnArtistIsRefused(t *testing.T) {
r := newArtistRig(t)
r.src.set(Found{Data: jpegBytes(t, 2500, 2500, color.White), Credit: credit}, nil) // 6 million pixels
if _, err := r.svc.ArtistImage(context.Background(), 7, Large); !errors.Is(err, ErrNoImage) {
t.Errorf("err = %v, want ErrNoImage: a copy of 960 pixels is never that big", err)
}
// The same picture is fine as the cover of an album (the limit there is larger).
if _, err := Process(r.src.found.Data); err != nil {
t.Errorf("Process: %v", err)
}
}

View file

@ -58,11 +58,13 @@ func (c *CommonsSource) active(ctx context.Context) (bool, error) {
return true, nil
}
// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. A request
// that was cancelled is not a failure of the provider and is returned as it is.
func failed(err error) error {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return err
// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. When the
// caller's own context has ended (it left, or ran out of time) that is returned instead:
// it is not the provider's fault. A deadline buried in the error is not a sign of that, since
// the outbound client's own timeout produces one when the provider is merely slow.
func failed(ctx context.Context, err error) error {
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
return fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
}
@ -90,7 +92,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
case errors.Is(err, musicbrainz.ErrNotFound), errors.Is(err, musicbrainz.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
return Found{}, failed(err)
return Found{}, failed(ctx, err)
case item == "":
return Found{}, ErrNoImage
}
@ -99,7 +101,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
case errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
return Found{}, failed(err)
return Found{}, failed(ctx, err)
case name == "":
return Found{}, ErrNoImage
}
@ -109,7 +111,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
errors.Is(err, commons.ErrUnusable), errors.Is(err, commons.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
return Found{}, failed(err)
return Found{}, failed(ctx, err)
}
// The admin may have switched the provider off while the chain was being followed:
// do not download, and do not hand over, what they no longer allow.
@ -120,10 +122,10 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
}
data, err := c.Commons.Download(ctx, file.ThumbURL)
switch {
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound):
case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable):
return Found{}, ErrNoImage
case err != nil:
return Found{}, failed(err)
return Found{}, failed(ctx, err)
}
if on, err := c.active(ctx); err != nil {
return Found{}, err

View file

@ -3,6 +3,7 @@ package images
import (
"context"
"errors"
"fmt"
"sync"
"testing"
@ -194,10 +195,38 @@ func TestSwitchedOffWhileItWasWorkingNothingIsReturned(t *testing.T) {
}
}
func TestACancelledRequestIsNotAFailureOfTheProvider(t *testing.T) {
func TestAnOutageThatEndsInADeadlineIsStillAnOutage(t *testing.T) {
// The outbound client's own timeout wraps context.DeadlineExceeded inside ErrUnavailable. It
// is a failure of the provider (so an older picture can be served), not a cancellation.
slow := fmt.Errorf("%w: %w", commons.ErrUnavailable, context.DeadlineExceeded)
for name, tt := range map[string]func(*chain){
"musicbrainz": func(c *chain) { c.mb.err = fmt.Errorf("%w: %w", musicbrainz.ErrUnavailable, context.DeadlineExceeded) },
"wikidata": func(c *chain) { c.cm.nameErr = slow },
"commons": func(c *chain) { c.cm.infoErr = slow },
"the download": func(c *chain) { c.cm.loadErr = slow },
} {
c := newChain()
tt(c)
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrSourceUnavailable) {
t.Errorf("%s: err = %v, want ErrSourceUnavailable", name, err)
}
}
}
func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) {
c := newChain()
ctx, cancel := context.WithCancel(context.Background())
c.mb.hook = cancel // the caller leaves while MusicBrainz is being asked
c.mb.err = context.Canceled
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, context.Canceled) {
if _, err := c.src.ArtistImage(ctx, 1); !errors.Is(err, context.Canceled) || errors.Is(err, ErrSourceUnavailable) {
t.Errorf("err = %v, want the cancellation itself", err)
}
}
func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) {
c := newChain()
c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit
if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) {
t.Errorf("err = %v, want ErrNoImage", err)
}
}

View file

@ -45,7 +45,11 @@ type Processed struct {
// Process decodes a GIF, JPEG or PNG and returns it scaled down to fit LargeEdge and
// SmallEdge (never enlarged), on a white background where it was transparent. The
// same input always gives the same bytes.
func Process(data []byte) (Processed, error) {
func Process(data []byte) (Processed, error) { return ProcessLimited(data, MaxPixels) }
// ProcessLimited is Process with a smaller limit on the number of pixels, for pictures that
// are known to be small (a 960 pixel copy is never millions of pixels high).
func ProcessLimited(data []byte, maxPixels int64) (Processed, error) {
if len(data) == 0 || len(data) > MaxSourceBytes {
return Processed{}, fmt.Errorf("%w: %d bytes", ErrUnusable, len(data))
}
@ -53,7 +57,7 @@ func Process(data []byte) (Processed, error) {
if err != nil {
return Processed{}, fmt.Errorf("%w: %v", ErrUnusable, err)
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > MaxPixels { // int64: a 32-bit int would overflow
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > maxPixels { // int64: a 32-bit int would overflow
return Processed{}, fmt.Errorf("%w: %dx%d pixels", ErrUnusable, cfg.Width, cfg.Height)
}
src, _, err := image.Decode(bytes.NewReader(data))

View file

@ -157,22 +157,31 @@ type Cover struct {
// Service builds covers from the user's own files, on the first request, and keeps
// them in the data folder.
type Service struct {
dir string
store Store
src Source
embedded func(path string) ([]byte, error)
now func() time.Time
flight singleflight.Group
slots chan struct{} // one per cover build allowed at the same time
artistSl chan struct{} // the same for artist pictures, which wait for the network
artists ArtistSource
tidied sync.Map // the folders whose old temporary files were already removed
dir string
store Store
src Source
embedded func(path string) ([]byte, error)
now func() time.Time
flight singleflight.Group
slots chan struct{} // one per cover build allowed at the same time
artistSl chan struct{} // the same for artist pictures, which wait for the network
artists ArtistSource
buildTimeout time.Duration // how long one build may take
// purges counts the times the provider's pictures were taken back (BeginArtistPurge). A
// build that started before one must not keep what it found: its files and its record
// would bring back what the admin just removed. Keeping a result takes the read side, so
// a purge waits for the results being kept and the next ones are refused.
purgeMu sync.RWMutex
purges uint64
tidied sync.Map // the folders whose old temporary files were already removed
}
// NewService builds a Service that keeps its files under dir.
func NewService(dir string, store Store, src Source) *Service {
return &Service{dir: dir, store: store, src: src, embedded: taglib.ReadImage, now: time.Now,
slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds)}
slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds),
buildTimeout: buildTimeout}
}
// SetArtistSource tells where pictures of artists come from. Without one, they are
@ -255,7 +264,7 @@ func (s *Service) open(kind string, id int64, size Size, rec Record) (*Cover, er
// build looks for a picture, stores it and records the outcome. If a file of the
// album changed while it was working, what it found may be out of date: it starts again.
func (s *Service) build(albumID int64) (Record, error) {
ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
defer cancel()
var rec Record
for attempt := 0; attempt < 3; attempt++ {

View file

@ -64,9 +64,11 @@ type Getter interface {
// Client talks to Wikidata and Commons. It is safe for concurrent use.
type Client struct {
http Getter
wikidata string
commons string
http Getter
downloader Getter // for the pictures themselves; the API client when nil
wikidata string
commons string
allowAny bool // tests only: the test servers are not on a Wikimedia host
}
// New builds a Client that sends its requests through g, to wikidata and commons
@ -75,6 +77,14 @@ func New(g Getter, wikidata, commons string) *Client {
return &Client{http: g, wikidata: wikidata, commons: commons}
}
// WithDownloader makes the pictures themselves be downloaded through g instead of the
// client of the API calls. They have their own limits (a picture is far bigger than an
// answer) and their own breaker: a picture host that misbehaves must not stop the API.
func (c *Client) WithDownloader(g Getter) *Client {
c.downloader = g
return c
}
// File is a Commons picture and what has to be shown with it.
type File struct {
Name string // the file name, without the "File:"
@ -228,10 +238,20 @@ func (c *Client) Info(ctx context.Context, name string) (File, error) {
// Download reads a picture from a Wikimedia picture host. Any other address is refused
// without a request.
func (c *Client) Download(ctx context.Context, pictureURL string) ([]byte, error) {
if !allowedPictureURL(pictureURL) {
if !c.allowAny && !allowedPictureURL(pictureURL) {
return nil, ErrBadRequest
}
return c.get(ctx, pictureURL)
g := c.downloader
if g == nil {
g = c.http
}
body, err := c.getWith(ctx, g, pictureURL)
// A redirect (never followed) or a picture over the limit is a fault of that picture,
// not an outage: there is no usable picture to show.
if errors.Is(err, outbound.ErrRedirect) || errors.Is(err, outbound.ErrResponseTooLarge) {
return nil, fmt.Errorf("%w: the picture host answered in a way that cannot be used", ErrUnusable)
}
return body, err
}
// Probe asks about one public item, to find out whether the service answers. It is a
@ -290,8 +310,10 @@ var tagRe = regexp.MustCompile(`<[^>]*>`)
// entities decoded, control characters dropped, spaces collapsed, at most maxAuthorLen.
func plainText(s string) string {
s = html.UnescapeString(tagRe.ReplaceAllString(s, ""))
// Control characters, and the invisible ones that reorder or hide text (a right-to-left
// override, a zero-width space): they could make an author read as somebody else.
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) && !unicode.IsSpace(r) {
if (unicode.IsControl(r) && !unicode.IsSpace(r)) || unicode.Is(unicode.Cf, r) {
return -1
}
return r
@ -329,7 +351,11 @@ func allowedPictureURL(raw string) bool {
// get sends one request and returns the body of a successful answer.
func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) {
resp, err := c.http.Get(ctx, rawURL)
return c.getWith(ctx, c.http, rawURL)
}
func (c *Client) getWith(ctx context.Context, g Getter, rawURL string) ([]byte, error) {
resp, err := g.Get(ctx, rawURL)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()

View file

@ -1,6 +1,7 @@
package commons
import (
"bytes"
"context"
"errors"
"net/http"
@ -212,6 +213,15 @@ func TestTheLicenceAddressIsKeptOnlyOnCreativeCommons(t *testing.T) {
}
}
func TestTheAuthorHasNoInvisibleOrReorderingCharacters(t *testing.T) {
// A right-to-left override or a zero-width character can make a name read as another.
body := strings.Replace(infoBody, `<a rel=\"nofollow\" class=\"external text\" href=\"https://www.flickr.com/photos/raph_ph/\">Raph_PH</a>`, `Ra\u202ephH\u200b_\u2066x`, 1)
got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg")
if err != nil || got.Author != "RaphH_x" {
t.Errorf("author = %q err = %v", got.Author, err)
}
}
func TestTheAuthorIsPlainText(t *testing.T) {
for artist, want := range map[string]string{
`<a href=\"https://x\">Raph_PH</a>`: "Raph_PH",
@ -284,6 +294,41 @@ func TestDownloadOnlyFromWikimediaPictureHosts(t *testing.T) {
}
}
// What the picture host does wrong is a problem of that picture, not of the service: it must
// not be taken for an outage, and it must not count against the breaker of the API client.
func TestDownloadTreatsARedirectOrAHugePictureAsUnusable(t *testing.T) {
redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "https://evil.example/", http.StatusFound)
}))
defer redirect.Close()
huge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(bytes.Repeat([]byte{1}, 5000))
}))
defer huge.Close()
for name, srv := range map[string]*httptest.Server{"redirect": redirect, "huge": huge} {
dl := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries, MaxBody: 1000})
c := New(nil, "", "").WithDownloader(dl)
c.allowAny = true // the test servers are not on a Wikimedia host
if _, err := c.Download(context.Background(), srv.URL+"/a.jpg"); !errors.Is(err, ErrUnusable) {
t.Errorf("%s: err = %v, want ErrUnusable", name, err)
}
}
}
func TestDownloadUsesItsOwnClient(t *testing.T) {
api := newServer(t, 200, `{"claims":{}}`)
pics := newServer(t, 200, "PICTURE")
c := api.client().WithDownloader(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}))
c.allowAny = true
got, err := c.Download(context.Background(), pics.URL+"/a.jpg")
if err != nil || string(got) != "PICTURE" {
t.Fatalf("got %q err %v", got, err)
}
if len(api.requests()) != 0 || len(pics.requests()) != 1 {
t.Errorf("api saw %v, pictures saw %v: the download must go through its own client", api.requests(), pics.requests())
}
}
func TestErrorsDoNotCopyTheProvidersText(t *testing.T) {
secret := "SECRET-FROM-THE-PROVIDER"
_, err := newServer(t, 200, `{"error":{"code":"x","info":"`+secret+`"}}`).client().FileName(context.Background(), "Q1")