diff --git a/cmd/sondad/main.go b/cmd/sondad/main.go
index 9298c17..4cb8c06 100644
--- a/cmd/sondad/main.go
+++ b/cmd/sondad/main.go
@@ -59,6 +59,7 @@ const (
// commonsMaxBody leaves room for a 960 pixel copy of a picture, which is some hundred kilobytes.
const (
commonsMinInterval = 300 * time.Millisecond
+ commonsAPIMaxBody = 1 << 20
commonsMaxBody = 8 << 20
)
@@ -159,11 +160,17 @@ func run() error {
return wikipediaClient.Probe(ctx)
})
// Pictures of artists from Wikimedia Commons, through Wikidata, under free licences only.
+ // The pictures are downloaded through a client of their own: they are far bigger than an
+ // answer, and a picture host that misbehaves must not open the breaker of the API.
commonsClient := commons.New(outbound.New(outbound.Options{
+ UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
+ MinInterval: commonsMinInterval,
+ MaxBody: commonsAPIMaxBody,
+ }), commons.WikidataURL, commons.CommonsURL).WithDownloader(outbound.New(outbound.Options{
UserAgentFunc: providerSettings.UserAgentFor(providers.Commons, version),
MinInterval: commonsMinInterval,
MaxBody: commonsMaxBody,
- }), commons.WikidataURL, commons.CommonsURL)
+ }))
providerSettings.SetChecker(providers.Commons, func(ctx context.Context, _ string) error {
return commonsClient.Probe(ctx)
})
@@ -177,6 +184,9 @@ func run() error {
Artists: discoveryRepo, Settings: providerSettings, MusicBrainz: musicbrainzClient, Commons: commonsClient,
})
providerSettings.SetPurger(func(ctx context.Context, name string) (providers.Purged, error) {
+ if name == providers.Commons {
+ pictures.BeginArtistPurge() // a picture being read right now must not be kept
+ }
out, err := discoveryRepo.Purge(ctx, name)
if err != nil || name != providers.Commons {
return out, err
diff --git a/internal/api/openapi/openapi.yaml b/internal/api/openapi/openapi.yaml
index e08ca70..7347b32 100644
--- a/internal/api/openapi/openapi.yaml
+++ b/internal/api/openapi/openapi.yaml
@@ -1088,7 +1088,8 @@ paths:
Whose the picture is and under which licence, to be shown next to it (the licences ask for
it). The author is plain text and may be empty; `license_url` may be empty when the licence
has no address; `page_url` is the page of the file on Commons, where everything is.
- It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a picture.
+ It answers like `GET /artists/{id}/image` (same errors), because the credit exists only with a
+ picture: the first request for an artist can take as long as the picture does (several seconds).
security: [{bearerAuth: []}]
responses:
'200':
diff --git a/internal/images/artist.go b/internal/images/artist.go
index c650dd8..15468f0 100644
--- a/internal/images/artist.go
+++ b/internal/images/artist.go
@@ -70,13 +70,33 @@ func (s *Service) ArtistImage(ctx context.Context, artistID int64, size Size) (*
return s.open(kindArtist, artistID, size, built)
}
+// MaxArtistPixels is the largest copy of an artist's picture that is decoded. The copy
+// asked from Commons is 960 pixels wide, so this leaves room for a very tall one.
+const MaxArtistPixels = 4_000_000
+
+// BeginArtistPurge must be called before the pictures of artists are taken back (their
+// records forgotten and their files removed). A read that started before it is not kept:
+// it starts again, and is refused if the provider is by then off.
+func (s *Service) BeginArtistPurge() {
+ s.purgeMu.Lock()
+ s.purges++
+ s.purgeMu.Unlock()
+}
+
+func (s *Service) purgeCount() uint64 {
+ s.purgeMu.RLock()
+ defer s.purgeMu.RUnlock()
+ return s.purges
+}
+
// buildArtist asks the source, stores the picture and records the outcome. If the
-// record was cleared while it worked, what it found may be out of date: it starts again.
+// record was cleared or the pictures were taken back while it worked, what it found may
+// be out of date: it starts again.
func (s *Service) buildArtist(artistID int64) (Record, error) {
if s.artists == nil {
return Record{}, ErrProviderDisabled
}
- ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
+ ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
defer cancel()
var rec Record
for attempt := 0; attempt < 3; attempt++ {
@@ -84,14 +104,27 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
if err != nil {
return Record{}, fmt.Errorf("read picture record: %w", err)
}
- rec, err = s.searchArtist(ctx, artistID)
+ epoch := s.purgeCount()
+ var pic *Processed
+ rec, pic, err = s.searchArtist(ctx, artistID)
+ if errors.Is(err, errPurged) {
+ continue
+ }
if err != nil {
+ // Nobody calls this context: when it ends it is the build that ran out of time,
+ // which is an outage of the provider and not something to hide as an internal error.
+ if ctx.Err() != nil && (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) {
+ err = fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
+ }
return Record{}, err
}
rec.Gen = cur.Gen
- saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
+ saved, err := s.saveArtist(ctx, artistID, rec, pic, epoch)
+ if errors.Is(err, errPurged) {
+ continue
+ }
if err != nil {
- return Record{}, fmt.Errorf("save picture record: %w", err)
+ return Record{}, err
}
if saved {
return rec, nil
@@ -100,27 +133,30 @@ func (s *Service) buildArtist(artistID int64) (Record, error) {
return rec, nil
}
-func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, error) {
+// errPurged says the pictures were taken back since the build began.
+var errPurged = errors.New("the pictures were taken back while this was being built")
+
+// searchArtist asks the source and, for a picture, prepares it (nil when there is none).
+// The files are written by saveArtist, together with the record, so that a purge cannot
+// fall between them.
+func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, *Processed, error) {
select {
case s.artistSl <- struct{}{}:
defer func() { <-s.artistSl }()
case <-ctx.Done():
- return Record{}, ctx.Err()
+ return Record{}, nil, ctx.Err()
}
none := Record{Status: StatusNone, CheckedAt: s.now()}
found, err := s.artists.ArtistImage(ctx, artistID)
switch {
case errors.Is(err, ErrNoImage):
- return none, nil
+ return none, nil, nil
case err != nil:
- return Record{}, err
+ return Record{}, nil, err
}
- p, err := Process(found.Data)
+ p, err := ProcessLimited(found.Data, MaxArtistPixels)
if err != nil {
- return none, nil // what the provider sent is not a picture: there is none to show
- }
- if err := s.write(kindArtist, artistID, p); err != nil {
- return Record{}, err
+ return none, nil, nil // what the provider sent is not a usable picture: there is none to show
}
// The credit is part of the identity: the same picture under another author or licence
// must not be taken for the same one by a client that kept the old credit.
@@ -130,7 +166,27 @@ func (s *Service) searchArtist(ctx context.Context, artistID int64) (Record, err
return Record{
Status: StatusOK, Source: SourceCommons, Credit: found.Credit, ETag: fmt.Sprintf("%x", h.Sum(nil)[:8]),
Width: p.Width, Height: p.Height, CheckedAt: none.CheckedAt,
- }, nil
+ }, &p, nil
+}
+
+// saveArtist keeps what was found: the files and the record, unless the pictures were
+// taken back since the build began (errPurged) or a change moved the record's generation.
+func (s *Service) saveArtist(ctx context.Context, artistID int64, rec Record, pic *Processed, epoch uint64) (bool, error) {
+ s.purgeMu.RLock()
+ defer s.purgeMu.RUnlock()
+ if s.purges != epoch {
+ return false, errPurged
+ }
+ if pic != nil {
+ if err := s.write(kindArtist, artistID, *pic); err != nil {
+ return false, err
+ }
+ }
+ saved, err := s.store.Save(ctx, kindArtist, artistID, rec)
+ if err != nil {
+ return false, fmt.Errorf("save picture record: %w", err)
+ }
+ return saved, nil
}
// RemoveArtistFiles deletes the stored pictures of artists, and only those, and says how
diff --git a/internal/images/artist_test.go b/internal/images/artist_test.go
index 98b012b..ef90483 100644
--- a/internal/images/artist_test.go
+++ b/internal/images/artist_test.go
@@ -21,15 +21,24 @@ type fakeArtists struct {
err error
calls int32
gate chan struct{}
+ gate2 bool // the gate is never opened for this call: it waits for the context instead
}
-func (f *fakeArtists) ArtistImage(_ context.Context, id int64) (Found, error) {
+func (f *fakeArtists) ArtistImage(ctx context.Context, id int64) (Found, error) {
atomic.AddInt32(&f.calls, 1)
f.mu.Lock()
gate, found, err := f.gate, f.found, f.err
f.mu.Unlock()
if gate != nil {
- <-gate
+ if f.gate2 {
+ select {
+ case <-gate:
+ case <-ctx.Done():
+ return Found{}, ctx.Err()
+ }
+ } else {
+ <-gate
+ }
}
return found, err
}
@@ -205,18 +214,77 @@ func TestManyAsksForAnArtistPictureAskTheProviderOnce(t *testing.T) {
}
}
-func TestAChangeDuringTheReadOfAnArtistPictureIsNotUndoneByIt(t *testing.T) {
+func TestAPurgeDuringTheReadIsNotUndoneByIt(t *testing.T) {
+ r := newArtistRig(t)
+ r.src.gate = make(chan struct{})
+ done := make(chan error, 1)
+ go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
+ time.Sleep(50 * time.Millisecond) // the read has the picture and is about to keep it
+ r.svc.BeginArtistPurge() // the admin takes the provider's data back...
+ r.src.set(Found{}, ErrProviderDisabled) // ...and switches it off
+ close(r.src.gate)
+ if err := <-done; !errors.Is(err, ErrProviderDisabled) {
+ t.Fatalf("err = %v, want the read to be refused once the provider is off", err)
+ }
+ if len(r.store.recs) != 0 {
+ t.Errorf("records = %v: what the read found before the purge came back", r.store.recs)
+ }
+ if entries, _ := os.ReadDir(filepath.Join(r.data, "images", "artist")); len(entries) != 0 {
+ t.Errorf("%d files left by a read that started before the purge", len(entries))
+ }
+}
+
+func TestAReadThatStartedBeforeAPurgeStartsAgainIfTheProviderIsStillOn(t *testing.T) {
r := newArtistRig(t)
r.src.gate = make(chan struct{})
done := make(chan error, 1)
go func() { _, err := r.svc.ArtistImage(context.Background(), 7, Large); done <- err }()
time.Sleep(50 * time.Millisecond)
- r.store.clear("artist", 7) // for example the purge of the provider
+ r.svc.BeginArtistPurge()
close(r.src.gate)
- <-done
- // What the read that started before the change found was not kept as the answer.
- if rec := r.store.recs["artist/7"]; rec.Status == StatusOK && rec.Gen == 0 {
- t.Errorf("record = %+v: an out of date build was saved over the change", rec)
+ if err := <-done; err != nil {
+ t.Fatalf("err = %v", err)
+ }
+ if n := atomic.LoadInt32(&r.src.calls); n != 2 {
+ t.Errorf("the provider was asked %d times, want 2: the first answer was from before the purge", n)
+ }
+ if rec := r.store.recs["artist/7"]; rec.Status != StatusOK {
+ t.Errorf("record = %+v", rec)
+ }
+}
+
+func TestABuildThatTakesTooLongIsAnOutageNotAnInternalError(t *testing.T) {
+ r := newArtistRig(t)
+ _, old := r.get(t, 7, Large) // a picture is stored
+ r.now = r.now.Add(ArtistTTL + time.Hour)
+ r.svc.buildTimeout = 40 * time.Millisecond
+ r.src.gate = make(chan struct{}) // never opened during the build
+ r.src.gate2 = true
+ c, again := r.get(t, 7, Large)
+ if !bytes.Equal(old, again) || c.Credit != credit {
+ t.Error("the old picture was not served when the build timed out")
+ }
+ // With nothing stored the caller is told it is an outage (502), not a failure of ours (500).
+ r2 := newArtistRig(t)
+ r2.svc.buildTimeout = 40 * time.Millisecond
+ r2.src.gate = make(chan struct{})
+ r2.src.gate2 = true
+ if _, err := r2.svc.ArtistImage(context.Background(), 8, Large); !errors.Is(err, ErrSourceUnavailable) {
+ t.Errorf("err = %v, want ErrSourceUnavailable", err)
+ }
+ close(r.src.gate)
+ close(r2.src.gate)
+}
+
+func TestAHugePictureOfAnArtistIsRefused(t *testing.T) {
+ r := newArtistRig(t)
+ r.src.set(Found{Data: jpegBytes(t, 2500, 2500, color.White), Credit: credit}, nil) // 6 million pixels
+ if _, err := r.svc.ArtistImage(context.Background(), 7, Large); !errors.Is(err, ErrNoImage) {
+ t.Errorf("err = %v, want ErrNoImage: a copy of 960 pixels is never that big", err)
+ }
+ // The same picture is fine as the cover of an album (the limit there is larger).
+ if _, err := Process(r.src.found.Data); err != nil {
+ t.Errorf("Process: %v", err)
}
}
diff --git a/internal/images/commons_source.go b/internal/images/commons_source.go
index bfbcac3..d75c416 100644
--- a/internal/images/commons_source.go
+++ b/internal/images/commons_source.go
@@ -58,11 +58,13 @@ func (c *CommonsSource) active(ctx context.Context) (bool, error) {
return true, nil
}
-// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. A request
-// that was cancelled is not a failure of the provider and is returned as it is.
-func failed(err error) error {
- if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
- return err
+// failed turns what went wrong into ErrSourceUnavailable, keeping the cause. When the
+// caller's own context has ended (it left, or ran out of time) that is returned instead:
+// it is not the provider's fault. A deadline buried in the error is not a sign of that, since
+// the outbound client's own timeout produces one when the provider is merely slow.
+func failed(ctx context.Context, err error) error {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return ctxErr
}
return fmt.Errorf("%w: %w", ErrSourceUnavailable, err)
}
@@ -90,7 +92,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
case errors.Is(err, musicbrainz.ErrNotFound), errors.Is(err, musicbrainz.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
- return Found{}, failed(err)
+ return Found{}, failed(ctx, err)
case item == "":
return Found{}, ErrNoImage
}
@@ -99,7 +101,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
case errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
- return Found{}, failed(err)
+ return Found{}, failed(ctx, err)
case name == "":
return Found{}, ErrNoImage
}
@@ -109,7 +111,7 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
errors.Is(err, commons.ErrUnusable), errors.Is(err, commons.ErrBadRequest):
return Found{}, ErrNoImage
case err != nil:
- return Found{}, failed(err)
+ return Found{}, failed(ctx, err)
}
// The admin may have switched the provider off while the chain was being followed:
// do not download, and do not hand over, what they no longer allow.
@@ -120,10 +122,10 @@ func (c *CommonsSource) ArtistImage(ctx context.Context, artistID int64) (Found,
}
data, err := c.Commons.Download(ctx, file.ThumbURL)
switch {
- case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound):
+ case errors.Is(err, commons.ErrBadRequest), errors.Is(err, commons.ErrNotFound), errors.Is(err, commons.ErrUnusable):
return Found{}, ErrNoImage
case err != nil:
- return Found{}, failed(err)
+ return Found{}, failed(ctx, err)
}
if on, err := c.active(ctx); err != nil {
return Found{}, err
diff --git a/internal/images/commons_source_test.go b/internal/images/commons_source_test.go
index ddf3894..529faa1 100644
--- a/internal/images/commons_source_test.go
+++ b/internal/images/commons_source_test.go
@@ -3,6 +3,7 @@ package images
import (
"context"
"errors"
+ "fmt"
"sync"
"testing"
@@ -194,10 +195,38 @@ func TestSwitchedOffWhileItWasWorkingNothingIsReturned(t *testing.T) {
}
}
-func TestACancelledRequestIsNotAFailureOfTheProvider(t *testing.T) {
+func TestAnOutageThatEndsInADeadlineIsStillAnOutage(t *testing.T) {
+ // The outbound client's own timeout wraps context.DeadlineExceeded inside ErrUnavailable. It
+ // is a failure of the provider (so an older picture can be served), not a cancellation.
+ slow := fmt.Errorf("%w: %w", commons.ErrUnavailable, context.DeadlineExceeded)
+ for name, tt := range map[string]func(*chain){
+ "musicbrainz": func(c *chain) { c.mb.err = fmt.Errorf("%w: %w", musicbrainz.ErrUnavailable, context.DeadlineExceeded) },
+ "wikidata": func(c *chain) { c.cm.nameErr = slow },
+ "commons": func(c *chain) { c.cm.infoErr = slow },
+ "the download": func(c *chain) { c.cm.loadErr = slow },
+ } {
+ c := newChain()
+ tt(c)
+ if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrSourceUnavailable) {
+ t.Errorf("%s: err = %v, want ErrSourceUnavailable", name, err)
+ }
+ }
+}
+
+func TestARequestThatTheCallerCancelledIsReturnedAsCancelled(t *testing.T) {
c := newChain()
+ ctx, cancel := context.WithCancel(context.Background())
+ c.mb.hook = cancel // the caller leaves while MusicBrainz is being asked
c.mb.err = context.Canceled
- if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, context.Canceled) {
+ if _, err := c.src.ArtistImage(ctx, 1); !errors.Is(err, context.Canceled) || errors.Is(err, ErrSourceUnavailable) {
t.Errorf("err = %v, want the cancellation itself", err)
}
}
+
+func TestAPictureThatCannotBeUsedFromTheHostIsNoPicture(t *testing.T) {
+ c := newChain()
+ c.cm.loadErr = commons.ErrUnusable // a redirect or a picture over the limit
+ if _, err := c.src.ArtistImage(context.Background(), 1); !errors.Is(err, ErrNoImage) {
+ t.Errorf("err = %v, want ErrNoImage", err)
+ }
+}
diff --git a/internal/images/process.go b/internal/images/process.go
index ceba78c..1b3716a 100644
--- a/internal/images/process.go
+++ b/internal/images/process.go
@@ -45,7 +45,11 @@ type Processed struct {
// Process decodes a GIF, JPEG or PNG and returns it scaled down to fit LargeEdge and
// SmallEdge (never enlarged), on a white background where it was transparent. The
// same input always gives the same bytes.
-func Process(data []byte) (Processed, error) {
+func Process(data []byte) (Processed, error) { return ProcessLimited(data, MaxPixels) }
+
+// ProcessLimited is Process with a smaller limit on the number of pixels, for pictures that
+// are known to be small (a 960 pixel copy is never millions of pixels high).
+func ProcessLimited(data []byte, maxPixels int64) (Processed, error) {
if len(data) == 0 || len(data) > MaxSourceBytes {
return Processed{}, fmt.Errorf("%w: %d bytes", ErrUnusable, len(data))
}
@@ -53,7 +57,7 @@ func Process(data []byte) (Processed, error) {
if err != nil {
return Processed{}, fmt.Errorf("%w: %v", ErrUnusable, err)
}
- if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > MaxPixels { // int64: a 32-bit int would overflow
+ if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > maxPixels { // int64: a 32-bit int would overflow
return Processed{}, fmt.Errorf("%w: %dx%d pixels", ErrUnusable, cfg.Width, cfg.Height)
}
src, _, err := image.Decode(bytes.NewReader(data))
diff --git a/internal/images/service.go b/internal/images/service.go
index 007f4cd..8abf91c 100644
--- a/internal/images/service.go
+++ b/internal/images/service.go
@@ -157,22 +157,31 @@ type Cover struct {
// Service builds covers from the user's own files, on the first request, and keeps
// them in the data folder.
type Service struct {
- dir string
- store Store
- src Source
- embedded func(path string) ([]byte, error)
- now func() time.Time
- flight singleflight.Group
- slots chan struct{} // one per cover build allowed at the same time
- artistSl chan struct{} // the same for artist pictures, which wait for the network
- artists ArtistSource
- tidied sync.Map // the folders whose old temporary files were already removed
+ dir string
+ store Store
+ src Source
+ embedded func(path string) ([]byte, error)
+ now func() time.Time
+ flight singleflight.Group
+ slots chan struct{} // one per cover build allowed at the same time
+ artistSl chan struct{} // the same for artist pictures, which wait for the network
+ artists ArtistSource
+ buildTimeout time.Duration // how long one build may take
+
+ // purges counts the times the provider's pictures were taken back (BeginArtistPurge). A
+ // build that started before one must not keep what it found: its files and its record
+ // would bring back what the admin just removed. Keeping a result takes the read side, so
+ // a purge waits for the results being kept and the next ones are refused.
+ purgeMu sync.RWMutex
+ purges uint64
+ tidied sync.Map // the folders whose old temporary files were already removed
}
// NewService builds a Service that keeps its files under dir.
func NewService(dir string, store Store, src Source) *Service {
return &Service{dir: dir, store: store, src: src, embedded: taglib.ReadImage, now: time.Now,
- slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds)}
+ slots: make(chan struct{}, MaxParallelBuilds), artistSl: make(chan struct{}, MaxParallelBuilds),
+ buildTimeout: buildTimeout}
}
// SetArtistSource tells where pictures of artists come from. Without one, they are
@@ -255,7 +264,7 @@ func (s *Service) open(kind string, id int64, size Size, rec Record) (*Cover, er
// build looks for a picture, stores it and records the outcome. If a file of the
// album changed while it was working, what it found may be out of date: it starts again.
func (s *Service) build(albumID int64) (Record, error) {
- ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
+ ctx, cancel := context.WithTimeout(context.Background(), s.buildTimeout)
defer cancel()
var rec Record
for attempt := 0; attempt < 3; attempt++ {
diff --git a/internal/providers/commons/commons.go b/internal/providers/commons/commons.go
index d769580..71255ef 100644
--- a/internal/providers/commons/commons.go
+++ b/internal/providers/commons/commons.go
@@ -64,9 +64,11 @@ type Getter interface {
// Client talks to Wikidata and Commons. It is safe for concurrent use.
type Client struct {
- http Getter
- wikidata string
- commons string
+ http Getter
+ downloader Getter // for the pictures themselves; the API client when nil
+ wikidata string
+ commons string
+ allowAny bool // tests only: the test servers are not on a Wikimedia host
}
// New builds a Client that sends its requests through g, to wikidata and commons
@@ -75,6 +77,14 @@ func New(g Getter, wikidata, commons string) *Client {
return &Client{http: g, wikidata: wikidata, commons: commons}
}
+// WithDownloader makes the pictures themselves be downloaded through g instead of the
+// client of the API calls. They have their own limits (a picture is far bigger than an
+// answer) and their own breaker: a picture host that misbehaves must not stop the API.
+func (c *Client) WithDownloader(g Getter) *Client {
+ c.downloader = g
+ return c
+}
+
// File is a Commons picture and what has to be shown with it.
type File struct {
Name string // the file name, without the "File:"
@@ -228,10 +238,20 @@ func (c *Client) Info(ctx context.Context, name string) (File, error) {
// Download reads a picture from a Wikimedia picture host. Any other address is refused
// without a request.
func (c *Client) Download(ctx context.Context, pictureURL string) ([]byte, error) {
- if !allowedPictureURL(pictureURL) {
+ if !c.allowAny && !allowedPictureURL(pictureURL) {
return nil, ErrBadRequest
}
- return c.get(ctx, pictureURL)
+ g := c.downloader
+ if g == nil {
+ g = c.http
+ }
+ body, err := c.getWith(ctx, g, pictureURL)
+ // A redirect (never followed) or a picture over the limit is a fault of that picture,
+ // not an outage: there is no usable picture to show.
+ if errors.Is(err, outbound.ErrRedirect) || errors.Is(err, outbound.ErrResponseTooLarge) {
+ return nil, fmt.Errorf("%w: the picture host answered in a way that cannot be used", ErrUnusable)
+ }
+ return body, err
}
// Probe asks about one public item, to find out whether the service answers. It is a
@@ -290,8 +310,10 @@ var tagRe = regexp.MustCompile(`<[^>]*>`)
// entities decoded, control characters dropped, spaces collapsed, at most maxAuthorLen.
func plainText(s string) string {
s = html.UnescapeString(tagRe.ReplaceAllString(s, ""))
+ // Control characters, and the invisible ones that reorder or hide text (a right-to-left
+ // override, a zero-width space): they could make an author read as somebody else.
s = strings.Map(func(r rune) rune {
- if unicode.IsControl(r) && !unicode.IsSpace(r) {
+ if (unicode.IsControl(r) && !unicode.IsSpace(r)) || unicode.Is(unicode.Cf, r) {
return -1
}
return r
@@ -329,7 +351,11 @@ func allowedPictureURL(raw string) bool {
// get sends one request and returns the body of a successful answer.
func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) {
- resp, err := c.http.Get(ctx, rawURL)
+ return c.getWith(ctx, c.http, rawURL)
+}
+
+func (c *Client) getWith(ctx context.Context, g Getter, rawURL string) ([]byte, error) {
+ resp, err := g.Get(ctx, rawURL)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
diff --git a/internal/providers/commons/commons_test.go b/internal/providers/commons/commons_test.go
index 578b49f..6a0edee 100644
--- a/internal/providers/commons/commons_test.go
+++ b/internal/providers/commons/commons_test.go
@@ -1,6 +1,7 @@
package commons
import (
+ "bytes"
"context"
"errors"
"net/http"
@@ -212,6 +213,15 @@ func TestTheLicenceAddressIsKeptOnlyOnCreativeCommons(t *testing.T) {
}
}
+func TestTheAuthorHasNoInvisibleOrReorderingCharacters(t *testing.T) {
+ // A right-to-left override or a zero-width character can make a name read as another.
+ body := strings.Replace(infoBody, `Raph_PH`, `Ra\u202ephH\u200b_\u2066x`, 1)
+ got, err := newServer(t, 200, body).client().Info(context.Background(), "Band.jpg")
+ if err != nil || got.Author != "RaphH_x" {
+ t.Errorf("author = %q err = %v", got.Author, err)
+ }
+}
+
func TestTheAuthorIsPlainText(t *testing.T) {
for artist, want := range map[string]string{
`Raph_PH`: "Raph_PH",
@@ -284,6 +294,41 @@ func TestDownloadOnlyFromWikimediaPictureHosts(t *testing.T) {
}
}
+// What the picture host does wrong is a problem of that picture, not of the service: it must
+// not be taken for an outage, and it must not count against the breaker of the API client.
+func TestDownloadTreatsARedirectOrAHugePictureAsUnusable(t *testing.T) {
+ redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, "https://evil.example/", http.StatusFound)
+ }))
+ defer redirect.Close()
+ huge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write(bytes.Repeat([]byte{1}, 5000))
+ }))
+ defer huge.Close()
+ for name, srv := range map[string]*httptest.Server{"redirect": redirect, "huge": huge} {
+ dl := outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries, MaxBody: 1000})
+ c := New(nil, "", "").WithDownloader(dl)
+ c.allowAny = true // the test servers are not on a Wikimedia host
+ if _, err := c.Download(context.Background(), srv.URL+"/a.jpg"); !errors.Is(err, ErrUnusable) {
+ t.Errorf("%s: err = %v, want ErrUnusable", name, err)
+ }
+ }
+}
+
+func TestDownloadUsesItsOwnClient(t *testing.T) {
+ api := newServer(t, 200, `{"claims":{}}`)
+ pics := newServer(t, 200, "PICTURE")
+ c := api.client().WithDownloader(outbound.New(outbound.Options{UserAgent: "Sonda/test ( https://example.org )", Retries: outbound.NoRetries}))
+ c.allowAny = true
+ got, err := c.Download(context.Background(), pics.URL+"/a.jpg")
+ if err != nil || string(got) != "PICTURE" {
+ t.Fatalf("got %q err %v", got, err)
+ }
+ if len(api.requests()) != 0 || len(pics.requests()) != 1 {
+ t.Errorf("api saw %v, pictures saw %v: the download must go through its own client", api.requests(), pics.requests())
+ }
+}
+
func TestErrorsDoNotCopyTheProvidersText(t *testing.T) {
secret := "SECRET-FROM-THE-PROVIDER"
_, err := newServer(t, 200, `{"error":{"code":"x","info":"`+secret+`"}}`).client().FileName(context.Background(), "Q1")